diff --git a/.claude/skills/pre-push-gate/SKILL.md b/.claude/skills/pre-push-gate/SKILL.md index c7198ad3a9..ddb54b47ca 100644 --- a/.claude/skills/pre-push-gate/SKILL.md +++ b/.claude/skills/pre-push-gate/SKILL.md @@ -273,6 +273,31 @@ of the server's stderr. Re-run a subset with - **A new server fails `test:scripts`** with "every server directory in the repo has a smoke entry": add it to `SERVERS` in `scripts/smoke-servers.mjs`. +### `interface-diff` + +The last stage diffs the `everything` server's MCP interface (capabilities, +tools, prompts, resources, resource templates) in this checkout's build +against the same server built from the merge-base with `origin/v2/main`, and +prints the report. **A changed interface does not fail it**: the report is +what the PR's sticky comment will show, so read it, and if a change is not +one you meant, that is the finding. It fails only when the diff cannot run, +with `❌ The interface diff could not run` and the reason: + +- **"origin/v2/main is not available"**: `git fetch origin v2/main`. +- **"src/everything/dist/index.js does not exist"**: you ran it on its own + before building: `npm run build -w src/everything`. +- **An `npm ci` or `npm run build` failure**: the _base_ did not install or + build. Offline with a cold npm cache is the usual cause; otherwise name a + base that builds with `npm run interface-diff -- --base `. +- **"Base server probe failed" / "Target probe failed"**: a server did not + answer the probe. A target failure is this checkout's server and is the + real defect; the smoke stage before it usually fails first. + +Re-run it on its own with `npm run interface-diff` (about ten seconds). What +it compares against in CI, and why it is a script rather than an action, is +in [`docs/quality-gate.md`](../../../docs/quality-gate.md) and the header of +`scripts/interface-diff.mjs`. + ## Waiting on the lease A gate that starts with diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml new file mode 100644 index 0000000000..16fe49c17d --- /dev/null +++ b/.github/workflows/everything-mcp-diff.yml @@ -0,0 +1,254 @@ +# Tracks public interface changes to the Everything MCP server. +# The Everything server is a reference implementation demonstrating all MCP features. +# This workflow helps reviewers see how tools, resources, prompts, and capabilities +# evolve over time - useful for SDK compliance validation and catching regressions. +# +# Contributed by @SamMorrowDrums in #3260 and adapted for v2/main in #4860: +# +# - The check is `npm run interface-diff` (scripts/interface-diff.mjs), which +# drives the `mcp-server-diff` CLI, an exact-pinned root devDependency. The +# same script is a `local:gate` stage, and `scripts/lib/workflow-gate.test.mjs` +# requires every npm script a workflow runs to be one, so CI and the gate run +# the same check. No third-party action runs here. +# - The base is chosen per event below rather than by the tool, whose default +# (the merge-base with origin/main) is the wrong base for PRs to v2/main. +# - Untrusted code (the PR's install, build and server) runs only in `diff`, +# which holds a read-only token and no persisted credentials. The sticky +# comment is posted by `comment`, which holds `pull-requests: write` and runs +# none of the PR's code, and only for PRs from this repository: a fork PR's +# token is read-only, so there the report is in the job summary and the +# `mcp-diff-report` artifact. `pull_request_target` is deliberately not used, +# since it would run the fork's build next to a write token. If fork PRs need +# the comment too, the way is a separate `workflow_run` workflow that posts +# the artifact without checking anything out. +# +# A changed interface does not fail the run: the diff is information for the +# reviewer. A diff that cannot run (a base that does not build, a server that +# cannot be probed) does. +# +# See: https://github.com/modelcontextprotocol/inspector/issues/1034 +name: Everything Server MCP Diff + +on: + push: + branches: [main, v2/main] + paths: + - "src/everything/**" + - "package.json" + - "package-lock.json" + - "tsconfig.json" + - "scripts/interface-diff.mjs" + - "scripts/lib/win-shell-args.mjs" + - ".github/workflows/everything-mcp-diff.yml" + # No branch filter: PRs to v2/main and to main both run, and so does a PR + # stacked on another PR's branch. The base is the PR's own base either way. + pull_request: + paths: + - "src/everything/**" + - "package.json" + - "package-lock.json" + - "tsconfig.json" + - "scripts/interface-diff.mjs" + - "scripts/lib/win-shell-args.mjs" + - ".github/workflows/everything-mcp-diff.yml" + # A published Release (a milestone, tagged vX.Y.Z on main; see RELEASING.md) + # surfaces the cumulative interface delta since the previous Release. This + # replaces #3260's `typescript-servers-*` tag trigger: those tags stopped in + # 2024, and no release since has used them. + release: + types: [published] + # Manual trigger: the branch chosen in "Use workflow from" is the head, and + # `compare_ref` any base. #3260 also took a `target_ref`, but a target that + # predates the interface-diff script has no `npm run interface-diff` to run, + # so the head is the dispatched branch instead. + workflow_dispatch: + inputs: + compare_ref: + description: "Base ref to compare against (commit SHA, tag, or branch; defaults to the merge-base with v2/main)" + required: false + default: "" + +permissions: + contents: read + +# Superseded runs on the same PR are cancelled. Every other run gets a group +# of its own: a group holds one running and one pending run, so a shared group +# would drop a push's run when two more queued behind it. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + diff: + name: Diff Everything Server Interface + timeout-minutes: 15 + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + # The base is built from history, so all of it is needed, tags + # included (the release base is the previous Release's tag). + fetch-depth: 0 + # The event's own commit (the merge commit for a PR), not a branch + # ref that a later push could move before this step runs. + ref: ${{ github.sha }} + persist-credentials: false + + - uses: actions/setup-node@v7 + with: + node-version: 22 + cache: npm + + # A root `npm ci` runs every workspace's `prepare`, which builds it, so + # the head build the diff probes exists after this step. + - name: Install dependencies and build + run: npm ci + + - name: Choose the base to compare against + id: base + env: + EVENT: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + BEFORE: ${{ github.event.before }} + INPUT_BASE: ${{ inputs.compare_ref }} + TAG: ${{ github.event.release.tag_name }} + PUBLISHED: ${{ github.event.release.published_at }} + REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + run: | + case "$EVENT" in + # The checkout is the PR's merge commit, so its base tip is + # exactly what the PR changes. + pull_request) base="$PR_BASE" ;; + # The branch as it was before this push; a new branch has none. + push) + if [ -n "$BEFORE" ] && [ "$BEFORE" != 0000000000000000000000000000000000000000 ]; then + base="$BEFORE" + else + base=$(git rev-parse HEAD^) + fi ;; + # The Release published just before this one (so a re-run of an + # older Release still compares against ITS predecessor). Not + # `git describe`: the date-stamped releases tagged commits that + # never reached main, so the nearest ancestor tag is years older + # than the last release. + release) + base=$(gh release list --repo "$REPO" --exclude-drafts --exclude-pre-releases --limit 50 \ + --json tagName,publishedAt \ + --jq "map(select(.tagName != \"$TAG\" and .publishedAt < \"$PUBLISHED\")) | sort_by(.publishedAt) | last | .tagName // empty") + if [ -z "$base" ]; then echo "no earlier Release to compare against" >&2; exit 1; fi ;; + workflow_dispatch) + if [ -n "$INPUT_BASE" ]; then + base="$INPUT_BASE" + else + base=$(git merge-base HEAD origin/v2/main) + fi ;; + *) echo "unexpected event: $EVENT" >&2; exit 1 ;; + esac + echo "Comparing against $base" + echo "ref=$base" >> "$GITHUB_OUTPUT" + + - name: Diff the interface + env: + BASE: ${{ steps.base.outputs.ref }} + run: npm run interface-diff -- --base "$BASE" --report-dir mcp-diff-report + + - name: Write the job summary + if: always() + run: | + { + if [ -f mcp-diff-report/report.md ]; then + # A step summary holds at most 1 MiB. + head -c 900000 mcp-diff-report/report.md + if [ "$(wc -c < mcp-diff-report/report.md)" -gt 900000 ]; then + echo "" + echo "_The report is truncated here; the full report is in the run's \`mcp-diff-report\` artifact._" + fi + else + echo "The interface diff produced no report; see the job log." + fi + echo "" + echo "---" + echo "" + echo "ℹ️ **Interpreting Results**" + echo "" + echo "The Everything server is a reference implementation that exercises all MCP features." + echo "Interface changes here may indicate:" + echo "- New MCP protocol features being demonstrated" + echo "- Updated tool schemas or descriptions" + echo "- New resources, prompts, or capabilities" + echo "" + echo "Review changes to ensure they align with the intended protocol updates." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the report + if: always() + uses: actions/upload-artifact@v7 + with: + name: mcp-diff-report + path: mcp-diff-report/ + if-no-files-found: ignore + + # Post (or update) a sticky PR comment so reviewers can see the diff without + # navigating to the Actions tab. One comment per PR, found again by its + # marker and edited on each push. This job checks nothing out and runs none + # of the PR's code; it only reads the report the `diff` job uploaded. + # Dependabot's PRs are same-repo but get a read-only token, so they are + # skipped like fork PRs and keep the job summary and the artifact. + comment: + name: Post the diff on the PR + needs: diff + if: >- + always() && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]' && + (needs.diff.result == 'success' || needs.diff.result == 'failure') + timeout-minutes: 5 + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Download the report + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: mcp-diff-report + path: mcp-diff-report + + - name: Post / update sticky PR comment + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + marker='' + { + echo "$marker" + if [ -f mcp-diff-report/report.md ]; then + # A comment holds at most 65536 characters. + head -c 60000 mcp-diff-report/report.md + if [ "$(wc -c < mcp-diff-report/report.md)" -gt 60000 ]; then + echo "" + echo "_The report is truncated here; the full report is in the run's \`mcp-diff-report\` artifact._" + fi + else + echo "## \`everything\`: MCP interface diff" + echo "" + echo "❌ The diff job produced no report; see the run's log." + fi + echo "" + echo "Updated by [this run]($RUN_URL) for \`$HEAD_SHA\`." + } > body.md + id=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n 1) + if [ -n "$id" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$id" -F body=@body.md > /dev/null + else + gh api "repos/$REPO/issues/$PR/comments" -F body=@body.md > /dev/null + fi diff --git a/AGENTS.md b/AGENTS.md index bba09c93db..7b4abbd327 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,7 @@ servers/ │ └── time/ Py mcp-server-time PyPI Time and timezone conversion ├── .claude/skills/ On-demand procedures (see the Skills index above) ├── .changeset/ Pending changesets for the TypeScript servers, and the changesets config -├── scripts/ The pre-push gate (gate-lease, smoke-servers, validate-py, coverage-py), its guards (verify-*), +├── scripts/ The pre-push gate (gate-lease, smoke-servers, validate-py, coverage-py, interface-diff), its guards (verify-*), │ the skills tooling, release tooling (npm-publish-guard, prepare-python-release, │ pack-and-verify, release-manifest, release-notes), and the issue-filing sweeps │ (dependency-refresh, dependabot-alerts, sdk-watch) @@ -61,8 +61,8 @@ servers/ │ overview of how the rules, skills, gates and sweeps fit together ├── .github/workflows/ typescript.yml, python.yml (per-package CI), release.yml (publishes on a GitHub Release), │ version-packages.yml (the changesets PR), prepare-python-release.yml (the CalVer PR), -│ claude.yml (@claude mentions), and the scheduled sweeps that file issues: -│ dependency-refresh.yml, dependabot-alerts.yml, sdk-watch.yml +│ claude.yml (@claude mentions), everything-mcp-diff.yml (everything's interface diff on PRs), +│ and the scheduled sweeps that file issues: dependency-refresh.yml, dependabot-alerts.yml, sdk-watch.yml ├── .github/ISSUE_TEMPLATE/ Bug and feature issue forms; config.yml routes security and new servers away ├── .github/pull_request_template.md The "issues, not PRs" banner that turns outside PRs away ├── RELEASING.md How packages are versioned and published, and how to recover a failed publish @@ -129,14 +129,15 @@ workspace's format check, lint, typecheck, build and tests), `coverage` (each TypeScript workspace's per-file coverage gate), `validate:py` (each Python server's locked sync, `ruff check`, `ruff format --check`, pyright, pytest and build), `coverage:py` (each Python server's per-file -coverage gate), `verify:skills:cli`, and `smoke` (every server -booted over each transport it implements). The stage-by-stage reference, and -what CI runs where, is [`docs/quality-gate.md`](./docs/quality-gate.md); +coverage gate), `verify:skills:cli`, `smoke` (every server booted over each +transport it implements), and `interface-diff` (the `everything` server's MCP +interface diffed against the base it branched from). The stage-by-stage +reference, and what CI runs where, is [`docs/quality-gate.md`](./docs/quality-gate.md); diagnosing a red stage is the `pre-push-gate` skill. - **`npm run validate` and the per-package commands above are the inner loop, - not a substitute.** They skip the other language, the pinned skills validator - and the boot smoke. + not a substitute.** They skip the other language, the pinned skills validator, + the boot smoke and the interface diff. - **Format before committing**: `npm run format` at the root (TypeScript), and `uv run --frozen ruff format .` in a Python server you changed. The gate only checks formatting; it never rewrites files. diff --git a/docs/quality-gate.md b/docs/quality-gate.md index 075b72de24..7c197a5b05 100644 --- a/docs/quality-gate.md +++ b/docs/quality-gate.md @@ -13,7 +13,7 @@ out; everything else points here. | --- | --- | --- | --- | | Inner loop | `npm run validate -w src/`, `npm run validate:py -- ` | Your machine | While iterating on one server | | Pre-push gate | `npm run local:gate` | Your machine, under a lease | Before every push | -| CI | `.github/workflows/typescript.yml`, `python.yml`; `dco.yml` | GitHub, on every push and pull request; `dco.yml` on pull requests only | Before merge | +| CI | `.github/workflows/typescript.yml`, `python.yml`, `everything-mcp-diff.yml`; `dco.yml` | GitHub, on every push and pull request (`everything-mcp-diff.yml`: when `everything` or what builds it changes); `dco.yml` on pull requests only | Before merge | **`npm run local:gate` runs every check CI runs.** CI splits the same checks into parallel jobs on a fresh install; the gate runs them in sequence on your @@ -49,6 +49,7 @@ in this order, and the first failure stops the run. | 7 | `coverage:py` | Per Python server: `uv sync --locked`, then `uv run --frozen pytest --cov --cov-report=term-missing --cov-report=json`; then **every file** in `coverage.json` must reach **90% on lines and 90% on branches** | `python.yml` → **Coverage \** (one leg each) | | 8 | `verify:skills:cli` | `claude plugin validate` on `.claude/skills`, at the pinned CLI version | `typescript.yml` → **Root guards** | | 9 | `smoke` | Every server boots over each transport it implements and answers one tool call | `typescript.yml` → **Boot smoke** | +| 10 | `interface-diff` | The `everything` server's MCP interface (capabilities, tools, prompts, resources, resource templates) in this checkout's build, diffed against the same server built from the base. It fails only when the diff cannot run; a changed interface is reported, not failed | `everything-mcp-diff.yml` → **Diff Everything Server Interface** (and the PR comment) | Notes on the stages: @@ -91,8 +92,9 @@ Notes on the stages: when a server's environment is missing or behind its lockfile, since `uv sync --locked` then downloads packages (and the pinned interpreter, if `uv` does not have it); `coverage:py` begins with the same sync, which is a - no-op once `validate:py` has run. With warm caches those are the only stages - that can fail offline. + no-op once `validate:py` has run; and `interface-diff` needs it for the + base's `npm ci`. With warm caches those are the only stages that can fail + offline. - **`smoke` launches what a user launches**: the built `dist/index.js` for a TypeScript server, the console script through `uv run --no-sync` for a Python one. stdio for all seven; HTTP+SSE and Streamable HTTP as well for @@ -102,6 +104,16 @@ Notes on the stages: than the default 3001. It runs after `validate` and `validate:py` because it needs the build and each Python server's synced environment; it creates neither. +- **`interface-diff` compares against the merge-base with `origin/v2/main`** + locally (`-- --base ` names another), and against the PR's base, the + commit before the push, or the previous release's tag in CI. It builds the + base from an export of that commit under `node_modules/.cache/` and removes + it afterwards; the head is the build `validate` made. It drives the + `mcp-server-diff` CLI, an exact-pinned root devDependency, which probes with + the 2026-07-28 `server/discover` first and falls back to the `initialize` + handshake, so it covers a server in either era. Its `npm ci` of the base + needs the network when the npm cache is cold, and it needs + `origin/v2/main` fetched. Only `everything` is diffed today (#4860). - `python.yml` also has a **Build \** job per server that re-runs pyright and `uv build` and uploads the built distribution. It checks nothing stage 6 does not. diff --git a/package-lock.json b/package-lock.json index d1390cede6..a9adead5f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "@modelcontextprotocol/client": "^2.3.1", "eslint": "^10.11.0", "globals": "^17.12.0", + "mcp-server-diff": "3.0.0", "prettier": "3.8.4", "proper-lockfile": "^4.1.2", "semver": "^7.8.4", @@ -31,6 +32,50 @@ "yaml": "^2.9.0" } }, + "node_modules/@actions/core": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-3.0.1.tgz", + "integrity": "sha512-a6d/Nwahm9fliVGRhdhofo40HjHQasUPusmc7vBfyky+7Z+P2A1J68zyFVaNcEclc/Se+eO595oAr5nwEIoIUA==", + "dev": true, + "dependencies": { + "@actions/exec": "^3.0.0", + "@actions/http-client": "^4.0.0" + } + }, + "node_modules/@actions/exec": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-3.0.0.tgz", + "integrity": "sha512-6xH/puSoNBXb72VPlZVm7vQ+svQpFyA96qdDBvhB8eNZOE8LtPf9L4oAsfzK/crCL8YZ+19fKYVnM63Sl+Xzlw==", + "dev": true, + "dependencies": { + "@actions/io": "^3.0.2" + } + }, + "node_modules/@actions/http-client": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-4.0.1.tgz", + "integrity": "sha512-+Nvd1ImaOZBSoPbsUtEhv+1z99H12xzncCkz0a3RuehINE81FZSe2QTj3uvAPTcJX/SCzUQHQ0D1GrPMbrPitg==", + "dev": true, + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^6.23.0" + } + }, + "node_modules/@actions/http-client/node_modules/undici": { + "version": "6.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.29.0.tgz", + "integrity": "sha512-R+RODBqp6i2pPflGdq+xIOUkl+RNfGgHwoinecKu/JCuf2uO06cOKoDbI2P7Dn6KcswdKwrczbU6IYJ6K8X+wg==", + "dev": true, + "engines": { + "node": ">=18.17" + } + }, + "node_modules/@actions/io": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-3.0.2.tgz", + "integrity": "sha512-nRBchcMM+QK1pdjO7/idu86rbJI5YHUKCvKs0KxnSYbVe3F51UfGxuZX4Qy/fWlp6l7gWFwIkrOzN+oUK03kfw==", + "dev": true + }, "node_modules/@babel/helper-string-parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", @@ -577,6 +622,19 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, + "node_modules/@hono/node-server": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.4.tgz", + "integrity": "sha512-2g1qeHl4BduwDn6L8nkAvDrPUWPMlkQ//a5NhP/zIyaXKMjZNRV+cG9Csu+BiJFf6+68pqzjf4Hi5sJGKHuS1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4 || ^5.0.0-0" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -796,6 +854,47 @@ "hono": "^4" } }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.32.1", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.32.1.tgz", + "integrity": "sha512-2DdE+SJDtzLEEWzY1ZjY7Q+VcPhcV1KisD3zI4u0XZyktsjHum1mwbMI+JaulUBi2OZk+KJAi2uPXzxichPkdw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, "node_modules/@modelcontextprotocol/server": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/server/-/server-2.3.1.tgz", @@ -1802,6 +1901,41 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -2784,6 +2918,23 @@ "fast-string-truncated-width": "^3.0.2" } }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fast-wrap-ansi": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", @@ -3402,6 +3553,20 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -3805,6 +3970,36 @@ "node": ">= 0.4" } }, + "node_modules/mcp-server-diff": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mcp-server-diff/-/mcp-server-diff-3.0.0.tgz", + "integrity": "sha512-vtbf+ZSfRFMZQipzIpQDP5l35qMhW/CtemsERMr7DjU5Z+uY6ZtL8gHBAamwZ2sS3VvsWtVqMg1Hk91HbavZrw==", + "dev": true, + "dependencies": { + "@actions/core": "^3.0.1", + "@actions/exec": "^3.0.0", + "@actions/io": "^3.0.2", + "@modelcontextprotocol/sdk": "^1.13.2", + "diff": "^9.0.0", + "undici": "^8.4.1", + "zod": "^4.4.3" + }, + "bin": { + "mcp-server-diff": "dist/cli/index.js" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/mcp-server-diff/node_modules/diff": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", + "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -4443,6 +4638,16 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.12", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", @@ -4972,6 +5177,15 @@ "license": "0BSD", "optional": true }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "dev": true, + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -5054,6 +5268,15 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/undici": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.11.2.tgz", + "integrity": "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ==", + "dev": true, + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -5427,6 +5650,16 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + }, "src/everything": { "name": "@modelcontextprotocol/server-everything", "version": "1.0.0", diff --git a/package.json b/package.json index a1d225897a..49ed4ba99e 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,11 @@ "verify:action-pins": "node scripts/verify-action-pins.mjs", "verify:dco": "node scripts/verify-dco.mjs", "smoke": "node scripts/smoke-servers.mjs", + "interface-diff": "node scripts/interface-diff.mjs", "pack:verify": "node scripts/pack-and-verify.mjs", "release:notes": "node scripts/release-notes.mjs", "local:gate": "node scripts/gate-lease.mjs npm run local:gate:stages", - "local:gate:stages": "npm run verify:install-fresh && npm run verify:dco && npm run validate && npm run coverage && npm run validate:py && npm run coverage:py && npm run verify:skills:cli && npm run smoke" + "local:gate:stages": "npm run verify:install-fresh && npm run verify:dco && npm run validate && npm run coverage && npm run validate:py && npm run coverage:py && npm run verify:skills:cli && npm run smoke && npm run interface-diff" }, "dependencies": { "@modelcontextprotocol/server-everything": "*", @@ -61,6 +62,7 @@ "@modelcontextprotocol/client": "^2.3.1", "eslint": "^10.11.0", "globals": "^17.12.0", + "mcp-server-diff": "3.0.0", "prettier": "3.8.4", "proper-lockfile": "^4.1.2", "semver": "^7.8.4", diff --git a/scripts/interface-diff.mjs b/scripts/interface-diff.mjs new file mode 100644 index 0000000000..59403326c4 --- /dev/null +++ b/scripts/interface-diff.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node +// The `everything` server's MCP interface diff (#4860), adapted from +// @SamMorrowDrums's workflow in #3260. +// +// It snapshots the public interface of two builds of `everything` (the server +// capabilities from the handshake, and its tools, prompts, resources and +// resource templates) with `mcp-server-diff`, and reports what changed between +// them: the build in this checkout (the head) against the same server built +// from a base commit. The spec refactor (#4857) leans on it: an SDK migration +// that is meant to be transparent proves it with an empty diff, and each new +// 2026-07-28 feature shows up as a reviewable one. +// +// Why a script, and not the `mcp-server-diff` GitHub Action #3260 used: +// +// - `local:gate` must run every check CI runs, and +// `scripts/lib/workflow-gate.test.mjs` derives that from the npm scripts a +// workflow invokes. A check behind an action is invisible to it; a check +// behind `npm run interface-diff` is enforced by it. CI +// (`everything-mcp-diff.yml`) and the gate run this same file. +// - The tool comes in as an exact-pinned root devDependency, so it is held +// by `package-lock.json`'s integrity hash, the way every other npm +// dependency here is, and no third-party action runs in CI at all. +// - The action picks the base itself, as the merge-base with `origin/main`. +// PRs here target `v2/main`, so that base is wrong; the caller names it. +// +// The base is built from a plain export of the base commit's tree (no git +// worktree is registered) under `node_modules/.cache/`, which git, Prettier and +// ESLint all ignore, and removed afterwards. It sits inside the checkout rather +// than under the system temp directory because `mcp-server-diff` splits a +// server's start command on whitespace, and a relative path from the checkout +// cannot contain any, where a temp directory under a Windows profile can. +// +// The head is NOT built here: inside `local:gate` the `validate` stage has +// already built it, and CI's root `npm ci` builds it. A missing build is an +// error, as it is for the boot smoke. +// +// Outcome: +// - unchanged / changed: exit 0. A changed interface is information for the +// reviewer, not a failure (#3260 ran with `fail_on_diff: false` too). +// - error: exit 1. A server that cannot be probed, or a base that does not +// build, is a broken check, so it fails the gate and the CI job. +// +// Usage: +// npm run interface-diff # base: merge-base with origin/v2/main +// npm run interface-diff -- --base # any commit, tag or branch +// npm run interface-diff -- --report-dir # also write report.md and result.json + +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { winShellArgs } from "./lib/win-shell-args.mjs"; + +const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", +); + +/** The server this diff covers. Start with `everything` only (#4860). */ +export const SERVER_DIR = "src/everything"; + +/** The entry point a client launches, relative to a checkout's root. */ +const ENTRY = `${SERVER_DIR}/dist/index.js`; + +/** The base when none is named: where this branch left the develop line. */ +export const DEFAULT_BASE_BRANCH = "origin/v2/main"; + +/** + * Parse the command line. + * + * @param {string[]} argv + * @returns {{ base: string | null, reportDir: string | null }} + */ +export function parseArgs(argv) { + const opts = { base: null, reportDir: null }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + const [flag, inline] = arg.startsWith("--") ? arg.split(/=(.*)/s) : [arg]; + const value = () => { + const v = inline ?? argv[++i]; + if (v === undefined || v === "") throw new Error(`${flag} needs a value`); + return v; + }; + if (flag === "--base") opts.base = value(); + else if (flag === "--report-dir") opts.reportDir = value(); + else throw new Error(`unknown argument: ${arg}`); + } + return opts; +} + +/** + * Turn `mcp-server-diff`'s JSON output into this diff's verdict. + * + * The CLI exits 1 both when the interfaces differ and when a probe fails, so + * its exit code cannot tell a change from a broken check; the JSON can. + * + * @param {string} stdout what `mcp-server-diff -o json -q` printed + * @param {string} [stderr] what it printed to stderr: a fatal error before the + * report goes there, so it is the reason when no report came out + * @returns {{ status: "unchanged" | "changed" | "error", error?: string, + * diffs: { endpoint: string, diff: string }[], + * baseCounts?: object, headCounts?: object }} + */ +export function classify(stdout, stderr = "") { + let parsed; + try { + parsed = JSON.parse(stdout); + } catch { + return { + status: "error", + error: `mcp-server-diff printed no JSON report:\n${[stderr, stdout] + .map((s) => s.trim()) + .filter(Boolean) + .join("\n") + .slice(0, 2000)}`, + diffs: [], + }; + } + const result = parsed?.results?.[0]; + if (!result || typeof result !== "object") + return { + status: "error", + error: "mcp-server-diff reported no comparison", + diffs: [], + }; + const diffs = Array.isArray(result.diffs) ? result.diffs : []; + if (result.error) { + // `error` is the bare exception; the entry in `diffs` says which build + // failed ("Base server probe failed: …" / "Target probe failed: …"). + const context = diffs.find((d) => d.endpoint === "error")?.diff; + return { status: "error", error: String(context ?? result.error), diffs }; + } + // A report without a verdict is not a pass. + if (typeof result.hasDifferences !== "boolean") + return { + status: "error", + error: "mcp-server-diff's report has no hasDifferences verdict", + diffs, + }; + return { + status: result.hasDifferences ? "changed" : "unchanged", + diffs, + baseCounts: result.baseCounts, + headCounts: result.targetCounts, + }; +} + +/** A Markdown code fence longer than any run of backticks in `text`. */ +export function fenceFor(text) { + const longest = Math.max( + 0, + ...[...text.matchAll(/`+/g)].map((m) => m[0].length), + ); + return "`".repeat(Math.max(3, longest + 1)); +} + +/** + * The human-readable report: the verdict first, then the counts, then each + * part of the interface that changed as a diff. + * + * @param {ReturnType} verdict + * @param {{ base: string, head: string }} labels + */ +export function renderReport(verdict, { base, head }) { + const lines = ["## `everything`: MCP interface diff", ""]; + lines.push(`Base: ${base} `, `Head: ${head}`, ""); + if (verdict.status === "unchanged") + lines.push("✅ **No interface changes detected.**"); + else if (verdict.status === "changed") + lines.push( + `⚠️ **Interface changes detected** in ${verdict.diffs.length} part(s) of the interface. Review them below to confirm they are intended.`, + ); + else lines.push("❌ **The interface diff could not run.**"); + lines.push(""); + + if (verdict.status === "error") { + const fence = fenceFor(verdict.error ?? ""); + lines.push(fence, verdict.error ?? "", fence, ""); + return lines.join("\n"); + } + + const row = (name, c = {}) => + `| ${name} | ${c.tools ?? "?"} | ${c.prompts ?? "?"} | ${c.resources ?? "?"} | ${c.resourceTemplates ?? "?"} |`; + lines.push( + "| | Tools | Prompts | Resources | Resource templates |", + "| --- | --- | --- | --- | --- |", + row("Base", verdict.baseCounts), + row("Head", verdict.headCounts), + "", + ); + + for (const { endpoint, diff } of verdict.diffs) { + const fence = fenceFor(diff); + lines.push(`### ${endpoint}`, "", `${fence}diff`, diff, fence, ""); + } + return lines.join("\n"); +} + +/** The last lines of a stream, for an error message; empty stays empty. */ +const tail = (text) => { + const lines = (text ?? "").trim(); + return lines ? `${lines.split("\n").slice(-25).join("\n")}\n` : ""; +}; + +/** + * Run a command to completion; throw with its output when it fails. Only + * `npm` needs a shell, and only on Windows (it is a `.cmd` shim there); its + * arguments are then quoted for `cmd.exe`. `git` is spawned without one. + */ +function run(command, args, cwd, env) { + const shell = process.platform === "win32" && command === "npm"; + const res = spawnSync(command, shell ? winShellArgs(args) : args, { + cwd, + env: env ?? process.env, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + shell, + }); + if (res.error) throw res.error; + if (res.status !== 0) + throw new Error( + `\`${command} ${args.join(" ")}\` exited ${res.status}:\n${tail(res.stdout)}${tail(res.stderr)}`, + ); + return res.stdout; +} + +const git = (...args) => run("git", args, repoRoot).trim(); + +/** The commit to compare against, as a full SHA. */ +function resolveBase(base) { + if (base) return git("rev-parse", "--verify", `${base}^{commit}`); + try { + git("rev-parse", "--verify", DEFAULT_BASE_BRANCH); + } catch { + throw new Error( + `${DEFAULT_BASE_BRANCH} is not available to compare against. Run \`git fetch origin v2/main\`, or name a base with --base .`, + ); + } + return git("merge-base", "HEAD", DEFAULT_BASE_BRANCH); +} + +/** + * Build `everything` as it was at `sha` into `dir`: export the commit's tree + * through a throwaway index (no worktree registered, the real index + * untouched), install that workspace's locked dependencies, build it. + */ +function buildBase(sha, dir) { + const index = `${dir}.index`; + const env = { ...process.env, GIT_INDEX_FILE: index }; + try { + run("git", ["read-tree", sha], repoRoot, env); + run( + "git", + ["checkout-index", "--all", `--prefix=${dir}${path.sep}`], + repoRoot, + env, + ); + } finally { + rmSync(index, { force: true }); + } + if (!existsSync(path.join(dir, SERVER_DIR, "package.json"))) + throw new Error(`${SERVER_DIR} does not exist at the base ${sha}`); + run( + "npm", + [ + "ci", + "--workspace", + SERVER_DIR, + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--prefer-offline", + ], + dir, + ); + run("npm", ["run", "build", "--workspace", SERVER_DIR], dir); +} + +/** Where the pinned `mcp-server-diff` CLI lives. */ +function cliPath() { + const require = createRequire(import.meta.url); + const manifest = require.resolve("mcp-server-diff/package.json"); + const { bin } = require("mcp-server-diff/package.json"); + const rel = typeof bin === "string" ? bin : bin["mcp-server-diff"]; + return path.join(path.dirname(manifest), rel); +} + +/** A short, readable label for a commit. */ +function describe(sha) { + const subject = git("log", "-1", "--format=%s", sha); + return `\`${sha.slice(0, 12)}\` (${subject.replaceAll("`", "'")})`; +} + +export async function main(argv = process.argv.slice(2)) { + let opts; + try { + opts = parseArgs(argv); + } catch (err) { + console.error(`interface-diff: ${err.message}`); + return 2; + } + + let verdict; + let labels = { base: "(unresolved)", head: "(unresolved)" }; + const cacheRoot = path.join(repoRoot, "node_modules", ".cache"); + let work; + try { + if (!existsSync(path.join(repoRoot, ENTRY))) + throw new Error( + `${ENTRY} does not exist — build first (npm run build -w ${SERVER_DIR}).`, + ); + const sha = resolveBase(opts.base); + const headSha = git("rev-parse", "HEAD"); + labels = { + base: describe(sha), + head: `the build in this checkout, at ${describe(headSha)}`, + }; + console.error(`interface-diff: building the base ${sha.slice(0, 12)} …`); + + mkdirSync(cacheRoot, { recursive: true }); + work = mkdtempSync(path.join(cacheRoot, "servers-interface-diff-")); + buildBase(sha, work); + + // Both start commands are relative to the checkout, so they hold no + // whitespace for mcp-server-diff to split on (see the header). + const baseEntry = path + .relative(repoRoot, path.join(work, ENTRY)) + .split(path.sep) + .join("/"); + console.error("interface-diff: probing the base and the head …"); + const res = spawnSync( + process.execPath, + [ + cliPath(), + "--base", + `node ${baseEntry} stdio`, + "--target", + `node ${ENTRY} stdio`, + "--output", + "json", + "--quiet", + ], + { cwd: repoRoot, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }, + ); + if (res.error) throw res.error; + verdict = classify(res.stdout, res.stderr); + } catch (err) { + verdict = { status: "error", error: err.message, diffs: [] }; + } finally { + if (work) rmSync(work, { recursive: true, force: true }); + } + + const report = renderReport(verdict, labels); + console.log(report); + if (opts.reportDir) { + mkdirSync(opts.reportDir, { recursive: true }); + writeFileSync(path.join(opts.reportDir, "report.md"), `${report}\n`); + writeFileSync( + path.join(opts.reportDir, "result.json"), + `${JSON.stringify({ status: verdict.status, diffCount: verdict.diffs.length }, null, 2)}\n`, + ); + } + return verdict.status === "error" ? 1 : 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) + process.exitCode = await main(); diff --git a/scripts/interface-diff.test.mjs b/scripts/interface-diff.test.mjs new file mode 100644 index 0000000000..90d6fc25ab --- /dev/null +++ b/scripts/interface-diff.test.mjs @@ -0,0 +1,138 @@ +// Tests for the `everything` interface diff (#4860). The pure halves are +// exercised here: the argument parser, the verdict drawn from +// `mcp-server-diff`'s JSON (the one thing that decides whether the gate and +// the CI job fail), and the report. Building a base and probing two servers is +// what `npm run interface-diff` itself does as a gate stage, so it is not +// repeated here. Run via `npm run test:scripts`. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + classify, + fenceFor, + main, + parseArgs, + renderReport, +} from "./interface-diff.mjs"; + +const counts = { tools: 13, prompts: 4, resources: 7, resourceTemplates: 2 }; +const cliJson = (result) => + JSON.stringify({ timestamp: "t", results: [result], summary: {} }); + +test("parseArgs reads both flags, in either spelling", () => { + assert.deepEqual(parseArgs([]), { base: null, reportDir: null }); + assert.deepEqual(parseArgs(["--base", "abc", "--report-dir", "out"]), { + base: "abc", + reportDir: "out", + }); + assert.deepEqual(parseArgs(["--base=v1.0.0", "--report-dir=a=b"]), { + base: "v1.0.0", + reportDir: "a=b", + }); +}); + +test("parseArgs refuses an unknown flag or a missing value", () => { + assert.throws(() => parseArgs(["--fail-on-diff"]), /unknown argument/); + assert.throws(() => parseArgs(["--base"]), /--base needs a value/); + assert.throws(() => parseArgs(["--base="]), /--base needs a value/); +}); + +test("main exits 2 on a bad command line, before building anything", async (t) => { + // The usage error it prints is expected; keep it out of the test output. + const error = t.mock.method(console, "error", () => {}); + assert.equal(await main(["--nope"]), 2); + assert.match(String(error.mock.calls[0]?.arguments[0]), /unknown argument/); +}); + +test("an identical interface is unchanged", () => { + const v = classify( + cliJson({ + hasDifferences: false, + baseCounts: counts, + targetCounts: counts, + diffs: [], + }), + ); + assert.equal(v.status, "unchanged"); + assert.deepEqual(v.headCounts, counts); +}); + +test("a difference is a change, not an error", () => { + const diffs = [{ endpoint: "tools", diff: "- a\n+ b" }]; + const v = classify( + cliJson({ + hasDifferences: true, + baseCounts: counts, + targetCounts: counts, + diffs, + }), + ); + assert.equal(v.status, "changed"); + assert.deepEqual(v.diffs, diffs); +}); + +test("a probe failure is an error, though the CLI reports it as a difference", () => { + // mcp-server-diff sets hasDifferences on a failed probe too, and exits 1 + // for both; only the `error` field tells them apart. + const v = classify( + cliJson({ + hasDifferences: true, + error: "boom", + diffs: [{ endpoint: "error", diff: "Target probe failed: boom" }], + }), + ); + assert.equal(v.status, "error"); + // The context (which build failed) comes from the diffs entry. + assert.equal(v.error, "Target probe failed: boom"); +}); + +test("output that is not the JSON report is an error, never a pass", () => { + assert.equal(classify("").status, "error"); + assert.equal(classify("Fatal error: x").status, "error"); + assert.equal(classify("{}").status, "error"); + assert.equal(classify(JSON.stringify({ results: [] })).status, "error"); + assert.equal(classify(JSON.stringify({ results: [{}] })).status, "error"); +}); + +test("a CLI crash before the report carries its stderr as the reason", () => { + const v = classify("", "Fatal error: Error: spawn node ENOENT"); + assert.equal(v.status, "error"); + assert.match(v.error, /no JSON report[\s\S]*spawn node ENOENT/); +}); + +test("fenceFor outruns any backticks in the content", () => { + assert.equal(fenceFor("plain"), "```"); + assert.equal(fenceFor("a ``` b"), "````"); + assert.equal(fenceFor("`````"), "``````"); +}); + +test("the report leads with the verdict and shows each changed part", () => { + const labels = { base: "`base`", head: "`head`" }; + assert.match( + renderReport( + { + status: "unchanged", + diffs: [], + baseCounts: counts, + headCounts: counts, + }, + labels, + ), + /No interface changes detected[\s\S]*\| Head \| 13 \| 4 \| 7 \| 2 \|/, + ); + const changed = renderReport( + { + status: "changed", + diffs: [{ endpoint: "tools", diff: "- x ``` y" }], + baseCounts: counts, + headCounts: counts, + }, + labels, + ); + assert.match(changed, /Interface changes detected/); + assert.match(changed, /### tools\n\n````diff\n- x ``` y\n````/); + assert.match( + renderReport({ status: "error", error: "boom", diffs: [] }, labels), + /could not run[\s\S]*boom/, + ); +}); diff --git a/scripts/lib/workflow-gate.test.mjs b/scripts/lib/workflow-gate.test.mjs index 6dee0ca4fd..c08ae56186 100644 --- a/scripts/lib/workflow-gate.test.mjs +++ b/scripts/lib/workflow-gate.test.mjs @@ -392,6 +392,7 @@ describe("the gate's name", () => { "coverage:py", "verify:skills:cli", "smoke", + "interface-diff", ]) { it(`runs \`${stage}\``, () => { assert.ok(