From d0d81b1123533459bc5c792fb3d8576b4ee203dd Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Wed, 28 Jan 2026 17:56:25 +0100 Subject: [PATCH 01/17] ci: add MCP interface diff workflow for Everything server Adds a GitHub Actions workflow that tracks public interface changes to the Everything MCP server using mcp-server-diff. Features: - Runs on PRs and pushes affecting src/everything/ - Auto-compares against merge-base (PRs) or previous state (pushes) - Manual workflow_dispatch for comparing any two refs - Generates diff reports showing tool, resource, prompt, and capability changes This helps catch unintended interface changes and provides clear visibility into how the reference server evolves over time. Related: https://github.com/modelcontextprotocol/inspector/issues/1034 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 70 +++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/everything-mcp-diff.yml diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml new file mode 100644 index 0000000000..6dd2db00e5 --- /dev/null +++ b/.github/workflows/everything-mcp-diff.yml @@ -0,0 +1,70 @@ +# Tracks public interface changes to the Everything MCP server. +# The Everything server is a reference implementation demonstrating all MCP features. +# This workflow helps reviewers see how tools, resources, prompts, and capabilities +# evolve over time - useful for SDK compliance validation and catching regressions. +# +# See: https://github.com/modelcontextprotocol/inspector/issues/1034 +name: Everything Server MCP Diff + +on: + push: + branches: [main] + paths: + - 'src/everything/**' + pull_request: + paths: + - 'src/everything/**' + # Manual trigger for comparing any two refs (commits, tags, branches) + workflow_dispatch: + inputs: + compare_ref: + description: 'Base ref to compare against (commit SHA, tag, or branch)' + required: false + default: '' + target_ref: + description: 'Target ref to compare (defaults to current branch if empty)' + required: false + default: '' + +permissions: + contents: read + +jobs: + mcp-diff: + name: Diff Everything Server Interface + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.target_ref || github.ref }} + + - name: Run MCP Server Diff + uses: SamMorrowDrums/mcp-server-diff@4ddeb31dd8b3f98f2c20d24d60d4d9b2b9aa6e3b # v2.1.1 + with: + setup_node: 'true' + node_version: '22' + install_command: npm ci + build_command: npm run build + start_command: node dist/index.js stdio + compare_ref: ${{ github.event.inputs.compare_ref || '' }} + server_timeout: '15' + working-directory: src/everything + + - name: Add summary context + if: always() + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "---" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "โ„น๏ธ **Interpreting Results**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The Everything server is a reference implementation that exercises all MCP features." >> $GITHUB_STEP_SUMMARY + echo "Interface changes here may indicate:" >> $GITHUB_STEP_SUMMARY + echo "- New MCP protocol features being demonstrated" >> $GITHUB_STEP_SUMMARY + echo "- Updated tool schemas or descriptions" >> $GITHUB_STEP_SUMMARY + echo "- New resources, prompts, or capabilities" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Review changes to ensure they align with the intended protocol updates." >> $GITHUB_STEP_SUMMARY From 4948cdc8a0faa90b832aec39cedb9b887d577e91 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Wed, 28 Jan 2026 18:09:02 +0100 Subject: [PATCH 02/17] fix: update mcp-server-diff to v2.3.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 6dd2db00e5..3473f664a9 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -42,7 +42,7 @@ jobs: ref: ${{ github.event.inputs.target_ref || github.ref }} - name: Run MCP Server Diff - uses: SamMorrowDrums/mcp-server-diff@4ddeb31dd8b3f98f2c20d24d60d4d9b2b9aa6e3b # v2.1.1 + uses: SamMorrowDrums/mcp-server-diff@a5555e85d68eaa014a334ae7d12b73787f2c49cc # v2.3.5 with: setup_node: 'true' node_version: '22' From e245576e30bd100bac484b54fd1cb5b17878c581 Mon Sep 17 00:00:00 2001 From: Sam Morrow Date: Wed, 28 Jan 2026 18:17:34 +0100 Subject: [PATCH 03/17] fix: use full paths instead of working-directory Composite actions don't support working-directory at step level. Use full paths in commands instead. Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 3473f664a9..16168537c3 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -46,12 +46,11 @@ jobs: with: setup_node: 'true' node_version: '22' - install_command: npm ci - build_command: npm run build - start_command: node dist/index.js stdio + install_command: cd src/everything && npm ci + build_command: cd src/everything && npm run build + start_command: node src/everything/dist/index.js stdio compare_ref: ${{ github.event.inputs.compare_ref || '' }} server_timeout: '15' - working-directory: src/everything - name: Add summary context if: always() From 9f05f499add22661dad6ab86a2a3266c5d7bda50 Mon Sep 17 00:00:00 2001 From: SamMorrowDrums <48113580+SamMorrowDrums@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:43:02 +0200 Subject: [PATCH 04/17] ci: add concurrency group and fix mcp-server-diff SHA pin Address review feedback: - Add concurrency group so superseded runs on the same ref are cancelled - Repin SamMorrowDrums/mcp-server-diff to the correct v2.3.5 commit SHA (the previous SHA did not exist in the action's repo) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 16168537c3..49dfee3064 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -29,6 +29,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: mcp-diff: name: Diff Everything Server Interface @@ -42,7 +46,7 @@ jobs: ref: ${{ github.event.inputs.target_ref || github.ref }} - name: Run MCP Server Diff - uses: SamMorrowDrums/mcp-server-diff@a5555e85d68eaa014a334ae7d12b73787f2c49cc # v2.3.5 + uses: SamMorrowDrums/mcp-server-diff@f7e5e58a4b0c4f68a5827adad2292953bb1ab9ef # v2.3.5 with: setup_node: 'true' node_version: '22' From 124b28ff1409f32a86484525b892bdaa6e3e0efc Mon Sep 17 00:00:00 2001 From: SamMorrowDrums <48113580+SamMorrowDrums@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:04:00 +0200 Subject: [PATCH 05/17] ci: post sticky PR comment with MCP diff report Surface the Everything server interface diff directly on the PR instead of requiring reviewers to dig into the Actions tab. - Add 'pull-requests: write' permission - Capture mcp-server-diff status output via step id - Build a comment body that summarises the status and includes the full report in a collapsed
block - Post via marocchino/sticky-pull-request-comment (SHA-pinned to v3.0.4) using a stable header so subsequent pushes update the same comment - Guarded to same-repo PRs only; fork PRs still get the summary and uploaded artifact Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 49dfee3064..0eabe2264e 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -28,6 +28,7 @@ on: permissions: contents: read + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -46,6 +47,7 @@ jobs: ref: ${{ github.event.inputs.target_ref || github.ref }} - name: Run MCP Server Diff + id: mcp_diff uses: SamMorrowDrums/mcp-server-diff@f7e5e58a4b0c4f68a5827adad2292953bb1ab9ef # v2.3.5 with: setup_node: 'true' @@ -71,3 +73,43 @@ jobs: echo "- New resources, prompts, or capabilities" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Review changes to ensure they align with the intended protocol updates." >> $GITHUB_STEP_SUMMARY + + # Post (or update) a sticky PR comment so reviewers can see the diff without + # navigating to the Actions tab. The same comment is updated on subsequent + # pushes via the `header` key. Skipped for PRs from forks where GITHUB_TOKEN + # is read-only โ€” the report is still available in the workflow summary and + # as an uploaded artifact. + - name: Build PR comment body + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + id: comment_body + run: | + { + echo 'body<Full report' + echo '' + tail -n +5 conformance-report/CONFORMANCE_REPORT.md + echo '' + echo '
' + else + echo '_Report file not found โ€” see the workflow logs for details._' + fi + echo '' + echo "Updated by [\`${{ github.workflow }}\`](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) ยท commit \`${{ github.event.pull_request.head.sha }}\`" + echo 'MCP_DIFF_EOF' + } >> "$GITHUB_OUTPUT" + + - name: Post / update sticky PR comment + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4 + with: + header: everything-server-mcp-diff + message: ${{ steps.comment_body.outputs.body }} From afc07115bd6883c72613fb020cd8877d18ba51ce Mon Sep 17 00:00:00 2001 From: SamMorrowDrums <48113580+SamMorrowDrums@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:16:35 +0200 Subject: [PATCH 06/17] ci: also run MCP diff on TypeScript release tags Mirrors github/github-mcp-server's pattern of pushing on tags so each release surfaces its cumulative interface delta. The everything server ships as part of the typescript-servers monorepo bundle, so we trigger on typescript-servers-* tags and let mcp-server-diff auto-compare against the previous matching tag. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 0eabe2264e..798367fe8e 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -11,6 +11,11 @@ on: branches: [main] paths: - 'src/everything/**' + # TypeScript release tags also trigger a run, comparing against the + # previous typescript-servers-* tag (auto-detected by mcp-server-diff). + # This surfaces the cumulative interface delta for each release. + tags: + - 'typescript-servers-*' pull_request: paths: - 'src/everything/**' From e5ca90c71eed9ecdd791a7e541730011b3412ac5 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 02:13:33 -0400 Subject: [PATCH 07/17] ci(everything): run the interface diff as an npm script on v2/main (#4860) Adapt #3260's workflow for v2/main: - Run the check as `npm run interface-diff` (scripts/interface-diff.mjs), driving the mcp-server-diff 3.0.0 CLI as an exact-pinned root devDependency instead of the SHA-pinned v2.3.5 action. The same script is a new last stage of local:gate, which workflow-gate.test.mjs now requires, so the gate keeps running every check CI runs. - Choose the base per event (the PR's base, the commit before a push, the previous release tag, or a dispatch input) instead of the tool's default merge-base with origin/main, which is wrong for v2/main PRs. - Trigger pushes on main and v2/main, PRs on any base, and published Releases instead of the retired typescript-servers-* tags. - Run the PR's code only in a read-only job with no persisted credentials; post the sticky comment from a separate job that holds pull-requests: write, runs no PR code, and only runs for same-repo PRs. Fork PRs keep the job summary and the report artifact. - Give every job a timeout. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 255 +++++++++++----- package-lock.json | 93 ++++++ package.json | 4 +- scripts/interface-diff.mjs | 345 ++++++++++++++++++++++ scripts/interface-diff.test.mjs | 127 ++++++++ scripts/lib/workflow-gate.test.mjs | 1 + 6 files changed, 749 insertions(+), 76 deletions(-) create mode 100644 scripts/interface-diff.mjs create mode 100644 scripts/interface-diff.test.mjs diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 798367fe8e..30f556c8e6 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -3,118 +3,223 @@ # This workflow helps reviewers see how tools, resources, prompts, and capabilities # evolve over time - useful for SDK compliance validation and catching regressions. # +# Contributed by @SamMorrowDrums in #3260 and adapted for v2/main in #4860: +# +# - The check is `npm run interface-diff` (scripts/interface-diff.mjs), which +# drives the `mcp-server-diff` CLI, an exact-pinned root devDependency. The +# same script is a `local:gate` stage, and `scripts/lib/workflow-gate.test.mjs` +# requires every npm script a workflow runs to be one, so CI and the gate run +# the same check. No third-party action runs here. +# - The base is chosen per event below rather than by the tool, whose default +# (the merge-base with origin/main) is the wrong base for PRs to v2/main. +# - Untrusted code (the PR's install, build and server) runs only in `diff`, +# which holds a read-only token and no persisted credentials. The sticky +# comment is posted by `comment`, which holds `pull-requests: write` and runs +# none of the PR's code, and only for PRs from this repository: a fork PR's +# token is read-only, so there the report is in the job summary and the +# `mcp-diff-report` artifact. `pull_request_target` is deliberately not used, +# since it would run the fork's build next to a write token. If fork PRs need +# the comment too, the way is a separate `workflow_run` workflow that posts +# the artifact without checking anything out. +# +# A changed interface does not fail the run: the diff is information for the +# reviewer. A diff that cannot run (a base that does not build, a server that +# cannot be probed) does. +# # See: https://github.com/modelcontextprotocol/inspector/issues/1034 name: Everything Server MCP Diff on: push: - branches: [main] + branches: [main, v2/main] paths: - - 'src/everything/**' - # TypeScript release tags also trigger a run, comparing against the - # previous typescript-servers-* tag (auto-detected by mcp-server-diff). - # This surfaces the cumulative interface delta for each release. - tags: - - 'typescript-servers-*' + - "src/everything/**" + - "package.json" + - "package-lock.json" + - "scripts/interface-diff.mjs" + - ".github/workflows/everything-mcp-diff.yml" + # No branch filter: PRs to v2/main and to main both run, and so does a PR + # stacked on another PR's branch. The base is the PR's own base either way. pull_request: paths: - - 'src/everything/**' + - "src/everything/**" + - "package.json" + - "package-lock.json" + - "scripts/interface-diff.mjs" + - ".github/workflows/everything-mcp-diff.yml" + # A published Release (a milestone, tagged vX.Y.Z on main; see RELEASING.md) + # surfaces the cumulative interface delta since the previous release's tag. + release: + types: [published] # Manual trigger for comparing any two refs (commits, tags, branches) workflow_dispatch: inputs: compare_ref: - description: 'Base ref to compare against (commit SHA, tag, or branch)' + description: "Base ref to compare against (commit SHA, tag, or branch; defaults to the merge-base with v2/main)" required: false - default: '' + default: "" target_ref: - description: 'Target ref to compare (defaults to current branch if empty)' + description: "Target ref to compare (defaults to the ref the workflow runs on)" required: false - default: '' + default: "" permissions: contents: read - pull-requests: write +# Superseded runs on the same PR are cancelled; pushes and releases each keep +# their own run. concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - mcp-diff: + diff: name: Diff Everything Server Interface + timeout-minutes: 15 runs-on: ubuntu-latest - + permissions: + contents: read steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v6 with: + # The base is built from history, so all of it is needed, tags + # included (the release base is the previous tag). fetch-depth: 0 - ref: ${{ github.event.inputs.target_ref || github.ref }} + ref: ${{ inputs.target_ref || github.ref }} + persist-credentials: false - - name: Run MCP Server Diff - id: mcp_diff - uses: SamMorrowDrums/mcp-server-diff@f7e5e58a4b0c4f68a5827adad2292953bb1ab9ef # v2.3.5 + - uses: actions/setup-node@v7 with: - setup_node: 'true' - node_version: '22' - install_command: cd src/everything && npm ci - build_command: cd src/everything && npm run build - start_command: node src/everything/dist/index.js stdio - compare_ref: ${{ github.event.inputs.compare_ref || '' }} - server_timeout: '15' + node-version: 22 + cache: npm - - name: Add summary context - if: always() + # A root `npm ci` runs every workspace's `prepare`, which builds it, so + # the head build the diff probes exists after this step. + - name: Install dependencies and build + run: npm ci + + - name: Choose the base to compare against + id: base + env: + EVENT: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + BEFORE: ${{ github.event.before }} + INPUT_BASE: ${{ inputs.compare_ref }} run: | - echo "" >> $GITHUB_STEP_SUMMARY - echo "---" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "โ„น๏ธ **Interpreting Results**" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The Everything server is a reference implementation that exercises all MCP features." >> $GITHUB_STEP_SUMMARY - echo "Interface changes here may indicate:" >> $GITHUB_STEP_SUMMARY - echo "- New MCP protocol features being demonstrated" >> $GITHUB_STEP_SUMMARY - echo "- Updated tool schemas or descriptions" >> $GITHUB_STEP_SUMMARY - echo "- New resources, prompts, or capabilities" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Review changes to ensure they align with the intended protocol updates." >> $GITHUB_STEP_SUMMARY + case "$EVENT" in + # The checkout is the PR's merge commit, so its base tip is + # exactly what the PR changes. + pull_request) base="$PR_BASE" ;; + # The branch as it was before this push; a new branch has none. + push) + if [ -n "$BEFORE" ] && [ "$BEFORE" != 0000000000000000000000000000000000000000 ]; then + base="$BEFORE" + else + base=$(git rev-parse HEAD^) + fi ;; + release) base=$(git describe --tags --abbrev=0 HEAD^) ;; + workflow_dispatch) + if [ -n "$INPUT_BASE" ]; then + base="$INPUT_BASE" + else + base=$(git merge-base HEAD origin/v2/main) + fi ;; + *) echo "unexpected event: $EVENT" >&2; exit 1 ;; + esac + echo "Comparing against $base" + echo "ref=$base" >> "$GITHUB_OUTPUT" - # Post (or update) a sticky PR comment so reviewers can see the diff without - # navigating to the Actions tab. The same comment is updated on subsequent - # pushes via the `header` key. Skipped for PRs from forks where GITHUB_TOKEN - # is read-only โ€” the report is still available in the workflow summary and - # as an uploaded artifact. - - name: Build PR comment body - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - id: comment_body + - name: Diff the interface + env: + BASE: ${{ steps.base.outputs.ref }} + run: npm run interface-diff -- --base "$BASE" --report-dir mcp-diff-report + + - name: Write the job summary + if: always() run: | { - echo 'body<Full report' - echo '' - tail -n +5 conformance-report/CONFORMANCE_REPORT.md - echo '' - echo '' - else - echo '_Report file not found โ€” see the workflow logs for details._' - fi - echo '' - echo "Updated by [\`${{ github.workflow }}\`](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) ยท commit \`${{ github.event.pull_request.head.sha }}\`" - echo 'MCP_DIFF_EOF' - } >> "$GITHUB_OUTPUT" + echo "" + echo "---" + echo "" + echo "โ„น๏ธ **Interpreting Results**" + echo "" + echo "The Everything server is a reference implementation that exercises all MCP features." + echo "Interface changes here may indicate:" + echo "- New MCP protocol features being demonstrated" + echo "- Updated tool schemas or descriptions" + echo "- New resources, prompts, or capabilities" + echo "" + echo "Review changes to ensure they align with the intended protocol updates." + } >> "$GITHUB_STEP_SUMMARY" - - name: Post / update sticky PR comment - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4 + - name: Upload the report + if: always() + uses: actions/upload-artifact@v7 + with: + name: mcp-diff-report + path: mcp-diff-report/ + if-no-files-found: ignore + + # Post (or update) a sticky PR comment so reviewers can see the diff without + # navigating to the Actions tab. One comment per PR, found again by its + # marker and edited on each push. This job checks nothing out and runs none + # of the PR's code; it only reads the report the `diff` job uploaded. + comment: + name: Post the diff on the PR + needs: diff + if: >- + always() && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + (needs.diff.result == 'success' || needs.diff.result == 'failure') + timeout-minutes: 5 + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Download the report + uses: actions/download-artifact@v8 + continue-on-error: true with: - header: everything-server-mcp-diff - message: ${{ steps.comment_body.outputs.body }} + name: mcp-diff-report + path: mcp-diff-report + + - name: Post / update sticky PR comment + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + marker='' + { + echo "$marker" + if [ -f mcp-diff-report/report.md ]; then + # A comment holds at most 65536 characters. + head -c 60000 mcp-diff-report/report.md + if [ "$(wc -c < mcp-diff-report/report.md)" -gt 60000 ]; then + echo "" + echo "_The report is truncated here; the full report is in the run's \`mcp-diff-report\` artifact._" + fi + else + echo "## \`everything\`: MCP interface diff" + echo "" + echo "โŒ The diff job produced no report; see the run's log." + fi + echo "" + echo "Updated by [this run]($RUN_URL) for \`$HEAD_SHA\`." + } > body.md + id=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n 1) + if [ -n "$id" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$id" -F body=@body.md > /dev/null + else + gh api "repos/$REPO/issues/$PR/comments" -F body=@body.md > /dev/null + fi diff --git a/package-lock.json b/package-lock.json index d1390cede6..6359f597db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "@modelcontextprotocol/client": "^2.3.1", "eslint": "^10.11.0", "globals": "^17.12.0", + "mcp-server-diff": "3.0.0", "prettier": "3.8.4", "proper-lockfile": "^4.1.2", "semver": "^7.8.4", @@ -31,6 +32,50 @@ "yaml": "^2.9.0" } }, + "node_modules/@actions/core": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-3.0.1.tgz", + "integrity": "sha512-a6d/Nwahm9fliVGRhdhofo40HjHQasUPusmc7vBfyky+7Z+P2A1J68zyFVaNcEclc/Se+eO595oAr5nwEIoIUA==", + "dev": true, + "dependencies": { + "@actions/exec": "^3.0.0", + "@actions/http-client": "^4.0.0" + } + }, + "node_modules/@actions/exec": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-3.0.0.tgz", + "integrity": "sha512-6xH/puSoNBXb72VPlZVm7vQ+svQpFyA96qdDBvhB8eNZOE8LtPf9L4oAsfzK/crCL8YZ+19fKYVnM63Sl+Xzlw==", + "dev": true, + "dependencies": { + "@actions/io": "^3.0.2" + } + }, + "node_modules/@actions/http-client": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-4.0.1.tgz", + "integrity": "sha512-+Nvd1ImaOZBSoPbsUtEhv+1z99H12xzncCkz0a3RuehINE81FZSe2QTj3uvAPTcJX/SCzUQHQ0D1GrPMbrPitg==", + "dev": true, + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^6.23.0" + } + }, + "node_modules/@actions/http-client/node_modules/undici": { + "version": "6.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.29.0.tgz", + "integrity": "sha512-R+RODBqp6i2pPflGdq+xIOUkl+RNfGgHwoinecKu/JCuf2uO06cOKoDbI2P7Dn6KcswdKwrczbU6IYJ6K8X+wg==", + "dev": true, + "engines": { + "node": ">=18.17" + } + }, + "node_modules/@actions/io": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-3.0.2.tgz", + "integrity": "sha512-nRBchcMM+QK1pdjO7/idu86rbJI5YHUKCvKs0KxnSYbVe3F51UfGxuZX4Qy/fWlp6l7gWFwIkrOzN+oUK03kfw==", + "dev": true + }, "node_modules/@babel/helper-string-parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", @@ -3805,6 +3850,36 @@ "node": ">= 0.4" } }, + "node_modules/mcp-server-diff": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mcp-server-diff/-/mcp-server-diff-3.0.0.tgz", + "integrity": "sha512-vtbf+ZSfRFMZQipzIpQDP5l35qMhW/CtemsERMr7DjU5Z+uY6ZtL8gHBAamwZ2sS3VvsWtVqMg1Hk91HbavZrw==", + "dev": true, + "dependencies": { + "@actions/core": "^3.0.1", + "@actions/exec": "^3.0.0", + "@actions/io": "^3.0.2", + "@modelcontextprotocol/sdk": "^1.13.2", + "diff": "^9.0.0", + "undici": "^8.4.1", + "zod": "^4.4.3" + }, + "bin": { + "mcp-server-diff": "dist/cli/index.js" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/mcp-server-diff/node_modules/diff": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", + "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -4972,6 +5047,15 @@ "license": "0BSD", "optional": true }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "dev": true, + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -5054,6 +5138,15 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/undici": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.11.2.tgz", + "integrity": "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ==", + "dev": true, + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index a1d225897a..49ed4ba99e 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,11 @@ "verify:action-pins": "node scripts/verify-action-pins.mjs", "verify:dco": "node scripts/verify-dco.mjs", "smoke": "node scripts/smoke-servers.mjs", + "interface-diff": "node scripts/interface-diff.mjs", "pack:verify": "node scripts/pack-and-verify.mjs", "release:notes": "node scripts/release-notes.mjs", "local:gate": "node scripts/gate-lease.mjs npm run local:gate:stages", - "local:gate:stages": "npm run verify:install-fresh && npm run verify:dco && npm run validate && npm run coverage && npm run validate:py && npm run coverage:py && npm run verify:skills:cli && npm run smoke" + "local:gate:stages": "npm run verify:install-fresh && npm run verify:dco && npm run validate && npm run coverage && npm run validate:py && npm run coverage:py && npm run verify:skills:cli && npm run smoke && npm run interface-diff" }, "dependencies": { "@modelcontextprotocol/server-everything": "*", @@ -61,6 +62,7 @@ "@modelcontextprotocol/client": "^2.3.1", "eslint": "^10.11.0", "globals": "^17.12.0", + "mcp-server-diff": "3.0.0", "prettier": "3.8.4", "proper-lockfile": "^4.1.2", "semver": "^7.8.4", diff --git a/scripts/interface-diff.mjs b/scripts/interface-diff.mjs new file mode 100644 index 0000000000..3b46b27bc0 --- /dev/null +++ b/scripts/interface-diff.mjs @@ -0,0 +1,345 @@ +#!/usr/bin/env node +// The `everything` server's MCP interface diff (#4860), adapted from +// @SamMorrowDrums's workflow in #3260. +// +// It snapshots the public interface of two builds of `everything` (the server +// capabilities from the handshake, and its tools, prompts, resources and +// resource templates) with `mcp-server-diff`, and reports what changed between +// them: the build in this checkout (the head) against the same server built +// from a base commit. The spec refactor (#4857) leans on it: an SDK migration +// that is meant to be transparent proves it with an empty diff, and each new +// 2026-07-28 feature shows up as a reviewable one. +// +// Why a script, and not the `mcp-server-diff` GitHub Action #3260 used: +// +// - `local:gate` must run every check CI runs, and +// `scripts/lib/workflow-gate.test.mjs` derives that from the npm scripts a +// workflow invokes. A check behind an action is invisible to it; a check +// behind `npm run interface-diff` is enforced by it. CI +// (`everything-mcp-diff.yml`) and the gate run this same file. +// - The tool comes in as an exact-pinned root devDependency, so it is held +// by `package-lock.json`'s integrity hash, the way every other npm +// dependency here is, and no third-party action runs in CI at all. +// - The action picks the base itself, as the merge-base with `origin/main`. +// PRs here target `v2/main`, so that base is wrong; the caller names it. +// +// The base is built from a plain export of the base commit's tree (no git +// worktree is registered) under `node_modules/.cache/`, which git, Prettier and +// ESLint all ignore, and removed afterwards. It sits inside the checkout rather +// than under the system temp directory because `mcp-server-diff` splits a +// server's start command on whitespace, and a relative path from the checkout +// cannot contain any, where a temp directory under a Windows profile can. +// +// The head is NOT built here: inside `local:gate` the `validate` stage has +// already built it, and CI's root `npm ci` builds it. A missing build is an +// error, as it is for the boot smoke. +// +// Outcome: +// - unchanged / changed: exit 0. A changed interface is information for the +// reviewer, not a failure (#3260 ran with `fail_on_diff: false` too). +// - error: exit 1. A server that cannot be probed, or a base that does not +// build, is a broken check, so it fails the gate and the CI job. +// +// Usage: +// npm run interface-diff # base: merge-base with origin/v2/main +// npm run interface-diff -- --base # any commit, tag or branch +// npm run interface-diff -- --report-dir # also write report.md and result.json + +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", +); + +/** The server this diff covers. Start with `everything` only (#4860). */ +export const SERVER_DIR = "src/everything"; + +/** The entry point a client launches, relative to a checkout's root. */ +const ENTRY = `${SERVER_DIR}/dist/index.js`; + +/** The base when none is named: where this branch left the develop line. */ +export const DEFAULT_BASE_BRANCH = "origin/v2/main"; + +/** + * Parse the command line. + * + * @param {string[]} argv + * @returns {{ base: string | null, reportDir: string | null }} + */ +export function parseArgs(argv) { + const opts = { base: null, reportDir: null }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + const [flag, inline] = arg.startsWith("--") ? arg.split(/=(.*)/s) : [arg]; + const value = () => { + const v = inline ?? argv[++i]; + if (v === undefined || v === "") throw new Error(`${flag} needs a value`); + return v; + }; + if (flag === "--base") opts.base = value(); + else if (flag === "--report-dir") opts.reportDir = value(); + else throw new Error(`unknown argument: ${arg}`); + } + return opts; +} + +/** + * Turn `mcp-server-diff`'s JSON output into this diff's verdict. + * + * The CLI exits 1 both when the interfaces differ and when a probe fails, so + * its exit code cannot tell a change from a broken check; the JSON can. + * + * @param {string} stdout what `mcp-server-diff -o json -q` printed + * @returns {{ status: "unchanged" | "changed" | "error", error?: string, + * diffs: { endpoint: string, diff: string }[], + * baseCounts?: object, headCounts?: object }} + */ +export function classify(stdout) { + let parsed; + try { + parsed = JSON.parse(stdout); + } catch { + return { + status: "error", + error: `mcp-server-diff printed no JSON report:\n${stdout.trim().slice(0, 2000)}`, + diffs: [], + }; + } + const result = parsed?.results?.[0]; + if (!result || typeof result !== "object") + return { + status: "error", + error: "mcp-server-diff reported no comparison", + diffs: [], + }; + const diffs = Array.isArray(result.diffs) ? result.diffs : []; + if (result.error) + return { status: "error", error: String(result.error), diffs }; + return { + status: result.hasDifferences ? "changed" : "unchanged", + diffs, + baseCounts: result.baseCounts, + headCounts: result.targetCounts, + }; +} + +/** A Markdown code fence longer than any run of backticks in `text`. */ +export function fenceFor(text) { + const longest = Math.max( + 0, + ...[...text.matchAll(/`+/g)].map((m) => m[0].length), + ); + return "`".repeat(Math.max(3, longest + 1)); +} + +/** + * The human-readable report: the verdict first, then the counts, then each + * part of the interface that changed as a diff. + * + * @param {ReturnType} verdict + * @param {{ base: string, head: string }} labels + */ +export function renderReport(verdict, { base, head }) { + const lines = ["## `everything`: MCP interface diff", ""]; + lines.push(`Base: ${base} `, `Head: ${head}`, ""); + if (verdict.status === "unchanged") + lines.push("โœ… **No interface changes detected.**"); + else if (verdict.status === "changed") + lines.push( + `โš ๏ธ **Interface changes detected** in ${verdict.diffs.length} part(s) of the interface. Review them below to confirm they are intended.`, + ); + else lines.push("โŒ **The interface diff could not run.**"); + lines.push(""); + + if (verdict.status === "error") { + const fence = fenceFor(verdict.error ?? ""); + lines.push(fence, verdict.error ?? "", fence, ""); + return lines.join("\n"); + } + + const row = (name, c = {}) => + `| ${name} | ${c.tools ?? "?"} | ${c.prompts ?? "?"} | ${c.resources ?? "?"} | ${c.resourceTemplates ?? "?"} |`; + lines.push( + "| | Tools | Prompts | Resources | Resource templates |", + "| --- | --- | --- | --- | --- |", + row("Base", verdict.baseCounts), + row("Head", verdict.headCounts), + "", + ); + + for (const { endpoint, diff } of verdict.diffs) { + const fence = fenceFor(diff); + lines.push(`### ${endpoint}`, "", `${fence}diff`, diff, fence, ""); + } + return lines.join("\n"); +} + +/** Run a command to completion; throw with its output when it fails. */ +function run(command, args, cwd, env) { + const res = spawnSync(command, args, { + cwd, + env: env ?? process.env, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + shell: process.platform === "win32", + }); + if (res.error) throw res.error; + if (res.status !== 0) + throw new Error( + `\`${command} ${args.join(" ")}\` exited ${res.status}:\n${(res.stderr || res.stdout).trim().split("\n").slice(-25).join("\n")}`, + ); + return res.stdout; +} + +const git = (...args) => run("git", args, repoRoot).trim(); + +/** The commit to compare against, as a full SHA. */ +function resolveBase(base) { + if (base) return git("rev-parse", "--verify", `${base}^{commit}`); + try { + git("rev-parse", "--verify", DEFAULT_BASE_BRANCH); + } catch { + throw new Error( + `${DEFAULT_BASE_BRANCH} is not available to compare against. Run \`git fetch origin v2/main\`, or name a base with --base .`, + ); + } + return git("merge-base", "HEAD", DEFAULT_BASE_BRANCH); +} + +/** + * Build `everything` as it was at `sha` into `dir`: export the commit's tree + * through a throwaway index (no worktree registered, the real index + * untouched), install that workspace's locked dependencies, build it. + */ +function buildBase(sha, dir) { + const index = `${dir}.index`; + const env = { ...process.env, GIT_INDEX_FILE: index }; + try { + run("git", ["read-tree", sha], repoRoot, env); + run( + "git", + ["checkout-index", "--all", `--prefix=${dir}${path.sep}`], + repoRoot, + env, + ); + } finally { + rmSync(index, { force: true }); + } + if (!existsSync(path.join(dir, SERVER_DIR, "package.json"))) + throw new Error(`${SERVER_DIR} does not exist at the base ${sha}`); + run( + "npm", + [ + "ci", + "--workspace", + SERVER_DIR, + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--prefer-offline", + ], + dir, + ); + run("npm", ["run", "build", "--workspace", SERVER_DIR], dir); +} + +/** Where the pinned `mcp-server-diff` CLI lives. */ +function cliPath() { + const require = createRequire(import.meta.url); + const manifest = require.resolve("mcp-server-diff/package.json"); + const { bin } = require("mcp-server-diff/package.json"); + const rel = typeof bin === "string" ? bin : bin["mcp-server-diff"]; + return path.join(path.dirname(manifest), rel); +} + +/** A short, readable label for a commit. */ +function describe(sha) { + const subject = git("log", "-1", "--format=%s", sha); + return `\`${sha.slice(0, 12)}\` (${subject.replaceAll("`", "'")})`; +} + +export async function main(argv = process.argv.slice(2)) { + let opts; + try { + opts = parseArgs(argv); + } catch (err) { + console.error(`interface-diff: ${err.message}`); + return 2; + } + + let verdict; + let labels = { base: "(unresolved)", head: "(unresolved)" }; + const cacheRoot = path.join(repoRoot, "node_modules", ".cache"); + let work; + try { + if (!existsSync(path.join(repoRoot, ENTRY))) + throw new Error( + `${ENTRY} does not exist โ€” build first (npm run build -w ${SERVER_DIR}).`, + ); + const sha = resolveBase(opts.base); + const headSha = git("rev-parse", "HEAD"); + labels = { + base: describe(sha), + head: `the build in this checkout, at ${describe(headSha)}`, + }; + console.error(`interface-diff: building the base ${sha.slice(0, 12)} โ€ฆ`); + + mkdirSync(cacheRoot, { recursive: true }); + work = mkdtempSync(path.join(cacheRoot, "servers-interface-diff-")); + buildBase(sha, work); + + // Both start commands are relative to the checkout, so they hold no + // whitespace for mcp-server-diff to split on (see the header). + const baseEntry = path + .relative(repoRoot, path.join(work, ENTRY)) + .split(path.sep) + .join("/"); + console.error("interface-diff: probing the base and the head โ€ฆ"); + const res = spawnSync( + process.execPath, + [ + cliPath(), + "--base", + `node ${baseEntry} stdio`, + "--target", + `node ${ENTRY} stdio`, + "--output", + "json", + "--quiet", + ], + { cwd: repoRoot, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }, + ); + if (res.error) throw res.error; + verdict = classify(res.stdout); + } catch (err) { + verdict = { status: "error", error: err.message, diffs: [] }; + } finally { + if (work) rmSync(work, { recursive: true, force: true }); + } + + const report = renderReport(verdict, labels); + console.log(report); + if (opts.reportDir) { + mkdirSync(opts.reportDir, { recursive: true }); + writeFileSync(path.join(opts.reportDir, "report.md"), `${report}\n`); + writeFileSync( + path.join(opts.reportDir, "result.json"), + `${JSON.stringify({ status: verdict.status, diffCount: verdict.diffs.length }, null, 2)}\n`, + ); + } + return verdict.status === "error" ? 1 : 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) + process.exitCode = await main(); diff --git a/scripts/interface-diff.test.mjs b/scripts/interface-diff.test.mjs new file mode 100644 index 0000000000..e30008c34e --- /dev/null +++ b/scripts/interface-diff.test.mjs @@ -0,0 +1,127 @@ +// Tests for the `everything` interface diff (#4860). The pure halves are +// exercised here: the argument parser, the verdict drawn from +// `mcp-server-diff`'s JSON (the one thing that decides whether the gate and +// the CI job fail), and the report. Building a base and probing two servers is +// what `npm run interface-diff` itself does as a gate stage, so it is not +// repeated here. Run via `npm run test:scripts`. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + classify, + fenceFor, + main, + parseArgs, + renderReport, +} from "./interface-diff.mjs"; + +const counts = { tools: 13, prompts: 4, resources: 7, resourceTemplates: 2 }; +const cliJson = (result) => + JSON.stringify({ timestamp: "t", results: [result], summary: {} }); + +test("parseArgs reads both flags, in either spelling", () => { + assert.deepEqual(parseArgs([]), { base: null, reportDir: null }); + assert.deepEqual(parseArgs(["--base", "abc", "--report-dir", "out"]), { + base: "abc", + reportDir: "out", + }); + assert.deepEqual(parseArgs(["--base=v1.0.0", "--report-dir=a=b"]), { + base: "v1.0.0", + reportDir: "a=b", + }); +}); + +test("parseArgs refuses an unknown flag or a missing value", () => { + assert.throws(() => parseArgs(["--fail-on-diff"]), /unknown argument/); + assert.throws(() => parseArgs(["--base"]), /--base needs a value/); + assert.throws(() => parseArgs(["--base="]), /--base needs a value/); +}); + +test("main exits 2 on a bad command line, before building anything", async () => { + assert.equal(await main(["--nope"]), 2); +}); + +test("an identical interface is unchanged", () => { + const v = classify( + cliJson({ + hasDifferences: false, + baseCounts: counts, + targetCounts: counts, + diffs: [], + }), + ); + assert.equal(v.status, "unchanged"); + assert.deepEqual(v.headCounts, counts); +}); + +test("a difference is a change, not an error", () => { + const diffs = [{ endpoint: "tools", diff: "- a\n+ b" }]; + const v = classify( + cliJson({ + hasDifferences: true, + baseCounts: counts, + targetCounts: counts, + diffs, + }), + ); + assert.equal(v.status, "changed"); + assert.deepEqual(v.diffs, diffs); +}); + +test("a probe failure is an error, though the CLI reports it as a difference", () => { + // mcp-server-diff sets hasDifferences on a failed probe too, and exits 1 + // for both; only the `error` field tells them apart. + const v = classify( + cliJson({ + hasDifferences: true, + error: "Target probe failed: boom", + diffs: [{ endpoint: "error", diff: "Target probe failed: boom" }], + }), + ); + assert.equal(v.status, "error"); + assert.match(v.error, /boom/); +}); + +test("output that is not the JSON report is an error, never a pass", () => { + assert.equal(classify("").status, "error"); + assert.equal(classify("Fatal error: x").status, "error"); + assert.equal(classify("{}").status, "error"); + assert.equal(classify(JSON.stringify({ results: [] })).status, "error"); +}); + +test("fenceFor outruns any backticks in the content", () => { + assert.equal(fenceFor("plain"), "```"); + assert.equal(fenceFor("a ``` b"), "````"); + assert.equal(fenceFor("`````"), "``````"); +}); + +test("the report leads with the verdict and shows each changed part", () => { + const labels = { base: "`base`", head: "`head`" }; + assert.match( + renderReport( + { + status: "unchanged", + diffs: [], + baseCounts: counts, + headCounts: counts, + }, + labels, + ), + /No interface changes detected[\s\S]*\| Head \| 13 \| 4 \| 7 \| 2 \|/, + ); + const changed = renderReport( + { + status: "changed", + diffs: [{ endpoint: "tools", diff: "- x ``` y" }], + baseCounts: counts, + headCounts: counts, + }, + labels, + ); + assert.match(changed, /Interface changes detected/); + assert.match(changed, /### tools\n\n````diff\n- x ``` y\n````/); + assert.match( + renderReport({ status: "error", error: "boom", diffs: [] }, labels), + /could not run[\s\S]*boom/, + ); +}); diff --git a/scripts/lib/workflow-gate.test.mjs b/scripts/lib/workflow-gate.test.mjs index 6dee0ca4fd..c08ae56186 100644 --- a/scripts/lib/workflow-gate.test.mjs +++ b/scripts/lib/workflow-gate.test.mjs @@ -392,6 +392,7 @@ describe("the gate's name", () => { "coverage:py", "verify:skills:cli", "smoke", + "interface-diff", ]) { it(`runs \`${stage}\``, () => { assert.ok( From dde6224267e814de530f67ee6def459329f41c5a Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 02:13:33 -0400 Subject: [PATCH 08/17] docs: record the interface diff as a gate stage (#4860) Add `interface-diff` to the stage table in docs/quality-gate.md, to AGENTS.md's Before pushing and project tree, and a diagnosis section to the pre-push-gate skill. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .claude/skills/pre-push-gate/SKILL.md | 25 +++++++++++++++++++++++++ AGENTS.md | 17 +++++++++-------- docs/quality-gate.md | 18 +++++++++++++++--- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/.claude/skills/pre-push-gate/SKILL.md b/.claude/skills/pre-push-gate/SKILL.md index c7198ad3a9..ddb54b47ca 100644 --- a/.claude/skills/pre-push-gate/SKILL.md +++ b/.claude/skills/pre-push-gate/SKILL.md @@ -273,6 +273,31 @@ of the server's stderr. Re-run a subset with - **A new server fails `test:scripts`** with "every server directory in the repo has a smoke entry": add it to `SERVERS` in `scripts/smoke-servers.mjs`. +### `interface-diff` + +The last stage diffs the `everything` server's MCP interface (capabilities, +tools, prompts, resources, resource templates) in this checkout's build +against the same server built from the merge-base with `origin/v2/main`, and +prints the report. **A changed interface does not fail it**: the report is +what the PR's sticky comment will show, so read it, and if a change is not +one you meant, that is the finding. It fails only when the diff cannot run, +with `โŒ The interface diff could not run` and the reason: + +- **"origin/v2/main is not available"**: `git fetch origin v2/main`. +- **"src/everything/dist/index.js does not exist"**: you ran it on its own + before building: `npm run build -w src/everything`. +- **An `npm ci` or `npm run build` failure**: the _base_ did not install or + build. Offline with a cold npm cache is the usual cause; otherwise name a + base that builds with `npm run interface-diff -- --base `. +- **"Base server probe failed" / "Target probe failed"**: a server did not + answer the probe. A target failure is this checkout's server and is the + real defect; the smoke stage before it usually fails first. + +Re-run it on its own with `npm run interface-diff` (about ten seconds). What +it compares against in CI, and why it is a script rather than an action, is +in [`docs/quality-gate.md`](../../../docs/quality-gate.md) and the header of +`scripts/interface-diff.mjs`. + ## Waiting on the lease A gate that starts with diff --git a/AGENTS.md b/AGENTS.md index bba09c93db..7b4abbd327 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,7 @@ servers/ โ”‚ โ””โ”€โ”€ time/ Py mcp-server-time PyPI Time and timezone conversion โ”œโ”€โ”€ .claude/skills/ On-demand procedures (see the Skills index above) โ”œโ”€โ”€ .changeset/ Pending changesets for the TypeScript servers, and the changesets config -โ”œโ”€โ”€ scripts/ The pre-push gate (gate-lease, smoke-servers, validate-py, coverage-py), its guards (verify-*), +โ”œโ”€โ”€ scripts/ The pre-push gate (gate-lease, smoke-servers, validate-py, coverage-py, interface-diff), its guards (verify-*), โ”‚ the skills tooling, release tooling (npm-publish-guard, prepare-python-release, โ”‚ pack-and-verify, release-manifest, release-notes), and the issue-filing sweeps โ”‚ (dependency-refresh, dependabot-alerts, sdk-watch) @@ -61,8 +61,8 @@ servers/ โ”‚ overview of how the rules, skills, gates and sweeps fit together โ”œโ”€โ”€ .github/workflows/ typescript.yml, python.yml (per-package CI), release.yml (publishes on a GitHub Release), โ”‚ version-packages.yml (the changesets PR), prepare-python-release.yml (the CalVer PR), -โ”‚ claude.yml (@claude mentions), and the scheduled sweeps that file issues: -โ”‚ dependency-refresh.yml, dependabot-alerts.yml, sdk-watch.yml +โ”‚ claude.yml (@claude mentions), everything-mcp-diff.yml (everything's interface diff on PRs), +โ”‚ and the scheduled sweeps that file issues: dependency-refresh.yml, dependabot-alerts.yml, sdk-watch.yml โ”œโ”€โ”€ .github/ISSUE_TEMPLATE/ Bug and feature issue forms; config.yml routes security and new servers away โ”œโ”€โ”€ .github/pull_request_template.md The "issues, not PRs" banner that turns outside PRs away โ”œโ”€โ”€ RELEASING.md How packages are versioned and published, and how to recover a failed publish @@ -129,14 +129,15 @@ workspace's format check, lint, typecheck, build and tests), `coverage` (each TypeScript workspace's per-file coverage gate), `validate:py` (each Python server's locked sync, `ruff check`, `ruff format --check`, pyright, pytest and build), `coverage:py` (each Python server's per-file -coverage gate), `verify:skills:cli`, and `smoke` (every server -booted over each transport it implements). The stage-by-stage reference, and -what CI runs where, is [`docs/quality-gate.md`](./docs/quality-gate.md); +coverage gate), `verify:skills:cli`, `smoke` (every server booted over each +transport it implements), and `interface-diff` (the `everything` server's MCP +interface diffed against the base it branched from). The stage-by-stage +reference, and what CI runs where, is [`docs/quality-gate.md`](./docs/quality-gate.md); diagnosing a red stage is the `pre-push-gate` skill. - **`npm run validate` and the per-package commands above are the inner loop, - not a substitute.** They skip the other language, the pinned skills validator - and the boot smoke. + not a substitute.** They skip the other language, the pinned skills validator, + the boot smoke and the interface diff. - **Format before committing**: `npm run format` at the root (TypeScript), and `uv run --frozen ruff format .` in a Python server you changed. The gate only checks formatting; it never rewrites files. diff --git a/docs/quality-gate.md b/docs/quality-gate.md index 075b72de24..7c197a5b05 100644 --- a/docs/quality-gate.md +++ b/docs/quality-gate.md @@ -13,7 +13,7 @@ out; everything else points here. | --- | --- | --- | --- | | Inner loop | `npm run validate -w src/`, `npm run validate:py -- ` | Your machine | While iterating on one server | | Pre-push gate | `npm run local:gate` | Your machine, under a lease | Before every push | -| CI | `.github/workflows/typescript.yml`, `python.yml`; `dco.yml` | GitHub, on every push and pull request; `dco.yml` on pull requests only | Before merge | +| CI | `.github/workflows/typescript.yml`, `python.yml`, `everything-mcp-diff.yml`; `dco.yml` | GitHub, on every push and pull request (`everything-mcp-diff.yml`: when `everything` or what builds it changes); `dco.yml` on pull requests only | Before merge | **`npm run local:gate` runs every check CI runs.** CI splits the same checks into parallel jobs on a fresh install; the gate runs them in sequence on your @@ -49,6 +49,7 @@ in this order, and the first failure stops the run. | 7 | `coverage:py` | Per Python server: `uv sync --locked`, then `uv run --frozen pytest --cov --cov-report=term-missing --cov-report=json`; then **every file** in `coverage.json` must reach **90% on lines and 90% on branches** | `python.yml` โ†’ **Coverage \** (one leg each) | | 8 | `verify:skills:cli` | `claude plugin validate` on `.claude/skills`, at the pinned CLI version | `typescript.yml` โ†’ **Root guards** | | 9 | `smoke` | Every server boots over each transport it implements and answers one tool call | `typescript.yml` โ†’ **Boot smoke** | +| 10 | `interface-diff` | The `everything` server's MCP interface (capabilities, tools, prompts, resources, resource templates) in this checkout's build, diffed against the same server built from the base. It fails only when the diff cannot run; a changed interface is reported, not failed | `everything-mcp-diff.yml` โ†’ **Diff Everything Server Interface** (and the PR comment) | Notes on the stages: @@ -91,8 +92,9 @@ Notes on the stages: when a server's environment is missing or behind its lockfile, since `uv sync --locked` then downloads packages (and the pinned interpreter, if `uv` does not have it); `coverage:py` begins with the same sync, which is a - no-op once `validate:py` has run. With warm caches those are the only stages - that can fail offline. + no-op once `validate:py` has run; and `interface-diff` needs it for the + base's `npm ci`. With warm caches those are the only stages that can fail + offline. - **`smoke` launches what a user launches**: the built `dist/index.js` for a TypeScript server, the console script through `uv run --no-sync` for a Python one. stdio for all seven; HTTP+SSE and Streamable HTTP as well for @@ -102,6 +104,16 @@ Notes on the stages: than the default 3001. It runs after `validate` and `validate:py` because it needs the build and each Python server's synced environment; it creates neither. +- **`interface-diff` compares against the merge-base with `origin/v2/main`** + locally (`-- --base ` names another), and against the PR's base, the + commit before the push, or the previous release's tag in CI. It builds the + base from an export of that commit under `node_modules/.cache/` and removes + it afterwards; the head is the build `validate` made. It drives the + `mcp-server-diff` CLI, an exact-pinned root devDependency, which probes with + the 2026-07-28 `server/discover` first and falls back to the `initialize` + handshake, so it covers a server in either era. Its `npm ci` of the base + needs the network when the npm cache is cold, and it needs + `origin/v2/main` fetched. Only `everything` is diffed today (#4860). - `python.yml` also has a **Build \** job per server that re-runs pyright and `uv build` and uploads the built distribution. It checks nothing stage 6 does not. From b35e29de98458b5ca86f0d926b6e899401bb4ab3 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 02:14:23 -0400 Subject: [PATCH 09/17] ci(everything): diff a Release against the previous Release (#4860) `git describe` finds the nearest ancestor tag, but the date-stamped releases tagged commits that never reached main, so from main it lands on typescript-servers-0.6.2 (2024). Take the previous published Release's tag from the Releases list instead. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 30f556c8e6..27107e0026 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -48,7 +48,9 @@ on: - "scripts/interface-diff.mjs" - ".github/workflows/everything-mcp-diff.yml" # A published Release (a milestone, tagged vX.Y.Z on main; see RELEASING.md) - # surfaces the cumulative interface delta since the previous release's tag. + # surfaces the cumulative interface delta since the previous Release. This + # replaces #3260's `typescript-servers-*` tag trigger: those tags stopped in + # 2024, and no release since has used them. release: types: [published] # Manual trigger for comparing any two refs (commits, tags, branches) @@ -84,7 +86,7 @@ jobs: uses: actions/checkout@v6 with: # The base is built from history, so all of it is needed, tags - # included (the release base is the previous tag). + # included (the release base is the previous Release's tag). fetch-depth: 0 ref: ${{ inputs.target_ref || github.ref }} persist-credentials: false @@ -106,6 +108,9 @@ jobs: PR_BASE: ${{ github.event.pull_request.base.sha }} BEFORE: ${{ github.event.before }} INPUT_BASE: ${{ inputs.compare_ref }} + TAG: ${{ github.event.release.tag_name }} + REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} run: | case "$EVENT" in # The checkout is the PR's merge commit, so its base tip is @@ -118,7 +123,14 @@ jobs: else base=$(git rev-parse HEAD^) fi ;; - release) base=$(git describe --tags --abbrev=0 HEAD^) ;; + # The previous published Release's tag. Not `git describe`: the + # date-stamped releases tagged commits that never reached main, so + # the nearest ancestor tag is years older than the last release. + release) + base=$(gh release list --repo "$REPO" --exclude-drafts --exclude-pre-releases --limit 50 \ + --json tagName,publishedAt \ + --jq "map(select(.tagName != \"$TAG\")) | sort_by(.publishedAt) | last | .tagName // empty") + if [ -z "$base" ]; then echo "no earlier Release to compare against" >&2; exit 1; fi ;; workflow_dispatch) if [ -n "$INPUT_BASE" ]; then base="$INPUT_BASE" From f242458df7869e57b15f001c558ea277a0426885 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 03:17:42 -0400 Subject: [PATCH 10/17] fix(interface-diff): check out the event's SHA; report the CLI's stderr (#4860) Copilot round 1 on #4975: - Check out github.sha rather than github.ref, so a push run diffs the commit it was triggered for against that event's `before`, not a branch tip a later push moved. - When mcp-server-diff prints no JSON report, include its stderr (where a fatal error goes) in the error, so the reason is not blank. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 4 +++- scripts/interface-diff.mjs | 12 +++++++++--- scripts/interface-diff.test.mjs | 6 ++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 27107e0026..43fae86a20 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -88,7 +88,9 @@ jobs: # The base is built from history, so all of it is needed, tags # included (the release base is the previous Release's tag). fetch-depth: 0 - ref: ${{ inputs.target_ref || github.ref }} + # The event's own commit (the merge commit for a PR), not a branch + # ref that a later push could move before this step runs. + ref: ${{ inputs.target_ref || github.sha }} persist-credentials: false - uses: actions/setup-node@v7 diff --git a/scripts/interface-diff.mjs b/scripts/interface-diff.mjs index 3b46b27bc0..ce4c9ff63c 100644 --- a/scripts/interface-diff.mjs +++ b/scripts/interface-diff.mjs @@ -101,18 +101,24 @@ export function parseArgs(argv) { * its exit code cannot tell a change from a broken check; the JSON can. * * @param {string} stdout what `mcp-server-diff -o json -q` printed + * @param {string} [stderr] what it printed to stderr: a fatal error before the + * report goes there, so it is the reason when no report came out * @returns {{ status: "unchanged" | "changed" | "error", error?: string, * diffs: { endpoint: string, diff: string }[], * baseCounts?: object, headCounts?: object }} */ -export function classify(stdout) { +export function classify(stdout, stderr = "") { let parsed; try { parsed = JSON.parse(stdout); } catch { return { status: "error", - error: `mcp-server-diff printed no JSON report:\n${stdout.trim().slice(0, 2000)}`, + error: `mcp-server-diff printed no JSON report:\n${[stderr, stdout] + .map((s) => s.trim()) + .filter(Boolean) + .join("\n") + .slice(0, 2000)}`, diffs: [], }; } @@ -321,7 +327,7 @@ export async function main(argv = process.argv.slice(2)) { { cwd: repoRoot, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }, ); if (res.error) throw res.error; - verdict = classify(res.stdout); + verdict = classify(res.stdout, res.stderr); } catch (err) { verdict = { status: "error", error: err.message, diffs: [] }; } finally { diff --git a/scripts/interface-diff.test.mjs b/scripts/interface-diff.test.mjs index e30008c34e..0d36cbffbf 100644 --- a/scripts/interface-diff.test.mjs +++ b/scripts/interface-diff.test.mjs @@ -89,6 +89,12 @@ test("output that is not the JSON report is an error, never a pass", () => { assert.equal(classify(JSON.stringify({ results: [] })).status, "error"); }); +test("a CLI crash before the report carries its stderr as the reason", () => { + const v = classify("", "Fatal error: Error: spawn node ENOENT"); + assert.equal(v.status, "error"); + assert.match(v.error, /no JSON report[\s\S]*spawn node ENOENT/); +}); + test("fenceFor outruns any backticks in the content", () => { assert.equal(fenceFor("plain"), "```"); assert.equal(fenceFor("a ``` b"), "````"); From 0c2a61d62bd7206d9f43a4a2b94bf55cbed2c4b9 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 03:25:44 -0400 Subject: [PATCH 11/17] fix(interface-diff): diff a Release against its own predecessor (#4860) Copilot round 2 on #4975: taking the latest Release other than the current one picks a newer Release when an older one's run is re-run. Keep only Releases published before this event's Release. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 43fae86a20..adbc391f65 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -111,6 +111,7 @@ jobs: BEFORE: ${{ github.event.before }} INPUT_BASE: ${{ inputs.compare_ref }} TAG: ${{ github.event.release.tag_name }} + PUBLISHED: ${{ github.event.release.published_at }} REPO: ${{ github.repository }} GH_TOKEN: ${{ github.token }} run: | @@ -125,13 +126,15 @@ jobs: else base=$(git rev-parse HEAD^) fi ;; - # The previous published Release's tag. Not `git describe`: the - # date-stamped releases tagged commits that never reached main, so - # the nearest ancestor tag is years older than the last release. + # The Release published just before this one (so a re-run of an + # older Release still compares against ITS predecessor). Not + # `git describe`: the date-stamped releases tagged commits that + # never reached main, so the nearest ancestor tag is years older + # than the last release. release) base=$(gh release list --repo "$REPO" --exclude-drafts --exclude-pre-releases --limit 50 \ --json tagName,publishedAt \ - --jq "map(select(.tagName != \"$TAG\")) | sort_by(.publishedAt) | last | .tagName // empty") + --jq "map(select(.tagName != \"$TAG\" and .publishedAt < \"$PUBLISHED\")) | sort_by(.publishedAt) | last | .tagName // empty") if [ -z "$base" ]; then echo "no earlier Release to compare against" >&2; exit 1; fi ;; workflow_dispatch) if [ -n "$INPUT_BASE" ]; then From a793ed4d99ab997472c5a969f797791581028d59 Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 03:34:27 -0400 Subject: [PATCH 12/17] fix(interface-diff): Dependabot, root tsconfig, Windows quoting (#4860) Copilot round 3 on #4975 ("previously missed"): - Skip the comment job on Dependabot PRs, which are same-repo but get a read-only token; they keep the summary and artifact like fork PRs. - Trigger on the root tsconfig.json, which everything's tsconfig extends. - Spawn only npm through a shell on Windows, with its arguments quoted by winShellArgs; git needs no shell. - Silence the expected usage error in the script test. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 5 +++++ scripts/interface-diff.mjs | 12 +++++++++--- scripts/interface-diff.test.mjs | 5 ++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index adbc391f65..8add0835c9 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -36,6 +36,7 @@ on: - "src/everything/**" - "package.json" - "package-lock.json" + - "tsconfig.json" - "scripts/interface-diff.mjs" - ".github/workflows/everything-mcp-diff.yml" # No branch filter: PRs to v2/main and to main both run, and so does a PR @@ -45,6 +46,7 @@ on: - "src/everything/**" - "package.json" - "package-lock.json" + - "tsconfig.json" - "scripts/interface-diff.mjs" - ".github/workflows/everything-mcp-diff.yml" # A published Release (a milestone, tagged vX.Y.Z on main; see RELEASING.md) @@ -187,6 +189,8 @@ jobs: # navigating to the Actions tab. One comment per PR, found again by its # marker and edited on each push. This job checks nothing out and runs none # of the PR's code; it only reads the report the `diff` job uploaded. + # Dependabot's PRs are same-repo but get a read-only token, so they are + # skipped like fork PRs and keep the job summary and the artifact. comment: name: Post the diff on the PR needs: diff @@ -194,6 +198,7 @@ jobs: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]' && (needs.diff.result == 'success' || needs.diff.result == 'failure') timeout-minutes: 5 runs-on: ubuntu-latest diff --git a/scripts/interface-diff.mjs b/scripts/interface-diff.mjs index ce4c9ff63c..0f40d0d904 100644 --- a/scripts/interface-diff.mjs +++ b/scripts/interface-diff.mjs @@ -56,6 +56,7 @@ import { import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; +import { winShellArgs } from "./lib/win-shell-args.mjs"; const repoRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -191,14 +192,19 @@ export function renderReport(verdict, { base, head }) { return lines.join("\n"); } -/** Run a command to completion; throw with its output when it fails. */ +/** + * Run a command to completion; throw with its output when it fails. Only + * `npm` needs a shell, and only on Windows (it is a `.cmd` shim there); its + * arguments are then quoted for `cmd.exe`. `git` is spawned without one. + */ function run(command, args, cwd, env) { - const res = spawnSync(command, args, { + const shell = process.platform === "win32" && command === "npm"; + const res = spawnSync(command, shell ? winShellArgs(args) : args, { cwd, env: env ?? process.env, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, - shell: process.platform === "win32", + shell, }); if (res.error) throw res.error; if (res.status !== 0) diff --git a/scripts/interface-diff.test.mjs b/scripts/interface-diff.test.mjs index 0d36cbffbf..20134909a5 100644 --- a/scripts/interface-diff.test.mjs +++ b/scripts/interface-diff.test.mjs @@ -37,8 +37,11 @@ test("parseArgs refuses an unknown flag or a missing value", () => { assert.throws(() => parseArgs(["--base="]), /--base needs a value/); }); -test("main exits 2 on a bad command line, before building anything", async () => { +test("main exits 2 on a bad command line, before building anything", async (t) => { + // The usage error it prints is expected; keep it out of the test output. + const error = t.mock.method(console, "error", () => {}); assert.equal(await main(["--nope"]), 2); + assert.match(String(error.mock.calls[0]?.arguments[0]), /unknown argument/); }); test("an identical interface is unchanged", () => { From b15cf214094b773487490c24ba79aa9d81ca5f2e Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 03:44:53 -0400 Subject: [PATCH 13/17] fix(interface-diff): give each push and release run its own group (#4860) Copilot round 4 on #4975: a concurrency group keeps only one pending run, so a third rapid push cancelled the queued one even without cancel-in-progress. Group by ref only for PRs; by run id otherwise. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 8add0835c9..9520b235c7 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -70,10 +70,11 @@ on: permissions: contents: read -# Superseded runs on the same PR are cancelled; pushes and releases each keep -# their own run. +# Superseded runs on the same PR are cancelled. Every other run gets a group +# of its own: a group holds one running and one pending run, so a shared group +# would drop a push's run when two more queued behind it. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: From fe4cef392799f5462a2fa61745aa9f4fc06ab6cf Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 03:51:55 -0400 Subject: [PATCH 14/17] fix(interface-diff): never pass a report without a verdict (#4860) Copilot round 5 on #4975: - A result with no boolean hasDifferences is an error, not "unchanged". - On a probe failure, report the CLI's contextual message ("Base server probe failed" / "Target probe failed") rather than the bare exception. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- scripts/interface-diff.mjs | 15 +++++++++++++-- scripts/interface-diff.test.mjs | 6 ++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/interface-diff.mjs b/scripts/interface-diff.mjs index 0f40d0d904..58c084f127 100644 --- a/scripts/interface-diff.mjs +++ b/scripts/interface-diff.mjs @@ -131,8 +131,19 @@ export function classify(stdout, stderr = "") { diffs: [], }; const diffs = Array.isArray(result.diffs) ? result.diffs : []; - if (result.error) - return { status: "error", error: String(result.error), diffs }; + if (result.error) { + // `error` is the bare exception; the entry in `diffs` says which build + // failed ("Base server probe failed: โ€ฆ" / "Target probe failed: โ€ฆ"). + const context = diffs.find((d) => d.endpoint === "error")?.diff; + return { status: "error", error: String(context ?? result.error), diffs }; + } + // A report without a verdict is not a pass. + if (typeof result.hasDifferences !== "boolean") + return { + status: "error", + error: "mcp-server-diff's report has no hasDifferences verdict", + diffs, + }; return { status: result.hasDifferences ? "changed" : "unchanged", diffs, diff --git a/scripts/interface-diff.test.mjs b/scripts/interface-diff.test.mjs index 20134909a5..90d6fc25ab 100644 --- a/scripts/interface-diff.test.mjs +++ b/scripts/interface-diff.test.mjs @@ -77,12 +77,13 @@ test("a probe failure is an error, though the CLI reports it as a difference", ( const v = classify( cliJson({ hasDifferences: true, - error: "Target probe failed: boom", + error: "boom", diffs: [{ endpoint: "error", diff: "Target probe failed: boom" }], }), ); assert.equal(v.status, "error"); - assert.match(v.error, /boom/); + // The context (which build failed) comes from the diffs entry. + assert.equal(v.error, "Target probe failed: boom"); }); test("output that is not the JSON report is an error, never a pass", () => { @@ -90,6 +91,7 @@ test("output that is not the JSON report is an error, never a pass", () => { assert.equal(classify("Fatal error: x").status, "error"); assert.equal(classify("{}").status, "error"); assert.equal(classify(JSON.stringify({ results: [] })).status, "error"); + assert.equal(classify(JSON.stringify({ results: [{}] })).status, "error"); }); test("a CLI crash before the report carries its stderr as the reason", () => { From ff191795dcb1dc70f25da0adc638d49de1b5f50e Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 04:02:20 -0400 Subject: [PATCH 15/17] fix(interface-diff): bound the job summary; watch the Windows helper (#4860) Copilot round 6 on #4975: - Cap the job summary below GitHub's 1 MiB limit, pointing to the artifact when the report is longer. - Add scripts/lib/win-shell-args.mjs, which the script imports, to the path filters. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 9520b235c7..144eacfd79 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -38,6 +38,7 @@ on: - "package-lock.json" - "tsconfig.json" - "scripts/interface-diff.mjs" + - "scripts/lib/win-shell-args.mjs" - ".github/workflows/everything-mcp-diff.yml" # No branch filter: PRs to v2/main and to main both run, and so does a PR # stacked on another PR's branch. The base is the PR's own base either way. @@ -48,6 +49,7 @@ on: - "package-lock.json" - "tsconfig.json" - "scripts/interface-diff.mjs" + - "scripts/lib/win-shell-args.mjs" - ".github/workflows/everything-mcp-diff.yml" # A published Release (a milestone, tagged vX.Y.Z on main; see RELEASING.md) # surfaces the cumulative interface delta since the previous Release. This @@ -160,7 +162,12 @@ jobs: run: | { if [ -f mcp-diff-report/report.md ]; then - cat mcp-diff-report/report.md + # A step summary holds at most 1 MiB. + head -c 900000 mcp-diff-report/report.md + if [ "$(wc -c < mcp-diff-report/report.md)" -gt 900000 ]; then + echo "" + echo "_The report is truncated here; the full report is in the run's \`mcp-diff-report\` artifact._" + fi else echo "The interface diff produced no report; see the job log." fi From 9d7128ca3b5b98398e88c9ea90279b69b7f24c2b Mon Sep 17 00:00:00 2001 From: cliffhall Date: Sun, 4 Oct 2026 04:24:53 -0400 Subject: [PATCH 16/17] fix(interface-diff): drop target_ref; keep both streams in errors (#4860) Copilot round 7 on #4975: - Remove the dispatch `target_ref` input: a target that predates the script has no `npm run interface-diff`, so the run failed. The head is the branch chosen in "Use workflow from". - A failed base install or build now reports the tail of both stdout and stderr, so tsc diagnostics are not lost behind an npm footer. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- .github/workflows/everything-mcp-diff.yml | 11 +++++------ scripts/interface-diff.mjs | 8 +++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/everything-mcp-diff.yml b/.github/workflows/everything-mcp-diff.yml index 144eacfd79..16fe49c17d 100644 --- a/.github/workflows/everything-mcp-diff.yml +++ b/.github/workflows/everything-mcp-diff.yml @@ -57,17 +57,16 @@ on: # 2024, and no release since has used them. release: types: [published] - # Manual trigger for comparing any two refs (commits, tags, branches) + # Manual trigger: the branch chosen in "Use workflow from" is the head, and + # `compare_ref` any base. #3260 also took a `target_ref`, but a target that + # predates the interface-diff script has no `npm run interface-diff` to run, + # so the head is the dispatched branch instead. workflow_dispatch: inputs: compare_ref: description: "Base ref to compare against (commit SHA, tag, or branch; defaults to the merge-base with v2/main)" required: false default: "" - target_ref: - description: "Target ref to compare (defaults to the ref the workflow runs on)" - required: false - default: "" permissions: contents: read @@ -95,7 +94,7 @@ jobs: fetch-depth: 0 # The event's own commit (the merge commit for a PR), not a branch # ref that a later push could move before this step runs. - ref: ${{ inputs.target_ref || github.sha }} + ref: ${{ github.sha }} persist-credentials: false - uses: actions/setup-node@v7 diff --git a/scripts/interface-diff.mjs b/scripts/interface-diff.mjs index 58c084f127..59403326c4 100644 --- a/scripts/interface-diff.mjs +++ b/scripts/interface-diff.mjs @@ -203,6 +203,12 @@ export function renderReport(verdict, { base, head }) { return lines.join("\n"); } +/** The last lines of a stream, for an error message; empty stays empty. */ +const tail = (text) => { + const lines = (text ?? "").trim(); + return lines ? `${lines.split("\n").slice(-25).join("\n")}\n` : ""; +}; + /** * Run a command to completion; throw with its output when it fails. Only * `npm` needs a shell, and only on Windows (it is a `.cmd` shim there); its @@ -220,7 +226,7 @@ function run(command, args, cwd, env) { if (res.error) throw res.error; if (res.status !== 0) throw new Error( - `\`${command} ${args.join(" ")}\` exited ${res.status}:\n${(res.stderr || res.stdout).trim().split("\n").slice(-25).join("\n")}`, + `\`${command} ${args.join(" ")}\` exited ${res.status}:\n${tail(res.stdout)}${tail(res.stderr)}`, ); return res.stdout; } From 6eebb0c65045c839448e8623ddced0ea5ffd806e Mon Sep 17 00:00:00 2001 From: cliffhall Date: Fri, 9 Oct 2026 23:25:40 -0400 Subject: [PATCH 17/17] chore(interface-diff): refresh the lockfile after rebasing onto v2/main (#4860) v2/main no longer carries SDK v1 at the root (#5063), so mcp-server-diff's @modelcontextprotocol/sdk ^1.13.2 now resolves to its own dev-only copy. Co-Authored-By: Claude Opus 5.5 Signed-off-by: cliffhall --- package-lock.json | 140 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/package-lock.json b/package-lock.json index 6359f597db..a9adead5f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -622,6 +622,19 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, + "node_modules/@hono/node-server": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.4.tgz", + "integrity": "sha512-2g1qeHl4BduwDn6L8nkAvDrPUWPMlkQ//a5NhP/zIyaXKMjZNRV+cG9Csu+BiJFf6+68pqzjf4Hi5sJGKHuS1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4 || ^5.0.0-0" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -841,6 +854,47 @@ "hono": "^4" } }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.32.1", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.32.1.tgz", + "integrity": "sha512-2DdE+SJDtzLEEWzY1ZjY7Q+VcPhcV1KisD3zI4u0XZyktsjHum1mwbMI+JaulUBi2OZk+KJAi2uPXzxichPkdw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, "node_modules/@modelcontextprotocol/server": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/server/-/server-2.3.1.tgz", @@ -1847,6 +1901,41 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -2829,6 +2918,23 @@ "fast-string-truncated-width": "^3.0.2" } }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fast-wrap-ansi": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", @@ -3447,6 +3553,20 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -4518,6 +4638,16 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.12", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", @@ -5520,6 +5650,16 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + }, "src/everything": { "name": "@modelcontextprotocol/server-everything", "version": "1.0.0",