diff --git a/.claude/skills/mcp-sdk-tier-audit/README.md b/.claude/skills/mcp-sdk-tier-audit/README.md index 77fc8cf8..c0593831 100644 --- a/.claude/skills/mcp-sdk-tier-audit/README.md +++ b/.claude/skills/mcp-sdk-tier-audit/README.md @@ -44,6 +44,8 @@ For public repos, any authenticated token works (no special scopes needed — au --days Limit triage analysis to last N days --output json | markdown | terminal (default: terminal) --token GitHub token (defaults to GITHUB_TOKEN or gh auth token) +--spec-version Only run conformance scenarios for this spec version (also pins spec-tracking to it) +--sdk-release-tag Exact release tag of the submitted SDK version (pins spec-tracking to that submission) ``` ### What the CLI Checks diff --git a/.claude/skills/mcp-sdk-tier-audit/references/report-template.md b/.claude/skills/mcp-sdk-tier-audit/references/report-template.md index d77e1993..dc0215cc 100644 --- a/.claude/skills/mcp-sdk-tier-audit/references/report-template.md +++ b/.claude/skills/mcp-sdk-tier-audit/references/report-template.md @@ -28,7 +28,7 @@ Write two files to `results/` in the conformance repo: | 2b | Labels | 12 required labels | 12 required labels | {present}/{required} | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | | 3 | Critical Bug Resolution | All P0s within 7 days | All P0s within 2 weeks | {open P0 count} open | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | | 4 | Stable Release | Required + clear versioning | At least one stable release | {version} | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | -| 4b | Spec Tracking | Timeline agreed per release | Within 6 months | {days_gap}d gap ({PASS/FAIL}) | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | +| 4b | Spec Tracking | Timeline agreed per release | Within 6 months | {days_gap}d gap ({PASS/PARTIAL/FAIL}) | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | | 5 | Documentation | Comprehensive w/ examples | Basic docs for core features | {pass}/{total} features | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | | 6 | Dependency Policy | Published update policy | Published update policy | {Found/Not found} | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | | 7 | Roadmap | Published roadmap | Plan toward Tier 1 | {Found/Not found} | {PASS/FAIL} | {PASS/FAIL} | {detail or "None"} | diff --git a/src/tier-check/checks/spec-tracking.test.ts b/src/tier-check/checks/spec-tracking.test.ts new file mode 100644 index 00000000..d63bfcf4 --- /dev/null +++ b/src/tier-check/checks/spec-tracking.test.ts @@ -0,0 +1,497 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { Octokit } from '@octokit/rest'; +import { checkSpecTracking } from './spec-tracking'; + +const SPEC_OWNER = 'modelcontextprotocol'; +const SPEC_REPO = 'modelcontextprotocol'; +const SDK_OWNER = 'modelcontextprotocol'; +const SDK_REPO = 'typescript-sdk'; + +const DAY_MS = 1000 * 60 * 60 * 24; + +function isoDaysAfter(iso: string, days: number): string { + return new Date(new Date(iso).getTime() + days * DAY_MS).toISOString(); +} + +function notFound(tag: string): Error { + return Object.assign(new Error(`Not Found: ${tag}`), { status: 404 }); +} + +function apiError(): Error { + return new Error('service unavailable'); +} + +function release(overrides: { + tag_name: string; + published_at?: string | null; + created_at?: string | null; + draft?: boolean; + prerelease?: boolean; +}) { + return { + tag_name: overrides.tag_name, + published_at: overrides.published_at ?? null, + created_at: overrides.created_at ?? overrides.published_at ?? null, + draft: overrides.draft ?? false, + prerelease: overrides.prerelease ?? false + }; +} + +/** + * Builds a fake Octokit exposing only the methods checkSpecTracking uses. + * `getReleaseByTag` is routed by (owner, repo, tag) via the provided maps. + */ +function makeOctokit(opts: { + getReleaseByTag?: Record; + specListReleases?: unknown[] | Error; + sdkListReleases?: unknown[] | Error; +}): Octokit { + const getReleaseByTag = vi.fn( + async ({ + owner, + repo, + tag + }: { + owner: string; + repo: string; + tag: string; + }) => { + const key = `${owner}/${repo}@${tag}`; + const entry = opts.getReleaseByTag?.[key]; + if (entry === undefined) { + throw new Error(`unexpected getReleaseByTag call: ${key}`); + } + if (entry instanceof Error) throw entry; + return { data: entry }; + } + ); + + const listReleases = vi.fn( + async ({ owner, repo }: { owner: string; repo: string }) => { + const isSpec = owner === SPEC_OWNER && repo === SPEC_REPO; + const source = isSpec ? opts.specListReleases : opts.sdkListReleases; + if (source instanceof Error) throw source; + return { data: source ?? [] }; + } + ); + + return { + repos: { getReleaseByTag, listReleases } + } as unknown as Octokit; +} + +describe('checkSpecTracking — submitted mode (pinned spec + sdk tags)', () => { + const specTag = '2025-11-25'; + const sdkTag = 'v1.4.0'; + const specPublishedAt = '2025-11-25T00:00:00.000Z'; + + function octokitForGap(gapDays: number) { + return makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: release({ + tag_name: specTag, + published_at: specPublishedAt + }), + [`${SDK_OWNER}/${SDK_REPO}@${sdkTag}`]: release({ + tag_name: sdkTag, + published_at: isoDaysAfter(specPublishedAt, gapDays) + }) + } + }); + } + + it('passes at gap 0 (sdk released exactly on the spec date)', async () => { + const result = await checkSpecTracking( + octokitForGap(0), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(result.status).toBe('pass'); + expect(result.days_gap).toBe(0); + expect(result.meets_tier1_window).toBe(true); + expect(result.meets_tier2_window).toBe(true); + }); + + it('passes at a negative gap (sdk released before the spec)', async () => { + const result = await checkSpecTracking( + octokitForGap(-5), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(result.status).toBe('pass'); + expect(result.days_gap).toBe(-5); + expect(result.meets_tier1_window).toBe(true); + expect(result.meets_tier2_window).toBe(true); + }); + + it('is partial at gap 1', async () => { + const result = await checkSpecTracking( + octokitForGap(1), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(result.status).toBe('partial'); + expect(result.days_gap).toBe(1); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(true); + }); + + it('is partial at exactly gap 183 (six months, SEP-1730 Tier 2 window)', async () => { + const result = await checkSpecTracking( + octokitForGap(183), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(result.status).toBe('partial'); + expect(result.days_gap).toBe(183); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(true); + }); + + it('fails at gap 184', async () => { + const result = await checkSpecTracking( + octokitForGap(184), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(result.status).toBe('fail'); + expect(result.days_gap).toBe(184); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(false); + }); + + it('emits target_spec_tag and submitted_sdk_tag on success', async () => { + const result = await checkSpecTracking( + octokitForGap(0), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(result.target_spec_tag).toBe(specTag); + expect(result.submitted_sdk_tag).toBe(sdkTag); + }); + + it('404s on the submitted sdk tag → fail with a reason naming the tag, both windows false', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: release({ + tag_name: specTag, + published_at: specPublishedAt + }), + [`${SDK_OWNER}/${SDK_REPO}@${sdkTag}`]: notFound(sdkTag) + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('fail'); + expect(result.reason).toContain(sdkTag); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(false); + // The spec side had already resolved — that must not be lost. + expect(result.latest_spec_release).toBe(specPublishedAt); + expect(result.target_spec_tag).toBe(specTag); + expect(result.submitted_sdk_tag).toBe(sdkTag); + }); + + it('404s on the pinned spec tag → fail with a reason naming the tag, both windows false', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: notFound(specTag) + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('fail'); + expect(result.reason).toContain(specTag); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(false); + expect(result.target_spec_tag).toBe(specTag); + expect(result.submitted_sdk_tag).toBe(sdkTag); + }); + + it('a generic (non-404) error resolving the pinned spec tag → skipped, tags preserved', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: apiError() + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('skipped'); + expect(result.meets_tier1_window).toBeNull(); + expect(result.meets_tier2_window).toBeNull(); + expect(result.target_spec_tag).toBe(specTag); + expect(result.submitted_sdk_tag).toBe(sdkTag); + expect(result.latest_spec_release).toBeNull(); + }); + + it('a generic (non-404) error resolving the submitted sdk tag → skipped, tags and resolved spec date preserved', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: release({ + tag_name: specTag, + published_at: specPublishedAt + }), + [`${SDK_OWNER}/${SDK_REPO}@${sdkTag}`]: apiError() + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('skipped'); + expect(result.meets_tier1_window).toBeNull(); + expect(result.meets_tier2_window).toBeNull(); + expect(result.target_spec_tag).toBe(specTag); + expect(result.submitted_sdk_tag).toBe(sdkTag); + // The spec side had already resolved when the sdk lookup blew up. + expect(result.latest_spec_release).toBe(specPublishedAt); + }); + + it('resolves a pinned RC spec tag as-is (no normalization)', async () => { + const rcTag = '2026-07-28-RC'; + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${rcTag}`]: release({ + tag_name: rcTag, + published_at: '2026-07-28T00:00:00.000Z', + prerelease: true + }), + [`${SDK_OWNER}/${SDK_REPO}@${sdkTag}`]: release({ + tag_name: sdkTag, + published_at: '2026-07-28T00:00:00.000Z' + }) + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: rcTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('pass'); + expect(result.target_spec_tag).toBe(rcTag); + }); + + it('treats a draft release returned for the pinned spec tag as not a valid target', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: release({ + tag_name: specTag, + published_at: specPublishedAt, + draft: true + }) + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('fail'); + expect(result.reason).toContain(specTag); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(false); + expect(result.target_spec_tag).toBe(specTag); + expect(result.submitted_sdk_tag).toBe(sdkTag); + }); + + it('treats a draft release returned for the submitted tag as not a valid submission', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: release({ + tag_name: specTag, + published_at: specPublishedAt + }), + [`${SDK_OWNER}/${SDK_REPO}@${sdkTag}`]: release({ + tag_name: sdkTag, + published_at: specPublishedAt, + draft: true + }) + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('fail'); + expect(result.reason).toContain(sdkTag); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(false); + }); + + it('falls back to created_at when the submitted release has no published_at', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: release({ + tag_name: specTag, + published_at: specPublishedAt + }), + [`${SDK_OWNER}/${SDK_REPO}@${sdkTag}`]: release({ + tag_name: sdkTag, + published_at: null, + created_at: specPublishedAt + }) + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('pass'); + expect(result.days_gap).toBe(0); + }); + + it('falls back to created_at when the pinned spec release has no published_at', async () => { + const octokit = makeOctokit({ + getReleaseByTag: { + [`${SPEC_OWNER}/${SPEC_REPO}@${specTag}`]: release({ + tag_name: specTag, + published_at: null, + created_at: specPublishedAt + }), + [`${SDK_OWNER}/${SDK_REPO}@${sdkTag}`]: release({ + tag_name: sdkTag, + published_at: specPublishedAt + }) + } + }); + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO, { + specVersion: specTag, + sdkReleaseTag: sdkTag + }); + expect(result.status).toBe('pass'); + expect(result.days_gap).toBe(0); + expect(result.latest_spec_release).toBe(specPublishedAt); + }); + + it('sdk_release_within_30d is true at a 30d gap and false at a 31d gap', async () => { + const at30 = await checkSpecTracking( + octokitForGap(30), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(at30.sdk_release_within_30d).toBe(true); + expect(at30.status).toBe('partial'); + + const at31 = await checkSpecTracking( + octokitForGap(31), + SDK_OWNER, + SDK_REPO, + { specVersion: specTag, sdkReleaseTag: sdkTag } + ); + expect(at31.sdk_release_within_30d).toBe(false); + expect(at31.status).toBe('partial'); + }); +}); + +describe('checkSpecTracking — legacy mode (no opts)', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + const specPublishedAt = '2025-11-25T00:00:00.000Z'; + + it('selects the first non-draft SDK release published on/after the spec', async () => { + const octokit = makeOctokit({ + specListReleases: [ + release({ tag_name: '2025-11-25', published_at: specPublishedAt }) + ], + // API returns newest-first. + sdkListReleases: [ + release({ + tag_name: 'v1.6.0', + published_at: isoDaysAfter(specPublishedAt, 40) + }), + release({ + tag_name: 'v1.5.0', + published_at: isoDaysAfter(specPublishedAt, 10) + }), + release({ + tag_name: 'v1.4.0', + published_at: isoDaysAfter(specPublishedAt, -20) + }) + ] + }); + + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO); + expect(result.latest_sdk_release).toBe(isoDaysAfter(specPublishedAt, 10)); + expect(result.days_gap).toBe(10); + expect(result.status).toBe('partial'); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(true); + }); + + it('uses days elapsed since the spec when no SDK release follows it (fake timers)', async () => { + vi.setSystemTime(new Date(isoDaysAfter(specPublishedAt, 200))); + + const octokit = makeOctokit({ + specListReleases: [ + release({ tag_name: '2025-11-25', published_at: specPublishedAt }) + ], + sdkListReleases: [ + release({ + tag_name: 'v1.4.0', + published_at: isoDaysAfter(specPublishedAt, -20) + }) + ] + }); + + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO); + expect(result.days_gap).toBe(200); + expect(result.status).toBe('fail'); + expect(result.meets_tier1_window).toBe(false); + expect(result.meets_tier2_window).toBe(false); + }); + + it('returns skipped when listReleases throws', async () => { + const octokit = makeOctokit({ + specListReleases: new Error('network down') + }); + + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO); + expect(result.status).toBe('skipped'); + expect(result.meets_tier1_window).toBeNull(); + expect(result.meets_tier2_window).toBeNull(); + }); + + it('returns skipped when the sdk listReleases call throws after the spec resolved, preserving latest_spec_release', async () => { + const octokit = makeOctokit({ + specListReleases: [ + release({ tag_name: '2025-11-25', published_at: specPublishedAt }) + ], + sdkListReleases: new Error('network down') + }); + + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO); + expect(result.status).toBe('skipped'); + expect(result.latest_spec_release).toBe(specPublishedAt); + expect(result.meets_tier1_window).toBeNull(); + expect(result.meets_tier2_window).toBeNull(); + }); + + it('returns skipped with a reason when there are no SDK releases', async () => { + const octokit = makeOctokit({ + specListReleases: [ + release({ tag_name: '2025-11-25', published_at: specPublishedAt }) + ], + sdkListReleases: [] + }); + + const result = await checkSpecTracking(octokit, SDK_OWNER, SDK_REPO); + expect(result.status).toBe('skipped'); + expect(result.reason).toBeTruthy(); + expect(result.latest_spec_release).toBe(specPublishedAt); + expect(result.meets_tier1_window).toBeNull(); + expect(result.meets_tier2_window).toBeNull(); + }); +}); diff --git a/src/tier-check/checks/spec-tracking.ts b/src/tier-check/checks/spec-tracking.ts index ce4c101c..33cb61fe 100644 --- a/src/tier-check/checks/spec-tracking.ts +++ b/src/tier-check/checks/spec-tracking.ts @@ -1,21 +1,206 @@ import { Octokit } from '@octokit/rest'; import { SpecTrackingResult } from '../types'; +const MS_PER_DAY = 1000 * 60 * 60 * 24; + +// SEP-1730 Tier 2 window: an SDK release lands within six months of the +// spec version it targets. Replaces the previous hardcoded 30-day check. +const TIER2_WINDOW_DAYS = 183; + +interface ReleaseLike { + draft?: boolean; + prerelease?: boolean; + published_at?: string | null; + created_at?: string | null; +} + +export interface SpecTrackingOptions { + /** Exact release tag of the submitted SDK version (no normalization). */ + sdkReleaseTag?: string; + /** Exact release tag on modelcontextprotocol/modelcontextprotocol to pin to. */ + specVersion?: string; +} + +function isNotFoundError(err: unknown): boolean { + return ( + typeof err === 'object' && + err !== null && + 'status' in err && + (err as { status?: unknown }).status === 404 + ); +} + +// SEP-1730 window verdicts, derived once alongside the display status: Tier 1 +// requires landing before (or with) the spec release; Tier 2 tolerates up to +// six months after. +function windows(daysGap: number): { + status: SpecTrackingResult['status']; + meetsTier1Window: boolean; + meetsTier2Window: boolean; +} { + const meetsTier1Window = daysGap <= 0; + const meetsTier2Window = daysGap <= TIER2_WINDOW_DAYS; + const status = meetsTier1Window + ? 'pass' + : meetsTier2Window + ? 'partial' + : 'fail'; + return { status, meetsTier1Window, meetsTier2Window }; +} + +function gapResult(params: { + specPublishedAt: string; + sdkPublishedAt: string; + targetSpecTag: string | null; + submittedSdkTag: string | null; +}): SpecTrackingResult { + const daysGap = Math.round( + (new Date(params.sdkPublishedAt).getTime() - + new Date(params.specPublishedAt).getTime()) / + MS_PER_DAY + ); + const { status, meetsTier1Window, meetsTier2Window } = windows(daysGap); + + return { + status, + latest_spec_release: params.specPublishedAt, + latest_sdk_release: params.sdkPublishedAt, + sdk_release_within_30d: daysGap <= 30, + days_gap: daysGap, + target_spec_tag: params.targetSpecTag, + submitted_sdk_tag: params.submittedSdkTag, + meets_tier1_window: meetsTier1Window, + meets_tier2_window: meetsTier2Window + }; +} + export async function checkSpecTracking( octokit: Octokit, owner: string, - repo: string + repo: string, + opts: SpecTrackingOptions = {} ): Promise { - try { - // Get latest spec release from modelcontextprotocol/modelcontextprotocol - const { data: specReleases } = await octokit.repos.listReleases({ - owner: 'modelcontextprotocol', - repo: 'modelcontextprotocol', - per_page: 5 + const targetSpecTag = opts.specVersion ?? null; + const submittedSdkTag = opts.sdkReleaseTag ?? null; + + // Fail/skip results always carry the tags already known at the call site + // (from the function-scope consts above) plus latest_spec_release when the + // spec was already resolved — a single derivation site so no return path + // can silently drop information a caller would expect. + function failResult( + reason: string, + latestSpecRelease: string | null = null + ): SpecTrackingResult { + return { + status: 'fail', + latest_spec_release: latestSpecRelease, + latest_sdk_release: null, + sdk_release_within_30d: null, + days_gap: null, + target_spec_tag: targetSpecTag, + submitted_sdk_tag: submittedSdkTag, + meets_tier1_window: false, + meets_tier2_window: false, + reason + }; + } + + function skipResult( + reason: string, + latestSpecRelease: string | null = null + ): SpecTrackingResult { + return { + status: 'skipped', + latest_spec_release: latestSpecRelease, + latest_sdk_release: null, + sdk_release_within_30d: null, + days_gap: null, + target_spec_tag: targetSpecTag, + submitted_sdk_tag: submittedSdkTag, + meets_tier1_window: null, + meets_tier2_window: null, + reason + }; + } + + // --- Resolve the spec release to track against --- + let specRelease: ReleaseLike; + if (opts.specVersion) { + try { + const { data } = await octokit.repos.getReleaseByTag({ + owner: 'modelcontextprotocol', + repo: 'modelcontextprotocol', + tag: opts.specVersion + }); + specRelease = data; + } catch (err) { + if (isNotFoundError(err)) { + return failResult(`spec release tag not found: ${opts.specVersion}`); + } + return skipResult('github api error'); + } + + if (specRelease.draft) { + return failResult(`spec release is a draft: ${opts.specVersion}`); + } + } else { + try { + const { data: specReleases } = await octokit.repos.listReleases({ + owner: 'modelcontextprotocol', + repo: 'modelcontextprotocol', + per_page: 5 + }); + const latestSpec = specReleases.find((r) => !r.draft && !r.prerelease); + if (!latestSpec) { + return skipResult('no spec releases found'); + } + specRelease = latestSpec; + } catch { + return skipResult('github api error'); + } + } + + const specPublishedAt = specRelease.published_at ?? specRelease.created_at!; + const specDate = new Date(specPublishedAt); + + // --- Resolve the SDK release to compare against the spec --- + if (opts.sdkReleaseTag) { + let sdkRelease: ReleaseLike; + try { + const { data } = await octokit.repos.getReleaseByTag({ + owner, + repo, + tag: opts.sdkReleaseTag + }); + sdkRelease = data; + } catch (err) { + if (isNotFoundError(err)) { + return failResult( + `sdk release tag not found: ${opts.sdkReleaseTag}`, + specPublishedAt + ); + } + return skipResult('github api error', specPublishedAt); + } + + if (sdkRelease.draft) { + return failResult( + `submitted release is a draft: ${opts.sdkReleaseTag}`, + specPublishedAt + ); + } + + const sdkPublishedAt = sdkRelease.published_at ?? sdkRelease.created_at!; + return gapResult({ + specPublishedAt, + sdkPublishedAt, + targetSpecTag, + submittedSdkTag }); - const latestSpec = specReleases.find((r) => !r.draft && !r.prerelease); + } - // Get SDK releases (API returns newest-first) + // Legacy mode: first non-draft SDK release published on/after the spec. + try { const { data: sdkReleases } = await octokit.repos.listReleases({ owner, repo, @@ -23,57 +208,43 @@ export async function checkSpecTracking( }); const nonDraftSdkReleases = sdkReleases.filter((r) => !r.draft); - if (!latestSpec || nonDraftSdkReleases.length === 0) { - return { - status: 'skipped', - latest_spec_release: latestSpec?.published_at || null, - latest_sdk_release: nonDraftSdkReleases[0]?.published_at || null, - sdk_release_within_30d: null, - days_gap: null - }; + if (nonDraftSdkReleases.length === 0) { + return skipResult('no sdk releases found', specPublishedAt); } - const specDate = new Date(latestSpec.published_at!); - - // Reverse so oldest-first, then find the FIRST SDK release after the spec + // API returns newest-first; reverse so oldest-first, then find the + // FIRST SDK release published on/after the spec. const oldestFirst = [...nonDraftSdkReleases].reverse(); const firstSdkAfterSpec = oldestFirst.find( (r) => new Date(r.published_at!) >= specDate ); if (!firstSdkAfterSpec) { - // No SDK release after the latest spec release - const daysSinceSpec = Math.round( - (Date.now() - specDate.getTime()) / (1000 * 60 * 60 * 24) + const daysElapsed = Math.round( + (Date.now() - specDate.getTime()) / MS_PER_DAY ); + const { status, meetsTier1Window, meetsTier2Window } = + windows(daysElapsed); return { - status: daysSinceSpec <= 30 ? 'pass' : 'fail', - latest_spec_release: latestSpec.published_at, - latest_sdk_release: nonDraftSdkReleases[0]?.published_at || null, - sdk_release_within_30d: daysSinceSpec <= 30, - days_gap: daysSinceSpec + status, + latest_spec_release: specPublishedAt, + latest_sdk_release: nonDraftSdkReleases[0]?.published_at ?? null, + sdk_release_within_30d: daysElapsed <= 30, + days_gap: daysElapsed, + target_spec_tag: targetSpecTag, + submitted_sdk_tag: submittedSdkTag, + meets_tier1_window: meetsTier1Window, + meets_tier2_window: meetsTier2Window }; } - const sdkDate = new Date(firstSdkAfterSpec.published_at!); - const daysGap = Math.round( - (sdkDate.getTime() - specDate.getTime()) / (1000 * 60 * 60 * 24) - ); - - return { - status: daysGap <= 30 ? 'pass' : 'fail', - latest_spec_release: latestSpec.published_at, - latest_sdk_release: firstSdkAfterSpec.published_at, - sdk_release_within_30d: daysGap <= 30, - days_gap: daysGap - }; + return gapResult({ + specPublishedAt, + sdkPublishedAt: firstSdkAfterSpec.published_at!, + targetSpecTag, + submittedSdkTag + }); } catch { - return { - status: 'skipped', - latest_spec_release: null, - latest_sdk_release: null, - sdk_release_within_30d: null, - days_gap: null - }; + return skipResult('github api error', specPublishedAt); } } diff --git a/src/tier-check/index.test.ts b/src/tier-check/index.test.ts new file mode 100644 index 00000000..0aad8330 --- /dev/null +++ b/src/tier-check/index.test.ts @@ -0,0 +1,198 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('@octokit/rest', () => ({ + Octokit: vi.fn().mockImplementation(function () { + return {}; + }) +})); + +vi.mock('./checks/test-conformance-results', () => ({ + checkConformance: vi.fn().mockResolvedValue({ + status: 'pass', + pass_rate: 1, + passed: 1, + failed: 0, + total: 1, + details: [] + }), + checkClientConformance: vi.fn().mockResolvedValue({ + status: 'skipped', + pass_rate: 0, + passed: 0, + failed: 0, + total: 0, + details: [] + }) +})); + +vi.mock('./checks/labels', () => ({ + checkLabels: vi.fn().mockResolvedValue({ + status: 'pass', + present: 0, + required: 0, + missing: [], + found: [], + uses_issue_types: false + }) +})); + +vi.mock('./checks/triage', () => ({ + checkTriage: vi.fn().mockResolvedValue({ + status: 'pass', + compliance_rate: 1, + total_issues: 0, + triaged_within_sla: 0, + exceeding_sla: 0, + median_hours: 0, + p95_hours: 0, + days_analyzed: undefined + }) +})); + +vi.mock('./checks/p0', () => ({ + checkP0Resolution: vi.fn().mockResolvedValue({ + status: 'pass', + open_p0s: 0, + open_p0_details: [], + closed_within_7d: 0, + closed_within_14d: 0, + closed_total: 0, + all_p0s_resolved_within_7d: true, + all_p0s_resolved_within_14d: true + }) +})); + +vi.mock('./checks/release', () => ({ + checkStableRelease: vi.fn().mockResolvedValue({ + status: 'pass', + version: '1.0.0', + is_stable: true, + is_prerelease: false + }) +})); + +vi.mock('./checks/files', () => ({ + checkPolicySignals: vi.fn().mockResolvedValue({ + status: 'pass', + files: {} + }) +})); + +vi.mock('./checks/spec-tracking', () => ({ + checkSpecTracking: vi.fn().mockResolvedValue({ + status: 'pass', + latest_spec_release: '2025-11-25T00:00:00.000Z', + latest_sdk_release: '2025-11-25T00:00:00.000Z', + sdk_release_within_30d: true, + days_gap: 0, + target_spec_tag: '2025-11-25', + submitted_sdk_tag: 'v1.4.0', + meets_tier1_window: true, + meets_tier2_window: true + }) +})); + +import { createTierCheckCommand } from './index'; +import { checkSpecTracking } from './checks/spec-tracking'; + +describe('createTierCheckCommand', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(checkSpecTracking).mockResolvedValue({ + status: 'pass', + latest_spec_release: '2025-11-25T00:00:00.000Z', + latest_sdk_release: '2025-11-25T00:00:00.000Z', + sdk_release_within_30d: true, + days_gap: 0, + target_spec_tag: '2025-11-25', + submitted_sdk_tag: 'v1.4.0', + meets_tier1_window: true, + meets_tier2_window: true + }); + }); + + it('registers --sdk-release-tag as an option', () => { + const tierCheck = createTierCheckCommand(); + const flags = tierCheck.options.map((o) => o.long); + expect(flags).toContain('--sdk-release-tag'); + }); + + it('forwards --sdk-release-tag and the raw --spec-version to checkSpecTracking', async () => { + const tierCheck = createTierCheckCommand(); + await tierCheck.parseAsync( + [ + '--repo', + 'modelcontextprotocol/typescript-sdk', + '--skip-conformance', + '--token', + 'fake-token', + '--spec-version', + '2025-11-25', + '--sdk-release-tag', + 'v1.4.0', + '--output', + 'json' + ], + { from: 'user' } + ); + + expect(checkSpecTracking).toHaveBeenCalledWith( + expect.anything(), + 'modelcontextprotocol', + 'typescript-sdk', + { sdkReleaseTag: 'v1.4.0', specVersion: '2025-11-25' } + ); + }); + + it('maps --spec-version draft to undefined (no draft release to track against)', async () => { + const tierCheck = createTierCheckCommand(); + await tierCheck.parseAsync( + [ + '--repo', + 'modelcontextprotocol/typescript-sdk', + '--skip-conformance', + '--token', + 'fake-token', + '--spec-version', + 'draft', + '--sdk-release-tag', + 'v1.4.0', + '--output', + 'json' + ], + { from: 'user' } + ); + + expect(checkSpecTracking).toHaveBeenCalledWith( + expect.anything(), + 'modelcontextprotocol', + 'typescript-sdk', + { sdkReleaseTag: 'v1.4.0', specVersion: undefined } + ); + }); + + it('maps an absent --spec-version to undefined (legacy latest-stable mode)', async () => { + const tierCheck = createTierCheckCommand(); + await tierCheck.parseAsync( + [ + '--repo', + 'modelcontextprotocol/typescript-sdk', + '--skip-conformance', + '--token', + 'fake-token', + '--sdk-release-tag', + 'v1.4.0', + '--output', + 'json' + ], + { from: 'user' } + ); + + expect(checkSpecTracking).toHaveBeenCalledWith( + expect.anything(), + 'modelcontextprotocol', + 'typescript-sdk', + { sdkReleaseTag: 'v1.4.0', specVersion: undefined } + ); + }); +}); diff --git a/src/tier-check/index.ts b/src/tier-check/index.ts index 1cd32c19..b73c0759 100644 --- a/src/tier-check/index.ts +++ b/src/tier-check/index.ts @@ -14,6 +14,7 @@ import { computeTier } from './tier-logic'; import { formatJson, formatMarkdown, formatTerminal } from './output'; import { TierScorecard } from './types'; import { resolveSpecVersion } from '../scenarios'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; function parseRepo(repo: string): { owner: string; repo: string } { const parts = repo.split('/'); @@ -53,6 +54,10 @@ export function createTierCheckCommand(): Command { '--spec-version ', 'Only run conformance scenarios for this spec version' ) + .option( + '--sdk-release-tag ', + 'Exact release tag of the submitted SDK version, resolved with no normalization (pins spec-tracking to that submission)' + ) .action(async (options) => { const { owner, repo } = parseRepo(options.repo); let token = options.token || process.env.GITHUB_TOKEN; @@ -61,6 +66,20 @@ export function createTierCheckCommand(): Command { ? resolveSpecVersion(options.specVersion) : undefined; + // The CLI's --spec-version vocabulary is released dated versions plus + // the 'draft' alias — resolveSpecVersion above already exits the + // process on anything else. checkSpecTracking itself accepts any exact + // release tag (including pre-releases like an RC), which is exercised + // directly in its unit tests; pinning one of those from this CLI would + // need a wider --spec-version vocabulary, deliberately out of scope + // here. The draft spec version has no GitHub release to pin against, + // so draft runs fall back to tracking the latest stable spec release + // instead — during the graduation window, the latest stable release + // IS the version that was just the draft, so the fallback lands on + // the right release anyway. + const specVersionForTracking = + specVersion === DRAFT_PROTOCOL_VERSION ? undefined : specVersion; + if (!token) { // Try to get token from GitHub CLI try { @@ -133,7 +152,10 @@ export function createTierCheckCommand(): Command { console.error(' \u2713 Policy Signals'); return r; }), - checkSpecTracking(octokit, owner, repo).then((r) => { + checkSpecTracking(octokit, owner, repo, { + sdkReleaseTag: options.sdkReleaseTag, + specVersion: specVersionForTracking + }).then((r) => { console.error(' \u2713 Spec Tracking'); return r; }) diff --git a/src/tier-check/output.ts b/src/tier-check/output.ts index ab0d2fbe..aa6eefe2 100644 --- a/src/tier-check/output.ts +++ b/src/tier-check/output.ts @@ -198,7 +198,7 @@ export function formatMarkdown(scorecard: TierScorecard): string { .join(', ')} |` ); lines.push( - `| Spec Tracking | ${c.spec_tracking.status} | ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'} |` + `| Spec Tracking | ${c.spec_tracking.status} | ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : (c.spec_tracking.reason ?? 'N/A')} |` ); lines.push(''); @@ -335,7 +335,7 @@ export function formatTerminal(scorecard: TierScorecard): void { .join(', ')}` ); console.log( - ` ${statusIcon(c.spec_tracking.status)} Spec Tracking ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'}` + ` ${statusIcon(c.spec_tracking.status)} Spec Tracking ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : (c.spec_tracking.reason ?? 'N/A')}` ); if (scorecard.implied_tier.tier1_blockers.length > 0) { diff --git a/src/tier-check/tier-logic.test.ts b/src/tier-check/tier-logic.test.ts new file mode 100644 index 00000000..64438f82 --- /dev/null +++ b/src/tier-check/tier-logic.test.ts @@ -0,0 +1,169 @@ +import { describe, it, expect } from 'vitest'; +import { computeTier } from './tier-logic'; +import { TierScorecard } from './types'; + +/** A fully-passing fixture; tests override individual checks. */ +function buildChecks( + overrides: Partial = {} +): TierScorecard['checks'] { + return { + conformance: { + status: 'pass', + pass_rate: 1, + passed: 10, + failed: 0, + total: 10, + details: [] + }, + client_conformance: { + status: 'pass', + pass_rate: 1, + passed: 10, + failed: 0, + total: 10, + details: [] + }, + labels: { + status: 'pass', + present: 5, + required: 5, + missing: [], + found: [], + uses_issue_types: true + }, + triage: { + status: 'pass', + compliance_rate: 1, + total_issues: 10, + triaged_within_sla: 10, + exceeding_sla: 0, + median_hours: 1, + p95_hours: 2, + days_analyzed: 30 + }, + p0_resolution: { + status: 'pass', + open_p0s: 0, + open_p0_details: [], + closed_within_7d: 1, + closed_within_14d: 1, + closed_total: 1, + all_p0s_resolved_within_7d: true, + all_p0s_resolved_within_14d: true + }, + stable_release: { + status: 'pass', + version: '1.0.0', + is_stable: true, + is_prerelease: false + }, + policy_signals: { + status: 'pass', + files: {} + }, + spec_tracking: { + status: 'pass', + latest_spec_release: '2025-11-25T00:00:00.000Z', + latest_sdk_release: '2025-11-25T00:00:00.000Z', + sdk_release_within_30d: true, + days_gap: 0, + target_spec_tag: '2025-11-25', + submitted_sdk_tag: 'v1.4.0', + meets_tier1_window: true, + meets_tier2_window: true + }, + ...overrides + }; +} + +describe('computeTier — spec_tracking', () => { + it('pass: no blocker, tier 1', () => { + const result = computeTier(buildChecks()); + expect(result.tier1_blockers).not.toContain('spec_tracking'); + expect(result.tier).toBe(1); + }); + + it('partial: blocks tier 1 but tier2Met stays true (tier 2)', () => { + const checks = buildChecks({ + spec_tracking: { + status: 'partial', + latest_spec_release: '2025-11-25T00:00:00.000Z', + latest_sdk_release: '2026-01-01T00:00:00.000Z', + sdk_release_within_30d: false, + days_gap: 37, + target_spec_tag: '2025-11-25', + submitted_sdk_tag: 'v1.4.0', + meets_tier1_window: false, + meets_tier2_window: true + } + }); + const result = computeTier(checks); + expect(result.tier1_blockers).toContain('spec_tracking'); + expect(result.tier2_met).toBe(true); + expect(result.tier).toBe(2); + }); + + it('fail: blocks tier 1 and tier2Met becomes false (tier 3)', () => { + const checks = buildChecks({ + spec_tracking: { + status: 'fail', + latest_spec_release: '2025-11-25T00:00:00.000Z', + latest_sdk_release: '2026-06-01T00:00:00.000Z', + sdk_release_within_30d: false, + days_gap: 188, + target_spec_tag: '2025-11-25', + submitted_sdk_tag: 'v1.4.0', + meets_tier1_window: false, + meets_tier2_window: false + } + }); + const result = computeTier(checks); + expect(result.tier1_blockers).toContain('spec_tracking'); + expect(result.tier2_met).toBe(false); + expect(result.tier).toBe(3); + }); + + it('skipped: pushes "spec_tracking (skipped)" blocker, tier2Met tolerates it (tier 2)', () => { + const checks = buildChecks({ + spec_tracking: { + status: 'skipped', + latest_spec_release: null, + latest_sdk_release: null, + sdk_release_within_30d: null, + days_gap: null, + target_spec_tag: null, + submitted_sdk_tag: null, + meets_tier1_window: null, + meets_tier2_window: null, + reason: 'github api error' + } + }); + const result = computeTier(checks); + expect(result.tier1_blockers).toContain('spec_tracking (skipped)'); + expect(result.tier1_blockers).not.toContain('spec_tracking'); + expect(result.tier2_met).toBe(true); + expect(result.tier).toBe(2); + }); + + it('driven by meets_tier1_window/meets_tier2_window fields, not the status string', () => { + // A hypothetical 'partial' result whose window fields say Tier 1 is + // actually met should not block tier 1 — computeTier reads the fields, + // not the status label. + const checks = buildChecks({ + spec_tracking: { + status: 'partial', + latest_spec_release: '2025-11-25T00:00:00.000Z', + latest_sdk_release: '2025-11-20T00:00:00.000Z', + sdk_release_within_30d: true, + days_gap: -5, + target_spec_tag: '2025-11-25', + submitted_sdk_tag: 'v1.4.0', + meets_tier1_window: true, + meets_tier2_window: true + } + }); + const result = computeTier(checks); + expect(result.tier1_blockers).not.toContain('spec_tracking'); + expect(result.tier).toBe(1); + }); +}); diff --git a/src/tier-check/tier-logic.ts b/src/tier-check/tier-logic.ts index cf096c97..ce91850a 100644 --- a/src/tier-check/tier-logic.ts +++ b/src/tier-check/tier-logic.ts @@ -35,7 +35,9 @@ export function computeTier( // they feed into the skill's judgment-based evaluation but don't independently // block tier advancement since SEP-1730 doesn't list specific files. - if (checks.spec_tracking.status === 'fail') { + if (checks.spec_tracking.status === 'skipped') { + tier1Blockers.push('spec_tracking (skipped)'); + } else if (checks.spec_tracking.meets_tier1_window !== true) { tier1Blockers.push('spec_tracking'); } @@ -49,6 +51,8 @@ export function computeTier( checks.conformance.pass_rate >= 0.8) && (checks.client_conformance.status === 'skipped' || checks.client_conformance.pass_rate >= 0.8) && + (checks.spec_tracking.status === 'skipped' || + checks.spec_tracking.meets_tier2_window === true) && checks.p0_resolution.all_p0s_resolved_within_14d && checks.stable_release.is_stable; diff --git a/src/tier-check/types.ts b/src/tier-check/types.ts index eacbcbe6..627c9f5c 100644 --- a/src/tier-check/types.ts +++ b/src/tier-check/types.ts @@ -62,8 +62,17 @@ export interface PolicySignalsResult extends CheckResult { export interface SpecTrackingResult extends CheckResult { latest_spec_release: string | null; latest_sdk_release: string | null; + // Deprecated: no longer read by tier-logic (superseded by the SEP-1730 + // pass/partial/fail windows), kept only for backward-compatible output. sdk_release_within_30d: boolean | null; days_gap: number | null; + target_spec_tag: string | null; + submitted_sdk_tag: string | null; + // SEP-1730 window verdicts read by tier-logic; status is the display + // label derived from the same windows. + meets_tier1_window: boolean | null; + meets_tier2_window: boolean | null; + reason?: string; } export interface TierScorecard {