fix: tier-check branch support and draft/extension scoring by felixweinberger · Pull Request #153 · modelcontextprotocol/conformance · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 33 additions & 17 deletions .claude/skills/mcp-sdk-tier-audit/SKILL.md
21 changes: 19 additions & 2 deletions src/tier-check/checks/test-conformance-results.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,15 @@ import {
listActiveClientScenarios,
getScenarioSpecVersions
} from '../../scenarios';
import { ConformanceCheck } from '../../types';
import { ConformanceCheck, SpecVersion } from '../../types';

const NON_SCORING_VERSIONS: SpecVersion[] = ['draft', 'extension'];

/** Whether a scenario counts toward tier scoring (has at least one date-versioned spec). */
function isTierScoring(specVersions?: SpecVersion[]): boolean {
if (!specVersions || specVersions.length === 0) return true; // unknown = count it
return specVersions.some((v) => !NON_SCORING_VERSIONS.includes(v));
}

/**
* Parse conformance results from an output directory.
Expand Down Expand Up @@ -132,7 +140,16 @@ function reconcileWithExpected(
}
}

result.pass_rate = result.total > 0 ? result.passed / result.total : 0;
// pass_rate only counts tier-scoring scenarios (date-versioned, not draft/extension).
// passed/failed/total reflect ALL scenarios for full reporting; pass_rate and status
// reflect only tier-scoring scenarios for tier logic.
const tierDetails = result.details.filter((d) =>
isTierScoring(d.specVersions)
);
const tierPassed = tierDetails.filter((d) => d.passed).length;
const tierTotal = tierDetails.length;

result.pass_rate = tierTotal > 0 ? tierPassed / tierTotal : 0;
result.status =
result.pass_rate >= 1.0
? 'pass'
Expand Down
129 changes: 96 additions & 33 deletions src/tier-check/output.ts
Loading