FilesExpand file tree

main

/

conformance-weekly.yaml

Copy path

View runs

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

189 lines (178 loc) · 6.89 KB

main

/

conformance-weekly.yaml

Copy path

Top

File metadata and controls

• Code
• Blame

189 lines (178 loc) · 6.89 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

name: conformance-weekly

# Tracks the newest published @modelcontextprotocol/conformance; the

# on:pull_request pipeline pins an exact version instead. Also publishes

# score badges to the orphan `badges` branch, consumed by the README.

on:

schedule:

- cron: '0 6 * * 1' # Mondays 06:00 UTC

workflow_dispatch:

permissions:

contents: write

issues: write

jobs:

server:

name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

include:

- spec-version: '2025-11-25'

dist-tag: latest

path: '/'

baseline: conformance-baseline-2025-11-25.yml

- spec-version: '2026-07-28'

dist-tag: alpha

path: '/'

baseline: conformance-baseline-2026-07-28.yml

steps:

- uses: actions/checkout@v7

- uses: actions/setup-node@v7

with:

node-version: '22'

- run: composer install --prefer-dist --no-progress --no-interaction

- name: Start conformance server

run: |

mkdir -p tests/Conformance/sessions tests/Conformance/logs

chmod -R 777 tests/Conformance/sessions tests/Conformance/logs

docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d

sleep 5

- name: Run conformance tests

working-directory: ./tests/Conformance

run: >-

npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} server

--url http://localhost:8000${{ matrix.path }}

--suite all

--spec-version ${{ matrix.spec-version }}

--expected-failures ${{ matrix.baseline }}

--output-dir results

- name: Generate score badge

if: always()

run: php tests/Conformance/score.php server ${{ matrix.spec-version }}

- name: Show docker logs on failure

if: failure()

run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs

- name: Upload conformance results

if: failure()

uses: actions/upload-artifact@v7

with:

name: conformance-server-results-${{ matrix.spec-version }}

path: |

tests/Conformance/logs

tests/Conformance/results

- name: Upload score badge

if: always()

uses: actions/upload-artifact@v7

with:

name: badge-server-${{ matrix.spec-version }}

path: tests/Conformance/server-conformance-${{ matrix.spec-version }}.json

client:

name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

include:

- spec-version: '2025-11-25'

dist-tag: latest

baseline: conformance-baseline-2025-11-25.yml

- spec-version: '2026-07-28'

dist-tag: alpha

baseline: conformance-baseline-2026-07-28.yml

steps:

- uses: actions/checkout@v7

- uses: shivammathur/setup-php@v2

with:

php-version: '8.4'

coverage: none

- uses: actions/setup-node@v7

with:

node-version: '22'

- run: composer install --prefer-dist --no-progress --no-interaction

- run: mkdir -p tests/Conformance/logs

- name: Run conformance tests

working-directory: ./tests/Conformance

run: >-

npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} client

--command "php ${{ github.workspace }}/tests/Conformance/client.php"

--suite all

--spec-version ${{ matrix.spec-version }}

--expected-failures ${{ matrix.baseline }}

--output-dir results

- name: Generate score badge

if: always()

run: php tests/Conformance/score.php client ${{ matrix.spec-version }}

- name: Upload conformance results

if: failure()

uses: actions/upload-artifact@v7

with:

name: conformance-client-results-${{ matrix.spec-version }}

path: |

tests/Conformance/logs

tests/Conformance/results

- name: Upload score badge

if: always()

uses: actions/upload-artifact@v7

with:

name: badge-client-${{ matrix.spec-version }}

path: tests/Conformance/client-conformance-${{ matrix.spec-version }}.json

notify:

name: Open issue on failure

runs-on: ubuntu-latest

needs: [server, client]

if: failure()

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

GH_REPO: ${{ github.repository }}

EVENT_NAME: ${{ github.event_name }}

RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

steps:

- name: File or comment tracking issue

run: |

existing=$(gh issue list --label conformance-weekly --state open --json number --jq '.[0].number // empty')

if [ -n "$existing" ]; then

gh issue comment "$existing" --body "New failure on $(date -u +%FT%TZ): $RUN_URL"

else

gh issue create \

--title '[conformance] Weekly conformance run failed' \

--label conformance-weekly \

--body "Conformance against the newest published \`@modelcontextprotocol/conformance\` failed.

- Run: $RUN_URL

- Trigger: $EVENT_NAME

- Date: $(date -u +%FT%TZ)

Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to the matching \`tests/Conformance/conformance-baseline-<spec-version>.yml\`."

fi

publish:

name: Publish conformance badges

runs-on: ubuntu-latest

needs: [server, client]

if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }}

steps:

- uses: actions/checkout@v7

- uses: actions/download-artifact@v8

with:

pattern: badge-*

path: badges-in

merge-multiple: true

- name: Publish to badges branch

run: |

git config user.name "github-actions[bot]"

git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

if git ls-remote --exit-code --heads origin badges >/dev/null 2>&1; then

git fetch origin badges

git worktree add badges-wt badges

else

git worktree add --detach badges-wt

git -C badges-wt checkout --orphan badges

git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true

fi

rm -f badges-wt/server-conformance.json badges-wt/client-conformance.json

cp badges-in/*.json badges-wt/

cd badges-wt

git add -A

if git diff --cached --quiet; then

echo "Conformance scores unchanged."

else

git commit -m "Update conformance score badges"

git push origin badges

fi