FilesExpand file tree

main

/

nightly_tests.yml

Copy path

View runs

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

408 lines (379 loc) · 13.7 KB

main

/

nightly_tests.yml

Copy path

Top

File metadata and controls

• Code
• Blame

408 lines (379 loc) · 13.7 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

name: Nightly and release tests on main/release branch

on:

workflow_dispatch:

schedule:

- cron: "0 0 * * *" # every day at midnight

env:

DIFFUSERS_IS_CI: yes

HF_HUB_ENABLE_HF_TRANSFER: 1

OMP_NUM_THREADS: 8

MKL_NUM_THREADS: 8

PYTEST_TIMEOUT: 600

RUN_SLOW: yes

RUN_NIGHTLY: yes

PIPELINE_USAGE_CUTOFF: 5000

SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

jobs:

setup_torch_cuda_pipeline_matrix:

name: Setup Torch Pipelines CUDA Slow Tests Matrix

runs-on:

group: aws-general-8-plus

container:

image: diffusers/diffusers-pytorch-cpu

outputs:

pipeline_test_matrix: ${{ steps.fetch_pipeline_matrix.outputs.pipeline_test_matrix }}

steps:

- name: Checkout diffusers

uses: actions/checkout@v3

with:

fetch-depth: 2

- name: Install dependencies

run: |

pip install -e .[test]

pip install huggingface_hub

- name: Fetch Pipeline Matrix

id: fetch_pipeline_matrix

run: |

matrix=$(python utils/fetch_torch_cuda_pipeline_test_matrix.py)

echo $matrix

echo "pipeline_test_matrix=$matrix" >> $GITHUB_OUTPUT

- name: Pipeline Tests Artifacts

if: ${{ always() }}

uses: actions/upload-artifact@v4

with:

name: test-pipelines.json

path: reports

run_nightly_tests_for_torch_pipelines:

name: Nightly Torch Pipelines CUDA Tests

needs: setup_torch_cuda_pipeline_matrix

strategy:

fail-fast: false

max-parallel: 8

matrix:

module: ${{ fromJson(needs.setup_torch_cuda_pipeline_matrix.outputs.pipeline_test_matrix) }}

runs-on:

group: aws-g4dn-2xlarge

container:

image: diffusers/diffusers-pytorch-cuda

options: --shm-size "16gb" --ipc host --gpus 0

steps:

- name: Checkout diffusers

uses: actions/checkout@v3

with:

fetch-depth: 2

- name: NVIDIA-SMI

run: nvidia-smi

- name: Install dependencies

run: |

python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"

python -m uv pip install -e [quality,test]

pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git

python -m uv pip install pytest-reportlog

- name: Environment

run: |

python utils/print_env.py

- name: Pipeline CUDA Test

env:

HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}

# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms

CUBLAS_WORKSPACE_CONFIG: :16:8

run: |

python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \

-s -v -k "not Flax and not Onnx" \

--make-reports=tests_pipeline_${{ matrix.module }}_cuda \

--report-log=tests_pipeline_${{ matrix.module }}_cuda.log \

tests/pipelines/${{ matrix.module }}

- name: Failure short reports

if: ${{ failure() }}

run: |

cat reports/tests_pipeline_${{ matrix.module }}_cuda_stats.txt

cat reports/tests_pipeline_${{ matrix.module }}_cuda_failures_short.txt

- name: Test suite reports artifacts

if: ${{ always() }}

uses: actions/upload-artifact@v4

with:

name: pipeline_${{ matrix.module }}_test_reports

path: reports

- name: Generate Report and Notify Channel

if: always()

run: |

pip install slack_sdk tabulate

python utils/log_reports.py >> $GITHUB_STEP_SUMMARY

run_nightly_tests_for_other_torch_modules:

name: Nightly Torch CUDA Tests

runs-on:

group: aws-g4dn-2xlarge

container:

image: diffusers/diffusers-pytorch-cuda

options: --shm-size "16gb" --ipc host --gpus 0

defaults:

run:

shell: bash

strategy:

fail-fast: false

max-parallel: 2

matrix:

module: [models, schedulers, lora, others, single_file, examples]

steps:

- name: Checkout diffusers

uses: actions/checkout@v3

with:

fetch-depth: 2

- name: Install dependencies

run: |

python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"

python -m uv pip install -e [quality,test]

python -m uv pip install peft@git+https://github.com/huggingface/peft.git

pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git

python -m uv pip install pytest-reportlog

- name: Environment

run: python utils/print_env.py

- name: Run nightly PyTorch CUDA tests for non-pipeline modules

if: ${{ matrix.module != 'examples'}}

env:

HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}

# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms

CUBLAS_WORKSPACE_CONFIG: :16:8

run: |

python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \

-s -v -k "not Flax and not Onnx" \

--make-reports=tests_torch_${{ matrix.module }}_cuda \

--report-log=tests_torch_${{ matrix.module }}_cuda.log \

tests/${{ matrix.module }}

- name: Run nightly example tests with Torch

if: ${{ matrix.module == 'examples' }}

env:

HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}

# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms

CUBLAS_WORKSPACE_CONFIG: :16:8

run: |

python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \

-s -v --make-reports=examples_torch_cuda \

--report-log=examples_torch_cuda.log \

examples/

- name: Failure short reports

if: ${{ failure() }}

run: |

cat reports/tests_torch_${{ matrix.module }}_cuda_stats.txt

cat reports/tests_torch_${{ matrix.module }}_cuda_failures_short.txt

- name: Test suite reports artifacts

if: ${{ always() }}

uses: actions/upload-artifact@v4

with:

name: torch_${{ matrix.module }}_cuda_test_reports

path: reports

- name: Generate Report and Notify Channel

if: always()

run: |

pip install slack_sdk tabulate

python utils/log_reports.py >> $GITHUB_STEP_SUMMARY

run_flax_tpu_tests:

name: Nightly Flax TPU Tests

runs-on: docker-tpu

if: github.event_name == 'schedule'

container:

image: diffusers/diffusers-flax-tpu

options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ --privileged

defaults:

run:

shell: bash

steps:

- name: Checkout diffusers

uses: actions/checkout@v3

with:

fetch-depth: 2

- name: Install dependencies

run: |

python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"

python -m uv pip install -e [quality,test]

pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git

python -m uv pip install pytest-reportlog

- name: Environment

run: python utils/print_env.py

- name: Run nightly Flax TPU tests

env:

HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}

run: |

python -m pytest -n 0 \

-s -v -k "Flax" \

--make-reports=tests_flax_tpu \

--report-log=tests_flax_tpu.log \

tests/

- name: Failure short reports

if: ${{ failure() }}

run: |

cat reports/tests_flax_tpu_stats.txt

cat reports/tests_flax_tpu_failures_short.txt

- name: Test suite reports artifacts

if: ${{ always() }}

uses: actions/upload-artifact@v4

with:

name: flax_tpu_test_reports

path: reports

- name: Generate Report and Notify Channel

if: always()

run: |

pip install slack_sdk tabulate

python utils/log_reports.py >> $GITHUB_STEP_SUMMARY

run_nightly_onnx_tests:

name: Nightly ONNXRuntime CUDA tests on Ubuntu

runs-on:

group: aws-g4dn-2xlarge

container:

image: diffusers/diffusers-onnxruntime-cuda

options: --gpus 0 --shm-size "16gb" --ipc host

steps:

- name: Checkout diffusers

uses: actions/checkout@v3

with:

fetch-depth: 2

- name: NVIDIA-SMI

run: nvidia-smi

- name: Install dependencies

run: |

python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"

python -m uv pip install -e [quality,test]

pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git

python -m uv pip install pytest-reportlog

- name: Environment

run: python utils/print_env.py

- name: Run Nightly ONNXRuntime CUDA tests

env:

HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}

run: |

python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \

-s -v -k "Onnx" \

--make-reports=tests_onnx_cuda \

--report-log=tests_onnx_cuda.log \

tests/

- name: Failure short reports

if: ${{ failure() }}

run: |

cat reports/tests_onnx_cuda_stats.txt

cat reports/tests_onnx_cuda_failures_short.txt

- name: Test suite reports artifacts

if: ${{ always() }}

uses: actions/upload-artifact@v4

with:

name: tests_onnx_cuda_reports

path: reports

- name: Generate Report and Notify Channel

if: always()

run: |

pip install slack_sdk tabulate

python utils/log_reports.py >> $GITHUB_STEP_SUMMARY

# M1 runner currently not well supported

# TODO: (Dhruv) add these back when we setup better testing for Apple Silicon

# run_nightly_tests_apple_m1:

# name: Nightly PyTorch MPS tests on MacOS

# runs-on: [ self-hosted, apple-m1 ]

# if: github.event_name == 'schedule'

#

# steps:

# - name: Checkout diffusers

# uses: actions/checkout@v3

# with:

# fetch-depth: 2

#

# - name: Clean checkout

# shell: arch -arch arm64 bash {0}

# run: |

# git clean -fxd

# - name: Setup miniconda

# uses: ./.github/actions/setup-miniconda

# with:

# python-version: 3.9

#

# - name: Install dependencies

# shell: arch -arch arm64 bash {0}

# run: |

# ${CONDA_RUN} python -m pip install --upgrade pip uv

# ${CONDA_RUN} python -m uv pip install -e [quality,test]

# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu

# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate

# ${CONDA_RUN} python -m uv pip install pytest-reportlog

# - name: Environment

# shell: arch -arch arm64 bash {0}

# run: |

# ${CONDA_RUN} python utils/print_env.py

# - name: Run nightly PyTorch tests on M1 (MPS)

# shell: arch -arch arm64 bash {0}

# env:

# HF_HOME: /System/Volumes/Data/mnt/cache

# HF_TOKEN: ${{ secrets.HF_TOKEN }}

# run: |

# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \

# --report-log=tests_torch_mps.log \

# tests/

# - name: Failure short reports

# if: ${{ failure() }}

# run: cat reports/tests_torch_mps_failures_short.txt

#

# - name: Test suite reports artifacts

# if: ${{ always() }}

# uses: actions/upload-artifact@v4

# with:

# name: torch_mps_test_reports

# path: reports

#

# - name: Generate Report and Notify Channel

# if: always()

# run: |

# pip install slack_sdk tabulate

# python utils/log_reports.py >> $GITHUB_STEP_SUMMARY run_nightly_tests_apple_m1:

# name: Nightly PyTorch MPS tests on MacOS

# runs-on: [ self-hosted, apple-m1 ]

# if: github.event_name == 'schedule'

#

# steps:

# - name: Checkout diffusers

# uses: actions/checkout@v3

# with:

# fetch-depth: 2

#

# - name: Clean checkout

# shell: arch -arch arm64 bash {0}

# run: |

# git clean -fxd

# - name: Setup miniconda

# uses: ./.github/actions/setup-miniconda

# with:

# python-version: 3.9

#

# - name: Install dependencies

# shell: arch -arch arm64 bash {0}

# run: |

# ${CONDA_RUN} python -m pip install --upgrade pip uv

# ${CONDA_RUN} python -m uv pip install -e [quality,test]

# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu

# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate

# ${CONDA_RUN} python -m uv pip install pytest-reportlog

# - name: Environment

# shell: arch -arch arm64 bash {0}

# run: |

# ${CONDA_RUN} python utils/print_env.py

# - name: Run nightly PyTorch tests on M1 (MPS)

# shell: arch -arch arm64 bash {0}

# env:

# HF_HOME: /System/Volumes/Data/mnt/cache

# HF_TOKEN: ${{ secrets.HF_TOKEN }}

# run: |

# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \

# --report-log=tests_torch_mps.log \

# tests/

# - name: Failure short reports

# if: ${{ failure() }}

# run: cat reports/tests_torch_mps_failures_short.txt

#

# - name: Test suite reports artifacts

# if: ${{ always() }}

# uses: actions/upload-artifact@v4

# with:

# name: torch_mps_test_reports

# path: reports

#

# - name: Generate Report and Notify Channel

# if: always()

# run: |

# pip install slack_sdk tabulate

# python utils/log_reports.py >> $GITHUB_STEP_SUMMARY