Skip to content
Navigation Menu
{{ message }}
forked from NVIDIA/cuda-python
-
Notifications
You must be signed in to change notification settings - Fork 0
278 lines (250 loc) · 11.6 KB
/
Copy pathtest-wheel-linux.yml
File metadata and controls
278 lines (250 loc) · 11.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
#
# SPDX-License-Identifier: Apache-2.0
name: "CI: Test wheels"
on:
workflow_call:
inputs:
host-platform:
type: string
required: true
python-version:
type: string
required: true
build-ctk-ver:
type: string
required: true
cuda-version:
type: string
required: true
local-ctk:
type: string
required: true
runner:
type: string
required: true
jobs:
test:
# The build stage could fail but we want the CI to keep moving.
if: ${{ github.repository_owner == 'nvidia' && !cancelled() }}
runs-on: ${{ (inputs.runner == 'default' && inputs.host-platform == 'linux-64' && 'linux-amd64-gpu-v100-latest-1') ||
(inputs.runner == 'default' && inputs.host-platform == 'linux-aarch64' && 'linux-arm64-gpu-a100-latest-1') ||
(inputs.runner == 'H100' && 'linux-amd64-gpu-h100-latest-1') }}
# Our self-hosted runners require a container
# TODO: use a different (nvidia?) container
container:
options: -u root --security-opt seccomp=unconfined --shm-size 16g
image: ubuntu:22.04
env:
NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
defaults:
run:
shell: bash --noprofile --norc -xeuo pipefail {0}
steps:
- name: Ensure GPU is working
run: nvidia-smi
- name: Checkout ${{ github.event.repository.name }}
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set environment variables
run: |
PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
if [[ "${{ inputs.host-platform }}" == linux* ]]; then
REPO_DIR=$(pwd)
elif [[ "${{ inputs.host-platform }}" == win* ]]; then
PWD=$(pwd)
REPO_DIR=$(cygpath -w $PWD)
fi
BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ inputs.build-ctk-ver }})"
TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ inputs.cuda-version }})"
if [[ $BUILD_CUDA_MAJOR != $TEST_CUDA_MAJOR ]]; then
SKIP_CUDA_BINDINGS_TEST=1
SKIP_CUDA_CORE_CYTHON_TEST=0
else
SKIP_CUDA_BINDINGS_TEST=0
BUILD_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ inputs.build-ctk-ver }})"
TEST_CUDA_MINOR="$(cut -d '.' -f 2 <<< ${{ inputs.cuda-version }})"
if [[ $BUILD_CUDA_MINOR != $TEST_CUDA_MINOR ]]; then
SKIP_CUDA_CORE_CYTHON_TEST=1
else
SKIP_CUDA_CORE_CYTHON_TEST=0
fi
fi
# make outputs from the previous job as env vars
CUDA_CORE_ARTIFACT_BASENAME="cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}"
echo "PYTHON_VERSION_FORMATTED=${PYTHON_VERSION_FORMATTED}" >> $GITHUB_ENV
echo "CUDA_CORE_ARTIFACT_BASENAME=${CUDA_CORE_ARTIFACT_BASENAME}" >> $GITHUB_ENV
echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
CUDA_BINDINGS_ARTIFACT_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.build-ctk-ver }}-${{ inputs.host-platform }}"
echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV
echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}-${{ github.sha }}" >> $GITHUB_ENV
echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
echo "SKIP_CUDA_CORE_CYTHON_TEST=${SKIP_CUDA_CORE_CYTHON_TEST}" >> $GITHUB_ENV
- name: Install dependencies
uses: ./.github/actions/install_unix_deps
continue-on-error: false
with:
# gcc for Cython tests, jq/wget for artifact fetching
dependencies: "build-essential jq wget"
dependent_exes: "gcc jq wget"
- name: Download cuda-python build artifacts
if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
uses: actions/download-artifact@v4
with:
name: cuda-python-wheel
path: .
- name: Download cuda.bindings build artifacts
if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0'}}
uses: actions/download-artifact@v4
with:
name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
- name: Download cuda-python & cuda.bindings build artifacts from the prior branch
if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '1'}}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt.
# gh is needed for artifact fetching.
mkdir -p -m 755 /etc/apt/keyrings \
&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
&& cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
&& apt update \
&& apt install gh -y
OLD_BRANCH=$(cat .github/BACKPORT_BRANCH)
OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
LATEST_PRIOR_RUN_ID=$(gh run list -b ${OLD_BRANCH} -L 1 -w "CI: Build and test" -s completed -R NVIDIA/cuda-python --json databaseId | jq '.[]| .databaseId')
if [[ "$LATEST_PRIOR_RUN_ID" == "" ]]; then
echo "LATEST_PRIOR_RUN_ID not found!"
exit 1
fi
gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python
ls -al $OLD_BASENAME
mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
rmdir $OLD_BASENAME
gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python
ls -al cuda-python-wheel
mv cuda-python-wheel/*.whl .
rmdir cuda-python-wheel
- name: Display structure of downloaded cuda-python artifacts
run: |
pwd
ls -lahR .
- name: Display structure of downloaded cuda.bindings artifacts
run: |
pwd
ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR
- name: Download cuda.core build artifacts
uses: actions/download-artifact@v4
with:
name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
- name: Display structure of downloaded cuda.core build artifacts
run: |
pwd
ls -lahR $CUDA_CORE_ARTIFACTS_DIR
- name: Set up Python ${{ inputs.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
env:
# we use self-hosted runners on which setup-python behaves weirdly...
AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"
- name: Set up mini CTK
if: ${{ inputs.local-ctk == '1' }}
uses: ./.github/actions/fetch_ctk
continue-on-error: false
with:
host-platform: ${{ inputs.host-platform }}
cuda-version: ${{ inputs.cuda-version }}
- name: Set up compute-sanitizer
run: |
# We don't test compute-sanitizer on CTK<12 because backporting fixes is too much effort
# We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix
# Only local ctk installs have compute-sanitizer; there is not wheel for it
if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.cuda-version }}" != "11.8.0" && "${{ inputs.local-ctk }}" == 1 ]]; then
COMPUTE_SANITIZER="${CUDA_HOME}/bin/compute-sanitizer"
COMPUTE_SANITIZER_VERSION=$(${COMPUTE_SANITIZER} --version | grep -Eo "[0-9]{4}\.[0-9]\.[0-9]" | sed -e 's/\.//g')
SANITIZER_CMD="${COMPUTE_SANITIZER} --target-processes=all --launch-timeout=0 --tool=memcheck --error-exitcode=1"
if [[ "$COMPUTE_SANITIZER_VERSION" -ge 202111 ]]; then
SANITIZER_CMD="${SANITIZER_CMD} --padding=32"
fi
echo "CUDA_PYTHON_TESTING_WITH_COMPUTE_SANITIZER=1" >> $GITHUB_ENV
else
SANITIZER_CMD=""
fi
echo "SANITIZER_CMD=${SANITIZER_CMD}" >> $GITHUB_ENV
- name: Run cuda.bindings tests
if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
run: |
pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
if [[ "${{ inputs.local-ctk }}" == 1 ]]; then
ls $CUDA_PATH
pip install *.whl
else
pip install $(ls *.whl)[all]
fi
popd
pushd ./cuda_bindings
pip install -r requirements.txt
${SANITIZER_CMD} pytest -rxXs -v tests/
# It is a bit convoluted to run the Cython tests against CTK wheels,
# so let's just skip them.
if [[ "${{ inputs.local-ctk }}" == 1 ]]; then
if [[ "${{ inputs.host-platform }}" == linux* ]]; then
bash tests/cython/build_tests.sh
elif [[ "${{ inputs.host-platform }}" == win* ]]; then
# TODO: enable this once win-64 runners are up
exit 1
fi
${SANITIZER_CMD} pytest -rxXs -v tests/cython
fi
popd
- name: Run cuda.core tests
run: |
# If build/test majors match: cuda.bindings is installed in the previous step.
# If mismatch: cuda.bindings is installed from the backport branch.
if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then
pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
if [[ "${{ inputs.local-ctk }}" == 1 ]]; then
pip install *.whl
else
pip install $(ls *.whl)[all]
fi
popd
fi
TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ inputs.cuda-version }})"
pushd "${CUDA_CORE_ARTIFACTS_DIR}"
pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}"]
popd
pushd ./cuda_core
pip install -r "tests/requirements-cu${TEST_CUDA_MAJOR}.txt"
${SANITIZER_CMD} pytest -rxXs -v tests/
# It is a bit convoluted to run the Cython tests against CTK wheels,
# so let's just skip them. Also, currently our CI always installs the
# latest bindings (from either major version). This is not compatible
# with the test requirements.
if [[ "${{ inputs.local-ctk }}" == 1 && "${SKIP_CUDA_CORE_CYTHON_TEST}" == 0 ]]; then
pip install cython setuptools # setuptools needed starting PY312
if [[ "${{ inputs.host-platform }}" == linux* ]]; then
bash tests/cython/build_tests.sh
elif [[ "${{ inputs.host-platform }}" == win* ]]; then
# TODO: enable this once win-64 runners are up
exit 1
fi
${SANITIZER_CMD} pytest -rxXs -v tests/cython
fi
popd
- name: Ensure cuda-python installable
run: |
if [[ "${{ inputs.local-ctk }}" == 1 ]]; then
pip install cuda_python*.whl
else
pip install $(ls cuda_python*.whl)[all]
fi
You can’t perform that action at this time.