{{ message }}
Repository navigation
Expand file tree
/
Copy pathgenerate_sbom.py
More file actions
410 lines (344 loc) · 14.7 KB
/
Copy pathgenerate_sbom.py
File metadata and controls
410 lines (344 loc) · 14.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
"""Tool for generating Software Bill of Materials (SBOM) for Python's dependencies"""
import glob
import hashlib
import json
import os
import random
import re
import subprocess
import sys
import time
import typing
import urllib.error
import urllib.request
from pathlib import Path, PurePosixPath, PureWindowsPath
CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent
# Before adding a new entry to this list, double check that
# the license expression is a valid SPDX license expression:
# See: https://spdx.org/licenses
ALLOWED_LICENSE_EXPRESSIONS = {
"Apache-2.0",
"Apache-2.0 OR BSD-2-Clause",
"BSD-2-Clause",
"BSD-3-Clause",
"CC0-1.0",
"ISC",
"LGPL-2.1-only",
"MIT",
"MPL-2.0",
"Python-2.0.1",
}
# Properties which are required for our purposes.
REQUIRED_PROPERTIES_PACKAGE = frozenset([
"SPDXID",
"name",
"versionInfo",
"downloadLocation",
"checksums",
"licenseConcluded",
"externalRefs",
"primaryPackagePurpose",
])
class PackageFiles(typing.NamedTuple):
"""Structure for describing the files of a package"""
include: list[str] | None
exclude: list[str] | None = None
# SBOMS don't have a method to specify the sources of files
# so we need to do that external to the SBOM itself. Add new
# values to 'exclude' if we create new files within tracked
# directories that aren't sourced from third-party packages.
PACKAGE_TO_FILES = {
"expat": PackageFiles(
include=["Modules/expat/**"],
exclude=[
"Modules/expat/expat_config.h",
"Modules/expat/pyexpatns.h",
"Modules/expat/refresh.sh",
]
),
"macholib": PackageFiles(
include=["Lib/ctypes/macholib/**"],
exclude=[
"Lib/ctypes/macholib/README.ctypes",
"Lib/ctypes/macholib/fetch_macholib",
"Lib/ctypes/macholib/fetch_macholib.bat",
],
),
"hacl-star": PackageFiles(
include=["Modules/_hacl/**"],
exclude=[
"Modules/_hacl/refresh.sh",
"Modules/_hacl/README.md",
"Modules/_hacl/python_hacl_namespace.h",
]
),
}
def spdx_id(value: str) -> str:
"""Encode a value into characters that are valid in an SPDX ID"""
return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value)
def error_if(value: bool, error_message: str) -> None:
"""Prints an error if a comparison fails along with a link to the devguide"""
if value:
print(error_message)
print("See 'https://devguide.python.org/developer-workflow/sbom' for more information.")
sys.exit(1)
def is_root_directory_git_index() -> bool:
"""Checks if the root directory is a git index"""
try:
subprocess.check_call(
["git", "-C", str(CPYTHON_ROOT_DIR), "rev-parse"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except subprocess.CalledProcessError:
return False
return True
def filter_gitignored_paths(paths: list[str]) -> list[str]:
"""
Filter out paths excluded by the gitignore file.
The output of 'git check-ignore --non-matching --verbose' looks
like this for non-matching (included) files:
'::<whitespace><path>'
And looks like this for matching (excluded) files:
'.gitignore:9:*.a Tools/lib.a'
"""
# No paths means no filtering to be done.
if not paths:
return []
# Filter out files in gitignore.
# Non-matching files show up as '::<whitespace><path>'
git_check_ignore_proc = subprocess.run(
["git", "check-ignore", "--verbose", "--non-matching", *paths],
cwd=CPYTHON_ROOT_DIR,
check=False,
stdout=subprocess.PIPE,
)
# 1 means matches, 0 means no matches.
assert git_check_ignore_proc.returncode in (0, 1)
# Paths may or may not be quoted, Windows quotes paths.
git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z")
# Return the list of paths sorted
git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines()
git_check_not_ignored = []
for line in git_check_ignore_lines:
if match := git_check_ignore_re.fullmatch(line):
git_check_not_ignored.append(match.group(2) or match.group(3))
return sorted(git_check_not_ignored)
def get_externals() -> list[str]:
"""
Parses 'PCbuild/get_externals.bat' for external libraries.
Returns a list of (git tag, name, version) tuples.
"""
get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat"
externals = re.findall(
r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s",
get_externals_bat_path.read_text()
)
return externals
def download_with_retries(download_location: str,
max_retries: int = 7,
base_delay: float = 2.25,
max_jitter: float = 1.0) -> typing.Any:
"""Download a file with exponential backoff retry."""
for attempt in range(max_retries + 1):
try:
resp = urllib.request.urlopen(download_location)
except (urllib.error.URLError, ConnectionError) as ex:
if attempt == max_retries:
msg = f"Download from {download_location} failed."
raise OSError(msg) from ex
time.sleep(base_delay**attempt + random.uniform(0, max_jitter))
else:
return resp
def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None:
"""Make a bunch of assertions about the SBOM package data to ensure it's consistent."""
for package in sbom_data["packages"]:
# Properties and ID must be properly formed.
error_if(
"name" not in package,
"Package is missing the 'name' field"
)
# Verify that the checksum matches the expected value
# and that the download URL is valid.
if "checksums" not in package or "CI" in os.environ:
download_location = package["downloadLocation"]
resp = download_with_retries(download_location)
error_if(resp.status != 200, f"Couldn't access URL: {download_location}'")
package["checksums"] = [{
"algorithm": "SHA256",
"checksumValue": hashlib.sha256(resp.read()).hexdigest()
}]
missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys())
error_if(
bool(missing_required_keys),
f"Package '{package['name']}' is missing required fields: {missing_required_keys}",
)
error_if(
package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"),
f"Package '{package['name']}' has a malformed SPDXID",
)
# Version must be in the download and external references.
version = package["versionInfo"]
error_if(
version not in package["downloadLocation"],
f"Version '{version}' for package '{package['name']} not in 'downloadLocation' field",
)
error_if(
any(version not in ref["referenceLocator"] for ref in package["externalRefs"]),
(
f"Version '{version}' for package '{package['name']} not in "
f"all 'externalRefs[].referenceLocator' fields"
),
)
# HACL* specifies its expected rev in a refresh script.
if package["name"] == "hacl-star":
hacl_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/_hacl/refresh.sh").read_text()
hacl_expected_rev_match = re.search(
r"expected_hacl_star_rev=([0-9a-f]{40})",
hacl_refresh_sh
)
hacl_expected_rev = hacl_expected_rev_match and hacl_expected_rev_match.group(1)
error_if(
hacl_expected_rev != version,
"HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'"
)
# libexpat specifies its expected rev in a refresh script.
if package["name"] == "expat":
libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text()
libexpat_expected_version_match = re.search(
r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"",
libexpat_refresh_sh
)
libexpat_expected_sha256_match = re.search(
r"expected_libexpat_sha256=\"([a-f0-9]{64})\"",
libexpat_refresh_sh
)
libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1)
libexpat_expected_sha256 = libexpat_expected_sha256_match and libexpat_expected_sha256_match.group(1)
error_if(
libexpat_expected_version != version,
"libexpat SBOM version doesn't match value in 'Modules/expat/refresh.sh'"
)
error_if(
package["checksums"] != [{
"algorithm": "SHA256",
"checksumValue": libexpat_expected_sha256
}],
"libexpat SBOM checksum doesn't match value in 'Modules/expat/refresh.sh'"
)
# License must be on the approved list for SPDX.
license_concluded = package["licenseConcluded"]
error_if(
license_concluded != "NOASSERTION",
"License identifier must be 'NOASSERTION'"
)
def create_source_sbom() -> None:
sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
sbom_data = json.loads(sbom_path.read_bytes())
# We regenerate all of this information. Package information
# should be preserved though since that is edited by humans.
sbom_data["files"] = []
sbom_data["relationships"] = []
# Ensure all packages in this tool are represented also in the SBOM file.
actual_names = {package["name"] for package in sbom_data["packages"]}
expected_names = set(PACKAGE_TO_FILES)
error_if(
actual_names != expected_names,
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
)
check_sbom_packages(sbom_data)
# We call 'sorted()' here a lot to avoid filesystem scan order issues.
for name, files in sorted(PACKAGE_TO_FILES.items()):
package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
exclude = files.exclude or ()
for include in sorted(files.include or ()):
# Find all the paths and then filter them through .gitignore.
paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True)
paths = filter_gitignored_paths(paths)
error_if(
len(paths) == 0,
f"No valid paths found at path '{include}' for package '{name}",
)
for path in paths:
# Normalize the filename from any combination of slashes.
path = str(PurePosixPath(PureWindowsPath(path)))
# Skip directories and excluded files
if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude:
continue
# SPDX requires SHA1 to be used for files, but we provide SHA256 too.
data = (CPYTHON_ROOT_DIR / path).read_bytes()
# We normalize line-endings for consistent checksums.
# This is a rudimentary check for binary files.
if b"\x00" not in data:
data = data.replace(b"\r\n", b"\n")
checksum_sha1 = hashlib.sha1(data).hexdigest()
checksum_sha256 = hashlib.sha256(data).hexdigest()
file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}")
sbom_data["files"].append({
"SPDXID": file_spdx_id,
"fileName": path,
"checksums": [
{"algorithm": "SHA1", "checksumValue": checksum_sha1},
{"algorithm": "SHA256", "checksumValue": checksum_sha256},
],
})
# Tie each file back to its respective package.
sbom_data["relationships"].append({
"spdxElementId": package_spdx_id,
"relatedSpdxElement": file_spdx_id,
"relationshipType": "CONTAINS",
})
# Update the SBOM on disk
sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
def create_externals_sbom() -> None:
sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json"
sbom_data = json.loads(sbom_path.read_bytes())
externals = get_externals()
externals_name_to_version = {}
externals_name_to_git_tag = {}
for git_tag in externals:
name, _, version = git_tag.rpartition("-")
externals_name_to_version[name] = version
externals_name_to_git_tag[name] = git_tag
# Ensure all packages in this tool are represented also in the SBOM file.
actual_names = {package["name"] for package in sbom_data["packages"]}
expected_names = set(externals_name_to_version)
error_if(
actual_names != expected_names,
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
)
# Set the versionInfo and downloadLocation fields for all packages.
for package in sbom_data["packages"]:
package_version = externals_name_to_version[package["name"]]
# Update the version information in all the locations.
package["versionInfo"] = package_version
for external_ref in package["externalRefs"]:
if external_ref["referenceType"] != "cpe23Type":
continue
# Version is the fifth field of a CPE.
cpe23ref = external_ref["referenceLocator"]
external_ref["referenceLocator"] = re.sub(
r"\A(cpe(?::[^:]+){4}):[^:]+:",
fr"\1:{package_version}:",
cpe23ref
)
download_location = (
f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz"
)
download_location_changed = download_location != package["downloadLocation"]
package["downloadLocation"] = download_location
# If the download URL has changed we want one to get recalulated.
if download_location_changed:
package.pop("checksums", None)
check_sbom_packages(sbom_data)
# Update the SBOM on disk
sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
def main() -> None:
# Don't regenerate the SBOM if we're not a git repository.
if not is_root_directory_git_index():
print("Skipping SBOM generation due to not being a git repository")
return
create_source_sbom()
create_externals_sbom()
if __name__ == "__main__":
main()
You can’t perform that action at this time.