{{ message }}
-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Expand file tree
/
Copy pathcodspeed-environment.py
More file actions
executable file
·306 lines (263 loc) · 11.7 KB
/
Copy pathcodspeed-environment.py
File metadata and controls
executable file
·306 lines (263 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
#!/usr/bin/env python3
"""Name the machine a CodSpeed measurement was taken on, and hold a branch to it.
CodSpeed reports a run as a change against a base run, and the runner uploads a
description of the machine beside the profile: `SystemInfo` in
`CodSpeedHQ/runner` carries the OS and its version, the architecture, the CPU's
vendor id, brand and flags, the physical core count and the installed memory.
When the two runs disagree on it the report says "different runtime
environments detected" and the numbers below that line are not a comparison of
two trees. A hosted runner pool hands out whatever machine is free, so two runs
of the same workflow do not share one by construction.
A simulated measurement does not move with how fast the machine is -- the cache
geometry the runner hands Valgrind is fixed, and two runs whose wall clock
differs by half still report the same counts. It does move with what the
machine makes the process execute: glibc resolves `memcpy`, `memset` and the
string routines through IFUNC selectors keyed on CPU flags, so one binary has
one instruction stream per selection. The toolchain is the same class of input
one level up, since a new stable rustc emits different code. So is the CPython
these benchmarks are linked against: `benches/execution.rs` and
`benches/microbenchmarks.rs` run each program under CPython through pyo3 as well
as under RustPython, and the runner's default interpreter moves with its image.
The rule this enforces: `main` defines the environment. Every run on it records
what it measured on, and a run on any other ref measures only when it matches
that recording. Anything else is skipped rather than uploaded, because an
uploaded cross-environment run is published as a regression of the branch that
happened to draw the other machine. A recording that cannot be read is
"anything else" too: an unknown machine is not a matching one, and reading it as
one would leave the guard open exactly while it has nothing to check against.
codspeed-environment.py --record build/codspeed-environment.json
codspeed-environment.py --record CURRENT --reference REFERENCE --github-output
Exits 0 whether or not the environments match; the verdict is the `match`
output, so the caller decides what a mismatch costs.
"""
import argparse
import hashlib
import json
import math
import os
import platform
import re
import subprocess
import sys
# The fields a comparison is made over. `SystemInfo` also carries the host name
# and the invoking user, which a hosted runner changes on every run and which
# therefore cannot be part of a rule that any run is expected to satisfy.
COMPARED_FIELDS = (
"os",
"arch",
"cpu_vendor_id",
"cpu_brand",
"cpu_cores",
"total_memory_gb",
"cpu_flags",
"rustc",
"libc",
"python",
)
def _command_output(argv):
try:
out = subprocess.run(
argv, capture_output=True, text=True, check=False, timeout=60
)
except (OSError, subprocess.SubprocessError):
return ""
# A probe that fails contributes nothing rather than an error string, so a
# field absent on one platform stays absent instead of becoming a value.
return out.stdout.strip() if out.returncode == 0 else ""
def _os_release():
"""The distribution and its version, as `SupportedOs` spells them."""
try:
with open("/etc/os-release", encoding="utf-8") as handle:
fields = dict(
line.rstrip("\n").split("=", 1)
for line in handle
if "=" in line and not line.startswith("#")
)
except OSError:
return f"{platform.system()} {platform.release()}"
name = fields.get("ID", "linux").strip('"')
version = fields.get("VERSION_ID", "").strip('"')
return f"{name} {version}".strip()
def _linux_cpu():
with open("/proc/cpuinfo", encoding="utf-8") as handle:
return parse_cpuinfo(handle.read())
def parse_cpuinfo(text):
blocks = [block for block in text.split("\n\n") if block.strip()]
def field(block, name):
match = re.search(rf"^{name}\s*:\s*(.*)$", block, re.MULTILINE)
return match.group(1).strip() if match else ""
first = blocks[0] if blocks else ""
# Count distinct physical cores the way `sysinfo::System::physical_core_count`
# does: one entry per (socket, core), so a hyperthreaded pair counts once.
# aarch64 names neither, and the file ends in a machine-wide block that no
# `processor` line introduces, so the fallback counts those lines instead.
cores = {(field(block, "physical id"), field(block, "core id")) for block in blocks}
cores.discard(("", ""))
logical = len(re.findall(r"^processor\s*:", text, re.MULTILINE))
# x86_64 spells the CPU's identity one way and aarch64 another; a field
# missing on one architecture is constant there and carries no signal.
vendor = field(first, "vendor_id") or field(first, "CPU implementer")
brand = (
field(first, "model name")
or " ".join(
field(first, name) for name in ("CPU part", "CPU variant", "CPU revision")
).strip()
)
flags = field(first, "flags") or field(first, "Features")
return {
"cpu_vendor_id": vendor,
"cpu_brand": brand,
"cpu_cores": len(cores) or logical,
"cpu_flags": sorted(flags.split()),
}
def _darwin_cpu():
def sysctl(name):
return _command_output(["sysctl", "-n", name])
flags = " ".join(
sysctl(name) for name in ("machdep.cpu.features", "hw.optional.arm.caps")
)
return {
"cpu_vendor_id": sysctl("machdep.cpu.vendor") or platform.machine(),
"cpu_brand": sysctl("machdep.cpu.brand_string"),
"cpu_cores": int(sysctl("hw.physicalcpu") or 0),
"cpu_flags": sorted(flags.split()),
}
def _total_memory_gb():
"""Installed memory in whole GiB, rounded up as `sysinfo` reports it."""
try:
with open("/proc/meminfo", encoding="utf-8") as handle:
for line in handle:
if line.startswith("MemTotal:"):
return math.ceil(int(line.split()[1]) * 1024 / 1024**3)
except OSError:
pass
size = _command_output(["sysctl", "-n", "hw.memsize"])
return math.ceil(int(size) / 1024**3) if size.isdigit() else 0
def _libc_version():
"""The C library whose IFUNC selectors decide the string routines."""
version = _command_output(["ldd", "--version"])
return version.splitlines()[0].strip() if version else ""
def _python_version():
"""The interpreter the benchmarks measure beside RustPython.
Read from the process running this script rather than from a subprocess, so
it names the `python3` the workflow put on PATH -- which is the one the
pyo3 build resolves libpython from.
"""
return f"{platform.python_implementation()} {platform.python_version()}"
def collect():
cpu = _linux_cpu() if sys.platform.startswith("linux") else _darwin_cpu()
environment = {
"os": _os_release(),
"arch": platform.machine(),
"total_memory_gb": _total_memory_gb(),
"rustc": _command_output(["rustc", "--version"]),
"libc": _libc_version(),
"python": _python_version(),
**cpu,
}
environment["digest"] = digest(environment)
return environment
def digest(environment):
payload = json.dumps(
{name: environment.get(name) for name in COMPARED_FIELDS},
sort_keys=True,
separators=(",", ":"),
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
def differences(current, reference):
return [
(name, reference.get(name), current.get(name))
for name in COMPARED_FIELDS
if reference.get(name) != current.get(name)
]
def _render(environment):
for name in COMPARED_FIELDS:
value = environment.get(name)
if name == "cpu_flags":
value = f"{len(value)} flags"
print(f" {name:<17}{value}")
print(f" {'digest':<17}{environment['digest']}")
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument(
"--record", metavar="PATH", help="write this machine's environment as JSON"
)
parser.add_argument(
"--reference",
metavar="PATH",
help="compare against a previously recorded environment; a file that"
" cannot be read leaves the machine unidentified, which is not a match",
)
parser.add_argument(
"--github-output",
action="store_true",
help="append `match` and `digest` to $GITHUB_OUTPUT",
)
args = parser.parse_args(argv)
current = collect()
print("Benchmark environment:")
_render(current)
match = True
annotation = ""
if args.reference:
try:
with open(args.reference, encoding="utf-8") as handle:
reference = json.load(handle)
except (OSError, ValueError):
match = False
# A warning rather than a notice: a mismatch is the guard working,
# while an unreadable recording means it has nothing to work from,
# and a job that quietly measures nothing forever looks the same as
# one that measures.
annotation = (
"::warning title=CodSpeed benchmarks skipped::No recording of the"
" machine the comparison base was measured on could be read at"
f" {args.reference}, so a measurement taken here cannot be shown"
" to be comparable with it."
)
print(f"\nNo reference environment at {args.reference}.")
else:
moved = differences(current, reference)
if moved:
match = False
annotation = (
"::notice title=CodSpeed benchmarks skipped::"
f"This runner ({current['cpu_brand']}, {current['digest']}) is"
" not the machine the comparison base was measured on, and an"
" upload from it would be published as a regression."
)
# Digest the reference's own fields rather than reading back the
# digest stored beside them, so a recording written by an older
# field set is named by what it actually holds.
print(
f"\nThis machine is not the one the reference was taken on"
f" ({digest(reference)} -> {current['digest']}):"
)
for name, was, now in moved:
if name == "cpu_flags":
was_set, now_set = set(was or ()), set(now or ())
was = " ".join(sorted(was_set - now_set)) or "-"
now = " ".join(sorted(now_set - was_set)) or "-"
print(f" {name}: {was} -> {now}")
else:
print(f"\nThis machine matches the reference ({current['digest']}).")
if args.record:
directory = os.path.dirname(args.record)
if directory:
os.makedirs(directory, exist_ok=True)
with open(args.record, "w", encoding="utf-8") as handle:
json.dump(current, handle, indent=2, sort_keys=True)
handle.write("\n")
if args.github_output:
# A skipped job is otherwise indistinguishable from one that never had
# benchmarks, and the reason is the one fact a reader needs.
if annotation:
print(annotation)
output = os.environ.get("GITHUB_OUTPUT")
if output:
with open(output, "a", encoding="utf-8") as handle:
handle.write(f"match={'true' if match else 'false'}\n")
handle.write(f"digest={current['digest']}\n")
return 0
if __name__ == "__main__":
sys.exit(main())
You can’t perform that action at this time.