{{ message }}
Repository navigation
Expand file tree
/
Copy pathcoverage.sh
More file actions
executable file
·290 lines (256 loc) · 11.3 KB
/
Copy pathcoverage.sh
File metadata and controls
executable file
·290 lines (256 loc) · 11.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
#!/bin/bash
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the Apache 2.0 License.
# Aggregate LLVM coverage data produced by test runs and produce coverage
# statistics, including overall summaries and details of uncovered lines.
set -euo pipefail
usage() {
cat <<EOF
Usage: $(basename "$0") [OPTIONS] [<binary> ...]
Aggregate LLVM coverage data from test runs and produce coverage reports.
Build CCF with -DCOVERAGE=ON, run the desired tests (e.g. via ctest -L unit),
then call this script from the build directory.
Options:
-d <dir> Directory to search for .profraw files (default: .)
-o <file> Output merged profile file (default: <dir>/coverage.profdata)
--add-profile <file>
Also merge <file>, a .profraw or .profdata file such as one
produced with --merge-only on another machine (repeatable)
--merge-only Only merge profiles into the output file, without reading
binaries or producing reports
--html <dir> Generate an HTML coverage report in <dir>
--show-uncovered Print files and line numbers with zero coverage
-h, --help Show this help
Binaries:
One or more instrumented executables may be given as positional arguments.
If none are given, the script reads the list from <dir>/coverage_binaries.txt,
which is generated automatically by CMake when -DCOVERAGE=ON is set.
Examples:
# From the build directory after running unit tests:
ctest -L unit
../scripts/coverage.sh
# Generate an HTML report:
../scripts/coverage.sh --html ./coverage_html
# Show which specific lines are uncovered:
../scripts/coverage.sh --show-uncovered
# Specify binaries explicitly:
../scripts/coverage.sh map_test crypto_test
# Combine with coverage collected in another build directory, for example
# on another machine or platform:
../scripts/coverage.sh --merge-only -o other.profdata # in that directory
../scripts/coverage.sh --add-profile other.profdata # in this directory
Notes:
- Tests must be built and run with -DCOVERAGE=ON. The build system
automatically sets LLVM_PROFILE_FILE so each test writes its own
uniquely-named .profraw file.
- Profiles added with --add-profile should come from builds of the same
source revision, compiler and compile options. llvm-cov reports
functions compiled differently as having mismatched data, and excludes
them from reports.
- Reports include framework code under src/ and include/, excluding tests
and performance code.
- Requires llvm-profdata, llvm-cov and python3 (any of -18 / -15 suffixed
llvm variants are also accepted).
- Line coverage, per file and in the TOTAL row, is corrected to count each
physical source line once (scripts/coverage_lines.py), rather than
llvm-cov's own per-function summary, which distorts C++ line counts. All
other columns, including branch coverage, are unmodified llvm-cov output.
See scripts/coverage_lines.py for why.
EOF
exit 0
}
PROFRAW_DIR="."
OUTPUT_FILE=""
ADDITIONAL_PROFILES=()
MERGE_ONLY=0
HTML_DIR=""
SHOW_UNCOVERED=0
BINARIES=()
SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
FRAMEWORK_SOURCE_DIRS=("${SOURCE_DIR}/src" "${SOURCE_DIR}/include")
while [[ $# -gt 0 ]]; do
case "$1" in
-d) PROFRAW_DIR="$2"; shift 2 ;;
-o) OUTPUT_FILE="$2"; shift 2 ;;
--add-profile) ADDITIONAL_PROFILES+=("$2"); shift 2 ;;
--merge-only) MERGE_ONLY=1; shift ;;
--html) HTML_DIR="$2"; shift 2 ;;
--show-uncovered) SHOW_UNCOVERED=1; shift ;;
-h|--help) usage ;;
--) shift; BINARIES+=("$@"); break ;;
-*) echo "Unknown option: $1" >&2; exit 1 ;;
*) BINARIES+=("$1"); shift ;;
esac
done
if [[ -z "${OUTPUT_FILE}" ]]; then
OUTPUT_FILE="${PROFRAW_DIR}/coverage.profdata"
fi
if [[ "${MERGE_ONLY}" -eq 1 ]] &&
[[ -n "${HTML_DIR}" || "${SHOW_UNCOVERED}" -eq 1 || ${#BINARIES[@]} -gt 0 ]]; then
echo "Error: --merge-only cannot be combined with binaries or report options." >&2
exit 1
fi
# ---------------------------------------------------------------------------
# Locate llvm tools, accepting versioned suffixes
# ---------------------------------------------------------------------------
find_tool() {
local tool="$1"
for candidate in "${tool}" "${tool}-18" "${tool}-15"; do
if command -v "${candidate}" &>/dev/null; then
printf '%s' "${candidate}"
return 0
fi
done
echo "Error: '${tool}' not found in PATH (also tried -18 and -15 suffixes)" >&2
return 1
}
LLVM_PROFDATA=$(find_tool llvm-profdata)
if [[ "${MERGE_ONLY}" -eq 0 ]]; then
LLVM_COV=$(find_tool llvm-cov)
fi
# ---------------------------------------------------------------------------
# Auto-discover binaries from coverage_binaries.txt if none given explicitly
# ---------------------------------------------------------------------------
if [[ "${MERGE_ONLY}" -eq 0 && ${#BINARIES[@]} -eq 0 ]]; then
BINARIES_FILE="${PROFRAW_DIR}/coverage_binaries.txt"
if [[ ! -f "${BINARIES_FILE}" ]]; then
echo "Error: no binaries specified and '${BINARIES_FILE}' not found." >&2
echo "Either pass binary paths as arguments or build with -DCOVERAGE=ON." >&2
exit 1
fi
mapfile -t BINARIES < "${BINARIES_FILE}"
# Drop blank lines (file(GENERATE) may add a trailing newline)
readarray -t BINARIES < <(printf '%s\n' "${BINARIES[@]}" | grep -v '^[[:space:]]*$')
fi
if [[ "${MERGE_ONLY}" -eq 0 && ${#BINARIES[@]} -eq 0 ]]; then
echo "Error: no instrumented binaries found." >&2
exit 1
fi
# ---------------------------------------------------------------------------
# Collect .profraw files
# ---------------------------------------------------------------------------
mapfile -t PROFRAW_FILES < <(find "${PROFRAW_DIR}" -name "*.profraw" -type f | sort)
if [[ ${#PROFRAW_FILES[@]} -eq 0 ]]; then
echo "No .profraw files found in '${PROFRAW_DIR}'." >&2
echo "Make sure tests were built and run with -DCOVERAGE=ON." >&2
exit 1
fi
echo "Found ${#PROFRAW_FILES[@]} .profraw file(s) in '${PROFRAW_DIR}'"
# ---------------------------------------------------------------------------
# Merge profile data
# ---------------------------------------------------------------------------
for profile in "${ADDITIONAL_PROFILES[@]}"; do
echo "Including additional profile '${profile}'"
done
echo "Merging coverage data into '${OUTPUT_FILE}'..."
"${LLVM_PROFDATA}" merge -sparse "${PROFRAW_FILES[@]}" "${ADDITIONAL_PROFILES[@]}" \
-o "${OUTPUT_FILE}"
if [[ "${MERGE_ONLY}" -eq 1 ]]; then
exit 0
fi
# ---------------------------------------------------------------------------
# Detect -ffile-prefix-map and compute -compilation-dir for llvm-cov.
# The build uses -ffile-prefix-map=<real>=<mapped> (e.g.
# /workspaces/CCF=CCF) to make binaries reproducible. This turns all
# recorded source paths into relative paths like CCF/include/... By
# default llvm-cov resolves these relative to the (also-mapped)
# compilation directory, producing non-existent doubled paths such as
# CCF/build/CCF/include/... Setting -compilation-dir to the parent of
# the real source tree lets llvm-cov resolve the mapped paths correctly:
# e.g. CCF/include/... relative to /workspaces -> /workspaces/CCF/include/...
# Path equivalence also handles source directories not named CCF.
# ---------------------------------------------------------------------------
COMPILATION_DIR=""
PATH_EQUIVALENCE=""
COMPILE_DB="${PROFRAW_DIR}/compile_commands.json"
if [[ -f "${COMPILE_DB}" ]]; then
prefix_map=$(grep -m1 -o '\-ffile-prefix-map=[^ "]*' "${COMPILE_DB}" | sed 's/-ffile-prefix-map=//' || true)
if [[ -n "${prefix_map}" ]]; then
real_path="${prefix_map%%=*}"
COMPILATION_DIR=$(dirname "${real_path}")
PATH_EQUIVALENCE="${COMPILATION_DIR}/${prefix_map#*=},${real_path}"
echo "Detected file-prefix-map, using compilation-dir: ${COMPILATION_DIR}"
fi
fi
# ---------------------------------------------------------------------------
# Build common llvm-cov argument list
# ---------------------------------------------------------------------------
build_cov_args() {
# First binary is the positional argument; additional binaries use -object
local args=("${BINARIES[0]}" "-instr-profile=${OUTPUT_FILE}")
for bin in "${BINARIES[@]:1}"; do
args+=("-object" "${bin}")
done
# Exclude third-party and test code: coverage should measure the framework
# being tested, not the tests exercising it.
args+=("-ignore-filename-regex=3rdparty/|/test/|/tests/|_test\.(cpp|h)$|/perf/")
# Override compilation directory so llvm-cov can resolve mapped source paths
if [[ -n "${COMPILATION_DIR}" ]]; then
args+=("-compilation-dir=${COMPILATION_DIR}")
args+=("-path-equivalence=${PATH_EQUIVALENCE}")
fi
args+=("--sources" "${FRAMEWORK_SOURCE_DIRS[@]}")
printf '%s\n' "${args[@]}"
}
mapfile -t COV_ARGS < <(build_cov_args)
# ---------------------------------------------------------------------------
# Overall coverage summary
#
# llvm-cov's own report computes line coverage per function then sums per
# file, which distorts C++ totals (see scripts/coverage_lines.py for why).
# The line count/missed/coverage of each per-file row and of the TOTAL row is
# patched below to count each physical source line once instead; all other
# columns are unmodified llvm-cov output.
# ---------------------------------------------------------------------------
REPORT_FILE=$(mktemp)
trap 'rm -f "${REPORT_FILE}"' EXIT
"${LLVM_COV}" report "${COV_ARGS[@]}" > "${REPORT_FILE}"
echo ""
echo "=== Coverage Summary ==="
"${LLVM_COV}" export -format=lcov "${COV_ARGS[@]}" |
python3 "${SOURCE_DIR}/scripts/coverage_lines.py" "${REPORT_FILE}"
# ---------------------------------------------------------------------------
# Uncovered lines detail
# ---------------------------------------------------------------------------
if [[ "${SHOW_UNCOVERED}" -eq 1 ]]; then
echo ""
echo "=== Uncovered Lines ==="
# llvm-cov show --format=text emits blocks like:
# <spaces><line_num>|<count>|<source>
# A file header line ends with a bare colon: "/path/to/file.cpp:"
# Lines whose count field is exactly 0 are unexecuted executable lines.
"${LLVM_COV}" show "${COV_ARGS[@]}" --format=text |
awk '
# Match a file header (ends with colon, no leading whitespace)
/^[^[:space:]].*:$/ {
current_file = substr($0, 1, length($0) - 1)
printed_header = 0
next
}
# Match lines with a zero execution count: " <N>| 0|<source>"
/^[[:space:]]+[0-9]+\|[[:space:]]+0\|/ {
if (!printed_header) {
print current_file ":"
printed_header = 1
}
# Strip leading whitespace and reformat as " <line>: <source>"
line = $0
sub(/^[[:space:]]+/, "", line)
n = split(line, parts, "|")
printf " %s: %s\n", parts[1], parts[3]
}
'
fi
# ---------------------------------------------------------------------------
# HTML report
# ---------------------------------------------------------------------------
if [[ -n "${HTML_DIR}" ]]; then
echo ""
echo "Generating HTML report in '${HTML_DIR}'..."
mkdir -p "${HTML_DIR}"
"${LLVM_COV}" show "${COV_ARGS[@]}" --format=html --output-dir="${HTML_DIR}"
echo "HTML report written to '${HTML_DIR}/index.html'"
echo "Note: the HTML index totals use llvm-cov's own per-function summary,"
echo "not the corrected line counts above. Each file's own source view is"
echo "unaffected, and already shows every line once."
fi
You can’t perform that action at this time.