{{ message }}
Repository navigation
Expand file tree
/
Copy pathall.bash
More file actions
executable file
·394 lines (349 loc) · 11 KB
/
Copy pathall.bash
File metadata and controls
executable file
·394 lines (349 loc) · 11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
#!/usr/bin/env bash
# Copyright 2019 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
source devtools/lib.sh || { echo "Are you at repo root?"; exit 1; }
GO=go
# Support ** in globs, for check_script_hashes.
shopt -s globstar
warnout() {
while read line; do
warn "$line"
done
}
# filter FILES GLOB1 GLOB2 ...
# returns the files in FILES that match any of the glob patterns.
filter() {
local infiles=$1
shift
local patterns=$*
local outfiles=
for pat in $patterns; do
for f in $infiles; do
if [[ $f == $pat ]]; then
outfiles="$outfiles $f"
fi
done
done
echo $outfiles
}
# Return the files that are modified or added.
# If there are such files in the working directory, whether or not
# they are staged for commit, use those.
# Otherwise, use the files changed since the previous commit.
modified_files() {
local working=$(working_files)
if [[ $working != ' ' ]]; then
echo $working
elif [[ $(git rev-parse HEAD) = $(git rev-parse master) ]]; then
echo ""
else
diff_files HEAD^
fi
}
# Return the files in the working directory that have not been added to the commit.
working_files() {
echo "$(diff_files '') $(diff_files --cached)"
}
# Helper for modified_files. It asks git for all modified, added or deleted
# files, and keeps only the latter two.
diff_files() {
git diff --name-status $* | awk '$1 ~ /^R/ { print $3; next } $1 != "D" { print $2 }'
}
# codedirs lists directories that contain discovery code. If they include
# directories containing external code, those directories must be excluded in
# findcode below.
codedirs=(
"cmd"
"internal"
"migrations"
"static"
)
# verify_header checks that all given files contain the standard header for Go
# projects.
verify_header() {
if [[ "$@" != "" ]]; then
for FILE in $@
do
# Allow for the copyright header to start on either of the first three
# lines, to accommodate conventions for CSS and HTML.
line="$(head -4 $FILE)"
if [[ ! $line == *"The Go Authors. All rights reserved."* ]] &&
[[ ! $line == "// DO NOT EDIT. This file was copied from" ]]; then
err "missing license header: $FILE"
fi
done
fi
}
# findcode finds source files in the repo, skipping third-party source.
findcode() {
find ${codedirs[@]} \
-not -path '*/third_party/*' \
\( -name '*.go' -o -name '*.sql' -o -name '*.tmpl' -o -name '*.css' -o -name '*.js' -o -name '*.ts' \)
}
# check_headers checks that all source files that have been staged in this
# commit, and all other non-third-party files in the repo, have a license
# header.
check_headers() {
if [[ $# -gt 0 ]]; then
info "Checking listed files for license header"
verify_header $*
else
info "Checking staged files for license header"
# Check code files that have been modified or added.
verify_header $(git diff --cached --name-status | grep -vE "^D" | cut -f 2- | grep -E ".go$|.sql$|.sh$")
info "Checking internal files for license header"
verify_header $(findcode)
fi
}
# bad_migrations outputs migrations with bad sequence numbers.
bad_migrations() {
ls migrations | cut -d _ -f 1 | sort | uniq -c | grep -vE '^\s+2 '
}
# check_bad_migrations looks for sql migration files with bad sequence numbers,
# possibly resulting from a bad merge.
check_bad_migrations() {
info "Checking for bad migrations"
bad_migrations | while read line
do
err "unexpected number of migrations: $line"
done
}
# check_unparam runs unparam on source files.
check_unparam() {
echo "unparam temporarily skipped: failing at tip"
#runcmd $GO tool unparam ./...
}
# check_vet runs go vet on source files.
check_vet() {
runcmd $GO vet -all ./...
}
# check_staticcheck runs staticcheck on source files.
check_staticcheck() {
echo "staticcheck temporarily skipped: failing at tip"
#runcmd $GO tool staticcheck $(go list ./... | grep -v third_party | grep -v internal/doc | grep -v internal/render)
}
# check_misspell runs misspell on source files.
check_misspell() {
runcmd $GO tool misspell cmd/**/*.{go,sh} internal/**/* README.md
}
# check_templates runs go-template-lint on template files. Unfortunately it
# doesn't handler the /helpers/ fileglob correctly, so it is too noisy to be
# included in standard checks.
check_templates() {
runcmd $GO tool go-template-lint \
-f=internal/frontend/server.go \
-t=internal/frontend/server.go \
-td=static | warnout
}
script_hash_glob='static/**/*.tmpl'
# check_script_hashes checks that our CSP hashes match the ones
# for our HTML scripts.
check_script_hashes() {
runcmd $GO run ./devtools/cmd/csphash $script_hash_glob
}
# run_build_static builds JavaScript output from TypeScript source files.
run_build_static() {
runcmd $GO run ./devtools/cmd/static
files=$(working_files)
if [[ $(filter "$files" 'static/**/*.min*') != '' ]]; then
err "minimized CSS files are not consistent with unminimized ones; run ./devtools/cmd/static to regenerate them"
fi
}
run_npm() {
npmcmd=${GO_DISCOVERY_NPM_CMD:-"./devtools/nodejs.sh npm"}
# Run npm install if node_modules directory does not exist.
if [ ! -d "node_modules" ]; then
runcmd $npmcmd install --quiet
fi
runcmd $npmcmd $@
}
run_npx() {
npxcmd=${GO_DISCOVERY_NPX_CMD:-"./devtools/nodejs.sh npx"}
# Run npm install if node_modules directory does not exist.
if [ ! -d "node_modules" ]; then
run_npm install --quiet
fi
runcmd $npxcmd $@
}
prettier_file_globs='**/*.md'
# run_prettier runs prettier on CSS, JS, and MD files. Uses globally
# installed prettier if available or a dockerized installation as a
# fallback.
run_prettier() {
local files=$*
if [[ $files = '' ]]; then
files=$prettier_file_globs
fi
run_npx prettier --write $files
}
go_linters() {
check_vet
check_staticcheck
check_misspell
check_unparam
}
standard_linters() {
run_build_static
check_headers
check_bad_migrations
go_linters
check_script_hashes
}
usage() {
cat <<EOUSAGE
Usage: $0 [subcommand]
Available subcommands:
help - display this help message
(empty) - run all standard checks and tests
ci - run checks and tests suitable for continuous integration
test - (ci) run tests with race detector and test database
cl - run checks and tests on the current CL, suitable for a commit or pre-push hook
lint - run all standard linters below:
headers - (lint) check source files for the license disclaimer
migrations - (lint) check migration sequence numbers
misspell - (lint) run misspell on source files
npm - run npm commands or scripts from package.json
npx - run a command from a local or remote npm package
script_hashes - (lint) check script hashes
script_output - (lint) check script output
staticcheck - (lint) run staticcheck on source files
unparam - (lint) run unparam on source files
prettier - (lint, nonstandard) run prettier on .js and .css files.
templates - (lint, nonstandard) run go-template-lint on templates
vet - (lint) run vet on source files
EOUSAGE
}
# Packages to run without the race detector on CI.
# (They time out with -race.)
declare -A no_race
no_race=(
[golang.org/x/pkgsite/internal/frontend]=1
[golang.org/x/pkgsite/internal/worker]=1
[golang.org/x/pkgsite/internal/testing/integration]=1
)
# ci_tests runs the Go test suite for continuous integration.
#
# Note: when invoked in CI via TestIntegration ("allbash test"), TestLint in
# integration_test.go will be skipped by "go test" because GO_DISCOVERY_TESTDB
# is set. (TestLint already ran linters on the host runner before TestIntegration.)
ci_tests() {
# Explicitly mark the working directory as safe in CI.
# https://github.com/docker-library/golang/issues/452
local wd=$(pwd)
runcmd git config --system --add safe.directory ${wd}
local pkgs=$*
if [[ $pkgs == '' ]]; then
pkgs=$($GO list ./... | grep -v node_modules)
fi
for pkg in $pkgs; do
if [[ ${no_race[$pkg]} = '' ]]; then
race="$race $pkg"
else
norace="$norace $pkg"
fi
done
if [[ $race != '' ]]; then
runcmd env GO_DISCOVERY_TESTDB=true $GO test -race -count=1 $race
fi
if [[ $norace != '' ]]; then
runcmd env GO_DISCOVERY_TESTDB=true $GO test -count=1 $norace
fi
}
main() {
case "$1" in
"-h" | "--help" | "help")
usage
exit 0
;;
"")
standard_linters
run_prettier
run_npm run lint -- --fix
run_npm run test
runcmd $GO mod tidy
runcmd env GO_DISCOVERY_TESTDB=true go test ./...
runcmd $GO test ./internal/secrets
run_npm audit
;;
cl)
# Similar to the above, but only run checks that apply to files in this commit.
files=$(modified_files)
if [[ $files = '' ]]; then
info "No modified files; nothing to do."
exit 0
fi
info "Running checks on:"
info " " $files
if [[ $(filter "$files" $script_hash_glob) != '' ]]; then
check_script_hashes
run_build_static
fi
check_headers $(filter "$files" '*.go' '*.sql' '*.sh')
if [[ $(filter "$files" 'migrations/*') != '' ]]; then
check_bad_migrations
fi
if [[ $(filter "$files" '*.go') != '' ]]; then
go_linters
fi
pfiles=$(filter "$files" $prettier_file_globs)
if [[ $pfiles != '' ]]; then
run_prettier $pfiles
fi
if [[ $(filter "$files" 'static/**') != '' ]]; then
run_npm run lint -- --fix
run_npm run test
fi
runcmd $GO mod tidy
runcmd env GO_DISCOVERY_TESTDB=true go test ./...
runcmd $GO test ./internal/secrets
;;
ci)
# Similar to the no-arg mode, but omit actions that require GCP
# permissions or that don't test the code.
# Also, run the race detector on most tests.
local start=`date +%s`
standard_linters
# Print how long it takes to download dependencies and run the standard
# linters in CI.
local end=`date +%s`
echo
echo "--------------------"
echo "DONE: $((end-start)) seconds"
echo "--------------------"
shift
ci_tests $*
;;
test)
# Run only the Go tests for continuous integration (without linters).
# Used by TestIntegration in Docker, where linters have already been run
# on the host by TestLint (which skips itself when GO_DISCOVERY_TESTDB is set).
shift
ci_tests $*
;;
lint) standard_linters ;;
headers) check_headers ;;
migrations) check_bad_migrations ;;
misspell) check_misspell ;;
staticcheck) check_staticcheck ;;
prettier)
shift
run_prettier $*
;;
templates) check_templates ;;
unparam) check_unparam ;;
vet) check_vet ;;
script_hashes) check_script_hashes ;;
build_static) run_build_static ;;
npm) run_npm ${@:2} ;;
npx) run_npx ${@:2} ;;
*)
usage
exit 1
esac
if [[ $EXIT_CODE != 0 ]]; then
err "FAILED; see errors above"
fi
exit $EXIT_CODE
}
main $@
You can’t perform that action at this time.