Merge branch 'master' into johnjing/subcolumn-aggregate-tuple · ClickHouse/ClickHouse@fe61ffb · GitHub
Skip to content

Commit fe61ffb

Browse files
nihalzpnihalzp
committed
Merge branch 'master' into johnjing/subcolumn-aggregate-tuple
2 parents 7570ee6 + e6a9b21 commit fe61ffb

561 files changed

Lines changed: 40718 additions & 19204 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 8 additions & 4 deletions

‎ci/defs/job_configs.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,7 @@ class JobConfigs:
489489
],
490490
),
491491
timeout=900,
492+
post_hooks=["python3 ./ci/jobs/scripts/job_hooks/docker_clean_up_hook.py"],
492493
).parametrize(
493494
Job.ParamSet(
494495
parameter="amd_release",
@@ -1378,6 +1379,7 @@ class JobConfigs:
13781379
include_paths=[
13791380
"./docs",
13801381
"./ci/jobs/docs_job_mintlify.py",
1382+
"./ci/jobs/scripts/docs",
13811383
],
13821384
# Exclude everything currently in ./docs so that this job runs only
13831385
# on files that are NOT part of the legacy docs tree (i.e. the new

‎ci/jobs/ast_fuzzer_job.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,9 +307,24 @@ def run_fuzz_job(check_name: str):
307307
sanitizer_oom = Shell.get_output(
308308
f"rg --text 'Sanitizer:? (out-of-memory|out of memory|failed to allocate)|Child process was terminated by signal 9' {server_log}"
309309
)
310-
if sanitizer_oom:
310+
# Sanitizer shadow memory is invisible to the server's memory tracker,
311+
# so the kernel OOM killer may SIGKILL the server before any limit
312+
# fires. It may also kill the watchdog, losing the "terminated by
313+
# signal 9" message in the server log. A SIGKILLed server (exit 137)
314+
# with no sanitizer report is an OOM, not a bug.
315+
has_sanitizer_report = any(WORKSPACE_PATH.glob("sanitizer.log.*"))
316+
kernel_oom_kill = (
317+
server_died and server_exit_code == 137 and not has_sanitizer_report
318+
)
319+
if sanitizer_oom or kernel_oom_kill:
311320
print("Sanitizer OOM")
312-
info.append("WARNING: Sanitizer OOM - test considered passed")
321+
if sanitizer_oom:
322+
info.append("WARNING: Sanitizer OOM - test considered passed")
323+
else:
324+
info.append(
325+
"WARNING: Server was killed by the kernel OOM killer "
326+
"(sanitizer build) - test considered passed"
327+
)
313328
status = Result.Status.OK
314329
is_failed = False
315330
else:

‎ci/jobs/buzzhouse_job.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ def main():
166166

167167
allow_hardcoded_inserts = random.choice([True, False])
168168
min_nested_rows = random.randint(0, 5)
169-
max_nested_rows = min_nested_rows + (5 if allow_hardcoded_inserts else 100)
169+
max_nested_rows = min_nested_rows + (5 if allow_hardcoded_inserts else 30)
170170
min_insert_rows = random.randint(1, 100)
171171
max_insert_rows = min_insert_rows + (10 if allow_hardcoded_inserts else 3000)
172172
min_string_length = random.randint(0, 100)
@@ -270,7 +270,6 @@ def main():
270270
"hot_table_settings": [
271271
"allow_coalescing_columns_in_partition_or_order_key",
272272
# "allow_experimental_replacing_merge_with_cleanup",
273-
"allow_experimental_reverse_key",
274273
"allow_floating_point_partition_key",
275274
"allow_nullable_key",
276275
"allow_summing_columns_in_partition_or_order_key",

‎ci/jobs/docs_job_mintlify.py‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,42 @@
1-
import os
2-
1+
from ci.jobs.scripts.docs.check_readonly_copies import check_readonly_copies
2+
from ci.jobs.scripts.docs.mintlify_docs_check import DEFAULT_CHECKS
3+
from ci.praktika.info import Info
34
from ci.praktika.result import Result
4-
from ci.praktika.utils import Shell, Utils
5+
from ci.praktika.utils import Utils
56

6-
if __name__ == "__main__":
77

8-
results = []
9-
stop_watch = Utils.Stopwatch()
10-
temp_dir = f"{Utils.cwd()}/ci/tmp/"
8+
def _readonly_copies_guard():
9+
# One-way sync: fail edits to docs folders whose canonical source lives in
10+
# another repo (declared in ci/jobs/scripts/docs/readonly_copies.json). This
11+
# is aggregator-only -- the consuming repos are the source of truth, so this
12+
# is deliberately not part of the shared DEFAULT_CHECKS.
13+
changed_files = Info().get_changed_files()
14+
if changed_files is None:
15+
# Fail close: without the changed-file list we cannot prove the PR does
16+
# not touch read-only copies, so do not report success.
17+
print("Error: the changed-file list is unavailable, cannot run the check.")
18+
return False
19+
return check_readonly_copies(changed_files)
20+
21+
22+
if __name__ == "__main__":
1123

1224
docs_dir = f"{Utils.cwd()}/docs"
1325

14-
results.append(
15-
Result.from_commands_run(
16-
name="Verify Mintlify docs.json file is valid",
17-
command=[
18-
"mint validate",
19-
],
20-
workdir=docs_dir,
21-
)
22-
)
26+
# The mint check definitions are shared with the standalone driver
27+
# (ci/jobs/scripts/docs/mintlify_docs_check.py). This job already runs inside
28+
# the docs-builder image with the docs present natively, so it runs them
29+
# directly; add new checks to DEFAULT_CHECKS, not here.
30+
results = [
31+
Result.from_commands_run(name=name, command=command, workdir=docs_dir)
32+
for name, command in DEFAULT_CHECKS
33+
]
2334

2435
results.append(
2536
Result.from_commands_run(
26-
name="Check for broken links",
27-
command=[
28-
"mint broken-links",
29-
],
30-
workdir=docs_dir,
37+
name="No direct edits to read-only copied docs",
38+
command=_readonly_copies_guard,
3139
)
3240
)
3341

34-
Result.create_from(results=results).complete_job()
42+
Result.create_from(results=results).complete_job()

‎ci/jobs/jepsen_check.py‎

Lines changed: 66 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -178,58 +178,81 @@ def main():
178178
logging.info("Not jepsen test label in labels list, skipping")
179179
sys.exit(0)
180180

181+
image_name = KEEPER_IMAGE_NAME if args.program == "keeper" else SERVER_IMAGE_NAME
182+
# Pull by the immutable praktika digest tag instead of mutable ":latest".
183+
# Jepsen runs docker directly, so use the architecture-specific tag built by praktika.
184+
docker_tag = info.docker_tag(image_name)
185+
if docker_tag:
186+
if Utils.is_arm():
187+
docker_tag += "_arm"
188+
elif Utils.is_amd():
189+
docker_tag += "_amd"
190+
else:
191+
raise RuntimeError("Unsupported CPU architecture")
192+
docker_image = f"{image_name}:{docker_tag}"
193+
else:
194+
if not info.is_local_run:
195+
raise RuntimeError(
196+
f"No praktika digest tag for image [{image_name}] "
197+
f"in workflow [{info.workflow_name}]"
198+
)
199+
logging.warning(
200+
"No praktika digest tag for %s in local run; falling back to mutable ':latest' "
201+
"(may be served stale through the pull-through cache)",
202+
image_name,
203+
)
204+
docker_image = image_name
205+
181206
temp_path.mkdir(parents=True, exist_ok=True)
182207

183208
result_path = temp_path / "result_path"
184209
result_path.mkdir(parents=True, exist_ok=True)
185210

186-
instances = prepare_autoscaling_group_and_get_hostnames(
187-
KEEPER_DESIRED_INSTANCE_COUNT
188-
if args.program == "keeper"
189-
else SERVER_DESIRED_INSTANCE_COUNT
190-
)
191-
nodes_path = save_nodes_to_file(
192-
instances[:KEEPER_DESIRED_INSTANCE_COUNT], temp_path
193-
)
194-
195-
# always use latest
196-
docker_image = KEEPER_IMAGE_NAME if args.program == "keeper" else SERVER_IMAGE_NAME
211+
try:
212+
instances = prepare_autoscaling_group_and_get_hostnames(
213+
KEEPER_DESIRED_INSTANCE_COUNT
214+
if args.program == "keeper"
215+
else SERVER_DESIRED_INSTANCE_COUNT
216+
)
217+
nodes_path = save_nodes_to_file(
218+
instances[:KEEPER_DESIRED_INSTANCE_COUNT], temp_path
219+
)
197220

198-
# build amd_binary assumed to be always ready on the master as it's part of the merge queue workflow
199-
urls = read_build_urls(temp_path, "amd_binary")
200-
build_url = None
201-
for url in urls:
202-
if url.endswith("clickhouse"):
203-
build_url = url
204-
assert build_url, "No build url found in the report"
221+
# build amd_binary assumed to be always ready on the master as it's part of the merge queue workflow
222+
urls = read_build_urls(temp_path, "amd_binary")
223+
build_url = None
224+
for url in urls:
225+
if url.endswith("clickhouse"):
226+
build_url = url
227+
assert build_url, "No build url found in the report"
205228

206-
extra_args = ""
207-
if args.program == "server":
208-
extra_args = f"-e KEEPER_NODE={instances[-1]}"
229+
extra_args = ""
230+
if args.program == "server":
231+
extra_args = f"-e KEEPER_NODE={instances[-1]}"
209232

210-
ssh_key_secret = Secret.Config(
211-
name="jepsen_ssh_key", type=Secret.Type.AWS_SSM_PARAMETER
212-
)
213-
with SSHKey(key_value=ssh_key_secret.get_value() + "\n"):
214-
ssh_auth_sock = os.environ["SSH_AUTH_SOCK"]
215-
auth_sock_dir = os.path.dirname(ssh_auth_sock)
216-
cmd = get_run_command(
217-
ssh_auth_sock,
218-
auth_sock_dir,
219-
info,
220-
nodes_path,
221-
Utils.cwd(),
222-
build_url,
223-
result_path,
224-
extra_args,
225-
docker_image,
233+
ssh_key_secret = Secret.Config(
234+
name="jepsen_ssh_key", type=Secret.Type.AWS_SSM_PARAMETER
226235
)
227-
logging.info("Going to run jepsen: %s", cmd)
228-
229-
if Shell.run(cmd) != 0:
230-
logging.error("Jepsen run command failed")
231-
232-
clear_autoscaling_group()
236+
with SSHKey(key_value=ssh_key_secret.get_value() + "\n"):
237+
ssh_auth_sock = os.environ["SSH_AUTH_SOCK"]
238+
auth_sock_dir = os.path.dirname(ssh_auth_sock)
239+
cmd = get_run_command(
240+
ssh_auth_sock,
241+
auth_sock_dir,
242+
info,
243+
nodes_path,
244+
Utils.cwd(),
245+
build_url,
246+
result_path,
247+
extra_args,
248+
docker_image,
249+
)
250+
logging.info("Going to run jepsen: %s", cmd)
251+
252+
if Shell.run(cmd) != 0:
253+
logging.error("Jepsen run command failed")
254+
finally:
255+
clear_autoscaling_group()
233256

234257
status = Result.Status.OK
235258
description = "No invalid analysis found ヽ(‘ー`)ノ"

‎ci/jobs/scripts/docs/check_readonly_copies.py‎

Lines changed: 107 additions & 0 deletions

0 commit comments

Comments
 (0)