Merge branch 'main' into jb/malloc_profiles · DataDog/java-profiler@d061e9a · GitHub
Skip to content

Commit d061e9a

Browse files
jbachorikjbachorik
authored
Merge branch 'main' into jb/malloc_profiles
2 parents bc3ec7d + b05a6ca commit d061e9a

69 files changed

Lines changed: 2491 additions & 414 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 2 deletions

‎.github/workflows/codecheck.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ jobs:
7676

7777
# Initializes the CodeQL tools for scanning.
7878
- name: Initialize CodeQL
79-
uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
79+
uses: github/codeql-action/init@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1
8080
with:
8181
languages: ${{ matrix.language }}
8282
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -85,4 +85,4 @@ jobs:
8585
# queries: ./path/to/local/query, your-org/your-repo/queries@main
8686
- run: ./gradlew -x test assembleReleaseJar
8787
- name: Perform CodeQL Analysis
88-
uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
88+
uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1

‎.github/workflows/release-validated.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
fetch-depth: 0
4444

4545
- name: Setup Java
46-
uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0
46+
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
4747
with:
4848
distribution: 'zulu'
4949
java-version: '21'
@@ -204,7 +204,7 @@ jobs:
204204
git checkout $GITHUB_REF_NAME
205205
206206
- name: Setup Java
207-
uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0
207+
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
208208
with:
209209
distribution: 'zulu'
210210
java-version: '21'

‎.gitlab/fuzzing/.gitlab-ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ fuzz_infra:
88
image: registry.ddbuild.io/images/docker:27.3.1
99
tags: ["arch:amd64"]
1010
stage: fuzz
11-
timeout: 30m
11+
timeout: 1h30m
1212
allow_failure: true
1313
id_tokens:
1414
DDSIGN_ID_TOKEN:

‎.gitlab/reliability/chaos_check.sh‎

Lines changed: 11 additions & 172 deletions
Original file line numberDiff line numberDiff line change
@@ -1,178 +1,17 @@
11
#!/usr/bin/env bash
2-
3-
set +e # Disable exit on error
2+
#
3+
# CI entry point for the chaos harness. Thin wrapper around
4+
# utils/run-chaos-harness.sh — this file only supplies the CI-specific
5+
# caching paths (so repeated scheduled runs on the same runner reuse the
6+
# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's
7+
# `artifacts:` block expects at the repo root. All the actual build/run logic
8+
# lives in the standalone script, which is also runnable by hand.
49

510
HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
611
ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )"
712

8-
RUNTIME=${1}
9-
CONFIG=${2:-profiler+tracer}
10-
ALLOCATOR=${3:-gmalloc}
11-
12-
echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}"
13-
14-
CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}"
15-
# CHAOS_JDK uses sdkman notation (<version>-<dist>); extract major for Adoptium API.
16-
JDK_MAJOR="${CHAOS_JDK%%.*}"
17-
JDK_ARCH=$(uname -m | sed 's/x86_64/x64/')
18-
JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}"
19-
20-
if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
21-
TMP=$(mktemp -d)
22-
DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse"
23-
echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..."
24-
if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then
25-
echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2
26-
rm -rf "${TMP}"
27-
exit 1
28-
fi
29-
mkdir -p "${JDK_INSTALL_DIR}"
30-
tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1
31-
rm -rf "${TMP}"
32-
fi
33-
34-
if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
35-
echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2
36-
exit 1
37-
fi
38-
export JAVA_HOME="${JDK_INSTALL_DIR}"
39-
export PATH="${JAVA_HOME}/bin:${PATH}"
40-
ACTIVE_JDK=$(java -version 2>&1 | head -1)
41-
# Check major version only — patch may differ from the Adoptium latest GA.
42-
if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then
43-
echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2
44-
exit 1
45-
fi
46-
47-
# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot.
48-
# Running mvn from /tmp avoids the empty pom.xml at the repo root.
49-
DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1)
50-
if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then
51-
DDPROF_JAR="${DDPROF_JAR_LOCAL}"
52-
echo "Using local ddprof jar: ${DDPROF_JAR}"
53-
else
54-
if [ -z "${CURRENT_VERSION:-}" ]; then
55-
echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2
56-
exit 1
57-
fi
58-
echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots"
59-
(cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \
60-
-DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \
61-
-Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION})
62-
DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar"
63-
fi
64-
65-
if [ ! -f "${DDPROF_JAR}" ]; then
66-
echo "FAIL:ddprof jar unavailable" >&2
67-
exit 1
68-
fi
69-
70-
mkdir -p /var/lib/datadog
71-
wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer'
72-
73-
# chaos.jar is produced once per pipeline by the chaos:build job (stresstest
74-
# stage) and pulled here as an artifact. Fall back to an inline build if the
75-
# artifact is absent (e.g. local repro outside CI).
76-
CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar"
77-
if [ ! -f "${CHAOS_JAR}" ]; then
78-
echo "chaos.jar artifact not present — building inline"
79-
( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q )
80-
fi
81-
82-
if [ ! -f "${CHAOS_JAR}" ]; then
83-
echo "FAIL:chaos.jar unavailable" >&2
84-
exit 1
85-
fi
86-
87-
# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's
88-
# (relocated) profiler classes match the version under test.
89-
DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \
90-
DDPROF_JAR=${DDPROF_JAR} \
91-
OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \
92-
"${ROOT}/utils/patch-dd-java-agent.sh"
93-
94-
if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then
95-
echo "FAIL:dd-java-agent patching failed" >&2
96-
exit 1
97-
fi
98-
99-
PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar
100-
101-
case $CONFIG in
102-
profiler)
103-
echo "Running with profiler only"
104-
ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false"
105-
# @Trace is a no-op without the tracer, so trace-context is excluded here.
106-
ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
107-
;;
108-
profiler+tracer)
109-
echo "Running with profiler and tracer"
110-
ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true"
111-
ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
112-
;;
113-
*)
114-
echo "Unknown configuration: $CONFIG"
115-
exit 1
116-
;;
117-
esac
118-
119-
case $ALLOCATOR in
120-
gmalloc)
121-
echo "Running with gmalloc"
122-
;;
123-
tcmalloc)
124-
echo "Running with tcmalloc"
125-
export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4')
126-
# thread-churn/dump-storm antagonists cycle many short-lived threads;
127-
# tcmalloc's defaults are slow to return their per-thread caches to the
128-
# OS, which was inflating container RSS past the OOM limit on aarch64.
129-
export TCMALLOC_RELEASE_RATE=10
130-
export TCMALLOC_AGGRESSIVE_DECOMMIT=1
131-
;;
132-
jemalloc)
133-
echo "Running with jemalloc"
134-
export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so')
135-
;;
136-
*)
137-
echo "Unknown allocator: $ALLOCATOR"
138-
echo "Valid values are: gmalloc, tcmalloc, jemalloc"
139-
exit 1
140-
;;
141-
esac
142-
143-
echo "LD_PRELOAD=$LD_PRELOAD"
144-
145-
timeout "$((RUNTIME + 300))" \
146-
java -javaagent:${PATCHED_AGENT} \
147-
${ENABLEMENT} \
148-
-Ddd.profiling.upload.period=10 \
149-
-Ddd.profiling.start-force-first=true \
150-
-Ddd.profiling.ddprof.liveheap.enabled=true \
151-
-Ddd.profiling.ddprof.alloc.enabled=true \
152-
-Ddd.profiling.ddprof.wall.enabled=true \
153-
-Ddd.profiling.ddprof.nativemem.enabled=true \
154-
-Ddd.env=java-profiler-stability \
155-
-Ddd.service=java-profiler-chaos \
156-
-Xmx2g -Xms2g \
157-
-XX:MaxMetaspaceSize=384m \
158-
-XX:ErrorFile=${HERE}/../../hs_err.log \
159-
-XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \
160-
-jar ${CHAOS_JAR} \
161-
--duration ${RUNTIME}s \
162-
--antagonists ${ANTAGONISTS}
163-
164-
RC=$?
165-
echo "RC=$RC"
13+
export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}"
14+
export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}"
15+
export CHAOS_ERROR_FILE="${ROOT}/hs_err.log"
16616

167-
if [ $RC -ne 0 ]; then
168-
CRASH_MSG="Chaos harness crashed (RC=${RC})"
169-
HS_ERR="${HERE}/../../hs_err.log"
170-
if [ -f "${HS_ERR}" ]; then
171-
SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120)
172-
FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120)
173-
[ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}"
174-
[ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}"
175-
fi
176-
echo "FAIL:${CRASH_MSG}" >&2
177-
exit 1
178-
fi
17+
exec "${ROOT}/utils/run-chaos-harness.sh" "$@"

‎build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,21 @@ object ConfigurationPresets {
3737
project.logger.lifecycle("Setting up standard build configurations for $currentPlatform-$currentArch")
3838
project.logger.lifecycle("Using compiler: $compiler")
3939

40+
// Opt-in compile-time fault injection. When -PenableFaultInjection is
41+
// passed we append -D__FAULT_INJECTION__ to the standard release/debug
42+
// library builds, so the documented buildRelease / buildDebug workflows
43+
// produce a fault-injected libjavaProfiler.so. It is intentionally NOT
44+
// applied to asan/tsan/fuzzer: those configs install their own SIGSEGV
45+
// interception, which conflicts with the deliberately-faulting loads.
46+
val faultInjection = project.hasProperty("enableFaultInjection")
4047
extension.buildConfigurations.apply {
4148
register("release") {
4249
configureRelease(this, currentPlatform, currentArch, version)
50+
if (faultInjection) compilerArgs.add("-D__FAULT_INJECTION__")
4351
}
4452
register("debug") {
4553
configureDebug(this, currentPlatform, currentArch, version)
54+
if (faultInjection) compilerArgs.add("-D__FAULT_INJECTION__")
4655
}
4756
register("asan") {
4857
configureAsan(this, currentPlatform, currentArch, version, rootDir, compiler)

‎build-logic/conventions/src/main/kotlin/com/datadoghq/native/fuzz/FuzzTargetsPlugin.kt‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,26 @@ class FuzzTargetsPlugin : Plugin<Project> {
108108
val crashDir = extension.crashDir.get().asFile
109109
val duration = extension.duration.get()
110110

111+
// Profiler sources are identical for every fuzz target (same compiler args and
112+
// includes), so compile them once and share the resulting objects across targets
113+
// instead of recompiling the whole profiler per target.
114+
val sharedObjDir = project.file("${project.layout.buildDirectory.get()}/obj/fuzz/_profiler")
115+
val compileProfilerTask = project.tasks.register("compileFuzzProfilerSources", NativeCompileTask::class.java) {
116+
onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") }
117+
group = "build"
118+
description = "Compile the profiler sources shared by all fuzz targets"
119+
120+
this.compiler.set(compiler)
121+
this.compilerArgs.set(compilerArgs)
122+
sources.from(
123+
extension.profilerSourceDir.map { dir ->
124+
project.fileTree(dir) { include("**/*.cpp") }
125+
}
126+
)
127+
includes.from(includeFiles)
128+
objectFileDir.set(sharedObjDir)
129+
}
130+
111131
// Discover and create tasks for each fuzz target
112132
if (fuzzSourceDir.exists()) {
113133
fuzzSourceDir.listFiles()?.filter { file -> file.name.endsWith(".cpp") }?.forEach { fuzzFile ->
@@ -119,34 +139,33 @@ class FuzzTargetsPlugin : Plugin<Project> {
119139
val binary = project.file("$binDir/$fuzzName")
120140
val targetCorpusDir = File(corpusBaseDir, fuzzName)
121141

122-
// Compile task
142+
// Compile task - only compiles this target's own fuzz driver; profiler
143+
// sources come from the shared compileFuzzProfilerSources task.
123144
val compileTask = project.tasks.register("compileFuzz_$fuzzName", NativeCompileTask::class.java) {
124145
onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") }
125146
group = "build"
126147
description = "Compile the fuzz target $fuzzName"
127148

128149
this.compiler.set(compiler)
129150
this.compilerArgs.set(compilerArgs)
130-
sources.from(
131-
extension.profilerSourceDir.map { dir ->
132-
project.fileTree(dir) { include("**/*.cpp") }
133-
},
134-
fuzzFile
135-
)
151+
sources.from(fuzzFile)
136152
includes.from(includeFiles)
137153
objectFileDir.set(objDir)
138154
}
139155

140156
// Link task
141157
val linkTask = project.tasks.register("linkFuzz_$fuzzName", NativeLinkExecutableTask::class.java) {
142158
onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") }
143-
dependsOn(compileTask)
159+
dependsOn(compileProfilerTask, compileTask)
144160
group = "build"
145161
description = "Link the fuzz target $fuzzName"
146162

147163
linker.set(compiler)
148164
this.linkerArgs.set(linkerArgs)
149-
objectFiles.from(project.fileTree(objDir) { include("*.o") })
165+
objectFiles.from(
166+
project.fileTree(sharedObjDir) { include("*.o") },
167+
project.fileTree(objDir) { include("*.o") }
168+
)
150169
outputFile.set(binary)
151170
}
152171

‎build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,16 @@ class GtestTaskBuilder(
257257
// Mark unit-test builds so test-only production APIs are compiled in.
258258
args.add("-DUNIT_TEST")
259259

260+
// NOTE: -D__FAULT_INJECTION__ is intentionally NOT added here. It is
261+
// applied per-config in ConfigurationPresets (release/debug only) and we
262+
// start from config.compilerArgs.get() above, so the release/debug gtest
263+
// builds inherit it automatically under -PenableFaultInjection while the
264+
// asan/tsan/fuzzer configs correctly never receive it. Re-deriving it
265+
// from the project property here would be redundant for release/debug
266+
// (they already carry it) and would wrongly leak the deliberately-
267+
// faulting loads into the sanitizer configs, which install their own
268+
// SIGSEGV interception.
269+
260270
return args
261271
}
262272

‎build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt‎

Lines changed: 17 additions & 0 deletions

0 commit comments

Comments
 (0)