DataDog · jbachorik · Jul 16, 2026 · Jul 16, 2026 · Jul 16, 2026 · Jul 16, 2026
@@ -1,178 +1,17 @@
 #!/usr/bin/env bash
-
-set +e # Disable exit on error
+#
+# CI entry point for the chaos harness. Thin wrapper around
+# utils/run-chaos-harness.sh — this file only supplies the CI-specific
+# caching paths (so repeated scheduled runs on the same runner reuse the
+# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's
+# `artifacts:` block expects at the repo root. All the actual build/run logic
+# lives in the standalone script, which is also runnable by hand.
 
 HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )"
 
-RUNTIME=${1}
-CONFIG=${2:-profiler+tracer}
-ALLOCATOR=${3:-gmalloc}
-
-echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}"
-
-CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}"
-# CHAOS_JDK uses sdkman notation (<version>-<dist>); extract major for Adoptium API.
-JDK_MAJOR="${CHAOS_JDK%%.*}"
-JDK_ARCH=$(uname -m | sed 's/x86_64/x64/')
-JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}"
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  TMP=$(mktemp -d)
-  DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse"
-  echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..."
-  if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then
-    echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2
-    rm -rf "${TMP}"
-    exit 1
-  fi
-  mkdir -p "${JDK_INSTALL_DIR}"
-  tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1
-  rm -rf "${TMP}"
-fi
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2
-  exit 1
-fi
-export JAVA_HOME="${JDK_INSTALL_DIR}"
-export PATH="${JAVA_HOME}/bin:${PATH}"
-ACTIVE_JDK=$(java -version 2>&1 | head -1)
-# Check major version only — patch may differ from the Adoptium latest GA.
-if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then
-  echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2
-  exit 1
-fi
-
-# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot.
-# Running mvn from /tmp avoids the empty pom.xml at the repo root.
-DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1)
-if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then
-  DDPROF_JAR="${DDPROF_JAR_LOCAL}"
-  echo "Using local ddprof jar: ${DDPROF_JAR}"
-else
-  if [ -z "${CURRENT_VERSION:-}" ]; then
-    echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2
-    exit 1
-  fi
-  echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots"
-  (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \
-      -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \
-      -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION})
-  DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar"
-fi
-
-if [ ! -f "${DDPROF_JAR}" ]; then
-  echo "FAIL:ddprof jar unavailable" >&2
-  exit 1
-fi
-
-mkdir -p /var/lib/datadog
-wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer'
-
-# chaos.jar is produced once per pipeline by the chaos:build job (stresstest
-# stage) and pulled here as an artifact. Fall back to an inline build if the
-# artifact is absent (e.g. local repro outside CI).
-CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar"
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "chaos.jar artifact not present — building inline"
-  ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q )
-fi
-
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "FAIL:chaos.jar unavailable" >&2
-  exit 1
-fi
-
-# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's
-# (relocated) profiler classes match the version under test.
-DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \
-DDPROF_JAR=${DDPROF_JAR} \
-OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \
-"${ROOT}/utils/patch-dd-java-agent.sh"
-
-if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then
-  echo "FAIL:dd-java-agent patching failed" >&2
-  exit 1
-fi
-
-PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar
-
-case $CONFIG in
-  profiler)
-    echo "Running with profiler only"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false"
-    # @Trace is a no-op without the tracer, so trace-context is excluded here.
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  profiler+tracer)
-    echo "Running with profiler and tracer"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true"
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  *)
-    echo "Unknown configuration: $CONFIG"
-    exit 1
-    ;;
-esac
-
-case $ALLOCATOR in
-  gmalloc)
-    echo "Running with gmalloc"
-    ;;
-  tcmalloc)
-    echo "Running with tcmalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4')
-    # thread-churn/dump-storm antagonists cycle many short-lived threads;
-    # tcmalloc's defaults are slow to return their per-thread caches to the
-    # OS, which was inflating container RSS past the OOM limit on aarch64.
-    export TCMALLOC_RELEASE_RATE=10
-    export TCMALLOC_AGGRESSIVE_DECOMMIT=1
-    ;;
-  jemalloc)
-    echo "Running with jemalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so')
-    ;;
-  *)
-    echo "Unknown allocator: $ALLOCATOR"
-    echo "Valid values are: gmalloc, tcmalloc, jemalloc"
-    exit 1
-    ;;
-esac
-
-echo "LD_PRELOAD=$LD_PRELOAD"
-
-timeout "$((RUNTIME + 300))" \
-java -javaagent:${PATCHED_AGENT} \
-     ${ENABLEMENT} \
-     -Ddd.profiling.upload.period=10 \
-     -Ddd.profiling.start-force-first=true \
-     -Ddd.profiling.ddprof.liveheap.enabled=true \
-     -Ddd.profiling.ddprof.alloc.enabled=true \
-     -Ddd.profiling.ddprof.wall.enabled=true \
-     -Ddd.profiling.ddprof.nativemem.enabled=true \
-     -Ddd.env=java-profiler-stability \
-     -Ddd.service=java-profiler-chaos \
-     -Xmx2g -Xms2g \
-     -XX:MaxMetaspaceSize=384m \
-     -XX:ErrorFile=${HERE}/../../hs_err.log \
-     -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \
-     -jar ${CHAOS_JAR} \
-     --duration ${RUNTIME}s \
-     --antagonists ${ANTAGONISTS}
-
-RC=$?
-echo "RC=$RC"
+export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}"
+export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}"
+export CHAOS_ERROR_FILE="${ROOT}/hs_err.log"
 
-if [ $RC -ne 0 ]; then
-  CRASH_MSG="Chaos harness crashed (RC=${RC})"
-  HS_ERR="${HERE}/../../hs_err.log"
-  if [ -f "${HS_ERR}" ]; then
-    SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120)
-    FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120)
-    [ -n "${SIG}" ]   && CRASH_MSG="${CRASH_MSG};${SIG}"
-    [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}"
-  fi
-  echo "FAIL:${CRASH_MSG}" >&2
-  exit 1
-fi
+exec "${ROOT}/utils/run-chaos-harness.sh" "$@"
@@ -3,6 +3,7 @@ package com.datadoghq.profiler
 
 import com.datadoghq.native.NativeBuildExtension
 import com.datadoghq.native.model.BuildConfiguration
+import com.datadoghq.native.model.Platform
 import com.datadoghq.native.util.PlatformUtils
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
@@ -204,6 +205,22 @@ class ProfilerTestPlugin : Plugin<Project> {
 
         // Environment variables (explicit for consistency across both paths)
         val envVars = buildMap<String, String> {
+            // Turn silent glibc heap corruption (e.g. a use-after-free write into a
+            // freed chunk) into an immediate, attributable SIGABRT instead of a crash
+            // much later in an unrelated allocation. Only meaningful against glibc's
+            // own malloc: skip on musl (no MALLOC_CHECK_ support), and skip whenever a
+            // sanitizer/allocator already replaces malloc via LD_PRELOAD.
+            if (PlatformUtils.currentPlatform == Platform.LINUX &&
+                !PlatformUtils.isMusl() &&
+                !testEnv.containsKey("LD_PRELOAD")
+            ) {
+                val perturbByte = (1..255).random()
+                put("MALLOC_CHECK_", "3")
+                put("MALLOC_PERTURB_", perturbByte.toString())
+                // Logged so a glibc-detected corruption abort can be reproduced with the
+                // same perturb byte (the value is otherwise random per run).
+                project.logger.lifecycle("[$configName] MALLOC_PERTURB_=$perturbByte")
+            }
             putAll(testEnv)
             put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1")
             put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString())

@@ -15,6 +15,7 @@ the runner script.
 | `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation     |
 | `alloc-storm`     | Java alloc engine + GOT-patched libc malloc/free                         |
 | `trace-context`   | `setContext`/`clearContext` racing signals, span ID propagation          |
+| `vthread-context-cascade` | tracer-style context propagate/activate/restore across fan-out cascades of short-lived virtual threads, racing carrier-pin churn to trigger the `ContextStorageMode.THREAD` stale-carrier `DirectByteBuffer` use-after-free |
 
 ## Deferred
 

@@ -101,6 +101,7 @@ private void javaLoop() {
             buf[0] = (byte) idx;
             acc += buf[buf.length - 1];
             idx = (idx + 1) % SIZES.length;
+            MemoryGovernor.pace();
         }
         sink.addAndGet(acc);
     }
@@ -116,6 +117,7 @@ private void nativeLoop() {
                 long addr = (long) ALLOCATE_MEMORY.invoke(UNSAFE, NATIVE_BLOCK_SIZE);
                 acc += addr;
                 FREE_MEMORY.invoke(UNSAFE, addr);
+                MemoryGovernor.pace();
             }
         } catch (Throwable t) {
             // reflective failure; JVM crash is the signal we watch for

@@ -96,6 +96,7 @@ private void schedulePoolTasks(final int poolIdx) {
                     @Override
                     public void run() {
                         if (!running) return;
+                        MemoryGovernor.pace();
                         long seed = System.nanoTime();
                         sink.addAndGet(burn(seed));
                         ScheduledExecutorService sibling = pools[nextIdx];

@@ -104,6 +104,7 @@ private void rebalanceLoop() {
                 return;
             }
             if (!running) return;
+            MemoryGovernor.pace();
 
             // Interrupt all victims simultaneously (burst)
             Thread[] victims = new Thread[BURST_SIZE];

@@ -76,6 +76,7 @@ public void stopGracefully(Duration timeout) {
 
     private void startChain(final int chainId) {
         if (!running) return;
+        MemoryGovernor.pace();
         final long seed = ThreadLocalRandom.current().nextLong() ^ ((long) chainId << 32);
         CompletableFuture
                 .runAsync(new Runnable() {

@@ -88,6 +88,7 @@ private void ringLoop() {
             }
             slot = (slot + 1) % RING_SIZE;
             sizeIdx = (sizeIdx + 1) % RING_SIZES_BYTES.length;
+            MemoryGovernor.pace();
         }
         for (int i = 0; i < RING_SIZE; i++) {
             ring[i] = null;
@@ -114,6 +115,7 @@ private void burstLoop() {
                 Thread.currentThread().interrupt();
                 return;
             }
+            MemoryGovernor.pace();
         }
         sink.addAndGet(acc);
     }

@@ -79,6 +79,7 @@ private void loop() {
                     return;
                 }
             }
+            MemoryGovernor.pace();
         }
     }
 

@@ -110,6 +110,7 @@ private void loop() {
                 Thread.currentThread().interrupt();
                 return;
             }
+            MemoryGovernor.pace();
         }
     }
 

@@ -42,16 +42,18 @@ public static void main(String[] args) throws Exception {
         Args parsed = Args.parse(args);
         log("starting duration=" + parsed.duration + " antagonists=" + parsed.antagonists);
 
-        List<Antagonist> running = new ArrayList<>();
-        for (String name : parsed.antagonists) {
-            Antagonist a = create(name);
-            a.start();
-            running.add(a);
-            log("antagonist started: " + a.name());
-        }
+        MemoryGovernor.start();
 
-        long deadlineNanos = System.nanoTime() + parsed.duration.toNanos();
+        List<Antagonist> running = new ArrayList<>();
         try {
+            for (String name : parsed.antagonists) {
+                Antagonist a = create(name);
+                a.start();
+                running.add(a);
+                log("antagonist started: " + a.name());
+            }
+
+            long deadlineNanos = System.nanoTime() + parsed.duration.toNanos();
             while (System.nanoTime() < deadlineNanos) {
                 Thread.sleep(1_000L);
             }
@@ -72,6 +74,8 @@ private static Antagonist create(String name) {
                 return new AllocStormAntagonist();
             case "vthread-churn":
                 return new VirtualThreadChurnAntagonist();
+            case "vthread-context-cascade":
+                return new VirtualThreadContextCascadeAntagonist();
             case "classloader-churn":
                 return new ClassLoaderChurnAntagonist();
             case "trace-context":
-Original file line number
+Diff line change
@@ Expand Up / @@ -79,6 +79,7 @@ private void loop() { @@
                         return;
                     }
                 }
+                MemoryGovernor.pace();
             }
         }
@@ Expand Down @@