From 192c8cb09b1147d9f90ecae27c049d3cd4cda7f7 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 16 Jul 2026 15:58:23 +0200 Subject: [PATCH 1/5] Gate FrameType::isRawPointer on ENCODED_MASK and HotSpot A raw, unencoded ASGCT BCI can coincidentally have bit 30 set on non-HotSpot VMs; isRawPointer() must require ENCODED_MASK before trusting bit 30, not just the HotSpot check. --- ddprof-lib/src/main/cpp/frame.h | 10 ++++--- ddprof-lib/src/test/cpp/frame_ut.cpp | 40 ++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/ddprof-lib/src/main/cpp/frame.h b/ddprof-lib/src/main/cpp/frame.h index 5c00603ac7..9c5e37cb21 100644 --- a/ddprof-lib/src/main/cpp/frame.h +++ b/ddprof-lib/src/main/cpp/frame.h @@ -79,11 +79,13 @@ class FrameType { return (FrameTypeId)((bci >> TYPE_SHIFT) & FRAME_TYPE_MASK); } - // Raw pointers only exist on HotSpot (see hotspotSupport.cpp fillFrameRaw). - // On other VMs an unencoded, VM-supplied bci can coincidentally have bit 30 - // set; that must not be mistaken for our raw-pointer encoding. + // Raw pointers only exist on HotSpot (see hotspotSupport.cpp fillFrameRaw), + // and RAW_POINTER_MASK is only ever set by encode(..., rawPointer=true), + // which unconditionally also sets ENCODED_MASK. Requiring both HotSpot and + // ENCODED_MASK prevents a raw, VM-supplied ASGCT BCI from false-positiving + // just because it happens to have bit 30 set. static inline bool isRawPointer(int bci) { - return VM::isHotspot() && bci > 0 && (bci & RAW_POINTER_MASK) != 0; + return VM::isHotspot() && (bci > 0 && (bci & ENCODED_MASK) != 0 && (bci & RAW_POINTER_MASK) != 0); } }; diff --git a/ddprof-lib/src/test/cpp/frame_ut.cpp b/ddprof-lib/src/test/cpp/frame_ut.cpp index 5b96efb831..951db75fb8 100644 --- a/ddprof-lib/src/test/cpp/frame_ut.cpp +++ b/ddprof-lib/src/test/cpp/frame_ut.cpp @@ -7,14 +7,29 @@ #include "../../main/cpp/frame.h" #include "../../main/cpp/gtest_crash_handler.h" -// VMTestAccessor — friend of VM, lets tests toggle VM::isHotspot() since -// isRawPointer() now gates on it (raw pointers only exist on HotSpot). +// Test-only friend accessor for VM internals. It exists solely so these unit +// tests can exercise the VM-specific raw-pointer path in FrameType and must +// never be reused from production code. class VMTestAccessor { public: static bool getHotspot() { return VM::_hotspot; } static void setHotspot(bool v) { VM::_hotspot = v; } }; +class VMHotspotGuard { +private: + bool _saved; + +public: + explicit VMHotspotGuard(bool hotspot) : _saved(VMTestAccessor::getHotspot()) { + VMTestAccessor::setHotspot(hotspot); + } + + ~VMHotspotGuard() { + VMTestAccessor::setHotspot(_saved); + } +}; + static constexpr char FRAME_TEST_NAME[] = "FrameTest"; class GlobalSetup { @@ -150,26 +165,22 @@ TEST(FrameTypeIsRawPointerTest, FalseForEncodedWithoutFlag) { TEST(FrameTypeIsRawPointerTest, TrueWhenBit30IsSetOnHotspot) { // Manually set the raw-pointer flag (bit 30) on an encoded value. // Raw pointers only exist on HotSpot, so isRawPointer() must be gated on it. - bool saved = VMTestAccessor::getHotspot(); - VMTestAccessor::setHotspot(true); + VMHotspotGuard hotspot(true); int base = FrameType::encode(FRAME_JIT_COMPILED, 0); int withFlag = base | (1 << 30); EXPECT_TRUE(FrameType::isRawPointer(withFlag)) << "isRawPointer() must be true when bit 30 is set, value is positive, and VM is HotSpot"; - VMTestAccessor::setHotspot(saved); } TEST(FrameTypeIsRawPointerTest, FalseWhenBit30IsSetOnNonHotspot) { // Non-HotSpot VMs can coincidentally produce an unencoded bci with bit 30 // set (e.g. OpenJ9's raw AsyncGetCallTrace output); that must not be // mistaken for HotSpot's raw-pointer encoding. - bool saved = VMTestAccessor::getHotspot(); - VMTestAccessor::setHotspot(false); + VMHotspotGuard hotspot(false); int base = FrameType::encode(FRAME_JIT_COMPILED, 0); int withFlag = base | (1 << 30); EXPECT_FALSE(FrameType::isRawPointer(withFlag)) << "isRawPointer() must be false when VM is not HotSpot, even with bit 30 set"; - VMTestAccessor::setHotspot(saved); } TEST(FrameTypeIsRawPointerTest, FalseWithOnlyBit30SetOnNegative) { @@ -179,3 +190,16 @@ TEST(FrameTypeIsRawPointerTest, FalseWithOnlyBit30SetOnNegative) { EXPECT_LT(negativeWithBit30, 0); EXPECT_FALSE(FrameType::isRawPointer(negativeWithBit30)); } + +TEST(FrameTypeIsRawPointerTest, FalseForRawAsgctBciWithBit30SetButNoEncodedMarker) { + // Regression test: a raw, unencoded ASGCT BCI (bit 20 / ENCODED_MASK not set) + // that incidentally has bit 30 set must NOT be reported as a raw pointer. + // Such values occur on non-HotSpot VMs (e.g. J9), which never call encode() + // and therefore never set ENCODED_MASK; only encode(..., rawPointer=true) + // (HotSpot-only, per its own assert) is allowed to set RAW_POINTER_MASK. + int rawAsgctBciWithBit30 = 1 << 30; + VMHotspotGuard hotspot(true); + EXPECT_FALSE(FrameType::isRawPointer(rawAsgctBciWithBit30)) + << "isRawPointer() must require ENCODED_MASK (bit 20) before trusting bit 30, " + << "otherwise raw ASGCT BCIs that never went through encode() can false-positive"; +} From 32997b17d02b8fc9ccec6ca7a86a2530427ece98 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 16 Jul 2026 16:21:27 +0200 Subject: [PATCH 2/5] Add vthread-context-cascade chaos antagonist Races tracer-style context propagate/activate/restore semantics against virtual-thread carrier churn, targeting the ContextStorageMode.THREAD stale-carrier DirectByteBuffer use-after-free. --- ddprof-stresstest/src/chaos/README.md | 1 + .../com/datadoghq/profiler/chaos/Main.java | 2 + ...VirtualThreadContextCascadeAntagonist.java | 545 ++++++++++++++++++ 3 files changed, 548 insertions(+) create mode 100644 ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java diff --git a/ddprof-stresstest/src/chaos/README.md b/ddprof-stresstest/src/chaos/README.md index 5dfcabb707..2c72a4eba0 100644 --- a/ddprof-stresstest/src/chaos/README.md +++ b/ddprof-stresstest/src/chaos/README.md @@ -15,6 +15,7 @@ the runner script. | `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation | | `alloc-storm` | Java alloc engine + GOT-patched libc malloc/free | | `trace-context` | `setContext`/`clearContext` racing signals, span ID propagation | +| `vthread-context-cascade` | tracer-style context propagate/activate/restore across fan-out cascades of short-lived virtual threads, racing carrier-pin churn to trigger the `ContextStorageMode.THREAD` stale-carrier `DirectByteBuffer` use-after-free | ## Deferred diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java index 9b2cccc432..1594b0ad66 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java @@ -72,6 +72,8 @@ private static Antagonist create(String name) { return new AllocStormAntagonist(); case "vthread-churn": return new VirtualThreadChurnAntagonist(); + case "vthread-context-cascade": + return new VirtualThreadContextCascadeAntagonist(); case "classloader-churn": return new ClassLoaderChurnAntagonist(); case "trace-context": diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java new file mode 100644 index 0000000000..78de52e195 --- /dev/null +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java @@ -0,0 +1,545 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.datadoghq.profiler.chaos; + +import datadog.trace.api.Trace; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Simulates a tracer's context propagation across a cascade of short-lived virtual threads, and + * deliberately races the carrier-thread churn needed to expose a stale-carrier use-after-free. + * + *

Unlike a hand-rolled context simulation, every context write here is driven by the real + * Datadog tracer: {@link #runCascadeNode}/{@link #nestedOp}/{@link #staleRacerBefore}/{@link + * #staleRacerAfter} are {@link Trace}-annotated methods. Under a {@code dd-java-agent}- + * instrumented JVM the tracer intercepts each one, activating a real span on entry ({@code + * JavaProfiler.setContext}) and deactivating it on exit ({@code clearContext}) — exactly what a + * genuinely-instrumented application does. {@code dd-trace-api} is a {@code compileOnly} + * dependency (see {@code TraceContextAntagonist}): at runtime the patched {@code dd-java-agent} + * provides the (relocated) classes and intercepts the annotation, so this antagonist never touches + * {@code com.datadoghq.profiler.*} directly. + * + *

Each cascade node ({@link #runCascadeNode}) runs a handful of nested sub-operations, each + * activating a new child span and sleeping briefly ({@link #forceUnmount}) to force the virtual + * thread off its current carrier before the span's exit (and the next nested entry) write context + * again — the high context-change-rate dimension. It then fans out to child virtual threads while + * its own span is still active, handing the tracer's own async/virtual-thread instrumentation the + * job of propagating that span as parent context onto the next hop (the chain continues on a new + * thread/carrier). On exit, the span's close restores whatever was logically active underneath it. + * + *

A second, independent driver ({@link #pinningChurnLoop()}) continuously pins short-lived + * virtual threads to their carrier (a {@code synchronized} block held across a blocking sleep) + * across a rotating set of monitors, forcing the JDK's virtual-thread scheduler to spin up extra + * compensating carrier platform threads while the pins are held. + * + *

Targets: {@code com.datadoghq.profiler.OtelContextStorage}'s {@code ContextStorageMode.THREAD} + * fallback (the default on JDK 21+ without {@code --add-exports + * java.base/jdk.internal.misc=ALL-UNNAMED}). Under that mode the {@code DirectByteBuffer} conduit + * for a virtual thread's OTEL context (which backs both the span id fields written by {@code + * setContext} and any custom attributes) is cached in a plain {@code ThreadLocal} the first time + * the thread writes context, bound to whichever carrier happened to be mounted at that moment; the + * conduit wraps memory embedded directly inside that carrier's native {@code ProfiledThread} + * (see {@code threadLocalData.h}), which is {@code delete}d — a real {@code free()} — the moment + * the carrier's JVMTI {@code ThreadEnd} fires. The nested-op loop in {@link #runCascadeNode} races + * this cheaply: a nested span's entry writes, {@link #forceUnmount} forces a short real unmount, + * then the span's exit (or the next nested entry) writes again — if the virtual thread was + * remounted elsewhere, that second write reuses the (possibly now-stale) cached conduit. + * + *

Catching genuine memory reuse (not just misattribution) needs the original carrier + * to actually be torn down first. {@link #driverLoop} runs a duty cycle for the cascade/pin-churn + * side only (a short active burst followed by a quiet phase with no new work, long enough for the + * JDK's default {@code ForkJoinPool} scheduler — 30s keep-alive, see {@code + * java.lang.VirtualThread#createDefaultScheduler()} — to actually reap idle carriers). Stale- + * buffer racers ({@link #runStaleBufferRacer}) don't wait on that: every racer runs on a virtual + * thread bound to a purpose-built {@link #fastCarrierScheduler} — a {@link ThreadPoolExecutor} + * with a 200ms keep-alive — via the package-private {@code VirtualThread(Executor, String, int, + * Runnable)} constructor (JDK 21+; see {@code java.lang.VirtualThread}, requires {@code + * --add-opens java.base/java.lang=ALL-UNNAMED}). {@link #staleRacerLoop} keeps up to {@link + * #MAX_STALE_RACERS} of these racers in flight continuously, independent of the cascade/pin duty + * cycle above — sustained concurrent hammering rather than one batch per cycle, since more + * concurrent carrier churn means more chances for a freed {@code ProfiledThread} slot to actually + * get reused before a stale racer touches it. Each racer writes context once ({@link + * #staleRacerBefore}), sleeps a randomized interval straddling the scheduler's keep-alive ({@link + * #FAST_STALE_RACER_SLEEP_MIN_MILLIS}–{@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}), then writes + * again ({@link #staleRacerAfter}) — varying the wait means a run samples both "carrier likely + * still tearing down" and "carrier long gone, memory possibly already reused" timings, instead of + * only ever probing one fixed instant. + * + *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully no-ops the whole + * antagonist on older runtimes where it's absent. But on a JDK where it is present, the + * {@code VirtualThread} constructor above is required, not optional: it's this antagonist's only + * way to reach the stale-carrier UAF without a slow duty-cycle wait, so {@link #start} fails fast + * with an {@link IllegalStateException} (crashing the chaos harness process, which the CI runner + * already treats as a failure) rather than silently losing that coverage if a missing {@code + * --add-opens} flag or a JDK build that changed the constructor's signature makes it + * unreachable. Live virtual thread count is bounded by semaphore permit pools so cascades, + * pinning churn, and stale-buffer racers cannot runaway the JVM. + */ +public final class VirtualThreadContextCascadeAntagonist implements Antagonist { + + private static final Method OF_VIRTUAL = resolveOfVirtual(); + private static final Method BUILDER_START = resolveBuilderStart(); + + private static final int FAN_OUT = 3; + private static final int MAX_DEPTH = 4; + private static final int NESTED_OPS_PER_HOP = 3; + private static final int MAX_LIVE_VTHREADS = 512; + private static final int ROOT_BATCH = 16; + + // Carrier-pinning churn: rotating monitors held across a blocking sleep to force the + // scheduler to spin up (and later reap) compensating carrier platform threads. + private static final int PIN_LOCK_COUNT = 32; + private static final int PIN_BATCH = 24; + private static final int MAX_PIN_VTHREADS = 128; + private static final Object[] PIN_LOCKS = new Object[PIN_LOCK_COUNT]; + + static { + for (int i = 0; i < PIN_LOCK_COUNT; i++) { + PIN_LOCKS[i] = new Object(); + } + } + + // Duty cycle driving genuine carrier teardown: an active burst of cascade/pin-churn work, + // then a quiet phase long enough to clear the JDK's default 30s carrier keep-alive so idle + // carriers actually exit before the stale-buffer racers spawned in the burst wake back up. + private static final long ACTIVE_PHASE_MILLIS = 8_000L; + private static final long QUIET_PHASE_MILLIS = 40_000L; + + // Stale-buffer racers run continuously on their own driver, decoupled from the cascade/pin + // duty cycle above — that cycle only exists to let the JDK default scheduler's 30s keep-alive + // reap cascade/pin carriers, which is irrelevant to racers bound to fastCarrierScheduler. + private static final int STALE_RACER_BATCH = 64; + private static final int MAX_STALE_RACERS = 1024; + + // A short keep-alive forces genuine carrier-thread exit (and thus a real ProfiledThread + // free()) in well under a second instead of 30s. The racer's sleep is randomized across a + // range straddling this keep-alive so runs cover both "carrier likely still exiting" and + // "carrier long gone, memory possibly already reused" timings instead of one fixed instant. + private static final long FAST_SCHEDULER_KEEPALIVE_MILLIS = 200L; + private static final long FAST_STALE_RACER_SLEEP_MIN_MILLIS = 50L; + private static final long FAST_STALE_RACER_SLEEP_MAX_MILLIS = FAST_SCHEDULER_KEEPALIVE_MILLIS * 4; + private static final Constructor VTHREAD_CTOR = resolveVirtualThreadCtor(); + + private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); + private final Semaphore livePinVthreads = new Semaphore(MAX_PIN_VTHREADS); + private final Semaphore liveStaleRacers = new Semaphore(MAX_STALE_RACERS); + private final Executor fastCarrierScheduler = VTHREAD_CTOR != null ? newFastCarrierScheduler() : null; + private volatile boolean running; + private volatile boolean activePhase = true; + private Thread driver; + private Thread pinningDriver; + private Thread staleRacerDriver; + private final AtomicLong sink = new AtomicLong(); + + @Override + public String name() { + return "vthread-context-cascade"; + } + + @Override + public void start() { + if (OF_VIRTUAL != null && BUILDER_START != null && VTHREAD_CTOR == null) { + // We're on a VT-capable JDK, so the fast-carrier constructor should be reachable — + // its absence (missing --add-opens, or a JDK build that changed the constructor) + // would otherwise silently drop stale-racer coverage instead of failing the run. + throw new IllegalStateException( + "vthread-context-cascade: java.lang.VirtualThread(Executor, String, int, " + + "Runnable) is unreachable on a VT-capable JDK — pass --add-opens " + + "java.base/java.lang=ALL-UNNAMED, or this antagonist loses its core " + + "stale-carrier UAF coverage"); + } + running = true; + driver = new Thread(this::driverLoop, "chaos-vthread-cascade-driver"); + driver.setDaemon(true); + driver.start(); + pinningDriver = new Thread(this::pinningChurnLoop, "chaos-vthread-cascade-pin-driver"); + pinningDriver.setDaemon(true); + pinningDriver.start(); + staleRacerDriver = new Thread(this::staleRacerLoop, "chaos-vthread-cascade-racer-driver"); + staleRacerDriver.setDaemon(true); + staleRacerDriver.start(); + } + + @Override + public void stopGracefully(Duration timeout) { + running = false; + long deadlineNanos = System.nanoTime() + timeout.toNanos(); + try { + driver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + pinningDriver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + staleRacerDriver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. + try { + liveVthreads.tryAcquire(MAX_LIVE_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + livePinVthreads.tryAcquire(MAX_PIN_VTHREADS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + // Stale-buffer racers sleep well past stopGracefully's own timeout budget, so this is + // best-effort only — outstanding racers are daemon threads and die with the JVM. + try { + liveStaleRacers.tryAcquire(MAX_STALE_RACERS, remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** Milliseconds remaining until {@code deadlineNanos}, clamped to zero (never negative). */ + private static long remainingMillis(long deadlineNanos) { + long remainingNanos = deadlineNanos - System.nanoTime(); + return remainingNanos <= 0L ? 0L : TimeUnit.NANOSECONDS.toMillis(remainingNanos); + } + + /** + * Independently pins short-lived virtual threads to their carrier and releases them again, + * forcing the scheduler to grow and shrink its compensating carrier pool while cascade nodes + * are racing carrier migration on the other driver. + */ + private void pinningChurnLoop() { + if (OF_VIRTUAL == null || BUILDER_START == null) { + return; + } + while (running) { + if (!activePhase) { + // Quiet phase: stay out of the way so idle carriers can actually be reaped. + try { + Thread.sleep(50L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + continue; + } + for (int i = 0; i < PIN_BATCH && running && activePhase; i++) { + if (!livePinVthreads.tryAcquire()) { + continue; + } + final Object lock = PIN_LOCKS[ThreadLocalRandom.current().nextInt(PIN_LOCK_COUNT)]; + try { + Object builder = OF_VIRTUAL.invoke(null); + BUILDER_START.invoke( + builder, + (Runnable) + () -> { + try { + synchronized (lock) { + // Sleeping while holding a monitor pins the + // carrier: the scheduler must compensate with an + // extra platform thread for other runnable + // virtual threads, then reap it once idle. + Thread.sleep(2L); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + livePinVthreads.release(); + } + }); + } catch (Throwable t) { + livePinVthreads.release(); + } + } + try { + Thread.sleep(1L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + private void driverLoop() { + if (OF_VIRTUAL == null || BUILDER_START == null) { + System.out.println("[chaos] vthread-context-cascade: skipping (Thread.ofVirtual not available)"); + return; + } + while (running) { + // Active phase: mint cascade work. + activePhase = true; + long activeDeadlineNanos = System.nanoTime() + Duration.ofMillis(ACTIVE_PHASE_MILLIS).toNanos(); + while (running && System.nanoTime() < activeDeadlineNanos) { + for (int i = 0; i < ROOT_BATCH && running; i++) { + // Mint one chain-of-operations: a fresh synthetic trace, its root span + // activated by the tracer the moment runCascadeNode is entered. + spawnCascadeNode(0); + } + try { + Thread.sleep(1L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + if (!running) { + return; + } + // Quiet phase: mint nothing new so short-lived virtual threads drain and carriers + // can actually sit idle past the scheduler's 30s keep-alive and get reaped. + activePhase = false; + try { + Thread.sleep(QUIET_PHASE_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** + * Independent driver that keeps the stale-buffer racer population saturated at {@link + * #MAX_STALE_RACERS} for as long as the antagonist runs, regardless of the cascade/pin duty + * cycle — the fast-carrier scheduler needs no quiet phase to reap its carriers, so there's no + * reason to gate racer spawning on one. This is the "many vthreads hammering" dimension: + * sustained concurrent pressure on the fast-carrier scheduler's pool, not periodic bursts. + */ + private void staleRacerLoop() { + while (running) { + boolean spawnedAny = spawnStaleBufferRacers(); + try { + // Once the pool is fully saturated, back off instead of re-polling 1000x/sec for + // permits that free up far slower (racers sleep up to FAST_STALE_RACER_SLEEP_MAX_MILLIS). + Thread.sleep(spawnedAny ? 1L : 20L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + /** Returns whether at least one racer was actually started this pass. */ + private boolean spawnStaleBufferRacers() { + boolean spawnedAny = false; + for (int i = 0; i < STALE_RACER_BATCH && running; i++) { + if (!liveStaleRacers.tryAcquire()) { + continue; + } + if (startFastCarrierVirtualThread(this::runStaleBufferRacer) == null) { + liveStaleRacers.release(); + } else { + spawnedAny = true; + } + } + return spawnedAny; + } + + /** + * Activates a span once, sleeps past the carrier's keep-alive, then activates another. Under + * {@code ContextStorageMode.THREAD} the second activation reuses whatever {@code + * DirectByteBuffer} was cached on the first — if the original carrier has since been reaped + * and its {@code ProfiledThread} memory reused, this is a genuine use-after-free. + * + *

The sleep is randomized per racer across [{@link #FAST_STALE_RACER_SLEEP_MIN_MILLIS}, + * {@link #FAST_STALE_RACER_SLEEP_MAX_MILLIS}] rather than fixed, so a sustained run samples a + * spread of timings relative to {@link #FAST_SCHEDULER_KEEPALIVE_MILLIS} instead of only ever + * probing the same instant. + */ + private void runStaleBufferRacer() { + try { + staleRacerBefore(); + long sleepMillis = + ThreadLocalRandom.current() + .nextLong( + FAST_STALE_RACER_SLEEP_MIN_MILLIS, + FAST_STALE_RACER_SLEEP_MAX_MILLIS + 1); + Thread.sleep(sleepMillis); + staleRacerAfter(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + liveStaleRacers.release(); + } + } + + @Trace(operationName = "chaos.stale.before", resourceName = "chaos.stale.before") + private void staleRacerBefore() { + // Entry/exit alone is the "write" — the tracer activates/deactivates a span around this + // no-op body, exactly like TraceContextAntagonist's inner()/outer(). + } + + @Trace(operationName = "chaos.stale.after", resourceName = "chaos.stale.after") + private void staleRacerAfter() {} + + /** Spawns one virtual thread to run a cascade node, respecting the live-thread budget. */ + private void spawnCascadeNode(int depth) { + if (!running || !liveVthreads.tryAcquire()) { + return; + } + try { + Object builder = OF_VIRTUAL.invoke(null); + BUILDER_START.invoke( + builder, + (Runnable) + () -> { + try { + runCascadeNode(depth); + } finally { + liveVthreads.release(); + } + }); + } catch (Throwable t) { + liveVthreads.release(); + } + } + + /** + * One hop of the cascade, run as its own {@link Trace}-annotated span on a fresh virtual + * thread. The tracer activates this span on entry and deactivates it on exit; while it's + * active, this hop fans out to child hops on new virtual threads, handing the tracer's own + * virtual-thread instrumentation the job of propagating it as parent context onto each child. + */ + @Trace(operationName = "chaos.cascade.hop", resourceName = "chaos.cascade.hop") + private void runCascadeNode(int depth) { + long r = ThreadLocalRandom.current().nextLong() ^ ((long) depth << 48); + for (int op = 0; op < NESTED_OPS_PER_HOP && running; op++) { + r = nestedOp(r); + } + sink.addAndGet(r); + + // Propagate the chain to the next hop before this thread's own span is torn down. + if (depth < MAX_DEPTH && running) { + for (int i = 0; i < FAN_OUT; i++) { + spawnCascadeNode(depth + 1); + } + } + } + + /** + * A nested sub-operation within a hop: the tracer activates a child span on entry, {@link + * #forceUnmount} sleeps briefly to force the virtual thread off its current carrier, then the + * span's exit (or the next nested entry) writes context again — racing carrier migration + * against the cached context conduit. + */ + @Trace(operationName = "chaos.cascade.op", resourceName = "chaos.cascade.op") + private long nestedOp(long seed) { + return forceUnmount(seed); + } + + /** + * Burns some CPU (a stand-in for real work) then blocks briefly, forcing the virtual thread + * to genuinely unmount. On resumption it may be remounted on a different carrier than the + * one it was on when the context before this call was written. + */ + private static long forceUnmount(long seed) { + long r = seed; + for (int i = 0; i < 2_000; i++) { + r = r * 6364136223846793005L + 1442695040888963407L; + } + try { + Thread.sleep(1L + (r & 1)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return r; + } + + /** + * Resolves {@code java.lang.VirtualThread}'s package-private {@code (Executor, String, int, + * Runnable)} constructor, the only way to bind a virtual thread to a custom scheduler on + * mainline OpenJDK 21+ (there is no public {@code Thread.ofVirtual().scheduler(Executor)} + * API). Requires {@code --add-opens java.base/java.lang=ALL-UNNAMED}; returns {@code null} + * (caller falls back to the default-scheduler path) if that flag is absent, the JVM is + * pre-21, or the constructor's signature changes on some future JDK build. + */ + private static Constructor resolveVirtualThreadCtor() { + try { + Class vthreadClass = Class.forName("java.lang.VirtualThread"); + Constructor ctor = + vthreadClass.getDeclaredConstructor( + Executor.class, String.class, int.class, Runnable.class); + ctor.setAccessible(true); + return ctor; + } catch (Throwable t) { + return null; + } + } + + /** + * A carrier pool with a short keep-alive instead of the JDK default scheduler's fixed 30s, so + * an idle carrier's OS thread actually exits (and its {@code ProfiledThread} is freed) in + * well under a second. + */ + private static Executor newFastCarrierScheduler() { + return new ThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + FAST_SCHEDULER_KEEPALIVE_MILLIS, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), + r -> { + Thread carrier = new Thread(r, "chaos-vthread-cascade-fast-carrier"); + carrier.setDaemon(true); + return carrier; + }); + } + + /** + * Starts {@code task} on a virtual thread bound to {@link #fastCarrierScheduler}. Returns + * {@code null} (never starts anything) if {@link #VTHREAD_CTOR} wasn't resolved or the + * reflective construction fails; {@link #start} already fails fast when {@link #VTHREAD_CTOR} + * is unreachable on a VT-capable JDK, so callers here only need to handle the rarer + * construction-failure case (e.g. transient reflection errors). + * + *

{@code 0} for the characteristics argument mirrors {@code Thread.ofVirtual()}'s default + * (the only known bit, {@code Thread.NO_INHERIT_THREAD_LOCALS}, is unset). + */ + private Thread startFastCarrierVirtualThread(Runnable task) { + if (VTHREAD_CTOR == null) { + return null; + } + try { + Thread t = (Thread) VTHREAD_CTOR.newInstance(fastCarrierScheduler, null, 0, task); + t.start(); + return t; + } catch (Throwable t) { + return null; + } + } + + private static Method resolveOfVirtual() { + try { + return Thread.class.getMethod("ofVirtual"); + } catch (NoSuchMethodException e) { + return null; + } + } + + private static Method resolveBuilderStart() { + try { + Class builder = Class.forName("java.lang.Thread$Builder"); + return builder.getMethod("start", Runnable.class); + } catch (ClassNotFoundException | NoSuchMethodException e) { + return null; + } + } +} From 163ee950899807116dfc34add092134dccc9f39b Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 16 Jul 2026 16:21:55 +0200 Subject: [PATCH 3/5] Add memory-pressure governor, bound in-flight vthreads in churn antagonist MemoryGovernor watches heap/direct-memory usage and gates DirectMemoryAntagonist.burstLoop() plus the other allocation-heavy antagonists on it, nudging GC once on a critical watermark and exiting cleanly on an early heap OOME while tolerating a late one. VirtualThreadChurnAntagonist bounds in-flight virtual threads with a semaphore: the default carrier pool has far fewer platform threads than a batch submits, so an unthrottled submission rate outpaces execution and the backlog of pending VirtualThread/Continuation objects grows without bound. --- .../profiler/chaos/AllocStormAntagonist.java | 2 + .../chaos/BoundedThreadPoolAntagonist.java | 1 + .../chaos/ConsumerGroupAntagonist.java | 1 + .../profiler/chaos/ContextHopAntagonist.java | 1 + .../chaos/DirectMemoryAntagonist.java | 2 + .../profiler/chaos/DumpStormAntagonist.java | 1 + .../chaos/HiddenClassChurnAntagonist.java | 1 + .../com/datadoghq/profiler/chaos/Main.java | 18 +- .../profiler/chaos/MemoryGovernor.java | 238 ++++++++++++++++++ .../chaos/VirtualThreadChurnAntagonist.java | 27 +- .../profiler/chaos/WeakRefWaveAntagonist.java | 1 + 11 files changed, 284 insertions(+), 9 deletions(-) create mode 100644 ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/AllocStormAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/AllocStormAntagonist.java index d6ee4c6970..5b33a94fae 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/AllocStormAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/AllocStormAntagonist.java @@ -101,6 +101,7 @@ private void javaLoop() { buf[0] = (byte) idx; acc += buf[buf.length - 1]; idx = (idx + 1) % SIZES.length; + MemoryGovernor.pace(); } sink.addAndGet(acc); } @@ -116,6 +117,7 @@ private void nativeLoop() { long addr = (long) ALLOCATE_MEMORY.invoke(UNSAFE, NATIVE_BLOCK_SIZE); acc += addr; FREE_MEMORY.invoke(UNSAFE, addr); + MemoryGovernor.pace(); } } catch (Throwable t) { // reflective failure; JVM crash is the signal we watch for diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/BoundedThreadPoolAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/BoundedThreadPoolAntagonist.java index 045bf090e2..2d129a87fd 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/BoundedThreadPoolAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/BoundedThreadPoolAntagonist.java @@ -96,6 +96,7 @@ private void schedulePoolTasks(final int poolIdx) { @Override public void run() { if (!running) return; + MemoryGovernor.pace(); long seed = System.nanoTime(); sink.addAndGet(burn(seed)); ScheduledExecutorService sibling = pools[nextIdx]; diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ConsumerGroupAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ConsumerGroupAntagonist.java index 12e14f3aee..cc38e33380 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ConsumerGroupAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ConsumerGroupAntagonist.java @@ -104,6 +104,7 @@ private void rebalanceLoop() { return; } if (!running) return; + MemoryGovernor.pace(); // Interrupt all victims simultaneously (burst) Thread[] victims = new Thread[BURST_SIZE]; diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ContextHopAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ContextHopAntagonist.java index 439444dacd..c0d994c553 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ContextHopAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ContextHopAntagonist.java @@ -76,6 +76,7 @@ public void stopGracefully(Duration timeout) { private void startChain(final int chainId) { if (!running) return; + MemoryGovernor.pace(); final long seed = ThreadLocalRandom.current().nextLong() ^ ((long) chainId << 32); CompletableFuture .runAsync(new Runnable() { diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DirectMemoryAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DirectMemoryAntagonist.java index 40bcba7862..5e1023e85c 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DirectMemoryAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DirectMemoryAntagonist.java @@ -88,6 +88,7 @@ private void ringLoop() { } slot = (slot + 1) % RING_SIZE; sizeIdx = (sizeIdx + 1) % RING_SIZES_BYTES.length; + MemoryGovernor.pace(); } for (int i = 0; i < RING_SIZE; i++) { ring[i] = null; @@ -114,6 +115,7 @@ private void burstLoop() { Thread.currentThread().interrupt(); return; } + MemoryGovernor.pace(); } sink.addAndGet(acc); } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java index 26ff07147c..beba4b4a7b 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java @@ -79,6 +79,7 @@ private void loop() { return; } } + MemoryGovernor.pace(); } } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java index d496b4060f..0fa42f2b8a 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java @@ -110,6 +110,7 @@ private void loop() { Thread.currentThread().interrupt(); return; } + MemoryGovernor.pace(); } } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java index 1594b0ad66..65b29c7c6e 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java @@ -42,16 +42,18 @@ public static void main(String[] args) throws Exception { Args parsed = Args.parse(args); log("starting duration=" + parsed.duration + " antagonists=" + parsed.antagonists); - List running = new ArrayList<>(); - for (String name : parsed.antagonists) { - Antagonist a = create(name); - a.start(); - running.add(a); - log("antagonist started: " + a.name()); - } + MemoryGovernor.start(); - long deadlineNanos = System.nanoTime() + parsed.duration.toNanos(); + List running = new ArrayList<>(); try { + for (String name : parsed.antagonists) { + Antagonist a = create(name); + a.start(); + running.add(a); + log("antagonist started: " + a.name()); + } + + long deadlineNanos = System.nanoTime() + parsed.duration.toNanos(); while (System.nanoTime() < deadlineNanos) { Thread.sleep(1_000L); } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java new file mode 100644 index 0000000000..3eaf6025eb --- /dev/null +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java @@ -0,0 +1,238 @@ +/* + * Copyright 2026, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.datadoghq.profiler.chaos; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Shared memory-pressure gate for antagonists whose allocation/thread-churn + * rate can outrun GC/cleanup and threaten either the container's cgroup + * ceiling or the JVM heap itself (alloc-storm, direct-memory, dump-storm, + * weakref-wave, hidden-class-churn, context-hop, bounded-pool, + * consumer-group). + * + *

A single background thread samples two independent signals every tick: + * cgroup memory usage/limit (v2 {@code memory.current}/{@code memory.max}, + * falling back to v1 {@code memory.usage_in_bytes}/{@code + * memory.limit_in_bytes}) and JVM heap usage ({@link + * java.lang.management.MemoryMXBean#getHeapMemoryUsage()}). Either signal + * crossing its high watermark sets the shared {@code throttled} flag; + * clearing it requires both signals to be back below their low watermark + * (hysteresis, so a single antagonist bouncing near one ceiling doesn't + * flap the gate for antagonists driving the other). A signal that has no + * readable ceiling (no cgroup limit, or a JVM without {@code -Xmx}) is + * treated as permanently low rather than guessed at. + * + *

Antagonists poll {@link #pace()} once per loop iteration — a volatile + * read plus, only while throttled, a short sleep. Above a second, higher + * critical watermark, {@code pace()} sleeps much longer and fires a one-off + * {@link System#gc()} to force reclaim before OOME hits — plain throttling + * wasn't always enough to stop a heap OOME once several antagonists kept + * allocating past the regular watermark (job 1861234807). + * + *

Even a 500ms background sampler was too coarse for an antagonist that + * filled the heap from 16% to 97% between two consecutive samples (job + * 1861521304) — by the time the sampler noticed, it was too late to react. + * {@code pace()} therefore also peeks at heap usage directly every {@link + * #INLINE_CHECK_STRIDE}th call, amortizing the cost of the check across + * many hot-loop iterations while reacting within that many allocations + * instead of within a fixed wall-clock interval. This inline path only ever + * escalates (sets {@code throttled}/{@code critical}); de-escalating still + * requires the background sampler's cgroup+heap hysteresis, since a single + * antagonist's local view can't tell whether the other signal has cleared. + */ +final class MemoryGovernor { + + private static final double CGROUP_HIGH_WATERMARK = 0.85; + private static final double CGROUP_LOW_WATERMARK = 0.70; + // Heap-space OOME is less recoverable than a cgroup kill (no headroom to + // shed off-heap first), so back off earlier and require more slack. + private static final double HEAP_HIGH_WATERMARK = 0.80; + private static final double HEAP_LOW_WATERMARK = 0.65; + // Critical tier: same signals, higher bar. Crossing this calls for a + // harder brake than pace()'s regular 5ms sleep, plus a one-time GC nudge. + private static final double CGROUP_CRITICAL_WATERMARK = 0.93; + private static final double HEAP_CRITICAL_WATERMARK = 0.90; + private static final long SAMPLE_INTERVAL_MS = 100; + private static final long THROTTLE_SLEEP_MS = 5; + private static final long CRITICAL_THROTTLE_SLEEP_MS = 50; + // Power of two so the stride check is a cheap mask instead of a modulo. + private static final long INLINE_CHECK_STRIDE = 64; + + private static final Path[] USAGE_PATHS = { + Paths.get("/sys/fs/cgroup/memory.current"), + Paths.get("/sys/fs/cgroup/memory/memory.usage_in_bytes"), + }; + private static final Path[] LIMIT_PATHS = { + Paths.get("/sys/fs/cgroup/memory.max"), + Paths.get("/sys/fs/cgroup/memory/memory.limit_in_bytes"), + }; + + private static volatile boolean throttled; + private static volatile boolean critical; + // Last cgroup reading from the background sampler, for the inline path's + // log line only — the inline path itself never reads cgroup state (that + // requires a file read, too costly to do on every antagonist's hot loop). + private static volatile double lastCgroupFraction; + private static final AtomicLong paceCalls = new AtomicLong(); + + private MemoryGovernor() { + } + + /** Starts the background sampler. Cgroup and/or heap signals no-op individually if unreadable. Call once from Main. */ + static void start() { + Path usagePath = firstReadable(USAGE_PATHS); + long limitBytes = firstLimit(LIMIT_PATHS); + if (usagePath == null || limitBytes <= 0) { + usagePath = null; + limitBytes = -1; + } + Path finalUsagePath = usagePath; + long finalLimitBytes = limitBytes; + Thread sampler = new Thread(() -> sampleLoop(finalUsagePath, finalLimitBytes), "chaos-memory-governor"); + sampler.setDaemon(true); + sampler.start(); + } + + /** Called from an antagonist's hot allocation loop. No-op unless throttled. */ + static void pace() { + if ((paceCalls.incrementAndGet() & (INLINE_CHECK_STRIDE - 1)) == 0) { + inlineHeapCheck(); + } + if (critical) { + try { + Thread.sleep(CRITICAL_THROTTLE_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (throttled) { + try { + Thread.sleep(THROTTLE_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * Fast path called from {@link #pace()} every {@link + * #INLINE_CHECK_STRIDE}th call. Escalates on the caller's own thread as + * soon as heap usage crosses a watermark, instead of waiting for the + * next 500ms sampler tick. + */ + private static void inlineHeapCheck() { + MemoryUsage heap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + if (heap.getMax() <= 0) { + return; + } + double heapFraction = (double) heap.getUsed() / (double) heap.getMax(); + if (heapFraction >= HEAP_CRITICAL_WATERMARK && !critical) { + critical = true; + log(lastCgroupFraction, heapFraction, "critical (inline)"); + System.gc(); + } else if (heapFraction >= HEAP_HIGH_WATERMARK && !throttled) { + throttled = true; + log(lastCgroupFraction, heapFraction, "throttling (inline)"); + } + } + + private static void sampleLoop(Path usagePath, long limitBytes) { + while (true) { + try { + Thread.sleep(SAMPLE_INTERVAL_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + try { + update(usagePath, limitBytes); + } catch (IOException | NumberFormatException | OutOfMemoryError e) { + // Transient read/allocation failure; keep last known state and retry next tick. + } + } + } + + private static void update(Path usagePath, long limitBytes) throws IOException { + double cgroupFraction = 0.0; + if (usagePath != null) { + cgroupFraction = (double) readLong(usagePath) / (double) limitBytes; + } + lastCgroupFraction = cgroupFraction; + double heapFraction = 0.0; + MemoryUsage heap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + if (heap.getMax() > 0) { + heapFraction = (double) heap.getUsed() / (double) heap.getMax(); + } + + boolean high = cgroupFraction >= CGROUP_HIGH_WATERMARK || heapFraction >= HEAP_HIGH_WATERMARK; + boolean low = cgroupFraction <= CGROUP_LOW_WATERMARK && heapFraction <= HEAP_LOW_WATERMARK; + boolean crit = cgroupFraction >= CGROUP_CRITICAL_WATERMARK || heapFraction >= HEAP_CRITICAL_WATERMARK; + if (high && !throttled) { + throttled = true; + log(cgroupFraction, heapFraction, "throttling"); + } else if (low && throttled) { + throttled = false; + log(cgroupFraction, heapFraction, "released"); + } + // else: in the dead zone between watermarks — keep current state. + + if (crit && !critical) { + critical = true; + log(cgroupFraction, heapFraction, "critical"); + // One-shot nudge on the crossing, not every sample tick, so a + // sustained critical state doesn't turn into a GC storm on top + // of the memory pressure it's meant to relieve. + System.gc(); + } else if (low && critical) { + critical = false; + log(cgroupFraction, heapFraction, "critical-released"); + } + } + + private static void log(double cgroupFraction, double heapFraction, String state) { + System.out.println("[chaos] memory-governor " + state + + " cgroup=" + String.format("%.2f", cgroupFraction) + + " heap=" + String.format("%.2f", heapFraction)); + } + + private static Path firstReadable(Path[] candidates) { + for (Path p : candidates) { + if (Files.isReadable(p)) { + return p; + } + } + return null; + } + + private static long firstLimit(Path[] candidates) { + for (Path p : candidates) { + if (Files.isReadable(p)) { + try { + return readLong(p); + } catch (IOException | NumberFormatException e) { + // "max" (v2, unlimited) or unreadable; try the next candidate. + } + } + } + return -1; + } + + private static long readLong(Path path) throws IOException { + String s = new String(Files.readAllBytes(path), StandardCharsets.US_ASCII).trim(); + return Long.parseLong(s); + } +} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java index 3a91f941d8..0dcd969131 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java @@ -11,6 +11,7 @@ import java.lang.reflect.Method; import java.time.Duration; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicLong; /** @@ -19,6 +20,12 @@ * *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully * no-ops on older runtimes. + * + *

In-flight virtual threads are bounded by a semaphore: the default + * carrier pool has far fewer platform threads than a batch submits, so an + * unthrottled submission rate outpaces execution and the backlog of pending + * {@code VirtualThread}/{@code Continuation} objects grows without bound + * instead of churning. */ public final class VirtualThreadChurnAntagonist implements Antagonist { @@ -26,6 +33,7 @@ public final class VirtualThreadChurnAntagonist implements Antagonist { private static final Method BUILDER_START = resolveBuilderStart(); private final int batchSize; + private final Semaphore inFlight; private volatile boolean running; private Thread driver; @@ -37,6 +45,7 @@ public VirtualThreadChurnAntagonist() { public VirtualThreadChurnAntagonist(int batchSize) { this.batchSize = batchSize; + this.inFlight = new Semaphore(batchSize * 4); } @Override @@ -70,10 +79,26 @@ private void loop() { while (running) { for (int i = 0; i < batchSize && running; i++) { final long seed = System.nanoTime() ^ i; + try { + // Blocks once the backlog is full, throttling submission to the + // carrier pool's actual drain rate instead of hoarding pending + // VirtualThread/Continuation objects. + inFlight.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } try { Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke(builder, (Runnable) () -> sink.addAndGet(burn(seed))); + BUILDER_START.invoke(builder, (Runnable) () -> { + try { + sink.addAndGet(burn(seed)); + } finally { + inFlight.release(); + } + }); } catch (Throwable t) { + inFlight.release(); return; } } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/WeakRefWaveAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/WeakRefWaveAntagonist.java index c58aa8915d..3b3aee3d90 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/WeakRefWaveAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/WeakRefWaveAntagonist.java @@ -84,6 +84,7 @@ private void waveLoop() { obj[0] = (byte) i; strongRefs.add(obj); weakRefs.add(new WeakReference(obj)); + MemoryGovernor.pace(); } // Publish filled list to reader currentWave = weakRefs; From 8517bb96bb339f0f43084ab613fb52bb6ac26cec Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 16 Jul 2026 16:22:28 +0200 Subject: [PATCH 4/5] Rework chaos_check.sh into a standalone runner; default glibc heap-corruption detection utils/run-chaos-harness.sh is the new standalone entry point (usable from CI or by hand); chaos_check.sh becomes a thin wrapper around it. Also defaults MALLOC_CHECK_/MALLOC_PERTURB_ for glibc-backed test runs (this script and the shared ddprof-test Gradle test-task configuration), turning silent heap corruption into an immediate, attributable SIGABRT instead of a crash much later in an unrelated allocation. Skipped on musl and whenever a sanitizer/allocator already replaces malloc via LD_PRELOAD. The chosen MALLOC_PERTURB_ byte is logged so a corruption abort can be reproduced. --- .gitlab/reliability/chaos_check.sh | 183 +---------- .../datadoghq/profiler/ProfilerTestPlugin.kt | 17 + utils/run-chaos-harness.sh | 305 ++++++++++++++++++ 3 files changed, 333 insertions(+), 172 deletions(-) create mode 100755 utils/run-chaos-harness.sh diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh index 22b99e130d..6ac5c6605e 100755 --- a/.gitlab/reliability/chaos_check.sh +++ b/.gitlab/reliability/chaos_check.sh @@ -1,178 +1,17 @@ #!/usr/bin/env bash - -set +e # Disable exit on error +# +# CI entry point for the chaos harness. Thin wrapper around +# utils/run-chaos-harness.sh — this file only supplies the CI-specific +# caching paths (so repeated scheduled runs on the same runner reuse the +# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's +# `artifacts:` block expects at the repo root. All the actual build/run logic +# lives in the standalone script, which is also runnable by hand. HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )" -RUNTIME=${1} -CONFIG=${2:-profiler+tracer} -ALLOCATOR=${3:-gmalloc} - -echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" - -CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}" -# CHAOS_JDK uses sdkman notation (-); extract major for Adoptium API. -JDK_MAJOR="${CHAOS_JDK%%.*}" -JDK_ARCH=$(uname -m | sed 's/x86_64/x64/') -JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}" - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - TMP=$(mktemp -d) - DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse" - echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..." - if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then - echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2 - rm -rf "${TMP}" - exit 1 - fi - mkdir -p "${JDK_INSTALL_DIR}" - tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1 - rm -rf "${TMP}" -fi - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2 - exit 1 -fi -export JAVA_HOME="${JDK_INSTALL_DIR}" -export PATH="${JAVA_HOME}/bin:${PATH}" -ACTIVE_JDK=$(java -version 2>&1 | head -1) -# Check major version only — patch may differ from the Adoptium latest GA. -if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then - echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2 - exit 1 -fi - -# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot. -# Running mvn from /tmp avoids the empty pom.xml at the repo root. -DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1) -if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then - DDPROF_JAR="${DDPROF_JAR_LOCAL}" - echo "Using local ddprof jar: ${DDPROF_JAR}" -else - if [ -z "${CURRENT_VERSION:-}" ]; then - echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2 - exit 1 - fi - echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots" - (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ - -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION}) - DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar" -fi - -if [ ! -f "${DDPROF_JAR}" ]; then - echo "FAIL:ddprof jar unavailable" >&2 - exit 1 -fi - -mkdir -p /var/lib/datadog -wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' - -# chaos.jar is produced once per pipeline by the chaos:build job (stresstest -# stage) and pulled here as an artifact. Fall back to an inline build if the -# artifact is absent (e.g. local repro outside CI). -CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar" -if [ ! -f "${CHAOS_JAR}" ]; then - echo "chaos.jar artifact not present — building inline" - ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q ) -fi - -if [ ! -f "${CHAOS_JAR}" ]; then - echo "FAIL:chaos.jar unavailable" >&2 - exit 1 -fi - -# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's -# (relocated) profiler classes match the version under test. -DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \ -DDPROF_JAR=${DDPROF_JAR} \ -OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \ -"${ROOT}/utils/patch-dd-java-agent.sh" - -if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then - echo "FAIL:dd-java-agent patching failed" >&2 - exit 1 -fi - -PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar - -case $CONFIG in - profiler) - echo "Running with profiler only" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - # @Trace is a no-op without the tracer, so trace-context is excluded here. - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - profiler+tracer) - echo "Running with profiler and tracer" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - *) - echo "Unknown configuration: $CONFIG" - exit 1 - ;; -esac - -case $ALLOCATOR in - gmalloc) - echo "Running with gmalloc" - ;; - tcmalloc) - echo "Running with tcmalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') - # thread-churn/dump-storm antagonists cycle many short-lived threads; - # tcmalloc's defaults are slow to return their per-thread caches to the - # OS, which was inflating container RSS past the OOM limit on aarch64. - export TCMALLOC_RELEASE_RATE=10 - export TCMALLOC_AGGRESSIVE_DECOMMIT=1 - ;; - jemalloc) - echo "Running with jemalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') - ;; - *) - echo "Unknown allocator: $ALLOCATOR" - echo "Valid values are: gmalloc, tcmalloc, jemalloc" - exit 1 - ;; -esac - -echo "LD_PRELOAD=$LD_PRELOAD" - -timeout "$((RUNTIME + 300))" \ -java -javaagent:${PATCHED_AGENT} \ - ${ENABLEMENT} \ - -Ddd.profiling.upload.period=10 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.ddprof.liveheap.enabled=true \ - -Ddd.profiling.ddprof.alloc.enabled=true \ - -Ddd.profiling.ddprof.wall.enabled=true \ - -Ddd.profiling.ddprof.nativemem.enabled=true \ - -Ddd.env=java-profiler-stability \ - -Ddd.service=java-profiler-chaos \ - -Xmx2g -Xms2g \ - -XX:MaxMetaspaceSize=384m \ - -XX:ErrorFile=${HERE}/../../hs_err.log \ - -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \ - -jar ${CHAOS_JAR} \ - --duration ${RUNTIME}s \ - --antagonists ${ANTAGONISTS} - -RC=$? -echo "RC=$RC" +export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}" +export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}" +export CHAOS_ERROR_FILE="${ROOT}/hs_err.log" -if [ $RC -ne 0 ]; then - CRASH_MSG="Chaos harness crashed (RC=${RC})" - HS_ERR="${HERE}/../../hs_err.log" - if [ -f "${HS_ERR}" ]; then - SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120) - FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120) - [ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}" - [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}" - fi - echo "FAIL:${CRASH_MSG}" >&2 - exit 1 -fi +exec "${ROOT}/utils/run-chaos-harness.sh" "$@" diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt index 0aac0a91ae..4d85bc78ba 100644 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -3,6 +3,7 @@ package com.datadoghq.profiler import com.datadoghq.native.NativeBuildExtension import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform import com.datadoghq.native.util.PlatformUtils import org.gradle.api.DefaultTask import org.gradle.api.GradleException @@ -204,6 +205,22 @@ class ProfilerTestPlugin : Plugin { // Environment variables (explicit for consistency across both paths) val envVars = buildMap { + // Turn silent glibc heap corruption (e.g. a use-after-free write into a + // freed chunk) into an immediate, attributable SIGABRT instead of a crash + // much later in an unrelated allocation. Only meaningful against glibc's + // own malloc: skip on musl (no MALLOC_CHECK_ support), and skip whenever a + // sanitizer/allocator already replaces malloc via LD_PRELOAD. + if (PlatformUtils.currentPlatform == Platform.LINUX && + !PlatformUtils.isMusl() && + !testEnv.containsKey("LD_PRELOAD") + ) { + val perturbByte = (1..255).random() + put("MALLOC_CHECK_", "3") + put("MALLOC_PERTURB_", perturbByte.toString()) + // Logged so a glibc-detected corruption abort can be reproduced with the + // same perturb byte (the value is otherwise random per run). + project.logger.lifecycle("[$configName] MALLOC_PERTURB_=$perturbByte") + } putAll(testEnv) put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1") put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString()) diff --git a/utils/run-chaos-harness.sh b/utils/run-chaos-harness.sh new file mode 100755 index 0000000000..0409978e45 --- /dev/null +++ b/utils/run-chaos-harness.sh @@ -0,0 +1,305 @@ +#!/usr/bin/env bash +# +# Standalone chaos-harness runner. Builds/locates the ddprof jar, patches a +# dd-java-agent build with it, and runs the long-running chaos harness +# (ddprof-stresstest/src/chaos) with a chosen antagonist set and allocator. +# +# Usable both from CI (see .gitlab/reliability/chaos_check.sh, a thin wrapper +# around this script) and by hand for local repro — no CI-only assumptions +# (no fixed /opt or /var/lib paths, no CI-artifact env vars required). +# +# Usage: run-chaos-harness.sh [config] [allocator] [antagonists] +# config: profiler | profiler+tracer (default: profiler+tracer) +# allocator: gmalloc | tcmalloc | jemalloc (default: gmalloc) +# antagonists: comma-separated antagonist names; overrides the config default +# +# Env vars: +# CHAOS_JDK sdkman-style version (e.g. 21.0.3-tem) to require/download. +# If unset, uses whatever `java` is already on PATH. +# CHAOS_JDK_DIR where to install a downloaded JDK (default: under +# CHAOS_WORK_DIR; set to a persistent path to cache across runs). +# CHAOS_WORK_DIR scratch dir for the patched agent, downloaded jars, and +# hs_err.log (default: /.chaos-harness). +# CHAOS_REFRESH_AGENT=1 force re-download of dd-java-agent.jar even if cached. + +set +e + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT="$( cd "${HERE}/.." >/dev/null 2>&1 && pwd )" + +RUNTIME=${1} +CONFIG=${2:-profiler+tracer} +ALLOCATOR=${3:-gmalloc} +ANTAGONISTS_OVERRIDE=${4:-} + +if [ -z "${RUNTIME}" ]; then + echo "usage: $0 [config] [allocator] [antagonists]" >&2 + exit 1 +fi + +WORK_DIR="${CHAOS_WORK_DIR:-${ROOT}/.chaos-harness}" +mkdir -p "${WORK_DIR}" + +echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" + +# --- JDK resolution --------------------------------------------------------- +# Only fetch a JDK if the caller explicitly pinned one via CHAOS_JDK and it +# doesn't match what's already active; otherwise just use `java` as found. +if [ -n "${CHAOS_JDK:-}" ]; then + JDK_MAJOR="${CHAOS_JDK%%.*}" + ACTIVE_MAJOR=$(java -version 2>&1 | head -1 | grep -oE '"[0-9]+' | tr -d '"') + if [ "${ACTIVE_MAJOR}" != "${JDK_MAJOR}" ]; then + JDK_ARCH=$(uname -m | sed 's/x86_64/x64/') + JDK_INSTALL_DIR="${CHAOS_JDK_DIR:-${WORK_DIR}/jdk-${CHAOS_JDK}}" + if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then + TMP=$(mktemp -d) + DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse" + echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..." + if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then + echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2 + rm -rf "${TMP}" + exit 1 + fi + mkdir -p "${JDK_INSTALL_DIR}" + tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1 + rm -rf "${TMP}" + fi + export JAVA_HOME="${JDK_INSTALL_DIR}" + export PATH="${JAVA_HOME}/bin:${PATH}" + fi +fi + +if ! command -v java >/dev/null 2>&1; then + echo "FAIL:no java on PATH (set CHAOS_JDK to have one downloaded)" >&2 + exit 1 +fi +echo "Using: $(java -version 2>&1 | head -1)" + +# --- ddprof jar -------------------------------------------------------------- +# Prefer a local build artifact. If absent and CURRENT_VERSION is set (CI +# passes it in from the get-versions job), fall back to the Maven snapshot. +DDPROF_JAR=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1) +if [ -z "${DDPROF_JAR}" ] || [ ! -f "${DDPROF_JAR}" ]; then + if [ -z "${CURRENT_VERSION:-}" ]; then + echo "FAIL:no local ddprof jar found and CURRENT_VERSION is empty — build one first: ./gradlew :ddprof-lib:jar" >&2 + exit 1 + fi + echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots" + ( cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ + -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ + -Dartifact=com.datadoghq:ddprof:"${CURRENT_VERSION}" ) + DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar" + if [ ! -f "${DDPROF_JAR}" ]; then + echo "FAIL:ddprof jar unavailable (Maven snapshot download failed)" >&2 + exit 1 + fi +fi +echo "Using ddprof jar: ${DDPROF_JAR}" + +# --- dd-java-agent ------------------------------------------------------------ +DD_AGENT_JAR="${WORK_DIR}/dd-java-agent.jar" +if [ ! -f "${DD_AGENT_JAR}" ] || [ "${CHAOS_REFRESH_AGENT:-0}" = "1" ]; then + echo "Fetching dd-java-agent.jar..." + wget -q --timeout=120 --tries=3 -O "${DD_AGENT_JAR}" 'https://dtdg.co/latest-java-tracer' +fi +if [ ! -f "${DD_AGENT_JAR}" ]; then + echo "FAIL:dd-java-agent.jar download failed" >&2 + exit 1 +fi + +CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar" +if [ ! -f "${CHAOS_JAR}" ]; then + echo "chaos.jar not present — building inline" + # --no-daemon: a lingering Gradle daemon shares this container's cgroup with + # the chaos JVM we're about to launch and eats into the same memory ceiling + # the MemoryGovernor is trying to protect. + ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q --no-daemon ) +fi +if [ ! -f "${CHAOS_JAR}" ]; then + echo "FAIL:chaos.jar unavailable" >&2 + exit 1 +fi + +# Patch dd-java-agent.jar with the locally built ddprof contents so the +# agent's (relocated) profiler classes match the version under test. +PATCHED_AGENT="${WORK_DIR}/dd-java-agent-patched.jar" +if ! DD_AGENT_JAR="${DD_AGENT_JAR}" DDPROF_JAR="${DDPROF_JAR}" OUTPUT_JAR="${PATCHED_AGENT}" \ + "${ROOT}/utils/patch-dd-java-agent.sh"; then + echo "FAIL:dd-java-agent patching failed" >&2 + exit 1 +fi +if [ ! -f "${PATCHED_AGENT}" ]; then + echo "FAIL:dd-java-agent patching reported success but output jar is missing" >&2 + exit 1 +fi + +case $CONFIG in + profiler) + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" + # @Trace is a no-op without the tracer, so trace-context and + # vthread-context-cascade (which is driven entirely by @Trace-annotated + # methods) are excluded here. + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ;; + profiler+tracer) + ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" + DEFAULT_ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,vthread-context-cascade,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" + ;; + *) + echo "Unknown configuration: $CONFIG (valid: profiler, profiler+tracer)" >&2 + exit 1 + ;; +esac +ANTAGONISTS="${ANTAGONISTS_OVERRIDE:-${DEFAULT_ANTAGONISTS}}" + +case $ALLOCATOR in + gmalloc) + # Turn silent heap corruption (e.g. a stale-pointer UAF write into a freed + # chunk) into an immediate, attributable SIGABRT instead of a crash much + # later in an unrelated allocation. Only meaningful against glibc's own + # malloc, not the LD_PRELOAD-replaced allocators below. + export MALLOC_CHECK_=3 + export MALLOC_PERTURB_=$((RANDOM % 255 + 1)) + # Logged so a glibc-detected corruption abort can be reproduced with the + # same perturb byte (the value is otherwise random per run). + echo "MALLOC_PERTURB_=${MALLOC_PERTURB_}" + # thread-churn/dump-storm/vthread-context-cascade cycle many short-lived + # threads; glibc's per-thread arenas are slow to trim back to the OS, + # which was inflating container RSS past the OOM limit on aarch64 + # (mirrors the tcmalloc/jemalloc tuning below). + export MALLOC_ARENA_MAX=2 + export MALLOC_TRIM_THRESHOLD_=65536 + # glibc >= 2.34 moved MALLOC_CHECK_ support out of the main libc; it is a + # silent no-op unless libc_malloc_debug is preloaded (see glibc's "Heap + # Consistency Checking" docs). Detect that case and preload the debug + # library explicitly, rather than silently running with corruption + # detection disabled. + if command -v ldd >/dev/null 2>&1; then + GLIBC_VERSION=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$') + if [ -n "${GLIBC_VERSION}" ] && [ "$(printf '%s\n' "2.34" "${GLIBC_VERSION}" | sort -V | head -1)" = "2.34" ]; then + MALLOC_DEBUG_LIB=$(find /usr/lib/ /usr/lib64/ /lib/ /lib64/ -name 'libc_malloc_debug.so*' 2>/dev/null | head -1) + if [ -z "${MALLOC_DEBUG_LIB}" ]; then + echo "FAIL:glibc ${GLIBC_VERSION} requires libc_malloc_debug to be preloaded for MALLOC_CHECK_ to take effect, but it could not be found" >&2 + exit 1 + fi + export LD_PRELOAD="${MALLOC_DEBUG_LIB}${LD_PRELOAD:+:${LD_PRELOAD}}" + echo "glibc ${GLIBC_VERSION} detected — preloading ${MALLOC_DEBUG_LIB} for MALLOC_CHECK_" + fi + fi + ;; + tcmalloc) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libtcmalloc_minimal.so.4' -o -name 'libtcmalloc.dylib' 2>/dev/null | head -1) + # thread-churn/dump-storm antagonists cycle many short-lived threads; + # tcmalloc's defaults are slow to return their per-thread caches to the + # OS, which was inflating container RSS past the OOM limit on aarch64. + export TCMALLOC_RELEASE_RATE=10 + export TCMALLOC_AGGRESSIVE_DECOMMIT=1 + ;; + jemalloc) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libjemalloc.so' -o -name 'libjemalloc.dylib' 2>/dev/null | head -1) + # Same aarch64 RSS-inflation issue as tcmalloc above: jemalloc's default + # decay times leave dirty/muzzy pages resident under heavy thread churn. + export MALLOC_CONF="background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000" + ;; + *) + echo "Unknown allocator: $ALLOCATOR (valid: gmalloc, tcmalloc, jemalloc)" >&2 + exit 1 + ;; +esac +echo "LD_PRELOAD=${LD_PRELOAD:-}" + +# Log the actual container memory ceiling (if any) so OOMKilled failures are +# diagnosable from the CI job log directly, instead of having to infer it +# from RSS measurements on a separate repro host. +CGROUP_MEM_LIMIT=$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes 2>/dev/null || cat /sys/fs/cgroup/memory.max 2>/dev/null || echo "unknown") +echo "cgroup memory limit: ${CGROUP_MEM_LIMIT}" + +# Container ceiling is 6GiB (see logged cgroup limit above). Job 1858271921 +# (profiler+tracer, jemalloc, 21.0.3-tem, aarch64) OOMKilled within seconds of +# all 13 antagonists starting: several of them (alloc-storm, direct-memory, +# dump-storm) deliberately burst off-heap/native memory hard and fast, sharing +# the same cgroup limit as the heap. Reproduced under a 6GiB-capped cgroup and +# confirmed MemoryGovernor (see chaos/MemoryGovernor.java, wired into the +# antagonists' hot loops) keeps that same config under the limit. 2560m still +# ran within a few percent of java.lang.OutOfMemoryError under the profiler +# config (job 1860026763); 3072m keeps comparable headroom to the off-heap +# side while giving the heap itself real margin. +HEAP_MB=3072 + +HS_ERR="${CHAOS_ERROR_FILE:-${WORK_DIR}/hs_err.log}" +rm -f "${HS_ERR}" + +# dd-trace-java's profiling uploader writes rotated JFR chunks under +# /tmp/ddprof_root/pid_/jfr/... (the path seen in the +# jfrStreamWriterHost.inline.hpp write-guarantee crash log). On a CI runner +# with no real intake, stalled/failed uploads can leave chunks from a +# previous run sitting on disk, starving the next run's JFR writer. Clean +# stale chunks before we start and again on exit, and log disk usage so a +# future disk-full crash is diagnosable from the job log directly. +DDPROF_ROOT="/tmp/ddprof_root" +rm -rf "${DDPROF_ROOT}" +trap 'rm -rf "${DDPROF_ROOT}"' EXIT +echo "disk usage ($(dirname "${DDPROF_ROOT}")): $(df -h "$(dirname "${DDPROF_ROOT}")" | awk 'NR==2{print $3" used / "$2" total ("$5" full)"}')" + +# -XX:+ExitOnOutOfMemoryError: without it, a heap OOME is "recoverable" and +# every other allocating thread throws its own OutOfMemoryError over the +# following seconds instead of the process exiting once. Job 1861521304 rode +# that cascade into a java.lang.instrument allocation-failure assertion and +# then a SIGSEGV with no hs_err.log at all — exit immediately on the first +# OOME instead, so a failure is a single, diagnosable event. +CHAOS_START=$(date +%s) +timeout "$((RUNTIME + 300))" \ +java -javaagent:${PATCHED_AGENT} \ + --add-opens java.base/java.lang=ALL-UNNAMED \ + --add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ + ${ENABLEMENT} \ + -Ddd.profiling.upload.period=10 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.ddprof.liveheap.enabled=true \ + -Ddd.profiling.ddprof.alloc.enabled=true \ + -Ddd.profiling.ddprof.wall.enabled=true \ + -Ddd.profiling.ddprof.nativemem.enabled=true \ + -Ddd.env=java-profiler-stability \ + -Ddd.service=java-profiler-chaos \ + -Xmx${HEAP_MB}m -Xms${HEAP_MB}m \ + -XX:MaxMetaspaceSize=384m \ + -XX:NativeMemoryTracking=summary \ + -XX:ErrorFile=${HS_ERR} \ + -XX:+ExitOnOutOfMemoryError \ + -jar ${CHAOS_JAR} \ + --duration ${RUNTIME}s \ + --antagonists ${ANTAGONISTS} + +RC=$? +CHAOS_ELAPSED=$(( $(date +%s) - CHAOS_START )) +echo "RC=$RC (elapsed ${CHAOS_ELAPSED}s of ${RUNTIME}s requested)" + +# Exit code 3 is HotSpot's own -XX:+ExitOnOutOfMemoryError termination — a +# clean exit, not a crash. Antagonists deliberately stress allocation, so +# hitting the heap ceiling under their combined load isn't itself a bug. +# It only counts as a failure here if it happened too early for the rest of +# the antagonist mix to have had a fair shot at running (see chaos_check.sh +# job 1862133932: this run recovered from four heap-pressure cycles before a +# fifth finally exhausted it at ~80% of the run). +if [ $RC -eq 3 ] && [ "${CHAOS_ELAPSED}" -ge "$((RUNTIME / 2))" ]; then + echo "Chaos run hit the heap ceiling at ${CHAOS_ELAPSED}s/${RUNTIME}s — treating as a tolerated outcome, not a failure" + exit 0 +fi + +if [ $RC -ne 0 ]; then + CRASH_MSG="Chaos harness crashed (RC=${RC})" + if [ $RC -eq 3 ]; then + CRASH_MSG="Chaos harness hit heap OOME too early (RC=3 at ${CHAOS_ELAPSED}s/${RUNTIME}s) — antagonists didn't get a fair run" + fi + if [ -f "${HS_ERR}" ]; then + SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120) + FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120) + [ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}" + [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}" + echo "hs_err written to ${HS_ERR}" + fi + echo "FAIL:${CRASH_MSG}" >&2 + exit 1 +fi + +echo "Chaos run completed cleanly" From cdc7c95e416abbe9ee6af5c73bddc4520d3f9b7b Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 16 Jul 2026 16:38:34 +0200 Subject: [PATCH 5/5] Address review polish: exactly-once GC nudge, uncontended pacing, bounded fast-carrier pool MemoryGovernor: throttled/critical are now AtomicBoolean with compareAndSet-guarded transitions, so a watermark crossing observed by several antagonist threads at once escalates (and fires System.gc()) exactly once instead of racing. paceCalls moves from a shared AtomicLong to a per-thread counter, removing a cache-line contention point on every antagonist's hot allocation loop. VirtualThreadContextCascadeAntagonist: fastCarrierScheduler's max pool size is now bounded by MAX_STALE_RACERS instead of Integer.MAX_VALUE (no headroom lost, since that's already the hard cap on concurrent racers). stopGracefully null-checks its driver threads so it's safe to call even if start() never ran or failed partway through; same guard added to VirtualThreadChurnAntagonist. run-chaos-harness.sh: the allocator-detection find commands now cap -maxdepth 4 instead of unboundedly recursing /usr/lib, /usr/lib64, etc. --- .../profiler/chaos/MemoryGovernor.java | 41 ++++++++++--------- .../chaos/VirtualThreadChurnAntagonist.java | 12 ++++-- ...VirtualThreadContextCascadeAntagonist.java | 41 +++++++++++++------ utils/run-chaos-harness.sh | 6 +-- 4 files changed, 61 insertions(+), 39 deletions(-) diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java index 3eaf6025eb..daf0702b90 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/MemoryGovernor.java @@ -16,7 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicBoolean; /** * Shared memory-pressure gate for antagonists whose allocation/thread-churn @@ -82,13 +82,21 @@ final class MemoryGovernor { Paths.get("/sys/fs/cgroup/memory/memory.limit_in_bytes"), }; - private static volatile boolean throttled; - private static volatile boolean critical; + // CAS-guarded so a watermark crossing is escalated exactly once even when + // several antagonist threads observe it in the same instant (both via the + // background sampler and the per-thread inline path below). + private static final AtomicBoolean throttled = new AtomicBoolean(); + private static final AtomicBoolean critical = new AtomicBoolean(); // Last cgroup reading from the background sampler, for the inline path's // log line only — the inline path itself never reads cgroup state (that // requires a file read, too costly to do on every antagonist's hot loop). private static volatile double lastCgroupFraction; - private static final AtomicLong paceCalls = new AtomicLong(); + // Per-thread rather than a shared counter: this is incremented on every + // pace() call across every antagonist's hot loop, so a single shared + // AtomicLong would become a cache-line contention point. Exactness across + // threads isn't needed — each thread just needs to trigger the inline + // check roughly every INLINE_CHECK_STRIDE calls. + private static final ThreadLocal paceCallCounter = ThreadLocal.withInitial(() -> new long[1]); private MemoryGovernor() { } @@ -110,16 +118,17 @@ static void start() { /** Called from an antagonist's hot allocation loop. No-op unless throttled. */ static void pace() { - if ((paceCalls.incrementAndGet() & (INLINE_CHECK_STRIDE - 1)) == 0) { + long[] counter = paceCallCounter.get(); + if ((++counter[0] & (INLINE_CHECK_STRIDE - 1)) == 0) { inlineHeapCheck(); } - if (critical) { + if (critical.get()) { try { Thread.sleep(CRITICAL_THROTTLE_SLEEP_MS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - } else if (throttled) { + } else if (throttled.get()) { try { Thread.sleep(THROTTLE_SLEEP_MS); } catch (InterruptedException e) { @@ -140,12 +149,10 @@ private static void inlineHeapCheck() { return; } double heapFraction = (double) heap.getUsed() / (double) heap.getMax(); - if (heapFraction >= HEAP_CRITICAL_WATERMARK && !critical) { - critical = true; + if (heapFraction >= HEAP_CRITICAL_WATERMARK && critical.compareAndSet(false, true)) { log(lastCgroupFraction, heapFraction, "critical (inline)"); System.gc(); - } else if (heapFraction >= HEAP_HIGH_WATERMARK && !throttled) { - throttled = true; + } else if (heapFraction >= HEAP_HIGH_WATERMARK && throttled.compareAndSet(false, true)) { log(lastCgroupFraction, heapFraction, "throttling (inline)"); } } @@ -181,24 +188,20 @@ private static void update(Path usagePath, long limitBytes) throws IOException { boolean high = cgroupFraction >= CGROUP_HIGH_WATERMARK || heapFraction >= HEAP_HIGH_WATERMARK; boolean low = cgroupFraction <= CGROUP_LOW_WATERMARK && heapFraction <= HEAP_LOW_WATERMARK; boolean crit = cgroupFraction >= CGROUP_CRITICAL_WATERMARK || heapFraction >= HEAP_CRITICAL_WATERMARK; - if (high && !throttled) { - throttled = true; + if (high && throttled.compareAndSet(false, true)) { log(cgroupFraction, heapFraction, "throttling"); - } else if (low && throttled) { - throttled = false; + } else if (low && throttled.compareAndSet(true, false)) { log(cgroupFraction, heapFraction, "released"); } // else: in the dead zone between watermarks — keep current state. - if (crit && !critical) { - critical = true; + if (crit && critical.compareAndSet(false, true)) { log(cgroupFraction, heapFraction, "critical"); // One-shot nudge on the crossing, not every sample tick, so a // sustained critical state doesn't turn into a GC storm on top // of the memory pressure it's meant to relieve. System.gc(); - } else if (low && critical) { - critical = false; + } else if (low && critical.compareAndSet(true, false)) { log(cgroupFraction, heapFraction, "critical-released"); } } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java index 0dcd969131..7406b3de4d 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java @@ -64,10 +64,14 @@ public void start() { @Override public void stopGracefully(Duration timeout) { running = false; - try { - driver.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // driver is only non-null once start() has actually spawned it; guard against + // stopGracefully being called when start() never ran. + if (driver != null) { + try { + driver.join(timeout.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } } diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java index 78de52e195..ecc3c951e6 100644 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java +++ b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadContextCascadeAntagonist.java @@ -134,6 +134,13 @@ public final class VirtualThreadContextCascadeAntagonist implements Antagonist { private static final long FAST_SCHEDULER_KEEPALIVE_MILLIS = 200L; private static final long FAST_STALE_RACER_SLEEP_MIN_MILLIS = 50L; private static final long FAST_STALE_RACER_SLEEP_MAX_MILLIS = FAST_SCHEDULER_KEEPALIVE_MILLIS * 4; + + // Caps the fast-carrier pool's platform-thread growth. MAX_STALE_RACERS is + // already the hard ceiling on concurrent virtual threads that could each + // need a carrier, so this doesn't reduce headroom — it just removes the + // unbounded-growth characteristic of Integer.MAX_VALUE. + private static final int MAX_FAST_CARRIER_THREADS = MAX_STALE_RACERS; + private static final Constructor VTHREAD_CTOR = resolveVirtualThreadCtor(); private final Semaphore liveVthreads = new Semaphore(MAX_LIVE_VTHREADS); @@ -180,20 +187,28 @@ public void start() { public void stopGracefully(Duration timeout) { running = false; long deadlineNanos = System.nanoTime() + timeout.toNanos(); - try { - driver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // Driver threads are only non-null once start() has actually spawned them; guard against + // stopGracefully being called when start() never ran or failed partway through. + if (driver != null) { + try { + driver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } - try { - pinningDriver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + if (pinningDriver != null) { + try { + pinningDriver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } - try { - staleRacerDriver.join(remainingMillis(deadlineNanos)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + if (staleRacerDriver != null) { + try { + staleRacerDriver.join(remainingMillis(deadlineNanos)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } // Best-effort drain of in-flight cascades so the JVM doesn't exit mid-fan-out. try { @@ -492,7 +507,7 @@ private static Constructor resolveVirtualThreadCtor() { private static Executor newFastCarrierScheduler() { return new ThreadPoolExecutor( 0, - Integer.MAX_VALUE, + MAX_FAST_CARRIER_THREADS, FAST_SCHEDULER_KEEPALIVE_MILLIS, TimeUnit.MILLISECONDS, new SynchronousQueue<>(), diff --git a/utils/run-chaos-harness.sh b/utils/run-chaos-harness.sh index 0409978e45..ba8f695208 100755 --- a/utils/run-chaos-harness.sh +++ b/utils/run-chaos-harness.sh @@ -177,7 +177,7 @@ case $ALLOCATOR in if command -v ldd >/dev/null 2>&1; then GLIBC_VERSION=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$') if [ -n "${GLIBC_VERSION}" ] && [ "$(printf '%s\n' "2.34" "${GLIBC_VERSION}" | sort -V | head -1)" = "2.34" ]; then - MALLOC_DEBUG_LIB=$(find /usr/lib/ /usr/lib64/ /lib/ /lib64/ -name 'libc_malloc_debug.so*' 2>/dev/null | head -1) + MALLOC_DEBUG_LIB=$(find /usr/lib/ /usr/lib64/ /lib/ /lib64/ -maxdepth 4 -name 'libc_malloc_debug.so*' 2>/dev/null | head -1) if [ -z "${MALLOC_DEBUG_LIB}" ]; then echo "FAIL:glibc ${GLIBC_VERSION} requires libc_malloc_debug to be preloaded for MALLOC_CHECK_ to take effect, but it could not be found" >&2 exit 1 @@ -188,7 +188,7 @@ case $ALLOCATOR in fi ;; tcmalloc) - export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libtcmalloc_minimal.so.4' -o -name 'libtcmalloc.dylib' 2>/dev/null | head -1) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -maxdepth 4 -name 'libtcmalloc_minimal.so.4' -o -name 'libtcmalloc.dylib' 2>/dev/null | head -1) # thread-churn/dump-storm antagonists cycle many short-lived threads; # tcmalloc's defaults are slow to return their per-thread caches to the # OS, which was inflating container RSS past the OOM limit on aarch64. @@ -196,7 +196,7 @@ case $ALLOCATOR in export TCMALLOC_AGGRESSIVE_DECOMMIT=1 ;; jemalloc) - export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -name 'libjemalloc.so' -o -name 'libjemalloc.dylib' 2>/dev/null | head -1) + export LD_PRELOAD=$(find /usr/lib/ /usr/lib64/ /opt/homebrew/lib/ /usr/local/lib/ -maxdepth 4 -name 'libjemalloc.so' -o -name 'libjemalloc.dylib' 2>/dev/null | head -1) # Same aarch64 RSS-inflation issue as tcmalloc above: jemalloc's default # decay times leave dirty/muzzy pages resident under heavy thread churn. export MALLOC_CONF="background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000"