diff --git a/dd-java-agent/agent-bootstrap/src/jmh/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperBenchmark.java b/dd-java-agent/agent-bootstrap/src/jmh/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperBenchmark.java new file mode 100644 index 00000000000..f58261fb1a7 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/jmh/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperBenchmark.java @@ -0,0 +1,47 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(1) +public class TaskBlockHelperBenchmark { + + /** Measures the rejected-entry fast path. */ + @Benchmark + public TaskBlockHelper.State captureRejected(BenchmarkState state) { + return TaskBlockHelper.captureForSleep(state.rejected); + } + + /** Measures an accepted entry paired with synchronous completion. */ + @Benchmark + public void captureAndFinishAccepted(BenchmarkState state) { + TaskBlockHelper.finish(state.acceptedState); + } + + /** Measures the null-state completion fast path. */ + @Benchmark + public void finishNull() { + TaskBlockHelper.finish(null); + } + + @State(Scope.Thread) + public static class BenchmarkState { + final ProfilingContextIntegration rejected = ProfilingContextIntegration.NoOp.INSTANCE; + final TaskBlockHelper.State acceptedState = + new TaskBlockHelper.State(ProfilingContextIntegration.NoOp.INSTANCE, 17L); + } +} diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java new file mode 100644 index 00000000000..7f3a11f117e --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java @@ -0,0 +1,88 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import java.util.Collections; +import java.util.Map; +import java.util.WeakHashMap; + +/** Helper for profiling {@code LockSupport.park*} intervals from bootstrap instrumentation. */ +public final class LockSupportHelper { + /** + * Best-effort association between a parked thread and the most recent {@code unpark} caller's + * active span. Weak keys avoid retaining terminated threads; the synchronized wrapper protects + * calls made concurrently by parked and unparking threads. + */ + static final Map UNPARKING_SPAN = Collections.synchronizedMap(new WeakHashMap<>()); + + private LockSupportHelper() {} + + /** State required to balance a park entry accepted by a profiling integration. */ + public static final class ParkState { + private final ProfilingContextIntegration profiling; + private final long blockerHash; + + ParkState(ProfilingContextIntegration profiling, long blockerHash) { + this.profiling = profiling; + this.blockerHash = blockerHash; + } + } + + /** Captures a park entry through the currently installed profiling integration. */ + public static ParkState captureState(Object blocker) { + return captureState(blocker, AgentTracer.get().getProfilingContext()); + } + + static ParkState captureState(Object blocker, ProfilingContextIntegration profiling) { + if (profiling == null) { + return null; + } + try { + if (!profiling.parkEnter()) { + return null; + } + return new ParkState( + profiling, + blocker == null ? 0L : Integer.toUnsignedLong(System.identityHashCode(blocker))); + } catch (Throwable ignored) { + return null; + } + } + + /** Drains unpark attribution and balances an accepted park entry. */ + public static void finish(ParkState state) { + Long unblockingSpanId = UNPARKING_SPAN.remove(Thread.currentThread()); + finish(state, unblockingSpanId == null ? 0L : unblockingSpanId); + } + + static void finish(ParkState state, long unblockingSpanId) { + if (state == null) { + return; + } + try { + state.profiling.parkExit(state.blockerHash, unblockingSpanId); + } catch (Throwable ignored) { + } + } + + /** + * Records the latest unpark caller for {@code thread}. An untraced call explicitly clears an + * older traced caller so the association follows last-writer semantics. + */ + public static void recordUnpark(Thread thread) { + if (thread == null) { + return; + } + AgentSpan span = AgentTracer.activeSpan(); + AgentSpanContext context = span == null ? null : span.spanContext(); + if (context instanceof ProfilerContext) { + UNPARKING_SPAN.put(thread, ((ProfilerContext) context).getSpanId()); + } else { + UNPARKING_SPAN.remove(thread); + } + } +} diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper.java new file mode 100644 index 00000000000..a27dddb2e64 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper.java @@ -0,0 +1,53 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; + +/** Helper for synchronously bracketing untraced {@code Thread.sleep} intervals. */ +public final class TaskBlockHelper { + private TaskBlockHelper() {} + + /** State required to balance a native TaskBlock interval accepted at sleep entry. */ + public static final class State { + final ProfilingContextIntegration profiling; + final long token; + + State(ProfilingContextIntegration profiling, long token) { + this.profiling = profiling; + this.token = token; + } + } + + /** Starts a synchronous sleeping interval through the currently installed integration. */ + public static State captureForSleep() { + try { + return captureForSleep(AgentTracer.get().getProfilingContext()); + } catch (Throwable ignored) { + return null; + } + } + + static State captureForSleep(ProfilingContextIntegration profiling) { + try { + if (profiling == null) { + return null; + } + long token = profiling.beginTaskBlock(ProfilingContextIntegration.BLOCKING_STATE_SLEEPING); + return token == 0L ? null : new State(profiling, token); + } catch (Throwable ignored) { + return null; + } + } + + /** Completes a sleeping interval accepted by {@link #captureForSleep()}. */ + public static void finish(State state) { + if (state == null) { + return; + } + try { + state.profiling.endTaskBlock(state.token, 0L, 0L); + } catch (Throwable ignored) { + } + } +} diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelperTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelperTest.java new file mode 100644 index 00000000000..f7e8d4a4f46 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelperTest.java @@ -0,0 +1,167 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class LockSupportHelperTest { + private interface ProfilerSpanContext extends AgentSpanContext, ProfilerContext {} + + private AgentTracer.TracerAPI previousTracer; + + @BeforeEach + void setUp() { + previousTracer = AgentTracer.get(); + AgentTracer.forceRegister(tracerWithActiveSpan(null)); + LockSupportHelper.UNPARKING_SPAN.clear(); + } + + @AfterEach + void tearDown() { + LockSupportHelper.UNPARKING_SPAN.clear(); + AgentTracer.forceRegister(previousTracer); + } + + @Test + void nullIntegrationDoesNotAcceptEntry() { + assertNull(LockSupportHelper.captureState(new Object(), null)); + } + + @Test + void rejectedEntryDoesNotRequireExit() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.parkEnter()).thenReturn(false); + + LockSupportHelper.ParkState state = LockSupportHelper.captureState(new Object(), profiling); + LockSupportHelper.finish(state, 17L); + + assertNull(state); + verify(profiling).parkEnter(); + verify(profiling, never()).parkExit(0L, 17L); + } + + @Test + void acceptedEntryIsAlwaysPairedWithExit() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.parkEnter()).thenReturn(true); + Object blocker = new Object(); + + LockSupportHelper.ParkState state = LockSupportHelper.captureState(blocker, profiling); + LockSupportHelper.finish(state, 23L); + + assertNotNull(state); + verify(profiling).parkEnter(); + verify(profiling).parkExit(Integer.toUnsignedLong(System.identityHashCode(blocker)), 23L); + } + + @Test + void nullBlockerUsesZeroIdentity() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.parkEnter()).thenReturn(true); + + LockSupportHelper.finish(LockSupportHelper.captureState(null, profiling), 0L); + + verify(profiling).parkExit(0L, 0L); + } + + @Test + void entryAndExitFailuresDoNotEscapeInstrumentation() { + ProfilingContextIntegration entryFailure = mock(ProfilingContextIntegration.class); + doThrow(new IllegalStateException("enter")).when(entryFailure).parkEnter(); + assertNull(LockSupportHelper.captureState(null, entryFailure)); + + ProfilingContextIntegration exitFailure = mock(ProfilingContextIntegration.class); + when(exitFailure.parkEnter()).thenReturn(true); + doThrow(new IllegalStateException("exit")).when(exitFailure).parkExit(0L, 0L); + LockSupportHelper.ParkState state = LockSupportHelper.captureState(null, exitFailure); + + assertDoesNotThrow(() -> LockSupportHelper.finish(state, 0L)); + } + + @Test + void finishAlwaysDrainsUnparkAttribution() { + Thread current = Thread.currentThread(); + LockSupportHelper.UNPARKING_SPAN.put(current, 31L); + + LockSupportHelper.finish(null); + + assertNull(LockSupportHelper.UNPARKING_SPAN.get(current)); + } + + @Test + void latestTracedUnparkWins() { + Thread target = new Thread(() -> {}); + installActiveSpan(41L); + LockSupportHelper.recordUnpark(target); + installActiveSpan(42L); + + LockSupportHelper.recordUnpark(target); + + assertEquals(42L, LockSupportHelper.UNPARKING_SPAN.get(target)); + } + + @Test + void laterUntracedUnparkClearsOlderTracedCaller() { + Thread target = new Thread(() -> {}); + installActiveSpan(51L); + LockSupportHelper.recordUnpark(target); + AgentTracer.forceRegister(tracerWithActiveSpan(null)); + + LockSupportHelper.recordUnpark(target); + + assertNull(LockSupportHelper.UNPARKING_SPAN.get(target)); + } + + @Test + void nonProfilerSpanContextClearsOlderTracedCaller() { + Thread target = new Thread(() -> {}); + installActiveSpan(61L); + LockSupportHelper.recordUnpark(target); + AgentSpan span = mock(AgentSpan.class); + when(span.spanContext()).thenReturn(mock(AgentSpanContext.class)); + AgentTracer.forceRegister(tracerWithActiveSpan(span)); + + LockSupportHelper.recordUnpark(target); + + assertNull(LockSupportHelper.UNPARKING_SPAN.get(target)); + } + + @Test + void nullUnparkTargetIsNoOp() { + installActiveSpan(71L); + + LockSupportHelper.recordUnpark(null); + + assertEquals(0, LockSupportHelper.UNPARKING_SPAN.size()); + } + + private static void installActiveSpan(long spanId) { + AgentSpan span = mock(AgentSpan.class); + ProfilerSpanContext context = mock(ProfilerSpanContext.class); + when(span.spanContext()).thenReturn(context); + when(context.getSpanId()).thenReturn(spanId); + AgentTracer.forceRegister(tracerWithActiveSpan(span)); + } + + private static AgentTracer.TracerAPI tracerWithActiveSpan(AgentSpan span) { + AgentTracer.TracerAPI tracer = mock(AgentTracer.TracerAPI.class); + when(tracer.activeSpan()).thenReturn(span); + return tracer; + } +} diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperTest.java new file mode 100644 index 00000000000..d46f6ed7c82 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperTest.java @@ -0,0 +1,93 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import org.junit.jupiter.api.Test; + +class TaskBlockHelperTest { + private static final long TOKEN = 313L; + + @Test + void nullIntegrationDoesNotArmState() { + assertNull(TaskBlockHelper.captureForSleep(null)); + } + + @Test + void zeroIsTheOnlyInvalidToken() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.beginTaskBlock(ProfilingContextIntegration.BLOCKING_STATE_SLEEPING)) + .thenReturn(0L); + + assertNull(TaskBlockHelper.captureForSleep(profiling)); + verify(profiling).beginTaskBlock(ProfilingContextIntegration.BLOCKING_STATE_SLEEPING); + verify(profiling, never()).endTaskBlock(0L, 0L, 0L); + } + + @Test + void positiveTokenIsRetainedWithItsIntegration() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.beginTaskBlock(ProfilingContextIntegration.BLOCKING_STATE_SLEEPING)) + .thenReturn(TOKEN); + + TaskBlockHelper.State state = TaskBlockHelper.captureForSleep(profiling); + + assertNotNull(state); + assertEquals(profiling, state.profiling); + assertEquals(TOKEN, state.token); + } + + @Test + void negativeNonzeroTokenIsValid() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.beginTaskBlock(ProfilingContextIntegration.BLOCKING_STATE_SLEEPING)) + .thenReturn(Long.MIN_VALUE); + + TaskBlockHelper.State state = TaskBlockHelper.captureForSleep(profiling); + TaskBlockHelper.finish(state); + + assertNotNull(state); + verify(profiling).endTaskBlock(Long.MIN_VALUE, 0L, 0L); + } + + @Test + void finishAlwaysBalancesAcceptedToken() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + TaskBlockHelper.State state = new TaskBlockHelper.State(profiling, TOKEN); + + TaskBlockHelper.finish(state); + + verify(profiling).endTaskBlock(TOKEN, 0L, 0L); + } + + @Test + void finishIgnoresNullState() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + + TaskBlockHelper.finish(null); + + verifyNoInteractions(profiling); + } + + @Test + void entryAndExitFailuresAreContained() { + ProfilingContextIntegration entryFailure = mock(ProfilingContextIntegration.class); + when(entryFailure.beginTaskBlock(ProfilingContextIntegration.BLOCKING_STATE_SLEEPING)) + .thenThrow(new IllegalStateException("entry")); + ProfilingContextIntegration exitFailure = mock(ProfilingContextIntegration.class); + doThrow(new IllegalStateException("exit")).when(exitFailure).endTaskBlock(TOKEN, 0L, 0L); + + assertDoesNotThrow(() -> TaskBlockHelper.captureForSleep(entryFailure)); + assertDoesNotThrow(() -> TaskBlockHelper.finish(new TaskBlockHelper.State(exitFailure, TOKEN))); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index df1b75789db..30bb1661ab0 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package com.datadog.profiling.ddprof; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.enableJMethodIDOptim; @@ -14,6 +15,7 @@ import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallCollapsing; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallContextFilter; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallInterval; +import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallPrecheck; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isAllocationProfilingEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isCpuProfilerEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isLiveHeapSizeTrackingEnabled; @@ -36,6 +38,7 @@ import com.datadog.profiling.utils.ProfilingMode; import com.datadoghq.profiler.ContextSetter; import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.TaskBlockBridge; import datadog.environment.JavaVirtualMachine; import datadog.libs.ddprof.DdprofLibraryLoader; import datadog.trace.api.config.ProfilingConfig; @@ -101,9 +104,12 @@ public static DatadogProfiler newInstance(ConfigProvider configProvider) { return new DatadogProfiler(configProvider); } - private final AtomicBoolean recordingFlag = new AtomicBoolean(false); + // JavaProfiler owns one process-wide native recording; all wrapper instances must observe the + // same lifecycle so early-created integrations can dispatch hooks while the controller records. + private static final AtomicBoolean RECORDING = new AtomicBoolean(false); private final ConfigProvider configProvider; private final JavaProfiler profiler; + private final TaskBlockBridge taskBlockBridge; private final Set profilingModes = EnumSet.noneOf(ProfilingMode.class); private final ContextSetter contextSetter; @@ -244,6 +250,7 @@ void reset() { private DatadogProfiler(ConfigProvider configProvider) { this.configProvider = configProvider; this.profiler = DdprofLibraryLoader.javaProfiler().getComponent(); + this.taskBlockBridge = new TaskBlockBridge(profiler); this.detailedDebugLogging = configProvider.getBoolean( PROFILING_DETAILED_DEBUG_LOGGING, PROFILING_DETAILED_DEBUG_LOGGING_DEFAULT); @@ -355,7 +362,7 @@ public RecordingData stop(OngoingRecording recording) { /** A call-back from {@linkplain DatadogProfilerRecording#stop()} */ void stopProfiler() { - if (recordingFlag.compareAndSet(true, false)) { + if (RECORDING.compareAndSet(true, false)) { profiler.stop(); if (isActive()) { log.debug("Profiling is still active. Waiting to stop."); @@ -385,7 +392,7 @@ String executeProfilerCmd(String cmd) throws IOException { } Path newRecording() throws IOException, IllegalStateException { - if (recordingFlag.compareAndSet(false, true)) { + if (RECORDING.compareAndSet(false, true)) { Path recFile = Files.createTempFile(recordingsPath, "dd-profiler-", ".jfr"); String cmd = cmdStartProfiling(recFile); try { @@ -397,7 +404,7 @@ Path newRecording() throws IOException, IllegalStateException { } else { log.warn("Unable to start Datadog profiler recording: {}", e.getMessage()); } - recordingFlag.set(false); + RECORDING.set(false); throw e; } return recFile; @@ -410,6 +417,11 @@ void dump(Path path) { } String cmdStartProfiling(Path file) throws IllegalStateException { + return cmdStartProfiling(file, DdprofLibraryLoader.isFinalWallProtocolSupported()); + } + + String cmdStartProfiling(Path file, boolean finalWallProtocolSupported) + throws IllegalStateException { // 'start' = start, 'jfr=7' = store in JFR format ready for concatenation int safemode = getSafeMode(configProvider); if (safemode != ProfilingConfig.PROFILING_DATADOG_PROFILER_SAFEMODE_DEFAULT) { @@ -421,7 +433,6 @@ String cmdStartProfiling(Path file) throws IllegalStateException { } StringBuilder cmd = new StringBuilder("start,jfr"); cmd.append(",file=").append(file.toAbsolutePath()); - cmd.append(",loglevel=").append(getLogLevel(configProvider)); cmd.append(",jstackdepth=").append(getStackDepth(configProvider)); cmd.append(",cstack=").append(getCStack(configProvider)); cmd.append(",safemode=").append(getSafeMode(configProvider)); @@ -464,14 +475,17 @@ String cmdStartProfiling(Path file) throws IllegalStateException { cmd.append('~'); } cmd.append(getWallInterval(configProvider)).append('m'); - // ddprof quirk: if filter parameter is omitted, it defaults to "0" (enabled), - // not empty string (disabled). When enabled without tracing, no threads are added - // to the filter, resulting in zero samples. We must explicitly pass filter= (empty) - // to disable filtering and sample all threads. - if (getWallContextFilter(configProvider)) { - cmd.append(",filter=0"); + boolean contextScope = getWallContextFilter(configProvider); + if (finalWallProtocolSupported) { + cmd.append(",wallscope=").append(contextScope ? "context" : "all"); + if (getWallPrecheck(configProvider)) { + cmd.append(",wallprecheck=true"); + } } else { - cmd.append(",filter="); + // Legacy ddprof defaults to context filtering when filter is omitted. An empty value is + // therefore required to select all threads. wallprecheck is omitted because support for + // it was established together with wallscope by the non-mutating capability check. + cmd.append(contextScope ? ",filter=0" : ",filter="); } } cmd.append(",loglevel=").append(getLogLevel(configProvider)); @@ -534,6 +548,34 @@ public void clearSpanContext() { reapplyAppContext(); } + boolean hasParkTaskBlockSupport() { + return taskBlockBridge.hasParkSupport(); + } + + boolean parkEnter() { + if (!RECORDING.get()) { + return false; + } + taskBlockBridge.parkEnter(); + return true; + } + + void parkExit(long blocker, long unblockingSpanId) { + taskBlockBridge.parkExit(blocker, unblockingSpanId); + } + + boolean hasSynchronousTaskBlockSupport() { + return taskBlockBridge.hasSynchronousTaskBlockSupport(); + } + + long beginTaskBlock(int state) { + return RECORDING.get() ? taskBlockBridge.beginTaskBlock(state) : 0L; + } + + boolean endTaskBlock(long token, long blocker, long unblockingSpanId) { + return token != 0L && taskBlockBridge.endTaskBlock(token, blocker, unblockingSpanId); + } + public boolean setContextValue(int offset, String value) { if (contextSetter != null && offset >= 0 && value != null) { try { diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java index 8582e27c853..f9d5a17aaf0 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java @@ -58,6 +58,7 @@ import com.datadog.profiling.controller.ProfilingSupport; import datadog.environment.JavaVirtualMachine; import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.api.profiling.TaskBlockInstrumentationConfig; import datadog.trace.bootstrap.config.provider.ConfigProvider; import java.lang.management.ManagementFactory; import java.util.Collections; @@ -169,6 +170,10 @@ public static boolean getWallContextFilter(ConfigProvider configProvider) { PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT); } + public static boolean getWallPrecheck(ConfigProvider configProvider) { + return TaskBlockInstrumentationConfig.isWallPrecheckEnabled(configProvider); + } + static boolean isJmethodIDSafe() { return ProfilingSupport.isJmethodIDSafe(); } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 65462a46bce..44740502239 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package com.datadog.profiling.ddprof; import datadog.trace.api.EndpointTracker; @@ -78,6 +79,26 @@ public String name() { return "ddprof"; } + @Override + public boolean parkEnter() { + return DDPROF.parkEnter(); + } + + @Override + public void parkExit(long blocker, long unblockingSpanId) { + DDPROF.parkExit(blocker, unblockingSpanId); + } + + @Override + public long beginTaskBlock(int state) { + return DDPROF.beginTaskBlock(state); + } + + @Override + public boolean endTaskBlock(long token, long blocker, long unblockingSpanId) { + return DDPROF.endTaskBlock(token, blocker, unblockingSpanId); + } + public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java index 79d6985508e..6ae99538a0f 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package com.datadog.profiling.ddprof; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -72,4 +73,12 @@ void testSnapshot() throws Exception { assertNotNull(inputStream); assertTrue(inputStream.available() > 0); } + + @Test + void recordingLifecycleIsVisibleAcrossProfilerWrappers() { + DatadogProfiler integrationProfiler = DatadogProfiler.newInstance(); + + assertTrue(integrationProfiler.parkEnter()); + integrationProfiler.parkExit(0L, 0L); + } } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index 6dcb3e9d231..f1944d54c45 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -69,7 +69,7 @@ void test() throws Exception { IItemCollection events = JfrLoaderToolkit.loadEvents(data.getStream()); assertTrue(events.hasItems()); } finally { - recording.stop(); + recording.close(); } } else { log.warn("Datadog Profiler is not available. Skipping test."); @@ -104,8 +104,7 @@ void testStartCmd(boolean cpu, boolean wall, boolean alloc, boolean memleak) thr private static Stream profilingModes() { return IntStream.range(0, 1 << 4) .mapToObj( - x -> - Arguments.of((x & 0x1000) != 0, (x & 0x100) != 0, (x & 0x10) != 0, (x & 0x1) != 0)); + x -> Arguments.of((x & 0x8) != 0, (x & 0x4) != 0, (x & 0x2) != 0, (x & 0x1) != 0)); } @Test @@ -127,7 +126,8 @@ void testStartCmdEnableJMethodIDOptim() throws Exception { @ParameterizedTest @MethodSource("wallContextFilterModes") - void testWallContextFilter(boolean tracingEnabled, boolean contextFilterEnabled) + void testFinalWallScope( + boolean tracingEnabled, boolean contextFilterEnabled, String expectedWallScope) throws Exception { // Skip test if profiler native library is not available (e.g., on macOS) try { @@ -150,36 +150,90 @@ void testWallContextFilter(boolean tracingEnabled, boolean contextFilterEnabled) DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); Path targetFile = Paths.get("/tmp/target.jfr"); - String cmd = profiler.cmdStartProfiling(targetFile); + String cmd = profiler.cmdStartProfiling(targetFile, true); assertTrue(cmd.contains("wall="), "Command should contain wall profiling: " + cmd); + assertTrue(cmd.contains(",wallscope=" + expectedWallScope), cmd); + assertFalse(cmd.contains(",filter="), cmd); + } - if (tracingEnabled && contextFilterEnabled) { - assertTrue( - cmd.contains(",filter=0"), - "Command should contain ',filter=0' when tracing and context filter are enabled: " + cmd); - } else { - assertTrue( - cmd.contains(",filter="), - "Command should contain ',filter=' when tracing is disabled or context filter is disabled: " - + cmd); - if (cmd.contains(",filter=0")) { - throw new AssertionError( - "Command should not contain ',filter=0' when tracing is disabled or context filter is disabled: " - + cmd); + @ParameterizedTest + @MethodSource("wallPrecheckModes") + void testWallPrecheck(boolean wallPrecheckEnabled) throws Exception { + try { + Throwable reason = DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(); + if (reason != null) { + Assumptions.assumeTrue(false, "Profiler not available: " + reason.getMessage()); } + } catch (Throwable e) { + Assumptions.assumeTrue(false, "Profiler not available: " + e.getMessage()); + } + + Properties props = new Properties(); + props.put(ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED, "true"); + props.put( + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK, + Boolean.toString(wallPrecheckEnabled)); + + DatadogProfiler profiler = + DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); + + Path targetFile = Paths.get("/tmp/target.jfr"); + String cmd = profiler.cmdStartProfiling(targetFile, true); + + assertTrue(cmd.contains("wall="), cmd); + if (wallPrecheckEnabled) { + assertTrue(cmd.contains(",wallprecheck=true"), cmd); + } else { + assertFalse(cmd.contains("wallprecheck="), cmd); } } + @ParameterizedTest + @MethodSource("legacyWallModes") + void testLegacyWallProtocol( + boolean tracingEnabled, + boolean contextFilterEnabled, + boolean wallPrecheckEnabled, + String expectedFilter) + throws Exception { + Properties props = new Properties(); + props.put(ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED, "true"); + props.put(TraceInstrumentationConfig.TRACE_ENABLED, Boolean.toString(tracingEnabled)); + props.put( + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, + Boolean.toString(contextFilterEnabled)); + props.put( + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK, + Boolean.toString(wallPrecheckEnabled)); + + DatadogProfiler profiler = + DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); + String cmd = profiler.cmdStartProfiling(Paths.get("/tmp/target.jfr"), false); + + assertTrue(cmd.contains(expectedFilter), cmd); + assertFalse(cmd.contains("wallscope="), cmd); + assertFalse(cmd.contains("wallprecheck="), cmd); + } + + private static Stream wallPrecheckModes() { + return Stream.of(Arguments.of(true), Arguments.of(false)); + } + private static Stream wallContextFilterModes() { return Stream.of( - Arguments.of(true, true), // tracing enabled, context filter enabled -> filter=0 - Arguments.of(true, false), // tracing enabled, context filter disabled -> filter= - Arguments.of( - false, true), // tracing disabled, context filter enabled -> filter= (tracing disabled - // overrides) - Arguments.of(false, false) // tracing disabled, context filter disabled -> filter= - ); + Arguments.of(true, true, "context"), + Arguments.of(true, false, "all"), + Arguments.of(false, true, "all"), + Arguments.of(false, false, "all")); + } + + private static Stream legacyWallModes() { + return Stream.of( + Arguments.of(true, true, true, ",filter=0"), + Arguments.of(true, false, true, ",filter="), + Arguments.of(false, true, true, ",filter="), + Arguments.of(false, false, false, ",filter=")); } @Test diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/datadog/libs/ddprof/DdprofLibraryLoaderTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/datadog/libs/ddprof/DdprofLibraryLoaderTest.java new file mode 100644 index 00000000000..d262ad54614 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/datadog/libs/ddprof/DdprofLibraryLoaderTest.java @@ -0,0 +1,46 @@ +// Copyright 2026 Datadog, Inc. +package datadog.libs.ddprof; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +class DdprofLibraryLoaderTest { + + @Test + void finalProtocolProbeUsesNonMutatingCheckAction() { + AtomicReference command = new AtomicReference<>(); + + boolean supported = + DdprofLibraryLoader.probeFinalWallProtocol( + value -> { + command.set(value); + return "OK"; + }); + + assertTrue(supported); + assertEquals("check,wall=10m,wallscope=all,wallprecheck=true", command.get()); + } + + @Test + void finalProtocolProbeRejectsParserFailure() { + assertFalse( + DdprofLibraryLoader.probeFinalWallProtocol( + command -> { + throw new IllegalArgumentException("Unknown argument: wallscope"); + })); + } + + @Test + void finalProtocolProbeRejectsCheckFailure() { + assertFalse( + DdprofLibraryLoader.probeFinalWallProtocol( + command -> { + throw new IOException("wall engine unavailable"); + })); + } +} diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index 36cedaf6081..69ce02b0d84 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -70,6 +70,7 @@ 1 java.util.concurrent.ConcurrentHashMap* 1 java.util.concurrent.atomic.* 1 java.util.concurrent.locks.* +0 java.util.concurrent.locks.LockSupport 0 java.util.logging.* # allow capturing JVM shutdown 0 java.lang.Shutdown diff --git a/dd-java-agent/ddprof-lib/src/main/java/com/datadoghq/profiler/TaskBlockBridge.java b/dd-java-agent/ddprof-lib/src/main/java/com/datadoghq/profiler/TaskBlockBridge.java new file mode 100644 index 00000000000..040c6d9df04 --- /dev/null +++ b/dd-java-agent/ddprof-lib/src/main/java/com/datadoghq/profiler/TaskBlockBridge.java @@ -0,0 +1,116 @@ +// Copyright 2026 Datadog, Inc. +package com.datadoghq.profiler; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +/** + * Compatibility bridge for TaskBlock hooks added to newer ddprof artifacts. + * + *

This class deliberately lives in the profiler package: that gives its lookup access to the + * package-private park hooks without making those hooks public. Hook names are resolved from + * strings so the agent remains binary-compatible with older ddprof artifacts. Resolved method + * handles are cached and invoked directly on the hot path. + */ +public final class TaskBlockBridge { + private static final MethodHandle PARK_ENTER = findVirtual("parkEnter", void.class); + private static final MethodHandle PARK_EXIT = + findVirtual("parkExit", void.class, long.class, long.class); + private static final MethodHandle BEGIN_TASK_BLOCK = + findVirtual("beginTaskBlock", long.class, int.class); + private static final MethodHandle END_TASK_BLOCK = + findVirtual("endTaskBlock", boolean.class, long.class, long.class, long.class); + + private final JavaProfiler profiler; + + /** Creates a bridge bound to the loaded profiler singleton. */ + public TaskBlockBridge(JavaProfiler profiler) { + this.profiler = profiler; + } + + /** Returns whether both native park hooks are available. */ + public boolean hasParkSupport() { + return PARK_ENTER != null && PARK_EXIT != null; + } + + /** Returns whether the synchronous paired TaskBlock API is available. */ + public boolean hasSynchronousTaskBlockSupport() { + return BEGIN_TASK_BLOCK != null && END_TASK_BLOCK != null; + } + + /** Invokes the native park-entry hook when it is available. */ + public boolean parkEnter() { + if (!hasParkSupport()) { + return false; + } + try { + PARK_ENTER.invokeExact(profiler); + return true; + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + /** Invokes the native park-exit hook when it is available. */ + public void parkExit(long blocker, long unblockingSpanId) { + if (!hasParkSupport()) { + return; + } + try { + PARK_EXIT.invokeExact(profiler, blocker, unblockingSpanId); + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + /** Begins a synchronous TaskBlock interval, returning {@code 0} when unsupported. */ + public long beginTaskBlock(int state) { + if (!hasSynchronousTaskBlockSupport()) { + return 0L; + } + try { + return (long) BEGIN_TASK_BLOCK.invokeExact(profiler, state); + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + /** Ends a synchronous TaskBlock interval when the paired API is available. */ + public boolean endTaskBlock(long token, long blocker, long unblockingSpanId) { + if (!hasSynchronousTaskBlockSupport()) { + return false; + } + try { + return (boolean) END_TASK_BLOCK.invokeExact(profiler, token, blocker, unblockingSpanId); + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + private static MethodHandle findVirtual( + String methodName, Class returnType, Class... parameterTypes) { + return findVirtual(JavaProfiler.class, methodName, returnType, parameterTypes); + } + + static MethodHandle findVirtual( + Class targetClass, String methodName, Class returnType, Class... parameterTypes) { + try { + return MethodHandles.lookup() + .findVirtual(targetClass, methodName, MethodType.methodType(returnType, parameterTypes)); + } catch (NoSuchMethodException | IllegalAccessException ignored) { + return null; + } + } + + private static RuntimeException propagate(Throwable throwable) { + if (throwable instanceof RuntimeException) { + return (RuntimeException) throwable; + } + if (throwable instanceof Error) { + throw (Error) throwable; + } + return new IllegalStateException( + "Unexpected checked exception from ddprof TaskBlock hook", throwable); + } +} diff --git a/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java b/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java index b637dc69309..8becf1335d3 100644 --- a/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java +++ b/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java @@ -24,6 +24,9 @@ */ public final class DdprofLibraryLoader { private static final Logger log = LoggerFactory.getLogger(DdprofLibraryLoader.class.getName()); + private static final String FINAL_WALL_PROTOCOL_CHECK = + "check,wall=10m,wallscope=all,wallprecheck=true"; + private static volatile boolean finalWallProtocolSupported; public abstract static class ComponentHolder { private volatile boolean loaded = false; @@ -123,6 +126,17 @@ public static OTelContextHolder otelContext() { return OTEL_CONTEXT_HOLDER; } + /** + * Returns whether the loaded native profiler accepts the final wall-clock command protocol. + * + *

Calling this method initializes the profiler holder when necessary. The capability is + * established with the profiler's non-mutating {@code check} action. + */ + public static boolean isFinalWallProtocolSupported() { + PROFILER_HOLDER.getComponent(); + return finalWallProtocolSupported; + } + private static JavaProfilerHolder initJavaProfiler() { JavaProfiler profiler; Throwable reasonNotLoaded = null; @@ -135,6 +149,11 @@ private static JavaProfilerHolder initJavaProfiler() { scratch); // sanity test - force load Datadog profiler to catch it not being available early profiler.execute("status"); + finalWallProtocolSupported = probeFinalWallProtocol(profiler::execute); + if (!finalWallProtocolSupported) { + log.debug( + "Loaded ddprof does not accept wallscope and wallprecheck; using legacy wall command options"); + } } catch (Throwable t) { reasonNotLoaded = t; profiler = null; @@ -142,6 +161,21 @@ private static JavaProfilerHolder initJavaProfiler() { return new JavaProfilerHolder(profiler, reasonNotLoaded); } + static boolean probeFinalWallProtocol(ProfilerCommandExecutor executor) { + try { + executor.execute(FINAL_WALL_PROTOCOL_CHECK); + return true; + } catch (IllegalArgumentException | IllegalStateException | IOException ignored) { + return false; + } + } + + @FunctionalInterface + interface ProfilerCommandExecutor { + String execute(String command) + throws IllegalArgumentException, IllegalStateException, IOException; + } + private static JVMAccessHolder initJVMAccess() { ConfigProvider configProvider = ConfigProvider.getInstance(); AtomicReference reasonNotLoaded = new AtomicReference<>(); diff --git a/dd-java-agent/ddprof-lib/src/test/java/com/datadoghq/profiler/TaskBlockBridgeTest.java b/dd-java-agent/ddprof-lib/src/test/java/com/datadoghq/profiler/TaskBlockBridgeTest.java new file mode 100644 index 00000000000..a0f92a7ca9d --- /dev/null +++ b/dd-java-agent/ddprof-lib/src/test/java/com/datadoghq/profiler/TaskBlockBridgeTest.java @@ -0,0 +1,34 @@ +// Copyright 2026 Datadog, Inc. +package com.datadoghq.profiler; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class TaskBlockBridgeTest { + @Test + void resolvesPackagePrivateHooksFromProfilerPackage() { + assertNotNull(TaskBlockBridge.findVirtual(SupportedHooks.class, "parkEnter", void.class)); + assertNotNull( + TaskBlockBridge.findVirtual(SupportedHooks.class, "beginTaskBlock", long.class, int.class)); + } + + @Test + void missingHooksAreReportedWithoutFailingClassInitialization() { + assertNull(TaskBlockBridge.findVirtual(UnsupportedHooks.class, "parkEnter", void.class)); + assertNull( + TaskBlockBridge.findVirtual( + UnsupportedHooks.class, "beginTaskBlock", long.class, int.class)); + } + + static final class SupportedHooks { + void parkEnter() {} + + long beginTaskBlock(int state) { + return state; + } + } + + static final class UnsupportedHooks {} +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle new file mode 100644 index 00000000000..f2f91cdc7fd --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle @@ -0,0 +1,12 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + coreJdk() + } +} + +dependencies { + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java new file mode 100644 index 00000000000..805cc11d8a3 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -0,0 +1,109 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.locksupport; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isStatic; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Config; +import datadog.trace.api.profiling.TaskBlockInstrumentationConfig; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.bootstrap.instrumentation.java.concurrent.LockSupportHelper; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * Instruments {@link java.util.concurrent.locks.LockSupport} park and unpark operations for + * TaskBlock profiling. The native hooks own platform-thread park intervals only; virtual-thread + * calls are rejected by ddprof without touching carrier-thread ownership. + */ +@AutoService(InstrumenterModule.class) +public class LockSupportProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + /** Creates the LockSupport profiling instrumentation module. */ + public LockSupportProfilingInstrumentation() { + super("lock-support"); + } + + @Override + public boolean isEnabled() { + return super.isEnabled() + && TaskBlockInstrumentationConfig.isEnabled(Config.get(), ConfigProvider.getInstance()); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.util.concurrent.locks.LockSupport"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + parkMethod().and(takesArgument(0, named("java.lang.Object"))), + getClass().getName() + "$ParkWithBlockerAdvice"); + transformer.applyAdvice( + parkMethod().and(not(takesArgument(0, named("java.lang.Object")))), + getClass().getName() + "$ParkWithoutBlockerAdvice"); + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(named("unpark")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$UnparkAdvice"); + } + + private static ElementMatcher.Junction parkMethod() { + return isMethod() + .and(isStatic()) + .and(nameStartsWith("park")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))); + } + + /** Advice for park variants whose first argument is the blocker object. */ + public static final class ParkWithBlockerAdvice { + /** Starts the paired profiling lifecycle before the park. */ + @Advice.OnMethodEnter(suppress = Throwable.class) + public static LockSupportHelper.ParkState before(@Advice.Argument(0) Object blocker) { + return LockSupportHelper.captureState(blocker); + } + + /** Completes an accepted profiling lifecycle after the park. */ + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter LockSupportHelper.ParkState state) { + LockSupportHelper.finish(state); + } + } + + /** Advice for park variants without an explicit blocker object. */ + public static final class ParkWithoutBlockerAdvice { + /** Starts the paired profiling lifecycle before the park. */ + @Advice.OnMethodEnter(suppress = Throwable.class) + public static LockSupportHelper.ParkState before() { + return LockSupportHelper.captureState(null); + } + + /** Completes an accepted profiling lifecycle after the park. */ + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter LockSupportHelper.ParkState state) { + LockSupportHelper.finish(state); + } + } + + /** Advice that records the active span of the latest unpark caller. */ + public static final class UnparkAdvice { + /** Updates best-effort unpark attribution before dispatching to the JDK. */ + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before(@Advice.Argument(0) Thread thread) { + LockSupportHelper.recordUnpark(thread); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingDisabledForkedTest.groovy b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingDisabledForkedTest.groovy new file mode 100644 index 00000000000..2f88f4393d1 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingDisabledForkedTest.groovy @@ -0,0 +1,31 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.locksupport + +import datadog.trace.agent.test.InstrumentationSpecification + +import java.util.concurrent.locks.LockSupport + +class LockSupportProfilingDisabledForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + injectSysConfig('dd.profiling.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.precheck', 'false') + injectSysConfig('dd.profiling.ddprof.wall.context.filter', 'false') + super.configurePreAgent() + } + + def setup() { + TEST_PROFILING_CONTEXT_INTEGRATION.clear() + } + + def 'disabled TaskBlock gate leaves LockSupport uninstrumented'() { + when: + LockSupport.parkNanos(1L) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkEnterCalls.get() == 0 + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitCalls.get() == 0 + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingInstrumentationForkedTest.groovy b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingInstrumentationForkedTest.groovy new file mode 100644 index 00000000000..0e651f8a086 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingInstrumentationForkedTest.groovy @@ -0,0 +1,67 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.locksupport + +import datadog.trace.agent.test.InstrumentationSpecification + +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.LockSupport + +class LockSupportProfilingInstrumentationForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + injectSysConfig('dd.profiling.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.precheck', 'true') + injectSysConfig('dd.profiling.ddprof.wall.context.filter', 'false') + super.configurePreAgent() + } + + def setup() { + TEST_PROFILING_CONTEXT_INTEGRATION.clear() + } + + def cleanup() { + TEST_PROFILING_CONTEXT_INTEGRATION.clear() + } + + def 'transformed park with blocker dispatches one balanced lifecycle on the calling thread'() { + setup: + Object blocker = new Object() + Thread callingThread = Thread.currentThread() + + when: + LockSupport.parkNanos(blocker, TimeUnit.MILLISECONDS.toNanos(1)) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkEnterCalls.get() >= 1 + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitCalls.get() >= 1 + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitThreads.contains(callingThread) + TEST_PROFILING_CONTEXT_INTEGRATION.lastParkBlocker.get() == Integer.toUnsignedLong(System.identityHashCode(blocker)) + } + + def 'transformed park without blocker dispatches a zero blocker'() { + setup: + Thread callingThread = Thread.currentThread() + + when: + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitThreads.contains(callingThread) + TEST_PROFILING_CONTEXT_INTEGRATION.lastParkBlocker.get() == 0L + } + + def 'rejected entry is not followed by an exit'() { + setup: + TEST_PROFILING_CONTEXT_INTEGRATION.acceptParkEntries = false + Thread callingThread = Thread.currentThread() + + when: + LockSupport.parkNanos(1L) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkEnterCalls.get() >= 1 + !TEST_PROFILING_CONTEXT_INTEGRATION.parkExitThreads.contains(callingThread) + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/build.gradle new file mode 100644 index 00000000000..71a051b606b --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/build.gradle @@ -0,0 +1,15 @@ +// Copyright 2026 Datadog, Inc. + +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + coreJdk() + } +} + +dependencies { + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito + testImplementation libs.bytebuddy +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepCallSiteMatcher.java b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepCallSiteMatcher.java new file mode 100644 index 00000000000..df1955b3a02 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepCallSiteMatcher.java @@ -0,0 +1,60 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.threadsleep; + +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.SLEEP_DURATION_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.SLEEP_JI_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.SLEEP_J_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.THREAD_INTERNAL; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.TIME_UNIT_INTERNAL; + +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.jar.asm.Opcodes; +import net.bytebuddy.pool.TypePool; + +/** Shared validation for sleep call sites seen by the scanner and rewriting visitor. */ +final class ThreadSleepCallSiteMatcher { + private static final TypeDescription THREAD = TypeDescription.ForLoadedType.of(Thread.class); + + private ThreadSleepCallSiteMatcher() {} + + static boolean isSupported( + int opcode, String owner, String name, String descriptor, TypePool typePool) { + if (!"sleep".equals(name)) { + return false; + } + if (opcode == Opcodes.INVOKEVIRTUAL) { + return TIME_UNIT_INTERNAL.equals(owner) && SLEEP_J_DESC.equals(descriptor); + } + if (opcode != Opcodes.INVOKESTATIC || !isSupportedThreadDescriptor(descriptor)) { + return false; + } + if (THREAD_INTERNAL.equals(owner)) { + return true; + } + if (typePool == null) { + return false; + } + try { + TypePool.Resolution resolution = typePool.describe(owner.replace('/', '.')); + return resolution.isResolved() && resolution.resolve().isAssignableTo(THREAD); + } catch (RuntimeException ignored) { + return false; + } + } + + static boolean isPotentiallySupported(int opcode, String owner, String name, String descriptor) { + if (!"sleep".equals(name)) { + return false; + } + return (opcode == Opcodes.INVOKESTATIC && isSupportedThreadDescriptor(descriptor)) + || (opcode == Opcodes.INVOKEVIRTUAL + && TIME_UNIT_INTERNAL.equals(owner) + && SLEEP_J_DESC.equals(descriptor)); + } + + private static boolean isSupportedThreadDescriptor(String descriptor) { + return SLEEP_J_DESC.equals(descriptor) + || SLEEP_JI_DESC.equals(descriptor) + || SLEEP_DURATION_DESC.equals(descriptor); + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepCallSiteMethodVisitor.java b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepCallSiteMethodVisitor.java new file mode 100644 index 00000000000..e55687385aa --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepCallSiteMethodVisitor.java @@ -0,0 +1,249 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.threadsleep; + +import datadog.trace.agent.tooling.bytebuddy.reqctx.LocalVariablesSorter; +import net.bytebuddy.jar.asm.Label; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.jar.asm.Opcodes; +import net.bytebuddy.jar.asm.Type; +import net.bytebuddy.pool.TypePool; +import net.bytebuddy.utility.OpenedClassReader; + +/** + * Wraps each {@code INVOKESTATIC java/lang/Thread.sleep(...)V} and {@code INVOKEVIRTUAL + * java/util/concurrent/TimeUnit.sleep(J)V} call site in a TaskBlock capture/finish pair with a + * try-finally so the {@code finish} runs even when the sleep throws {@code InterruptedException}. + * + *

For {@code Thread.sleep(J)V} the emitted shape at each call site is: + * + * 

+ *   ; incoming stack: [..., millis]
+ *   LSTORE  <tmpLong>                            ; cache the arg
+ *   INVOKESTATIC TaskBlockHelper.captureForSleep()State
+ *   ASTORE  <state>
+ *   tryStart:
+ *   LLOAD   <tmpLong>
+ *   INVOKESTATIC Thread.sleep(J)V                     ; the original call
+ *   ALOAD   <state>
+ *   INVOKESTATIC TaskBlockHelper.finish(State)V
+ *   GOTO end
+ *   tryEnd / handler:
+ *   ASTORE  <throwable>
+ *   ALOAD   <state>
+ *   INVOKESTATIC TaskBlockHelper.finish(State)V
+ *   ALOAD   <throwable>
+ *   ATHROW
+ *   end:
+ *
+ * + *

For {@code Thread.sleep(JI)V} the incoming stack is {@code [..., millis, nanos]} (int on top); + * ISTORE pops first, LSTORE second, and the load sequence mirrors that. + * + *

For {@code Thread.sleep(java.time.Duration)V} (JDK 19+) the incoming stack is {@code [..., + * duration]} (one object reference on top); ASTORE pops the reference and ALOAD re-pushes it before + * the original call. The {@code Duration} overload cannot be relied upon to delegate to + * {@code sleep(J,I)V} at the bytecode level — the internal delegation call lives inside {@code + * java.lang.Thread} which is excluded from instrumentation. Each {@code Thread.sleep(Duration)} + * call site in user code is therefore wrapped directly. + */ +final class ThreadSleepCallSiteMethodVisitor extends LocalVariablesSorter { + + static final String THREAD_INTERNAL = "java/lang/Thread"; + static final String TIME_UNIT_INTERNAL = "java/util/concurrent/TimeUnit"; + static final String STATE_INTERNAL = + "datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper$State"; + static final String STATE_DESC = "L" + STATE_INTERNAL + ";"; + static final String TASK_BLOCK_HELPER = + "datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper"; + static final String CAPTURE_FOR_SLEEP_DESC = "()" + STATE_DESC; + static final String FINISH_DESC = "(" + STATE_DESC + ")V"; + + static final String SLEEP_J_DESC = "(J)V"; + static final String SLEEP_JI_DESC = "(JI)V"; + static final String SLEEP_DURATION_DESC = "(Ljava/time/Duration;)V"; + + private static final Type LONG_TYPE = Type.LONG_TYPE; + private static final Type INT_TYPE = Type.INT_TYPE; + private static final Type DURATION_TYPE = Type.getObjectType("java/time/Duration"); + private static final Type TIME_UNIT_TYPE = Type.getObjectType(TIME_UNIT_INTERNAL); + private static final Type STATE_TYPE = Type.getObjectType(STATE_INTERNAL); + private static final Type THROWABLE_TYPE = Type.getObjectType("java/lang/Throwable"); + private final TypePool typePool; + + ThreadSleepCallSiteMethodVisitor( + final int access, + final String descriptor, + final MethodVisitor delegate, + final TypePool typePool) { + super(OpenedClassReader.ASM_API, access, descriptor, delegate); + this.typePool = typePool; + } + + @Override + public void visitMethodInsn( + final int opcode, + final String owner, + final String name, + final String descriptor, + final boolean isInterface) { + boolean supported = + ThreadSleepCallSiteMatcher.isSupported(opcode, owner, name, descriptor, typePool); + if (supported && opcode == Opcodes.INVOKESTATIC) { + if (SLEEP_DURATION_DESC.equals(descriptor)) { + emitWrappedDurationSleepCall(owner, name, isInterface); + } else { + emitWrappedSleepCall(owner, name, descriptor, isInterface); + } + return; + } + if (supported) { + emitWrappedTimeUnitSleepCall(owner, name, descriptor, isInterface); + return; + } + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + + private void emitWrappedSleepCall( + final String owner, final String name, final String descriptor, final boolean isInterface) { + final boolean isJI = SLEEP_JI_DESC.equals(descriptor); + final int tmpLong = newLocal(LONG_TYPE); + final int tmpInt = isJI ? newLocal(INT_TYPE) : -1; + final int stateLocal = newLocal(STATE_TYPE); + final int throwableLocal = newLocal(THROWABLE_TYPE); + + final Label tryStart = new Label(); + final Label tryEnd = new Label(); + final Label handler = new Label(); + final Label end = new Label(); + + super.visitTryCatchBlock(tryStart, tryEnd, handler, null); + + // Pop args into locals in reverse stack order (top first). + if (isJI) { + visitNewLocalVarInsn(Opcodes.ISTORE, tmpInt); + } + visitNewLocalVarInsn(Opcodes.LSTORE, tmpLong); + + // captureForSleep() is null whenever the integration or native entry rejects the interval. + super.visitMethodInsn( + Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "captureForSleep", CAPTURE_FOR_SLEEP_DESC, false); + visitNewLocalVarInsn(Opcodes.ASTORE, stateLocal); + + // Protected region: the original Thread.sleep call. + super.visitLabel(tryStart); + visitNewLocalVarInsn(Opcodes.LLOAD, tmpLong); + if (isJI) { + visitNewLocalVarInsn(Opcodes.ILOAD, tmpInt); + } + super.visitMethodInsn(Opcodes.INVOKESTATIC, owner, name, descriptor, isInterface); + visitNewLocalVarInsn(Opcodes.ALOAD, stateLocal); + super.visitMethodInsn(Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "finish", FINISH_DESC, false); + super.visitJumpInsn(Opcodes.GOTO, end); + + // Exception handler: finish first, then rethrow. finish() tolerates null State, so the no-op + // case is harmless. + super.visitLabel(tryEnd); + super.visitLabel(handler); + visitNewLocalVarInsn(Opcodes.ASTORE, throwableLocal); + visitNewLocalVarInsn(Opcodes.ALOAD, stateLocal); + super.visitMethodInsn(Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "finish", FINISH_DESC, false); + visitNewLocalVarInsn(Opcodes.ALOAD, throwableLocal); + super.visitInsn(Opcodes.ATHROW); + + super.visitLabel(end); + } + + /** + * Emits the wrapped form for {@code TimeUnit.sleep(long)}. The incoming stack has {@code [..., + * timeUnit, timeout]}; the long argument is popped first, then the receiver reference. + */ + private void emitWrappedTimeUnitSleepCall( + final String owner, final String name, final String descriptor, final boolean isInterface) { + final int tmpLong = newLocal(LONG_TYPE); + final int tmpTimeUnit = newLocal(TIME_UNIT_TYPE); + final int stateLocal = newLocal(STATE_TYPE); + final int throwableLocal = newLocal(THROWABLE_TYPE); + + final Label tryStart = new Label(); + final Label tryEnd = new Label(); + final Label handler = new Label(); + final Label end = new Label(); + + super.visitTryCatchBlock(tryStart, tryEnd, handler, null); + + visitNewLocalVarInsn(Opcodes.LSTORE, tmpLong); + visitNewLocalVarInsn(Opcodes.ASTORE, tmpTimeUnit); + + super.visitMethodInsn( + Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "captureForSleep", CAPTURE_FOR_SLEEP_DESC, false); + visitNewLocalVarInsn(Opcodes.ASTORE, stateLocal); + + super.visitLabel(tryStart); + visitNewLocalVarInsn(Opcodes.ALOAD, tmpTimeUnit); + visitNewLocalVarInsn(Opcodes.LLOAD, tmpLong); + super.visitMethodInsn(Opcodes.INVOKEVIRTUAL, owner, name, descriptor, isInterface); + visitNewLocalVarInsn(Opcodes.ALOAD, stateLocal); + super.visitMethodInsn(Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "finish", FINISH_DESC, false); + super.visitJumpInsn(Opcodes.GOTO, end); + + super.visitLabel(tryEnd); + super.visitLabel(handler); + visitNewLocalVarInsn(Opcodes.ASTORE, throwableLocal); + visitNewLocalVarInsn(Opcodes.ALOAD, stateLocal); + super.visitMethodInsn(Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "finish", FINISH_DESC, false); + visitNewLocalVarInsn(Opcodes.ALOAD, throwableLocal); + super.visitInsn(Opcodes.ATHROW); + + super.visitLabel(end); + } + + /** + * Emits the wrapped form for {@code Thread.sleep(java.time.Duration)V}. The incoming stack has + * one object reference (the {@code Duration}); we ASTORE it, call captureForSleep, then ALOAD + * before the original call, mirroring the LSTORE/LLOAD pattern used for primitive overloads. + */ + private void emitWrappedDurationSleepCall( + final String owner, final String name, final boolean isInterface) { + final int tmpDuration = newLocal(DURATION_TYPE); + final int stateLocal = newLocal(STATE_TYPE); + final int throwableLocal = newLocal(THROWABLE_TYPE); + + final Label tryStart = new Label(); + final Label tryEnd = new Label(); + final Label handler = new Label(); + final Label end = new Label(); + + super.visitTryCatchBlock(tryStart, tryEnd, handler, null); + + // Pop the Duration reference into a local. + visitNewLocalVarInsn(Opcodes.ASTORE, tmpDuration); + + // captureForSleep() is null whenever the integration or native entry rejects the interval. + super.visitMethodInsn( + Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "captureForSleep", CAPTURE_FOR_SLEEP_DESC, false); + visitNewLocalVarInsn(Opcodes.ASTORE, stateLocal); + + // Protected region: the original Thread.sleep(Duration) call. + super.visitLabel(tryStart); + visitNewLocalVarInsn(Opcodes.ALOAD, tmpDuration); + super.visitMethodInsn(Opcodes.INVOKESTATIC, owner, name, SLEEP_DURATION_DESC, isInterface); + visitNewLocalVarInsn(Opcodes.ALOAD, stateLocal); + super.visitMethodInsn(Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "finish", FINISH_DESC, false); + super.visitJumpInsn(Opcodes.GOTO, end); + + // Exception handler: finish first, then rethrow. + super.visitLabel(tryEnd); + super.visitLabel(handler); + visitNewLocalVarInsn(Opcodes.ASTORE, throwableLocal); + visitNewLocalVarInsn(Opcodes.ALOAD, stateLocal); + super.visitMethodInsn(Opcodes.INVOKESTATIC, TASK_BLOCK_HELPER, "finish", FINISH_DESC, false); + visitNewLocalVarInsn(Opcodes.ALOAD, throwableLocal); + super.visitInsn(Opcodes.ATHROW); + + super.visitLabel(end); + } + + private void visitNewLocalVarInsn(final int opcode, final int local) { + mv.visitVarInsn(opcode, local); + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepProfilingInstrumentation.java new file mode 100644 index 00000000000..fec7f1744d4 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepProfilingInstrumentation.java @@ -0,0 +1,92 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.threadsleep; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.nameStartsWith; +import static net.bytebuddy.matcher.ElementMatchers.not; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Config; +import datadog.trace.api.profiling.TaskBlockInstrumentationConfig; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * Bracket {@code Thread.sleep} call sites at class-load time so a {@code datadog.TaskBlock} JFR + * event covers the sleep interval. + * + *

Why caller-site rewriting rather than {@code @Advice} on {@code Thread.sleep} itself: the JDK + * implements sleep independently of {@code Object.wait()}, so JVMTI {@code MonitorWait}/{@code + * MonitorWaited} callbacks do not bracket it. The native wallprecheck OS-thread-state filter can + * suppress {@code SIGVTALRM} for sleeping threads (when {@code wallprecheck=true}), but it does not + * emit a {@code datadog.TaskBlock} event. Rewriting application call sites provides the missing + * interval without transforming {@code java.lang.Thread}. + * + *

Coverage is purely opt-in by the user's bytecode: any supported {@code Thread.sleep(...)} or + * {@code TimeUnit.sleep(long)} call site in a non-JDK class is wrapped. Reflection-driven sleeps + * and JNI-driven sleeps remain uncovered (intentional: out-of-band call paths). + * + *

Active on every JDK when enabled via {@code profiling.ddprof.wall.precheck=true} (opt-in; + * default is off). The native JVMTI monitor callbacks cover {@code Object.wait()} and synchronized + * contention but not {@code Thread.sleep}, so sleep coverage is provided exclusively by this + * call-site instrumentation. The helper synchronously brackets an eligible platform-thread sleep + * with native TaskBlock ownership. Native entry rejects traced and virtual threads, so Java does + * not retain span or carrier-thread state. + * + *

Performance note: matched classes are scanned cheaply first. The {@link + * net.bytebuddy.asm.AsmVisitorWrapper} and its {@code COMPUTE_FRAMES} cost are only attached when a + * supported sleep call site is found, or when scanning fails open. + */ +@AutoService(InstrumenterModule.class) +public class ThreadSleepProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasTypeAdvice { + + public ThreadSleepProfilingInstrumentation() { + super("thread-sleep"); + } + + @Override + public boolean isEnabled() { + return super.isEnabled() + && TaskBlockInstrumentationConfig.isEnabled(Config.get(), ConfigProvider.getInstance()); + } + + @Override + public String hierarchyMarkerType() { + // null = no specific marker type; match broadly across all user-loaded classes. + return null; + } + + @Override + public ElementMatcher hierarchyMatcher() { + // Match every loaded class - sleep call sites can appear anywhere. The per-method + // visitor is a pass-through for methods without supported sleep calls, so cost of + // inspecting irrelevant classes is bounded. + // + // JDK / agent / bytebuddy internals are excluded to avoid bootstrap re-entry and + // self-instrumentation; in particular Thread.sleep's own callers inside java.lang.* would + // create a class-load loop because TaskBlockHelper itself sits in agent-bootstrap. + // + // JDK-generated dynamic proxy classes do not contain Thread.sleep INVOKESTATIC call sites, so + // retaining jdk.proxy* would add class-load overhead with zero coverage benefit. + return not( + nameStartsWith("java.") + .or(nameStartsWith("javax.")) + .or(nameStartsWith("jdk.")) + .or(nameStartsWith("sun.")) + .or(nameStartsWith("com.sun.")) + .or(nameStartsWith("datadog.")) + .or(nameStartsWith("net.bytebuddy."))); + } + + @Override + public void typeAdvice(TypeTransformer transformer) { + transformer.applyAdvice( + (builder, typeDescription, classLoader, module, pd) -> + ThreadSleepScanner.containsThreadSleepCallSite(classLoader, typeDescription) + ? builder.visit(new ThreadSleepRewritingVisitor()) + : builder); + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepRewritingVisitor.java b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepRewritingVisitor.java new file mode 100644 index 00000000000..496b03fa086 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepRewritingVisitor.java @@ -0,0 +1,76 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.threadsleep; + +import net.bytebuddy.asm.AsmVisitorWrapper; +import net.bytebuddy.description.field.FieldDescription; +import net.bytebuddy.description.field.FieldList; +import net.bytebuddy.description.method.MethodList; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.implementation.Implementation; +import net.bytebuddy.jar.asm.ClassReader; +import net.bytebuddy.jar.asm.ClassVisitor; +import net.bytebuddy.jar.asm.ClassWriter; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.pool.TypePool; +import net.bytebuddy.utility.OpenedClassReader; + +/** + * ASM visitor wrapper that wraps each supported {@code Thread.sleep(...)} and {@code + * TimeUnit.sleep(long)} call site in a TaskBlock capture/finish pair. + * + *

{@code COMPUTE_FRAMES} is requested via {@link #mergeWriter} so ASM recomputes stack-map + * frames after the added synthetic locals and try-finally handler. We pair it with {@code + * ClassReader.EXPAND_FRAMES} on the reader side — without that, ASM refuses to recompute frames on + * class files that already carry stack maps (Java 7+, bytecode v51+), throwing {@code + * IllegalStateException: ClassReader.accept() should be called with EXPAND_FRAMES flag}. + */ +public final class ThreadSleepRewritingVisitor implements AsmVisitorWrapper { + + @Override + public int mergeWriter(final int flags) { + return flags | ClassWriter.COMPUTE_FRAMES; + } + + @Override + public int mergeReader(final int flags) { + return flags | ClassReader.EXPAND_FRAMES; + } + + @Override + public ClassVisitor wrap( + final TypeDescription instrumentedType, + final ClassVisitor classVisitor, + final Implementation.Context implementationContext, + final TypePool typePool, + final FieldList fields, + final MethodList methods, + final int writerFlags, + final int readerFlags) { + return new ThreadSleepClassVisitor(classVisitor, typePool); + } + + /** Per-class visitor: dispatches per-method rewriting. */ + static final class ThreadSleepClassVisitor extends ClassVisitor { + private final TypePool typePool; + + ThreadSleepClassVisitor(final ClassVisitor cv, final TypePool typePool) { + super(OpenedClassReader.ASM_API, cv); + this.typePool = typePool; + } + + @Override + public MethodVisitor visitMethod( + final int access, + final String name, + final String descriptor, + final String signature, + final String[] exceptions) { + final MethodVisitor delegate = + super.visitMethod(access, name, descriptor, signature, exceptions); + if (delegate == null) { + return null; + } + return new ThreadSleepCallSiteMethodVisitor(access, descriptor, delegate, typePool); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepScanner.java b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepScanner.java new file mode 100644 index 00000000000..a1d585efe3a --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/main/java/datadog/trace/instrumentation/threadsleep/ThreadSleepScanner.java @@ -0,0 +1,80 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.threadsleep; + +import java.io.InputStream; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.jar.asm.ClassReader; +import net.bytebuddy.jar.asm.ClassVisitor; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.utility.OpenedClassReader; + +/** + * Scans class bytecode to determine whether a class contains at least one {@code Thread.sleep} or + * {@code TimeUnit.sleep} call site, without triggering the full {@code COMPUTE_FRAMES} analysis. + * + *

Used by {@link ThreadSleepProfilingInstrumentation} to avoid attaching {@link + * ThreadSleepRewritingVisitor} (and its {@code COMPUTE_FRAMES} cost) to classes that contain no + * sleep call sites. + * + *

Fails open (returns {@code true}) on any error so the transformation still runs when bytes + * cannot be read — preserving the pre-change safety guarantee. + */ +public final class ThreadSleepScanner { + + private ThreadSleepScanner() {} + + /** + * Returns {@code true} if the class identified by {@code typeDescription} contains at least one + * {@code Thread.sleep} or {@code TimeUnit.sleep} call site, {@code false} if it provably does + * not. + */ + public static boolean containsThreadSleepCallSite( + ClassLoader classLoader, TypeDescription typeDescription) { + if (classLoader == null) { + // Bootstrap classloader — type matcher already excludes java.* but guard defensively. + return true; + } + String resource = typeDescription.getInternalName() + ".class"; + try (InputStream is = classLoader.getResourceAsStream(resource)) { + if (is == null) { + // Runtime-generated class (dynamic proxy, ASM-generated) — no bytes on classpath. + return true; + } + return scan(new ClassReader(is)); + } catch (Exception e) { + return true; + } + } + + /** Package-private for unit testing. */ + static boolean scan(ClassReader classReader) { + SleepDetector detector = new SleepDetector(); + classReader.accept(detector, ClassReader.SKIP_FRAMES | ClassReader.SKIP_DEBUG); + return detector.found; + } + + private static final class SleepDetector extends ClassVisitor { + + boolean found = false; + + SleepDetector() { + super(OpenedClassReader.ASM_API); + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String desc, String sig, String[] exceptions) { + if (found) { + return null; // short-circuit: skip remaining methods entirely + } + return new MethodVisitor(OpenedClassReader.ASM_API) { + @Override + public void visitMethodInsn( + int opcode, String owner, String mName, String mDesc, boolean itf) { + if (found) return; + found = ThreadSleepCallSiteMatcher.isPotentiallySupported(opcode, owner, mName, mDesc); + } + }; + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/test/java/datadog/trace/instrumentation/threadsleep/ThreadSleepRewritingVisitorTest.java b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/test/java/datadog/trace/instrumentation/threadsleep/ThreadSleepRewritingVisitorTest.java new file mode 100644 index 00000000000..03dc89dbcc3 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/test/java/datadog/trace/instrumentation/threadsleep/ThreadSleepRewritingVisitorTest.java @@ -0,0 +1,629 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.threadsleep; + +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.CAPTURE_FOR_SLEEP_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.FINISH_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.SLEEP_DURATION_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.SLEEP_JI_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.SLEEP_J_DESC; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.TASK_BLOCK_HELPER; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.THREAD_INTERNAL; +import static datadog.trace.instrumentation.threadsleep.ThreadSleepCallSiteMethodVisitor.TIME_UNIT_INTERNAL; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import net.bytebuddy.jar.asm.ClassReader; +import net.bytebuddy.jar.asm.ClassVisitor; +import net.bytebuddy.jar.asm.ClassWriter; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.jar.asm.Opcodes; +import net.bytebuddy.pool.TypePool; +import net.bytebuddy.utility.OpenedClassReader; +import org.junit.jupiter.api.Test; + +/** + * ASM-level shape tests for {@link ThreadSleepRewritingVisitor}. Verifies that every {@code + * INVOKESTATIC Thread.sleep} call site is wrapped in a {@code captureForSleep} / {@code finish} + * pair, that the original args are preserved, and that the protected region covers both the normal + * and exceptional exit paths. + */ +class ThreadSleepRewritingVisitorTest { + + @Test + void singleSleepJ_isWrappedWithCaptureAndFinishAndPreservesArg() throws IOException { + List insns = rewriteAndScan(SingleSleepJFixture.class, "doSleep"); + + int sleepIdx = indexOfThreadSleep(insns, SLEEP_J_DESC); + assertTrue(sleepIdx >= 0, "expected INVOKESTATIC Thread.sleep(J)V to be present"); + + int captureIdx = indexOfInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC); + assertTrue( + captureIdx >= 0 && captureIdx < sleepIdx, "captureForSleep must precede Thread.sleep"); + + int finishAfterSleep = nextInvokeStatic(insns, sleepIdx + 1, "finish", FINISH_DESC); + assertTrue( + finishAfterSleep > sleepIdx, + "finish call must follow Thread.sleep on the normal exit path"); + + // The wrapped form must also have a second finish on the exception-handler path so finish + // runs when Thread.sleep throws InterruptedException. + int finishCount = countInvokeStatic(insns, "finish", FINISH_DESC); + assertEquals( + 2, + finishCount, + "expected two finish calls (normal exit + exception handler) per sleep site"); + + // The original LSTORE/LLOAD round-trip is necessary so the long arg is preserved across the + // injected captureForSleep call. + assertTrue(countOpcode(insns, Opcodes.LSTORE) >= 1, "expected LSTORE for cached millis"); + assertTrue(countOpcode(insns, Opcodes.LLOAD) >= 1, "expected LLOAD to re-push millis"); + + int aThrowCount = countOpcode(insns, Opcodes.ATHROW); + assertTrue( + aThrowCount >= 1, + "expected ATHROW in synthetic exception handler to rethrow the caught Throwable"); + } + + @Test + void singleSleepJI_isWrappedAndArgOrderPreserved() throws IOException { + List insns = rewriteAndScan(SingleSleepJIFixture.class, "doSleep"); + + int sleepIdx = indexOfThreadSleep(insns, SLEEP_JI_DESC); + assertTrue(sleepIdx >= 0, "expected INVOKESTATIC Thread.sleep(JI)V to be present"); + + // The (JI)V overload requires ISTORE (top of stack: int) before LSTORE (next: long). + assertTrue(countOpcode(insns, Opcodes.ISTORE) >= 1, "expected ISTORE for cached nanos arg"); + assertTrue(countOpcode(insns, Opcodes.LSTORE) >= 1, "expected LSTORE for cached millis arg"); + + // Stack-rebuild before the sleep call: LLOAD then ILOAD. + int lloadIdx = previousOpcode(insns, sleepIdx, Opcodes.LLOAD); + int iloadIdx = previousOpcode(insns, sleepIdx, Opcodes.ILOAD); + assertTrue( + lloadIdx >= 0 && iloadIdx >= 0 && lloadIdx < iloadIdx, + "LLOAD must precede ILOAD when rebuilding the (long, int) args"); + } + + @Test + void multipleSleepSites_eachReceiveIndependentWrap() throws IOException { + List insns = rewriteAndScan(MultipleSleepFixture.class, "doWork"); + + int sleepCount = countThreadSleep(insns); + assertEquals(3, sleepCount, "fixture should have three Thread.sleep call sites"); + + int captureCount = countInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC); + int finishCount = countInvokeStatic(insns, "finish", FINISH_DESC); + assertEquals(3, captureCount, "expected one captureForSleep per sleep site"); + assertEquals( + 6, finishCount, "expected two finish calls per sleep site (normal + handler) = 6 total"); + } + + @Test + void otherInvokeStatic_isNotRewritten() throws IOException { + List insns = rewriteAndScan(OtherInvokeFixture.class, "doWork"); + assertEquals( + 0, + countInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC), + "non-Thread.sleep INVOKESTATIC sites must be left alone"); + assertEquals( + 0, + countInvokeStatic(insns, "finish", FINISH_DESC), + "non-Thread.sleep INVOKESTATIC sites must be left alone"); + } + + @Test + void singleSleepDuration_isWrappedWithCaptureAndFinish() { + // The fixture bytecode is generated with ASM directly rather than compiled from a Java + // source inner class because Thread.sleep(Duration) was introduced in JDK 19; compiling a + // source reference to it would break builds on JDK 8/11/17. + List insns = rewriteAndScan(sleepDurationFixtureBytes(), "doSleep"); + + int sleepIdx = indexOfThreadSleep(insns, SLEEP_DURATION_DESC); + assertTrue(sleepIdx >= 0, "expected INVOKESTATIC Thread.sleep(Duration)V to be present"); + + int captureIdx = indexOfInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC); + assertTrue( + captureIdx >= 0 && captureIdx < sleepIdx, "captureForSleep must precede Thread.sleep"); + + int finishAfterSleep = nextInvokeStatic(insns, sleepIdx + 1, "finish", FINISH_DESC); + assertTrue( + finishAfterSleep > sleepIdx, + "finish call must follow Thread.sleep on the normal exit path"); + + int finishCount = countInvokeStatic(insns, "finish", FINISH_DESC); + assertEquals( + 2, + finishCount, + "expected two finish calls (normal exit + exception handler) per sleep site"); + + // The Duration reference must be stashed in a local (ASTORE) and reloaded (ALOAD) so the + // captureForSleep call does not drop it from the stack. + assertTrue(countOpcode(insns, Opcodes.ASTORE) >= 1, "expected ASTORE for cached Duration ref"); + assertTrue(countOpcode(insns, Opcodes.ALOAD) >= 1, "expected ALOAD to re-push Duration ref"); + + int aThrowCount = countOpcode(insns, Opcodes.ATHROW); + assertTrue( + aThrowCount >= 1, + "expected ATHROW in synthetic exception handler to rethrow the caught Throwable"); + } + + @Test + void classWithoutSleepCall_scanReturnsFalse() throws IOException { + assertFalse( + ThreadSleepScanner.scan(new ClassReader(classBytes(OtherInvokeFixture.class))), + "scanner must not flag a class with no Thread.sleep call sites"); + } + + @Test + void timeUnitSleep_isWrappedWithCaptureAndFinishAndPreservesReceiverAndArg() throws IOException { + List insns = rewriteAndScan(TimeUnitSleepFixture.class, "doSleep"); + + int sleepIdx = indexOfTimeUnitSleep(insns); + assertTrue(sleepIdx >= 0, "expected INVOKEVIRTUAL TimeUnit.sleep(J)V to be present"); + + int captureIdx = indexOfInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC); + assertTrue( + captureIdx >= 0 && captureIdx < sleepIdx, "captureForSleep must precede TimeUnit.sleep"); + + int aloadIdx = previousOpcode(insns, sleepIdx, Opcodes.ALOAD); + int lloadIdx = previousOpcode(insns, sleepIdx, Opcodes.LLOAD); + assertTrue(aloadIdx >= 0 && lloadIdx >= 0 && aloadIdx < lloadIdx); + + int finishCount = countInvokeStatic(insns, "finish", FINISH_DESC); + assertEquals( + 2, + finishCount, + "expected two finish calls (normal exit + exception handler) per sleep site"); + } + + @Test + void inheritedStaticSleepOwnedByThreadSubclassIsWrapped() throws IOException { + List insns = rewriteAndScan(ThreadSubclassSleepFixture.class, "doSleep"); + + assertEquals(1, countInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC)); + assertEquals(2, countInvokeStatic(insns, "finish", FINISH_DESC)); + } + + @Test + void unrelatedStaticSleepOwnerIsNotWrapped() { + List insns = + rewriteAndScan(staticSleepOwnerFixtureBytes("java/lang/String"), "doSleep"); + + assertEquals(0, countInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC)); + assertEquals(0, countInvokeStatic(insns, "finish", FINISH_DESC)); + } + + @Test + void unresolvedStaticSleepOwnerIsNotWrapped() { + List insns = + rewriteAndScan(staticSleepOwnerFixtureBytes("example/missing/ThreadLike"), "doSleep"); + + assertEquals(0, countInvokeStatic(insns, "captureForSleep", CAPTURE_FOR_SLEEP_DESC)); + assertEquals(0, countInvokeStatic(insns, "finish", FINISH_DESC)); + } + + @Test + void rewrittenLongAndLongIntSleepsExecuteNormally() throws Exception { + Class longFixture = loadRewritten(SingleSleepJFixture.class); + Method longSleep = longFixture.getDeclaredMethod("doSleep", long.class); + longSleep.setAccessible(true); + longSleep.invoke(null, 1L); + + Class longIntFixture = loadRewritten(SingleSleepJIFixture.class); + Method longIntSleep = longIntFixture.getDeclaredMethod("doSleep", long.class, int.class); + longIntSleep.setAccessible(true); + longIntSleep.invoke(null, 0L, 1); + } + + @Test + void rewrittenTimeUnitSleepExecutesNormally() throws Exception { + Class fixture = loadRewritten(TimeUnitSleepFixture.class); + Method sleep = fixture.getDeclaredMethod("doSleep", long.class); + sleep.setAccessible(true); + + sleep.invoke(null, 1L); + } + + @Test + void rewrittenDurationSleepExecutesNormallyWhenAvailable() throws Exception { + try { + Thread.class.getMethod("sleep", Duration.class); + } catch (NoSuchMethodException ignored) { + assumeTrue(false, "Thread.sleep(Duration) requires JDK 19 or later"); + } + + Class fixture = + new ByteArrayClassLoader(getClass().getClassLoader()) + .define(rewrite(sleepDurationFixtureBytes())); + Method sleep = fixture.getDeclaredMethod("doSleep", Duration.class); + sleep.setAccessible(true); + + sleep.invoke(null, Duration.ofMillis(1L)); + } + + @Test + void rewrittenSleepPreservesInterruptedException() throws Exception { + Class fixture = loadRewritten(SingleSleepJFixture.class); + Method sleep = fixture.getDeclaredMethod("doSleep", long.class); + sleep.setAccessible(true); + Thread.currentThread().interrupt(); + try { + InvocationTargetException expected = + assertThrows(InvocationTargetException.class, () -> sleep.invoke(null, 10L)); + assertTrue(expected.getCause() instanceof InterruptedException); + } finally { + Thread.interrupted(); + } + } + + // ------------------------------------------------------------------------------------------ + // Fixtures + // ------------------------------------------------------------------------------------------ + + @SuppressWarnings("unused") + static final class SingleSleepJFixture { + static void doSleep(long ms) throws InterruptedException { + Thread.sleep(ms); + } + } + + @SuppressWarnings("unused") + static final class SingleSleepJIFixture { + static void doSleep(long ms, int nanos) throws InterruptedException { + Thread.sleep(ms, nanos); + } + } + + @SuppressWarnings("unused") + static final class MultipleSleepFixture { + static void doWork() throws InterruptedException { + Thread.sleep(10); + Thread.sleep(20); + Thread.sleep(30, 100); + } + } + + @SuppressWarnings("unused") + static final class OtherInvokeFixture { + static long doWork() { + // Same shape (INVOKESTATIC with a long arg) as Thread.sleep but different owner/name — + // must not be rewritten. + return Math.abs(-42L); + } + } + + @SuppressWarnings("unused") + static final class TimeUnitSleepFixture { + static void doSleep(long timeout) throws InterruptedException { + TimeUnit.MILLISECONDS.sleep(timeout); + } + } + + static final class ThreadSubclass extends Thread {} + + static final class ThreadSubclassSleepFixture { + static void doSleep(long timeout) throws InterruptedException { + ThreadSubclass.sleep(timeout); + } + } + + // ------------------------------------------------------------------------------------------ + // Helpers (mirroring SynchronizedRewritingVisitorTest) + // ------------------------------------------------------------------------------------------ + + /** + * Generates the bytecode for a class equivalent to: + * + * 

+   *   static void doSleep(java.time.Duration d) throws InterruptedException {
+   *     Thread.sleep(d);
+   *   }
+   *
+ * + * using ASM directly so that the test compiles on JDK 8/11/17 even though {@code + * Thread.sleep(Duration)} was only introduced in JDK 19. + */ + private static byte[] sleepDurationFixtureBytes() { + ClassWriter cw = + new ClassWriter(ClassWriter.COMPUTE_FRAMES) { + @Override + protected String getCommonSuperClass(final String type1, final String type2) { + if (type1.equals(type2)) return type1; + try { + return super.getCommonSuperClass(type1, type2); + } catch (Exception ignored) { + return "java/lang/Object"; + } + } + }; + cw.visit( + Opcodes.V11, + Opcodes.ACC_FINAL | Opcodes.ACC_SUPER, + "datadog/trace/instrumentation/threadsleep/SleepDurationFixture", + null, + "java/lang/Object", + null); + MethodVisitor mv = + cw.visitMethod( + Opcodes.ACC_STATIC, + "doSleep", + "(Ljava/time/Duration;)V", + null, + new String[] {"java/lang/InterruptedException"}); + mv.visitCode(); + mv.visitVarInsn(Opcodes.ALOAD, 0); + mv.visitMethodInsn( + Opcodes.INVOKESTATIC, "java/lang/Thread", "sleep", "(Ljava/time/Duration;)V", false); + mv.visitInsn(Opcodes.RETURN); + mv.visitMaxs(1, 1); + mv.visitEnd(); + cw.visitEnd(); + return cw.toByteArray(); + } + + private static byte[] staticSleepOwnerFixtureBytes(String owner) { + ClassWriter writer = new ClassWriter(0); + writer.visit( + Opcodes.V11, + Opcodes.ACC_FINAL | Opcodes.ACC_SUPER, + "datadog/trace/instrumentation/threadsleep/StaticSleepOwnerFixture", + null, + "java/lang/Object", + null); + MethodVisitor method = + writer.visitMethod( + Opcodes.ACC_STATIC, + "doSleep", + "(J)V", + null, + new String[] {"java/lang/InterruptedException"}); + method.visitCode(); + method.visitVarInsn(Opcodes.LLOAD, 0); + method.visitMethodInsn(Opcodes.INVOKESTATIC, owner, "sleep", SLEEP_J_DESC, false); + method.visitInsn(Opcodes.RETURN); + method.visitMaxs(2, 2); + method.visitEnd(); + writer.visitEnd(); + return writer.toByteArray(); + } + + private static List rewriteAndScan(final Class cls, final String method) + throws IOException { + byte[] rewritten = rewrite(classBytes(cls)); + return scanMethod(rewritten, method); + } + + private static List rewriteAndScan( + final byte[] classBytes, final String method) { + byte[] rewritten = rewrite(classBytes); + return scanMethod(rewritten, method); + } + + private static byte[] rewrite(final byte[] in) { + ClassReader reader = new ClassReader(in); + ClassWriter writer = + new ClassWriter(reader, ClassWriter.COMPUTE_FRAMES) { + @Override + protected String getCommonSuperClass(final String type1, final String type2) { + if (type1.equals(type2)) return type1; + try { + return super.getCommonSuperClass(type1, type2); + } catch (Exception ignored) { + return "java/lang/Object"; + } + } + }; + ThreadSleepRewritingVisitor wrapper = new ThreadSleepRewritingVisitor(); + ClassVisitor cv = + new ThreadSleepRewritingVisitor.ThreadSleepClassVisitor( + writer, TypePool.Default.ofSystemLoader()); + reader.accept(cv, ClassReader.EXPAND_FRAMES); + assertEquals(ClassWriter.COMPUTE_FRAMES, wrapper.mergeWriter(0) & ClassWriter.COMPUTE_FRAMES); + assertEquals(ClassReader.EXPAND_FRAMES, wrapper.mergeReader(0) & ClassReader.EXPAND_FRAMES); + return writer.toByteArray(); + } + + private static byte[] classBytes(final Class cls) throws IOException { + String resource = cls.getName().replace('.', '/') + ".class"; + try (InputStream in = cls.getClassLoader().getResourceAsStream(resource)) { + if (in == null) { + throw new IOException("Could not find class resource: " + resource); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int n; + while ((n = in.read(buf)) > 0) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } + } + + private static Class loadRewritten(Class fixture) throws IOException { + return new ByteArrayClassLoader(fixture.getClassLoader()).define(rewrite(classBytes(fixture))); + } + + private static final class ByteArrayClassLoader extends ClassLoader { + private ByteArrayClassLoader(ClassLoader parent) { + super(parent); + } + + private Class define(byte[] bytes) { + return defineClass(null, bytes, 0, bytes.length); + } + } + + private static List scanMethod(final byte[] bytes, final String methodName) { + ClassReader reader = new ClassReader(bytes); + List result = new ArrayList<>(); + reader.accept( + new ClassVisitor(OpenedClassReader.ASM_API) { + @Override + public MethodVisitor visitMethod( + final int access, + final String name, + final String descriptor, + final String signature, + final String[] exceptions) { + if (!name.equals(methodName)) { + return null; + } + return new MethodVisitor(OpenedClassReader.ASM_API) { + @Override + public void visitInsn(final int opcode) { + result.add(InstructionRecord.simple(opcode)); + } + + @Override + public void visitVarInsn(final int opcode, final int var) { + InstructionRecord r = InstructionRecord.simple(opcode); + r.var = var; + result.add(r); + } + + @Override + public void visitMethodInsn( + final int opcode, + final String owner, + final String name, + final String descriptor, + final boolean isInterface) { + InstructionRecord r = InstructionRecord.simple(opcode); + r.methodOwner = owner; + r.methodName = name; + r.methodDescriptor = descriptor; + result.add(r); + } + }; + } + }, + ClassReader.SKIP_FRAMES); + return result; + } + + private static int indexOfThreadSleep(final List insns, final String desc) { + for (int i = 0; i < insns.size(); i++) { + InstructionRecord r = insns.get(i); + if (r.opcode == Opcodes.INVOKESTATIC + && THREAD_INTERNAL.equals(r.methodOwner) + && "sleep".equals(r.methodName) + && desc.equals(r.methodDescriptor)) { + return i; + } + } + return -1; + } + + private static int indexOfTimeUnitSleep(final List insns) { + for (int i = 0; i < insns.size(); i++) { + InstructionRecord r = insns.get(i); + if (r.opcode == Opcodes.INVOKEVIRTUAL + && TIME_UNIT_INTERNAL.equals(r.methodOwner) + && "sleep".equals(r.methodName) + && SLEEP_J_DESC.equals(r.methodDescriptor)) { + return i; + } + } + return -1; + } + + private static int countThreadSleep(final List insns) { + int n = 0; + for (InstructionRecord r : insns) { + if (r.opcode == Opcodes.INVOKESTATIC + && THREAD_INTERNAL.equals(r.methodOwner) + && "sleep".equals(r.methodName) + && (SLEEP_J_DESC.equals(r.methodDescriptor) + || SLEEP_JI_DESC.equals(r.methodDescriptor) + || SLEEP_DURATION_DESC.equals(r.methodDescriptor))) { + n++; + } + } + return n; + } + + private static int indexOfInvokeStatic( + final List insns, final String name, final String desc) { + for (int i = 0; i < insns.size(); i++) { + InstructionRecord r = insns.get(i); + if (r.opcode == Opcodes.INVOKESTATIC + && TASK_BLOCK_HELPER.equals(r.methodOwner) + && name.equals(r.methodName) + && desc.equals(r.methodDescriptor)) { + return i; + } + } + return -1; + } + + private static int nextInvokeStatic( + final List insns, final int from, final String name, final String desc) { + for (int i = from; i < insns.size(); i++) { + InstructionRecord r = insns.get(i); + if (r.opcode == Opcodes.INVOKESTATIC + && TASK_BLOCK_HELPER.equals(r.methodOwner) + && name.equals(r.methodName) + && desc.equals(r.methodDescriptor)) { + return i; + } + } + return -1; + } + + private static int countInvokeStatic( + final List insns, final String name, final String desc) { + int n = 0; + for (InstructionRecord r : insns) { + if (r.opcode == Opcodes.INVOKESTATIC + && TASK_BLOCK_HELPER.equals(r.methodOwner) + && name.equals(r.methodName) + && desc.equals(r.methodDescriptor)) { + n++; + } + } + return n; + } + + private static int countOpcode(final List insns, final int opcode) { + int n = 0; + for (InstructionRecord r : insns) { + if (r.opcode == opcode) n++; + } + return n; + } + + private static int previousOpcode( + final List insns, final int from, final int opcode) { + for (int i = from - 1; i >= 0; i--) { + if (insns.get(i).opcode == opcode) { + return i; + } + } + return -1; + } + + private static final class InstructionRecord { + int opcode; + int var; + String methodOwner; + String methodName; + String methodDescriptor; + + static InstructionRecord simple(final int opcode) { + InstructionRecord r = new InstructionRecord(); + r.opcode = opcode; + return r; + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/test/java/datadog/trace/instrumentation/threadsleep/ThreadSleepScannerTest.java b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/test/java/datadog/trace/instrumentation/threadsleep/ThreadSleepScannerTest.java new file mode 100644 index 00000000000..4e7372dfa87 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/thread-sleep/src/test/java/datadog/trace/instrumentation/threadsleep/ThreadSleepScannerTest.java @@ -0,0 +1,189 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.threadsleep; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.jar.asm.ClassReader; +import net.bytebuddy.jar.asm.ClassWriter; +import net.bytebuddy.jar.asm.MethodVisitor; +import net.bytebuddy.jar.asm.Opcodes; +import org.junit.jupiter.api.Test; + +class ThreadSleepScannerTest { + + @Test + void instrumentationEntrypoint_isPublicForAgentClassloaderAccess() throws NoSuchMethodException { + assertTrue(Modifier.isPublic(ThreadSleepScanner.class.getModifiers())); + Method method = + ThreadSleepScanner.class.getDeclaredMethod( + "containsThreadSleepCallSite", ClassLoader.class, TypeDescription.class); + assertTrue(Modifier.isPublic(method.getModifiers())); + } + + // --------------------------------------------------------------------------------- + // Positive cases: scan() must return true + // --------------------------------------------------------------------------------- + + @Test + void threadSleepJ_detected() throws IOException { + assertTrue( + ThreadSleepScanner.scan( + classReader(ThreadSleepRewritingVisitorTest.SingleSleepJFixture.class))); + } + + @Test + void threadSleepJI_detected() throws IOException { + assertTrue( + ThreadSleepScanner.scan( + classReader(ThreadSleepRewritingVisitorTest.SingleSleepJIFixture.class))); + } + + @Test + void threadSleepDuration_detected() { + assertTrue(ThreadSleepScanner.scan(new ClassReader(sleepDurationFixtureBytes()))); + } + + @Test + void timeUnitSleep_detected() throws IOException { + assertTrue( + ThreadSleepScanner.scan( + classReader(ThreadSleepRewritingVisitorTest.TimeUnitSleepFixture.class))); + } + + @Test + void inheritedStaticSleepCandidateDetectedForVisitorResolution() throws IOException { + assertTrue( + ThreadSleepScanner.scan( + classReader(ThreadSleepRewritingVisitorTest.ThreadSubclassSleepFixture.class))); + } + + @Test + void multipleSleepSites_detected() throws IOException { + assertTrue( + ThreadSleepScanner.scan( + classReader(ThreadSleepRewritingVisitorTest.MultipleSleepFixture.class))); + } + + // --------------------------------------------------------------------------------- + // Negative case: scan() must return false + // --------------------------------------------------------------------------------- + + @Test + void noSleepCall_notDetected() throws IOException { + assertFalse( + ThreadSleepScanner.scan( + classReader(ThreadSleepRewritingVisitorTest.OtherInvokeFixture.class))); + } + + // --------------------------------------------------------------------------------- + // Fail-open cases: containsThreadSleepCallSite() must return true + // --------------------------------------------------------------------------------- + + @Test + void nullClassLoader_returnsTrue() { + assertTrue( + ThreadSleepScanner.containsThreadSleepCallSite( + null, TypeDescription.ForLoadedType.of(Object.class))); + } + + @Test + void resourceNotFound_returnsTrue() { + ClassLoader emptyLoader = + new ClassLoader() { + @Override + public InputStream getResourceAsStream(String name) { + return null; + } + }; + assertTrue( + ThreadSleepScanner.containsThreadSleepCallSite( + emptyLoader, TypeDescription.ForLoadedType.of(Object.class))); + } + + @Test + void ioExceptionOnRead_returnsTrue() { + ClassLoader failingLoader = + new ClassLoader() { + @Override + public InputStream getResourceAsStream(String name) { + return new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("simulated I/O error"); + } + }; + } + }; + assertTrue( + ThreadSleepScanner.containsThreadSleepCallSite( + failingLoader, TypeDescription.ForLoadedType.of(Object.class))); + } + + // --------------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------------- + + private static ClassReader classReader(Class clazz) throws IOException { + String resource = clazz.getName().replace('.', '/') + ".class"; + try (InputStream in = clazz.getClassLoader().getResourceAsStream(resource)) { + assertNotNull(in, "Could not load test fixture: " + clazz.getName()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int n; + while ((n = in.read(buf)) > 0) { + out.write(buf, 0, n); + } + return new ClassReader(out.toByteArray()); + } + } + + /** + * Generates bytecode for a class with a single {@code Thread.sleep(java.time.Duration)} call site + * using ASM directly, so the test compiles on JDK 8/11/17 (the method was added in JDK 19). + */ + private static byte[] sleepDurationFixtureBytes() { + ClassWriter cw = + new ClassWriter(ClassWriter.COMPUTE_FRAMES) { + @Override + protected String getCommonSuperClass(final String type1, final String type2) { + if (type1.equals(type2)) return type1; + try { + return super.getCommonSuperClass(type1, type2); + } catch (Exception ignored) { + return "java/lang/Object"; + } + } + }; + cw.visit( + Opcodes.V11, + Opcodes.ACC_FINAL | Opcodes.ACC_SUPER, + "datadog/trace/instrumentation/threadsleep/SleepDurationScannerFixture", + null, + "java/lang/Object", + null); + MethodVisitor mv = + cw.visitMethod( + Opcodes.ACC_STATIC, + "doSleep", + "(Ljava/time/Duration;)V", + null, + new String[] {"java/lang/InterruptedException"}); + mv.visitCode(); + mv.visitVarInsn(Opcodes.ALOAD, 0); + mv.visitMethodInsn( + Opcodes.INVOKESTATIC, "java/lang/Thread", "sleep", "(Ljava/time/Duration;)V", false); + mv.visitInsn(Opcodes.RETURN); + mv.visitMaxs(1, 1); + mv.visitEnd(); + cw.visitEnd(); + return cw.toByteArray(); + } +} diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy index 370646949e7..35119f9e175 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package datadog.trace.agent.test import datadog.trace.api.EndpointTracker @@ -13,8 +14,11 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import java.util.concurrent.BlockingDeque +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.LinkedBlockingDeque +import java.util.Set import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong import static datadog.trace.api.profiling.Timer.TimerType.QUEUEING @@ -22,6 +26,12 @@ class TestProfilingContextIntegration implements ProfilingContextIntegration { final AtomicInteger attachments = new AtomicInteger() final AtomicInteger detachments = new AtomicInteger() final AtomicInteger counter = new AtomicInteger() + final AtomicInteger parkEnterCalls = new AtomicInteger() + final AtomicInteger parkExitCalls = new AtomicInteger() + final AtomicLong lastParkBlocker = new AtomicLong() + final AtomicLong lastUnblockingSpanId = new AtomicLong() + final Set parkExitThreads = ConcurrentHashMap.newKeySet() + volatile boolean acceptParkEntries = true final BlockingDeque closedTimings = new LinkedBlockingDeque<>() final Logger logger = LoggerFactory.getLogger(TestProfilingContextIntegration) @Override @@ -37,6 +47,26 @@ class TestProfilingContextIntegration implements ProfilingContextIntegration { void clear() { attachments.set(0) detachments.set(0) + parkEnterCalls.set(0) + parkExitCalls.set(0) + lastParkBlocker.set(0) + lastUnblockingSpanId.set(0) + parkExitThreads.clear() + acceptParkEntries = true + } + + @Override + boolean parkEnter() { + parkEnterCalls.incrementAndGet() + return acceptParkEntries + } + + @Override + void parkExit(long blocker, long unblockingSpanId) { + parkExitCalls.incrementAndGet() + lastParkBlocker.set(blocker) + lastUnblockingSpanId.set(unblockingSpanId) + parkExitThreads.add(Thread.currentThread()) } @Override diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/BlockingMixForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/BlockingMixForkedApp.java new file mode 100644 index 00000000000..5269f6ae3e5 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/BlockingMixForkedApp.java @@ -0,0 +1,117 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +public final class BlockingMixForkedApp { + private static final String THREAD_SLEEP = "blockingmix-sleep"; + private static final String THREAD_PARK = "blockingmix-park"; + private static final String THREAD_SYNC = "blockingmix-sync"; + + private static final int SLEEP_ITERATIONS = 20; + private static final int PARK_ITERATIONS = 20; + private static final int SYNC_ITERATIONS = 20; + private static final long PARK_NANOS = 50_000_000L; + private static final long SLEEP_MILLIS = 50L; + private static final long SYNC_HOLD_MILLIS = 50L; + + public static void main(String[] args) throws Exception { + BlockingMixForkedApp app = new BlockingMixForkedApp(); + runWorker(THREAD_SLEEP, app::runSleeps); + Thread.sleep(1500); + runWorker(THREAD_PARK, app::runParks); + runWorker(THREAD_SYNC, app::runSyncContention); + Thread.sleep(1500); + } + + private BlockingMixForkedApp() {} + + private static void runWorker(String name, InterruptibleTask task) throws Exception { + AtomicReference failure = new AtomicReference<>(); + Thread worker = + new Thread( + () -> { + try { + task.run(); + } catch (Throwable throwable) { + failure.set(throwable); + } + }, + name); + worker.start(); + worker.join(); + Throwable throwable = failure.get(); + if (throwable != null) { + if (throwable instanceof Exception) { + throw (Exception) throwable; + } + if (throwable instanceof Error) { + throw (Error) throwable; + } + throw new RuntimeException(throwable); + } + } + + private void runSleeps() throws InterruptedException { + for (int i = 0; i < SLEEP_ITERATIONS; i++) { + Thread.sleep(SLEEP_MILLIS); + } + } + + private void runParks() { + for (int i = 0; i < PARK_ITERATIONS; i++) { + long deadline = System.nanoTime() + PARK_NANOS; + long remaining; + while ((remaining = deadline - System.nanoTime()) > 0) { + LockSupport.parkNanos(remaining); + } + } + } + + private void runSyncContention() throws InterruptedException { + final Object lock = new Object(); + for (int i = 0; i < SYNC_ITERATIONS; i++) { + final CountDownLatch holderHasLock = new CountDownLatch(1); + final CountDownLatch holderMayRelease = new CountDownLatch(1); + Thread holder = + new Thread( + () -> { + synchronized (lock) { + holderHasLock.countDown(); + try { + holderMayRelease.await(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }, + "blockingmix-holder-" + i); + holder.setDaemon(true); + holder.start(); + holderHasLock.await(); + + new Thread( + () -> { + try { + Thread.sleep(SYNC_HOLD_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + holderMayRelease.countDown(); + }) + .start(); + synchronized (lock) { + // Acquire after contention. + } + holder.join(); + } + } + + @FunctionalInterface + private interface InterruptibleTask { + void run() throws Exception; + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/LockSupportTaskBlockForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/LockSupportTaskBlockForkedApp.java new file mode 100644 index 00000000000..dc8724e82fa --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/LockSupportTaskBlockForkedApp.java @@ -0,0 +1,127 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +/** Forked workload for platform and virtual-thread LockSupport TaskBlock smoke coverage. */ +public final class LockSupportTaskBlockForkedApp { + public static final String SPANLESS_PLATFORM_THREAD = "locksupport-spanless-platform"; + public static final String ACTIVE_PLATFORM_THREAD = "locksupport-active-platform"; + public static final String VIRTUAL_THREAD = "locksupport-virtual"; + public static final String SPANLESS_BLOCKER_MARKER = "LOCKSUPPORT_SPANLESS_BLOCKER="; + public static final String ACTIVE_BLOCKER_MARKER = "LOCKSUPPORT_ACTIVE_BLOCKER="; + public static final String VIRTUAL_BLOCKER_MARKER = "LOCKSUPPORT_VIRTUAL_BLOCKER="; + + private static final int PARK_ITERATIONS = 20; + private static final long PARK_NANOS = TimeUnit.MILLISECONDS.toNanos(40); + private static final Object SPANLESS_BLOCKER = new Object(); + private static final Object ACTIVE_BLOCKER = new Object(); + private static final Object VIRTUAL_BLOCKER = new Object(); + + private LockSupportTaskBlockForkedApp() {} + + /** Runs the smoke workload. */ + public static void main(String[] args) throws Exception { + printBlocker(SPANLESS_BLOCKER_MARKER, SPANLESS_BLOCKER); + printBlocker(ACTIVE_BLOCKER_MARKER, ACTIVE_BLOCKER); + printBlocker(VIRTUAL_BLOCKER_MARKER, VIRTUAL_BLOCKER); + Tracer tracer = GlobalTracer.get(); + runSpanlessPlatformWorker(tracer); + runActivePlatformWorker(tracer); + runVirtualWorker(); + Thread.sleep(1500L); + } + + private static void runSpanlessPlatformWorker(Tracer tracer) throws Exception { + CountDownLatch ready = new CountDownLatch(1); + Thread worker = + new Thread( + () -> { + ready.countDown(); + for (int i = 0; i < PARK_ITERATIONS; i++) { + LockSupport.parkNanos(SPANLESS_BLOCKER, PARK_NANOS); + } + }, + SPANLESS_PLATFORM_THREAD); + worker.start(); + if (!ready.await(5, TimeUnit.SECONDS)) { + throw new IllegalStateException("Spanless platform worker did not start"); + } + Thread.sleep(20L); + Span span = tracer.buildSpan("locksupport.unparker").start(); + try (Scope ignored = tracer.activateSpan(span)) { + LockSupport.unpark(worker); + } finally { + span.finish(); + } + join(worker); + } + + private static void runActivePlatformWorker(Tracer tracer) throws Exception { + Thread worker = + new Thread( + () -> { + Span span = tracer.buildSpan("locksupport.active").start(); + try (Scope ignored = tracer.activateSpan(span)) { + for (int i = 0; i < PARK_ITERATIONS; i++) { + LockSupport.parkNanos(ACTIVE_BLOCKER, PARK_NANOS); + } + } finally { + span.finish(); + } + }, + ACTIVE_PLATFORM_THREAD); + worker.start(); + join(worker); + } + + private static void runVirtualWorker() throws Exception { + Method startVirtualThread; + try { + startVirtualThread = Thread.class.getMethod("startVirtualThread", Runnable.class); + } catch (NoSuchMethodException unsupported) { + throw new IllegalStateException( + "Virtual-thread smoke coverage requires JDK 21+", unsupported); + } + Runnable workload = + () -> { + Thread.currentThread().setName(VIRTUAL_THREAD); + for (int i = 0; i < PARK_ITERATIONS; i++) { + LockSupport.parkNanos(VIRTUAL_BLOCKER, PARK_NANOS); + } + }; + Thread worker; + try { + worker = (Thread) startVirtualThread.invoke(null, workload); + } catch (InvocationTargetException error) { + Throwable cause = error.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new IllegalStateException(cause); + } + join(worker); + } + + private static void join(Thread worker) throws InterruptedException { + worker.join(TimeUnit.SECONDS.toMillis(10)); + if (worker.isAlive()) { + throw new IllegalStateException("Worker did not finish: " + worker.getName()); + } + } + + private static void printBlocker(String marker, Object blocker) { + System.out.println(marker + Integer.toUnsignedLong(System.identityHashCode(blocker))); + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/NativeIoTaskBlockForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/NativeIoTaskBlockForkedApp.java new file mode 100644 index 00000000000..92ce4897734 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/NativeIoTaskBlockForkedApp.java @@ -0,0 +1,53 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; + +/** Forked workload that blocks a platform thread in native accept and socket-read calls. */ +public final class NativeIoTaskBlockForkedApp { + private static final int ITERATIONS = 20; + + public static void main(String[] args) throws Exception { + Thread.currentThread().setName("native-io-spanless"); + for (int i = 0; i < ITERATIONS; i++) { + runBlockingAcceptAndRead(); + } + Thread.sleep(1500); + } + + private static void runBlockingAcceptAndRead() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { + Thread client = + new Thread( + () -> { + try { + Thread.sleep(50L); + try (Socket socket = + new Socket(InetAddress.getLoopbackAddress(), server.getLocalPort())) { + Thread.sleep(50L); + OutputStream output = socket.getOutputStream(); + output.write(1); + output.flush(); + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + }, + "native-io-client"); + client.start(); + + try (Socket accepted = server.accept()) { + accepted.setSoTimeout(5000); + InputStream input = accepted.getInputStream(); + if (input.read() != 1) { + throw new IllegalStateException("Unexpected socket payload"); + } + } + client.join(); + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/SynchronizedContentionForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/SynchronizedContentionForkedApp.java new file mode 100644 index 00000000000..bbef973747f --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/SynchronizedContentionForkedApp.java @@ -0,0 +1,100 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import java.util.concurrent.CountDownLatch; + +/** Forked workload for native monitor-contention TaskBlock smoke coverage. */ +public final class SynchronizedContentionForkedApp { + private static final int REPETITIONS = 10; + private static final Object BLOCK_LOCK = new Object(); + + public static void main(final String[] args) throws Exception { + SynchronizedContentionForkedApp app = new SynchronizedContentionForkedApp(); + for (int i = 0; i < REPETITIONS; i++) { + app.runBlockScenario(); + app.runInstanceMethodScenario(); + app.runStaticMethodScenario(); + } + Thread.sleep(1500); + } + + private final InstanceLockTarget instanceTarget = new InstanceLockTarget(); + + private void runBlockScenario() throws Exception { + CountDownLatch holderIn = new CountDownLatch(1); + Thread holder = + new Thread( + () -> { + synchronized (BLOCK_LOCK) { + holderIn.countDown(); + sleepWhileHoldingMonitor(); + } + }, + "sync-block-holder"); + holder.start(); + holderIn.await(); + + Thread contender = + new Thread( + () -> { + synchronized (BLOCK_LOCK) { + // entry-queue wait is the TaskBlock interval + } + }, + "sync-block-spanless"); + contender.start(); + contender.join(); + holder.join(); + } + + private void runInstanceMethodScenario() throws Exception { + CountDownLatch holderIn = new CountDownLatch(1); + Thread holder = new Thread(() -> instanceTarget.hold(holderIn), "sync-instance-holder"); + holder.start(); + holderIn.await(); + + Thread contender = new Thread(instanceTarget::contend, "sync-instance-spanless"); + contender.start(); + contender.join(); + holder.join(); + } + + private void runStaticMethodScenario() throws Exception { + CountDownLatch holderIn = new CountDownLatch(1); + Thread holder = new Thread(() -> StaticLockTarget.hold(holderIn), "sync-static-holder"); + holder.start(); + holderIn.await(); + + Thread contender = new Thread(StaticLockTarget::contend, "sync-static-spanless"); + contender.start(); + contender.join(); + holder.join(); + } + + private static void sleepWhileHoldingMonitor() { + try { + Thread.sleep(50L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + static final class InstanceLockTarget { + synchronized void hold(final CountDownLatch in) { + in.countDown(); + sleepWhileHoldingMonitor(); + } + + synchronized void contend() {} + } + + static final class StaticLockTarget { + static synchronized void hold(final CountDownLatch in) { + in.countDown(); + sleepWhileHoldingMonitor(); + } + + static synchronized void contend() {} + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/ThreadSleepTaskBlockForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/ThreadSleepTaskBlockForkedApp.java new file mode 100644 index 00000000000..4e3ee91a302 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/ThreadSleepTaskBlockForkedApp.java @@ -0,0 +1,99 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; + +/** Forked workload covering spanless, active-context, and virtual {@code Thread.sleep} calls. */ +public final class ThreadSleepTaskBlockForkedApp { + public static final String SPANLESS_PLATFORM_THREAD = "threadsleep-spanless"; + public static final String ACTIVE_PLATFORM_THREAD = "threadsleep-active"; + public static final String VIRTUAL_THREAD = "threadsleep-virtual"; + + private static final int SLEEP_ITERATIONS = 20; + private static final long SLEEP_MILLIS = 50L; + + private final Tracer tracer; + + private ThreadSleepTaskBlockForkedApp(Tracer tracer) { + this.tracer = tracer; + } + + public static void main(String[] args) throws Exception { + ThreadSleepTaskBlockForkedApp app = new ThreadSleepTaskBlockForkedApp(GlobalTracer.get()); + Thread spanless = new Thread(app::runSpanlessSleeps, SPANLESS_PLATFORM_THREAD); + Thread active = new Thread(app::runActiveSpanSleeps, ACTIVE_PLATFORM_THREAD); + spanless.start(); + active.start(); + Thread virtual = app.startVirtualWorkerIfSupported(); + spanless.join(); + active.join(); + if (virtual != null) { + virtual.join(); + } + Thread.sleep(1500L); + } + + private void runSpanlessSleeps() { + for (int i = 0; i < SLEEP_ITERATIONS; i++) { + sleep(); + } + } + + private void runActiveSpanSleeps() { + for (int i = 0; i < SLEEP_ITERATIONS; i++) { + Span span = tracer.buildSpan("threadsleep.active").start(); + try (Scope ignored = tracer.activateSpan(span)) { + sleep(); + } finally { + span.finish(); + } + } + } + + private Thread startVirtualWorkerIfSupported() throws Exception { + Method startVirtualThread; + try { + startVirtualThread = Thread.class.getMethod("startVirtualThread", Runnable.class); + } catch (NoSuchMethodException ignored) { + return null; + } + CountDownLatch named = new CountDownLatch(1); + Runnable task = + () -> { + await(named); + runSpanlessSleeps(); + }; + try { + Thread thread = (Thread) startVirtualThread.invoke(null, task); + thread.setName(VIRTUAL_THREAD); + named.countDown(); + return thread; + } catch (InvocationTargetException error) { + throw new IllegalStateException("Unable to start virtual sleep worker", error.getCause()); + } + } + + private static void sleep() { + try { + Thread.sleep(SLEEP_MILLIS); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(error); + } + } + + private static void await(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(error); + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/BlockingMixTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/BlockingMixTaskBlockProfilingTest.java new file mode 100644 index 00000000000..06ee51dbdb4 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/BlockingMixTaskBlockProfilingTest.java @@ -0,0 +1,400 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.agentShadowJar; +import static datadog.smoketest.SmokeTestUtils.buildDirectory; +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.SmokeTestUtils.javaPath; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; + +import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.test.util.Flaky; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.common.unit.UnitLookup; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** + * End-to-end mixed-blocking smoke / regression / demo test. Combines three roles in one fixture: + * + *
    + *
  1. Cross-workstream smoke: a single forked JVM under {@code -javaagent:} exercises + * {@code Thread.sleep}, {@code LockSupport.park*}, and native {@code synchronized} + * contention. Each population's events must be present. + *
  2. NoDoubleBracket: each blocking interval emits exactly one {@code + * datadog.TaskBlock} event. Overlapping TaskBlocks on one scenario thread point at a + * regression in the Java helper paths vs. the native JVMTI path. + *
  3. BlockingMix demo: the forked app is meant to be copy-pasted as a reproducer when + * triaging coverage issues. The runbook below lists JFR inspection commands and the expected + * scenario thread-name distribution. + *
+ * + *

Demo runbook (manual, off-CI)

+ * + * 
+ *   # 1. Run the forked app standalone to produce a JFR
+ *   ./gradlew :dd-smoke-tests:profiling-integration-tests:test \
+ *       --tests "*BlockingMixTaskBlockProfilingTest*" \
+ *       -Ddatadog.forkedTestRetainDumps=true
+ *
+ *   # 2. Inspect populations
+ *   jfr summary {dumpDir}/*.jfr | grep -E "datadog.TaskBlock|wall=" -A1
+ *
+ *   # 3. List per-scenario thread counts
+ *   jfr print --events "datadog.TaskBlock" {dumpDir}/*.jfr \
+ *       | grep -oE "eventThread = \\{[^}]+\\}" | sort | uniq -c
+ *
+ *   # 4. Expected (steady state):
+ *   #     N>=20 blockingmix-sleep
+ *   #     N=20  blockingmix-park
+ *   #     N=20  blockingmix-sync   (native JVMTI monitor callbacks)
+ *
+ *   # 5. Native counter snapshot:
+ *   jfr print --events "datadog.DatadogProfilerConfig" {dumpDir}/*.jfr
+ *
+ */ +@DisabledOnJ9 +@Flaky( + "TaskBlock/wall-clock sampler intermittently produces zero events across JDK versions; root cause is tracked separately") +final class BlockingMixTaskBlockProfilingTest { + + private static final byte[] JFR_MAGIC = new byte[] {'F', 'L', 'R', 0}; + private static final IAttribute SPAN_ID = attr("spanId", "spanId", "spanId", NUMBER); + private static final IAttribute LOCAL_ROOT_SPAN_ID = + attr("localRootSpanId", "localRootSpanId", "localRootSpanId", NUMBER); + private static final IAttribute START_TIME = + attr("startTime", "startTime", "startTime", NUMBER); + private static final IAttribute DURATION = + attr("duration", "duration", "duration", NUMBER); + private static final String THREAD_SLEEP = "blockingmix-sleep"; + private static final String THREAD_PARK = "blockingmix-park"; + private static final String THREAD_SYNC = "blockingmix-sync"; + + private static final Path LOG_FILE_BASE = + Paths.get( + buildDirectory(), + "reports", + "testProcess." + BlockingMixTaskBlockProfilingTest.class.getName()); + + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + Files.createDirectories(LOG_FILE_BASE); + logFilePath = + LOG_FILE_BASE.resolve( + testInfo.getTestMethod().map(method -> method.getName()).orElse("blockingMix") + + ".log"); + dumpDir = Files.createTempDirectory("dd-profiler-blockingmix-"); + } + + @AfterEach + void tearDown() throws IOException { + if (dumpDir != null && Files.exists(dumpDir)) { + Files.walk(dumpDir).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + + @Test + @DisplayName("Mixed sleep+park+sync workload emits one TaskBlock per blocking interval") + void mixedBlockingWorkloadEmitsExpectedPopulations() throws Exception { + Process targetProcess = createProcessBuilder().start(); + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + assumeDdprofNativeLibraryAvailable(); + + JfrStats stats = loadStats(); + + // ---- Smoke ----: every population must be present. + assertTrue( + stats.countByThread.getOrDefault(THREAD_SLEEP, 0L) > 0, + "Expected blockingmix.sleep TaskBlock events (thread-sleep call-site module); observed " + + stats.allCountByThread); + assertTrue( + stats.countByThread.getOrDefault(THREAD_PARK, 0L) > 0, + "Expected blockingmix.park TaskBlock events (existing lock-support module)"); + assertTrue( + stats.countByThread.getOrDefault(THREAD_SYNC, 0L) > 0, + "Expected blockingmix.sync TaskBlock events (native JVMTI monitor callbacks)"); + + // ---- NoDoubleBracket ----: no two TaskBlock events on the same thread with overlapping + // intervals for the same operation. This catches shifted start times as well as exact + // duplicates when Java and native paths both bracket one blocking interval. + assertFalse( + stats.hasOverlappingInterval(), + "Detected overlapping TaskBlock events on one scenario thread — double bracket regression. " + + "First overlap: " + + stats.firstOverlapDescription); + + // ---- Span context ----: all TaskBlock events in this workload must be spanless. + assertFalse( + stats.hasNonZeroSpanId, + "TaskBlock events from the mixed workload must all carry zero spanId"); + assertFalse( + stats.hasNonZeroLocalRootSpanId, + "TaskBlock events from the mixed workload must all carry zero localRootSpanId"); + + // ---- Health ----: no instrumentation classloading or rewrite failures in the forked log. + assertFalse( + logHasInstrumentationError(), + "Instrumentation produced classloading / rewrite errors in the forked log"); + } + + // ------------------------------------------------------------------------------------------ + // Process / JFR plumbing + // ------------------------------------------------------------------------------------------ + + private ProcessBuilder createProcessBuilder() { + String templateOverride = + BlockingMixTaskBlockProfilingTest.class + .getClassLoader() + .getResource("overrides.jfp") + .getFile(); + List command = + new ArrayList<>( + Arrays.asList( + javaPath(), + "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), + "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), + "-javaagent:" + agentShadowJar(), + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Ddd.service.name=smoke-test-blockingmix-taskblock", + "-Ddd.env=smoketest", + "-Ddd.version=99", + "-Ddd.profiling.enabled=true", + "-Ddd.profiling.ddprof.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_AUXILIARY_TYPE + "=async", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED + "=true", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_INTERVAL + "=10ms", + "-Ddd.profiling.agentless=false", + "-Ddd.profiling.start-delay=0", + "-Ddd." + ProfilingConfig.PROFILING_START_FORCE_FIRST + "=true", + "-Ddd.profiling.upload.period=1", + "-Ddd.profiling.hotspots.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES_SPAN_NAME_ENABLED + "=true", + "-Ddd.profiling.debug.dump_path=" + dumpDir, + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK + "=true", + "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", + "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug", + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UnlockCommercialFeatures", + "-XX:+FlightRecorder", + "-Ddd." + ProfilingConfig.PROFILING_TEMPLATE_OVERRIDE_FILE + "=" + templateOverride, + "-cp", + System.getProperty("java.class.path"), + com.datadog.smoketest.profiling.BlockingMixForkedApp.class.getName())); + if (System.getenv("TEST_LIBASYNC") != null) { + command.add( + command.size() - 3, + "-Ddd." + + ProfilingConfig.PROFILING_DATADOG_PROFILER_LIBPATH + + "=" + + System.getenv("TEST_LIBASYNC")); + } + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(new File(buildDirectory())); + processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")); + processBuilder.redirectErrorStream(true); + processBuilder.redirectOutput(ProcessBuilder.Redirect.to(logFilePath.toFile())); + return processBuilder; + } + + private JfrStats loadStats() throws Exception { + JfrStats stats = new JfrStats(); + List jfrFiles; + try (java.util.stream.Stream files = Files.walk(dumpDir)) { + jfrFiles = + files + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".jfr")) + .collect(Collectors.toList()); + } + int loaded = 0; + for (Path jfrFile : jfrFiles) { + IItemCollection events = tryLoadEvents(jfrFile); + if (events != null) { + stats.add(events); + loaded++; + } + } + if (loaded == 0 && !jfrFiles.isEmpty()) { + throw new RuntimeException( + "No JFR file in " + dumpDir + " could be parsed (tried " + jfrFiles.size() + " files)"); + } + return stats; + } + + private IItemCollection tryLoadEvents(Path path) { + try { + return JfrLoaderToolkit.loadEvents(path.toFile()); + } catch (Exception ignored) { + // fall through + } + try { + Path extracted = extractLastJfrStream(path); + if (!extracted.equals(path)) { + return JfrLoaderToolkit.loadEvents(extracted.toFile()); + } + } catch (Exception ignored) { + // fall through + } + return null; + } + + private Path extractLastJfrStream(Path path) throws IOException { + byte[] data = Files.readAllBytes(path); + int lastMagic = lastIndexOf(data, JFR_MAGIC); + if (lastMagic <= 0) { + return path; + } + Path extracted = dumpDir.resolve(path.getFileName() + ".ddprof.jfr"); + Files.write(extracted, Arrays.copyOfRange(data, lastMagic, data.length)); + return extracted; + } + + private static int lastIndexOf(byte[] data, byte[] needle) { + for (int i = data.length - needle.length; i >= 0; i--) { + boolean match = true; + for (int j = 0; j < needle.length; j++) { + if (data[i + j] != needle[j]) { + match = false; + break; + } + } + if (match) { + return i; + } + } + return -1; + } + + private boolean logHasInstrumentationError() throws IOException { + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + return log.contains("NoClassDefFoundError") + || log.contains("Failed to handle exception in instrumentation for"); + } + + private void assumeDdprofNativeLibraryAvailable() throws IOException { + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + assumeFalse( + log.contains("libjavaProfiler") && log.contains("not found on classpath"), + "ddprof native library is not available on this platform"); + } + + private static final class JfrStats { + final Map countByThread = new HashMap<>(); + final Map allCountByThread = new HashMap<>(); + boolean hasNonZeroSpanId; + boolean hasNonZeroLocalRootSpanId; + final Map> intervalsByThread = new HashMap<>(); + String firstOverlapDescription; + + void add(IItemCollection events) { + IItemCollection taskBlocks = events.apply(ItemFilters.type("datadog.TaskBlock")); + for (IItemIterable items : taskBlocks) { + IMemberAccessor span = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor root = LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor startTime = START_TIME.getAccessor(items.getType()); + IMemberAccessor duration = DURATION.getAccessor(items.getType()); + IMemberAccessor threadName = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + for (IItem item : items) { + String thread = threadName == null ? null : threadName.getMember(item); + allCountByThread.merge(String.valueOf(thread), 1L, Long::sum); + if (!THREAD_SLEEP.equals(thread) + && !THREAD_PARK.equals(thread) + && !THREAD_SYNC.equals(thread)) { + continue; + } + countByThread.merge(thread, 1L, Long::sum); + if (span != null) { + long spanId = span.getMember(item).longValue(); + hasNonZeroSpanId |= spanId != 0L; + } + if (root != null) { + long rootSpanId = root.getMember(item).longValue(); + hasNonZeroLocalRootSpanId |= rootSpanId != 0L; + } + if (startTime != null && duration != null && threadName != null) { + long startNanos = startTime.getMember(item).clampedLongValueIn(UnitLookup.EPOCH_NS); + long durationNanos = duration.getMember(item).clampedLongValueIn(UnitLookup.NANOSECOND); + intervalsByThread + .computeIfAbsent(thread, ignored -> new ArrayList<>()) + .add(new Interval(startNanos, saturatedAdd(startNanos, durationNanos))); + } + } + } + } + + private boolean hasOverlappingInterval() { + for (Map.Entry> entry : intervalsByThread.entrySet()) { + List intervals = entry.getValue(); + intervals.sort(Comparator.comparingLong(interval -> interval.startNanos)); + for (int i = 1; i < intervals.size(); i++) { + Interval previous = intervals.get(i - 1); + Interval current = intervals.get(i); + if (current.startNanos < previous.endNanos) { + firstOverlapDescription = + entry.getKey() + + " [" + + previous.startNanos + + "," + + previous.endNanos + + ") overlaps [" + + current.startNanos + + "," + + current.endNanos + + ")"; + return true; + } + } + } + return false; + } + + private static long saturatedAdd(long left, long right) { + if (right > 0L && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + return left + right; + } + } + + private static final class Interval { + private final long startNanos; + private final long endNanos; + + private Interval(long startNanos, long endNanos) { + this.startNanos = startNanos; + this.endNanos = endNanos; + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java new file mode 100644 index 00000000000..87c0f556770 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java @@ -0,0 +1,154 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; + +import com.datadog.smoketest.profiling.LockSupportTaskBlockForkedApp; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +@DisabledOnJ9 +final class LockSupportTaskBlockProfilingTest { + private static final IAttribute UNBLOCKING_SPAN_ID = + attr("unblockingSpanId", "unblockingSpanId", "unblockingSpanId", NUMBER); + + private Path logFilePath; + private Path dumpDir; + + @BeforeEach + void setUp(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + LockSupportTaskBlockProfilingTest.class, testInfo, "lockSupport"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-locksupport-"); + } + + @AfterEach + void cleanUp() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void onlySpanlessPlatformParksEmitTaskBlocks() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-locksupport-taskblock", + LockSupportTaskBlockForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + Stats stats = + new Stats( + markerValue(log, LockSupportTaskBlockForkedApp.SPANLESS_BLOCKER_MARKER), + markerValue(log, LockSupportTaskBlockForkedApp.ACTIVE_BLOCKER_MARKER), + markerValue(log, LockSupportTaskBlockForkedApp.VIRTUAL_BLOCKER_MARKER)); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + assertTrue(stats.spanlessPlatformCount > 0, "Expected spanless platform park TaskBlocks"); + assertEquals(0, stats.activePlatformCount, "Active-context parks must not emit TaskBlocks"); + assertEquals(0, stats.virtualCount, "Virtual-thread parks must not emit TaskBlocks"); + assertFalse(stats.spanlessHasContext, "Spanless park TaskBlocks must keep zero span context"); + assertTrue(stats.spanlessWithBlocker > 0, "Expected blocker identity on park TaskBlocks"); + assertTrue( + stats.spanlessWithUnblockingSpan > 0, "Expected best-effort unpark span attribution"); + assertFalse(stats.spanlessMissingThread, "TaskBlock events must resolve Event Thread"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, + "Failed to handle exception in instrumentation for java.util.concurrent.locks.LockSupport", + "Unexpected checked exception from ddprof TaskBlock hook"), + "LockSupport TaskBlock instrumentation failed"); + } + + private static long markerValue(String log, String marker) { + int start = log.indexOf(marker); + assertTrue(start >= 0, "Missing workload marker " + marker); + start += marker.length(); + int end = start; + while (end < log.length() && Character.isDigit(log.charAt(end))) { + end++; + } + return Long.parseLong(log.substring(start, end)); + } + + private static final class Stats { + private final long spanlessBlocker; + private final long activeBlocker; + private final long virtualBlocker; + private long spanlessPlatformCount; + private long activePlatformCount; + private long virtualCount; + private long spanlessWithBlocker; + private long spanlessWithUnblockingSpan; + private boolean spanlessHasContext; + private boolean spanlessMissingThread; + + private Stats(long spanlessBlocker, long activeBlocker, long virtualBlocker) { + this.spanlessBlocker = spanlessBlocker; + this.activeBlocker = activeBlocker; + this.virtualBlocker = virtualBlocker; + } + + private void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor threadAccessor = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + IMemberAccessor spanAccessor = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootAccessor = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blockerAccessor = BLOCKER.getAccessor(items.getType()); + IMemberAccessor unblockingAccessor = + UNBLOCKING_SPAN_ID.getAccessor(items.getType()); + for (IItem item : items) { + String thread = threadAccessor == null ? null : threadAccessor.getMember(item); + long blocker = blockerAccessor.getMember(item).longValue(); + if (blocker == spanlessBlocker) { + spanlessPlatformCount++; + spanlessHasContext |= + spanAccessor.getMember(item).longValue() != 0L + || rootAccessor.getMember(item).longValue() != 0L; + spanlessWithBlocker++; + spanlessWithUnblockingSpan += + unblockingAccessor.getMember(item).longValue() == 0L ? 0 : 1; + spanlessMissingThread |= + thread == null + || thread.isEmpty() + || !LockSupportTaskBlockForkedApp.SPANLESS_PLATFORM_THREAD.equals(thread); + } else if (blocker == activeBlocker) { + activePlatformCount++; + } else if (blocker == virtualBlocker) { + virtualCount++; + } + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/NativeIoTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/NativeIoTaskBlockProfilingTest.java new file mode 100644 index 00000000000..21f38c7316f --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/NativeIoTaskBlockProfilingTest.java @@ -0,0 +1,109 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** Linux smoke coverage for TaskBlocks emitted by native socket-I/O interposition. */ +@DisabledOnJ9 +@EnabledOnOs(OS.LINUX) +final class NativeIoTaskBlockProfilingTest { + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + NativeIoTaskBlockProfilingTest.class, testInfo, "nativeIo"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-nativeio-"); + } + + @AfterEach + void tearDown() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void blockingNativeSocketIoEmitsSpanlessTaskBlocks() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-nativeio-taskblock", + com.datadog.smoketest.profiling.NativeIoTaskBlockForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = new JfrStats(); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + + assertTrue(stats.count > 0, "Expected TaskBlocks from blocking native socket I/O"); + assertTrue(stats.nonZeroBlockerCount > 0, "Expected encoded native I/O blocker identity"); + assertFalse(stats.hasNonZeroSpanId, "Spanless native I/O TaskBlocks must keep spanId zero"); + assertFalse( + stats.hasNonZeroLocalRootSpanId, + "Spanless native I/O TaskBlocks must keep root spanId zero"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, + "native I/O hooks disabled", + "NoClassDefFoundError", + "Failed to handle exception"), + "native I/O TaskBlock path failed"); + } + + private static final class JfrStats { + long count; + long nonZeroBlockerCount; + boolean hasNonZeroSpanId; + boolean hasNonZeroLocalRootSpanId; + boolean hasMissingEventThread; + + void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor spanId = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootSpanId = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blocker = BLOCKER.getAccessor(items.getType()); + IMemberAccessor eventThread = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + for (IItem item : items) { + String thread = eventThread.getMember(item); + if (!"native-io-spanless".equals(thread)) { + continue; + } + count++; + nonZeroBlockerCount += blocker.getMember(item).longValue() == 0 ? 0 : 1; + hasNonZeroSpanId |= spanId.getMember(item).longValue() != 0; + hasNonZeroLocalRootSpanId |= rootSpanId.getMember(item).longValue() != 0; + hasMissingEventThread |= thread.isEmpty(); + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java new file mode 100644 index 00000000000..da0ce627b2c --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java @@ -0,0 +1,162 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; + +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** Smoke coverage for native {@code Object.wait} TaskBlock emission and context exclusion. */ +@DisabledOnJ9 +final class ObjectWaitTaskBlockProfilingTest { + private static final IAttribute UNBLOCKING_SPAN_ID = + attr("unblockingSpanId", "unblockingSpanId", "unblockingSpanId", NUMBER); + + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + ObjectWaitTaskBlockProfilingTest.class, testInfo, "objectWait"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-objectwait-"); + } + + @AfterEach + void tearDown() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void spanlessObjectWaitEmitsWhileTracedObjectWaitDoesNot() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-objectwait-taskblock", + ObjectWaitTaskBlockForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = new JfrStats(); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + + assertTrue(stats.spanlessCount > 0, "Expected Object.wait TaskBlocks outside trace context"); + assertTrue(stats.nonZeroBlockerCount > 0, "Expected monitor identity on Object.wait events"); + assertFalse(stats.hasNonZeroSpanId, "Spanless TaskBlocks must keep spanId zero"); + assertFalse(stats.hasNonZeroLocalRootSpanId, "Spanless TaskBlocks must keep root spanId zero"); + assertFalse(stats.hasNonZeroUnblockingSpanId, "Object.wait has no attributed notifier hook"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertFalse(stats.hasTracedThreadEvent, "Traced waits must not emit TaskBlock events"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, "NoClassDefFoundError", "Failed to handle exception in instrumentation"), + "Object.wait TaskBlock path failed"); + } + + public static final class ObjectWaitTaskBlockForkedApp { + private static final int ITERATIONS = 20; + private static final Object BLOCKER = new Object(); + + public static void main(String[] args) throws Exception { + Thread spanless = + new Thread(ObjectWaitTaskBlockForkedApp::waitRepeatedly, "objectwait-spanless"); + spanless.start(); + spanless.join(); + + Tracer tracer = GlobalTracer.get(); + Thread traced = + new Thread( + () -> { + Span span = tracer.buildSpan("objectwait.traced").start(); + try (Scope scope = tracer.activateSpan(span)) { + waitRepeatedly(); + } finally { + span.finish(); + } + }, + "objectwait-traced"); + traced.start(); + traced.join(); + Thread.sleep(1500); + } + + private static void waitRepeatedly() { + try { + for (int i = 0; i < ITERATIONS; i++) { + synchronized (BLOCKER) { + BLOCKER.wait(50L); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } + + private static final class JfrStats { + long spanlessCount; + long nonZeroBlockerCount; + boolean hasNonZeroSpanId; + boolean hasNonZeroLocalRootSpanId; + boolean hasNonZeroUnblockingSpanId; + boolean hasMissingEventThread; + boolean hasTracedThreadEvent; + + void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor spanId = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootSpanId = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blocker = BLOCKER.getAccessor(items.getType()); + IMemberAccessor unblockingSpanId = + UNBLOCKING_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor eventThread = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + for (IItem item : items) { + String thread = eventThread.getMember(item); + if ("objectwait-traced".equals(thread)) { + hasTracedThreadEvent = true; + } + if (!"objectwait-spanless".equals(thread)) { + continue; + } + spanlessCount++; + hasNonZeroSpanId |= spanId.getMember(item).longValue() != 0; + hasNonZeroLocalRootSpanId |= rootSpanId.getMember(item).longValue() != 0; + hasNonZeroUnblockingSpanId |= unblockingSpanId.getMember(item).longValue() != 0; + nonZeroBlockerCount += blocker.getMember(item).longValue() == 0 ? 0 : 1; + hasMissingEventThread |= thread.isEmpty(); + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/SynchronizedContentionProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/SynchronizedContentionProfilingTest.java new file mode 100644 index 00000000000..9b4f13b7a6b --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/SynchronizedContentionProfilingTest.java @@ -0,0 +1,118 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** Smoke coverage for native synchronized-contention TaskBlock emission. */ +@DisabledOnJ9 +final class SynchronizedContentionProfilingTest { + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + SynchronizedContentionProfilingTest.class, testInfo, "syncContention"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-synccontention-"); + } + + @AfterEach + void tearDown() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void spanlessSynchronizedContentionEmitsNativeTaskBlocks() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-synccontention-taskblock", + com.datadog.smoketest.profiling.SynchronizedContentionForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = new JfrStats(); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + + assertTrue(stats.blockScenarioCount > 0, "Expected synchronized-block TaskBlocks"); + assertTrue(stats.instanceMethodScenarioCount > 0, "Expected instance-method TaskBlocks"); + assertTrue(stats.staticMethodScenarioCount > 0, "Expected static-method TaskBlocks"); + assertFalse(stats.hasNonZeroSpanId, "Spanless TaskBlocks must keep spanId zero"); + assertFalse(stats.hasNonZeroLocalRootSpanId, "Spanless TaskBlocks must keep root spanId zero"); + assertTrue(stats.blockersWithNonZeroValue > 0, "Expected monitor identity on TaskBlocks"); + assertTrue(stats.distinctBlockerValues.size() > 1, "Expected per-monitor blocker identities"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, "NoClassDefFoundError", "Failed to handle exception", "VerifyError"), + "native synchronized-contention TaskBlock path failed"); + } + + private static final class JfrStats { + long blockScenarioCount; + long instanceMethodScenarioCount; + long staticMethodScenarioCount; + long blockersWithNonZeroValue; + final Set distinctBlockerValues = new HashSet<>(); + boolean hasNonZeroSpanId; + boolean hasNonZeroLocalRootSpanId; + boolean hasMissingEventThread; + + void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor spanId = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootSpanId = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blocker = BLOCKER.getAccessor(items.getType()); + IMemberAccessor eventThread = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + for (IItem item : items) { + String thread = eventThread.getMember(item); + if ("sync-block-spanless".equals(thread)) { + blockScenarioCount++; + } else if ("sync-instance-spanless".equals(thread)) { + instanceMethodScenarioCount++; + } else if ("sync-static-spanless".equals(thread)) { + staticMethodScenarioCount++; + } else { + continue; + } + hasNonZeroSpanId |= spanId.getMember(item).longValue() != 0; + hasNonZeroLocalRootSpanId |= rootSpanId.getMember(item).longValue() != 0; + long blockerValue = blocker.getMember(item).longValue(); + if (blockerValue != 0) { + blockersWithNonZeroValue++; + distinctBlockerValues.add(blockerValue); + } + hasMissingEventThread |= thread.isEmpty(); + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/TaskBlockProfilingTestSupport.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/TaskBlockProfilingTestSupport.java new file mode 100644 index 00000000000..6646c13167b --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/TaskBlockProfilingTestSupport.java @@ -0,0 +1,207 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.agentShadowJar; +import static datadog.smoketest.SmokeTestUtils.buildDirectory; +import static datadog.smoketest.SmokeTestUtils.javaPath; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; +import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; + +import datadog.trace.api.config.ProfilingConfig; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; + +/** Shared plumbing for native TaskBlock profiling smoke tests (JFR-loading, forked-JVM launch). */ +final class TaskBlockProfilingTestSupport { + static final byte[] JFR_MAGIC = new byte[] {'F', 'L', 'R', 0}; + static final IAttribute SPAN_ID = attr("spanId", "spanId", "spanId", NUMBER); + static final IAttribute LOCAL_ROOT_SPAN_ID = + attr("localRootSpanId", "localRootSpanId", "localRootSpanId", NUMBER); + static final IAttribute BLOCKER = attr("blocker", "blocker", "blocker", NUMBER); + static final IAttribute OPERATION = + attr("_dd.trace.operation", "_dd.trace.operation", "_dd.trace.operation", PLAIN_TEXT); + private static final Set REQUIRED_TASK_BLOCK_FIELDS = + new HashSet<>( + Arrays.asList( + "startTime", + "duration", + "eventThread", + "blocker", + "unblockingSpanId", + "stackTrace", + "observedBlockingState", + "spanId", + "localRootSpanId")); + + private TaskBlockProfilingTestSupport() {} + + static Path buildLogFilePath(Class testClass, TestInfo testInfo, String defaultName) + throws IOException { + Path logFileBase = Paths.get(buildDirectory(), "reports", "testProcess." + testClass.getName()); + Files.createDirectories(logFileBase); + return logFileBase.resolve( + testInfo.getTestMethod().map(Method::getName).orElse(defaultName) + ".log"); + } + + static Path createDumpDir(String prefix) throws IOException { + return Files.createTempDirectory(prefix); + } + + static void deleteRecursively(Path dir) throws IOException { + if (dir != null && Files.exists(dir)) { + Files.walk(dir).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + + static ProcessBuilder createTaskBlockProcessBuilder( + String serviceName, String mainClassName, Path dumpDir, Path logFilePath) throws IOException { + String templateOverride = + TaskBlockProfilingTestSupport.class.getClassLoader().getResource("overrides.jfp").getFile(); + List command = + new ArrayList<>( + Arrays.asList( + javaPath(), + "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), + "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), + "-javaagent:" + agentShadowJar(), + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Ddd.service.name=" + serviceName, + "-Ddd.env=smoketest", + "-Ddd.version=99", + "-Ddd.profiling.enabled=true", + "-Ddd.profiling.ddprof.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_AUXILIARY_TYPE + "=async", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED + "=true", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_INTERVAL + "=10ms", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER + "=false", + "-Ddd.profiling.agentless=false", + "-Ddd.profiling.start-delay=0", + "-Ddd." + ProfilingConfig.PROFILING_START_FORCE_FIRST + "=true", + "-Ddd.profiling.upload.period=1", + "-Ddd.profiling.hotspots.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES_SPAN_NAME_ENABLED + "=true", + "-Ddd.profiling.debug.dump_path=" + dumpDir, + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK + "=true", + "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", + "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug", + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UnlockCommercialFeatures", + "-XX:+FlightRecorder", + "-Ddd." + ProfilingConfig.PROFILING_TEMPLATE_OVERRIDE_FILE + "=" + templateOverride, + "-cp", + System.getProperty("java.class.path"), + mainClassName)); + if (System.getenv("TEST_LIBASYNC") != null) { + command.add( + command.size() - 3, + "-Ddd." + + ProfilingConfig.PROFILING_DATADOG_PROFILER_LIBPATH + + "=" + + System.getenv("TEST_LIBASYNC")); + } + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(new File(buildDirectory())); + processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")); + processBuilder.redirectErrorStream(true); + processBuilder.redirectOutput(ProcessBuilder.Redirect.to(logFilePath.toFile())); + return processBuilder; + } + + static List loadDumpedEvents(Path dumpDir) throws IOException { + List jfrFiles; + try (Stream files = Files.walk(dumpDir)) { + jfrFiles = + files + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".jfr")) + .collect(Collectors.toList()); + } + List events = new ArrayList<>(); + for (Path jfrFile : jfrFiles) { + events.add(loadEvents(dumpDir, jfrFile)); + } + return events; + } + + static boolean logContainsAny(Path logFilePath, String... needles) throws IOException { + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + for (String needle : needles) { + if (log.contains(needle)) { + return true; + } + } + return false; + } + + static void assertFinalTaskBlockSchema(IItemIterable items) { + Set actualFields = new HashSet<>(); + items + .getType() + .getAccessorKeys() + .keySet() + .forEach(accessorKey -> actualFields.add(accessorKey.getIdentifier())); + assertTrue( + actualFields.containsAll(REQUIRED_TASK_BLOCK_FIELDS), + "TaskBlock schema is missing fields. required=" + + REQUIRED_TASK_BLOCK_FIELDS + + ", actual=" + + actualFields); + } + + private static IItemCollection loadEvents(Path dumpDir, Path path) { + try { + return JfrLoaderToolkit.loadEvents(extractLastJfrStream(dumpDir, path).toFile()); + } catch (Exception e) { + throw new RuntimeException("Failed to load JFR " + path, e); + } + } + + private static Path extractLastJfrStream(Path dumpDir, Path path) throws IOException { + byte[] data = Files.readAllBytes(path); + int lastMagic = lastIndexOf(data, JFR_MAGIC); + if (lastMagic <= 0) { + return path; + } + Path extracted = dumpDir.resolve(path.getFileName() + ".ddprof.jfr"); + Files.write(extracted, Arrays.copyOfRange(data, lastMagic, data.length)); + return extracted; + } + + private static int lastIndexOf(byte[] data, byte[] needle) { + for (int i = data.length - needle.length; i >= 0; i--) { + boolean match = true; + for (int j = 0; j < needle.length; j++) { + if (data[i + j] != needle[j]) { + match = false; + break; + } + } + if (match) { + return i; + } + } + return -1; + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ThreadSleepTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ThreadSleepTaskBlockProfilingTest.java new file mode 100644 index 00000000000..f854a8e5f6c --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ThreadSleepTaskBlockProfilingTest.java @@ -0,0 +1,110 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datadog.smoketest.profiling.ThreadSleepTaskBlockForkedApp; +import datadog.trace.test.util.Flaky; +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** End-to-end coverage for synchronous TaskBlock emission around {@code Thread.sleep}. */ +@DisabledOnJ9 +@Flaky( + "TaskBlock/wall-clock sampler intermittently produces zero events across JDK versions; root cause is tracked separately") +final class ThreadSleepTaskBlockProfilingTest { + private Path logFilePath; + private Path dumpDir; + + @BeforeEach + void setUp(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + ThreadSleepTaskBlockProfilingTest.class, testInfo, "threadSleep"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-threadsleep-"); + } + + @AfterEach + void cleanUp() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void onlySpanlessPlatformSleepsEmitTaskBlocks() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-threadsleep-taskblock", + ThreadSleepTaskBlockForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + Stats stats = new Stats(); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + assertTrue(stats.spanlessPlatformCount > 0, "Expected spanless platform sleep TaskBlocks"); + assertEquals(0, stats.activePlatformCount, "Active-context sleeps must not emit TaskBlocks"); + assertEquals(0, stats.virtualCount, "Virtual-thread sleeps must not emit TaskBlocks"); + assertFalse(stats.spanlessHasContext, "Spanless sleep TaskBlocks must keep zero span context"); + assertFalse(stats.spanlessMissingThread, "Sleep TaskBlocks must resolve Event Thread"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, + "Failed to handle exception in instrumentation for " + + ThreadSleepTaskBlockForkedApp.class.getName(), + "Unexpected checked exception from ddprof TaskBlock hook"), + "Thread.sleep TaskBlock instrumentation failed"); + } + + private static final class Stats { + private long spanlessPlatformCount; + private long activePlatformCount; + private long virtualCount; + private boolean spanlessHasContext; + private boolean spanlessMissingThread; + + private void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor threadAccessor = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + IMemberAccessor spanAccessor = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootAccessor = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + for (IItem item : items) { + String thread = threadAccessor == null ? null : threadAccessor.getMember(item); + if (ThreadSleepTaskBlockForkedApp.SPANLESS_PLATFORM_THREAD.equals(thread)) { + spanlessPlatformCount++; + spanlessHasContext |= + spanAccessor.getMember(item).longValue() != 0L + || rootAccessor.getMember(item).longValue() != 0L; + spanlessMissingThread |= thread == null || thread.isEmpty(); + } else if (ThreadSleepTaskBlockForkedApp.ACTIVE_PLATFORM_THREAD.equals(thread)) { + activePlatformCount++; + } else if (ThreadSleepTaskBlockForkedApp.VIRTUAL_THREAD.equals(thread)) { + virtualCount++; + } + } + } + } + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java index 6d60ab686a7..d1b817b6fdb 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java @@ -97,6 +97,7 @@ public final class ProfilingConfig { public static final String PROFILING_DATADOG_PROFILER_SCRATCH = "profiling.ddprof.scratch"; public static final String PROFILING_DATADOG_PROFILER_LIBPATH = "profiling.ddprof.debug.lib"; + public static final String PROFILING_DATADOG_PROFILER_ALLOC_ENABLED = "profiling.ddprof.alloc.enabled"; public static final String PROFILING_DATADOG_PROFILER_ALLOC_INTERVAL = @@ -123,7 +124,11 @@ public final class ProfilingConfig { public static final String PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER = "profiling.ddprof.wall.context.filter"; - public static final boolean PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT = true; + public static final boolean PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT = false; + + public static final String PROFILING_DATADOG_PROFILER_WALL_PRECHECK = + "profiling.ddprof.wall.precheck"; + public static final boolean PROFILING_DATADOG_PROFILER_WALL_PRECHECK_DEFAULT = false; public static final String PROFILING_DATADOG_PROFILER_SCHEDULING_EVENT = "profiling.experimental.ddprof.scheduling.event"; diff --git a/internal-api/src/main/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfig.java b/internal-api/src/main/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfig.java new file mode 100644 index 00000000000..9c9f21553be --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfig.java @@ -0,0 +1,64 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.api.profiling; + +import static datadog.environment.JavaVirtualMachine.isJ9; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK_DEFAULT; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_ULTRA_MINIMAL; +import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; + +import datadog.trace.api.Config; +import datadog.trace.bootstrap.config.provider.ConfigProvider; + +/** Shared configuration gates for Java-level {@code datadog.TaskBlock} instrumentations. */ +public final class TaskBlockInstrumentationConfig { + private TaskBlockInstrumentationConfig() {} + + public static boolean isWallPrecheckEnabled(final ConfigProvider configProvider) { + return getDdprofBoolean( + configProvider, + PROFILING_DATADOG_PROFILER_WALL_PRECHECK, + PROFILING_DATADOG_PROFILER_WALL_PRECHECK_DEFAULT); + } + + /** Returns whether the effective wall-clock configuration samples all threads. */ + public static boolean isAllThreadWallScope(final ConfigProvider configProvider) { + boolean tracingEnabled = configProvider.getBoolean(TRACE_ENABLED, true); + return !tracingEnabled + || !getDdprofBoolean( + configProvider, + PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, + PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT); + } + + /** + * Common enablement gate shared by TaskBlock instrumentations. + * + *

The producer is useful only when ddprof and its effective wall-clock engine are enabled, + * wall precheck is requested, and the wall profiler samples all threads. In context-only scope, + * the existing context filter already excludes out-of-context threads. + */ + public static boolean isEnabled(final Config config, final ConfigProvider configProvider) { + return config.isDatadogProfilerEnabled() + && isWallClockProfilerEnabled(configProvider) + && isWallPrecheckEnabled(configProvider) + && isAllThreadWallScope(configProvider); + } + + private static boolean isWallClockProfilerEnabled(final ConfigProvider configProvider) { + boolean ultraMinimal = configProvider.getBoolean(PROFILING_ULTRA_MINIMAL, false); + boolean tracingEnabled = configProvider.getBoolean(TRACE_ENABLED, true); + boolean enabledByDefault = !ultraMinimal && tracingEnabled && !isJ9(); + return getDdprofBoolean( + configProvider, PROFILING_DATADOG_PROFILER_WALL_ENABLED, enabledByDefault); + } + + private static boolean getDdprofBoolean( + ConfigProvider configProvider, String key, boolean defaultValue) { + return configProvider.getBoolean( + key, configProvider.getBoolean(key.replace(".ddprof.", ".async."), defaultValue)); + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java index 4accced983a..5df9d7ab865 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package datadog.trace.bootstrap.instrumentation.api; import datadog.trace.api.EndpointCheckpointer; @@ -6,6 +7,9 @@ import datadog.trace.api.profiling.*; public interface ProfilingContextIntegration extends Profiling, EndpointCheckpointer, Timer { + /** Native {@code OSThreadState::SLEEPING}; used for {@code Thread.sleep} intervals. */ + int BLOCKING_STATE_SLEEPING = 7; + /** * invoked when the profiler is started, implementations must not initialise JFR before this is * called. @@ -34,6 +38,42 @@ default int encodeResourceName(CharSequence constant) { return 0; } + /** + * Called before the current thread enters {@code LockSupport.park*}. + * + * @return {@code true} when the integration accepted the entry dispatch and therefore requires a + * matching {@link #parkExit(long, long)}, even if profiling stops before the park returns + */ + default boolean parkEnter() { + return false; + } + + /** Completes a {@code LockSupport.park*} dispatch accepted by {@link #parkEnter()}. */ + default void parkExit(long blocker, long unblockingSpanId) {} + + /** + * Starts a synchronous TaskBlock interval on the current platform thread. + * + * @param state native blocking-state identifier + * @return an opaque token for {@link #endTaskBlock(long, long, long)}, or {@code 0} when the + * interval was not accepted + */ + default long beginTaskBlock(int state) { + return 0L; + } + + /** + * Completes a TaskBlock interval accepted by {@link #beginTaskBlock(int)}. + * + * @param token nonzero token returned by {@code beginTaskBlock} + * @param blocker identity hash of the blocking object, or {@code 0} when absent + * @param unblockingSpanId span that ended the interval, or {@code 0} when unavailable + * @return whether the native interval was completed and emitted + */ + default boolean endTaskBlock(long token, long blocker, long unblockingSpanId) { + return false; + } + String name(); final class NoOp implements ProfilingContextIntegration { diff --git a/internal-api/src/test/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfigTest.java b/internal-api/src/test/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfigTest.java new file mode 100644 index 00000000000..dfb223dfb55 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfigTest.java @@ -0,0 +1,118 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.api.profiling; + +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_ULTRA_MINIMAL; +import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.api.Config; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class TaskBlockInstrumentationConfigTest { + + @Test + void enabledRequiresEverySharedGate() { + Config config = config(true); + + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, true, true, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config(false), configProvider(true, true, true, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(false, true, true, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, false, true, false, false))); + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, true, false, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, true, true, true, false))); + } + + @Test + void disabledTracingForcesAllThreadScope() { + Properties properties = new Properties(); + properties.setProperty(TRACE_ENABLED, "false"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_ENABLED, "true"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_PRECHECK, "true"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, "true"); + + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(properties))); + assertTrue(TaskBlockInstrumentationConfig.isAllThreadWallScope(configProvider(properties))); + } + + @Test + void disabledTracingDisablesWallByDefault() { + Properties properties = new Properties(); + properties.setProperty(TRACE_ENABLED, "false"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_PRECHECK, "true"); + + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(properties))); + } + + @Test + void defaultsDoNotEnableTaskBlockInstrumentation() { + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(new Properties()))); + assertTrue( + TaskBlockInstrumentationConfig.isAllThreadWallScope( + ConfigProvider.withPropertiesOverride(new Properties()))); + } + + @Test + void asyncAliasesRemainSupported() { + Properties properties = new Properties(); + properties.setProperty("profiling.async.wall.enabled", "true"); + properties.setProperty("profiling.async.wall.precheck", "true"); + properties.setProperty("profiling.async.wall.context.filter", "false"); + + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(properties))); + } + + private static Config config(boolean ddprofEnabled) { + Config config = mock(Config.class); + when(config.isDatadogProfilerEnabled()).thenReturn(ddprofEnabled); + return config; + } + + private static ConfigProvider configProvider( + boolean wallEnabled, + boolean wallPrecheck, + boolean tracingEnabled, + boolean contextFilter, + boolean ultraMinimal) { + Properties properties = new Properties(); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_ENABLED, Boolean.toString(wallEnabled)); + properties.setProperty( + PROFILING_DATADOG_PROFILER_WALL_PRECHECK, Boolean.toString(wallPrecheck)); + properties.setProperty(TRACE_ENABLED, Boolean.toString(tracingEnabled)); + properties.setProperty( + PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, Boolean.toString(contextFilter)); + properties.setProperty(PROFILING_ULTRA_MINIMAL, Boolean.toString(ultraMinimal)); + return configProvider(properties); + } + + private static ConfigProvider configProvider(Properties properties) { + return ConfigProvider.withPropertiesOverride(properties); + } +} diff --git a/metadata/agent-jar-checks.properties b/metadata/agent-jar-checks.properties index fd04d637e35..c20190acd9d 100644 --- a/metadata/agent-jar-checks.properties +++ b/metadata/agent-jar-checks.properties @@ -201,6 +201,7 @@ expected.integrations = IastInstrumentation,\ synapse3-client,\ synapse3-server,\ testng,\ + thread-sleep,\ throwables,\ thymeleaf,\ tibco,\ diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 733f3736cbb..830266bdcb2 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -2769,6 +2769,14 @@ "aliases": [] } ], + "DD_PROFILING_ASYNC_WALL_PRECHECK": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], "DD_PROFILING_AUXILIARY": [ { "version": "A", @@ -2981,7 +2989,7 @@ { "version": "A", "type": "boolean", - "default": "true", + "default": "false", "aliases": [] } ], @@ -2993,6 +3001,14 @@ "aliases": [] } ], + "DD_PROFILING_DDPROF_WALL_PRECHECK": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], "DD_PROFILING_DDPROF_WALL_INTERVAL_MS": [ { "version": "A", @@ -7809,6 +7825,14 @@ "aliases": ["DD_TRACE_INTEGRATION_KOTLIN_COROUTINE_ENABLED", "DD_INTEGRATION_KOTLIN_COROUTINE_ENABLED"] } ], + "DD_TRACE_LOCK_SUPPORT_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_LOCK_SUPPORT_ENABLED", "DD_INTEGRATION_LOCK_SUPPORT_ENABLED"] + } + ], "DD_TRACE_LEGACY_E2E_DURATION_ENABLED": [ { "version": "A", @@ -10857,6 +10881,14 @@ "aliases": ["DD_TRACE_INTEGRATION_TEST_RETRY_ENABLED", "DD_INTEGRATION_TEST_RETRY_ENABLED"] } ], + "DD_TRACE_THREAD_SLEEP_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_THREAD_SLEEP_ENABLED", "DD_INTEGRATION_THREAD_SLEEP_ENABLED"] + } + ], "DD_TRACE_THREAD_POOL_EXECUTORS_EXCLUDE": [ { "version": "A", diff --git a/settings.gradle.kts b/settings.gradle.kts index 36b1d9b95df..8bc13a2f76a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -338,6 +338,8 @@ include( ":dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin", ":dd-java-agent:instrumentation:datadog:profiling:enable-wallclock-profiling", ":dd-java-agent:instrumentation:datadog:profiling:exception-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:lock-support", + ":dd-java-agent:instrumentation:datadog:profiling:thread-sleep", ":dd-java-agent:instrumentation:datadog:tracing:trace-annotation", ":dd-java-agent:instrumentation:datanucleus-4.0.5", ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0",