From 752ce26c37270fa0ffc9b6b30fb08780602ecc2c Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Thu, 16 Jul 2026 11:38:11 +0200 Subject: [PATCH 1/2] feat: add TaskBlock profiler integration plumbing --- .../profiling/ddprof/DatadogProfiler.java | 66 +++++- .../ddprof/DatadogProfilerConfig.java | 5 + .../ddprof/DatadogProfilerRecordingTest.java | 9 + .../profiling/ddprof/DatadogProfilerTest.java | 104 ++++++--- .../libs/ddprof/DdprofLibraryLoaderTest.java | 46 ++++ .../datadoghq/profiler/TaskBlockBridge.java | 116 ++++++++++ .../libs/ddprof/DdprofLibraryLoader.java | 34 +++ .../profiler/TaskBlockBridgeTest.java | 34 +++ .../profiling/NativeIoTaskBlockForkedApp.java | 53 +++++ .../SynchronizedContentionForkedApp.java | 100 +++++++++ .../NativeIoTaskBlockProfilingTest.java | 109 +++++++++ .../ObjectWaitTaskBlockProfilingTest.java | 162 ++++++++++++++ .../SynchronizedContentionProfilingTest.java | 118 ++++++++++ .../TaskBlockProfilingTestSupport.java | 207 ++++++++++++++++++ .../trace/api/config/ProfilingConfig.java | 7 +- .../TaskBlockInstrumentationConfig.java | 64 ++++++ .../TaskBlockInstrumentationConfigTest.java | 118 ++++++++++ metadata/supported-configurations.json | 18 +- 18 files changed, 1331 insertions(+), 39 deletions(-) create mode 100644 dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/datadog/libs/ddprof/DdprofLibraryLoaderTest.java create mode 100644 dd-java-agent/ddprof-lib/src/main/java/com/datadoghq/profiler/TaskBlockBridge.java create mode 100644 dd-java-agent/ddprof-lib/src/test/java/com/datadoghq/profiler/TaskBlockBridgeTest.java create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/NativeIoTaskBlockForkedApp.java create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/SynchronizedContentionForkedApp.java create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/NativeIoTaskBlockProfilingTest.java create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/SynchronizedContentionProfilingTest.java create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/TaskBlockProfilingTestSupport.java create mode 100644 internal-api/src/main/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfig.java create mode 100644 internal-api/src/test/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfigTest.java diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index df1b75789db..30bb1661ab0 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package com.datadog.profiling.ddprof; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.enableJMethodIDOptim; @@ -14,6 +15,7 @@ import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallCollapsing; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallContextFilter; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallInterval; +import static com.datadog.profiling.ddprof.DatadogProfilerConfig.getWallPrecheck; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isAllocationProfilingEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isCpuProfilerEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isLiveHeapSizeTrackingEnabled; @@ -36,6 +38,7 @@ import com.datadog.profiling.utils.ProfilingMode; import com.datadoghq.profiler.ContextSetter; import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.TaskBlockBridge; import datadog.environment.JavaVirtualMachine; import datadog.libs.ddprof.DdprofLibraryLoader; import datadog.trace.api.config.ProfilingConfig; @@ -101,9 +104,12 @@ public static DatadogProfiler newInstance(ConfigProvider configProvider) { return new DatadogProfiler(configProvider); } - private final AtomicBoolean recordingFlag = new AtomicBoolean(false); + // JavaProfiler owns one process-wide native recording; all wrapper instances must observe the + // same lifecycle so early-created integrations can dispatch hooks while the controller records. + private static final AtomicBoolean RECORDING = new AtomicBoolean(false); private final ConfigProvider configProvider; private final JavaProfiler profiler; + private final TaskBlockBridge taskBlockBridge; private final Set profilingModes = EnumSet.noneOf(ProfilingMode.class); private final ContextSetter contextSetter; @@ -244,6 +250,7 @@ void reset() { private DatadogProfiler(ConfigProvider configProvider) { this.configProvider = configProvider; this.profiler = DdprofLibraryLoader.javaProfiler().getComponent(); + this.taskBlockBridge = new TaskBlockBridge(profiler); this.detailedDebugLogging = configProvider.getBoolean( PROFILING_DETAILED_DEBUG_LOGGING, PROFILING_DETAILED_DEBUG_LOGGING_DEFAULT); @@ -355,7 +362,7 @@ public RecordingData stop(OngoingRecording recording) { /** A call-back from {@linkplain DatadogProfilerRecording#stop()} */ void stopProfiler() { - if (recordingFlag.compareAndSet(true, false)) { + if (RECORDING.compareAndSet(true, false)) { profiler.stop(); if (isActive()) { log.debug("Profiling is still active. Waiting to stop."); @@ -385,7 +392,7 @@ String executeProfilerCmd(String cmd) throws IOException { } Path newRecording() throws IOException, IllegalStateException { - if (recordingFlag.compareAndSet(false, true)) { + if (RECORDING.compareAndSet(false, true)) { Path recFile = Files.createTempFile(recordingsPath, "dd-profiler-", ".jfr"); String cmd = cmdStartProfiling(recFile); try { @@ -397,7 +404,7 @@ Path newRecording() throws IOException, IllegalStateException { } else { log.warn("Unable to start Datadog profiler recording: {}", e.getMessage()); } - recordingFlag.set(false); + RECORDING.set(false); throw e; } return recFile; @@ -410,6 +417,11 @@ void dump(Path path) { } String cmdStartProfiling(Path file) throws IllegalStateException { + return cmdStartProfiling(file, DdprofLibraryLoader.isFinalWallProtocolSupported()); + } + + String cmdStartProfiling(Path file, boolean finalWallProtocolSupported) + throws IllegalStateException { // 'start' = start, 'jfr=7' = store in JFR format ready for concatenation int safemode = getSafeMode(configProvider); if (safemode != ProfilingConfig.PROFILING_DATADOG_PROFILER_SAFEMODE_DEFAULT) { @@ -421,7 +433,6 @@ String cmdStartProfiling(Path file) throws IllegalStateException { } StringBuilder cmd = new StringBuilder("start,jfr"); cmd.append(",file=").append(file.toAbsolutePath()); - cmd.append(",loglevel=").append(getLogLevel(configProvider)); cmd.append(",jstackdepth=").append(getStackDepth(configProvider)); cmd.append(",cstack=").append(getCStack(configProvider)); cmd.append(",safemode=").append(getSafeMode(configProvider)); @@ -464,14 +475,17 @@ String cmdStartProfiling(Path file) throws IllegalStateException { cmd.append('~'); } cmd.append(getWallInterval(configProvider)).append('m'); - // ddprof quirk: if filter parameter is omitted, it defaults to "0" (enabled), - // not empty string (disabled). When enabled without tracing, no threads are added - // to the filter, resulting in zero samples. We must explicitly pass filter= (empty) - // to disable filtering and sample all threads. - if (getWallContextFilter(configProvider)) { - cmd.append(",filter=0"); + boolean contextScope = getWallContextFilter(configProvider); + if (finalWallProtocolSupported) { + cmd.append(",wallscope=").append(contextScope ? "context" : "all"); + if (getWallPrecheck(configProvider)) { + cmd.append(",wallprecheck=true"); + } } else { - cmd.append(",filter="); + // Legacy ddprof defaults to context filtering when filter is omitted. An empty value is + // therefore required to select all threads. wallprecheck is omitted because support for + // it was established together with wallscope by the non-mutating capability check. + cmd.append(contextScope ? ",filter=0" : ",filter="); } } cmd.append(",loglevel=").append(getLogLevel(configProvider)); @@ -534,6 +548,34 @@ public void clearSpanContext() { reapplyAppContext(); } + boolean hasParkTaskBlockSupport() { + return taskBlockBridge.hasParkSupport(); + } + + boolean parkEnter() { + if (!RECORDING.get()) { + return false; + } + taskBlockBridge.parkEnter(); + return true; + } + + void parkExit(long blocker, long unblockingSpanId) { + taskBlockBridge.parkExit(blocker, unblockingSpanId); + } + + boolean hasSynchronousTaskBlockSupport() { + return taskBlockBridge.hasSynchronousTaskBlockSupport(); + } + + long beginTaskBlock(int state) { + return RECORDING.get() ? taskBlockBridge.beginTaskBlock(state) : 0L; + } + + boolean endTaskBlock(long token, long blocker, long unblockingSpanId) { + return token != 0L && taskBlockBridge.endTaskBlock(token, blocker, unblockingSpanId); + } + public boolean setContextValue(int offset, String value) { if (contextSetter != null && offset >= 0 && value != null) { try { diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java index 8582e27c853..f9d5a17aaf0 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java @@ -58,6 +58,7 @@ import com.datadog.profiling.controller.ProfilingSupport; import datadog.environment.JavaVirtualMachine; import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.api.profiling.TaskBlockInstrumentationConfig; import datadog.trace.bootstrap.config.provider.ConfigProvider; import java.lang.management.ManagementFactory; import java.util.Collections; @@ -169,6 +170,10 @@ public static boolean getWallContextFilter(ConfigProvider configProvider) { PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT); } + public static boolean getWallPrecheck(ConfigProvider configProvider) { + return TaskBlockInstrumentationConfig.isWallPrecheckEnabled(configProvider); + } + static boolean isJmethodIDSafe() { return ProfilingSupport.isJmethodIDSafe(); } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java index 79d6985508e..6ae99538a0f 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package com.datadog.profiling.ddprof; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -72,4 +73,12 @@ void testSnapshot() throws Exception { assertNotNull(inputStream); assertTrue(inputStream.available() > 0); } + + @Test + void recordingLifecycleIsVisibleAcrossProfilerWrappers() { + DatadogProfiler integrationProfiler = DatadogProfiler.newInstance(); + + assertTrue(integrationProfiler.parkEnter()); + integrationProfiler.parkExit(0L, 0L); + } } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index 6dcb3e9d231..f1944d54c45 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -69,7 +69,7 @@ void test() throws Exception { IItemCollection events = JfrLoaderToolkit.loadEvents(data.getStream()); assertTrue(events.hasItems()); } finally { - recording.stop(); + recording.close(); } } else { log.warn("Datadog Profiler is not available. Skipping test."); @@ -104,8 +104,7 @@ void testStartCmd(boolean cpu, boolean wall, boolean alloc, boolean memleak) thr private static Stream profilingModes() { return IntStream.range(0, 1 << 4) .mapToObj( - x -> - Arguments.of((x & 0x1000) != 0, (x & 0x100) != 0, (x & 0x10) != 0, (x & 0x1) != 0)); + x -> Arguments.of((x & 0x8) != 0, (x & 0x4) != 0, (x & 0x2) != 0, (x & 0x1) != 0)); } @Test @@ -127,7 +126,8 @@ void testStartCmdEnableJMethodIDOptim() throws Exception { @ParameterizedTest @MethodSource("wallContextFilterModes") - void testWallContextFilter(boolean tracingEnabled, boolean contextFilterEnabled) + void testFinalWallScope( + boolean tracingEnabled, boolean contextFilterEnabled, String expectedWallScope) throws Exception { // Skip test if profiler native library is not available (e.g., on macOS) try { @@ -150,36 +150,90 @@ void testWallContextFilter(boolean tracingEnabled, boolean contextFilterEnabled) DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); Path targetFile = Paths.get("/tmp/target.jfr"); - String cmd = profiler.cmdStartProfiling(targetFile); + String cmd = profiler.cmdStartProfiling(targetFile, true); assertTrue(cmd.contains("wall="), "Command should contain wall profiling: " + cmd); + assertTrue(cmd.contains(",wallscope=" + expectedWallScope), cmd); + assertFalse(cmd.contains(",filter="), cmd); + } - if (tracingEnabled && contextFilterEnabled) { - assertTrue( - cmd.contains(",filter=0"), - "Command should contain ',filter=0' when tracing and context filter are enabled: " + cmd); - } else { - assertTrue( - cmd.contains(",filter="), - "Command should contain ',filter=' when tracing is disabled or context filter is disabled: " - + cmd); - if (cmd.contains(",filter=0")) { - throw new AssertionError( - "Command should not contain ',filter=0' when tracing is disabled or context filter is disabled: " - + cmd); + @ParameterizedTest + @MethodSource("wallPrecheckModes") + void testWallPrecheck(boolean wallPrecheckEnabled) throws Exception { + try { + Throwable reason = DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(); + if (reason != null) { + Assumptions.assumeTrue(false, "Profiler not available: " + reason.getMessage()); } + } catch (Throwable e) { + Assumptions.assumeTrue(false, "Profiler not available: " + e.getMessage()); + } + + Properties props = new Properties(); + props.put(ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED, "true"); + props.put( + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK, + Boolean.toString(wallPrecheckEnabled)); + + DatadogProfiler profiler = + DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); + + Path targetFile = Paths.get("/tmp/target.jfr"); + String cmd = profiler.cmdStartProfiling(targetFile, true); + + assertTrue(cmd.contains("wall="), cmd); + if (wallPrecheckEnabled) { + assertTrue(cmd.contains(",wallprecheck=true"), cmd); + } else { + assertFalse(cmd.contains("wallprecheck="), cmd); } } + @ParameterizedTest + @MethodSource("legacyWallModes") + void testLegacyWallProtocol( + boolean tracingEnabled, + boolean contextFilterEnabled, + boolean wallPrecheckEnabled, + String expectedFilter) + throws Exception { + Properties props = new Properties(); + props.put(ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED, "true"); + props.put(TraceInstrumentationConfig.TRACE_ENABLED, Boolean.toString(tracingEnabled)); + props.put( + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, + Boolean.toString(contextFilterEnabled)); + props.put( + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK, + Boolean.toString(wallPrecheckEnabled)); + + DatadogProfiler profiler = + DatadogProfiler.newInstance(ConfigProvider.withPropertiesOverride(props)); + String cmd = profiler.cmdStartProfiling(Paths.get("/tmp/target.jfr"), false); + + assertTrue(cmd.contains(expectedFilter), cmd); + assertFalse(cmd.contains("wallscope="), cmd); + assertFalse(cmd.contains("wallprecheck="), cmd); + } + + private static Stream wallPrecheckModes() { + return Stream.of(Arguments.of(true), Arguments.of(false)); + } + private static Stream wallContextFilterModes() { return Stream.of( - Arguments.of(true, true), // tracing enabled, context filter enabled -> filter=0 - Arguments.of(true, false), // tracing enabled, context filter disabled -> filter= - Arguments.of( - false, true), // tracing disabled, context filter enabled -> filter= (tracing disabled - // overrides) - Arguments.of(false, false) // tracing disabled, context filter disabled -> filter= - ); + Arguments.of(true, true, "context"), + Arguments.of(true, false, "all"), + Arguments.of(false, true, "all"), + Arguments.of(false, false, "all")); + } + + private static Stream legacyWallModes() { + return Stream.of( + Arguments.of(true, true, true, ",filter=0"), + Arguments.of(true, false, true, ",filter="), + Arguments.of(false, true, true, ",filter="), + Arguments.of(false, false, false, ",filter=")); } @Test diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/datadog/libs/ddprof/DdprofLibraryLoaderTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/datadog/libs/ddprof/DdprofLibraryLoaderTest.java new file mode 100644 index 00000000000..d262ad54614 --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/datadog/libs/ddprof/DdprofLibraryLoaderTest.java @@ -0,0 +1,46 @@ +// Copyright 2026 Datadog, Inc. +package datadog.libs.ddprof; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +class DdprofLibraryLoaderTest { + + @Test + void finalProtocolProbeUsesNonMutatingCheckAction() { + AtomicReference command = new AtomicReference<>(); + + boolean supported = + DdprofLibraryLoader.probeFinalWallProtocol( + value -> { + command.set(value); + return "OK"; + }); + + assertTrue(supported); + assertEquals("check,wall=10m,wallscope=all,wallprecheck=true", command.get()); + } + + @Test + void finalProtocolProbeRejectsParserFailure() { + assertFalse( + DdprofLibraryLoader.probeFinalWallProtocol( + command -> { + throw new IllegalArgumentException("Unknown argument: wallscope"); + })); + } + + @Test + void finalProtocolProbeRejectsCheckFailure() { + assertFalse( + DdprofLibraryLoader.probeFinalWallProtocol( + command -> { + throw new IOException("wall engine unavailable"); + })); + } +} diff --git a/dd-java-agent/ddprof-lib/src/main/java/com/datadoghq/profiler/TaskBlockBridge.java b/dd-java-agent/ddprof-lib/src/main/java/com/datadoghq/profiler/TaskBlockBridge.java new file mode 100644 index 00000000000..040c6d9df04 --- /dev/null +++ b/dd-java-agent/ddprof-lib/src/main/java/com/datadoghq/profiler/TaskBlockBridge.java @@ -0,0 +1,116 @@ +// Copyright 2026 Datadog, Inc. +package com.datadoghq.profiler; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +/** + * Compatibility bridge for TaskBlock hooks added to newer ddprof artifacts. + * + *

This class deliberately lives in the profiler package: that gives its lookup access to the + * package-private park hooks without making those hooks public. Hook names are resolved from + * strings so the agent remains binary-compatible with older ddprof artifacts. Resolved method + * handles are cached and invoked directly on the hot path. + */ +public final class TaskBlockBridge { + private static final MethodHandle PARK_ENTER = findVirtual("parkEnter", void.class); + private static final MethodHandle PARK_EXIT = + findVirtual("parkExit", void.class, long.class, long.class); + private static final MethodHandle BEGIN_TASK_BLOCK = + findVirtual("beginTaskBlock", long.class, int.class); + private static final MethodHandle END_TASK_BLOCK = + findVirtual("endTaskBlock", boolean.class, long.class, long.class, long.class); + + private final JavaProfiler profiler; + + /** Creates a bridge bound to the loaded profiler singleton. */ + public TaskBlockBridge(JavaProfiler profiler) { + this.profiler = profiler; + } + + /** Returns whether both native park hooks are available. */ + public boolean hasParkSupport() { + return PARK_ENTER != null && PARK_EXIT != null; + } + + /** Returns whether the synchronous paired TaskBlock API is available. */ + public boolean hasSynchronousTaskBlockSupport() { + return BEGIN_TASK_BLOCK != null && END_TASK_BLOCK != null; + } + + /** Invokes the native park-entry hook when it is available. */ + public boolean parkEnter() { + if (!hasParkSupport()) { + return false; + } + try { + PARK_ENTER.invokeExact(profiler); + return true; + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + /** Invokes the native park-exit hook when it is available. */ + public void parkExit(long blocker, long unblockingSpanId) { + if (!hasParkSupport()) { + return; + } + try { + PARK_EXIT.invokeExact(profiler, blocker, unblockingSpanId); + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + /** Begins a synchronous TaskBlock interval, returning {@code 0} when unsupported. */ + public long beginTaskBlock(int state) { + if (!hasSynchronousTaskBlockSupport()) { + return 0L; + } + try { + return (long) BEGIN_TASK_BLOCK.invokeExact(profiler, state); + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + /** Ends a synchronous TaskBlock interval when the paired API is available. */ + public boolean endTaskBlock(long token, long blocker, long unblockingSpanId) { + if (!hasSynchronousTaskBlockSupport()) { + return false; + } + try { + return (boolean) END_TASK_BLOCK.invokeExact(profiler, token, blocker, unblockingSpanId); + } catch (Throwable throwable) { + throw propagate(throwable); + } + } + + private static MethodHandle findVirtual( + String methodName, Class returnType, Class... parameterTypes) { + return findVirtual(JavaProfiler.class, methodName, returnType, parameterTypes); + } + + static MethodHandle findVirtual( + Class targetClass, String methodName, Class returnType, Class... parameterTypes) { + try { + return MethodHandles.lookup() + .findVirtual(targetClass, methodName, MethodType.methodType(returnType, parameterTypes)); + } catch (NoSuchMethodException | IllegalAccessException ignored) { + return null; + } + } + + private static RuntimeException propagate(Throwable throwable) { + if (throwable instanceof RuntimeException) { + return (RuntimeException) throwable; + } + if (throwable instanceof Error) { + throw (Error) throwable; + } + return new IllegalStateException( + "Unexpected checked exception from ddprof TaskBlock hook", throwable); + } +} diff --git a/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java b/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java index b637dc69309..8becf1335d3 100644 --- a/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java +++ b/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java @@ -24,6 +24,9 @@ */ public final class DdprofLibraryLoader { private static final Logger log = LoggerFactory.getLogger(DdprofLibraryLoader.class.getName()); + private static final String FINAL_WALL_PROTOCOL_CHECK = + "check,wall=10m,wallscope=all,wallprecheck=true"; + private static volatile boolean finalWallProtocolSupported; public abstract static class ComponentHolder { private volatile boolean loaded = false; @@ -123,6 +126,17 @@ public static OTelContextHolder otelContext() { return OTEL_CONTEXT_HOLDER; } + /** + * Returns whether the loaded native profiler accepts the final wall-clock command protocol. + * + *

Calling this method initializes the profiler holder when necessary. The capability is + * established with the profiler's non-mutating {@code check} action. + */ + public static boolean isFinalWallProtocolSupported() { + PROFILER_HOLDER.getComponent(); + return finalWallProtocolSupported; + } + private static JavaProfilerHolder initJavaProfiler() { JavaProfiler profiler; Throwable reasonNotLoaded = null; @@ -135,6 +149,11 @@ private static JavaProfilerHolder initJavaProfiler() { scratch); // sanity test - force load Datadog profiler to catch it not being available early profiler.execute("status"); + finalWallProtocolSupported = probeFinalWallProtocol(profiler::execute); + if (!finalWallProtocolSupported) { + log.debug( + "Loaded ddprof does not accept wallscope and wallprecheck; using legacy wall command options"); + } } catch (Throwable t) { reasonNotLoaded = t; profiler = null; @@ -142,6 +161,21 @@ private static JavaProfilerHolder initJavaProfiler() { return new JavaProfilerHolder(profiler, reasonNotLoaded); } + static boolean probeFinalWallProtocol(ProfilerCommandExecutor executor) { + try { + executor.execute(FINAL_WALL_PROTOCOL_CHECK); + return true; + } catch (IllegalArgumentException | IllegalStateException | IOException ignored) { + return false; + } + } + + @FunctionalInterface + interface ProfilerCommandExecutor { + String execute(String command) + throws IllegalArgumentException, IllegalStateException, IOException; + } + private static JVMAccessHolder initJVMAccess() { ConfigProvider configProvider = ConfigProvider.getInstance(); AtomicReference reasonNotLoaded = new AtomicReference<>(); diff --git a/dd-java-agent/ddprof-lib/src/test/java/com/datadoghq/profiler/TaskBlockBridgeTest.java b/dd-java-agent/ddprof-lib/src/test/java/com/datadoghq/profiler/TaskBlockBridgeTest.java new file mode 100644 index 00000000000..a0f92a7ca9d --- /dev/null +++ b/dd-java-agent/ddprof-lib/src/test/java/com/datadoghq/profiler/TaskBlockBridgeTest.java @@ -0,0 +1,34 @@ +// Copyright 2026 Datadog, Inc. +package com.datadoghq.profiler; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class TaskBlockBridgeTest { + @Test + void resolvesPackagePrivateHooksFromProfilerPackage() { + assertNotNull(TaskBlockBridge.findVirtual(SupportedHooks.class, "parkEnter", void.class)); + assertNotNull( + TaskBlockBridge.findVirtual(SupportedHooks.class, "beginTaskBlock", long.class, int.class)); + } + + @Test + void missingHooksAreReportedWithoutFailingClassInitialization() { + assertNull(TaskBlockBridge.findVirtual(UnsupportedHooks.class, "parkEnter", void.class)); + assertNull( + TaskBlockBridge.findVirtual( + UnsupportedHooks.class, "beginTaskBlock", long.class, int.class)); + } + + static final class SupportedHooks { + void parkEnter() {} + + long beginTaskBlock(int state) { + return state; + } + } + + static final class UnsupportedHooks {} +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/NativeIoTaskBlockForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/NativeIoTaskBlockForkedApp.java new file mode 100644 index 00000000000..92ce4897734 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/NativeIoTaskBlockForkedApp.java @@ -0,0 +1,53 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; + +/** Forked workload that blocks a platform thread in native accept and socket-read calls. */ +public final class NativeIoTaskBlockForkedApp { + private static final int ITERATIONS = 20; + + public static void main(String[] args) throws Exception { + Thread.currentThread().setName("native-io-spanless"); + for (int i = 0; i < ITERATIONS; i++) { + runBlockingAcceptAndRead(); + } + Thread.sleep(1500); + } + + private static void runBlockingAcceptAndRead() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { + Thread client = + new Thread( + () -> { + try { + Thread.sleep(50L); + try (Socket socket = + new Socket(InetAddress.getLoopbackAddress(), server.getLocalPort())) { + Thread.sleep(50L); + OutputStream output = socket.getOutputStream(); + output.write(1); + output.flush(); + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + }, + "native-io-client"); + client.start(); + + try (Socket accepted = server.accept()) { + accepted.setSoTimeout(5000); + InputStream input = accepted.getInputStream(); + if (input.read() != 1) { + throw new IllegalStateException("Unexpected socket payload"); + } + } + client.join(); + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/SynchronizedContentionForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/SynchronizedContentionForkedApp.java new file mode 100644 index 00000000000..bbef973747f --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/SynchronizedContentionForkedApp.java @@ -0,0 +1,100 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import java.util.concurrent.CountDownLatch; + +/** Forked workload for native monitor-contention TaskBlock smoke coverage. */ +public final class SynchronizedContentionForkedApp { + private static final int REPETITIONS = 10; + private static final Object BLOCK_LOCK = new Object(); + + public static void main(final String[] args) throws Exception { + SynchronizedContentionForkedApp app = new SynchronizedContentionForkedApp(); + for (int i = 0; i < REPETITIONS; i++) { + app.runBlockScenario(); + app.runInstanceMethodScenario(); + app.runStaticMethodScenario(); + } + Thread.sleep(1500); + } + + private final InstanceLockTarget instanceTarget = new InstanceLockTarget(); + + private void runBlockScenario() throws Exception { + CountDownLatch holderIn = new CountDownLatch(1); + Thread holder = + new Thread( + () -> { + synchronized (BLOCK_LOCK) { + holderIn.countDown(); + sleepWhileHoldingMonitor(); + } + }, + "sync-block-holder"); + holder.start(); + holderIn.await(); + + Thread contender = + new Thread( + () -> { + synchronized (BLOCK_LOCK) { + // entry-queue wait is the TaskBlock interval + } + }, + "sync-block-spanless"); + contender.start(); + contender.join(); + holder.join(); + } + + private void runInstanceMethodScenario() throws Exception { + CountDownLatch holderIn = new CountDownLatch(1); + Thread holder = new Thread(() -> instanceTarget.hold(holderIn), "sync-instance-holder"); + holder.start(); + holderIn.await(); + + Thread contender = new Thread(instanceTarget::contend, "sync-instance-spanless"); + contender.start(); + contender.join(); + holder.join(); + } + + private void runStaticMethodScenario() throws Exception { + CountDownLatch holderIn = new CountDownLatch(1); + Thread holder = new Thread(() -> StaticLockTarget.hold(holderIn), "sync-static-holder"); + holder.start(); + holderIn.await(); + + Thread contender = new Thread(StaticLockTarget::contend, "sync-static-spanless"); + contender.start(); + contender.join(); + holder.join(); + } + + private static void sleepWhileHoldingMonitor() { + try { + Thread.sleep(50L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + static final class InstanceLockTarget { + synchronized void hold(final CountDownLatch in) { + in.countDown(); + sleepWhileHoldingMonitor(); + } + + synchronized void contend() {} + } + + static final class StaticLockTarget { + static synchronized void hold(final CountDownLatch in) { + in.countDown(); + sleepWhileHoldingMonitor(); + } + + static synchronized void contend() {} + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/NativeIoTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/NativeIoTaskBlockProfilingTest.java new file mode 100644 index 00000000000..21f38c7316f --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/NativeIoTaskBlockProfilingTest.java @@ -0,0 +1,109 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** Linux smoke coverage for TaskBlocks emitted by native socket-I/O interposition. */ +@DisabledOnJ9 +@EnabledOnOs(OS.LINUX) +final class NativeIoTaskBlockProfilingTest { + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + NativeIoTaskBlockProfilingTest.class, testInfo, "nativeIo"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-nativeio-"); + } + + @AfterEach + void tearDown() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void blockingNativeSocketIoEmitsSpanlessTaskBlocks() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-nativeio-taskblock", + com.datadog.smoketest.profiling.NativeIoTaskBlockForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = new JfrStats(); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + + assertTrue(stats.count > 0, "Expected TaskBlocks from blocking native socket I/O"); + assertTrue(stats.nonZeroBlockerCount > 0, "Expected encoded native I/O blocker identity"); + assertFalse(stats.hasNonZeroSpanId, "Spanless native I/O TaskBlocks must keep spanId zero"); + assertFalse( + stats.hasNonZeroLocalRootSpanId, + "Spanless native I/O TaskBlocks must keep root spanId zero"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, + "native I/O hooks disabled", + "NoClassDefFoundError", + "Failed to handle exception"), + "native I/O TaskBlock path failed"); + } + + private static final class JfrStats { + long count; + long nonZeroBlockerCount; + boolean hasNonZeroSpanId; + boolean hasNonZeroLocalRootSpanId; + boolean hasMissingEventThread; + + void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor spanId = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootSpanId = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blocker = BLOCKER.getAccessor(items.getType()); + IMemberAccessor eventThread = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + for (IItem item : items) { + String thread = eventThread.getMember(item); + if (!"native-io-spanless".equals(thread)) { + continue; + } + count++; + nonZeroBlockerCount += blocker.getMember(item).longValue() == 0 ? 0 : 1; + hasNonZeroSpanId |= spanId.getMember(item).longValue() != 0; + hasNonZeroLocalRootSpanId |= rootSpanId.getMember(item).longValue() != 0; + hasMissingEventThread |= thread.isEmpty(); + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java new file mode 100644 index 00000000000..da0ce627b2c --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java @@ -0,0 +1,162 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; + +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** Smoke coverage for native {@code Object.wait} TaskBlock emission and context exclusion. */ +@DisabledOnJ9 +final class ObjectWaitTaskBlockProfilingTest { + private static final IAttribute UNBLOCKING_SPAN_ID = + attr("unblockingSpanId", "unblockingSpanId", "unblockingSpanId", NUMBER); + + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + ObjectWaitTaskBlockProfilingTest.class, testInfo, "objectWait"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-objectwait-"); + } + + @AfterEach + void tearDown() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void spanlessObjectWaitEmitsWhileTracedObjectWaitDoesNot() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-objectwait-taskblock", + ObjectWaitTaskBlockForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = new JfrStats(); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + + assertTrue(stats.spanlessCount > 0, "Expected Object.wait TaskBlocks outside trace context"); + assertTrue(stats.nonZeroBlockerCount > 0, "Expected monitor identity on Object.wait events"); + assertFalse(stats.hasNonZeroSpanId, "Spanless TaskBlocks must keep spanId zero"); + assertFalse(stats.hasNonZeroLocalRootSpanId, "Spanless TaskBlocks must keep root spanId zero"); + assertFalse(stats.hasNonZeroUnblockingSpanId, "Object.wait has no attributed notifier hook"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertFalse(stats.hasTracedThreadEvent, "Traced waits must not emit TaskBlock events"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, "NoClassDefFoundError", "Failed to handle exception in instrumentation"), + "Object.wait TaskBlock path failed"); + } + + public static final class ObjectWaitTaskBlockForkedApp { + private static final int ITERATIONS = 20; + private static final Object BLOCKER = new Object(); + + public static void main(String[] args) throws Exception { + Thread spanless = + new Thread(ObjectWaitTaskBlockForkedApp::waitRepeatedly, "objectwait-spanless"); + spanless.start(); + spanless.join(); + + Tracer tracer = GlobalTracer.get(); + Thread traced = + new Thread( + () -> { + Span span = tracer.buildSpan("objectwait.traced").start(); + try (Scope scope = tracer.activateSpan(span)) { + waitRepeatedly(); + } finally { + span.finish(); + } + }, + "objectwait-traced"); + traced.start(); + traced.join(); + Thread.sleep(1500); + } + + private static void waitRepeatedly() { + try { + for (int i = 0; i < ITERATIONS; i++) { + synchronized (BLOCKER) { + BLOCKER.wait(50L); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } + + private static final class JfrStats { + long spanlessCount; + long nonZeroBlockerCount; + boolean hasNonZeroSpanId; + boolean hasNonZeroLocalRootSpanId; + boolean hasNonZeroUnblockingSpanId; + boolean hasMissingEventThread; + boolean hasTracedThreadEvent; + + void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor spanId = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootSpanId = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blocker = BLOCKER.getAccessor(items.getType()); + IMemberAccessor unblockingSpanId = + UNBLOCKING_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor eventThread = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + for (IItem item : items) { + String thread = eventThread.getMember(item); + if ("objectwait-traced".equals(thread)) { + hasTracedThreadEvent = true; + } + if (!"objectwait-spanless".equals(thread)) { + continue; + } + spanlessCount++; + hasNonZeroSpanId |= spanId.getMember(item).longValue() != 0; + hasNonZeroLocalRootSpanId |= rootSpanId.getMember(item).longValue() != 0; + hasNonZeroUnblockingSpanId |= unblockingSpanId.getMember(item).longValue() != 0; + nonZeroBlockerCount += blocker.getMember(item).longValue() == 0 ? 0 : 1; + hasMissingEventThread |= thread.isEmpty(); + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/SynchronizedContentionProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/SynchronizedContentionProfilingTest.java new file mode 100644 index 00000000000..9b4f13b7a6b --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/SynchronizedContentionProfilingTest.java @@ -0,0 +1,118 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** Smoke coverage for native synchronized-contention TaskBlock emission. */ +@DisabledOnJ9 +final class SynchronizedContentionProfilingTest { + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + SynchronizedContentionProfilingTest.class, testInfo, "syncContention"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-synccontention-"); + } + + @AfterEach + void tearDown() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void spanlessSynchronizedContentionEmitsNativeTaskBlocks() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-synccontention-taskblock", + com.datadog.smoketest.profiling.SynchronizedContentionForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = new JfrStats(); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + + assertTrue(stats.blockScenarioCount > 0, "Expected synchronized-block TaskBlocks"); + assertTrue(stats.instanceMethodScenarioCount > 0, "Expected instance-method TaskBlocks"); + assertTrue(stats.staticMethodScenarioCount > 0, "Expected static-method TaskBlocks"); + assertFalse(stats.hasNonZeroSpanId, "Spanless TaskBlocks must keep spanId zero"); + assertFalse(stats.hasNonZeroLocalRootSpanId, "Spanless TaskBlocks must keep root spanId zero"); + assertTrue(stats.blockersWithNonZeroValue > 0, "Expected monitor identity on TaskBlocks"); + assertTrue(stats.distinctBlockerValues.size() > 1, "Expected per-monitor blocker identities"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, "NoClassDefFoundError", "Failed to handle exception", "VerifyError"), + "native synchronized-contention TaskBlock path failed"); + } + + private static final class JfrStats { + long blockScenarioCount; + long instanceMethodScenarioCount; + long staticMethodScenarioCount; + long blockersWithNonZeroValue; + final Set distinctBlockerValues = new HashSet<>(); + boolean hasNonZeroSpanId; + boolean hasNonZeroLocalRootSpanId; + boolean hasMissingEventThread; + + void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor spanId = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootSpanId = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blocker = BLOCKER.getAccessor(items.getType()); + IMemberAccessor eventThread = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + for (IItem item : items) { + String thread = eventThread.getMember(item); + if ("sync-block-spanless".equals(thread)) { + blockScenarioCount++; + } else if ("sync-instance-spanless".equals(thread)) { + instanceMethodScenarioCount++; + } else if ("sync-static-spanless".equals(thread)) { + staticMethodScenarioCount++; + } else { + continue; + } + hasNonZeroSpanId |= spanId.getMember(item).longValue() != 0; + hasNonZeroLocalRootSpanId |= rootSpanId.getMember(item).longValue() != 0; + long blockerValue = blocker.getMember(item).longValue(); + if (blockerValue != 0) { + blockersWithNonZeroValue++; + distinctBlockerValues.add(blockerValue); + } + hasMissingEventThread |= thread.isEmpty(); + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/TaskBlockProfilingTestSupport.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/TaskBlockProfilingTestSupport.java new file mode 100644 index 00000000000..6646c13167b --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/TaskBlockProfilingTestSupport.java @@ -0,0 +1,207 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.agentShadowJar; +import static datadog.smoketest.SmokeTestUtils.buildDirectory; +import static datadog.smoketest.SmokeTestUtils.javaPath; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; +import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; + +import datadog.trace.api.config.ProfilingConfig; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; + +/** Shared plumbing for native TaskBlock profiling smoke tests (JFR-loading, forked-JVM launch). */ +final class TaskBlockProfilingTestSupport { + static final byte[] JFR_MAGIC = new byte[] {'F', 'L', 'R', 0}; + static final IAttribute SPAN_ID = attr("spanId", "spanId", "spanId", NUMBER); + static final IAttribute LOCAL_ROOT_SPAN_ID = + attr("localRootSpanId", "localRootSpanId", "localRootSpanId", NUMBER); + static final IAttribute BLOCKER = attr("blocker", "blocker", "blocker", NUMBER); + static final IAttribute OPERATION = + attr("_dd.trace.operation", "_dd.trace.operation", "_dd.trace.operation", PLAIN_TEXT); + private static final Set REQUIRED_TASK_BLOCK_FIELDS = + new HashSet<>( + Arrays.asList( + "startTime", + "duration", + "eventThread", + "blocker", + "unblockingSpanId", + "stackTrace", + "observedBlockingState", + "spanId", + "localRootSpanId")); + + private TaskBlockProfilingTestSupport() {} + + static Path buildLogFilePath(Class testClass, TestInfo testInfo, String defaultName) + throws IOException { + Path logFileBase = Paths.get(buildDirectory(), "reports", "testProcess." + testClass.getName()); + Files.createDirectories(logFileBase); + return logFileBase.resolve( + testInfo.getTestMethod().map(Method::getName).orElse(defaultName) + ".log"); + } + + static Path createDumpDir(String prefix) throws IOException { + return Files.createTempDirectory(prefix); + } + + static void deleteRecursively(Path dir) throws IOException { + if (dir != null && Files.exists(dir)) { + Files.walk(dir).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + + static ProcessBuilder createTaskBlockProcessBuilder( + String serviceName, String mainClassName, Path dumpDir, Path logFilePath) throws IOException { + String templateOverride = + TaskBlockProfilingTestSupport.class.getClassLoader().getResource("overrides.jfp").getFile(); + List command = + new ArrayList<>( + Arrays.asList( + javaPath(), + "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), + "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), + "-javaagent:" + agentShadowJar(), + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Ddd.service.name=" + serviceName, + "-Ddd.env=smoketest", + "-Ddd.version=99", + "-Ddd.profiling.enabled=true", + "-Ddd.profiling.ddprof.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_AUXILIARY_TYPE + "=async", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED + "=true", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_INTERVAL + "=10ms", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER + "=false", + "-Ddd.profiling.agentless=false", + "-Ddd.profiling.start-delay=0", + "-Ddd." + ProfilingConfig.PROFILING_START_FORCE_FIRST + "=true", + "-Ddd.profiling.upload.period=1", + "-Ddd.profiling.hotspots.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES_SPAN_NAME_ENABLED + "=true", + "-Ddd.profiling.debug.dump_path=" + dumpDir, + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK + "=true", + "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", + "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug", + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UnlockCommercialFeatures", + "-XX:+FlightRecorder", + "-Ddd." + ProfilingConfig.PROFILING_TEMPLATE_OVERRIDE_FILE + "=" + templateOverride, + "-cp", + System.getProperty("java.class.path"), + mainClassName)); + if (System.getenv("TEST_LIBASYNC") != null) { + command.add( + command.size() - 3, + "-Ddd." + + ProfilingConfig.PROFILING_DATADOG_PROFILER_LIBPATH + + "=" + + System.getenv("TEST_LIBASYNC")); + } + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(new File(buildDirectory())); + processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")); + processBuilder.redirectErrorStream(true); + processBuilder.redirectOutput(ProcessBuilder.Redirect.to(logFilePath.toFile())); + return processBuilder; + } + + static List loadDumpedEvents(Path dumpDir) throws IOException { + List jfrFiles; + try (Stream files = Files.walk(dumpDir)) { + jfrFiles = + files + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".jfr")) + .collect(Collectors.toList()); + } + List events = new ArrayList<>(); + for (Path jfrFile : jfrFiles) { + events.add(loadEvents(dumpDir, jfrFile)); + } + return events; + } + + static boolean logContainsAny(Path logFilePath, String... needles) throws IOException { + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + for (String needle : needles) { + if (log.contains(needle)) { + return true; + } + } + return false; + } + + static void assertFinalTaskBlockSchema(IItemIterable items) { + Set actualFields = new HashSet<>(); + items + .getType() + .getAccessorKeys() + .keySet() + .forEach(accessorKey -> actualFields.add(accessorKey.getIdentifier())); + assertTrue( + actualFields.containsAll(REQUIRED_TASK_BLOCK_FIELDS), + "TaskBlock schema is missing fields. required=" + + REQUIRED_TASK_BLOCK_FIELDS + + ", actual=" + + actualFields); + } + + private static IItemCollection loadEvents(Path dumpDir, Path path) { + try { + return JfrLoaderToolkit.loadEvents(extractLastJfrStream(dumpDir, path).toFile()); + } catch (Exception e) { + throw new RuntimeException("Failed to load JFR " + path, e); + } + } + + private static Path extractLastJfrStream(Path dumpDir, Path path) throws IOException { + byte[] data = Files.readAllBytes(path); + int lastMagic = lastIndexOf(data, JFR_MAGIC); + if (lastMagic <= 0) { + return path; + } + Path extracted = dumpDir.resolve(path.getFileName() + ".ddprof.jfr"); + Files.write(extracted, Arrays.copyOfRange(data, lastMagic, data.length)); + return extracted; + } + + private static int lastIndexOf(byte[] data, byte[] needle) { + for (int i = data.length - needle.length; i >= 0; i--) { + boolean match = true; + for (int j = 0; j < needle.length; j++) { + if (data[i + j] != needle[j]) { + match = false; + break; + } + } + if (match) { + return i; + } + } + return -1; + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java index 6d60ab686a7..d1b817b6fdb 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/ProfilingConfig.java @@ -97,6 +97,7 @@ public final class ProfilingConfig { public static final String PROFILING_DATADOG_PROFILER_SCRATCH = "profiling.ddprof.scratch"; public static final String PROFILING_DATADOG_PROFILER_LIBPATH = "profiling.ddprof.debug.lib"; + public static final String PROFILING_DATADOG_PROFILER_ALLOC_ENABLED = "profiling.ddprof.alloc.enabled"; public static final String PROFILING_DATADOG_PROFILER_ALLOC_INTERVAL = @@ -123,7 +124,11 @@ public final class ProfilingConfig { public static final String PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER = "profiling.ddprof.wall.context.filter"; - public static final boolean PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT = true; + public static final boolean PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT = false; + + public static final String PROFILING_DATADOG_PROFILER_WALL_PRECHECK = + "profiling.ddprof.wall.precheck"; + public static final boolean PROFILING_DATADOG_PROFILER_WALL_PRECHECK_DEFAULT = false; public static final String PROFILING_DATADOG_PROFILER_SCHEDULING_EVENT = "profiling.experimental.ddprof.scheduling.event"; diff --git a/internal-api/src/main/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfig.java b/internal-api/src/main/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfig.java new file mode 100644 index 00000000000..9c9f21553be --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfig.java @@ -0,0 +1,64 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.api.profiling; + +import static datadog.environment.JavaVirtualMachine.isJ9; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK_DEFAULT; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_ULTRA_MINIMAL; +import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; + +import datadog.trace.api.Config; +import datadog.trace.bootstrap.config.provider.ConfigProvider; + +/** Shared configuration gates for Java-level {@code datadog.TaskBlock} instrumentations. */ +public final class TaskBlockInstrumentationConfig { + private TaskBlockInstrumentationConfig() {} + + public static boolean isWallPrecheckEnabled(final ConfigProvider configProvider) { + return getDdprofBoolean( + configProvider, + PROFILING_DATADOG_PROFILER_WALL_PRECHECK, + PROFILING_DATADOG_PROFILER_WALL_PRECHECK_DEFAULT); + } + + /** Returns whether the effective wall-clock configuration samples all threads. */ + public static boolean isAllThreadWallScope(final ConfigProvider configProvider) { + boolean tracingEnabled = configProvider.getBoolean(TRACE_ENABLED, true); + return !tracingEnabled + || !getDdprofBoolean( + configProvider, + PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, + PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER_DEFAULT); + } + + /** + * Common enablement gate shared by TaskBlock instrumentations. + * + *

The producer is useful only when ddprof and its effective wall-clock engine are enabled, + * wall precheck is requested, and the wall profiler samples all threads. In context-only scope, + * the existing context filter already excludes out-of-context threads. + */ + public static boolean isEnabled(final Config config, final ConfigProvider configProvider) { + return config.isDatadogProfilerEnabled() + && isWallClockProfilerEnabled(configProvider) + && isWallPrecheckEnabled(configProvider) + && isAllThreadWallScope(configProvider); + } + + private static boolean isWallClockProfilerEnabled(final ConfigProvider configProvider) { + boolean ultraMinimal = configProvider.getBoolean(PROFILING_ULTRA_MINIMAL, false); + boolean tracingEnabled = configProvider.getBoolean(TRACE_ENABLED, true); + boolean enabledByDefault = !ultraMinimal && tracingEnabled && !isJ9(); + return getDdprofBoolean( + configProvider, PROFILING_DATADOG_PROFILER_WALL_ENABLED, enabledByDefault); + } + + private static boolean getDdprofBoolean( + ConfigProvider configProvider, String key, boolean defaultValue) { + return configProvider.getBoolean( + key, configProvider.getBoolean(key.replace(".ddprof.", ".async."), defaultValue)); + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfigTest.java b/internal-api/src/test/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfigTest.java new file mode 100644 index 00000000000..dfb223dfb55 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/profiling/TaskBlockInstrumentationConfigTest.java @@ -0,0 +1,118 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.api.profiling; + +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_PRECHECK; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_ULTRA_MINIMAL; +import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.api.Config; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class TaskBlockInstrumentationConfigTest { + + @Test + void enabledRequiresEverySharedGate() { + Config config = config(true); + + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, true, true, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config(false), configProvider(true, true, true, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(false, true, true, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, false, true, false, false))); + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, true, false, false, false))); + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config, configProvider(true, true, true, true, false))); + } + + @Test + void disabledTracingForcesAllThreadScope() { + Properties properties = new Properties(); + properties.setProperty(TRACE_ENABLED, "false"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_ENABLED, "true"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_PRECHECK, "true"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, "true"); + + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(properties))); + assertTrue(TaskBlockInstrumentationConfig.isAllThreadWallScope(configProvider(properties))); + } + + @Test + void disabledTracingDisablesWallByDefault() { + Properties properties = new Properties(); + properties.setProperty(TRACE_ENABLED, "false"); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_PRECHECK, "true"); + + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(properties))); + } + + @Test + void defaultsDoNotEnableTaskBlockInstrumentation() { + assertFalse( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(new Properties()))); + assertTrue( + TaskBlockInstrumentationConfig.isAllThreadWallScope( + ConfigProvider.withPropertiesOverride(new Properties()))); + } + + @Test + void asyncAliasesRemainSupported() { + Properties properties = new Properties(); + properties.setProperty("profiling.async.wall.enabled", "true"); + properties.setProperty("profiling.async.wall.precheck", "true"); + properties.setProperty("profiling.async.wall.context.filter", "false"); + + assertTrue( + TaskBlockInstrumentationConfig.isEnabled( + config(true), ConfigProvider.withPropertiesOverride(properties))); + } + + private static Config config(boolean ddprofEnabled) { + Config config = mock(Config.class); + when(config.isDatadogProfilerEnabled()).thenReturn(ddprofEnabled); + return config; + } + + private static ConfigProvider configProvider( + boolean wallEnabled, + boolean wallPrecheck, + boolean tracingEnabled, + boolean contextFilter, + boolean ultraMinimal) { + Properties properties = new Properties(); + properties.setProperty(PROFILING_DATADOG_PROFILER_WALL_ENABLED, Boolean.toString(wallEnabled)); + properties.setProperty( + PROFILING_DATADOG_PROFILER_WALL_PRECHECK, Boolean.toString(wallPrecheck)); + properties.setProperty(TRACE_ENABLED, Boolean.toString(tracingEnabled)); + properties.setProperty( + PROFILING_DATADOG_PROFILER_WALL_CONTEXT_FILTER, Boolean.toString(contextFilter)); + properties.setProperty(PROFILING_ULTRA_MINIMAL, Boolean.toString(ultraMinimal)); + return configProvider(properties); + } + + private static ConfigProvider configProvider(Properties properties) { + return ConfigProvider.withPropertiesOverride(properties); + } +} diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 733f3736cbb..3b6fba6e674 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -2769,6 +2769,14 @@ "aliases": [] } ], + "DD_PROFILING_ASYNC_WALL_PRECHECK": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], "DD_PROFILING_AUXILIARY": [ { "version": "A", @@ -2981,7 +2989,7 @@ { "version": "A", "type": "boolean", - "default": "true", + "default": "false", "aliases": [] } ], @@ -2993,6 +3001,14 @@ "aliases": [] } ], + "DD_PROFILING_DDPROF_WALL_PRECHECK": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], "DD_PROFILING_DDPROF_WALL_INTERVAL_MS": [ { "version": "A", From b557c8bbfcd325b87f0211eeab9bd7288f01a8f8 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Thu, 16 Jul 2026 12:05:40 +0200 Subject: [PATCH 2/2] feat: add LockSupport TaskBlock profiling --- .../java/concurrent/LockSupportHelper.java | 88 +++++++++ .../concurrent/LockSupportHelperTest.java | 167 ++++++++++++++++++ .../ddprof/DatadogProfilingIntegration.java | 11 ++ .../bytebuddy/matcher/ignored_class_name.trie | 1 + .../profiling/lock-support/build.gradle | 12 ++ .../LockSupportProfilingInstrumentation.java | 109 ++++++++++++ ...kSupportProfilingDisabledForkedTest.groovy | 31 ++++ ...tProfilingInstrumentationForkedTest.groovy | 67 +++++++ .../TestProfilingContextIntegration.groovy | 30 ++++ .../LockSupportTaskBlockForkedApp.java | 127 +++++++++++++ .../LockSupportTaskBlockProfilingTest.java | 154 ++++++++++++++++ .../api/ProfilingContextIntegration.java | 14 ++ metadata/supported-configurations.json | 8 + settings.gradle.kts | 1 + 14 files changed, 820 insertions(+) create mode 100644 dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java create mode 100644 dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelperTest.java create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingDisabledForkedTest.groovy create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingInstrumentationForkedTest.groovy create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/LockSupportTaskBlockForkedApp.java create mode 100644 dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java new file mode 100644 index 00000000000..7f3a11f117e --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java @@ -0,0 +1,88 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import java.util.Collections; +import java.util.Map; +import java.util.WeakHashMap; + +/** Helper for profiling {@code LockSupport.park*} intervals from bootstrap instrumentation. */ +public final class LockSupportHelper { + /** + * Best-effort association between a parked thread and the most recent {@code unpark} caller's + * active span. Weak keys avoid retaining terminated threads; the synchronized wrapper protects + * calls made concurrently by parked and unparking threads. + */ + static final Map UNPARKING_SPAN = Collections.synchronizedMap(new WeakHashMap<>()); + + private LockSupportHelper() {} + + /** State required to balance a park entry accepted by a profiling integration. */ + public static final class ParkState { + private final ProfilingContextIntegration profiling; + private final long blockerHash; + + ParkState(ProfilingContextIntegration profiling, long blockerHash) { + this.profiling = profiling; + this.blockerHash = blockerHash; + } + } + + /** Captures a park entry through the currently installed profiling integration. */ + public static ParkState captureState(Object blocker) { + return captureState(blocker, AgentTracer.get().getProfilingContext()); + } + + static ParkState captureState(Object blocker, ProfilingContextIntegration profiling) { + if (profiling == null) { + return null; + } + try { + if (!profiling.parkEnter()) { + return null; + } + return new ParkState( + profiling, + blocker == null ? 0L : Integer.toUnsignedLong(System.identityHashCode(blocker))); + } catch (Throwable ignored) { + return null; + } + } + + /** Drains unpark attribution and balances an accepted park entry. */ + public static void finish(ParkState state) { + Long unblockingSpanId = UNPARKING_SPAN.remove(Thread.currentThread()); + finish(state, unblockingSpanId == null ? 0L : unblockingSpanId); + } + + static void finish(ParkState state, long unblockingSpanId) { + if (state == null) { + return; + } + try { + state.profiling.parkExit(state.blockerHash, unblockingSpanId); + } catch (Throwable ignored) { + } + } + + /** + * Records the latest unpark caller for {@code thread}. An untraced call explicitly clears an + * older traced caller so the association follows last-writer semantics. + */ + public static void recordUnpark(Thread thread) { + if (thread == null) { + return; + } + AgentSpan span = AgentTracer.activeSpan(); + AgentSpanContext context = span == null ? null : span.spanContext(); + if (context instanceof ProfilerContext) { + UNPARKING_SPAN.put(thread, ((ProfilerContext) context).getSpanId()); + } else { + UNPARKING_SPAN.remove(thread); + } + } +} diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelperTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelperTest.java new file mode 100644 index 00000000000..f7e8d4a4f46 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelperTest.java @@ -0,0 +1,167 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class LockSupportHelperTest { + private interface ProfilerSpanContext extends AgentSpanContext, ProfilerContext {} + + private AgentTracer.TracerAPI previousTracer; + + @BeforeEach + void setUp() { + previousTracer = AgentTracer.get(); + AgentTracer.forceRegister(tracerWithActiveSpan(null)); + LockSupportHelper.UNPARKING_SPAN.clear(); + } + + @AfterEach + void tearDown() { + LockSupportHelper.UNPARKING_SPAN.clear(); + AgentTracer.forceRegister(previousTracer); + } + + @Test + void nullIntegrationDoesNotAcceptEntry() { + assertNull(LockSupportHelper.captureState(new Object(), null)); + } + + @Test + void rejectedEntryDoesNotRequireExit() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.parkEnter()).thenReturn(false); + + LockSupportHelper.ParkState state = LockSupportHelper.captureState(new Object(), profiling); + LockSupportHelper.finish(state, 17L); + + assertNull(state); + verify(profiling).parkEnter(); + verify(profiling, never()).parkExit(0L, 17L); + } + + @Test + void acceptedEntryIsAlwaysPairedWithExit() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.parkEnter()).thenReturn(true); + Object blocker = new Object(); + + LockSupportHelper.ParkState state = LockSupportHelper.captureState(blocker, profiling); + LockSupportHelper.finish(state, 23L); + + assertNotNull(state); + verify(profiling).parkEnter(); + verify(profiling).parkExit(Integer.toUnsignedLong(System.identityHashCode(blocker)), 23L); + } + + @Test + void nullBlockerUsesZeroIdentity() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + when(profiling.parkEnter()).thenReturn(true); + + LockSupportHelper.finish(LockSupportHelper.captureState(null, profiling), 0L); + + verify(profiling).parkExit(0L, 0L); + } + + @Test + void entryAndExitFailuresDoNotEscapeInstrumentation() { + ProfilingContextIntegration entryFailure = mock(ProfilingContextIntegration.class); + doThrow(new IllegalStateException("enter")).when(entryFailure).parkEnter(); + assertNull(LockSupportHelper.captureState(null, entryFailure)); + + ProfilingContextIntegration exitFailure = mock(ProfilingContextIntegration.class); + when(exitFailure.parkEnter()).thenReturn(true); + doThrow(new IllegalStateException("exit")).when(exitFailure).parkExit(0L, 0L); + LockSupportHelper.ParkState state = LockSupportHelper.captureState(null, exitFailure); + + assertDoesNotThrow(() -> LockSupportHelper.finish(state, 0L)); + } + + @Test + void finishAlwaysDrainsUnparkAttribution() { + Thread current = Thread.currentThread(); + LockSupportHelper.UNPARKING_SPAN.put(current, 31L); + + LockSupportHelper.finish(null); + + assertNull(LockSupportHelper.UNPARKING_SPAN.get(current)); + } + + @Test + void latestTracedUnparkWins() { + Thread target = new Thread(() -> {}); + installActiveSpan(41L); + LockSupportHelper.recordUnpark(target); + installActiveSpan(42L); + + LockSupportHelper.recordUnpark(target); + + assertEquals(42L, LockSupportHelper.UNPARKING_SPAN.get(target)); + } + + @Test + void laterUntracedUnparkClearsOlderTracedCaller() { + Thread target = new Thread(() -> {}); + installActiveSpan(51L); + LockSupportHelper.recordUnpark(target); + AgentTracer.forceRegister(tracerWithActiveSpan(null)); + + LockSupportHelper.recordUnpark(target); + + assertNull(LockSupportHelper.UNPARKING_SPAN.get(target)); + } + + @Test + void nonProfilerSpanContextClearsOlderTracedCaller() { + Thread target = new Thread(() -> {}); + installActiveSpan(61L); + LockSupportHelper.recordUnpark(target); + AgentSpan span = mock(AgentSpan.class); + when(span.spanContext()).thenReturn(mock(AgentSpanContext.class)); + AgentTracer.forceRegister(tracerWithActiveSpan(span)); + + LockSupportHelper.recordUnpark(target); + + assertNull(LockSupportHelper.UNPARKING_SPAN.get(target)); + } + + @Test + void nullUnparkTargetIsNoOp() { + installActiveSpan(71L); + + LockSupportHelper.recordUnpark(null); + + assertEquals(0, LockSupportHelper.UNPARKING_SPAN.size()); + } + + private static void installActiveSpan(long spanId) { + AgentSpan span = mock(AgentSpan.class); + ProfilerSpanContext context = mock(ProfilerSpanContext.class); + when(span.spanContext()).thenReturn(context); + when(context.getSpanId()).thenReturn(spanId); + AgentTracer.forceRegister(tracerWithActiveSpan(span)); + } + + private static AgentTracer.TracerAPI tracerWithActiveSpan(AgentSpan span) { + AgentTracer.TracerAPI tracer = mock(AgentTracer.TracerAPI.class); + when(tracer.activeSpan()).thenReturn(span); + return tracer; + } +} diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 65462a46bce..2e56bdbccaf 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package com.datadog.profiling.ddprof; import datadog.trace.api.EndpointTracker; @@ -78,6 +79,16 @@ public String name() { return "ddprof"; } + @Override + public boolean parkEnter() { + return DDPROF.parkEnter(); + } + + @Override + public void parkExit(long blocker, long unblockingSpanId) { + DDPROF.parkExit(blocker, unblockingSpanId); + } + public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index 36cedaf6081..69ce02b0d84 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -70,6 +70,7 @@ 1 java.util.concurrent.ConcurrentHashMap* 1 java.util.concurrent.atomic.* 1 java.util.concurrent.locks.* +0 java.util.concurrent.locks.LockSupport 0 java.util.logging.* # allow capturing JVM shutdown 0 java.lang.Shutdown diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle new file mode 100644 index 00000000000..f2f91cdc7fd --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle @@ -0,0 +1,12 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + coreJdk() + } +} + +dependencies { + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java new file mode 100644 index 00000000000..805cc11d8a3 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -0,0 +1,109 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.locksupport; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isStatic; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Config; +import datadog.trace.api.profiling.TaskBlockInstrumentationConfig; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.bootstrap.instrumentation.java.concurrent.LockSupportHelper; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * Instruments {@link java.util.concurrent.locks.LockSupport} park and unpark operations for + * TaskBlock profiling. The native hooks own platform-thread park intervals only; virtual-thread + * calls are rejected by ddprof without touching carrier-thread ownership. + */ +@AutoService(InstrumenterModule.class) +public class LockSupportProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + /** Creates the LockSupport profiling instrumentation module. */ + public LockSupportProfilingInstrumentation() { + super("lock-support"); + } + + @Override + public boolean isEnabled() { + return super.isEnabled() + && TaskBlockInstrumentationConfig.isEnabled(Config.get(), ConfigProvider.getInstance()); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.util.concurrent.locks.LockSupport"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + parkMethod().and(takesArgument(0, named("java.lang.Object"))), + getClass().getName() + "$ParkWithBlockerAdvice"); + transformer.applyAdvice( + parkMethod().and(not(takesArgument(0, named("java.lang.Object")))), + getClass().getName() + "$ParkWithoutBlockerAdvice"); + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(named("unpark")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$UnparkAdvice"); + } + + private static ElementMatcher.Junction parkMethod() { + return isMethod() + .and(isStatic()) + .and(nameStartsWith("park")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))); + } + + /** Advice for park variants whose first argument is the blocker object. */ + public static final class ParkWithBlockerAdvice { + /** Starts the paired profiling lifecycle before the park. */ + @Advice.OnMethodEnter(suppress = Throwable.class) + public static LockSupportHelper.ParkState before(@Advice.Argument(0) Object blocker) { + return LockSupportHelper.captureState(blocker); + } + + /** Completes an accepted profiling lifecycle after the park. */ + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter LockSupportHelper.ParkState state) { + LockSupportHelper.finish(state); + } + } + + /** Advice for park variants without an explicit blocker object. */ + public static final class ParkWithoutBlockerAdvice { + /** Starts the paired profiling lifecycle before the park. */ + @Advice.OnMethodEnter(suppress = Throwable.class) + public static LockSupportHelper.ParkState before() { + return LockSupportHelper.captureState(null); + } + + /** Completes an accepted profiling lifecycle after the park. */ + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter LockSupportHelper.ParkState state) { + LockSupportHelper.finish(state); + } + } + + /** Advice that records the active span of the latest unpark caller. */ + public static final class UnparkAdvice { + /** Updates best-effort unpark attribution before dispatching to the JDK. */ + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before(@Advice.Argument(0) Thread thread) { + LockSupportHelper.recordUnpark(thread); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingDisabledForkedTest.groovy b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingDisabledForkedTest.groovy new file mode 100644 index 00000000000..2f88f4393d1 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingDisabledForkedTest.groovy @@ -0,0 +1,31 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.locksupport + +import datadog.trace.agent.test.InstrumentationSpecification + +import java.util.concurrent.locks.LockSupport + +class LockSupportProfilingDisabledForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + injectSysConfig('dd.profiling.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.precheck', 'false') + injectSysConfig('dd.profiling.ddprof.wall.context.filter', 'false') + super.configurePreAgent() + } + + def setup() { + TEST_PROFILING_CONTEXT_INTEGRATION.clear() + } + + def 'disabled TaskBlock gate leaves LockSupport uninstrumented'() { + when: + LockSupport.parkNanos(1L) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkEnterCalls.get() == 0 + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitCalls.get() == 0 + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingInstrumentationForkedTest.groovy b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingInstrumentationForkedTest.groovy new file mode 100644 index 00000000000..0e651f8a086 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/groovy/LockSupportProfilingInstrumentationForkedTest.groovy @@ -0,0 +1,67 @@ +// Copyright 2026 Datadog, Inc. +package datadog.trace.instrumentation.locksupport + +import datadog.trace.agent.test.InstrumentationSpecification + +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.LockSupport + +class LockSupportProfilingInstrumentationForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + injectSysConfig('dd.profiling.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.enabled', 'true') + injectSysConfig('dd.profiling.ddprof.wall.precheck', 'true') + injectSysConfig('dd.profiling.ddprof.wall.context.filter', 'false') + super.configurePreAgent() + } + + def setup() { + TEST_PROFILING_CONTEXT_INTEGRATION.clear() + } + + def cleanup() { + TEST_PROFILING_CONTEXT_INTEGRATION.clear() + } + + def 'transformed park with blocker dispatches one balanced lifecycle on the calling thread'() { + setup: + Object blocker = new Object() + Thread callingThread = Thread.currentThread() + + when: + LockSupport.parkNanos(blocker, TimeUnit.MILLISECONDS.toNanos(1)) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkEnterCalls.get() >= 1 + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitCalls.get() >= 1 + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitThreads.contains(callingThread) + TEST_PROFILING_CONTEXT_INTEGRATION.lastParkBlocker.get() == Integer.toUnsignedLong(System.identityHashCode(blocker)) + } + + def 'transformed park without blocker dispatches a zero blocker'() { + setup: + Thread callingThread = Thread.currentThread() + + when: + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkExitThreads.contains(callingThread) + TEST_PROFILING_CONTEXT_INTEGRATION.lastParkBlocker.get() == 0L + } + + def 'rejected entry is not followed by an exit'() { + setup: + TEST_PROFILING_CONTEXT_INTEGRATION.acceptParkEntries = false + Thread callingThread = Thread.currentThread() + + when: + LockSupport.parkNanos(1L) + + then: + TEST_PROFILING_CONTEXT_INTEGRATION.parkEnterCalls.get() >= 1 + !TEST_PROFILING_CONTEXT_INTEGRATION.parkExitThreads.contains(callingThread) + } +} diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy index 370646949e7..35119f9e175 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package datadog.trace.agent.test import datadog.trace.api.EndpointTracker @@ -13,8 +14,11 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import java.util.concurrent.BlockingDeque +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.LinkedBlockingDeque +import java.util.Set import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong import static datadog.trace.api.profiling.Timer.TimerType.QUEUEING @@ -22,6 +26,12 @@ class TestProfilingContextIntegration implements ProfilingContextIntegration { final AtomicInteger attachments = new AtomicInteger() final AtomicInteger detachments = new AtomicInteger() final AtomicInteger counter = new AtomicInteger() + final AtomicInteger parkEnterCalls = new AtomicInteger() + final AtomicInteger parkExitCalls = new AtomicInteger() + final AtomicLong lastParkBlocker = new AtomicLong() + final AtomicLong lastUnblockingSpanId = new AtomicLong() + final Set parkExitThreads = ConcurrentHashMap.newKeySet() + volatile boolean acceptParkEntries = true final BlockingDeque closedTimings = new LinkedBlockingDeque<>() final Logger logger = LoggerFactory.getLogger(TestProfilingContextIntegration) @Override @@ -37,6 +47,26 @@ class TestProfilingContextIntegration implements ProfilingContextIntegration { void clear() { attachments.set(0) detachments.set(0) + parkEnterCalls.set(0) + parkExitCalls.set(0) + lastParkBlocker.set(0) + lastUnblockingSpanId.set(0) + parkExitThreads.clear() + acceptParkEntries = true + } + + @Override + boolean parkEnter() { + parkEnterCalls.incrementAndGet() + return acceptParkEntries + } + + @Override + void parkExit(long blocker, long unblockingSpanId) { + parkExitCalls.incrementAndGet() + lastParkBlocker.set(blocker) + lastUnblockingSpanId.set(unblockingSpanId) + parkExitThreads.add(Thread.currentThread()) } @Override diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/LockSupportTaskBlockForkedApp.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/LockSupportTaskBlockForkedApp.java new file mode 100644 index 00000000000..dc8724e82fa --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/com/datadog/smoketest/profiling/LockSupportTaskBlockForkedApp.java @@ -0,0 +1,127 @@ +// Copyright 2026 Datadog, Inc. +package com.datadog.smoketest.profiling; + +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +/** Forked workload for platform and virtual-thread LockSupport TaskBlock smoke coverage. */ +public final class LockSupportTaskBlockForkedApp { + public static final String SPANLESS_PLATFORM_THREAD = "locksupport-spanless-platform"; + public static final String ACTIVE_PLATFORM_THREAD = "locksupport-active-platform"; + public static final String VIRTUAL_THREAD = "locksupport-virtual"; + public static final String SPANLESS_BLOCKER_MARKER = "LOCKSUPPORT_SPANLESS_BLOCKER="; + public static final String ACTIVE_BLOCKER_MARKER = "LOCKSUPPORT_ACTIVE_BLOCKER="; + public static final String VIRTUAL_BLOCKER_MARKER = "LOCKSUPPORT_VIRTUAL_BLOCKER="; + + private static final int PARK_ITERATIONS = 20; + private static final long PARK_NANOS = TimeUnit.MILLISECONDS.toNanos(40); + private static final Object SPANLESS_BLOCKER = new Object(); + private static final Object ACTIVE_BLOCKER = new Object(); + private static final Object VIRTUAL_BLOCKER = new Object(); + + private LockSupportTaskBlockForkedApp() {} + + /** Runs the smoke workload. */ + public static void main(String[] args) throws Exception { + printBlocker(SPANLESS_BLOCKER_MARKER, SPANLESS_BLOCKER); + printBlocker(ACTIVE_BLOCKER_MARKER, ACTIVE_BLOCKER); + printBlocker(VIRTUAL_BLOCKER_MARKER, VIRTUAL_BLOCKER); + Tracer tracer = GlobalTracer.get(); + runSpanlessPlatformWorker(tracer); + runActivePlatformWorker(tracer); + runVirtualWorker(); + Thread.sleep(1500L); + } + + private static void runSpanlessPlatformWorker(Tracer tracer) throws Exception { + CountDownLatch ready = new CountDownLatch(1); + Thread worker = + new Thread( + () -> { + ready.countDown(); + for (int i = 0; i < PARK_ITERATIONS; i++) { + LockSupport.parkNanos(SPANLESS_BLOCKER, PARK_NANOS); + } + }, + SPANLESS_PLATFORM_THREAD); + worker.start(); + if (!ready.await(5, TimeUnit.SECONDS)) { + throw new IllegalStateException("Spanless platform worker did not start"); + } + Thread.sleep(20L); + Span span = tracer.buildSpan("locksupport.unparker").start(); + try (Scope ignored = tracer.activateSpan(span)) { + LockSupport.unpark(worker); + } finally { + span.finish(); + } + join(worker); + } + + private static void runActivePlatformWorker(Tracer tracer) throws Exception { + Thread worker = + new Thread( + () -> { + Span span = tracer.buildSpan("locksupport.active").start(); + try (Scope ignored = tracer.activateSpan(span)) { + for (int i = 0; i < PARK_ITERATIONS; i++) { + LockSupport.parkNanos(ACTIVE_BLOCKER, PARK_NANOS); + } + } finally { + span.finish(); + } + }, + ACTIVE_PLATFORM_THREAD); + worker.start(); + join(worker); + } + + private static void runVirtualWorker() throws Exception { + Method startVirtualThread; + try { + startVirtualThread = Thread.class.getMethod("startVirtualThread", Runnable.class); + } catch (NoSuchMethodException unsupported) { + throw new IllegalStateException( + "Virtual-thread smoke coverage requires JDK 21+", unsupported); + } + Runnable workload = + () -> { + Thread.currentThread().setName(VIRTUAL_THREAD); + for (int i = 0; i < PARK_ITERATIONS; i++) { + LockSupport.parkNanos(VIRTUAL_BLOCKER, PARK_NANOS); + } + }; + Thread worker; + try { + worker = (Thread) startVirtualThread.invoke(null, workload); + } catch (InvocationTargetException error) { + Throwable cause = error.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new IllegalStateException(cause); + } + join(worker); + } + + private static void join(Thread worker) throws InterruptedException { + worker.join(TimeUnit.SECONDS.toMillis(10)); + if (worker.isAlive()) { + throw new IllegalStateException("Worker did not finish: " + worker.getName()); + } + } + + private static void printBlocker(String marker, Object blocker) { + System.out.println(marker + Integer.toUnsignedLong(System.identityHashCode(blocker))); + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java new file mode 100644 index 00000000000..87c0f556770 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java @@ -0,0 +1,154 @@ +// Copyright 2026 Datadog, Inc. +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.TaskBlockProfilingTestSupport.BLOCKER; +import static datadog.smoketest.TaskBlockProfilingTestSupport.LOCAL_ROOT_SPAN_ID; +import static datadog.smoketest.TaskBlockProfilingTestSupport.SPAN_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; + +import com.datadog.smoketest.profiling.LockSupportTaskBlockForkedApp; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +@DisabledOnJ9 +final class LockSupportTaskBlockProfilingTest { + private static final IAttribute UNBLOCKING_SPAN_ID = + attr("unblockingSpanId", "unblockingSpanId", "unblockingSpanId", NUMBER); + + private Path logFilePath; + private Path dumpDir; + + @BeforeEach + void setUp(TestInfo testInfo) throws IOException { + logFilePath = + TaskBlockProfilingTestSupport.buildLogFilePath( + LockSupportTaskBlockProfilingTest.class, testInfo, "lockSupport"); + dumpDir = TaskBlockProfilingTestSupport.createDumpDir("dd-profiler-locksupport-"); + } + + @AfterEach + void cleanUp() throws IOException { + TaskBlockProfilingTestSupport.deleteRecursively(dumpDir); + } + + @Test + void onlySpanlessPlatformParksEmitTaskBlocks() throws Exception { + Process targetProcess = + TaskBlockProfilingTestSupport.createTaskBlockProcessBuilder( + "smoke-test-locksupport-taskblock", + LockSupportTaskBlockForkedApp.class.getName(), + dumpDir, + logFilePath) + .start(); + + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + Stats stats = + new Stats( + markerValue(log, LockSupportTaskBlockForkedApp.SPANLESS_BLOCKER_MARKER), + markerValue(log, LockSupportTaskBlockForkedApp.ACTIVE_BLOCKER_MARKER), + markerValue(log, LockSupportTaskBlockForkedApp.VIRTUAL_BLOCKER_MARKER)); + for (IItemCollection events : TaskBlockProfilingTestSupport.loadDumpedEvents(dumpDir)) { + stats.add(events); + } + assertTrue(stats.spanlessPlatformCount > 0, "Expected spanless platform park TaskBlocks"); + assertEquals(0, stats.activePlatformCount, "Active-context parks must not emit TaskBlocks"); + assertEquals(0, stats.virtualCount, "Virtual-thread parks must not emit TaskBlocks"); + assertFalse(stats.spanlessHasContext, "Spanless park TaskBlocks must keep zero span context"); + assertTrue(stats.spanlessWithBlocker > 0, "Expected blocker identity on park TaskBlocks"); + assertTrue( + stats.spanlessWithUnblockingSpan > 0, "Expected best-effort unpark span attribution"); + assertFalse(stats.spanlessMissingThread, "TaskBlock events must resolve Event Thread"); + assertFalse( + TaskBlockProfilingTestSupport.logContainsAny( + logFilePath, + "Failed to handle exception in instrumentation for java.util.concurrent.locks.LockSupport", + "Unexpected checked exception from ddprof TaskBlock hook"), + "LockSupport TaskBlock instrumentation failed"); + } + + private static long markerValue(String log, String marker) { + int start = log.indexOf(marker); + assertTrue(start >= 0, "Missing workload marker " + marker); + start += marker.length(); + int end = start; + while (end < log.length() && Character.isDigit(log.charAt(end))) { + end++; + } + return Long.parseLong(log.substring(start, end)); + } + + private static final class Stats { + private final long spanlessBlocker; + private final long activeBlocker; + private final long virtualBlocker; + private long spanlessPlatformCount; + private long activePlatformCount; + private long virtualCount; + private long spanlessWithBlocker; + private long spanlessWithUnblockingSpan; + private boolean spanlessHasContext; + private boolean spanlessMissingThread; + + private Stats(long spanlessBlocker, long activeBlocker, long virtualBlocker) { + this.spanlessBlocker = spanlessBlocker; + this.activeBlocker = activeBlocker; + this.virtualBlocker = virtualBlocker; + } + + private void add(IItemCollection events) { + for (IItemIterable items : events.apply(ItemFilters.type("datadog.TaskBlock"))) { + TaskBlockProfilingTestSupport.assertFinalTaskBlockSchema(items); + IMemberAccessor threadAccessor = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + IMemberAccessor spanAccessor = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor rootAccessor = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blockerAccessor = BLOCKER.getAccessor(items.getType()); + IMemberAccessor unblockingAccessor = + UNBLOCKING_SPAN_ID.getAccessor(items.getType()); + for (IItem item : items) { + String thread = threadAccessor == null ? null : threadAccessor.getMember(item); + long blocker = blockerAccessor.getMember(item).longValue(); + if (blocker == spanlessBlocker) { + spanlessPlatformCount++; + spanlessHasContext |= + spanAccessor.getMember(item).longValue() != 0L + || rootAccessor.getMember(item).longValue() != 0L; + spanlessWithBlocker++; + spanlessWithUnblockingSpan += + unblockingAccessor.getMember(item).longValue() == 0L ? 0 : 1; + spanlessMissingThread |= + thread == null + || thread.isEmpty() + || !LockSupportTaskBlockForkedApp.SPANLESS_PLATFORM_THREAD.equals(thread); + } else if (blocker == activeBlocker) { + activePlatformCount++; + } else if (blocker == virtualBlocker) { + virtualCount++; + } + } + } + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java index 4accced983a..d78c8cde5dc 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java @@ -1,3 +1,4 @@ +// Copyright 2026 Datadog, Inc. package datadog.trace.bootstrap.instrumentation.api; import datadog.trace.api.EndpointCheckpointer; @@ -34,6 +35,19 @@ default int encodeResourceName(CharSequence constant) { return 0; } + /** + * Called before the current thread enters {@code LockSupport.park*}. + * + * @return {@code true} when the integration accepted the entry dispatch and therefore requires a + * matching {@link #parkExit(long, long)}, even if profiling stops before the park returns + */ + default boolean parkEnter() { + return false; + } + + /** Completes a {@code LockSupport.park*} dispatch accepted by {@link #parkEnter()}. */ + default void parkExit(long blocker, long unblockingSpanId) {} + String name(); final class NoOp implements ProfilingContextIntegration { diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 3b6fba6e674..8741e41f37e 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7825,6 +7825,14 @@ "aliases": ["DD_TRACE_INTEGRATION_KOTLIN_COROUTINE_ENABLED", "DD_INTEGRATION_KOTLIN_COROUTINE_ENABLED"] } ], + "DD_TRACE_LOCK_SUPPORT_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_LOCK_SUPPORT_ENABLED", "DD_INTEGRATION_LOCK_SUPPORT_ENABLED"] + } + ], "DD_TRACE_LEGACY_E2E_DURATION_ENABLED": [ { "version": "A", diff --git a/settings.gradle.kts b/settings.gradle.kts index 36b1d9b95df..08f5174dcc3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -338,6 +338,7 @@ include( ":dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin", ":dd-java-agent:instrumentation:datadog:profiling:enable-wallclock-profiling", ":dd-java-agent:instrumentation:datadog:profiling:exception-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:lock-support", ":dd-java-agent:instrumentation:datadog:tracing:trace-annotation", ":dd-java-agent:instrumentation:datanucleus-4.0.5", ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0",