From f2abd2c602303059f061e29bbd0bf0494661ae2f Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 14 Jul 2026 08:45:32 -0400 Subject: [PATCH 1/5] Recalibrate the UTF8 caches every N spans instead of per span MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TraceMapperV0_4 called TAG_CACHE.recalibrate()/VALUE_CACHE.recalibrate() on every span. recalibrate() is O(cacheSize) (decay every entry, purge cold ones, retune the promotion threshold), so per-span it dominated serializer-thread CPU — a PetClinic multi-endpoint JFR put GenerationalUtf8Cache.recalibrate at ~17% of the dd-trace-processor thread's samples, its #1 hotspot. Per-span was never the intended cadence: the caches adapt to shifts in tag-value cardinality on a timescale far longer than one span, so recalibrating once every RECALIBRATE_SPAN_INTERVAL (512) spans preserves their effectiveness while cutting recalibrate calls ~500x. Gated via a span counter and a pure, unit-tested shouldRecalibrate() predicate. Correctness of the caches is unaffected (recalibrate only decays/retunes; it never changes returned bytes). The win is CPU on the background serializer thread, which converts to application-thread throughput/latency under core contention (busy boxes, CPU-limited containers) and protects GC/JIT headroom elsewhere. Co-Authored-By: Claude Opus 4.8 --- .../writer/ddagent/TraceMapperV0_4.java | 22 ++++++++++- .../TraceMapperV0_4RecalibrateTest.java | 38 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java index a44ecc6aab1..82e6bf76c5f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import okhttp3.RequestBody; public final class TraceMapperV0_4 implements TraceMapper { @@ -36,6 +37,21 @@ public final class TraceMapperV0_4 implements TraceMapper { ? new GenerationalUtf8Cache(Config.get().getTagValueUtf8CacheSize()) : null; + // The UTF8 caches are recalibrated periodically (decay cold entries, retune the promotion + // threshold) rather than on every span. recalibrate() is O(cacheSize); driven per-span it was the + // single largest consumer of serializer-thread CPU (a PetClinic JFR put + // GenerationalUtf8Cache.recalibrate at ~17% of that thread). The cache adapts to shifts in + // tag-value cardinality on a timescale far longer than a single span, so recalibrating once every + // RECALIBRATE_SPAN_INTERVAL spans preserves its effectiveness while dropping the per-span cost. + // Must be a power of two (see the mask in shouldRecalibrate). + static final long RECALIBRATE_SPAN_INTERVAL = 512; + private static final AtomicLong SPAN_COUNTER = new AtomicLong(); + + /** True once every {@link #RECALIBRATE_SPAN_INTERVAL} spans; pure so it can be unit-tested. */ + static boolean shouldRecalibrate(long spanCount) { + return (spanCount & (RECALIBRATE_SPAN_INTERVAL - 1)) == 0; + } + private final int size; private boolean firstSpanWritten; @@ -68,8 +84,10 @@ MetaWriter forSpan(boolean firstInTrace, boolean lastInTrace, boolean firstInPay @Override public void accept(Metadata metadata) { - if (TAG_CACHE != null) TAG_CACHE.recalibrate(); - if (VALUE_CACHE != null) VALUE_CACHE.recalibrate(); + if (shouldRecalibrate(SPAN_COUNTER.incrementAndGet())) { + if (TAG_CACHE != null) TAG_CACHE.recalibrate(); + if (VALUE_CACHE != null) VALUE_CACHE.recalibrate(); + } TagMap tags = metadata.getTags(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java new file mode 100644 index 00000000000..aade95ccc1d --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java @@ -0,0 +1,38 @@ +package datadog.trace.common.writer.ddagent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class TraceMapperV0_4RecalibrateTest { + + @Test + void recalibratesOncePerInterval() { + final long interval = TraceMapperV0_4.RECALIBRATE_SPAN_INTERVAL; + + // The span counter starts at 1 (incrementAndGet), so no recalibrate until the first boundary. + assertFalse(TraceMapperV0_4.shouldRecalibrate(1)); + assertFalse(TraceMapperV0_4.shouldRecalibrate(interval - 1)); + assertTrue(TraceMapperV0_4.shouldRecalibrate(interval)); + assertFalse(TraceMapperV0_4.shouldRecalibrate(interval + 1)); + assertTrue(TraceMapperV0_4.shouldRecalibrate(2 * interval)); + + // Exactly one recalibrate per `interval` spans across a full sweep. + int recalibrations = 0; + for (long count = 1; count <= 4 * interval; count++) { + if (TraceMapperV0_4.shouldRecalibrate(count)) { + recalibrations++; + } + } + assertEquals(4, recalibrations); + } + + @Test + void intervalIsAPowerOfTwo() { + // shouldRecalibrate uses a bit mask, which is only correct for a power-of-two interval. + final long interval = TraceMapperV0_4.RECALIBRATE_SPAN_INTERVAL; + assertEquals(0, interval & (interval - 1), "RECALIBRATE_SPAN_INTERVAL must be a power of two"); + } +} From 0a7d73cd5b06e427f1d658e867ef305a710d7b5d Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 14 Jul 2026 09:37:10 -0400 Subject: [PATCH 2/5] Address review: simpler comment, counter inside shouldRecalibrate, cache guard - Trim the RECALIBRATE_SPAN_INTERVAL comment to explain intent (control recalibration frequency) without the profiling numbers (those live in the PR). - Move the span-counter increment inside shouldRecalibrate() so the gate is self-contained. - Only consult shouldRecalibrate() when a cache is actually present, so the cache-off config pays nothing (and the static-final guard folds away when caches are enabled). - Test now asserts the window-invariant (any 2*interval window crosses exactly two boundaries), since shouldRecalibrate() is now stateful. Co-Authored-By: Claude Opus 4.8 --- .../writer/ddagent/TraceMapperV0_4.java | 19 ++++++++----------- .../TraceMapperV0_4RecalibrateTest.java | 19 ++++++------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java index 82e6bf76c5f..d36fdde057a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java @@ -37,19 +37,16 @@ public final class TraceMapperV0_4 implements TraceMapper { ? new GenerationalUtf8Cache(Config.get().getTagValueUtf8CacheSize()) : null; - // The UTF8 caches are recalibrated periodically (decay cold entries, retune the promotion - // threshold) rather than on every span. recalibrate() is O(cacheSize); driven per-span it was the - // single largest consumer of serializer-thread CPU (a PetClinic JFR put - // GenerationalUtf8Cache.recalibrate at ~17% of that thread). The cache adapts to shifts in - // tag-value cardinality on a timescale far longer than a single span, so recalibrating once every - // RECALIBRATE_SPAN_INTERVAL spans preserves its effectiveness while dropping the per-span cost. - // Must be a power of two (see the mask in shouldRecalibrate). + // Controls how often the UTF8 caches are recalibrated. The caches adapt to shifts in tag-value + // cardinality over a timescale much longer than a single span, so recalibrating periodically + // rather than per-span preserves their effectiveness without paying recalibrate()'s O(cacheSize) + // cost on every span. Must be a power of two (see the mask in shouldRecalibrate). static final long RECALIBRATE_SPAN_INTERVAL = 512; private static final AtomicLong SPAN_COUNTER = new AtomicLong(); - /** True once every {@link #RECALIBRATE_SPAN_INTERVAL} spans; pure so it can be unit-tested. */ - static boolean shouldRecalibrate(long spanCount) { - return (spanCount & (RECALIBRATE_SPAN_INTERVAL - 1)) == 0; + /** True once every {@link #RECALIBRATE_SPAN_INTERVAL} spans; advances the span counter. */ + static boolean shouldRecalibrate() { + return (SPAN_COUNTER.incrementAndGet() & (RECALIBRATE_SPAN_INTERVAL - 1)) == 0; } private final int size; @@ -84,7 +81,7 @@ MetaWriter forSpan(boolean firstInTrace, boolean lastInTrace, boolean firstInPay @Override public void accept(Metadata metadata) { - if (shouldRecalibrate(SPAN_COUNTER.incrementAndGet())) { + if ((TAG_CACHE != null || VALUE_CACHE != null) && shouldRecalibrate()) { if (TAG_CACHE != null) TAG_CACHE.recalibrate(); if (VALUE_CACHE != null) VALUE_CACHE.recalibrate(); } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java index aade95ccc1d..770b7422210 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4RecalibrateTest.java @@ -1,8 +1,6 @@ package datadog.trace.common.writer.ddagent; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -12,21 +10,16 @@ class TraceMapperV0_4RecalibrateTest { void recalibratesOncePerInterval() { final long interval = TraceMapperV0_4.RECALIBRATE_SPAN_INTERVAL; - // The span counter starts at 1 (incrementAndGet), so no recalibrate until the first boundary. - assertFalse(TraceMapperV0_4.shouldRecalibrate(1)); - assertFalse(TraceMapperV0_4.shouldRecalibrate(interval - 1)); - assertTrue(TraceMapperV0_4.shouldRecalibrate(interval)); - assertFalse(TraceMapperV0_4.shouldRecalibrate(interval + 1)); - assertTrue(TraceMapperV0_4.shouldRecalibrate(2 * interval)); - - // Exactly one recalibrate per `interval` spans across a full sweep. + // shouldRecalibrate() advances a shared counter, so assert a property that holds regardless of + // the starting value: any window of 2*interval consecutive spans crosses exactly two interval + // boundaries. int recalibrations = 0; - for (long count = 1; count <= 4 * interval; count++) { - if (TraceMapperV0_4.shouldRecalibrate(count)) { + for (long i = 0; i < 2 * interval; i++) { + if (TraceMapperV0_4.shouldRecalibrate()) { recalibrations++; } } - assertEquals(4, recalibrations); + assertEquals(2, recalibrations); } @Test From 5b47a59e199ef41f49da4997ecf5bbfe87e3ae58 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 14 Jul 2026 09:54:36 -0400 Subject: [PATCH 3/5] Gate recalibrate via SPAN_COUNTER presence, cleaner call site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allocate SPAN_COUNTER only when a cache exists, and use its presence as the "recalibration enabled" gate inside shouldRecalibrate(). This drops the explicit (TAG_CACHE != null || VALUE_CACHE != null) guard from the call site — accept() now just checks shouldRecalibrate() — while keeping the cache-off config free (static-final null folds the whole block away). Co-Authored-By: Claude Opus 4.8 --- .../trace/common/writer/ddagent/TraceMapperV0_4.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java index d36fdde057a..55e9dbf2afc 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java @@ -42,11 +42,17 @@ public final class TraceMapperV0_4 implements TraceMapper { // rather than per-span preserves their effectiveness without paying recalibrate()'s O(cacheSize) // cost on every span. Must be a power of two (see the mask in shouldRecalibrate). static final long RECALIBRATE_SPAN_INTERVAL = 512; - private static final AtomicLong SPAN_COUNTER = new AtomicLong(); + + // Non-null only when a cache exists, so its presence doubles as the "recalibration enabled" + // gate: it keeps the call site a plain shouldRecalibrate() check, and (a static-final null when + // both caches are off) lets the JIT fold the whole recalibrate block away. + private static final AtomicLong SPAN_COUNTER = + (TAG_CACHE != null || VALUE_CACHE != null) ? new AtomicLong() : null; /** True once every {@link #RECALIBRATE_SPAN_INTERVAL} spans; advances the span counter. */ static boolean shouldRecalibrate() { - return (SPAN_COUNTER.incrementAndGet() & (RECALIBRATE_SPAN_INTERVAL - 1)) == 0; + return SPAN_COUNTER != null + && (SPAN_COUNTER.incrementAndGet() & (RECALIBRATE_SPAN_INTERVAL - 1)) == 0; } private final int size; @@ -81,7 +87,7 @@ MetaWriter forSpan(boolean firstInTrace, boolean lastInTrace, boolean firstInPay @Override public void accept(Metadata metadata) { - if ((TAG_CACHE != null || VALUE_CACHE != null) && shouldRecalibrate()) { + if (shouldRecalibrate()) { if (TAG_CACHE != null) TAG_CACHE.recalibrate(); if (VALUE_CACHE != null) VALUE_CACHE.recalibrate(); } From 8ccd4ff57616912122a67b242b475d5fc3a6abb9 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 14 Jul 2026 13:59:43 -0400 Subject: [PATCH 4/5] Suppress false-positive SpotBugs AT_STALE_THREAD_WRITE_OF_PRIMITIVE on firstSpanWritten firstSpanWritten is only touched on the single serializer thread. The new static AtomicLong recalibration counter makes SpotBugs treat this class as multithreaded, which retroactively flags the pre-existing non-volatile boolean. Suppress it, matching the same pattern already used in TracerHealthMetrics. Co-Authored-By: Claude Opus 4.8 --- .../datadog/trace/common/writer/ddagent/TraceMapperV0_4.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java index 55e9dbf2afc..a90370f617f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java @@ -17,6 +17,7 @@ import datadog.trace.core.Metadata; import datadog.trace.core.MetadataConsumer; import datadog.trace.core.PendingTrace; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; @@ -377,7 +378,11 @@ public int messageBufferSize() { return size; // 5MB } + // firstSpanWritten is only touched on the single serializer thread; SpotBugs flags it as a shared + // primitive only because the static AtomicLong recalibration counter makes it treat this class as + // multithreaded. Safe to suppress (same pattern as TracerHealthMetrics). @Override + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public void reset() { firstSpanWritten = false; } From 996c42e6aa1a1ab60ec1293f6c1d1bdc398509b2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 15 Jul 2026 08:39:59 -0400 Subject: [PATCH 5/5] Simplify recalibration counter to a plain long (no AtomicLong) Per review: the span counter drives a recalibration cadence, not an exact count, and the serializer is single-threaded, so AtomicLong is unnecessary -- a plain long suffices (a benign race would at most nudge when recalibrate fires). A static-final boolean gates it so the JIT still folds the whole block away when both caches are off. Also drops the now-useless firstSpanWritten SpotBugs suppression: with the AtomicLong gone the class is no longer flagged as multithreaded. Co-Authored-By: Claude Opus 4.8 --- .../writer/ddagent/TraceMapperV0_4.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java index a90370f617f..cbe9e54ee38 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java @@ -17,14 +17,12 @@ import datadog.trace.core.Metadata; import datadog.trace.core.MetadataConsumer; import datadog.trace.core.PendingTrace; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; import okhttp3.RequestBody; public final class TraceMapperV0_4 implements TraceMapper { @@ -44,16 +42,18 @@ public final class TraceMapperV0_4 implements TraceMapper { // cost on every span. Must be a power of two (see the mask in shouldRecalibrate). static final long RECALIBRATE_SPAN_INTERVAL = 512; - // Non-null only when a cache exists, so its presence doubles as the "recalibration enabled" - // gate: it keeps the call site a plain shouldRecalibrate() check, and (a static-final null when - // both caches are off) lets the JIT fold the whole recalibrate block away. - private static final AtomicLong SPAN_COUNTER = - (TAG_CACHE != null || VALUE_CACHE != null) ? new AtomicLong() : null; + // True when at least one cache exists. static-final so the JIT can fold the whole recalibrate + // block away when both caches are off. + private static final boolean RECALIBRATE = (TAG_CACHE != null || VALUE_CACHE != null); + + // Advanced by the single serializer thread. The value is a recalibration cadence, not an exact + // count, so a plain counter suffices -- no atomic/volatile needed; a benign race would at most + // nudge when recalibrate fires. + private static long spanCounter; /** True once every {@link #RECALIBRATE_SPAN_INTERVAL} spans; advances the span counter. */ static boolean shouldRecalibrate() { - return SPAN_COUNTER != null - && (SPAN_COUNTER.incrementAndGet() & (RECALIBRATE_SPAN_INTERVAL - 1)) == 0; + return RECALIBRATE && (++spanCounter & (RECALIBRATE_SPAN_INTERVAL - 1)) == 0; } private final int size; @@ -378,11 +378,7 @@ public int messageBufferSize() { return size; // 5MB } - // firstSpanWritten is only touched on the single serializer thread; SpotBugs flags it as a shared - // primitive only because the static AtomicLong recalibration counter makes it treat this class as - // multithreaded. Safe to suppress (same pattern as TracerHealthMetrics). @Override - @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public void reset() { firstSpanWritten = false; }