fix(profiler): complete malloc/native-socket hook-prefix-skip stackwalking

ddprof-lib/src/main/cpp/counters.h

@@ -118,6 +118,8 @@
   X(JVMTI_STACKS_INIT_OK, "jvmti_stacks_init_ok")                             \
   X(JVMTI_STACKS_INIT_FAILED, "jvmti_stacks_init_failed")                     \
   X(JVMTI_STACKS_REQUESTED, "jvmti_stacks_requested")                         \
+  X(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND, "native_trace_hook_prefix_not_found") \
+  X(NATIVE_HOOK_MARK_RESOLVE_FAILED, "native_hook_mark_resolve_failed")       \
   X(JVMTI_STACKS_FAILED_WRONG_PHASE, "jvmti_stacks_failed_wrong_phase")       \
   X(JVMTI_STACKS_FAILED_OTHER, "jvmti_stacks_failed_other")                  \
   /* Delegated stacks dropped at slot-lock. Rec-lock drops from all recording  \

ddprof-lib/src/main/cpp/event.h

@@ -34,6 +34,7 @@ enum EventType {
     EXECUTION_SAMPLE,
     WALL_CLOCK_SAMPLE,
     MALLOC_SAMPLE,
+    SOCKET_SAMPLE,
     INSTRUMENTED_METHOD,
     METHOD_TRACE,
     ALLOC_SAMPLE,

ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp

@@ -125,6 +125,8 @@ inline EventType eventTypeFromBCI(jint bci_type) {
             return PARK_SAMPLE;
         case BCI_NATIVE_MALLOC:
             return MALLOC_SAMPLE;
+        case BCI_NATIVE_SOCKET:
+            return SOCKET_SAMPLE;
         default:
             // For unknown or invalid BCI types, default to EXECUTION_SAMPLE
             // This maintains backward compatibility and prevents undefined behavior
@@ -305,7 +307,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
     bool anchor_recovery_used = false;
 
     // Show extended frame types and stub frames for execution-type events
-    bool details = event_type <= MALLOC_SAMPLE || features.mixed;
+    bool details = event_type <= SOCKET_SAMPLE || features.mixed;
 
     if (details && vm_thread != NULL && VMThread::isJavaThread(vm_thread)) {
         anchor = vm_thread->anchor();
@@ -678,18 +680,15 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
         } else {
             // Resolve native frame (may use remote symbolication if enabled)
             Profiler::NativeFrameResolution resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)pc, lock_index);
-            if (resolution.is_marked) {
-                // This is a marked C++ interpreter frame, terminate scan
-                break;
-            }
-            const char* method_name = resolution.method_name;
-            int frame_bci = resolution.bci;
-            char mark;
-            if (frame_bci != BCI_NATIVE_FRAME_REMOTE && method_name != NULL && (mark = NativeFunc::read_mark(method_name)) != 0) {
-                if (mark == MARK_ASYNC_PROFILER && event_type == MALLOC_SAMPLE) {
-                    // Skip all internal frames above malloc_hook functions, leave the hook itself
+            if (resolution.is_marked()) {
+                if (resolution.mark == MARK_ASYNC_PROFILER &&
+                    isHookPrefixedSample(event_type)) {
+                    // Discard frames captured above the malloc/socket hook boundary,
+                    // excluding the hook's own frame, and resume from the real
+                    // caller above it — mirrors the FP/DWARF skip-prefix logic in
+                    // Profiler::convertNativeTrace.
                     depth = 0;
-                } else if (mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) {
+                } else if (resolution.mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) {
                     // Insert current compile task as a pseudo Java frame
                     VMMethod* method = vm_thread->compiledMethod();
                     if (method != nullptr) {
@@ -698,13 +697,19 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                             fillFrame(frames[depth++], FRAME_JIT_COMPILED, 0, method_id, method);
                         }
                     }
-                } else if (mark == MARK_THREAD_ENTRY) {
+                } else if (resolution.mark == MARK_THREAD_ENTRY) {
                     // Thread entry point detected via pre-computed mark - this is the root frame
-                    // No need for expensive symbol resolution, just stop unwinding
                     Counters::increment(THREAD_ENTRY_MARK_DETECTIONS);
                     break;
+                } else {
+                    // Other marks (VM runtime / interpreter) terminate the scan.
+                    break;
                 }
-            } else if (method_name == NULL && details && !anchor_recovery_used
+                goto dwarf_unwind;
+            }
+            const char* method_name = resolution.method_name;
+            int frame_bci = resolution.bci;
+            if (method_name == NULL && details && !anchor_recovery_used
                        && profiler->findLibraryByAddress(pc) == NULL) {
                 // Try anchor recovery — prefer live anchor, fall back to saved data
                 anchor_recovery_used = true;
@@ -1194,7 +1199,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) {
   int java_frames = 0;
   if (features.mixed) {
     java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated);
-  } else if (request.event_type == BCI_NATIVE_MALLOC || request.event_type == BCI_NATIVE_SOCKET) {
+  } else if (isHookPrefixedSample(request.event_type)) {
     if (cstack >= CSTACK_VM) {
       java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated);
     } else {

ddprof-lib/src/main/cpp/jfrMetadata.cpp

@@ -303,8 +303,8 @@ void JfrMetadata::initialize(
               << field("name", T_STRING, "Name")
               << field("count", T_LONG, "Count"))
 
-          << (type("profiler.Malloc", T_MALLOC, "malloc")
-                  << category("Java Virtual Machine", "Native Memory")
+          << (type("datadog.NativeMemoryAllocation", T_MALLOC, "Native Memory Allocation")
+                  << category("Datadog", "Profiling")
                   << field("startTime", T_LONG, "Start Time", F_TIME_TICKS)
                   << field("eventThread", T_THREAD, "Event Thread", F_CPOOL)
                   << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL)

ddprof-lib/src/main/cpp/nativeSocketSampler.cpp

@@ -7,8 +7,11 @@
 
 #if defined(__linux__)
 
+#include "codeCache.h"
 #include "common.h"
+#include "counters.h"
 #include "flightRecorder.h"
+#include "libraries.h"
 #include "libraryPatcher.h"
 #include "log.h"
 #include "os.h"
@@ -27,6 +30,30 @@
 static thread_local PoissonSampler _send_sampler;
 static thread_local PoissonSampler _recv_sampler;
 
+// Marks the hook wrapper's own symbol as MARK_ASYNC_PROFILER so native call-stack
+// unwinding (Profiler::convertNativeTrace) can recognize the boundary between
+// profiler-internal frames and the real caller, mirroring MallocHooker::initialize().
+// Resolved by address (not by symbol-name predicate) because these are mangled
+// C++ static member functions, unlike malloc_hook's extern "C" free functions.
+// Returns false if the symbol could not be resolved/marked, in which case the
+// hook boundary is never recognized and every socket sample's native stack
+// comes back empty (see NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND).
+static bool markAsyncProfilerHook(void* fn_addr) {
+    CodeCache* lib = Libraries::instance()->findLibraryByAddress(fn_addr);
+    if (lib == nullptr) {
+        Counters::increment(NATIVE_HOOK_MARK_RESOLVE_FAILED);
+        return false;
+    }
+    const char* name = nullptr;
+    lib->binarySearch(fn_addr, &name);
+    if (name == nullptr) {
+        Counters::increment(NATIVE_HOOK_MARK_RESOLVE_FAILED);
+        return false;
+    }
+    NativeFunc::set_mark(name, MARK_ASYNC_PROFILER);
+    return true;
+}
+
 // Debug-only hook-fire counters, paired with TEST_LOG (common.h). Gated at
 // compile time to keep release hot paths free of cross-thread atomic writes.
 #ifdef DEBUG
@@ -386,6 +413,18 @@ Error NativeSocketSampler::start(Arguments &args) {
     TEST_LOG("NativeSocketSampler::start interval_ticks=%ld tsc_freq=%llu",
              init_interval, (unsigned long long)TSC::frequency());
 #endif
+    bool hooks_marked = markAsyncProfilerHook((void*)&NativeSocketSampler::send_hook);
+    hooks_marked &= markAsyncProfilerHook((void*)&NativeSocketSampler::recv_hook);
+    hooks_marked &= markAsyncProfilerHook((void*)&NativeSocketSampler::write_hook);
+    hooks_marked &= markAsyncProfilerHook((void*)&NativeSocketSampler::read_hook);
+    if (!hooks_marked) {
+        // Not fatal: hooks are still installed and sampling still works, but
+        // native stacks for socket samples will come back empty because the
+        // hook boundary frame can't be recognized during unwinding.
+        Log::warn("NativeSocketSampler: failed to mark one or more hook symbols; "
+                  "native call stacks for socket samples may be empty");
+    }
+
     if (!LibraryPatcher::patch_socket_functions()) {
         return Error("failed to install native socket hooks (dlsym returned NULL)");
     }

ddprof-lib/src/main/cpp/profiler.cpp

@@ -294,6 +294,7 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
   if (_cstack == CSTACK_NO ||
       (event_type == BCI_ALLOC || event_type == BCI_ALLOC_OUTSIDE_TLAB) ||
       (event_type != BCI_CPU && event_type != BCI_WALL &&
+       !isHookPrefixedSample(event_type) &&
        _cstack == CSTACK_DEFAULT)) {
     return 0;
   }
@@ -318,7 +319,9 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
                                          java_ctx, truncated);
   }
 
-  return convertNativeTrace(native_frames, callchain, frames, lock_index);
+  bool skip_hook_prefix = isHookPrefixedSample(event_type);
+  return convertNativeTrace(native_frames, callchain, frames, lock_index,
+                            skip_hook_prefix);
 }
 
 /**
@@ -375,24 +378,27 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
     char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0;
 
     if (mark != 0) {
-      return {nullptr, BCI_NATIVE_FRAME, true};  // Marked - stop processing
+      return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, mark);  // Marked - caller dispatches on mark
     }
 
     // Pack remote symbolication data using utility struct
     uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase();
     uint32_t lib_index = (uint32_t)lib->libIndex();
     unsigned long packed = RemoteFramePacker::pack(pc_offset, mark, lib_index);
 
-    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false);
+    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE);
   }
 
   // Traditional symbol resolution
   const char *method_name = nullptr;
   if (lib != nullptr) {
     lib->binarySearch((void*)pc, &method_name);
   }
-  if (method_name != nullptr && NativeFunc::is_marked(method_name)) {
-    return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, true);
+  if (method_name != nullptr) {
+    char mark = NativeFunc::read_mark(method_name);
+    if (mark != 0) {
+      return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, mark);
+    }
   }
 
   // No symbol but known library: pack for library-relative identification.
@@ -402,10 +408,10 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
     uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase();
     uint32_t lib_index = (uint32_t)lib->libIndex();
     unsigned long packed = RemoteFramePacker::pack(pc_offset, 0, lib_index);
-    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false);
+    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE);
   }
 
-  return NativeFrameResolution(method_name, BCI_NATIVE_FRAME, false);
+  return NativeFrameResolution(method_name, BCI_NATIVE_FRAME);
 }
 
 /**
@@ -418,9 +424,16 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
  * marked frames (JVM internals) that should terminate the stack walk.
  */
 int Profiler::convertNativeTrace(int native_frames, const void **callchain,
-                                 ASGCT_CallFrame *frames, int lock_index) {
+                                 ASGCT_CallFrame *frames, int lock_index,
+                                 bool skip_hook_prefix) {
   int depth = 0;
   void* prev_identifier = NULL;  // Can be jmethodID or frame pointer for remote
+  // skip_hook_prefix: the walk started inside profiler-internal code (e.g. the
+  // malloc/socket hook call chain), not at an interrupted user PC. Discard frames
+  // until the hook wrapper's own MARK_ASYNC_PROFILER-marked frame is reached, then
+  // resume normally from the real caller. Other mark kinds still terminate the
+  // scan immediately, same as the non-skipping case.
+  bool skipping = skip_hook_prefix;
 
   for (int i = 0; i < native_frames; i++) {
     uintptr_t pc = (uintptr_t)callchain[i];
@@ -436,9 +449,15 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
         char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0;
 
         if (mark != 0) {
+          if (skip_hook_prefix && mark == MARK_ASYNC_PROFILER) {
+            depth = 0;
+            skipping = false;
+            continue;
+          }
           // Terminate scan at marked frame
           return depth;
         }
+        if (skipping) continue;
 
         // Populate remote frame inline - no allocation needed!
         // Pass the mark we already retrieved to avoid duplicate binarySearch
@@ -456,10 +475,19 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
 
     // Fallback: Traditional symbol resolution
     const char *method_name = findNativeMethod((void*)pc);
-    if (method_name != nullptr && NativeFunc::is_marked(method_name)) {
-      // Terminate scan at marked frame
-      return depth;
+    if (method_name != nullptr) {
+      char mark = NativeFunc::read_mark(method_name);
+      if (mark != 0) {
+        if (skip_hook_prefix && mark == MARK_ASYNC_PROFILER) {
+          depth = 0;
+          skipping = false;
+          continue;
+        }
+        // Terminate scan at marked frame
+        return depth;
+      }
     }
+    if (skipping) continue;
 
     // Store standard frame
     jmethodID current_method = (jmethodID)method_name;
@@ -473,6 +501,11 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
     }
   }
 
+  if (skipping) {
+    // The hook-boundary (MARK_ASYNC_PROFILER) frame was never found in the
+    // callchain; every frame was discarded and the sample has no native stack.
+    Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+  }
   return depth;
 }
 

ddprof-lib/src/main/cpp/profiler.h

@@ -41,6 +41,19 @@ __asm__(".symver exp,exp@GLIBC_2.17");
 #endif
 #endif
 
+// A "hook-prefixed" sample (malloc, socket I/O, ...) is one whose native
+// callchain starts inside the profiler's own PLT/libc hook and must be
+// trimmed up to the MARK_ASYNC_PROFILER boundary frame before symbolication.
+// Single source of truth for that predicate across both BCI_* (FP/DWARF path)
+// and EventType (walkVM path) representations.
+inline bool isHookPrefixedSample(jint bci) {
+  return bci == BCI_NATIVE_MALLOC || bci == BCI_NATIVE_SOCKET;
+}
+
+inline bool isHookPrefixedSample(EventType event_type) {
+  return event_type == MALLOC_SAMPLE || event_type == SOCKET_SAMPLE;
+}
+
 #ifdef DEBUG
 #include <signal.h>
 static const char* force_stackwalk_crash_env = getenv("DDPROF_FORCE_STACKWALK_CRASH");
@@ -330,12 +343,13 @@ class alignas(alignof(SpinLock)) Profiler {
    *   Bits 47-61:  lib_index (15 bits, 32K libraries)
    *.  Bits 62-63:  reserved
    *
-   * Mark values indicate JVM internal frames that should terminate stack walks:
+   * Mark values identify JVM-internal frames the caller must dispatch on;
+   * they don't all mean "terminate the walk":
    *   0 = no mark (regular native frame)
-   *   MARK_VM_RUNTIME = 1
-   *   MARK_INTERPRETER = 2
-   *   MARK_COMPILER_ENTRY = 3
-   *   MARK_ASYNC_PROFILER = 4
+   *   MARK_VM_RUNTIME = 1      -- terminates the scan
+   *   MARK_INTERPRETER = 2     -- terminates the scan
+   *   MARK_COMPILER_ENTRY = 3  -- inserts a pseudo JIT-compile-task frame, then resumes unwinding
+   *   MARK_ASYNC_PROFILER = 4  -- resets depth and resumes unwinding above the hook boundary
    *
    * During stack walking, we perform symbol resolution (binarySearch) to check
    * marks and pack the mark value for later use. The performance is O(log n) for
@@ -389,20 +403,25 @@ class alignas(alignof(SpinLock)) Profiler {
   struct NativeFrameResolution {
     union {
       unsigned long packed_remote_frame;  // Packed remote frame data (pc_offset|mark|lib_index)
-      const char* method_name;            // Resolved method name 
+      const char* method_name;            // Resolved method name
     };
     int bci;                            // BCI_NATIVE_FRAME_REMOTE or BCI_NATIVE_FRAME
-    bool is_marked;                     // true if this is a marked C++ interpreter frame (stop processing)
-    NativeFrameResolution(const char* name, int bci_type, bool marked)
-      : method_name(name), bci(bci_type), is_marked(marked) {}
-    NativeFrameResolution(unsigned long packed, int bci_type, bool marked)
-      : packed_remote_frame(packed), bci(bci_type), is_marked(marked) {}
+    char mark;                         // the Mark value for this frame, 0 if unmarked
+    // True if this frame carries a JVM-internal mark; caller must inspect
+    // `mark` to decide whether to terminate the walk or dispatch/resume
+    // (see the Mark values table above resolveNativeFrameForWalkVM).
+    bool is_marked() const { return mark != 0; }
+    NativeFrameResolution(const char* name, int bci_type, char mark_value = 0)
+      : method_name(name), bci(bci_type), mark(mark_value) {}
+    NativeFrameResolution(unsigned long packed, int bci_type, char mark_value = 0)
+      : packed_remote_frame(packed), bci(bci_type), mark(mark_value) {}
   };
 
   void populateRemoteFrame(ASGCT_CallFrame* frame, uintptr_t pc, CodeCache* lib, char mark);
   NativeFrameResolution resolveNativeFrameForWalkVM(uintptr_t pc, int lock_index);
   int convertNativeTrace(int native_frames, const void **callchain,
-                         ASGCT_CallFrame *frames, int lock_index);
+                         ASGCT_CallFrame *frames, int lock_index,
+                         bool skip_hook_prefix);
   bool recordSample(void *ucontext, u64 weight, int tid, jint event_type,
                     u64 call_trace_id, Event *event,
                     u64 *recorded_call_trace_id = nullptr);