google · copybara-service · Jul 15, 2026
@@ -180,7 +180,7 @@ void MallocExtension_Internal_ProcessBackgroundActions() {
 
       if (Parameters::span_lifetime_tracking() ==
           tcmalloc::tcmalloc_internal::central_freelist_internal::
-              LifetimeTracking::kEnabled) {
+              PeriodicLifetimeTracking::kEnabled) {
         if (now - last_cfl_long_lived_check >= cfl_long_lived_check_period) {
           for (int i = 0; i < tcmalloc::tcmalloc_internal::kNumClasses; ++i) {
             tc_globals.central_freelist(i).HandleLongLivedSpans();

@@ -141,7 +141,12 @@ static constexpr size_t kSpanMoveStatBuckets =
 static constexpr size_t kSpansUsedStatBuckets =
     absl::bit_width(kMaxObjectsToMove);
 
-enum class LifetimeTracking : bool { kDisabled = false, kEnabled = true };
+enum class PeriodicLifetimeTracking : bool {
+  kDisabled = false,
+  kEnabled = true
+};
+
+enum class InlineLifetimeTracking : bool { kDisabled = false, kEnabled = true };
 
 template <size_t MaxSize, typename RunLength>
 inline int RecordSameSpanRuns(absl::Span<Span* const> batch,
@@ -182,12 +187,14 @@ class CentralFreeList {
         first_nonempty_index_(0),
         pages_per_span_(0),
         nonempty_(),
-        use_all_buckets_for_few_object_spans_(false) {}
+        use_all_buckets_for_few_object_spans_(false),
+        inline_lifetime_tracking_(InlineLifetimeTracking::kDisabled) {}
 
   CentralFreeList(const CentralFreeList&) = delete;
   CentralFreeList& operator=(const CentralFreeList&) = delete;
 
-  void Init(size_t size_class) ABSL_LOCKS_EXCLUDED(lock_);
+  void Init(size_t size_class, InlineLifetimeTracking inline_lifetime_tracking)
+      ABSL_LOCKS_EXCLUDED(lock_);
 
   // These methods all do internal locking.
 
@@ -426,11 +433,14 @@ class CentralFreeList {
   StatsCounters<kSpanMoveStatBuckets> long_lived_spans_moved_;
 
   ABSL_ATTRIBUTE_NO_UNIQUE_ADDRESS Forwarder forwarder_;
+
+  InlineLifetimeTracking inline_lifetime_tracking_;
 };
 
 // Like a constructor and hence we disable thread safety analysis.
 template <class Forwarder>
-inline void CentralFreeList<Forwarder>::Init(size_t size_class)
+inline void CentralFreeList<Forwarder>::Init(
+    size_t size_class, InlineLifetimeTracking inline_lifetime_tracking)
     ABSL_NO_THREAD_SAFETY_ANALYSIS {
   size_class_ = size_class;
   object_size_ = forwarder_.class_to_size(size_class);
@@ -456,6 +466,8 @@ inline void CentralFreeList<Forwarder>::Init(size_t size_class)
 
   TC_ASSERT_LE(absl::bit_width(objects_per_span_), kSpanUtilBucketCapacity);
   num_to_move_ = forwarder_.num_objects_to_move(size_class);
+  inline_lifetime_tracking_ = inline_lifetime_tracking;
+  inline_lifetime_tracking_ = InlineLifetimeTracking::kEnabled;
 }
 
 template <class Forwarder>
@@ -490,6 +502,16 @@ inline Span* CentralFreeList<Forwarder>::ReleaseToSpans(
     RecordSpanUtil(prev_bitwidth, /*increase=*/false);
     RecordSpanUtil(cur_bitwidth, /*increase=*/true);
   }
+  if (inline_lifetime_tracking_ == InlineLifetimeTracking::kEnabled) {
+    if (ABSL_PREDICT_FALSE(!span->is_long_lived_span())) {
+      const uint64_t now = forwarder_.clock_now();
+      const double freq = forwarder_.clock_frequency();
+      if ((now - span->AllocTime()) >
+          (absl::ToDoubleSeconds(kLongLivedSpanThreshold) * freq)) {
+        span->set_is_long_lived_span(true);
+      }
+    }
+  }
   // If span allocation changes so that it moved to a different nonempty_ list,
   // we remove it from the previous list and add it to the desired list indexed
   // by cur_index.
@@ -703,6 +725,18 @@ inline int CentralFreeList<Forwarder>::RemoveRange(absl::Span<void*> batch) {
         break;
       }
 
+      if (inline_lifetime_tracking_ == InlineLifetimeTracking::kEnabled) {
+        if (ABSL_PREDICT_FALSE(!span->is_long_lived_span())) {
+          const uint64_t now = forwarder_.clock_now();
+          const double frequency = forwarder_.clock_frequency();
+          const uint64_t threshold = static_cast<uint64_t>(
+              absl::ToDoubleSeconds(kLongLivedSpanThreshold) * frequency);
+          if (now - span->AllocTime() > threshold) {
+            span->set_is_long_lived_span(true);
+          }
+        }
+      }
+
       const uint16_t prev_allocated = span->Allocated();
       const uint8_t prev_bitwidth = absl::bit_width(prev_allocated);
       const uint8_t prev_index = span->nonempty_index();

@@ -49,7 +49,8 @@ void BM_Populate(benchmark::State& state) {
   int num_objects = 64 * 1024 * 1024 / object_size;
   const int num_batches = num_objects / batch_size;
 
-  Env env(object_size, span_bytes, batch_size);
+  Env env(object_size, span_bytes, batch_size,
+          central_freelist_internal::InlineLifetimeTracking::kDisabled);
 
   // Allocate an array large enough to hold 64 MiB of objects.
   std::vector<void*> buffer(num_objects);
@@ -99,7 +100,8 @@ void BM_Multithreaded(benchmark::State& state) {
 
   static Env* env;
   if (state.thread_index() == 0) {
-    env = new Env(object_size, span_bytes, batch_size);
+    env = new Env(object_size, span_bytes, batch_size,
+                  central_freelist_internal::InlineLifetimeTracking::kDisabled);
   }
 
   std::vector<void*> batch(batch_size);
@@ -134,7 +136,8 @@ void BM_MixAndReturn(benchmark::State& state) {
   int num_objects = 64 * 1024 * 1024 / object_size;
   const int num_batches = num_objects / batch_size;
 
-  Env env(object_size, span_bytes, batch_size);
+  Env env(object_size, span_bytes, batch_size,
+          central_freelist_internal::InlineLifetimeTracking::kDisabled);
 
   // Allocate an array large enough to hold 64 MiB of objects.
   std::vector<void*> buffer(num_objects);
@@ -187,7 +190,8 @@ void BM_SpanReuse(benchmark::State& state) {
   int num_objects = 64 * 1024 * 1024 / object_size;
   const int num_batches = num_objects / batch_size;
 
-  Env env(object_size, span_bytes, batch_size);
+  Env env(object_size, span_bytes, batch_size,
+          central_freelist_internal::InlineLifetimeTracking::kDisabled);
 
   // Array used to hold onto half of the objects
   std::vector<void*> held_objects(2 * num_objects);

@@ -117,13 +117,15 @@ template <class>
 inline constexpr bool always_false_v = false;
 
 void FuzzCFL(size_t object_size, Length num_pages, size_t num_objects_to_move,
-             const std::vector<Instruction>& instructions) {
+             const std::vector<Instruction>& instructions,
+             central_freelist_internal::InlineLifetimeTracking
+                 inline_lifetime_tracking) {
   // TODO(271282540): Add support for multiple size classes for fuzzing.
   if (!SizeMap::IsValidSizeClass(object_size, num_pages, num_objects_to_move)) {
     return;
   }
   CentralFreelistEnv env(object_size, Bytes(num_pages.in_bytes()),
-                         num_objects_to_move);
+                         num_objects_to_move, inline_lifetime_tracking);
   std::vector<void*> objects;
 
   for (const auto& instruction_wrapper : instructions) {
@@ -223,7 +225,9 @@ auto GetInstructionDomain() {
 FUZZ_TEST(CentralFreeListTest, FuzzCFL)
     .WithDomains(fuzztest::InRange<size_t>(0, kMaxSize), AnyLength(),
                  fuzztest::Arbitrary<size_t>(),
-                 fuzztest::VectorOf(GetInstructionDomain()));
+                 fuzztest::VectorOf(GetInstructionDomain()),
+                 fuzztest::Arbitrary<
+                     central_freelist_internal::InlineLifetimeTracking>());
 
 }  // namespace
 }  // namespace tcmalloc::tcmalloc_internal