diff --git a/tcmalloc/background.cc b/tcmalloc/background.cc index 563d9479b..6980a7215 100644 --- a/tcmalloc/background.cc +++ b/tcmalloc/background.cc @@ -180,7 +180,7 @@ void MallocExtension_Internal_ProcessBackgroundActions() { if (Parameters::span_lifetime_tracking() == tcmalloc::tcmalloc_internal::central_freelist_internal:: - LifetimeTracking::kEnabled) { + PeriodicLifetimeTracking::kEnabled) { if (now - last_cfl_long_lived_check >= cfl_long_lived_check_period) { for (int i = 0; i < tcmalloc::tcmalloc_internal::kNumClasses; ++i) { tc_globals.central_freelist(i).HandleLongLivedSpans(); diff --git a/tcmalloc/central_freelist.h b/tcmalloc/central_freelist.h index 2b93de377..899760365 100644 --- a/tcmalloc/central_freelist.h +++ b/tcmalloc/central_freelist.h @@ -141,7 +141,12 @@ static constexpr size_t kSpanMoveStatBuckets = static constexpr size_t kSpansUsedStatBuckets = absl::bit_width(kMaxObjectsToMove); -enum class LifetimeTracking : bool { kDisabled = false, kEnabled = true }; +enum class PeriodicLifetimeTracking : bool { + kDisabled = false, + kEnabled = true +}; + +enum class InlineLifetimeTracking : bool { kDisabled = false, kEnabled = true }; template inline int RecordSameSpanRuns(absl::Span batch, @@ -182,12 +187,14 @@ class CentralFreeList { first_nonempty_index_(0), pages_per_span_(0), nonempty_(), - use_all_buckets_for_few_object_spans_(false) {} + use_all_buckets_for_few_object_spans_(false), + inline_lifetime_tracking_(InlineLifetimeTracking::kDisabled) {} CentralFreeList(const CentralFreeList&) = delete; CentralFreeList& operator=(const CentralFreeList&) = delete; - void Init(size_t size_class) ABSL_LOCKS_EXCLUDED(lock_); + void Init(size_t size_class, InlineLifetimeTracking inline_lifetime_tracking) + ABSL_LOCKS_EXCLUDED(lock_); // These methods all do internal locking. @@ -426,11 +433,14 @@ class CentralFreeList { StatsCounters long_lived_spans_moved_; ABSL_ATTRIBUTE_NO_UNIQUE_ADDRESS Forwarder forwarder_; + + InlineLifetimeTracking inline_lifetime_tracking_; }; // Like a constructor and hence we disable thread safety analysis. template -inline void CentralFreeList::Init(size_t size_class) +inline void CentralFreeList::Init( + size_t size_class, InlineLifetimeTracking inline_lifetime_tracking) ABSL_NO_THREAD_SAFETY_ANALYSIS { size_class_ = size_class; object_size_ = forwarder_.class_to_size(size_class); @@ -456,6 +466,8 @@ inline void CentralFreeList::Init(size_t size_class) TC_ASSERT_LE(absl::bit_width(objects_per_span_), kSpanUtilBucketCapacity); num_to_move_ = forwarder_.num_objects_to_move(size_class); + inline_lifetime_tracking_ = inline_lifetime_tracking; + inline_lifetime_tracking_ = InlineLifetimeTracking::kEnabled; } template @@ -490,6 +502,16 @@ inline Span* CentralFreeList::ReleaseToSpans( RecordSpanUtil(prev_bitwidth, /*increase=*/false); RecordSpanUtil(cur_bitwidth, /*increase=*/true); } + if (inline_lifetime_tracking_ == InlineLifetimeTracking::kEnabled) { + if (ABSL_PREDICT_FALSE(!span->is_long_lived_span())) { + const uint64_t now = forwarder_.clock_now(); + const double freq = forwarder_.clock_frequency(); + if ((now - span->AllocTime()) > + (absl::ToDoubleSeconds(kLongLivedSpanThreshold) * freq)) { + span->set_is_long_lived_span(true); + } + } + } // If span allocation changes so that it moved to a different nonempty_ list, // we remove it from the previous list and add it to the desired list indexed // by cur_index. @@ -703,6 +725,18 @@ inline int CentralFreeList::RemoveRange(absl::Span batch) { break; } + if (inline_lifetime_tracking_ == InlineLifetimeTracking::kEnabled) { + if (ABSL_PREDICT_FALSE(!span->is_long_lived_span())) { + const uint64_t now = forwarder_.clock_now(); + const double frequency = forwarder_.clock_frequency(); + const uint64_t threshold = static_cast( + absl::ToDoubleSeconds(kLongLivedSpanThreshold) * frequency); + if (now - span->AllocTime() > threshold) { + span->set_is_long_lived_span(true); + } + } + } + const uint16_t prev_allocated = span->Allocated(); const uint8_t prev_bitwidth = absl::bit_width(prev_allocated); const uint8_t prev_index = span->nonempty_index(); diff --git a/tcmalloc/central_freelist_benchmark.cc b/tcmalloc/central_freelist_benchmark.cc index a6ae1b167..5917a35be 100644 --- a/tcmalloc/central_freelist_benchmark.cc +++ b/tcmalloc/central_freelist_benchmark.cc @@ -49,7 +49,8 @@ void BM_Populate(benchmark::State& state) { int num_objects = 64 * 1024 * 1024 / object_size; const int num_batches = num_objects / batch_size; - Env env(object_size, span_bytes, batch_size); + Env env(object_size, span_bytes, batch_size, + central_freelist_internal::InlineLifetimeTracking::kDisabled); // Allocate an array large enough to hold 64 MiB of objects. std::vector buffer(num_objects); @@ -99,7 +100,8 @@ void BM_Multithreaded(benchmark::State& state) { static Env* env; if (state.thread_index() == 0) { - env = new Env(object_size, span_bytes, batch_size); + env = new Env(object_size, span_bytes, batch_size, + central_freelist_internal::InlineLifetimeTracking::kDisabled); } std::vector batch(batch_size); @@ -134,7 +136,8 @@ void BM_MixAndReturn(benchmark::State& state) { int num_objects = 64 * 1024 * 1024 / object_size; const int num_batches = num_objects / batch_size; - Env env(object_size, span_bytes, batch_size); + Env env(object_size, span_bytes, batch_size, + central_freelist_internal::InlineLifetimeTracking::kDisabled); // Allocate an array large enough to hold 64 MiB of objects. std::vector buffer(num_objects); @@ -187,7 +190,8 @@ void BM_SpanReuse(benchmark::State& state) { int num_objects = 64 * 1024 * 1024 / object_size; const int num_batches = num_objects / batch_size; - Env env(object_size, span_bytes, batch_size); + Env env(object_size, span_bytes, batch_size, + central_freelist_internal::InlineLifetimeTracking::kDisabled); // Array used to hold onto half of the objects std::vector held_objects(2 * num_objects); diff --git a/tcmalloc/central_freelist_fuzz.cc b/tcmalloc/central_freelist_fuzz.cc index 4f594feda..58484179d 100644 --- a/tcmalloc/central_freelist_fuzz.cc +++ b/tcmalloc/central_freelist_fuzz.cc @@ -117,13 +117,15 @@ template inline constexpr bool always_false_v = false; void FuzzCFL(size_t object_size, Length num_pages, size_t num_objects_to_move, - const std::vector& instructions) { + const std::vector& instructions, + central_freelist_internal::InlineLifetimeTracking + inline_lifetime_tracking) { // TODO(271282540): Add support for multiple size classes for fuzzing. if (!SizeMap::IsValidSizeClass(object_size, num_pages, num_objects_to_move)) { return; } CentralFreelistEnv env(object_size, Bytes(num_pages.in_bytes()), - num_objects_to_move); + num_objects_to_move, inline_lifetime_tracking); std::vector objects; for (const auto& instruction_wrapper : instructions) { @@ -223,7 +225,9 @@ auto GetInstructionDomain() { FUZZ_TEST(CentralFreeListTest, FuzzCFL) .WithDomains(fuzztest::InRange(0, kMaxSize), AnyLength(), fuzztest::Arbitrary(), - fuzztest::VectorOf(GetInstructionDomain())); + fuzztest::VectorOf(GetInstructionDomain()), + fuzztest::Arbitrary< + central_freelist_internal::InlineLifetimeTracking>()); } // namespace } // namespace tcmalloc::tcmalloc_internal diff --git a/tcmalloc/central_freelist_test.cc b/tcmalloc/central_freelist_test.cc index 1248bd7c1..d31662c2a 100644 --- a/tcmalloc/central_freelist_test.cc +++ b/tcmalloc/central_freelist_test.cc @@ -390,7 +390,8 @@ namespace { using central_freelist_internal::kNumLists; using TypeParam = FakeCentralFreeListEnvironment< central_freelist_internal::CentralFreeList>; -using CentralFreeListTest = ::testing::TestWithParam; +using CentralFreeListTest = ::testing::TestWithParam>; TEST_P(CentralFreeListTest, IsolatedSmoke) { #if ABSL_HAVE_HWADDRESS_SANITIZER @@ -398,7 +399,9 @@ TEST_P(CentralFreeListTest, IsolatedSmoke) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; EXPECT_CALL(e.forwarder(), AllocateSpan).Times(1); absl::FixedArray batch(e.batch_size()); @@ -455,7 +458,9 @@ TEST_P(CentralFreeListTest, SpanUtilizationHistogram) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; constexpr size_t kNumSpans = 10; // Request kNumSpans spans. @@ -562,7 +567,9 @@ TEST_P(CentralFreeListTest, SinglePopulate) { // Make sure that we allocate up to kObjectsPerSpan objects in both the span // prioritization states. - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; // Try to fetch sufficiently large number of objects at startup. const int num_objects_to_fetch = kMaxObjectsToMove; std::vector objects(num_objects_to_fetch, nullptr); @@ -638,7 +645,9 @@ TEST_P(CentralFreeListTest, BitwidthIndexedNonEmptyLists) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; if (e.objects_per_span() <= 2 * kNumLists) { GTEST_SKIP() << "Skipping test as one hot encoding used for few object spans."; @@ -656,7 +665,9 @@ TEST_P(CentralFreeListTest, DirectIndexedEncodedNonEmptyLists) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; if (e.objects_per_span() > 2 * kNumLists) { GTEST_SKIP() << "Skipping test as one hot encoding not required."; } @@ -679,7 +690,9 @@ TEST_P(CentralFreeListTest, SpanPriority) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; // If the number of objects per span is less than 2, we do not use more than // one nonempty_ lists. So, we can not prioritize the spans based on how many @@ -868,7 +881,9 @@ TEST_P(CentralFreeListTest, HookTracing) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; static int insert_count = 0; static int remove_count = 0; @@ -903,7 +918,9 @@ TEST_P(CentralFreeListTest, SpanLifetime) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; // Skip the check for objects_per_span = 1 since such spans skip most of the // central freelist's logic. if (e.objects_per_span() == 1) { @@ -941,7 +958,9 @@ TEST_P(CentralFreeListTest, SpanAllocationTracker) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; const int objects_per_span = e.objects_per_span(); if (objects_per_span == 1) return; @@ -1015,8 +1034,9 @@ TEST_P(CentralFreeListTest, SameSpans) { #ifdef TCMALLOC_INTERNAL_LEGACY_LOCKING GTEST_SKIP() << "Stats are non-functional when optimization is not enabled."; #endif - const int num_to_move = GetParam().num_to_move; - TypeParam e(GetParam().size, GetParam().bytes, num_to_move); + const int num_to_move = std::get<0>(GetParam()).num_to_move; + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + num_to_move, std::get<1>(GetParam())); // Roundtrip a batch. void* batch[kMaxObjectsToMove]; @@ -1036,8 +1056,9 @@ TEST_P(CentralFreeListTest, SameSpans) { // Check the stats after the first insertion. { - std::string expected_stats = absl::StrFormat( - "class %3d [ %8zu bytes ] :", e.kSizeClass, GetParam().size); + std::string expected_stats = + absl::StrFormat("class %3d [ %8zu bytes ] :", e.kSizeClass, + std::get<0>(GetParam()).size); for (int i = 0; i < num_to_move; ++i) { const bool first_batch = e.objects_per_span() > 1 && i == got - pseudo_spans.size(); @@ -1072,7 +1093,9 @@ TEST_P(CentralFreeListTest, MultipleSpans) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; std::vector all_objects; constexpr size_t kNumSpans = 10; @@ -1158,7 +1181,9 @@ TEST_P(CentralFreeListTest, PassSpanDensityToPageheap) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; ASSERT_GE(e.objects_per_span(), 1); auto test_function = [&](size_t num_objects, AccessDensityPrediction density) { @@ -1187,7 +1212,9 @@ TEST_P(CentralFreeListTest, SpanLifetimeWithLongLivedSpans) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; const int objects_per_span = e.objects_per_span(); if (objects_per_span < 3) return; @@ -1282,7 +1309,9 @@ TEST_P(CentralFreeListTest, LongLivedSpansMovedHistogram) { << "Skipping under HWASan, which uses the top bits of the pointer."; #endif - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; const int objects_per_span = e.objects_per_span(); if (objects_per_span < 2) return; @@ -1363,7 +1392,9 @@ TEST_P(CentralFreeListTest, ParallelHandleLongLivedSpans) { #endif std::atomic done(false); - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + ; const int objects_per_span = e.objects_per_span(); if (objects_per_span < 2) return; @@ -1469,10 +1500,149 @@ TEST_P(CentralFreeListTest, ParallelHandleLongLivedSpans) { } } -INSTANTIATE_TEST_SUITE_P(CentralFreeList, CentralFreeListTest, - // We skip the first size class since it is set to 0. - testing::ValuesIn(kSizeClasses.classes.begin() + 1, - kSizeClasses.classes.end())); +TEST_P(CentralFreeListTest, InlineLifetimeTrackingSequential) { +#if ABSL_HAVE_HWADDRESS_SANITIZER + GTEST_SKIP() + << "Skipping under HWASan, which uses the top bits of the pointer."; +#endif + + TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).bytes, + std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam())); + + const int objects_per_span = e.objects_per_span(); + if (objects_per_span < 3) return; + + void* batch[kMaxObjectsToMove]; + + // --- Phase 1: InsertRange Promotion --- + { + // Allocate all objects from 1 span. + size_t fetched = 0; + std::vector objects; + while (fetched < objects_per_span) { + const size_t n = objects_per_span - fetched; + int got = e.central_freelist().RemoveRange( + absl::MakeSpan(batch, std::min(n, e.batch_size()))); + for (int i = 0; i < got; ++i) { + objects.push_back(batch[i]); + } + fetched += got; + } + + // Release one object to put the span in the tracker list. + e.central_freelist().InsertRange({&objects[0], 1}); + + // Verify it is in some normal list. + size_t long_lived_count = 0; + size_t normal_count = 0; + for (int i = 0; i < kNumLists; ++i) { + long_lived_count += e.central_freelist().NumSpansInList(i); + normal_count += e.central_freelist().NumSpansInList(i + kNumLists); + } + EXPECT_EQ(long_lived_count, 0); + EXPECT_EQ(normal_count, 1); + + // Advance clock beyond threshold. + e.forwarder().AdvanceClock( + central_freelist_internal::kLongLivedSpanThreshold + absl::Seconds(1)); + + // Release another object. + e.central_freelist().InsertRange({&objects[1], 1}); + + long_lived_count = 0; + normal_count = 0; + for (int i = 0; i < kNumLists; ++i) { + long_lived_count += e.central_freelist().NumSpansInList(i); + normal_count += e.central_freelist().NumSpansInList(i + kNumLists); + } + + if (std::get<1>(GetParam()) == + central_freelist_internal::InlineLifetimeTracking::kEnabled) { + EXPECT_EQ(long_lived_count, 1); + EXPECT_EQ(normal_count, 0); + } else { + EXPECT_EQ(long_lived_count, 0); + EXPECT_EQ(normal_count, 1); + } + + // Return all objects to completely free the span. + for (size_t i = 2; i < objects.size(); ++i) { + e.central_freelist().InsertRange({&objects[i], 1}); + } + } + + // --- Phase 2: RemoveRange Promotion --- + { + // Allocate all objects from 1 span. + size_t fetched = 0; + std::vector objects; + while (fetched < objects_per_span) { + const size_t n = objects_per_span - fetched; + int got = e.central_freelist().RemoveRange( + absl::MakeSpan(batch, std::min(n, e.batch_size()))); + for (int i = 0; i < got; ++i) { + objects.push_back(batch[i]); + } + fetched += got; + } + + // Release TWO objects so the span is in the tracker list and has room to + // allocate. + e.central_freelist().InsertRange({&objects[0], 1}); + e.central_freelist().InsertRange({&objects[1], 1}); + + // Verify it is in some normal list. + size_t long_lived_count = 0; + size_t normal_count = 0; + for (int i = 0; i < kNumLists; ++i) { + long_lived_count += e.central_freelist().NumSpansInList(i); + normal_count += e.central_freelist().NumSpansInList(i + kNumLists); + } + EXPECT_EQ(long_lived_count, 0); + EXPECT_EQ(normal_count, 1); + + // Advance clock beyond threshold. + e.forwarder().AdvanceClock( + central_freelist_internal::kLongLivedSpanThreshold + absl::Seconds(1)); + + // Allocate one object back. + int got = e.central_freelist().RemoveRange(absl::MakeSpan(batch, 1)); + EXPECT_EQ(got, 1); + void* allocated_object = batch[0]; + + long_lived_count = 0; + normal_count = 0; + for (int i = 0; i < kNumLists; ++i) { + long_lived_count += e.central_freelist().NumSpansInList(i); + normal_count += e.central_freelist().NumSpansInList(i + kNumLists); + } + + if (std::get<1>(GetParam()) == + central_freelist_internal::InlineLifetimeTracking::kEnabled) { + EXPECT_EQ(long_lived_count, 1); + EXPECT_EQ(normal_count, 0); + } else { + EXPECT_EQ(long_lived_count, 0); + EXPECT_EQ(normal_count, 1); + } + + // Return all objects. + e.central_freelist().InsertRange({&allocated_object, 1}); + for (size_t i = 2; i < objects.size(); ++i) { + e.central_freelist().InsertRange({&objects[i], 1}); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + CentralFreeList, CentralFreeListTest, + testing::Combine( + // We skip the first size class since it is set to 0. + testing::ValuesIn(kSizeClasses.classes.begin() + 1, + kSizeClasses.classes.end()), + testing::Values( + central_freelist_internal::InlineLifetimeTracking::kDisabled, + central_freelist_internal::InlineLifetimeTracking::kEnabled))); } // namespace } // namespace tcmalloc_internal diff --git a/tcmalloc/experiment_config.h b/tcmalloc/experiment_config.h index f3c087a33..3451c5b8f 100644 --- a/tcmalloc/experiment_config.h +++ b/tcmalloc/experiment_config.h @@ -34,6 +34,7 @@ enum class Experiment : int { TEST_ONLY_TCMALLOC_MADV_COLD_HUGEPAGE, // TODO: b/450368905 - Complete experiment. TEST_ONLY_TCMALLOC_POW2_SIZECLASS, TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, + TEST_ONLY_TCMALLOC_SPAN_INLINE_LIFETIME_TRACKING, // TODO: benreisner - Complete experiment. TEST_ONLY_TCMALLOC_SUBRELEASE_UNBACKED_PAGES, // TODO: b/525422238 - Complete experiment. // go/keep-sorted end kMaxExperimentID, diff --git a/tcmalloc/global_stats.cc b/tcmalloc/global_stats.cc index c9c76ac7b..941833173 100644 --- a/tcmalloc/global_stats.cc +++ b/tcmalloc/global_stats.cc @@ -666,11 +666,12 @@ void DumpStats(Printer& out, int level) { Parameters::back_small_allocations() ? 1 : 0); out.printf("PARAMETER tcmalloc_back_size_threshold_bytes %d\n", Parameters::back_size_threshold_bytes()); - out.printf("PARAMETER tcmalloc_span_lifetime_tracking %d\n", - Parameters::span_lifetime_tracking() == - central_freelist_internal::LifetimeTracking::kEnabled - ? 1 - : 0); + out.printf( + "PARAMETER tcmalloc_span_lifetime_tracking %d\n", + Parameters::span_lifetime_tracking() == + central_freelist_internal::PeriodicLifetimeTracking::kEnabled + ? 1 + : 0); out.printf("PARAMETER background_process_sleep_interval_ns %lld\n", absl::ToInt64Nanoseconds( Parameters::background_process_sleep_interval())); @@ -686,6 +687,12 @@ void DumpStats(Printer& out, int level) { EnableUnfilteredCollapse::kEnabled ? 1 : 0); + out.printf( + "PARAMETER tcmalloc_span_inline_lifetime_tracking %d\n", + Parameters::span_inline_lifetime_tracking() == + central_freelist_internal::InlineLifetimeTracking::kEnabled + ? 1 + : 0); } } @@ -937,9 +944,10 @@ void DumpStatsInPbtxt(Printer& out, int level) { region.PrintRaw("madvise", MadviseString()); region.PrintBool("tcmalloc_resize_size_class_max_capacity", Parameters::resize_size_class_max_capacity()); - region.PrintBool("tcmalloc_span_lifetime_tracking", - Parameters::span_lifetime_tracking() == - central_freelist_internal::LifetimeTracking::kEnabled); + region.PrintBool( + "tcmalloc_span_lifetime_tracking", + Parameters::span_lifetime_tracking() == + central_freelist_internal::PeriodicLifetimeTracking::kEnabled); region.PrintI64("background_process_sleep_interval_ns", absl::ToInt64Nanoseconds( @@ -948,6 +956,10 @@ void DumpStatsInPbtxt(Printer& out, int level) { region.PrintBool("tcmalloc_enable_unfiltered_collapse", Parameters::enable_unfiltered_collapse() == EnableUnfilteredCollapse::kEnabled); + region.PrintBool( + "tcmalloc_span_inline_lifetime_tracking", + Parameters::span_inline_lifetime_tracking() == + central_freelist_internal::InlineLifetimeTracking::kEnabled); } bool GetNumericProperty(const char* name_data, size_t name_size, diff --git a/tcmalloc/huge_page_filler.h b/tcmalloc/huge_page_filler.h index 853705e98..fc20b5f79 100644 --- a/tcmalloc/huge_page_filler.h +++ b/tcmalloc/huge_page_filler.h @@ -1331,7 +1331,7 @@ class HugePageFiller { // We group hugepages first by longest-free (as a measure of fragmentation), // then into kChunks chunks inside there by desirability of // allocation. - static constexpr size_t kChunks = 8; + static constexpr size_t kChunks = 1; // Which chunk should this hugepage be in? // This returns the largest possible value kChunks - 1 iff // pt has a single allocation. @@ -1349,6 +1349,7 @@ class HugePageFiller { // List of hugepages from which no pages have been released to the OS. PageTrackerLists regular_alloc_[AccessDensityPrediction::kPredictionCounts]; + static constexpr size_t kE = sizeof(PageTrackerLists); PageTrackerLists donated_alloc_; // Partially released ones that we are trying to release. // diff --git a/tcmalloc/mock_central_freelist.h b/tcmalloc/mock_central_freelist.h index 030b74875..12b35479e 100644 --- a/tcmalloc/mock_central_freelist.h +++ b/tcmalloc/mock_central_freelist.h @@ -43,7 +43,9 @@ class FakeCentralFreeListBase { FakeCentralFreeListBase(const FakeCentralFreeListBase&) = delete; FakeCentralFreeListBase& operator=(const FakeCentralFreeListBase&) = delete; - static constexpr void Init(size_t) {} + static constexpr void Init(size_t, + central_freelist_internal::InlineLifetimeTracking + inline_lifetime_tracking) {} }; // CentralFreeList implementation that backs onto the system's malloc. diff --git a/tcmalloc/mock_static_forwarder.h b/tcmalloc/mock_static_forwarder.h index dc11aa494..64ede3845 100644 --- a/tcmalloc/mock_static_forwarder.h +++ b/tcmalloc/mock_static_forwarder.h @@ -27,6 +27,7 @@ #include "absl/synchronization/mutex.h" #include "absl/time/time.h" #include "absl/types/span.h" +#include "tcmalloc/central_freelist.h" #include "tcmalloc/common.h" #include "tcmalloc/internal/hook_list.h" #include "tcmalloc/pages.h" @@ -230,11 +231,14 @@ class FakeCentralFreeListEnvironment { explicit FakeCentralFreeListEnvironment( size_t class_size, Bytes span_bytes, size_t num_objects_to_move, + central_freelist_internal::InlineLifetimeTracking + inline_lifetime_tracking, size_t page_size = kPageSize, + double clock_frequency = absl::ToDoubleNanoseconds(absl::Seconds(2))) { forwarder().Init(class_size, span_bytes, num_objects_to_move, page_size, clock_frequency); - cache_.Init(kSizeClass); + cache_.Init(kSizeClass, inline_lifetime_tracking); } ~FakeCentralFreeListEnvironment() { EXPECT_EQ(cache_.length(), 0); } diff --git a/tcmalloc/mock_transfer_cache.h b/tcmalloc/mock_transfer_cache.h index eba9f0644..6f95d582c 100644 --- a/tcmalloc/mock_transfer_cache.h +++ b/tcmalloc/mock_transfer_cache.h @@ -29,6 +29,7 @@ #include "absl/random/distributions.h" #include "absl/random/random.h" #include "absl/types/span.h" +#include "tcmalloc/central_freelist.h" #include "tcmalloc/common.h" #include "tcmalloc/mock_central_freelist.h" #include "tcmalloc/transfer_cache.h" @@ -145,7 +146,12 @@ class FakeTransferCacheEnvironment { ::tcmalloc::tcmalloc_internal::kMaxObjectsToMove; static constexpr int kBatchSize = Manager::num_objects_to_move(1); - FakeTransferCacheEnvironment() : manager_(), cache_(&manager_, 1) { Init(); } + FakeTransferCacheEnvironment() + : manager_(), + cache_(&manager_, 1, + central_freelist_internal::InlineLifetimeTracking::kDisabled) { + Init(); + } ~FakeTransferCacheEnvironment() { Drain(); } @@ -240,10 +246,12 @@ class ThreeSizeClassManager : public FakeTransferCacheManager { ThreeSizeClassManager() { for (int i = 0; i < 3; ++i) { caches_[i] = std::make_unique( - this, i); + this, i, + central_freelist_internal::InlineLifetimeTracking::kDisabled); } caches_[kColdSizeClass] = std::make_unique( - this, kColdSizeClass); + this, kColdSizeClass, + central_freelist_internal::InlineLifetimeTracking::kDisabled); } constexpr static size_t class_to_size(int size_class) { diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index babbc4de8..d52611a7a 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -324,37 +324,52 @@ HeapPartitioningMode Parameters::heap_partitioning_mode() { } bool ABSL_ATTRIBUTE_WEAK default_want_disable_span_lifetime_tracking(); -static central_freelist_internal::LifetimeTracking +static central_freelist_internal::PeriodicLifetimeTracking want_span_lifetime_tracking() { if (default_want_disable_span_lifetime_tracking != nullptr) { - return central_freelist_internal::LifetimeTracking::kDisabled; + return central_freelist_internal::PeriodicLifetimeTracking::kDisabled; } const char* e = thread_safe_getenv("TCMALLOC_DISABLE_SPAN_LIFETIME_TRACKING"); if (e) { switch (e[0]) { case '0': - return central_freelist_internal::LifetimeTracking::kDisabled; + return central_freelist_internal::PeriodicLifetimeTracking::kDisabled; case '1': - return central_freelist_internal::LifetimeTracking::kDisabled; + return central_freelist_internal::PeriodicLifetimeTracking::kDisabled; default: TC_BUG("bad env var '%s'", e); } } - return central_freelist_internal::LifetimeTracking::kDisabled; + return central_freelist_internal::PeriodicLifetimeTracking::kDisabled; } -central_freelist_internal::LifetimeTracking +central_freelist_internal::PeriodicLifetimeTracking Parameters::span_lifetime_tracking() { ABSL_CONST_INIT static absl::once_flag flag; ABSL_CONST_INIT static std::atomic< - central_freelist_internal::LifetimeTracking> - v{central_freelist_internal::LifetimeTracking::kDisabled}; + central_freelist_internal::PeriodicLifetimeTracking> + v{central_freelist_internal::PeriodicLifetimeTracking::kDisabled}; absl::base_internal::LowLevelCallOnce(&flag, [&]() { v.store(want_span_lifetime_tracking(), std::memory_order_relaxed); }); return v.load(std::memory_order_relaxed); } +central_freelist_internal::InlineLifetimeTracking +Parameters::span_inline_lifetime_tracking() { + ABSL_CONST_INIT static absl::once_flag flag; + ABSL_CONST_INIT static std::atomic< + central_freelist_internal::InlineLifetimeTracking> + v{central_freelist_internal::InlineLifetimeTracking::kDisabled}; + absl::base_internal::LowLevelCallOnce(&flag, [&]() { + v.store( + central_freelist_internal::InlineLifetimeTracking{IsExperimentActive( + Experiment::TEST_ONLY_TCMALLOC_SPAN_INLINE_LIFETIME_TRACKING)}, + std::memory_order_relaxed); + }); + return v.load(std::memory_order_relaxed); +} + int32_t Parameters::max_per_cpu_cache_size() { return tc_globals.cpu_cache().CacheLimit(); } diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index aa430aa47..051d96203 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -188,7 +188,11 @@ class Parameters { static HeapPartitioningMode heap_partitioning_mode(); - static central_freelist_internal::LifetimeTracking span_lifetime_tracking(); + static central_freelist_internal::PeriodicLifetimeTracking + span_lifetime_tracking(); + + static central_freelist_internal::InlineLifetimeTracking + span_inline_lifetime_tracking(); private: friend void ::TCMalloc_Internal_SetBackgroundReleaseRate(size_t v); diff --git a/tcmalloc/testing/get_stats_test.cc b/tcmalloc/testing/get_stats_test.cc index d5fdaa388..63e41236a 100644 --- a/tcmalloc/testing/get_stats_test.cc +++ b/tcmalloc/testing/get_stats_test.cc @@ -159,6 +159,14 @@ TEST_F(GetStatsTest, Pbtxt) { EXPECT_THAT(buf, ContainsRegex("max_cpu_cache_touched: [0-9]+")); } + if (IsExperimentActive( + Experiment::TEST_ONLY_TCMALLOC_SPAN_INLINE_LIFETIME_TRACKING)) { + EXPECT_THAT(buf, HasSubstr("tcmalloc_span_inline_lifetime_tracking: true")); + } else { + EXPECT_THAT(buf, + HasSubstr("tcmalloc_span_inline_lifetime_tracking: false")); + } + sized_delete(alloc, kSize); } @@ -220,6 +228,17 @@ TEST_F(GetStatsTest, Parameters) { HasSubstr( R"(PARAMETER tcmalloc_dense_trackers_sorted_on_spans_allocated 1)")); + if (IsExperimentActive( + Experiment::TEST_ONLY_TCMALLOC_SPAN_INLINE_LIFETIME_TRACKING)) { + EXPECT_THAT( + buf, + HasSubstr(R"(PARAMETER tcmalloc_span_inline_lifetime_tracking 1)")); + } else { + EXPECT_THAT( + buf, + HasSubstr(R"(PARAMETER tcmalloc_span_inline_lifetime_tracking 0)")); + } + #ifdef NDEBUG EXPECT_THAT( buf, HasSubstr(R"(PARAMETER tcmalloc_usermode_hugepage_collapse 1)")); diff --git a/tcmalloc/transfer_cache.h b/tcmalloc/transfer_cache.h index b0996f5d4..caabceded 100644 --- a/tcmalloc/transfer_cache.h +++ b/tcmalloc/transfer_cache.h @@ -111,7 +111,10 @@ class ProdCpuLayout { // Forwards calls to the unsharded TransferCache. class BackingTransferCache { public: - void Init(int size_class) { size_class_ = size_class; } + void Init(int size_class, central_freelist_internal::InlineLifetimeTracking + inline_lifetime_tracking) { + size_class_ = size_class; + } void InsertRange(absl::Span batch) const; [[nodiscard]] int RemoveRange(absl::Span batch) const; int size_class() const { return size_class_; } @@ -386,9 +389,12 @@ class ShardedTransferCacheManagerBase { : LargeCacheCapacity(size_class); new (&new_caches[size_class]) TransferCache(owner_, capacity.capacity > 0 ? size_class : 0, - {capacity.capacity, capacity.max_capacity}); - new_caches[size_class].freelist().Init(size_class); + {capacity.capacity, capacity.max_capacity}, + Parameters::span_inline_lifetime_tracking()); + new_caches[size_class].freelist().Init( + size_class, Parameters::span_inline_lifetime_tracking()); } + shard.transfer_caches = new_caches; active_shards_.fetch_add(1, std::memory_order_relaxed); shard.initialized.store(true, std::memory_order_release); @@ -484,7 +490,8 @@ class TransferCacheManager : public StaticForwarder { void InitCaches() { for (int i = 0; i < kNumClasses; ++i) { - new (&cache_[i].tc) TransferCache(this, i); + new (&cache_[i].tc) + TransferCache(this, i, Parameters::span_inline_lifetime_tracking()); } } @@ -579,7 +586,7 @@ class TransferCacheManager { void Init() { for (int i = 0; i < kNumClasses; ++i) { - freelist_[i].Init(i); + freelist_[i].Init(i, Parameters::span_inline_lifetime_tracking()); } } diff --git a/tcmalloc/transfer_cache_benchmark.cc b/tcmalloc/transfer_cache_benchmark.cc index fbe2dede2..5452c0284 100644 --- a/tcmalloc/transfer_cache_benchmark.cc +++ b/tcmalloc/transfer_cache_benchmark.cc @@ -18,6 +18,7 @@ #include "absl/random/distributions.h" #include "absl/random/random.h" #include "benchmark/benchmark.h" +#include "tcmalloc/central_freelist.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/mock_central_freelist.h" #include "tcmalloc/mock_transfer_cache.h" @@ -45,7 +46,14 @@ void BM_CrossThread(benchmark::State& state) { void* batch[kMaxObjectsToMove]; struct CrossThreadState { - CrossThreadState() : m{}, c{Cache(&m, 1), Cache(&m, 1)} {} + CrossThreadState() + : m{}, + c{Cache(&m, 1, + central_freelist_internal::InlineLifetimeTracking::kDisabled), + Cache( + &m, 1, + central_freelist_internal::InlineLifetimeTracking::kDisabled)} { + } FakeTransferCacheManager m; Cache c[2]; }; diff --git a/tcmalloc/transfer_cache_internals.h b/tcmalloc/transfer_cache_internals.h index ade6ddd25..c7427b3c5 100644 --- a/tcmalloc/transfer_cache_internals.h +++ b/tcmalloc/transfer_cache_internals.h @@ -32,6 +32,7 @@ #include "absl/base/optimization.h" #include "absl/base/thread_annotations.h" #include "absl/types/span.h" +#include "tcmalloc/central_freelist.h" #include "tcmalloc/common.h" #include "tcmalloc/internal/allocation_guard.h" #include "tcmalloc/internal/atomic_stats_counter.h" @@ -85,15 +86,20 @@ class TransferCache { using Manager = TransferCacheManager; using FreeList = CentralFreeList; - TransferCache(Manager *owner, int size_class) - : TransferCache(owner, size_class, CapacityNeeded(size_class)) {} + TransferCache(Manager* owner, int size_class, + central_freelist_internal::InlineLifetimeTracking + inline_lifetime_tracking) + : TransferCache(owner, size_class, CapacityNeeded(size_class), + inline_lifetime_tracking) {} struct Capacity { int capacity; int max_capacity; }; - TransferCache(Manager *owner, int size_class, Capacity capacity) + TransferCache(Manager* owner, int size_class, Capacity capacity, + central_freelist_internal::InlineLifetimeTracking + inline_lifetime_tracking) : lock_(absl::base_internal::SCHEDULE_KERNEL_ONLY), low_water_mark_(0), slot_info_(SizeInfo({0, capacity.capacity})), @@ -101,7 +107,7 @@ class TransferCache { freelist_do_not_access_directly_(), owner_(owner), max_capacity_(capacity.max_capacity) { - freelist().Init(size_class); + freelist().Init(size_class, inline_lifetime_tracking); slots_ = max_capacity_ != 0 ? reinterpret_cast(owner_->Alloc( max_capacity_ * sizeof(void*))) : nullptr; diff --git a/tcmalloc/variants.bzl b/tcmalloc/variants.bzl index 4729ab148..e4d3eb29f 100644 --- a/tcmalloc/variants.bzl +++ b/tcmalloc/variants.bzl @@ -277,6 +277,12 @@ test_variants = [ "deps": ["//tcmalloc:common_8k_pages"], "env": {"BORG_EXPERIMENTS": "TCMALLOC_EAGER_BACKING_V2"}, }, + { + "name": "tcmalloc_span_inline_lifetime_tracking", + "malloc": "//tcmalloc", + "deps": ["//tcmalloc:common_8k_pages"], + "env": {"BORG_EXPERIMENTS": "TEST_ONLY_TCMALLOC_SPAN_INLINE_LIFETIME_TRACKING"}, + }, ] def create_tcmalloc_library(