From a16381efc1c7a14a7ab12664c9bd2c7485dc4c02 Mon Sep 17 00:00:00 2001 From: TCMalloc Team Date: Sun, 11 Oct 2026 14:44:22 -0700 Subject: [PATCH] Preserve broken hugepage state in PageMap across HugeCache/HugeAllocator reuse. Use `PackedPtr` in `HugePageAwareAllocator` and `StaticForwarder` to encode broken hugepage state in the low bit of `PageMap` entries across whole-hugepage releases and subsequent reuse. PiperOrigin-RevId: 997556385 --- tcmalloc/BUILD | 3 + tcmalloc/CMakeLists.txt | 11 +++ tcmalloc/huge_page_aware_allocator.h | 41 +++++++++- tcmalloc/huge_page_aware_allocator_fuzz.cc | 28 ++++++- tcmalloc/huge_page_aware_allocator_test.cc | 93 +++++++++++++++++++++- tcmalloc/huge_page_options.h | 5 ++ tcmalloc/mock_huge_page_static_forwarder.h | 14 ++-- tcmalloc/parameters.cc | 2 + tcmalloc/parameters.h | 5 ++ tcmalloc/static_forwarder.h | 10 ++- tcmalloc/static_forwarder_test.cc | 4 +- 11 files changed, 196 insertions(+), 20 deletions(-) diff --git a/tcmalloc/BUILD b/tcmalloc/BUILD index 51303e4af..e22a42e77 100644 --- a/tcmalloc/BUILD +++ b/tcmalloc/BUILD @@ -98,6 +98,7 @@ tcmalloc_deps = [ "//tcmalloc/internal:logging", "//tcmalloc/internal:memory_tag", "//tcmalloc/internal:optimization", + "//tcmalloc/internal:packed_ptr", "//tcmalloc/internal:page_allocation_status", "//tcmalloc/internal:percpu", "//tcmalloc/internal:range_tracker", @@ -331,6 +332,7 @@ create_tcmalloc_libraries( "//tcmalloc/internal:mincore", "//tcmalloc/internal:numa", "//tcmalloc/internal:optimization", + "//tcmalloc/internal:packed_ptr", "//tcmalloc/internal:page_allocation_status", "//tcmalloc/internal:page_allocator_hooks", "//tcmalloc/internal:page_size", @@ -712,6 +714,7 @@ cc_library( "//tcmalloc/internal:config", "//tcmalloc/internal:logging", "//tcmalloc/internal:memory_tag", + "//tcmalloc/internal:packed_ptr", "//tcmalloc/internal:system_allocator", "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", diff --git a/tcmalloc/CMakeLists.txt b/tcmalloc/CMakeLists.txt index d3dd7283a..b3124a8a1 100644 --- a/tcmalloc/CMakeLists.txt +++ b/tcmalloc/CMakeLists.txt @@ -76,6 +76,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -127,6 +128,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -178,6 +180,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -351,6 +354,7 @@ tcmalloc_cc_library_variants( "tcmalloc::internal_mincore" "tcmalloc::internal_numa" "tcmalloc::internal_optimization" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_allocator_hooks" "tcmalloc::internal_page_size" @@ -485,6 +489,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -536,6 +541,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -588,6 +594,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -639,6 +646,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -690,6 +698,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -743,6 +752,7 @@ tcmalloc_cc_library( "tcmalloc::internal_memory_tag" "tcmalloc::internal_optimization" "tcmalloc::internal_overflow" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_page_allocation_status" "tcmalloc::internal_page_size" "tcmalloc::internal_percpu" @@ -869,6 +879,7 @@ tcmalloc_cc_library( "tcmalloc::internal_config" "tcmalloc::internal_logging" "tcmalloc::internal_memory_tag" + "tcmalloc::internal_packed_ptr" "tcmalloc::internal_system_allocator" ) diff --git a/tcmalloc/huge_page_aware_allocator.h b/tcmalloc/huge_page_aware_allocator.h index 872f1e5f8..3b6244c88 100644 --- a/tcmalloc/huge_page_aware_allocator.h +++ b/tcmalloc/huge_page_aware_allocator.h @@ -41,6 +41,7 @@ #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/metadata_allocator.h" +#include "tcmalloc/internal/packed_ptr.h" #include "tcmalloc/internal/pageflags.h" #include "tcmalloc/internal/parameter_accessors.h" #include "tcmalloc/internal/prefetch.h" @@ -332,9 +333,16 @@ class HugePageAwareAllocator final : public PageAllocatorInterface { MetadataObjectAllocator region_allocator_ ABSL_GUARDED_BY(pageheap_lock); + enum class HugePageSplitState : bool { + kUnbroken, + kBroken, + }; + [[nodiscard]] FillerType::Tracker* GetTracker(HugePage p) const; + [[nodiscard]] bool IsHugepageBroken(HugePage p) const; void SetTracker(HugePage p, FillerType::Tracker* pt); + void ClearTracker(HugePage p, HugePageSplitState split_state); [[nodiscard]] AddressRange AllocAndReport(size_t bytes, size_t align) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); @@ -436,24 +444,45 @@ inline HugePageAwareAllocator::HugePageAwareAllocator( template inline typename HugePageAwareAllocator::FillerType::Tracker* HugePageAwareAllocator::GetTracker(HugePage p) const { - void* v = forwarder_.GetHugepage(p); - FillerType::Tracker* pt = reinterpret_cast(v); + FillerType::Tracker* pt = forwarder_.GetHugepage(p).Get(); TC_ASSERT(pt == nullptr || pt->location() == p); return pt; } +template +inline bool HugePageAwareAllocator::IsHugepageBroken( + HugePage p) const { + return forwarder_.GetHugepage(p).Bits() != 0; +} + template inline void HugePageAwareAllocator::SetTracker( HugePage p, HugePageAwareAllocator::FillerType::Tracker* pt) { - forwarder_.SetHugepage(p, pt); + forwarder_.SetHugepage(p, PackedPtr(pt)); +} + +template +inline void HugePageAwareAllocator::ClearTracker( + HugePage p, HugePageSplitState split_state) { + forwarder_.SetHugepage( + p, + PackedPtr( + /*ptr=*/nullptr, split_state == HugePageSplitState::kBroken ? 1 : 0)); } template inline PageId HugePageAwareAllocator::AllocAndContribute( HugePage p, Length n, SpanAllocInfo span_alloc_info, bool donated) { TC_CHECK_NE(p.start_addr(), nullptr); + const bool was_broken = IsHugepageBroken(p); FillerType::Tracker* pt = tracker_allocator_.New(p, donated, forwarder_.clock().now()); + if (was_broken && + forwarder_.track_broken_hugepages() == TrackBrokenHugepages::kEnabled) { + pt->set_unbroken(/*status=*/false); + } + // TODO(b/572530574): When allocating from an unbacked range (*from_released), + // mark the unallocated tail pages [n, kPagesPerHugePage) unbacked in pt. TC_ASSERT_GE(pt->longest_free_range(), n); TC_ASSERT_EQ(pt->was_donated(), donated); // if the page was donated, we track its size so that we can potentially @@ -839,8 +868,12 @@ inline void HugePageAwareAllocator::ReleaseHugepage( } HugeRange r = {pt->location(), NHugePages(1)}; - SetTracker(pt->location(), nullptr); + ClearTracker(pt->location(), pt->unbroken() ? HugePageSplitState::kUnbroken + : HugePageSplitState::kBroken); + // TODO(b/572530574): Avoid coalescing or preferring broken hugepages ahead of + // intact THPs in HugeCache, and evict broken cached pages first in + // ShrinkCache. if (pt->released()) { cache_.ReleaseUnbacked(r); } else { diff --git a/tcmalloc/huge_page_aware_allocator_fuzz.cc b/tcmalloc/huge_page_aware_allocator_fuzz.cc index d787a35f0..3c570b321 100644 --- a/tcmalloc/huge_page_aware_allocator_fuzz.cc +++ b/tcmalloc/huge_page_aware_allocator_fuzz.cc @@ -418,6 +418,17 @@ struct SetSubreleaseUnbackedHugepages { } }; +struct SetTrackBrokenHugepages { + bool value; + + void Perform(State& state) const; + + template + friend void AbslStringify(Sink& sink, const SetTrackBrokenHugepages& s) { + absl::Format(&sink, "SetTrackBrokenHugepages{.value=%v}", s.value); + } +}; + struct SetReleaseSucceeds { bool value; @@ -590,10 +601,10 @@ using ParamOp = std::variant< SetFillerSkipSubreleaseLongInterval, SetFillerSkipSubreleaseColdShortInterval, SetFillerSkipSubreleaseColdLongInterval, SetReleasePartialAllocPages, - SetHpaaSubrelease, SetSubreleaseUnbackedHugepages, SetReleaseSucceeds, - SetCollapseSucceeds, SetHugeRegionAdaptiveRelease, SetAllocateSucceeds, - SetBackAllocations, SetBackSizeThresholdBytes, ReentrantSubprogram, - SetEnableUnfilteredCollapse, SetReleaseMaxFillerPages, + SetHpaaSubrelease, SetSubreleaseUnbackedHugepages, SetTrackBrokenHugepages, + SetReleaseSucceeds, SetCollapseSucceeds, SetHugeRegionAdaptiveRelease, + SetAllocateSucceeds, SetBackAllocations, SetBackSizeThresholdBytes, + ReentrantSubprogram, SetEnableUnfilteredCollapse, SetReleaseMaxFillerPages, SetEnableReleaseStalePages, SetMadvNoHugepageHugeRegions, UpdateBitmaps, SetUsageLimitPressure>; @@ -1159,6 +1170,11 @@ void SetSubreleaseUnbackedHugepages::Perform(State& state) const { : SubreleaseUnbackedMode::kDisabled); } +void SetTrackBrokenHugepages::Perform(State& state) const { + state.allocator.forwarder().track_broken_hugepages.set( + value ? TrackBrokenHugepages::kEnabled : TrackBrokenHugepages::kDisabled); +} + void SetReleaseSucceeds::Perform(State& state) const { state.allocator.forwarder().set_release_succeeds(value); } @@ -1331,6 +1347,8 @@ fuzztest::Domain GetChangeParamDomain(int depth) { fuzztest::Map( [](SetSubreleaseUnbackedHugepages s) { return ChangeParam{s}; }, fuzztest::Arbitrary()), + fuzztest::Map([](SetTrackBrokenHugepages s) { return ChangeParam{s}; }, + fuzztest::Arbitrary()), fuzztest::Map([](SetReleaseSucceeds s) { return ChangeParam{s}; }, fuzztest::Arbitrary()), fuzztest::Map([](SetCollapseSucceeds s) { return ChangeParam{s}; }, @@ -1976,6 +1994,8 @@ TEST(HugePageAwareAllocatorTest, PrinterTest) { "SetCollapseSucceeds{.value=true}"); EXPECT_EQ(absl::StrCat(SetSubreleaseUnbackedHugepages{.value = false}), "SetSubreleaseUnbackedHugepages{.value=false}"); + EXPECT_EQ(absl::StrCat(SetTrackBrokenHugepages{.value = false}), + "SetTrackBrokenHugepages{.value=false}"); EXPECT_EQ(absl::StrCat(SetUsageLimitPressure{.pages = 3}), "SetUsageLimitPressure{.pages=3}"); } diff --git a/tcmalloc/huge_page_aware_allocator_test.cc b/tcmalloc/huge_page_aware_allocator_test.cc index e0715803d..0f9dcc2ea 100644 --- a/tcmalloc/huge_page_aware_allocator_test.cc +++ b/tcmalloc/huge_page_aware_allocator_test.cc @@ -311,6 +311,26 @@ class HugePageAwareAllocatorTest return allocator_->region().UseHugeRegionMoreOften(); } + struct HugepageEntryState { + PageTracker* tracker = nullptr; + uint64_t bits = 0; + bool unbroken = false; + Length released_pages; + }; + + [[nodiscard]] HugepageEntryState GetHugepageState(HugePage hp) { + PageHeapSpinLockHolder l; + const auto packed = allocator_->forwarder().GetHugepage(hp); + HugepageEntryState state; + state.tracker = packed.Get(); + state.bits = packed.Bits(); + if (state.tracker != nullptr) { + state.unbroken = state.tracker->unbroken(); + state.released_pages = state.tracker->released_pages(); + } + return state; + } + std::string Print() { return PrintToString(1 << 20, [&](Printer& p) { PageFlags pageflags; @@ -858,7 +878,7 @@ TEST_P(HugePageAwareAllocatorTest, ParkedDonatedHugepageIsNotAbandoned) { ASSERT_TRUE(large); ASSERT_TRUE(large.donated); const HugePage hp = HugePageContaining(large.r.p); - auto* pt = static_cast(allocator_->forwarder().GetHugepage(hp)); + PageTracker* pt = allocator_->forwarder().GetHugepage(hp).Get(); ASSERT_NE(pt, nullptr); HugeLength donated_huge_pages; @@ -2571,6 +2591,77 @@ TEST_P(HugePageAwareAllocatorTest, GetPageAllocationStatus) { } } +TEST_P(HugePageAwareAllocatorTest, BrokenHugepageStatePersistsAcrossReuse) { + allocator_->forwarder().hpaa_subrelease.set(true); + allocator_->forwarder().release_partial_alloc_pages.set(true); + + const SpanAllocInfo kSpanInfo = {1, AccessDensityPrediction::kSparse}; + std::vector keep_alive; + + for (const TrackBrokenHugepages track_broken_hugepages : + {TrackBrokenHugepages::kDisabled, TrackBrokenHugepages::kEnabled}) { + allocator_->forwarder().track_broken_hugepages.set(track_broken_hugepages); + + // 1. Subrelease free pages on `hp`, refill it so `released_pages() == 0` + // while `!pt->unbroken()`, and drain it into `HugeCache`. + AllocationState a0 = New(Length(1), kSpanInfo); + ASSERT_TRUE(a0); + const HugePage hp = HugePageContaining(a0.r.p); + ReleasePages(kPagesPerHugePage - Length(1), + PageReleaseReason::kProcessBackgroundActions); + HugepageEntryState state = GetHugepageState(hp); + ASSERT_NE(state.tracker, nullptr); + EXPECT_FALSE(state.unbroken); + EXPECT_EQ(state.released_pages, kPagesPerHugePage - Length(1)); + + AllocationState fill_rest = New(kPagesPerHugePage - Length(1), kSpanInfo); + ASSERT_TRUE(fill_rest); + ASSERT_EQ(HugePageContaining(fill_rest.r.p), hp); + state = GetHugepageState(hp); + ASSERT_NE(state.tracker, nullptr); + EXPECT_FALSE(state.unbroken); + EXPECT_EQ(state.released_pages, Length(0)); + + // Draining both spans returns `hp` to `HugeCache`; PageMap preserves the + // low bit, and reuse restores `!pt->unbroken()` iff + // `track_broken_hugepages` is enabled. + Delete(a0, 1); + Delete(fill_rest, 1); + state = GetHugepageState(hp); + EXPECT_EQ(state.tracker, nullptr); + EXPECT_EQ(state.bits, 1u); + + AllocationState reused_from_cache = New(Length(1), kSpanInfo); + ASSERT_TRUE(reused_from_cache); + ASSERT_EQ(HugePageContaining(reused_from_cache.r.p), hp); + state = GetHugepageState(hp); + ASSERT_NE(state.tracker, nullptr); + EXPECT_EQ(state.unbroken, + track_broken_hugepages == TrackBrokenHugepages::kDisabled); + + // 2. Subreleasing and freeing all allocations (`HandleFullyFreedTracker`) + // returns `hp` to `HugeAllocator` unbacked; reusing `hp` restores + // `!pt->unbroken()` iff `track_broken_hugepages` is enabled. + ReleasePages(kPagesPerHugePage - Length(1), + PageReleaseReason::kProcessBackgroundActions); + Delete(reused_from_cache, 1); + AllocationState reused_from_allocator = New(Length(1), kSpanInfo); + ASSERT_TRUE(reused_from_allocator); + ASSERT_EQ(HugePageContaining(reused_from_allocator.r.p), hp); + state = GetHugepageState(hp); + ASSERT_NE(state.tracker, nullptr); + EXPECT_EQ(state.unbroken, + track_broken_hugepages == TrackBrokenHugepages::kDisabled); + + keep_alive.push_back(reused_from_allocator); + keep_alive.push_back(New(kPagesPerHugePage - Length(1), kSpanInfo)); + } + + for (AllocationState& s : keep_alive) { + Delete(s, 1); + } +} + } // namespace } // namespace tcmalloc_internal } // namespace tcmalloc diff --git a/tcmalloc/huge_page_options.h b/tcmalloc/huge_page_options.h index c9eb0e998..d815a68f3 100644 --- a/tcmalloc/huge_page_options.h +++ b/tcmalloc/huge_page_options.h @@ -59,6 +59,11 @@ enum class MadviseRegionsNoHugepage : bool { kEnabled = true, }; +enum class TrackBrokenHugepages : bool { + kDisabled = false, + kEnabled = true, +}; + } // namespace tcmalloc::tcmalloc_internal GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/mock_huge_page_static_forwarder.h b/tcmalloc/mock_huge_page_static_forwarder.h index a6ee579cc..0754f93dd 100644 --- a/tcmalloc/mock_huge_page_static_forwarder.h +++ b/tcmalloc/mock_huge_page_static_forwarder.h @@ -41,6 +41,7 @@ #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/memory_tag.h" +#include "tcmalloc/internal/packed_ptr.h" #include "tcmalloc/internal/system_allocator.h" #include "tcmalloc/pages.h" #include "tcmalloc/parameters.h" @@ -89,6 +90,8 @@ class FakeHugePageStaticForwarder : private Parameters { FakeParameter hpaa_subrelease{Parameters::hpaa_subrelease()}; FakeParameter subrelease_unbacked_hugepages{ Parameters::subrelease_unbacked_hugepages()}; + FakeParameter track_broken_hugepages{ + Parameters::track_broken_hugepages()}; FakeParameter huge_region_adaptive_release{ Parameters::huge_region_adaptive_release()}; FakeParameter background_release_rate{ @@ -140,10 +143,10 @@ class FakeHugePageStaticForwarder : private Parameters { // TODO(b/242550501): We should just swap out a PageMap instead of doing // these things. See the description of cl/169552275 for the original TODO. // While we're here, also look into consolidating forwarders. - [[nodiscard]] void* GetHugepage(HugePage p) const { + [[nodiscard]] PackedPtr GetHugepage(HugePage p) const { auto it = trackers_.find(p); if (it == trackers_.end()) { - return nullptr; + return PackedPtr(); } return it->second; } @@ -151,8 +154,7 @@ class FakeHugePageStaticForwarder : private Parameters { ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock) { return true; } - void SetHugepage(HugePage p, void* pt) { trackers_[p] = pt; } - + void SetHugepage(HugePage p, PackedPtr pt) { trackers_[p] = pt; } // SystemAlloc state. [[nodiscard]] AddressRange AllocatePages(size_t bytes, size_t align, @@ -266,9 +268,9 @@ class FakeHugePageStaticForwarder : private Parameters { } }; - absl::flat_hash_map, + absl::flat_hash_map, absl::Hash, std::equal_to, - AllocAdaptor>> + AllocAdaptor>>> trackers_; // Live spans by first page, with their length. SetSpan runs outside // pageheap_lock and ReleasePages runs with or without it, so the map has its diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index 2200ab42f..67cc39079 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -262,6 +262,8 @@ ABSL_CONST_INIT std::atomic Parameters::profile_sampling_interval_( ABSL_CONST_INIT std::atomic Parameters::subrelease_unbacked_hugepages_( SubreleaseUnbackedMode::kDisabled); +ABSL_CONST_INIT std::atomic + Parameters::track_broken_hugepages_(TrackBrokenHugepages::kDisabled); // TODO: b/134694141 - Remove this opt out. ABSL_CONST_INIT std::atomic Parameters::back_small_allocations_(false); diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index 0782c7a6f..712810ad9 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -107,6 +107,10 @@ class Parameters { [[nodiscard]] static SubreleaseUnbackedMode subrelease_unbacked_hugepages(); + [[nodiscard]] static TrackBrokenHugepages track_broken_hugepages() { + return track_broken_hugepages_.load(std::memory_order_relaxed); + } + [[nodiscard]] static bool back_small_allocations() { return back_small_allocations_.load(std::memory_order_relaxed); } @@ -315,6 +319,7 @@ class Parameters { static std::atomic per_cpu_caches_dynamic_slab_grow_threshold_; static std::atomic per_cpu_caches_dynamic_slab_shrink_threshold_; static std::atomic subrelease_unbacked_hugepages_; + static std::atomic track_broken_hugepages_; static std::atomic usermode_hugepage_collapse_enabled_; static std::atomic back_small_allocations_; static std::atomic back_size_threshold_bytes_; diff --git a/tcmalloc/static_forwarder.h b/tcmalloc/static_forwarder.h index 4f8f6b776..a1da8cfd9 100644 --- a/tcmalloc/static_forwarder.h +++ b/tcmalloc/static_forwarder.h @@ -40,6 +40,7 @@ #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/internal/numa.h" +#include "tcmalloc/internal/packed_ptr.h" #include "tcmalloc/internal/prefetch.h" #include "tcmalloc/internal/system_allocator.h" #include "tcmalloc/page_allocator_interface.h" @@ -86,6 +87,7 @@ class StaticForwarder : private Parameters { using Parameters::release_partial_alloc_pages; using Parameters::release_stale_pages; using Parameters::subrelease_unbacked_hugepages; + using Parameters::track_broken_hugepages; [[nodiscard]] static size_t class_to_size(int size_class) { return state.sizemap().class_to_size(size_class); @@ -326,15 +328,15 @@ class StaticForwarder : private Parameters { } // PageMap state. - [[nodiscard]] static void* GetHugepage(HugePage p) { - return state.pagemap().GetHugepage(p.first_page()); + [[nodiscard]] static PackedPtr GetHugepage(HugePage p) { + return PackedPtr(state.pagemap().GetHugepage(p.first_page())); } [[nodiscard]] static bool Ensure(Range r) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock) { return state.pagemap().Ensure(r); } - static void SetHugepage(HugePage p, void* pt) { - state.pagemap().SetHugepage(p.first_page(), pt); + static void SetHugepage(HugePage p, PackedPtr pt) { + state.pagemap().SetHugepage(p.first_page(), pt.Raw()); } // Error reporting diff --git a/tcmalloc/static_forwarder_test.cc b/tcmalloc/static_forwarder_test.cc index fbbd291c4..c76bbd4e9 100644 --- a/tcmalloc/static_forwarder_test.cc +++ b/tcmalloc/static_forwarder_test.cc @@ -213,7 +213,7 @@ TEST(StaticForwarderTest, PageMap) { const PageId p = span->first_page(); const HugePage hp = HugePageContaining(p); - EXPECT_EQ(ProdForwarder::GetHugepage(hp), + EXPECT_EQ(ProdForwarder::GetHugepage(hp).Raw(), tc_globals.pagemap().GetHugepage(hp.first_page())); { PageHeapSpinLockHolder l; @@ -262,6 +262,8 @@ TEST(StaticForwarderTest, Parameters) { Parameters::release_stale_pages()); EXPECT_EQ(ProdForwarder::subrelease_unbacked_hugepages(), Parameters::subrelease_unbacked_hugepages()); + EXPECT_EQ(ProdForwarder::track_broken_hugepages(), + Parameters::track_broken_hugepages()); } class StaticForwarderSpanTest : public testing::TestWithParam {