Skip to content

Commit d2afea2

Browse files
mpucciompuccio
committed
GenericTracking: start and import BoundedAllocator
1 parent 86084e0 commit d2afea2

25 files changed

Lines changed: 461 additions & 191 deletions

‎Detectors/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ add_subdirectory(HMPID)
2525
add_subdirectory(TOF)
2626
add_subdirectory(ZDC)
2727

28+
add_subdirectory(GenericTracking)
2829
add_subdirectory(ITSMFT)
2930
add_subdirectory(External) # sensitive external (CAD-derived) detectors; uses ITSMFT hit type
3031
add_subdirectory(TRD)

‎Detectors/GenericTracking/CMakeLists.txt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
# All rights not expressly granted are reserved.
4+
#
5+
# This software is distributed under the terms of the GNU General Public
6+
# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
#
8+
# In applying this license CERN does not waive the privileges and immunities
9+
# granted to it by virtue of its status as an Intergovernmental Organization
10+
# or submit itself to any jurisdiction.
11+
o2_add_library(GenericTracking
12+
SOURCES src/BoundedAllocator.cxx
13+
PRIVATE_LINK_LIBRARIES O2::GPUCommon O2::FrameworkLogger)
14+
15+
add_subdirectory(test)

‎Detectors/GenericTracking/include/GenericTracking/BoundedAllocator.h‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
///
12+
/// \file BoundedAllocator.h
13+
/// \brief Shared budgeted host allocation and bounded-vector utilities.
14+
///
15+
/// Reconciles GTRK's overflow-safe allocator, profiling counters and vector
16+
/// rebuilds with the owned-resource, mutable-budget and stage-peak contracts
17+
/// used by the upstream legacy ITS tracker and vertexer.
18+
///
19+
20+
#ifndef ALICEO2_GENERICTRACKING_BOUNDEDALLOCATOR_H_
21+
#define ALICEO2_GENERICTRACKING_BOUNDEDALLOCATOR_H_
22+
23+
#include <algorithm>
24+
#include <array>
25+
#include <atomic>
26+
#include <limits>
27+
#include <memory>
28+
#include <memory_resource>
29+
#include <new>
30+
#include <string>
31+
#include <utility>
32+
#include <vector>
33+
34+
namespace o2::gtrk
35+
{
36+
37+
// #define BOUNDED_MR_STATS
38+
class BoundedMemoryResource final : public std::pmr::memory_resource
39+
{
40+
public:
41+
class MemoryLimitExceeded final : public std::bad_alloc
42+
{
43+
public:
44+
MemoryLimitExceeded(size_t attempted, size_t used, size_t max);
45+
const char* what() const noexcept final;
46+
47+
private:
48+
std::string mMsg;
49+
};
50+
51+
static std::pmr::memory_resource* cachingUpstream();
52+
53+
BoundedMemoryResource(size_t maxBytes = std::numeric_limits<size_t>::max(),
54+
std::pmr::memory_resource* upstream = nullptr);
55+
56+
BoundedMemoryResource(std::unique_ptr<std::pmr::memory_resource> upstream,
57+
size_t maxBytes = std::numeric_limits<size_t>::max());
58+
59+
[[nodiscard]] size_t getUsedMemory() const noexcept;
60+
[[nodiscard]] size_t getMaxMemory() const noexcept;
61+
[[nodiscard]] size_t getThrowCount() const noexcept;
62+
[[nodiscard]] size_t getPeakMemory() const noexcept;
63+
[[nodiscard]] size_t getPeakMemoryDelta() const noexcept;
64+
65+
// Reset the stage peak/baseline, returning the previous peak for nested profiling.
66+
size_t resetPeakMemory() noexcept;
67+
void raisePeakMemory(size_t peak) noexcept;
68+
[[nodiscard]] size_t getAllocationCount() const noexcept { return mAllocations.load(std::memory_order_relaxed); }
69+
[[nodiscard]] size_t getAllocatedBytes() const noexcept { return mAllocatedBytes.load(std::memory_order_relaxed); }
70+
void setMaxMemory(size_t max);
71+
72+
#if !defined(__HIPCC__) && !defined(__CUDACC__)
73+
std::string asString() const;
74+
void print() const;
75+
#endif
76+
77+
private:
78+
void* do_allocate(size_t bytes, size_t alignment) final;
79+
void do_deallocate(void* p, size_t bytes, size_t alignment) final;
80+
bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final;
81+
82+
std::atomic<size_t> mMaxMemory{std::numeric_limits<size_t>::max()};
83+
std::atomic<size_t> mCountThrow{0};
84+
std::atomic<size_t> mUsedMemory{0};
85+
std::atomic<size_t> mAllocations{0};
86+
std::atomic<size_t> mAllocatedBytes{0};
87+
std::atomic<size_t> mPeakUsedMemory{0};
88+
std::atomic<size_t> mPeakBaselineMemory{0};
89+
std::unique_ptr<std::pmr::memory_resource> mOwnedUpstream;
90+
std::pmr::memory_resource* mUpstream{nullptr};
91+
92+
#ifdef BOUNDED_MR_STATS
93+
struct Stats {
94+
std::atomic<size_t> peak{0};
95+
std::atomic<size_t> live{0};
96+
std::atomic<size_t> nAlloc{0};
97+
std::atomic<size_t> nFree{0};
98+
std::atomic<size_t> totalAlloc{0};
99+
std::atomic<size_t> totalFreed{0};
100+
std::atomic<size_t> maxAlign{0};
101+
std::atomic<size_t> upstreamFailures{0};
102+
};
103+
Stats mStats{};
104+
#endif
105+
};
106+
107+
template <typename T>
108+
using BoundedVector = std::pmr::vector<T>;
109+
110+
template <typename T>
111+
using bounded_vector = BoundedVector<T>;
112+
113+
template <typename T>
114+
inline void deepVectorClear(std::vector<T>& vec)
115+
{
116+
std::vector<T>().swap(vec);
117+
}
118+
119+
template <typename T>
120+
inline void deepVectorClear(bounded_vector<T>& vec, std::pmr::memory_resource* mr = nullptr)
121+
{
122+
std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource();
123+
std::destroy_at(&vec);
124+
new (&vec) bounded_vector<T>(std::pmr::polymorphic_allocator<T>{tmr});
125+
}
126+
127+
template <typename T>
128+
inline void deepVectorClear(std::vector<bounded_vector<T>>& vec, std::pmr::memory_resource* mr = nullptr)
129+
{
130+
for (auto& v : vec) {
131+
deepVectorClear(v, mr);
132+
}
133+
}
134+
135+
template <typename T, size_t S>
136+
inline void deepVectorClear(std::array<bounded_vector<T>, S>& arr, std::pmr::memory_resource* mr = nullptr)
137+
{
138+
for (size_t i{0}; i < S; ++i) {
139+
deepVectorClear(arr[i], mr);
140+
}
141+
}
142+
143+
template <typename T>
144+
inline void clearResizeBoundedVector(bounded_vector<T>& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T())
145+
{
146+
// Rebuild a valid empty vector before allocation or element construction can
147+
// throw. The old allocation is released first, as required by bounded pools.
148+
deepVectorClear(vec, mr);
149+
vec.assign(sz, def);
150+
}
151+
152+
// GTRK's capacity-reusing resize: retained storage is intended for arrays
153+
// rewritten by the caller. A non-null different resource requires a rebuild.
154+
template <typename T>
155+
inline void resizeBoundedVector(BoundedVector<T>& vec, size_t sz, std::pmr::memory_resource* mr = nullptr)
156+
{
157+
if (sz > vec.capacity() || (mr != nullptr && mr != vec.get_allocator().resource())) {
158+
clearResizeBoundedVector(vec, sz, mr);
159+
return;
160+
}
161+
vec.resize(sz);
162+
}
163+
164+
// Release several bounded vectors onto one resource.
165+
template <typename... T>
166+
inline void deepVectorClear(std::pmr::memory_resource* mr, BoundedVector<T>&... vecs)
167+
{
168+
(deepVectorClear(vecs, mr), ...);
169+
}
170+
171+
template <typename T>
172+
inline void clearResizeBoundedVector(std::vector<bounded_vector<T>>& vec, size_t size, std::pmr::memory_resource* mr)
173+
{
174+
vec.clear();
175+
vec.reserve(size);
176+
for (size_t i = 0; i < size; ++i) {
177+
vec.emplace_back(std::pmr::polymorphic_allocator<bounded_vector<T>>{mr});
178+
}
179+
}
180+
181+
template <typename T, size_t S>
182+
inline void clearResizeBoundedArray(std::array<bounded_vector<T>, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T())
183+
{
184+
for (size_t i{0}; i < S; ++i) {
185+
clearResizeBoundedVector(arr[i], size, mr, def);
186+
}
187+
}
188+
189+
template <typename T>
190+
inline std::vector<T> toSTDVector(const bounded_vector<T>& b)
191+
{
192+
std::vector<T> t(b.size());
193+
std::copy(b.cbegin(), b.cend(), t.begin());
194+
return t;
195+
}
196+
197+
} // namespace o2::gtrk
198+
199+
#endif /* ALICEO2_GENERICTRACKING_BOUNDEDALLOCATOR_H_ */

Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx renamed to Detectors/GenericTracking/src/BoundedAllocator.cxx

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,19 @@
99
// granted to it by virtue of its status as an Intergovernmental Organization
1010
// or submit itself to any jurisdiction.
1111

12-
#include "ITSMFTTracking/BoundedAllocator.h"
12+
#include "GenericTracking/BoundedAllocator.h"
1313

1414
#include <cstdio>
1515
#include <format>
1616

1717
#include "GPUCommonLogger.h"
18-
#include "ITSMFTTracking/Constants.h"
1918

20-
namespace o2::itsmft::tracking
19+
namespace o2::gtrk
2120
{
21+
namespace
22+
{
23+
constexpr double kGigabyte = 1024. * 1024. * 1024.;
24+
}
2225

2326
BoundedMemoryResource::MemoryLimitExceeded::MemoryLimitExceeded(size_t attempted, size_t used, size_t max)
2427
{
@@ -56,13 +59,19 @@ void* BoundedMemoryResource::do_allocate(size_t bytes, size_t alignment)
5659
{
5760
size_t newUsed{0};
5861
size_t currentUsed{mUsedMemory.load(std::memory_order_relaxed)};
59-
do {
60-
newUsed = currentUsed + bytes;
61-
if (newUsed > mMaxMemory.load(std::memory_order_relaxed)) {
62+
for (;;) {
63+
const size_t limit = mMaxMemory.load(std::memory_order_relaxed);
64+
// Test before adding, so even SIZE_MAX requests cannot wrap under budget.
65+
if (currentUsed > limit || bytes > limit - currentUsed) {
66+
const size_t attempted = bytes > std::numeric_limits<size_t>::max() - currentUsed ? std::numeric_limits<size_t>::max() : currentUsed + bytes;
6267
mCountThrow.fetch_add(1, std::memory_order_relaxed);
63-
throw MemoryLimitExceeded(newUsed, currentUsed, mMaxMemory.load(std::memory_order_relaxed));
68+
throw MemoryLimitExceeded(attempted, currentUsed, limit);
69+
}
70+
newUsed = currentUsed + bytes;
71+
if (mUsedMemory.compare_exchange_weak(currentUsed, newUsed, std::memory_order_acq_rel, std::memory_order_relaxed)) {
72+
break;
6473
}
65-
} while (!mUsedMemory.compare_exchange_weak(currentUsed, newUsed, std::memory_order_acq_rel, std::memory_order_relaxed));
74+
}
6675

6776
void* p{nullptr};
6877
try {
@@ -75,6 +84,9 @@ void* BoundedMemoryResource::do_allocate(size_t bytes, size_t alignment)
7584
throw;
7685
}
7786

87+
mAllocations.fetch_add(1, std::memory_order_relaxed);
88+
mAllocatedBytes.fetch_add(bytes, std::memory_order_relaxed);
89+
7890
size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed);
7991
while (newUsed > peak && !mPeakUsedMemory.compare_exchange_weak(peak, newUsed, std::memory_order_relaxed)) {
8092
}
@@ -137,11 +149,18 @@ size_t BoundedMemoryResource::getPeakMemoryDelta() const noexcept
137149
return peak > baseline ? peak - baseline : 0;
138150
}
139151

140-
void BoundedMemoryResource::resetPeakMemory() noexcept
152+
size_t BoundedMemoryResource::resetPeakMemory() noexcept
141153
{
142154
const size_t used = mUsedMemory.load(std::memory_order_acquire);
143155
mPeakBaselineMemory.store(used, std::memory_order_release);
144-
mPeakUsedMemory.store(used, std::memory_order_release);
156+
return mPeakUsedMemory.exchange(used, std::memory_order_acq_rel);
157+
}
158+
159+
void BoundedMemoryResource::raisePeakMemory(size_t peak) noexcept
160+
{
161+
size_t current = mPeakUsedMemory.load(std::memory_order_relaxed);
162+
while (peak > current && !mPeakUsedMemory.compare_exchange_weak(current, peak, std::memory_order_relaxed)) {
163+
}
145164
}
146165

147166
void BoundedMemoryResource::setMaxMemory(size_t max)
@@ -174,18 +193,18 @@ std::string BoundedMemoryResource::asString() const
174193
const auto maxMemory = mMaxMemory.load(std::memory_order_relaxed);
175194
std::string result;
176195
if (maxMemory == std::numeric_limits<size_t>::max()) {
177-
result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / o2::its::constants::GB, peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB);
196+
result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / kGigabyte, peak / kGigabyte, peakDelta / kGigabyte);
178197
} else {
179-
result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast<double>(maxMemory) / o2::its::constants::GB, used / o2::its::constants::GB, 100.0 * used / static_cast<double>(maxMemory), peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB);
198+
result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast<double>(maxMemory) / kGigabyte, used / kGigabyte, 100.0 * used / static_cast<double>(maxMemory), peak / kGigabyte, peakDelta / kGigabyte);
180199
}
181200
#ifdef BOUNDED_MR_STATS
182201
result += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}",
183-
static_cast<float>(mStats.peak.load(std::memory_order_relaxed)) / o2::its::constants::GB,
202+
static_cast<float>(mStats.peak.load(std::memory_order_relaxed)) / kGigabyte,
184203
mStats.live.load(std::memory_order_relaxed),
185204
mStats.nAlloc.load(std::memory_order_relaxed),
186205
mStats.nFree.load(std::memory_order_relaxed),
187-
static_cast<float>(mStats.totalAlloc.load(std::memory_order_relaxed)) / o2::its::constants::GB,
188-
static_cast<float>(mStats.totalFreed.load(std::memory_order_relaxed)) / o2::its::constants::GB,
206+
static_cast<float>(mStats.totalAlloc.load(std::memory_order_relaxed)) / kGigabyte,
207+
static_cast<float>(mStats.totalFreed.load(std::memory_order_relaxed)) / kGigabyte,
189208
mStats.maxAlign.load(std::memory_order_relaxed),
190209
mStats.upstreamFailures.load(std::memory_order_relaxed));
191210
#endif
@@ -197,4 +216,4 @@ void BoundedMemoryResource::print() const
197216
LOGP(info, "{}", asString());
198217
}
199218

200-
} // namespace o2::itsmft::tracking
219+
} // namespace o2::gtrk

‎Detectors/GenericTracking/test/CMakeLists.txt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
# All rights not expressly granted are reserved.
4+
#
5+
# This software is distributed under the terms of the GNU General Public
6+
# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
#
8+
# In applying this license CERN does not waive the privileges and immunities
9+
# granted to it by virtue of its status as an Intergovernmental Organization
10+
# or submit itself to any jurisdiction.
11+
o2_add_test(bounded-allocator
12+
SOURCES testBoundedAllocator.cxx
13+
COMPONENT_NAME generic-tracking
14+
LABELS "generic-tracking"
15+
PUBLIC_LINK_LIBRARIES O2::GenericTracking)

0 commit comments

Comments
 (0)