Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

segment_map: Also segment the bucket array #112

Merged
merged 2 commits into from
Oct 5, 2024
Merged

segment_map: Also segment the bucket array #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4a8bbeb
segment_map: Also segment the bucket array
StephanDollberg Mar 6, 2024
c7fe207
segment_map: Allow also specifying a custom container for the bucket …
StephanDollberg Mar 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 106 additions & 52 deletions include/ankerl/unordered_dense.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@ ANKERL_UNORDERED_DENSE_PACK(struct big {
namespace detail {

struct nonesuch {};
struct default_container_t {};

template <class Default, class AlwaysVoid, template <class...> class Op, class... Args>
struct detector {
Expand Down Expand Up @@ -796,6 +797,7 @@ template <class Key,
class KeyEqual,
class AllocatorOrContainer,
class Bucket,
class BucketContainer,
bool IsSegmented>
class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, base_table_type_set> {
using underlying_value_type = typename std::conditional_t<is_map_v<T>, std::pair<Key, T>, Key>;
Expand All @@ -810,7 +812,12 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
private:
using bucket_alloc =
typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
using bucket_alloc_traits = std::allocator_traits<bucket_alloc>;
using default_bucket_container_type =
std::conditional_t<IsSegmented, segmented_vector<Bucket, bucket_alloc>, std::vector<Bucket, bucket_alloc>>;

using bucket_container_type = std::conditional_t<std::is_same_v<BucketContainer, detail::default_container_t>,
default_bucket_container_type,
BucketContainer>;

static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets
static constexpr float default_max_load_factor = 0.8F;
Expand Down Expand Up @@ -839,24 +846,26 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");

value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes.
using bucket_pointer = typename std::allocator_traits<bucket_alloc>::pointer;
bucket_pointer m_buckets{};
size_t m_num_buckets = 0;
bucket_container_type m_buckets{};
size_t m_max_bucket_capacity = 0;
float m_max_load_factor = default_max_load_factor;
Hash m_hash{};
KeyEqual m_equal{};
uint8_t m_shifts = initial_shifts;

[[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets)
return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == bucket_count())
? 0
: static_cast<value_idx_type>(bucket_idx + 1U);
}

// Helper to access bucket through pointer types
[[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& {
return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset));
[[nodiscard]] static constexpr auto at(bucket_container_type& bucket, size_t offset) -> Bucket& {
return bucket[offset];
}

[[nodiscard]] static constexpr auto at(const bucket_container_type& bucket, size_t offset) -> const Bucket& {
return bucket[offset];
}

// use the dist_inc and dist_dec functions so that uint16_t types work without warning
Expand Down Expand Up @@ -946,7 +955,13 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
} else {
m_shifts = other.m_shifts;
allocate_buckets_from_shift();
std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count());
if constexpr (IsSegmented || !std::is_same_v<BucketContainer, default_container_t>) {
for (auto i = 0UL; i < bucket_count(); ++i) {
at(m_buckets, i) = at(other.m_buckets, i);
}
} else {
std::memcpy(m_buckets.data(), other.m_buckets.data(), sizeof(Bucket) * bucket_count());
}
}
}

Expand All @@ -958,30 +973,38 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
}

void deallocate_buckets() {
auto ba = bucket_alloc(m_values.get_allocator());
if (nullptr != m_buckets) {
bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
m_buckets = nullptr;
}
m_num_buckets = 0;
m_buckets.clear();
m_buckets.shrink_to_fit();
m_max_bucket_capacity = 0;
}

void allocate_buckets_from_shift() {
auto ba = bucket_alloc(m_values.get_allocator());
m_num_buckets = calc_num_buckets(m_shifts);
m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets);
if (m_num_buckets == max_bucket_count()) {
auto num_buckets = calc_num_buckets(m_shifts);
if constexpr (IsSegmented || !std::is_same_v<BucketContainer, default_container_t>) {
if constexpr (has_reserve<bucket_container_type>) {
m_buckets.reserve(num_buckets);
}
for (size_t i = m_buckets.size(); i < num_buckets; ++i) {
m_buckets.emplace_back();
}
} else {
m_buckets.resize(num_buckets);
}
if (num_buckets == max_bucket_count()) {
// reached the maximum, make sure we can use each bucket
m_max_bucket_capacity = max_bucket_count();
} else {
m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor());
m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(num_buckets) * max_load_factor());
}
}

void clear_buckets() {
if (m_buckets != nullptr) {
std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count());
if constexpr (IsSegmented || !std::is_same_v<BucketContainer, default_container_t>) {
for (auto&& e : m_buckets) {
std::memset(&e, 0, sizeof(e));
}
} else {
std::memset(m_buckets.data(), 0, sizeof(Bucket) * bucket_count());
}
}

Expand All @@ -1004,7 +1027,9 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
on_error_bucket_overflow();
}
--m_shifts;
deallocate_buckets();
if constexpr (!IsSegmented || std::is_same_v<BucketContainer, default_container_t>) {
deallocate_buckets();
}
allocate_buckets_from_shift();
clear_and_fill_buckets_from_values();
}
Expand Down Expand Up @@ -1178,6 +1203,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
KeyEqual const& equal = KeyEqual(),
allocator_type const& alloc_or_container = allocator_type())
: m_values(alloc_or_container)
, m_buckets(alloc_or_container)
, m_hash(hash)
, m_equal(equal) {
if (0 != bucket_count) {
Expand Down Expand Up @@ -1253,12 +1279,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
: table(init, bucket_count, hash, KeyEqual(), alloc) {}

~table() {
if (nullptr != m_buckets) {
auto ba = bucket_alloc(m_values.get_allocator());
bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
}
}
~table() {}

auto operator=(table const& other) -> table& {
if (&other != this) {
Expand All @@ -1283,8 +1304,8 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas

// we can only reuse m_buckets when both maps have the same allocator!
if (get_allocator() == other.get_allocator()) {
m_buckets = std::exchange(other.m_buckets, nullptr);
m_num_buckets = std::exchange(other.m_num_buckets, 0);
m_buckets = std::move(other.m_buckets);
other.m_buckets.clear();
m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
m_shifts = std::exchange(other.m_shifts, initial_shifts);
m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
Expand Down Expand Up @@ -1421,7 +1442,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
on_error_too_many_elements();
}
auto shifts = calc_shifts_for_size(container.size());
if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
if (0 == bucket_count() || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
m_shifts = shifts;
deallocate_buckets();
allocate_buckets_from_shift();
Expand Down Expand Up @@ -1821,7 +1842,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
// bucket interface ///////////////////////////////////////////////////////

auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
return m_num_buckets;
return m_buckets.size();
}

static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
Expand All @@ -1840,7 +1861,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas

void max_load_factor(float ml) {
m_max_load_factor = ml;
if (m_num_buckets != max_bucket_count()) {
if (bucket_count() != max_bucket_count()) {
m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
}
}
Expand All @@ -1864,7 +1885,7 @@ class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, bas
m_values.reserve(capa);
}
auto shifts = calc_shifts_for_size((std::max)(capa, size()));
if (0 == m_num_buckets || shifts < m_shifts) {
if (0 == bucket_count() || shifts < m_shifts) {
m_shifts = shifts;
deallocate_buckets();
allocate_buckets_from_shift();
Expand Down Expand Up @@ -1925,30 +1946,34 @@ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
class Bucket = bucket_type::standard>
using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
class Bucket = bucket_type::standard,
class BucketContainer = detail::default_container_t>
using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false>;

ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class T,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
class Bucket = bucket_type::standard>
using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
class Bucket = bucket_type::standard,
class BucketContainer = detail::default_container_t>
using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, true>;

ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class AllocatorOrContainer = std::allocator<Key>,
class Bucket = bucket_type::standard>
using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
class Bucket = bucket_type::standard,
class BucketContainer = detail::default_container_t>
using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false>;

ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class AllocatorOrContainer = std::allocator<Key>,
class Bucket = bucket_type::standard>
using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
class Bucket = bucket_type::standard,
class BucketContainer = detail::default_container_t>
using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, true>;

# if defined(ANKERL_UNORDERED_DENSE_PMR)

Expand All @@ -1959,29 +1984,54 @@ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class Bucket = bucket_type::standard>
using map =
detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, false>;
using map = detail::table<Key,
T,
Hash,
KeyEqual,
ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>,
Bucket,
detail::default_container_t,
false>;

ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class T,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class Bucket = bucket_type::standard>
using segmented_map =
detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, true>;
using segmented_map = detail::table<Key,
T,
Hash,
KeyEqual,
ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>,
Bucket,
detail::default_container_t,
true>;

ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class Bucket = bucket_type::standard>
using set = detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, false>;
using set = detail::table<Key,
void,
Hash,
KeyEqual,
ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>,
Bucket,
detail::default_container_t,
false>;

ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class Hash = hash<Key>,
class KeyEqual = std::equal_to<Key>,
class Bucket = bucket_type::standard>
using segmented_set =
detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, true>;
using segmented_set = detail::table<Key,
void,
Hash,
KeyEqual,
ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>,
Bucket,
detail::default_container_t,
true>;

} // namespace pmr

Expand All @@ -2006,11 +2056,15 @@ ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
class AllocatorOrContainer,
class Bucket,
class Pred,
class BucketContainer,
bool IsSegmented>
// NOLINTNEXTLINE(cert-dcl58-cpp)
auto erase_if(ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>& map,
Pred pred) -> size_t {
using map_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>;
auto erase_if(
ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, IsSegmented>&
map,
Pred pred) -> size_t {
using map_t = ankerl::unordered_dense::detail::
table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, IsSegmented>;

// going back to front because erase() invalidates the end iterator
auto const old_size = map.size();
Expand Down
19 changes: 12 additions & 7 deletions test/app/doctest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,25 @@ template <class Key,
class Hash = ankerl::unordered_dense::hash<Key>,
class KeyEqual = std::equal_to<Key>,
class AllocatorOrContainer = std::deque<std::pair<Key, T>>,
class Bucket = ankerl::unordered_dense::bucket_type::standard>
class deque_map : public ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false> {
using base_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
class Bucket = ankerl::unordered_dense::bucket_type::standard,
class BucketContainer = std::deque<Bucket>>
class deque_map : public ankerl::unordered_dense::detail::
table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false> {
using base_t =
ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false>;
using base_t::base_t;
};

template <class Key,
class Hash = ankerl::unordered_dense::hash<Key>,
class KeyEqual = std::equal_to<Key>,
class AllocatorOrContainer = std::deque<Key>,
class Bucket = ankerl::unordered_dense::bucket_type::standard>
class deque_set
: public ankerl::unordered_dense::detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false> {
using base_t = ankerl::unordered_dense::detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
class Bucket = ankerl::unordered_dense::bucket_type::standard,
class BucketContainer = std::deque<Bucket>>
class deque_set : public ankerl::unordered_dense::detail::
table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false> {
using base_t = ankerl::unordered_dense::detail::
table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, BucketContainer, false>;
using base_t::base_t;
};

Expand Down
4 changes: 3 additions & 1 deletion test/unit/custom_container.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ TEST_CASE_MAP("custom_container",
std::string,
ankerl::unordered_dense::hash<int>,
std::equal_to<int>,
std::deque<std::pair<int, std::string>>) {
std::deque<std::pair<int, std::string>>,
ankerl::unordered_dense::bucket_type::standard,
std::deque<ankerl::unordered_dense::bucket_type::standard>) {
auto map = map_t();

for (int i = 0; i < 10; ++i) {
Expand Down