Skip to content

Commit

Permalink
Merge pull request #216 from greg7mdp/try_emplace_fine_locking
Browse files Browse the repository at this point in the history 
For parallel maps with read-capable mutexes, do read lock for lookup in `try_emplace_l`
  • Loading branch information
@greg7mdp
greg7mdp authored Nov 15, 2023
2 parents 401552d + ff7bd78 commit 0046905
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 38 deletions.
108 changes: 70 additions & 38 deletions parallel_hashmap/phmap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1547,12 +1547,13 @@ class raw_hash_set

template <class K = key_type, class F>
iterator lazy_emplace_with_hash(const key_arg<K>& key, size_t hashval, F&& f) {
auto res = find_or_prepare_insert(key, hashval);
if (res.second) {
lazy_emplace_at(res.first, std::forward<F>(f));
this->set_ctrl(res.first, H2(hashval));
size_t offset = _find_key(key, hashval);
if (offset == (size_t)-1) {
offset = prepare_insert(hashval);
lazy_emplace_at(offset, std::forward<F>(f));
this->set_ctrl(offset, H2(hashval));
}
return iterator_at(res.first);
return iterator_at(offset);
}

template <class K = key_type, class F>
Expand All @@ -1564,12 +1565,13 @@ class raw_hash_set

template <class K = key_type, class F>
void emplace_single_with_hash(const key_arg<K>& key, size_t hashval, F&& f) {
auto res = find_or_prepare_insert(key, hashval);
if (res.second) {
lazy_emplace_at(res.first, std::forward<F>(f));
this->set_ctrl(res.first, H2(hashval));
size_t offset = _find_key(key, hashval);
if (offset == (size_t)-1) {
offset = prepare_insert(hashval);
lazy_emplace_at(offset, std::forward<F>(f));
this->set_ctrl(offset, H2(hashval));
} else
_erase(iterator_at(res.first));
_erase(iterator_at(offset));
}


Expand Down Expand Up @@ -1897,12 +1899,14 @@ class raw_hash_set
std::pair<iterator, bool> emplace_decomposable(const K& key, size_t hashval,
Args&&... args)
{
auto res = find_or_prepare_insert(key, hashval);
if (res.second) {
emplace_at(res.first, std::forward<Args>(args)...);
this->set_ctrl(res.first, H2(hashval));
size_t offset = _find_key(key, hashval);
if (offset == (size_t)-1) {
offset = prepare_insert(hashval);
emplace_at(offset, std::forward<Args>(args)...);
this->set_ctrl(offset, H2(hashval));
return {iterator_at(offset), true};
}
return {iterator_at(res.first), res.second};
return {iterator_at(offset), false};
}

struct EmplaceDecomposable
Expand Down Expand Up @@ -2194,20 +2198,29 @@ class raw_hash_set

protected:
template <class K>
std::pair<size_t, bool> find_or_prepare_insert(const K& key, size_t hashval) {
size_t _find_key(const K& key, size_t hashval) {
auto seq = probe(hashval);
while (true) {
Group g{ctrl_ + seq.offset()};
for (uint32_t i : g.Match((h2_t)H2(hashval))) {
if (PHMAP_PREDICT_TRUE(PolicyTraits::apply(
EqualElement<K>{key, eq_ref()},
PolicyTraits::element(slots_ + seq.offset((size_t)i)))))
return {seq.offset((size_t)i), false};
return seq.offset((size_t)i);
}
if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break;
seq.next();
}
return {prepare_insert(hashval), true};
return (size_t)-1;
}


template <class K>
std::pair<size_t, bool> find_or_prepare_insert(const K& key, size_t hashval) {
size_t offset = _find_key(key, hashval);
if (offset == (size_t)-1)
return {prepare_insert(hashval), true};
return {offset, false};
}

size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE {
Expand Down Expand Up @@ -2479,26 +2492,30 @@ class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc>
template <class K, class V>
std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) {
size_t hashval = this->hash(k);
auto res = this->find_or_prepare_insert(k, hashval);
if (res.second) {
this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v));
this->set_ctrl(res.first, H2(hashval));
} else
Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v);
return {this->iterator_at(res.first), res.second};
size_t offset = this->_find_key(k, hashval);
if (offset == (size_t)-1) {
offset = this->prepare_insert(hashval);
this->emplace_at(offset, std::forward<K>(k), std::forward<V>(v));
this->set_ctrl(offset, H2(hashval));
return {this->iterator_at(offset), true};
}
Policy::value(&*this->iterator_at(offset)) = std::forward<V>(v);
return {this->iterator_at(offset), false};
}

template <class K = key_type, class... Args>
std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) {
size_t hashval = this->hash(k);
auto res = this->find_or_prepare_insert(k, hashval);
if (res.second) {
this->emplace_at(res.first, std::piecewise_construct,
size_t offset = this->_find_key(k, hashval);
if (offset == (size_t)-1) {
offset = this->prepare_insert(hashval);
this->emplace_at(offset, std::piecewise_construct,
std::forward_as_tuple(std::forward<K>(k)),
std::forward_as_tuple(std::forward<Args>(args)...));
this->set_ctrl(res.first, H2(hashval));
this->set_ctrl(offset, H2(hashval));
return {this->iterator_at(offset), true};
}
return {this->iterator_at(res.first), res.second};
return {this->iterator_at(offset), false};
}
};

Expand Down Expand Up @@ -2565,6 +2582,7 @@ class parallel_hash_set

protected:
using Lockable = phmap::LockableImpl<Mtx_>;


// --------------------------------------------------------------------
struct Inner : public Lockable
Expand Down Expand Up @@ -3324,7 +3342,7 @@ class parallel_hash_set
template <class K = key_type, class FExists, class FEmplace>
bool lazy_emplace_l(const key_arg<K>& key, FExists&& fExists, FEmplace&& fEmplace) {
size_t hashval = this->hash(key);
typename Lockable::UniqueLock m;
typename Lockable::ReadWriteLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, key, m);
Inner* inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand Down Expand Up @@ -3781,17 +3799,31 @@ class parallel_hash_set

template <class K>
std::tuple<Inner*, size_t, bool>
find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) {
find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::ReadWriteLock &mutexlock) {
Inner& inner = sets_[subidx(hashval)];
auto& set = inner.set_;
mutexlock = std::move(typename Lockable::UniqueLock(inner));
auto p = set.find_or_prepare_insert(key, hashval); // std::pair<size_t, bool>
return std::make_tuple(&inner, p.first, p.second);
mutexlock = std::move(typename Lockable::ReadWriteLock(inner));
size_t offset = set._find_key(key, hashval);
if (offset == (size_t)-1) {
if (mutexlock.switch_to_unique()) {
// we did an unlock/lock, and another thread could have inserted the same key, so we need to
// do a find() again.
offset = set._find_key(key, hashval);
if (offset == (size_t)-1) {
offset = set.prepare_insert(hashval);
return std::make_tuple(&inner, offset, true);
}
} else {
offset = set.prepare_insert(hashval);
return std::make_tuple(&inner, offset, true);
}
}
return std::make_tuple(&inner, offset, false);
}

template <class K>
std::tuple<Inner*, size_t, bool>
find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) {
find_or_prepare_insert(const K& key, typename Lockable::ReadWriteLock &mutexlock) {
return find_or_prepare_insert_with_hash<K>(this->hash(key), key, mutexlock);
}

Expand Down Expand Up @@ -4060,7 +4092,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
template <class K, class V>
std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) {
size_t hashval = this->hash(k);
typename Lockable::UniqueLock m;
typename Lockable::ReadWriteLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
typename Base::Inner *inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand All @@ -4080,7 +4112,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash

template <class K = key_type, class... Args>
std::pair<iterator, bool> try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) {
typename Lockable::UniqueLock m;
typename Lockable::ReadWriteLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
typename Base::Inner *inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand Down
103 changes: 103 additions & 0 deletions parallel_hashmap/phmap_base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4631,6 +4631,11 @@ class LockableBaseImpl
DoNothing& operator=(DoNothing&&) noexcept { return *this; }
void swap(DoNothing &) {}
bool owns_lock() const noexcept { return true; }
void lock() {}
void unlock() {}
void lock_shared() {}
void unlock_shared() {}
bool switch_to_unique() { return false; }
};

// ----------------------------------------------------
Expand Down Expand Up @@ -4705,6 +4710,8 @@ class LockableBaseImpl
}

mutex_type *mutex() const noexcept { return m_; }

bool switch_to_unique() { return false; }

private:
mutex_type *m_;
Expand Down Expand Up @@ -4784,8 +4791,99 @@ class LockableBaseImpl

mutex_type *mutex() const noexcept { return m_; }

bool switch_to_unique() { return false; }

private:
mutex_type *m_;
bool locked_;
};

// ----------------------------------------------------
class ReadWriteLock
{
public:
using mutex_type = MutexType;

ReadWriteLock() : m_(nullptr), locked_(false), locked_shared_(false) {}

explicit ReadWriteLock(mutex_type &m) : m_(&m), locked_(false), locked_shared_(true) {
m_->lock_shared();
}

ReadWriteLock(mutex_type& m, defer_lock_t) noexcept :
m_(&m), locked_(false), locked_shared_(false)
{}

ReadWriteLock(ReadWriteLock &&o) noexcept :
m_(std::move(o.m_)), locked_(o.locked_), locked_shared_(o.locked_shared_) {
o.locked_ = false;
o.locked_shared_ = false;
o.m_ = nullptr;
}

ReadWriteLock& operator=(ReadWriteLock&& other) noexcept {
ReadWriteLock temp(std::move(other));
swap(temp);
return *this;
}

~ReadWriteLock() {
if (locked_shared_)
m_->unlock_shared();
else if (locked_)
m_->unlock();
}

void lock_shared() {
if (!locked_shared_) {
m_->lock_shared();
locked_shared_ = true;
}
}

void unlock_shared() {
if (locked_shared_) {
m_->unlock_shared();
locked_shared_ = false;
}
}

void lock() {
if (!locked_) {
m_->lock();
locked_ = true;
}
}

void unlock() {
if (locked_) {
m_->unlock();
locked_ = false;
}
}

bool owns_lock() const noexcept { return locked_; }
bool owns_shared_lock() const noexcept { return locked_shared_; }

void swap(ReadWriteLock &o) noexcept {
std::swap(m_, o.m_);
std::swap(locked_, o.locked_);
std::swap(locked_shared_, o.locked_shared_);
}

mutex_type *mutex() const noexcept { return m_; }

bool switch_to_unique() {
assert(locked_shared_);
unlock_shared();
lock();
return true;
}


private:
mutex_type *m_;
bool locked_shared_;
bool locked_;
};

Expand Down Expand Up @@ -4877,6 +4975,7 @@ class LockableImpl : public Mtx_
using SharedLock = typename Base::WriteLock;
using UpgradeLock = typename Base::WriteLock;
using UniqueLock = typename Base::WriteLock;
using ReadWriteLock = typename Base::WriteLock;
using SharedLocks = typename Base::WriteLocks;
using UniqueLocks = typename Base::WriteLocks;
using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership
Expand All @@ -4893,6 +4992,7 @@ class LockableImpl<phmap::NullMutex>: public phmap::NullMutex
using Base = LockableBaseImpl<phmap::NullMutex>;
using SharedLock = typename Base::DoNothing;
using UpgradeLock = typename Base::DoNothing;
using ReadWriteLock = typename Base::DoNothing;
using UniqueLock = typename Base::DoNothing;
using UpgradeToUnique = typename Base::DoNothing;
using SharedLocks = typename Base::DoNothing;
Expand Down Expand Up @@ -4921,6 +5021,7 @@ class LockableImpl<phmap::NullMutex>: public phmap::NullMutex
using mutex_type = phmap::AbslMutex;
using Base = LockableBaseImpl<phmap::AbslMutex>;
using SharedLock = typename Base::ReadLock;
using ReadWriteLock = typename Base::ReadWriteLock;
using UpgradeLock = typename Base::WriteLock;
using UniqueLock = typename Base::WriteLock;
using SharedLocks = typename Base::ReadLocks;
Expand All @@ -4943,6 +5044,7 @@ class LockableImpl<phmap::NullMutex>: public phmap::NullMutex
using mutex_type = boost::shared_mutex;
using Base = LockableBaseImpl<boost::shared_mutex>;
using SharedLock = boost::shared_lock<mutex_type>;
using ReadWriteLock = typename Base::ReadWriteLock;
using UpgradeLock = boost::unique_lock<mutex_type>; // assume can't upgrade
using UniqueLock = boost::unique_lock<mutex_type>;
using SharedLocks = typename Base::ReadLocks;
Expand All @@ -4965,6 +5067,7 @@ class LockableImpl<phmap::NullMutex>: public phmap::NullMutex
using mutex_type = std::shared_mutex;
using Base = LockableBaseImpl<std::shared_mutex>;
using SharedLock = std::shared_lock<mutex_type>;
using ReadWriteLock = typename Base::ReadWriteLock;
using UpgradeLock = std::unique_lock<mutex_type>; // assume can't upgrade
using UniqueLock = std::unique_lock<mutex_type>;
using SharedLocks = typename Base::ReadLocks;
Expand Down

0 comments on commit 0046905

Please sign in to comment.