From d5b579fbd66e23aed538419f544c5b3068545fc5 Mon Sep 17 00:00:00 2001 From: Davi de Castro Reis Date: Sun, 3 Jun 2012 03:13:06 -0300 Subject: [PATCH] Generalized mph_map for trade-offs. --- configure.ac | 2 +- cxxmph/bm_map.cc | 22 ++++++++++----- cxxmph/mph_index.h | 68 +++++++++++++++++++++++++++++++++++++++------- cxxmph/mph_map.h | 66 ++++++++++++++++++++++++++------------------ 4 files changed, 114 insertions(+), 44 deletions(-) diff --git a/configure.ac b/configure.ac index a149229..6cb34c3 100644 --- a/configure.ac +++ b/configure.ac @@ -34,7 +34,7 @@ LDFLAGS="$LIBM $LDFLAGS" CFLAGS="-Wall" AC_PROG_CXX -CXXFLAGS="-Wall -Wno-unused-function -DNDEBUG -O3 -fomit-frame-pointer $CXXFLAGS" +CXXFLAGS="$CXXFLAGS -Wall -Wno-unused-function -DNDEBUG -O3 -fomit-frame-pointer" AC_ENABLE_CXXMPH if test x$cxxmph = xtrue; then AC_COMPILE_STDCXX_0X diff --git a/cxxmph/bm_map.cc b/cxxmph/bm_map.cc index f777a2b..4dd071c 100644 --- a/cxxmph/bm_map.cc +++ b/cxxmph/bm_map.cc @@ -4,16 +4,13 @@ #include "bm_common.h" #include "mph_map.h" -using cxxmph::mph_map; using std::string; -using std::unordered_map; // Another reference benchmark: // http://blog.aggregateknowledge.com/tag/bigmemory/ namespace cxxmph { - template const T* myfind(const MapType& mymap, const T& k) { auto it = mymap.find(k); @@ -100,13 +97,24 @@ using namespace cxxmph; int main(int argc, char** argv) { srandom(4); + Benchmark::Register(new BM_CreateUrls>("URLS100k")); + Benchmark::Register(new BM_CreateUrls>("URLS100k")); Benchmark::Register(new BM_CreateUrls>("URLS100k")); - Benchmark::Register(new BM_CreateUrls>("URLS100k")); + Benchmark::Register(new BM_CreateUrls>("URLS100k")); + + Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0)); + Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0)); Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0)); - Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0)); + Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0)); + + Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0.9)); + Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0.9)); Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0.9)); - Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0.9)); + Benchmark::Register(new BM_SearchUrls>("URLS100k", 10*1000 * 1000, 0.9)); + + Benchmark::Register(new BM_SearchUint64>); + Benchmark::Register(new BM_SearchUint64>); Benchmark::Register(new BM_SearchUint64>); - Benchmark::Register(new BM_SearchUint64>); + Benchmark::Register(new BM_SearchUint64>); Benchmark::RunAll(); } diff --git a/cxxmph/mph_index.h b/cxxmph/mph_index.h index 1f6ace1..57e9e64 100644 --- a/cxxmph/mph_index.h +++ b/cxxmph/mph_index.h @@ -48,8 +48,8 @@ namespace cxxmph { class MPHIndex { public: - MPHIndex(double c = 1.23, uint8_t b = 7) : - c_(c), b_(b), m_(0), n_(0), k_(0), r_(1), + MPHIndex(bool square = false, double c = 1.23, uint8_t b = 7) : + c_(c), b_(b), m_(0), n_(0), k_(0), square_(square), r_(1), ranktable_(NULL), ranktable_size_(0) { } ~MPHIndex(); @@ -66,6 +66,8 @@ class MPHIndex { uint32_t perfect_hash_size() const { return n_; } template // must agree with Reset uint32_t perfect_hash(const Key& x) const; // way faster than the minimal + template // must agree with Reset + uint32_t perfect_square(const Key& x) const; // even faster but needs square=true uint32_t minimal_perfect_hash_size() const { return size(); } template // must agree with Reset uint32_t minimal_perfect_hash(const Key& x) const; @@ -93,6 +95,7 @@ class MPHIndex { uint32_t m_; // edges count uint32_t n_; // vertex count uint32_t k_; // kth index in ranktable, $k = log_2(n=3r)\varepsilon$ + bool square_; // make bit vector size a power of 2 // Values used during search @@ -124,7 +127,7 @@ bool MPHIndex::Reset( if ((r_ % 2) == 0) r_ += 1; // This can be used to speed mods, but increases occupation too much. // Needs to try http://gmplib.org/manual/Integer-Exponentiation.html instead - // r_ = nextpoweroftwo(r_); + if (square_) r_ = nextpoweroftwo(r_); nest_displacement_[0] = 0; nest_displacement_[1] = r_; nest_displacement_[2] = (r_ << 1); @@ -173,6 +176,21 @@ bool MPHIndex::Mapping( return false; } +template +uint32_t MPHIndex::perfect_square(const Key& key) const { + if (!g_.size()) return 0; + h128 h = SeededHashFcn().hash128(key, hash_seed_[0]); + h[0] = (h[0] & (r_-1)) + nest_displacement_[0]; + h[1] = (h[1] & (r_-1)) + nest_displacement_[1]; + h[2] = (h[2] & (r_-1)) + nest_displacement_[2]; + assert((h[0]) < g_.size()); + assert((h[1]) < g_.size()); + assert((h[2]) < g_.size()); + uint8_t nest = threebit_mod3[g_[h[0]] + g_[h[1]] + g_[h[2]]]; + uint32_t vertex = h[nest]; + return vertex; +} + template uint32_t MPHIndex::perfect_hash(const Key& key) const { if (!g_.size()) return 0; @@ -180,17 +198,14 @@ uint32_t MPHIndex::perfect_hash(const Key& key) const { h[0] = (h[0] % r_) + nest_displacement_[0]; h[1] = (h[1] % r_) + nest_displacement_[1]; h[2] = (h[2] % r_) + nest_displacement_[2]; - // h[0] = (h[0] & (r_-1)) + nest_displacement_[0]; - // h[1] = (h[1] & (r_-1)) + nest_displacement_[1]; - // h[2] = (h[2] & (r_-1)) + nest_displacement_[2]; assert((h[0]) < g_.size()); assert((h[1]) < g_.size()); assert((h[2]) < g_.size()); - uint8_t nest = threebit_mod3[ - g_[h[0]] + g_[h[1]] + g_[h[2]]]; + uint8_t nest = threebit_mod3[g_[h[0]] + g_[h[1]] + g_[h[2]]]; uint32_t vertex = h[nest]; return vertex; } + template uint32_t MPHIndex::minimal_perfect_hash(const Key& key) const { return Rank(perfect_hash(key)); @@ -206,15 +221,48 @@ uint32_t MPHIndex::index(const Key& key) const { template >::hash_function> class SimpleMPHIndex : public MPHIndex { public: + SimpleMPHIndex(bool advanced_usage = false) : MPHIndex(advanced_usage) {} template bool Reset(ForwardIterator begin, ForwardIterator end, uint32_t size) { return MPHIndex::Reset(begin, end, size); } uint32_t index(const Key& key) const { return MPHIndex::index(key); } - uint32_t perfect_hash(const Key& key) const { return MPHIndex::perfect_hash(key); } - uint32_t minimal_perfect_hash(const Key& key) const { return MPHIndex::minimal_perfect_hash(key); } }; +// The parameters minimal and square trade memory usage for evaluation speed. +// Minimal decreases speed and memory usage, and square does the opposite. +// Using minimal=true and square=false is the same as SimpleMPHIndex. +template +struct FlexibleMPHIndex {}; + +template +struct FlexibleMPHIndex + : public SimpleMPHIndex { + FlexibleMPHIndex() : SimpleMPHIndex(false) {} + uint32_t index(const Key& key) const { + return MPHIndex::minimal_perfect_hash(key); } + uint32_t size() const { return MPHIndex::minimal_perfect_hash_size(); } +}; +template +struct FlexibleMPHIndex + : public SimpleMPHIndex { + FlexibleMPHIndex() : SimpleMPHIndex(true) {} + uint32_t index(const Key& key) const { + return MPHIndex::perfect_square(key); } + uint32_t size() const { return MPHIndex::perfect_hash_size(); } +}; +template +struct FlexibleMPHIndex + : public SimpleMPHIndex { + FlexibleMPHIndex() : SimpleMPHIndex(false) {} + uint32_t index(const Key& key) const { + return MPHIndex::index(key); } + uint32_t size() const { return MPHIndex::perfect_hash_size(); } +}; +// From a trade-off perspective this case does not make much sense. +// template +// class FlexibleMPHIndex + } // namespace cxxmph #endif // __CXXMPH_MPH_INDEX_H__ diff --git a/cxxmph/mph_map.h b/cxxmph/mph_map.h index 6563232..3fc83d0 100644 --- a/cxxmph/mph_map.h +++ b/cxxmph/mph_map.h @@ -5,12 +5,12 @@ // // This class not necessarily faster than unordered_map (or ext/hash_map). // Benchmark your code before using it. If you do not call rehash() before -// starting your reads, it will be definitively slower than unordered_map. +// starting your reads, it will be very likely slower than unordered_map. // -// For large sets of urls, which are a somewhat expensive to compare, I found -// this class to be about 10% faster than unordered_map. +// For large sets of urls (>100k), which are a somewhat expensive to compare, I +// found this class to be about 10%-30% faster than unordered_map. // -// The space overhead of this map is 1.93 bits per bucket and it achieves 100% +// The space overhead of this map is 2.6 bits per bucket and it achieves 100% // occupation with a rehash call. #include @@ -30,17 +30,18 @@ namespace cxxmph { using std::pair; using std::make_pair; -using std::unordered_map; using std::vector; // Save on repetitive typing. -#define MPH_MAP_TMPL_SPEC template -#define MPH_MAP_CLASS_SPEC mph_map +#define MPH_MAP_TMPL_SPEC \ + template +#define MPH_MAP_CLASS_SPEC mph_map_base #define MPH_MAP_METHOD_DECL(r, m) MPH_MAP_TMPL_SPEC typename MPH_MAP_CLASS_SPEC::r MPH_MAP_CLASS_SPEC::m #define MPH_MAP_INLINE_METHOD_DECL(r, m) MPH_MAP_TMPL_SPEC inline typename MPH_MAP_CLASS_SPEC::r MPH_MAP_CLASS_SPEC::m -template , class EqualKey = std::equal_to, class Alloc = std::allocator > -class mph_map { +template , class EqualKey = std::equal_to, class Alloc = std::allocator > +class mph_map_base { public: typedef Key key_type; typedef Data data_type; @@ -63,8 +64,8 @@ class mph_map { typedef bool bool_type; typedef pair insert_return_type; - mph_map(); - ~mph_map(); + mph_map_base(); + ~mph_map_base(); iterator begin(); iterator end(); @@ -83,7 +84,7 @@ class mph_map { data_type& operator[](const key_type &k); const data_type& operator[](const key_type &k) const; - size_type bucket_count() const { return index_.minimal_perfect_hash_size() + slack_.bucket_count(); } + size_type bucket_count() const { return index_.size() + slack_.bucket_count(); } void rehash(size_type nbuckets /*ignored*/); protected: // mimicking STL implementation @@ -106,9 +107,9 @@ class mph_map { void pack(); vector values_; vector present_; - SimpleMPHIndex::hash_function> index_; + FlexibleMPHIndex::hash_function> index_; // TODO(davi) optimize slack to use hash from index rather than calculate its own - typedef unordered_map slack_type; + typedef std::unordered_map slack_type; slack_type slack_; size_type size_; typename seeded_hash::hash_function hasher128_; @@ -119,13 +120,11 @@ bool operator==(const MPH_MAP_CLASS_SPEC& lhs, const MPH_MAP_CLASS_SPEC& rhs) { return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin()); } -MPH_MAP_TMPL_SPEC MPH_MAP_CLASS_SPEC::mph_map() : size_(0) { +MPH_MAP_TMPL_SPEC MPH_MAP_CLASS_SPEC::mph_map_base() : size_(0) { clear(); pack(); } - -MPH_MAP_TMPL_SPEC MPH_MAP_CLASS_SPEC::~mph_map() { -} +MPH_MAP_TMPL_SPEC MPH_MAP_CLASS_SPEC::~mph_map_base() { } MPH_MAP_METHOD_DECL(insert_return_type, insert)(const value_type& x) { auto it = find(x.first); @@ -154,13 +153,13 @@ MPH_MAP_METHOD_DECL(void_type, pack)() { make_iterator_first(begin()), make_iterator_first(end()), size_); if (!success) { exit(-1); } - vector new_values(index_.minimal_perfect_hash_size()); + vector new_values(index_.size()); new_values.reserve(new_values.size() * 2); - vector new_present(index_.minimal_perfect_hash_size(), false); + vector new_present(index_.size(), false); new_present.reserve(new_present.size() * 2); for (iterator it = begin(), it_end = end(); it != it_end; ++it) { - size_type id = index_.minimal_perfect_hash(it->first); - assert(id < index_.minimal_perfect_hash_size()); + size_type id = index_.index(it->first); + assert(id < index_.size()); assert(id < new_values.size()); new_values[id] = *it; new_present[id] = true; @@ -216,10 +215,10 @@ MPH_MAP_INLINE_METHOD_DECL(my_int32_t, index)(const key_type& k) const { auto sit = slack_.find(hasher128_.hash128(k, 0)); if (sit != slack_.end()) return sit->second; } - if (__builtin_expect(index_.minimal_perfect_hash_size(), 1)) { - auto minimal_perfect_hash = index_.minimal_perfect_hash(k); - if (__builtin_expect(present_[minimal_perfect_hash], true)) { - return minimal_perfect_hash; + if (__builtin_expect(index_.size(), 1)) { + auto id = index_.index(k); + if (__builtin_expect(present_[id], true)) { + return id; } } return -1; @@ -235,6 +234,21 @@ MPH_MAP_METHOD_DECL(void_type, rehash)(size_type nbuckets) { slack_type().swap(slack_); } +#define MPH_MAP_PREAMBLE template , class EqualKey = std::equal_to,\ + class Alloc = std::allocator > + +MPH_MAP_PREAMBLE class mph_map : public mph_map_base< + false, false, Key, Data, HashFcn, EqualKey, Alloc> {}; +MPH_MAP_PREAMBLE class unordered_map : public mph_map_base< + false, false, Key, Data, HashFcn, EqualKey, Alloc> {}; +MPH_MAP_PREAMBLE class hash_map : public mph_map_base< + false, false, Key, Data, HashFcn, EqualKey, Alloc> {}; + +MPH_MAP_PREAMBLE class dense_hash_map : public mph_map_base< + false, true, Key, Data, HashFcn, EqualKey, Alloc> {}; +MPH_MAP_PREAMBLE class sparse_hash_map : public mph_map_base< + true, false, Key, Data, HashFcn, EqualKey, Alloc> {}; } // namespace cxxmph