motiejus
/
turbonss
1
Fork 0
Code Releases Activity

Tests pass, but it segfaults at the benchmark. Need further investigation, but the core for the cuckoo stuff is already there.

This commit is contained in:
Davi Reis 2012-03-14 04:51:55 -03:00
parent 86797b6402
commit a4d96e6cb2
7 changed files with 111 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cxxmph/Makefile.am
View File

@ -3,7 +3,7 @@ check_PROGRAMS = hollow_iterator_test mph_map_test mph_index_test trigraph_test
noinst_PROGRAMS = bm_index bm_map
bin_PROGRAMS = cxxmph
lib_LTLIBRARIES = libcxxmph.la
libcxxmph_la_SOURCES = MurmurHash3.h MurmurHash3.cpp trigragh.h trigraph.cc mph_index.h mph_index.cc seeded_hash.h stringpiece.h benchmark.h benchmark.cc
libcxxmph_la_SOURCES = MurmurHash3.h MurmurHash3.cpp trigragh.h trigraph.cc mph_index.h mph_index.cc seeded_hash.h stringpiece.h benchmark.h benchmark.cc mph_bits.h mph_bits.cc
libcxxmph_la_LDFLAGS = -version-info 0:0:0
cxxmph_includedir = $(includedir)/cxxmph/
cxxmph_include_HEADERS = mph_map.h mph_index.h MurmurHash3.h trigraph.h seeded_hash.h stringpiece.h hollow_iterator.h

2
cxxmph/bm_map.cc
View File

@ -89,10 +89,12 @@ using namespace cxxmph;
int main(int argc, char** argv) {
srandom(4);
/*
Benchmark::Register(new BM_CreateUrls<mph_map<StringPiece, StringPiece>>("URLS100k"));
Benchmark::Register(new BM_CreateUrls<unordered_map<StringPiece, StringPiece>>("URLS100k"));
Benchmark::Register(new BM_SearchUrls<mph_map<StringPiece, StringPiece>>("URLS100k", 10*1000 * 1000, 0));
Benchmark::Register(new BM_SearchUrls<unordered_map<StringPiece, StringPiece, Murmur3StringPiece>>("URLS100k", 10*1000 * 1000, 0));
*/
Benchmark::Register(new BM_SearchUrls<mph_map<StringPiece, StringPiece>>("URLS100k", 10*1000 * 1000, 0.9));
Benchmark::Register(new BM_SearchUrls<unordered_map<StringPiece, StringPiece, Murmur3StringPiece>>("URLS100k", 10*1000 * 1000, 0.9));
Benchmark::Register(new BM_SearchUint64<mph_map<uint64_t, uint64_t>>);

4
cxxmph/mph_bits.cc Normal file
View File

@ -0,0 +1,4 @@
#include "mph_bits.h"
namespace cxxmph {
}

18
cxxmph/mph_bits.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef __CXXMPH_MPH_BITS_H__
#define __CXXMPH_MPH_BITS_H__
#include <stdint.h> // for uint32_t and friends
namespace cxxmph {
static const uint8_t valuemask[] = { 0xfc, 0xf3, 0xcf, 0x3f};
static void set_2bit_value(uint8_t *d, uint32_t i, uint8_t v) {
d[(i >> 2)] &= ((v << ((i & 3) << 1)) | valuemask[i & 3]);
}
static uint32_t get_2bit_value(const uint8_t* d, uint32_t i) {
return (d[(i >> 2)] >> (((i & 3) << 1)) & 3);
}
} // namespace cxxmph
#endif

54
cxxmph/mph_index.h
View File

@ -36,6 +36,7 @@ using std::cerr;
using std::endl;
#include "seeded_hash.h"
#include "mph_bits.h"
#include "trigraph.h"
namespace cxxmph {
@ -43,7 +44,7 @@ namespace cxxmph {
class MPHIndex {
public:
MPHIndex(double c = 1.23, uint8_t b = 7) :
c_(c), b_(b), m_(0), n_(0), k_(0), r_(0),
c_(c), b_(b), m_(0), n_(0), k_(0), r_(1),
g_(NULL), g_size_(0), ranktable_(NULL), ranktable_size_(0),
deserialized_(false) { }
~MPHIndex();
@ -65,8 +66,12 @@ class MPHIndex {
uint32_t minimal_perfect_hash(const Key& x) const;
// Crazy functions. Ignore.
template <class SeededHashFcn> // must agree with Reset
uint32_t cuckoo_hash(const uint32_t* h, uint8_t nest) const;
template <class SeededHashFcn, class Key> // must agree with Reset
uint32_t cuckoo_hash(const Key& x, const uint32_t* h, uint8_t nest) const;
uint8_t cuckoo_nest(const Key& x, const uint32_t* h) const;
template <class SeededHashFcn, class Key> // must agree with Reset
uint32_t cuckoo_nest_index(const Key& x, uint32_t* h) const;
template <class SeededHashFcn, class Key> // must agree with Reset
void hash_vector(const Key& x, uint32_t* h) const;
@ -117,26 +122,8 @@ class MPHIndex {
bool deserialized_;
static const uint8_t valuemask[];
static void set_2bit_value(uint8_t *d, uint32_t i, uint8_t v) {
d[(i >> 2)] &= ((v << ((i & 3) << 1)) | valuemask[i & 3]);
}
static uint32_t get_2bit_value(const uint8_t* d, uint32_t i) {
return (d[(i >> 2)] >> (((i & 3) << 1)) & 3);
}
};
template <class T>
T nexthigher(T k) {
if (k == 0)
return 1;
k--;
for (int i=1; i<sizeof(T)*CHAR_BIT; i<<=1)
k = k | k >> i;
return k+1;
}
// Template method needs to go in the header file.
template <class SeededHashFcn, class ForwardIterator>
bool MPHIndex::Reset(
@ -153,7 +140,7 @@ bool MPHIndex::Reset(
nest_displacement_[2] = (r_ << 1);
// This can be used to speed mods, but increases occupation too much.
// Needs to try http://gmplib.org/manual/Integer-Exponentiation.html instead
// r_ = nexthigher(r_);
// r_ = nextpoweroftwo(r_);
n_ = 3*r_;
k_ = 1U << b_;
@ -200,30 +187,40 @@ bool MPHIndex::Mapping(
return false;
}
template <class SeededHashFcn, class Key>
uint32_t MPHIndex::cuckoo_hash(const Key& key, const uint32_t* h, uint8_t nest) const {
template <class SeededHashFcn>
uint32_t MPHIndex::cuckoo_hash(const uint32_t* h, uint8_t nest) const {
return (h[nest] % r_) + nest_displacement_[nest];
}
template <class SeededHashFcn, class Key>
void MPHIndex::hash_vector(const Key& key, uint32_t* h) const {
SeededHashFcn().hash64(key, hash_seed_[0], reinterpret_cast<uint32_t*>(&h));
SeededHashFcn().hash64(key, hash_seed_[0], h);
}
template <class SeededHashFcn, class Key>
uint8_t MPHIndex::cuckoo_nest(const Key& key, const uint32_t* h) const {
uint32_t x[4];
x[0] = (h[0] % r_) + nest_displacement_[0];
x[1] = (h[1] % r_) + nest_displacement_[1];
x[2] = (h[2] % r_) + nest_displacement_[2];
return (get_2bit_value(g_, x[0]) + get_2bit_value(g_, x[1]) + get_2bit_value(g_, x[2])) % 3;
}
template <class SeededHashFcn, class Key>
uint32_t MPHIndex::perfect_hash(const Key& key) const {
uint32_t h[4];
SeededHashFcn().hash64(key, hash_seed_[0], reinterpret_cast<uint32_t*>(&h));
SeededHashFcn().hash64(key, hash_seed_[0], h);
// for (int i = 0; i < 3; ++i) h[i] = SeededHashFcn()(key, hash_seed_[i]);
h[0] = (h[0] % r_) + nest_displacement_[0];
h[1] = (h[1] % r_) + nest_displacement_[1];
h[2] = (h[2] % r_) + nest_displacement_[2];
assert(g_size_);
if (!g_size_) return 0;
// cerr << "g_.size() " << g_size_ << " h0 >> 2 " << (h[0] >> 2) << endl;
assert((h[0] >> 2) <g_size_);
assert((h[1] >> 2) <g_size_);
assert((h[2] >> 2) <g_size_);
uint32_t vertex = h[(get_2bit_value(g_, h[0]) + get_2bit_value(g_, h[1]) + get_2bit_value(g_, h[2])) % 3];
uint8_t nest = (get_2bit_value(g_, h[0]) + get_2bit_value(g_, h[1]) + get_2bit_value(g_, h[2])) % 3;
uint32_t vertex = h[nest];
return vertex;
}
template <class SeededHashFcn, class Key>
@ -248,6 +245,9 @@ class SimpleMPHIndex : public MPHIndex {
uint32_t index(const Key& key) const { return MPHIndex::index<HashFcn>(key); }
uint32_t perfect_hash(const Key& key) const { return MPHIndex::perfect_hash<HashFcn>(key); }
uint32_t minimal_perfect_hash(const Key& key) const { return MPHIndex::minimal_perfect_hash<HashFcn>(key); }
uint8_t cuckoo_nest(const Key& key, const uint32_t* h) const { return MPHIndex::cuckoo_nest<HashFcn>(key, h); }
uint32_t cuckoo_hash(const uint32_t* h, uint8_t nest) const { return MPHIndex::cuckoo_hash<HashFcn>(h, nest); }
void hash_vector(const Key& key, uint32_t* h) const { MPHIndex::hash_vector<HashFcn>(key, h); }
};
} // namespace cxxmph

65
cxxmph/mph_map.h
View File

@ -9,8 +9,9 @@
//
// See http://www.strchr.com/crc32_popcnt and new Murmur3 function to try to beat stl
#include <iostream>
#include <algorithm>
#include <iostream>
#include <limits>
#include <unordered_map>
#include <vector>
#include <utility> // for std::pair
@ -100,17 +101,19 @@ class mph_map {
return hollow_const_iterator<std::vector<value_type>>(&values_, &present_, it);
}
static void set_2bit_value(uint8_t *d, uint32_t i, uint8_t v) {
d[(i >> 2)] &= ((v << ((i & 3) << 1)) | valuemask[i & 3]);
}
static uint32_t get_2bit_value(const uint8_t* d, uint32_t i) {
return (d[(i >> 2)] >> (((i & 3) << 1)) & 3);
iterator slow_find(const key_type& k);
const_iterator slow_find(const key_type& k) const;
static const uint8_t kNestCollision = 3; // biggest 2 bit value
uint32_t nest_index(const key_type& k, uint32_t* h) const {
index_.hash_vector(k, h);
// Use a pivot to prevent branch in the fast path
return h[3] % (index_.perfect_hash_size() + 1);
}
void pack();
std::vector<value_type> values_;
std::vector<bool> present_;
const uint8_t* nests_;
std::vector<uint8_t> nests_;
SimpleMPHIndex<Key, typename seeded_hash<HashFcn>::hash_function> index_;
// TODO(davi) optimize slack to no hold a copy of the key
typedef unordered_map<Key, uint32_t, HashFcn, EqualKey, Alloc> slack_type;
@ -124,6 +127,7 @@ bool operator==(const MPH_MAP_CLASS_SPEC& lhs, const MPH_MAP_CLASS_SPEC& rhs) {
}
MPH_MAP_TMPL_SPEC MPH_MAP_CLASS_SPEC::mph_map() : size_(0) {
clear();
pack();
}
@ -140,6 +144,10 @@ MPH_MAP_METHOD_DECL(insert_return_type, insert)(const value_type& x) {
}
values_.push_back(x);
present_.push_back(true);
nests_.resize(ceil(values_.size() / 2.0), std::numeric_limits<uint8_t>::max());
uint32_t h[4];
auto index = nest_index(x.first, h);
set_2bit_value(&(nests_[0]), index, kNestCollision);
++size_;
slack_.insert(make_pair(x.first, values_.size() - 1));
if (should_pack) pack();
@ -157,14 +165,28 @@ MPH_MAP_METHOD_DECL(void_type, pack)() {
new_values.reserve(new_values.size() * 2);
std::vector<bool> new_present(index_.perfect_hash_size(), false);
new_present.reserve(new_present.size() * 2);
std::vector<uint8_t> new_nests(ceil(index_.perfect_hash_size() / 2.0), std::numeric_limits<uint8_t>::max());
new_nests.reserve(new_nests.size() * 2);
vector<bool> used_nests(new_nests.size() * 2);
for (iterator it = begin(), it_end = end(); it != it_end; ++it) {
size_type id = index_.perfect_hash(it->first);
assert(id < new_values.size());
new_values[id] = *it;
new_present[id] = true;
uint32_t h[4];
uint32_t index = nest_index(it->first, h);
if (used_nests[index]) {
set_2bit_value(&(new_nests[0]), index, kNestCollision);
}
else {
set_2bit_value(&(new_nests[0]), index, index_.cuckoo_nest(it->first, h));
assert(index_.perfect_hash(it->first) == index_.cuckoo_hash(h, index_.cuckoo_nest(it->first, h)));
used_nests[index] = true;
}
}
values_.swap(new_values);
present_.swap(new_present);
nests_.swap(new_nests);
slack_type().swap(slack_);
}
@ -180,11 +202,15 @@ MPH_MAP_METHOD_DECL(void_type, clear)() {
present_.clear();
slack_.clear();
index_.clear();
nests_.clear();
nests_.push_back(std::numeric_limits<uint8_t>::max());
size_ = 0;
}
MPH_MAP_METHOD_DECL(void_type, erase)(iterator pos) {
present_[pos - begin] = false;
uint32_t h[4];
nests_[nest_index(pos->first, h)] = kNestCollision;
*pos = value_type();
--size_;
}
@ -196,7 +222,7 @@ MPH_MAP_METHOD_DECL(void_type, erase)(const key_type& k) {
MPH_MAP_METHOD_DECL(const_iterator, find)(const key_type& k) const {
uint32_t h[4];
auto nest = nests_[index_.hash_vector(k, reinterpret_cast<uint32_t*>(&h))];
auto nest = get_2bit_value(&(nests_[0]), nest_index(k, h));
if (nest != kNestCollision) {
auto vit = values_.begin() + h[nest];
if (equal_(k, vit->first)) return make_iterator(vit);
@ -205,37 +231,44 @@ MPH_MAP_METHOD_DECL(const_iterator, find)(const key_type& k) const {
}
MPH_MAP_METHOD_DECL(const_iterator, slow_find)(const key_type& k) const {
if (index_.perfect_hash_size()) {
auto id = index_.perfect_hash(k);
if (!present_[id]) return end();
if (present_[id]) {
auto vit = values_.begin() + id;
if (equal_(k, vit->first)) return make_iterator(vit);
}
}
if (__builtin_expect(!slack_.empty(), 0)) {
auto sit = slack_.find(k);
if (it != slack_.end()) return make_iterator(values_.begin() + sit->second);
if (sit != slack_.end()) return make_iterator(values_.begin() + sit->second);
}
return end();
}
MPH_MAP_METHOD_DECL(iterator, find)(const key_type& k) {
uint32_t h[4];
auto nest = nests_[index_.hash_vector(k, reinterpret_cast<uint32_t*>(&h))];
auto index = nest_index(k, h);
assert(nests_.size());
assert(nests_.size() > index / 2);
auto nest = get_2bit_value(&(nests_[0]), index);
if (nest != kNestCollision) {
auto vit = values_.begin() + h[nest];
auto vit = values_.begin() + index_.cuckoo_hash(h, nest);
if (equal_(k, vit->first)) return make_iterator(vit);
}
return slow_find(k);
}
MPH_MAP_METHOD_DECL(iterator, slow_find)(const key_type& k) {
if (index_.perfect_hash_size()) {
auto id = index_.perfect_hash(k);
if (!present_[id]) return end();
if (present_[id]) {
auto vit = values_.begin() + id;
if (equal_(k, vit->first)) return make_iterator(vit);
}
}
if (__builtin_expect(!slack_.empty(), 0)) {
auto sit = slack_.find(k);
if (it != slack_.end()) return make_iterator(values_.begin() + sit->second);
if (sit != slack_.end()) return make_iterator(values_.begin() + sit->second);
}
return end();
}

4
cxxmph/mph_map_test.cc
View File

@ -17,6 +17,10 @@ int main(int argc, char** argv) {
}
for (int i = 0; i < num_keys; ++i) {
auto it = b.find(i);
if (it == b.end()) {
std::cerr << "Failed to find " << i << std::endl;
exit(-1);
}
if (it->first != it->second || it->first != i) {
std::cerr << "Found " << it->first << " looking for " << i << std::endl;
exit(-1);