motiejus/turbonss
1
Fork 0
Code Releases Activity

Tests pass, but it segfaults at the benchmark. Need further investigation, but the core for the cuckoo stuff is already there.

Browse Source
This commit is contained in:
Davi Reis 2012-03-14 04:51:55 -03:00
parent 86797b6402
commit a4d96e6cb2
7 changed files with 111 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cxxmph/Makefile.am
View File

@ -3,7 +3,7 @@ check_PROGRAMS = hollow_iterator_test mph_map_test mph_index_test trigraph_test
noinst_PROGRAMS = bm_index bm_map
bin_PROGRAMS = cxxmph
lib_LTLIBRARIES = libcxxmph.la
libcxxmph_la_SOURCES = MurmurHash3.h MurmurHash3.cpp trigragh.h trigraph.cc mph_index.h mph_index.cc seeded_hash.h stringpiece.h benchmark.h benchmark.cc
libcxxmph_la_SOURCES = MurmurHash3.h MurmurHash3.cpp trigragh.h trigraph.cc mph_index.h mph_index.cc seeded_hash.h stringpiece.h benchmark.h benchmark.cc mph_bits.h mph_bits.cc
libcxxmph_la_LDFLAGS = -version-info 0:0:0
cxxmph_includedir = $(includedir)/cxxmph/
cxxmph_include_HEADERS = mph_map.h mph_index.h MurmurHash3.h trigraph.h seeded_hash.h stringpiece.h hollow_iterator.h

2
cxxmph/bm_map.cc
View File

@ -89,10 +89,12 @@ using namespace cxxmph;
int main(int argc, char** argv) {
srandom(4);
/*
Benchmark::Register(new BM_CreateUrls<mph_map<StringPiece, StringPiece>>("URLS100k"));
Benchmark::Register(new BM_CreateUrls<unordered_map<StringPiece, StringPiece>>("URLS100k"));
Benchmark::Register(new BM_SearchUrls<mph_map<StringPiece, StringPiece>>("URLS100k", 10*1000 * 1000, 0));
Benchmark::Register(new BM_SearchUrls<unordered_map<StringPiece, StringPiece, Murmur3StringPiece>>("URLS100k", 10*1000 * 1000, 0));
*/
Benchmark::Register(new BM_SearchUrls<mph_map<StringPiece, StringPiece>>("URLS100k", 10*1000 * 1000, 0.9));
Benchmark::Register(new BM_SearchUrls<unordered_map<StringPiece, StringPiece, Murmur3StringPiece>>("URLS100k", 10*1000 * 1000, 0.9));
Benchmark::Register(new BM_SearchUint64<mph_map<uint64_t, uint64_t>>);

4
cxxmph/mph_bits.cc Normal file
View File

@ -0,0 +1,4 @@
#include "mph_bits.h"
namespace cxxmph {
}

18
cxxmph/mph_bits.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef __CXXMPH_MPH_BITS_H__
#define __CXXMPH_MPH_BITS_H__
#include <stdint.h> // for uint32_t and friends
namespace cxxmph {
static const uint8_t valuemask[] = { 0xfc, 0xf3, 0xcf, 0x3f};
static void set_2bit_value(uint8_t *d, uint32_t i, uint8_t v) {
d[(i >> 2)] &= ((v << ((i & 3) << 1)) | valuemask[i & 3]);
}
static uint32_t get_2bit_value(const uint8_t* d, uint32_t i) {
return (d[(i >> 2)] >> (((i & 3) << 1)) & 3);
}
} // namespace cxxmph
#endif

54
cxxmph/mph_index.h
View File

@ -36,6 +36,7 @@ using std::cerr;
using std::endl;
#include "seeded_hash.h"
#include "mph_bits.h"
#include "trigraph.h"
namespace cxxmph {
@ -43,7 +44,7 @@ namespace cxxmph {
class MPHIndex {
public:
MPHIndex(double c = 1.23, uint8_t b = 7) :
c_(c), b_(b), m_(0), n_(0), k_(0), r_(0),
c_(c), b_(b), m_(0), n_(0), k_(0), r_(1),
g_(NULL), g_size_(0), ranktable_(NULL), ranktable_size_(0),
deserialized_(false) { }
~MPHIndex();
@ -65,8 +66,12 @@ class MPHIndex {
uint32_t minimal_perfect_hash(const Key& x) const;
// Crazy functions. Ignore.
template <class SeededHashFcn> // must agree with Reset
uint32_t cuckoo_hash(const uint32_t* h, uint8_t nest) const;
template <class SeededHashFcn, class Key> // must agree with Reset
uint32_t cuckoo_hash(const Key& x, const uint32_t* h, uint8_t nest) const;
uint8_t cuckoo_nest(const Key& x, const uint32_t* h) const;
template <class SeededHashFcn, class Key> // must agree with Reset
uint32_t cuckoo_nest_index(const Key& x, uint32_t* h) const;
template <class SeededHashFcn, class Key> // must agree with Reset
void hash_vector(const Key& x, uint32_t* h) const;
@ -117,26 +122,8 @@ class MPHIndex {
bool deserialized_;
static const uint8_t valuemask[];
static void set_2bit_value(uint8_t *d, uint32_t i, uint8_t v) {
d[(i >> 2)] &= ((v << ((i & 3) << 1)) | valuemask[i & 3]);
}
static uint32_t get_2bit_value(const uint8_t* d, uint32_t i) {
return (d[(i >> 2)] >> (((i & 3) << 1)) & 3);
}
};
template <class T>
T nexthigher(T k) {
if (k == 0)
return 1;
k--;
for (int i=1; i<sizeof(T)*CHAR_BIT; i<<=1)
k = k | k >> i;
return k+1;
}
// Template method needs to go in the header file.
template <class SeededHashFcn, class ForwardIterator>
bool MPHIndex::Reset(
@ -153,7 +140,7 @@ bool MPHIndex::Reset(
nest_displacement_[2] = (r_ << 1);
// This can be used to speed mods, but increases occupation too much.
// Needs to try http://gmplib.org/manual/Integer-Exponentiation.html instead
// r_ = nexthigher(r_);
// r_ = nextpoweroftwo(r_);
n_ = 3*r_;
k_ = 1U << b_;
@ -200,30 +187,40 @@ bool MPHIndex::Mapping(
return false;
}
template <class SeededHashFcn, class Key>
uint32_t MPHIndex::cuckoo_hash(const Key& key, const uint32_t* h, uint8_t nest) const {
template <class SeededHashFcn>
uint32_t MPHIndex::cuckoo_hash(const uint32_t* h, uint8_t nest) const {
return (h[nest] % r_) + nest_displacement_[nest];
}
template <class SeededHashFcn, class Key>
void MPHIndex::hash_vector(const Key& key, uint32_t* h) const {
SeededHashFcn().hash64(key, hash_seed_[0], reinterpret_cast<uint32_t*>(&h));
SeededHashFcn().hash64(key, hash_seed_[0], h);
}
template <class SeededHashFcn, class Key>
uint8_t MPHIndex::cuckoo_nest(const Key& key, const uint32_t* h) const {
uint32_t x[4];
x[0] = (h[0] % r_) + nest_displacement_[0];
x[1] = (h[1] % r_) + nest_displacement_[1];
x[2] = (h[2] % r_) + nest_displacement_[2];
return (get_2bit_value(g_, x[0]) + get_2bit_value(g_, x[1]) + get_2bit_value(g_, x[2])) % 3;
}
template <class SeededHashFcn, class Key>
uint32_t MPHIndex::perfect_hash(const Key& key) const {
uint32_t h[4];
SeededHashFcn().hash64(key, hash_seed_[0], reinterpret_cast<uint32_t*>(&h));
SeededHashFcn().hash64(key, hash_seed_[0], h);
// for (int i = 0; i < 3; ++i) h[i] = SeededHashFcn()(key, hash_seed_[i]);
h[0] = (h[0] % r_) + nest_displacement_[0];
h[1] = (h[1] % r_) + nest_displacement_[1];
h[2] = (h[2] % r_) + nest_displacement_[2];
assert(g_size_);
if (!g_size_) return 0;
// cerr << "g_.size() " << g_size_ << " h0 >> 2 " << (h[0] >> 2) << endl;
assert((h[0] >> 2) <g_size_);
assert((h[1] >> 2) <g_size_);
assert((h[2] >> 2) <g_size_);
uint32_t vertex = h[(get_2bit_value(g_, h[0]) + get_2bit_value(g_, h[1]) + get_2bit_value(g_, h[2])) % 3];
uint8_t nest = (get_2bit_value(g_, h[0]) + get_2bit_value(g_, h[1]) + get_2bit_value(g_, h[2])) % 3;
uint32_t vertex = h[nest];
return vertex;
}
template <class SeededHashFcn, class Key>
@ -248,6 +245,9 @@ class SimpleMPHIndex : public MPHIndex {
uint32_t index(const Key& key) const { return MPHIndex::index<HashFcn>(key); }
uint32_t perfect_hash(const Key& key) const { return MPHIndex::perfect_hash<HashFcn>(key); }
uint32_t minimal_perfect_hash(const Key& key) const { return MPHIndex::minimal_perfect_hash<HashFcn>(key); }
uint8_t cuckoo_nest(const Key& key, const uint32_t* h) const { return MPHIndex::cuckoo_nest<HashFcn>(key, h); }
uint32_t cuckoo_hash(const uint32_t* h, uint8_t nest) const { return MPHIndex::cuckoo_hash<HashFcn>(h, nest); }
void hash_vector(const Key& key, uint32_t* h) const { MPHIndex::hash_vector<HashFcn>(key, h); }
};
} // namespace cxxmph

65
cxxmph/mph_map.h
View File

@ -9,8 +9,9 @@
//
// See http://www.strchr.com/crc32_popcnt and new Murmur3 function to try to beat stl
#include <iostream>
#include <algorithm>
#include <iostream>
#include <limits>
#include <unordered_map>
#include <vector>
#include <utility> // for std::pair
@ -100,17 +101,19 @@ class mph_map {
return hollow_const_iterator<std::vector<value_type>>(&values_, &present_, it);
}
static void set_2bit_value(uint8_t *d, uint32_t i, uint8_t v) {
d[(i >> 2)] &= ((v << ((i & 3) << 1)) | valuemask[i & 3]);
}
static uint32_t get_2bit_value(const uint8_t* d, uint32_t i) {
return (d[(i >> 2)] >> (((i & 3) << 1)) & 3);
iterator slow_find(const key_type& k);
const_iterator slow_find(const key_type& k) const;
static const uint8_t kNestCollision = 3; // biggest 2 bit value
uint32_t nest_index(const key_type& k, uint32_t* h) const {
index_.hash_vector(k, h);
// Use a pivot to prevent branch in the fast path
return h[3] % (index_.perfect_hash_size() + 1);
}
void pack();
std::vector<value_type> values_;
std::vector<bool> present_;
const uint8_t* nests_;
std::vector<uint8_t> nests_;
SimpleMPHIndex<Key, typename seeded_hash<HashFcn>::hash_function> index_;
// TODO(davi) optimize slack to no hold a copy of the key
typedef unordered_map<Key, uint32_t, HashFcn, EqualKey, Alloc> slack_type;
@ -124,6 +127,7 @@ bool operator==(const MPH_MAP_CLASS_SPEC& lhs, const MPH_MAP_CLASS_SPEC& rhs) {
}
MPH_MAP_TMPL_SPEC MPH_MAP_CLASS_SPEC::mph_map() : size_(0) {
clear();
pack();
}
@ -140,6 +144,10 @@ MPH_MAP_METHOD_DECL(insert_return_type, insert)(const value_type& x) {
}
values_.push_back(x);
present_.push_back(true);
nests_.resize(ceil(values_.size() / 2.0), std::numeric_limits<uint8_t>::max());
uint32_t h[4];
auto index = nest_index(x.first, h);
set_2bit_value(&(nests_[0]), index, kNestCollision);
++size_;
slack_.insert(make_pair(x.first, values_.size() - 1));
if (should_pack) pack();
@ -157,14 +165,28 @@ MPH_MAP_METHOD_DECL(void_type, pack)() {
new_values.reserve(new_values.size() * 2);
std::vector<bool> new_present(index_.perfect_hash_size(), false);
new_present.reserve(new_present.size() * 2);
std::vector<uint8_t> new_nests(ceil(index_.perfect_hash_size() / 2.0), std::numeric_limits<uint8_t>::max());
new_nests.reserve(new_nests.size() * 2);
vector<bool> used_nests(new_nests.size() * 2);
for (iterator it = begin(), it_end = end(); it != it_end; ++it) {
size_type id = index_.perfect_hash(it->first);
assert(id < new_values.size());
new_values[id] = *it;
new_present[id] = true;
uint32_t h[4];
uint32_t index = nest_index(it->first, h);
if (used_nests[index]) {
set_2bit_value(&(new_nests[0]), index, kNestCollision);
}
else {
set_2bit_value(&(new_nests[0]), index, index_.cuckoo_nest(it->first, h));
assert(index_.perfect_hash(it->first) == index_.cuckoo_hash(h, index_.cuckoo_nest(it->first, h)));
used_nests[index] = true;
}
}
values_.swap(new_values);
present_.swap(new_present);
nests_.swap(new_nests);
slack_type().swap(slack_);
}
@ -180,11 +202,15 @@ MPH_MAP_METHOD_DECL(void_type, clear)() {
present_.clear();
slack_.clear();
index_.clear();
nests_.clear();
nests_.push_back(std::numeric_limits<uint8_t>::max());
size_ = 0;
}
MPH_MAP_METHOD_DECL(void_type, erase)(iterator pos) {
present_[pos - begin] = false;
uint32_t h[4];
nests_[nest_index(pos->first, h)] = kNestCollision;
*pos = value_type();
--size_;
}
@ -196,7 +222,7 @@ MPH_MAP_METHOD_DECL(void_type, erase)(const key_type& k) {
MPH_MAP_METHOD_DECL(const_iterator, find)(const key_type& k) const {
uint32_t h[4];
auto nest = nests_[index_.hash_vector(k, reinterpret_cast<uint32_t*>(&h))];
auto nest = get_2bit_value(&(nests_[0]), nest_index(k, h));
if (nest != kNestCollision) {
auto vit = values_.begin() + h[nest];
if (equal_(k, vit->first)) return make_iterator(vit);
@ -205,37 +231,44 @@ MPH_MAP_METHOD_DECL(const_iterator, find)(const key_type& k) const {
}
MPH_MAP_METHOD_DECL(const_iterator, slow_find)(const key_type& k) const {
if (index_.perfect_hash_size()) {
auto id = index_.perfect_hash(k);
if (!present_[id]) return end();
if (present_[id]) {
auto vit = values_.begin() + id;
if (equal_(k, vit->first)) return make_iterator(vit);
}
}
if (__builtin_expect(!slack_.empty(), 0)) {
auto sit = slack_.find(k);
if (it != slack_.end()) return make_iterator(values_.begin() + sit->second);
if (sit != slack_.end()) return make_iterator(values_.begin() + sit->second);
}
return end();
}
MPH_MAP_METHOD_DECL(iterator, find)(const key_type& k) {
uint32_t h[4];
auto nest = nests_[index_.hash_vector(k, reinterpret_cast<uint32_t*>(&h))];
auto index = nest_index(k, h);
assert(nests_.size());
assert(nests_.size() > index / 2);
auto nest = get_2bit_value(&(nests_[0]), index);
if (nest != kNestCollision) {
auto vit = values_.begin() + h[nest];
auto vit = values_.begin() + index_.cuckoo_hash(h, nest);
if (equal_(k, vit->first)) return make_iterator(vit);
}
return slow_find(k);
}
MPH_MAP_METHOD_DECL(iterator, slow_find)(const key_type& k) {
if (index_.perfect_hash_size()) {
auto id = index_.perfect_hash(k);
if (!present_[id]) return end();
if (present_[id]) {
auto vit = values_.begin() + id;
if (equal_(k, vit->first)) return make_iterator(vit);
}
}
if (__builtin_expect(!slack_.empty(), 0)) {
auto sit = slack_.find(k);
if (it != slack_.end()) return make_iterator(values_.begin() + sit->second);
if (sit != slack_.end()) return make_iterator(values_.begin() + sit->second);
}
return end();
}

4
cxxmph/mph_map_test.cc
View File

@ -17,6 +17,10 @@ int main(int argc, char** argv) {
}
for (int i = 0; i < num_keys; ++i) {
auto it = b.find(i);
if (it == b.end()) {
std::cerr << "Failed to find " << i << std::endl;
exit(-1);
}
if (it->first != it->second || it->first != i) {
std::cerr << "Found " << it->first << " looking for " << i << std::endl;
exit(-1);