motiejus
/
turbonss
1
Fork 0
Code Releases Activity

BDZ_PH added

This commit is contained in:
fc_botelho 2008-03-25 20:20:55 +00:00
parent 825d8abca6
commit 963fede7f4
9 changed files with 677 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.t2t
View File

@ -44,7 +44,7 @@ The CMPH Library encapsulates the newest and more efficient algorithms in an eas
%html% - [BDZ Algorithm bdz.html]. %html% - [BDZ Algorithm bdz.html].
%txt% - BDZ Algorithm. %txt% - BDZ Algorithm.
The fastest algorithm to build MPHFs. It is based on random 3-graphs. A 3-graph is a The fastest algorithm to build PHFs and MPHFs. It is based on random 3-graphs. A 3-graph is a
generalization of a graph where each edge connects 3 vertices instead of only 2. The generalization of a graph where each edge connects 3 vertices instead of only 2. The
resulting functions are not order preserving and can be stored in only //(2 + x)cn// resulting functions are not order preserving and can be stored in only //(2 + x)cn//
bits, where //c// should be larger than or equal to //1.23// and //x// is a constant bits, where //c// should be larger than or equal to //1.23// and //x// is a constant
@ -81,6 +81,7 @@ The CMPH Library encapsulates the newest and more efficient algorithms in an eas
==News for version 0.8 (Coming soon)== ==News for version 0.8 (Coming soon)==
- [An algorithm to generate MPHFs that require around 2.6 bits per key to be stored bdz.html], which is referred to as BDZ algorithm. The algorithm is the fastest one available in the literature for sets that can be treated in internal memory. - [An algorithm to generate MPHFs that require around 2.6 bits per key to be stored bdz.html], which is referred to as BDZ algorithm. The algorithm is the fastest one available in the literature for sets that can be treated in internal memory.
- [An algorithm to generate PHFs with range m = cn, for c > 1.22 bdz.html], which is referred to as BDZ_PH algorithm. It is actually the BDZ algorithm without the ranking step. The resulting functions can be stored in 1.95 bits per key for //c = 1.23// and are considerably faster than the MPHFs generated by the BDZ algorithm.
- The hash functions djb2, fnv and sdbm were removed because they do not use random seeds and therefore are not useful for MPHFs algorithms. - The hash functions djb2, fnv and sdbm were removed because they do not use random seeds and therefore are not useful for MPHFs algorithms.
- All reported bugs and suggestions have been corrected and included as well. - All reported bugs and suggestions have been corrected and included as well.
@ -204,6 +205,7 @@ Minimum perfect hashing tool
* brz * brz
* fch * fch
* bdz * bdz
* bdz_ph
-f hash function (may be used multiple times) - valid values are -f hash function (may be used multiple times) - valid values are
* jenkins * jenkins
-V print version number and exit -V print version number and exit

1
src/Makefile.am
View File

@ -15,6 +15,7 @@ libcmph_la_SOURCES = debug.h\
bmz.h bmz_structs.h bmz.c\ bmz.h bmz_structs.h bmz.c\
bmz8.h bmz8_structs.h bmz8.c\ bmz8.h bmz8_structs.h bmz8.c\
bdz.h bdz_structs.h bdz.c\ bdz.h bdz_structs.h bdz.c\
bdz_ph.h bdz_structs_ph.h bdz_ph.c\
buffer_manager.h buffer_manager.c\ buffer_manager.h buffer_manager.c\
buffer_entry.h buffer_entry.c\ buffer_entry.h buffer_entry.c\
brz.h brz_structs.h brz.c\ brz.h brz_structs.h brz.c\

12
src/bdz.c
View File

@ -412,26 +412,26 @@ static void assigning(bdz_config_data_t *bdz, bdz_graph3_t* graph3, bdz_queue_t
if(!GETBIT(marked_vertices, v0)){ if(!GETBIT(marked_vertices, v0)){
if(!GETBIT(marked_vertices,v1)) if(!GETBIT(marked_vertices,v1))
{ {
SETVALUE(bdz->g, v1, UNASSIGNED); SETVALUE1(bdz->g, v1, UNASSIGNED);
SETBIT(marked_vertices, v1); SETBIT(marked_vertices, v1);
} }
if(!GETBIT(marked_vertices,v2)) if(!GETBIT(marked_vertices,v2))
{ {
SETVALUE(bdz->g, v2, UNASSIGNED); SETVALUE1(bdz->g, v2, UNASSIGNED);
SETBIT(marked_vertices, v2); SETBIT(marked_vertices, v2);
} }
SETVALUE(bdz->g, v0, (6-(GETVALUE(bdz->g, v1) + GETVALUE(bdz->g,v2)))%3); SETVALUE1(bdz->g, v0, (6-(GETVALUE(bdz->g, v1) + GETVALUE(bdz->g,v2)))%3);
SETBIT(marked_vertices, v0); SETBIT(marked_vertices, v0);
} else if(!GETBIT(marked_vertices, v1)) { } else if(!GETBIT(marked_vertices, v1)) {
if(!GETBIT(marked_vertices, v2)) if(!GETBIT(marked_vertices, v2))
{ {
SETVALUE(bdz->g, v2, UNASSIGNED); SETVALUE1(bdz->g, v2, UNASSIGNED);
SETBIT(marked_vertices, v2); SETBIT(marked_vertices, v2);
} }
SETVALUE(bdz->g, v1, (7-(GETVALUE(bdz->g, v0)+GETVALUE(bdz->g, v2)))%3); SETVALUE1(bdz->g, v1, (7-(GETVALUE(bdz->g, v0)+GETVALUE(bdz->g, v2)))%3);
SETBIT(marked_vertices, v1); SETBIT(marked_vertices, v1);
}else { }else {
SETVALUE(bdz->g, v2, (8-(GETVALUE(bdz->g,v0)+GETVALUE(bdz->g, v1)))%3); SETVALUE1(bdz->g, v2, (8-(GETVALUE(bdz->g,v0)+GETVALUE(bdz->g, v1)))%3);
SETBIT(marked_vertices, v2); SETBIT(marked_vertices, v2);
} }
DEBUGP("A:%u %u %u -- %u %u %u\n", v0, v1, v2, GETVALUE(bdz->g, v0), GETVALUE(bdz->g, v1), GETVALUE(bdz->g, v2)); DEBUGP("A:%u %u %u -- %u %u %u\n", v0, v1, v2, GETVALUE(bdz->g, v0), GETVALUE(bdz->g, v1), GETVALUE(bdz->g, v2));

522
src/bdz_ph.c Executable file
View File

@ -0,0 +1,522 @@
#include "bdz_ph.h"
#include "cmph_structs.h"
#include "bdz_structs_ph.h"
#include "hash.h"
#include "bitbool.h"
#include <math.h>
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <string.h>
//#define DEBUG
#include "debug.h"
#define UNASSIGNED 3
#define NULL_EDGE 0xffffffff
static int pow3_table[5] = {1,3,9,27,81};
static int lookup_table[5][256] = {
{0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0},
{0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1},
{0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1},
{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
};
typedef struct
{
cmph_uint32 vertices[3];
cmph_uint32 next_edges[3];
}bdz_ph_edge_t;
typedef cmph_uint32 * bdz_ph_queue_t;
static void bdz_ph_alloc_queue(bdz_ph_queue_t * queuep, cmph_uint32 nedges)
{
(*queuep)=malloc(nedges*sizeof(cmph_uint32));
};
static void bdz_ph_free_queue(bdz_ph_queue_t * queue)
{
free(*queue);
};
typedef struct
{
cmph_uint32 nedges;
bdz_ph_edge_t * edges;
cmph_uint32 * first_edge;
cmph_uint8 * vert_degree;
}bdz_ph_graph3_t;
static void bdz_ph_alloc_graph3(bdz_ph_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices)
{
graph3->edges=malloc(nedges*sizeof(bdz_ph_edge_t));
graph3->first_edge=malloc(nvertices*sizeof(cmph_uint32));
graph3->vert_degree=malloc(nvertices);
};
static void bdz_ph_init_graph3(bdz_ph_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices)
{
memset(graph3->first_edge,0xff,nvertices*sizeof(cmph_uint32));
memset(graph3->vert_degree,0,nvertices);
graph3->nedges=0;
};
static void bdz_ph_free_graph3(bdz_ph_graph3_t *graph3)
{
free(graph3->edges);
free(graph3->first_edge);
free(graph3->vert_degree);
};
static void bdz_ph_partial_free_graph3(bdz_ph_graph3_t *graph3)
{
free(graph3->first_edge);
free(graph3->vert_degree);
graph3->first_edge = NULL;
graph3->vert_degree = NULL;
};
static void bdz_ph_add_edge(bdz_ph_graph3_t * graph3, cmph_uint32 v0, cmph_uint32 v1, cmph_uint32 v2)
{
graph3->edges[graph3->nedges].vertices[0]=v0;
graph3->edges[graph3->nedges].vertices[1]=v1;
graph3->edges[graph3->nedges].vertices[2]=v2;
graph3->edges[graph3->nedges].next_edges[0]=graph3->first_edge[v0];
graph3->edges[graph3->nedges].next_edges[1]=graph3->first_edge[v1];
graph3->edges[graph3->nedges].next_edges[2]=graph3->first_edge[v2];
graph3->first_edge[v0]=graph3->first_edge[v1]=graph3->first_edge[v2]=graph3->nedges;
graph3->vert_degree[v0]++;
graph3->vert_degree[v1]++;
graph3->vert_degree[v2]++;
graph3->nedges++;
};
static void bdz_ph_dump_graph(bdz_ph_graph3_t* graph3, cmph_uint32 nedges, cmph_uint32 nvertices)
{
int i;
for(i=0;i<nedges;i++){
printf("\nedge %d %d %d %d ",i,graph3->edges[i].vertices[0],
graph3->edges[i].vertices[1],graph3->edges[i].vertices[2]);
printf(" nexts %d %d %d",graph3->edges[i].next_edges[0],
graph3->edges[i].next_edges[1],graph3->edges[i].next_edges[2]);
};
for(i=0;i<nvertices;i++){
printf("\nfirst for vertice %d %d ",i,graph3->first_edge[i]);
};
};
static void bdz_ph_remove_edge(bdz_ph_graph3_t * graph3, cmph_uint32 curr_edge)
{
cmph_uint32 i,j=0,vert,edge1,edge2;
for(i=0;i<3;i++){
vert=graph3->edges[curr_edge].vertices[i];
edge1=graph3->first_edge[vert];
edge2=NULL_EDGE;
while(edge1!=curr_edge&&edge1!=NULL_EDGE){
edge2=edge1;
if(graph3->edges[edge1].vertices[0]==vert){
j=0;
} else if(graph3->edges[edge1].vertices[1]==vert){
j=1;
} else
j=2;
edge1=graph3->edges[edge1].next_edges[j];
};
if(edge1==NULL_EDGE){
printf("\nerror remove edge %d dump graph",curr_edge);
bdz_ph_dump_graph(graph3,graph3->nedges,graph3->nedges+graph3->nedges/4);
exit(-1);
};
if(edge2!=NULL_EDGE){
graph3->edges[edge2].next_edges[j] =
graph3->edges[edge1].next_edges[i];
} else
graph3->first_edge[vert]=
graph3->edges[edge1].next_edges[i];
graph3->vert_degree[vert]--;
};
};
static int bdz_ph_generate_queue(cmph_uint32 nedges, cmph_uint32 nvertices, bdz_ph_queue_t queue, bdz_ph_graph3_t* graph3)
{
cmph_uint32 i,v0,v1,v2;
cmph_uint32 queue_head=0,queue_tail=0;
cmph_uint32 curr_edge;
cmph_uint32 tmp_edge;
cmph_uint8 * marked_edge =malloc((nedges >> 3) + 1);
memset(marked_edge, 0, (nedges >> 3) + 1);
for(i=0;i<nedges;i++){
v0=graph3->edges[i].vertices[0];
v1=graph3->edges[i].vertices[1];
v2=graph3->edges[i].vertices[2];
if(graph3->vert_degree[v0]==1 ||
graph3->vert_degree[v1]==1 ||
graph3->vert_degree[v2]==1){
if(!GETBIT(marked_edge,i)) {
queue[queue_head++]=i;
SETBIT(marked_edge,i);
}
};
};
while(queue_tail!=queue_head){
curr_edge=queue[queue_tail++];
bdz_ph_remove_edge(graph3,curr_edge);
v0=graph3->edges[curr_edge].vertices[0];
v1=graph3->edges[curr_edge].vertices[1];
v2=graph3->edges[curr_edge].vertices[2];
if(graph3->vert_degree[v0]==1 ) {
tmp_edge=graph3->first_edge[v0];
if(!GETBIT(marked_edge,tmp_edge)) {
queue[queue_head++]=tmp_edge;
SETBIT(marked_edge,tmp_edge);
};
};
if(graph3->vert_degree[v1]==1) {
tmp_edge=graph3->first_edge[v1];
if(!GETBIT(marked_edge,tmp_edge)){
queue[queue_head++]=tmp_edge;
SETBIT(marked_edge,tmp_edge);
};
};
if(graph3->vert_degree[v2]==1){
tmp_edge=graph3->first_edge[v2];
if(!GETBIT(marked_edge,tmp_edge)){
queue[queue_head++]=tmp_edge;
SETBIT(marked_edge,tmp_edge);
};
};
};
free(marked_edge);
return queue_head-nedges;/* returns 0 if successful otherwies return negative number*/
};
static int bdz_ph_mapping(cmph_config_t *mph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue);
static void assigning(bdz_ph_config_data_t *bdz_ph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue);
static void bdz_ph_optimization(bdz_ph_config_data_t *bdz_ph);
bdz_ph_config_data_t *bdz_ph_config_new()
{
bdz_ph_config_data_t *bdz_ph;
bdz_ph = (bdz_ph_config_data_t *)malloc(sizeof(bdz_ph_config_data_t));
assert(bdz_ph);
memset(bdz_ph, 0, sizeof(bdz_ph_config_data_t));
bdz_ph->hashfunc = CMPH_HASH_JENKINS;
bdz_ph->g = NULL;
bdz_ph->hl = NULL;
return bdz_ph;
}
void bdz_ph_config_destroy(cmph_config_t *mph)
{
bdz_ph_config_data_t *data = (bdz_ph_config_data_t *)mph->data;
DEBUGP("Destroying algorithm dependent data\n");
free(data);
}
void bdz_ph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
{
bdz_ph_config_data_t *bdz_ph = (bdz_ph_config_data_t *)mph->data;
CMPH_HASH *hashptr = hashfuncs;
cmph_uint32 i = 0;
while(*hashptr != CMPH_HASH_COUNT)
{
if (i >= 1) break; //bdz_ph only uses one linear hash function
bdz_ph->hashfunc = *hashptr;
++i, ++hashptr;
}
}
cmph_t *bdz_ph_new(cmph_config_t *mph, float c)
{
cmph_t *mphf = NULL;
bdz_ph_data_t *bdz_phf = NULL;
cmph_uint32 iterations;
bdz_ph_queue_t edges;
bdz_ph_graph3_t graph3;
bdz_ph_config_data_t *bdz_ph = (bdz_ph_config_data_t *)mph->data;
if (c == 0) c = 1.23; // validating restrictions over parameter c.
DEBUGP("c: %f\n", c);
bdz_ph->m = mph->key_source->nkeys;
bdz_ph->r = ceil((c * mph->key_source->nkeys)/3);
bdz_ph->n = 3*bdz_ph->r;
bdz_ph_alloc_graph3(&graph3, bdz_ph->m, bdz_ph->n);
bdz_ph_alloc_queue(&edges,bdz_ph->m);
DEBUGP("Created hypergraph\n");
DEBUGP("m (edges): %u n (vertices): %u r: %u c: %f \n", bdz_ph->m, bdz_ph->n, bdz_ph->r, c);
// Mapping step
iterations = 100;
if (mph->verbosity)
{
fprintf(stderr, "Entering mapping step for mph creation of %u keys with graph sized %u\n", bdz_ph->m, bdz_ph->n);
}
while(1)
{
int ok;
DEBUGP("linear hash function \n");
bdz_ph->hl = hash_state_new(bdz_ph->hashfunc, 15);
ok = bdz_ph_mapping(mph, &graph3, edges);
if (!ok)
{
--iterations;
hash_state_destroy(bdz_ph->hl);
bdz_ph->hl = NULL;
DEBUGP("%u iterations remaining\n", iterations);
if (mph->verbosity)
{
fprintf(stderr, "acyclic graph creation failure - %u iterations remaining\n", iterations);
}
if (iterations == 0) break;
}
else break;
}
if (iterations == 0)
{
// free(bdz_ph->g);
bdz_ph_free_queue(&edges);
bdz_ph_free_graph3(&graph3);
return NULL;
}
bdz_ph_partial_free_graph3(&graph3);
// Assigning step
if (mph->verbosity)
{
fprintf(stderr, "Entering assigning step for mph creation of %u keys with graph sized %u\n", bdz_ph->m, bdz_ph->n);
}
assigning(bdz_ph, &graph3, edges);
bdz_ph_free_queue(&edges);
bdz_ph_free_graph3(&graph3);
if (mph->verbosity)
{
fprintf(stderr, "Starting optimization step\n");
}
bdz_ph_optimization(bdz_ph);
mphf = (cmph_t *)malloc(sizeof(cmph_t));
mphf->algo = mph->algo;
bdz_phf = (bdz_ph_data_t *)malloc(sizeof(bdz_ph_data_t));
bdz_phf->g = bdz_ph->g;
bdz_ph->g = NULL; //transfer memory ownership
bdz_phf->hl = bdz_ph->hl;
bdz_ph->hl = NULL; //transfer memory ownership
bdz_phf->n = bdz_ph->n;
bdz_phf->m = bdz_ph->m;
bdz_phf->r = bdz_ph->r;
mphf->data = bdz_phf;
mphf->size = bdz_ph->n;
DEBUGP("Successfully generated minimal perfect hash\n");
if (mph->verbosity)
{
fprintf(stderr, "Successfully generated minimal perfect hash function\n");
}
return mphf;
}
static int bdz_ph_mapping(cmph_config_t *mph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue)
{
cmph_uint32 e;
int cycles = 0;
cmph_uint32 hl[3];
bdz_ph_config_data_t *bdz_ph = (bdz_ph_config_data_t *)mph->data;
bdz_ph_init_graph3(graph3, bdz_ph->m, bdz_ph->n);
mph->key_source->rewind(mph->key_source->data);
for (e = 0; e < mph->key_source->nkeys; ++e)
{
cmph_uint32 h0, h1, h2;
cmph_uint32 keylen;
char *key = NULL;
mph->key_source->read(mph->key_source->data, &key, &keylen);
hash_vector(bdz_ph->hl, key, keylen, hl);
h0 = hl[0] % bdz_ph->r;
h1 = hl[1] % bdz_ph->r + bdz_ph->r;
h2 = hl[2] % bdz_ph->r + (bdz_ph->r << 1);
mph->key_source->dispose(mph->key_source->data, key, keylen);
bdz_ph_add_edge(graph3,h0,h1,h2);
}
cycles = bdz_ph_generate_queue(bdz_ph->m, bdz_ph->n, queue, graph3);
return (cycles == 0);
}
static void assigning(bdz_ph_config_data_t *bdz_ph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue)
{
cmph_uint32 i;
cmph_uint32 nedges=graph3->nedges;
cmph_uint32 curr_edge;
cmph_uint32 v0,v1,v2;
cmph_uint8 * marked_vertices =malloc((bdz_ph->n >> 3) + 1);
bdz_ph->g = (cmph_uint8 *)calloc((bdz_ph->n >> 2)+1, sizeof(cmph_uint8));
memset(marked_vertices, 0, (bdz_ph->n >> 3) + 1);
//memset(bdz_ph->g, 0xff, (bdz_ph->n >> 2) + 1);
for(i=nedges-1;i+1>=1;i--){
curr_edge=queue[i];
v0=graph3->edges[curr_edge].vertices[0];
v1=graph3->edges[curr_edge].vertices[1];
v2=graph3->edges[curr_edge].vertices[2];
DEBUGP("B:%u %u %u -- %u %u %u\n", v0, v1, v2, GETVALUE(bdz_ph->g, v0), GETVALUE(bdz_ph->g, v1), GETVALUE(bdz_ph->g, v2));
if(!GETBIT(marked_vertices, v0)){
if(!GETBIT(marked_vertices,v1))
{
//SETVALUE(bdz_ph->g, v1, UNASSIGNED);
SETBIT(marked_vertices, v1);
}
if(!GETBIT(marked_vertices,v2))
{
//SETVALUE(bdz_ph->g, v2, UNASSIGNED);
SETBIT(marked_vertices, v2);
}
SETVALUE0(bdz_ph->g, v0, (6-(GETVALUE(bdz_ph->g, v1) + GETVALUE(bdz_ph->g,v2)))%3);
SETBIT(marked_vertices, v0);
} else if(!GETBIT(marked_vertices, v1)) {
if(!GETBIT(marked_vertices, v2))
{
//SETVALUE(bdz_ph->g, v2, UNASSIGNED);
SETBIT(marked_vertices, v2);
}
SETVALUE0(bdz_ph->g, v1, (7 - (GETVALUE(bdz_ph->g, v0)+GETVALUE(bdz_ph->g, v2)))%3);
SETBIT(marked_vertices, v1);
}else {
SETVALUE0(bdz_ph->g, v2, (8-(GETVALUE(bdz_ph->g,v0)+GETVALUE(bdz_ph->g, v1)))%3);
SETBIT(marked_vertices, v2);
}
DEBUGP("A:%u %u %u -- %u %u %u\n", v0, v1, v2, GETVALUE(bdz_ph->g, v0), GETVALUE(bdz_ph->g, v1), GETVALUE(bdz_ph->g, v2));
};
free(marked_vertices);
}
static void bdz_ph_optimization(bdz_ph_config_data_t *bdz_ph)
{
cmph_uint32 i;
cmph_uint8 byte = 0;
cmph_uint8 * new_g = (cmph_uint8 *)calloc((bdz_ph->n/5)+1, sizeof(cmph_uint8));
cmph_uint8 value;
cmph_uint32 idx;
for(i = 0; i < bdz_ph->n; i++)
{
idx = i/5;
byte = new_g[idx];
value = GETVALUE(bdz_ph->g, i);
byte += value*pow3_table[i%5];
new_g[idx] = byte;
}
free(bdz_ph->g);
bdz_ph->g = new_g;
}
int bdz_ph_dump(cmph_t *mphf, FILE *fd)
{
char *buf = NULL;
cmph_uint32 buflen;
bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data;
__cmph_dump(mphf, fd);
hash_state_dump(data->hl, &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd);
fwrite(buf, buflen, 1, fd);
free(buf);
fwrite(&(data->n), sizeof(cmph_uint32), 1, fd);
fwrite(&(data->m), sizeof(cmph_uint32), 1, fd);
fwrite(&(data->r), sizeof(cmph_uint32), 1, fd);
fwrite(data->g, sizeof(cmph_uint8)*((data->n/5)+1), 1, fd);
#ifdef DEBUG
cmph_uint32 i;
fprintf(stderr, "G: ");
for (i = 0; i < data->n; ++i) fprintf(stderr, "%u ", GETVALUE(data->g, i));
fprintf(stderr, "\n");
#endif
return 1;
}
void bdz_ph_load(FILE *f, cmph_t *mphf)
{
char *buf = NULL;
cmph_uint32 buflen;
bdz_ph_data_t *bdz_ph = (bdz_ph_data_t *)malloc(sizeof(bdz_ph_data_t));
DEBUGP("Loading bdz_ph mphf\n");
mphf->data = bdz_ph;
fread(&buflen, sizeof(cmph_uint32), 1, f);
DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc(buflen);
fread(buf, buflen, 1, f);
bdz_ph->hl = hash_state_load(buf, buflen);
free(buf);
DEBUGP("Reading m and n\n");
fread(&(bdz_ph->n), sizeof(cmph_uint32), 1, f);
fread(&(bdz_ph->m), sizeof(cmph_uint32), 1, f);
fread(&(bdz_ph->r), sizeof(cmph_uint32), 1, f);
bdz_ph->g = (cmph_uint8 *)calloc((bdz_ph->n/5)+1, sizeof(cmph_uint8));
fread(bdz_ph->g, ((bdz_ph->n/5)+1)*sizeof(cmph_uint8), 1, f);
#ifdef DEBUG
fprintf(stderr, "G: ");
for (i = 0; i < bdz_ph->n; ++i) fprintf(stderr, "%u ", GETVALUE(bdz_ph->g,i));
fprintf(stderr, "\n");
#endif
return;
}
cmph_uint32 bdz_ph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen)
{
bdz_ph_data_t *bdz_ph = mphf->data;
cmph_uint32 hl[3];
cmph_uint8 byte0, byte1, byte2;
cmph_uint32 vertex;
hash_vector(bdz_ph->hl, key, keylen,hl);
hl[0] = hl[0] % bdz_ph->r;
hl[1] = hl[1] % bdz_ph->r + bdz_ph->r;
hl[2] = hl[2] % bdz_ph->r + (bdz_ph->r << 1);
byte0 = bdz_ph->g[hl[0]/5];
byte1 = bdz_ph->g[hl[1]/5];
byte2 = bdz_ph->g[hl[2]/5];
byte0 = lookup_table[hl[0]%5][byte0];
byte1 = lookup_table[hl[1]%5][byte1];
byte2 = lookup_table[hl[2]%5][byte2];
vertex = hl[(byte0 + byte1 + byte2)%3];
return vertex;
}
void bdz_ph_destroy(cmph_t *mphf)
{
bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data;
free(data->g);
hash_state_destroy(data->hl);
free(data);
free(mphf);
}

18
src/bdz_ph.h Executable file
View File

@ -0,0 +1,18 @@
#ifndef __CMPH_BDZ_PH_H__
#define __CMPH_BDZ_PH_H__
#include "cmph.h"
typedef struct __bdz_ph_data_t bdz_ph_data_t;
typedef struct __bdz_ph_config_data_t bdz_ph_config_data_t;
bdz_ph_config_data_t *bdz_ph_config_new();
void bdz_ph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void bdz_ph_config_destroy(cmph_config_t *mph);
cmph_t *bdz_ph_new(cmph_config_t *mph, float c);
void bdz_ph_load(FILE *f, cmph_t *mphf);
int bdz_ph_dump(cmph_t *mphf, FILE *f);
void bdz_ph_destroy(cmph_t *mphf);
cmph_uint32 bdz_ph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen);
#endif

26
src/bdz_structs_ph.h Executable file
View File

@ -0,0 +1,26 @@
#ifndef __CMPH_BDZ_STRUCTS_PH_H__
#define __CMPH_BDZ_STRUCTS_PH_H__
#include "hash_state.h"
struct __bdz_ph_data_t
{
cmph_uint32 m; //edges (words) count
cmph_uint32 n; //vertex count
cmph_uint32 r; //partition vertex count
cmph_uint8 *g;
hash_state_t *hl; // linear hashing
};
struct __bdz_ph_config_data_t
{
CMPH_HASH hashfunc;
cmph_uint32 m; //edges (words) count
cmph_uint32 n; //vertex count
cmph_uint32 r; //partition vertex count
cmph_uint8 *g;
hash_state_t *hl; // linear hashing
};
#endif

59
src/bitbool.h
View File

@ -3,8 +3,32 @@
#include "cmph_types.h" #include "cmph_types.h"
extern const cmph_uint8 bitmask[]; extern const cmph_uint8 bitmask[];
/** \def GETBIT(array, i)
* \brief get the value of an 1-bit integer stored in an array.
* \param array to get 1-bit integer values from
* \param i is the index in array to get the 1-bit integer value from
*
* GETBIT(array, i) is a macro that gets the value of an 1-bit integer stored in array.
*/
#define GETBIT(array, i) ((array[i >> 3] & bitmask[i & 0x00000007]) >> (i & 0x00000007)) #define GETBIT(array, i) ((array[i >> 3] & bitmask[i & 0x00000007]) >> (i & 0x00000007))
/** \def SETBIT(array, i)
* \brief set 1 to an 1-bit integer stored in an array.
* \param array to store 1-bit integer values
* \param i is the index in array to set the the bit to 1
*
* SETBIT(array, i) is a macro that sets 1 to an 1-bit integer stored in an array.
*/
#define SETBIT(array, i) (array[i >> 3] |= bitmask[i & 0x00000007]) #define SETBIT(array, i) (array[i >> 3] |= bitmask[i & 0x00000007])
/** \def UNSETBIT(array, i)
* \brief set 0 to an 1-bit integer stored in an array.
* \param array to store 1-bit integer values
* \param i is the index in array to set the the bit to 0
*
* UNSETBIT(array, i) is a macro that sets 0 to an 1-bit integer stored in an array.
*/
#define UNSETBIT(array, i) (array[i >> 3] &= (~(bitmask[i & 0x00000007]))) #define UNSETBIT(array, i) (array[i >> 3] &= (~(bitmask[i & 0x00000007])))
//#define GETBIT(array, i) (array[(i) / 8] & bitmask[(i) % 8]) //#define GETBIT(array, i) (array[(i) / 8] & bitmask[(i) % 8])
@ -12,7 +36,40 @@ extern const cmph_uint8 bitmask[];
//#define UNSETBIT(array, i) (array[(i) / 8] &= (~(bitmask[(i) % 8]))) //#define UNSETBIT(array, i) (array[(i) / 8] &= (~(bitmask[(i) % 8])))
extern const cmph_uint8 valuemask[]; extern const cmph_uint8 valuemask[];
#define SETVALUE(array, i, v) (array[i >> 2] &= ((v << ((i & 0x00000003) << 1)) | valuemask[i & 0x00000003]))
/** \def SETVALUE1(array, i, v)
* \brief set a value for a 2-bit integer stored in an array initialized with 1s.
* \param array to store 2-bit integer values
* \param i is the index in array to set the value v
* \param v is the value to be set
*
* SETVALUE1(array, i, v) is a macro that set a value for a 2-bit integer stored in an array.
* The array should be initialized with all bits set to 1. For example:
* memset(array, 0xff, arraySize);
*/
#define SETVALUE1(array, i, v) (array[i >> 2] &= ((v << ((i & 0x00000003) << 1)) | valuemask[i & 0x00000003]))
/** \def SETVALUE0(array, i, v)
* \brief set a value for a 2-bit integer stored in an array initialized with 0s.
* \param array to store 2-bit integer values
* \param i is the index in array to set the value v
* \param v is the value to be set
*
* SETVALUE0(array, i, v) is a macro that set a value for a 2-bit integer stored in an array.
* The array should be initialized with all bits set to 0. For example:
* memset(array, 0, arraySize);
*/
#define SETVALUE0(array, i, v) (array[i >> 2] |= (v << ((i & 0x00000003) << 1)))
/** \def GETVALUE(array, i)
* \brief get a value for a 2-bit integer stored in an array.
* \param array to get 2-bit integer values from
* \param i is the index in array to get the value from
*
* GETVALUE(array, i) is a macro that get a value for a 2-bit integer stored in an array.
*/
#define GETVALUE(array, i) ((array[i >> 2] >> ((i & 0x00000003) << 1)) & 0x00000003) #define GETVALUE(array, i) ((array[i >> 2] >> ((i & 0x00000003) << 1)) & 0x00000003)
#endif #endif

32
src/cmph.c
View File

@ -6,6 +6,7 @@
#include "brz.h" /* included -- Fabiano */ #include "brz.h" /* included -- Fabiano */
#include "fch.h" /* included -- Fabiano */ #include "fch.h" /* included -- Fabiano */
#include "bdz.h" /* included -- Fabiano */ #include "bdz.h" /* included -- Fabiano */
#include "bdz_ph.h" /* included -- Fabiano */
#include <stdlib.h> #include <stdlib.h>
#include <assert.h> #include <assert.h>
@ -13,7 +14,8 @@
//#define DEBUG //#define DEBUG
#include "debug.h" #include "debug.h"
const char *cmph_names[] = { "bmz", "bmz8", "chm", "brz", "fch", "bdz", NULL }; /* included -- Fabiano */ const char *cmph_names[] = {"bmz", "bmz8", "chm", "brz", "fch", "bdz",
"bdz_ph", NULL }; /* included -- Fabiano */
typedef struct typedef struct
{ {
@ -227,6 +229,9 @@ void cmph_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo)
case CMPH_BDZ: case CMPH_BDZ:
bdz_config_destroy(mph); bdz_config_destroy(mph);
break; break;
case CMPH_BDZ_PH:
bdz_ph_config_destroy(mph);
break;
default: default:
assert(0); assert(0);
} }
@ -250,6 +255,9 @@ void cmph_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo)
case CMPH_BDZ: case CMPH_BDZ:
mph->data = bdz_config_new(); mph->data = bdz_config_new();
break; break;
case CMPH_BDZ_PH:
mph->data = bdz_ph_config_new();
break;
default: default:
assert(0); assert(0);
} }
@ -317,6 +325,9 @@ void cmph_config_destroy(cmph_config_t *mph)
case CMPH_BDZ: /* included -- Fabiano */ case CMPH_BDZ: /* included -- Fabiano */
bdz_config_destroy(mph); bdz_config_destroy(mph);
break; break;
case CMPH_BDZ_PH: /* included -- Fabiano */
bdz_ph_config_destroy(mph);
break;
default: default:
assert(0); assert(0);
} }
@ -350,6 +361,9 @@ void cmph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
case CMPH_BDZ: /* included -- Fabiano */ case CMPH_BDZ: /* included -- Fabiano */
bdz_config_set_hashfuncs(mph, hashfuncs); bdz_config_set_hashfuncs(mph, hashfuncs);
break; break;
case CMPH_BDZ_PH: /* included -- Fabiano */
bdz_ph_config_set_hashfuncs(mph, hashfuncs);
break;
default: default:
break; break;
} }
@ -395,6 +409,10 @@ cmph_t *cmph_new(cmph_config_t *mph)
DEBUGP("Creating bdz hash\n"); DEBUGP("Creating bdz hash\n");
mphf = bdz_new(mph, c); mphf = bdz_new(mph, c);
break; break;
case CMPH_BDZ_PH: /* included -- Fabiano */
DEBUGP("Creating bdz_ph hash\n");
mphf = bdz_ph_new(mph, c);
break;
default: default:
assert(0); assert(0);
} }
@ -417,6 +435,8 @@ int cmph_dump(cmph_t *mphf, FILE *f)
return fch_dump(mphf, f); return fch_dump(mphf, f);
case CMPH_BDZ: /* included -- Fabiano */ case CMPH_BDZ: /* included -- Fabiano */
return bdz_dump(mphf, f); return bdz_dump(mphf, f);
case CMPH_BDZ_PH: /* included -- Fabiano */
return bdz_ph_dump(mphf, f);
default: default:
assert(0); assert(0);
} }
@ -456,6 +476,10 @@ cmph_t *cmph_load(FILE *f)
DEBUGP("Loading bdz algorithm dependent parts\n"); DEBUGP("Loading bdz algorithm dependent parts\n");
bdz_load(f, mphf); bdz_load(f, mphf);
break; break;
case CMPH_BDZ_PH: /* included -- Fabiano */
DEBUGP("Loading bdz_ph algorithm dependent parts\n");
bdz_ph_load(f, mphf);
break;
default: default:
assert(0); assert(0);
} }
@ -486,6 +510,9 @@ cmph_uint32 cmph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen)
case CMPH_BDZ: /* included -- Fabiano */ case CMPH_BDZ: /* included -- Fabiano */
DEBUGP("bdz algorithm search\n"); DEBUGP("bdz algorithm search\n");
return bdz_search(mphf, key, keylen); return bdz_search(mphf, key, keylen);
case CMPH_BDZ_PH: /* included -- Fabiano */
DEBUGP("bdz_ph algorithm search\n");
return bdz_ph_search(mphf, key, keylen);
default: default:
assert(0); assert(0);
} }
@ -520,6 +547,9 @@ void cmph_destroy(cmph_t *mphf)
case CMPH_BDZ: /* included -- Fabiano */ case CMPH_BDZ: /* included -- Fabiano */
bdz_destroy(mphf); bdz_destroy(mphf);
return; return;
case CMPH_BDZ_PH: /* included -- Fabiano */
bdz_ph_destroy(mphf);
return;
default: default:
assert(0); assert(0);
} }

3
src/cmph_types.h
View File

@ -8,7 +8,8 @@ typedef float cmph_float32;
typedef enum { CMPH_HASH_JENKINS, CMPH_HASH_COUNT } CMPH_HASH; typedef enum { CMPH_HASH_JENKINS, CMPH_HASH_COUNT } CMPH_HASH;
extern const char *cmph_hash_names[]; extern const char *cmph_hash_names[];
typedef enum { CMPH_BMZ, CMPH_BMZ8, CMPH_CHM, CMPH_BRZ, CMPH_FCH, CMPH_BDZ, CMPH_COUNT } CMPH_ALGO; /* included -- Fabiano */ typedef enum { CMPH_BMZ, CMPH_BMZ8, CMPH_CHM, CMPH_BRZ, CMPH_FCH,
CMPH_BDZ, CMPH_BDZ_PH, CMPH_COUNT } CMPH_ALGO; /* included -- Fabiano */
extern const char *cmph_names[]; extern const char *cmph_names[];
#endif #endif