From cfd0019e93ea060374b5e663c49e27b9236d11d3 Mon Sep 17 00:00:00 2001 From: fc_botelho Date: Tue, 25 Mar 2008 20:20:55 +0000 Subject: [PATCH] BDZ_PH added --- README.t2t | 4 +- src/Makefile.am | 1 + src/bdz.c | 12 +- src/bdz_ph.c | 522 +++++++++++++++++++++++++++++++++++++++++++ src/bdz_ph.h | 18 ++ src/bdz_structs_ph.h | 26 +++ src/bitbool.h | 59 ++++- src/cmph.c | 52 ++++- src/cmph_types.h | 3 +- 9 files changed, 677 insertions(+), 20 deletions(-) create mode 100755 src/bdz_ph.c create mode 100755 src/bdz_ph.h create mode 100755 src/bdz_structs_ph.h diff --git a/README.t2t b/README.t2t index bcde386..4f29513 100644 --- a/README.t2t +++ b/README.t2t @@ -44,7 +44,7 @@ The CMPH Library encapsulates the newest and more efficient algorithms in an eas %html% - [BDZ Algorithm bdz.html]. %txt% - BDZ Algorithm. - The fastest algorithm to build MPHFs. It is based on random 3-graphs. A 3-graph is a + The fastest algorithm to build PHFs and MPHFs. It is based on random 3-graphs. A 3-graph is a generalization of a graph where each edge connects 3 vertices instead of only 2. The resulting functions are not order preserving and can be stored in only //(2 + x)cn// bits, where //c// should be larger than or equal to //1.23// and //x// is a constant @@ -81,6 +81,7 @@ The CMPH Library encapsulates the newest and more efficient algorithms in an eas ==News for version 0.8 (Coming soon)== - [An algorithm to generate MPHFs that require around 2.6 bits per key to be stored bdz.html], which is referred to as BDZ algorithm. The algorithm is the fastest one available in the literature for sets that can be treated in internal memory. +- [An algorithm to generate PHFs with range m = cn, for c > 1.22 bdz.html], which is referred to as BDZ_PH algorithm. It is actually the BDZ algorithm without the ranking step. The resulting functions can be stored in 1.95 bits per key for //c = 1.23// and are considerably faster than the MPHFs generated by the BDZ algorithm. - The hash functions djb2, fnv and sdbm were removed because they do not use random seeds and therefore are not useful for MPHFs algorithms. - All reported bugs and suggestions have been corrected and included as well. @@ -204,6 +205,7 @@ Minimum perfect hashing tool * brz * fch * bdz + * bdz_ph -f hash function (may be used multiple times) - valid values are * jenkins -V print version number and exit diff --git a/src/Makefile.am b/src/Makefile.am index ab7befe..b79b496 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -15,6 +15,7 @@ libcmph_la_SOURCES = debug.h\ bmz.h bmz_structs.h bmz.c\ bmz8.h bmz8_structs.h bmz8.c\ bdz.h bdz_structs.h bdz.c\ + bdz_ph.h bdz_structs_ph.h bdz_ph.c\ buffer_manager.h buffer_manager.c\ buffer_entry.h buffer_entry.c\ brz.h brz_structs.h brz.c\ diff --git a/src/bdz.c b/src/bdz.c index de67381..ed0a405 100755 --- a/src/bdz.c +++ b/src/bdz.c @@ -412,26 +412,26 @@ static void assigning(bdz_config_data_t *bdz, bdz_graph3_t* graph3, bdz_queue_t if(!GETBIT(marked_vertices, v0)){ if(!GETBIT(marked_vertices,v1)) { - SETVALUE(bdz->g, v1, UNASSIGNED); + SETVALUE1(bdz->g, v1, UNASSIGNED); SETBIT(marked_vertices, v1); } if(!GETBIT(marked_vertices,v2)) { - SETVALUE(bdz->g, v2, UNASSIGNED); + SETVALUE1(bdz->g, v2, UNASSIGNED); SETBIT(marked_vertices, v2); } - SETVALUE(bdz->g, v0, (6-(GETVALUE(bdz->g, v1) + GETVALUE(bdz->g,v2)))%3); + SETVALUE1(bdz->g, v0, (6-(GETVALUE(bdz->g, v1) + GETVALUE(bdz->g,v2)))%3); SETBIT(marked_vertices, v0); } else if(!GETBIT(marked_vertices, v1)) { if(!GETBIT(marked_vertices, v2)) { - SETVALUE(bdz->g, v2, UNASSIGNED); + SETVALUE1(bdz->g, v2, UNASSIGNED); SETBIT(marked_vertices, v2); } - SETVALUE(bdz->g, v1, (7-(GETVALUE(bdz->g, v0)+GETVALUE(bdz->g, v2)))%3); + SETVALUE1(bdz->g, v1, (7-(GETVALUE(bdz->g, v0)+GETVALUE(bdz->g, v2)))%3); SETBIT(marked_vertices, v1); }else { - SETVALUE(bdz->g, v2, (8-(GETVALUE(bdz->g,v0)+GETVALUE(bdz->g, v1)))%3); + SETVALUE1(bdz->g, v2, (8-(GETVALUE(bdz->g,v0)+GETVALUE(bdz->g, v1)))%3); SETBIT(marked_vertices, v2); } DEBUGP("A:%u %u %u -- %u %u %u\n", v0, v1, v2, GETVALUE(bdz->g, v0), GETVALUE(bdz->g, v1), GETVALUE(bdz->g, v2)); diff --git a/src/bdz_ph.c b/src/bdz_ph.c new file mode 100755 index 0000000..3ca4700 --- /dev/null +++ b/src/bdz_ph.c @@ -0,0 +1,522 @@ +#include "bdz_ph.h" +#include "cmph_structs.h" +#include "bdz_structs_ph.h" +#include "hash.h" +#include "bitbool.h" + +#include +#include +#include +#include +#include +//#define DEBUG +#include "debug.h" +#define UNASSIGNED 3 +#define NULL_EDGE 0xffffffff + + +static int pow3_table[5] = {1,3,9,27,81}; +static int lookup_table[5][256] = { + {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0}, + {0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 0, 0, 0, 1}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, +}; + +typedef struct +{ + cmph_uint32 vertices[3]; + cmph_uint32 next_edges[3]; +}bdz_ph_edge_t; + +typedef cmph_uint32 * bdz_ph_queue_t; + +static void bdz_ph_alloc_queue(bdz_ph_queue_t * queuep, cmph_uint32 nedges) +{ + (*queuep)=malloc(nedges*sizeof(cmph_uint32)); +}; +static void bdz_ph_free_queue(bdz_ph_queue_t * queue) +{ + free(*queue); +}; + +typedef struct +{ + cmph_uint32 nedges; + bdz_ph_edge_t * edges; + cmph_uint32 * first_edge; + cmph_uint8 * vert_degree; +}bdz_ph_graph3_t; + + +static void bdz_ph_alloc_graph3(bdz_ph_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices) +{ + graph3->edges=malloc(nedges*sizeof(bdz_ph_edge_t)); + graph3->first_edge=malloc(nvertices*sizeof(cmph_uint32)); + graph3->vert_degree=malloc(nvertices); +}; +static void bdz_ph_init_graph3(bdz_ph_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices) +{ + memset(graph3->first_edge,0xff,nvertices*sizeof(cmph_uint32)); + memset(graph3->vert_degree,0,nvertices); + graph3->nedges=0; +}; +static void bdz_ph_free_graph3(bdz_ph_graph3_t *graph3) +{ + free(graph3->edges); + free(graph3->first_edge); + free(graph3->vert_degree); +}; + +static void bdz_ph_partial_free_graph3(bdz_ph_graph3_t *graph3) +{ + free(graph3->first_edge); + free(graph3->vert_degree); + graph3->first_edge = NULL; + graph3->vert_degree = NULL; +}; + +static void bdz_ph_add_edge(bdz_ph_graph3_t * graph3, cmph_uint32 v0, cmph_uint32 v1, cmph_uint32 v2) +{ + graph3->edges[graph3->nedges].vertices[0]=v0; + graph3->edges[graph3->nedges].vertices[1]=v1; + graph3->edges[graph3->nedges].vertices[2]=v2; + graph3->edges[graph3->nedges].next_edges[0]=graph3->first_edge[v0]; + graph3->edges[graph3->nedges].next_edges[1]=graph3->first_edge[v1]; + graph3->edges[graph3->nedges].next_edges[2]=graph3->first_edge[v2]; + graph3->first_edge[v0]=graph3->first_edge[v1]=graph3->first_edge[v2]=graph3->nedges; + graph3->vert_degree[v0]++; + graph3->vert_degree[v1]++; + graph3->vert_degree[v2]++; + graph3->nedges++; +}; + +static void bdz_ph_dump_graph(bdz_ph_graph3_t* graph3, cmph_uint32 nedges, cmph_uint32 nvertices) +{ + int i; + for(i=0;iedges[i].vertices[0], + graph3->edges[i].vertices[1],graph3->edges[i].vertices[2]); + printf(" nexts %d %d %d",graph3->edges[i].next_edges[0], + graph3->edges[i].next_edges[1],graph3->edges[i].next_edges[2]); + }; + + for(i=0;ifirst_edge[i]); + + }; +}; + +static void bdz_ph_remove_edge(bdz_ph_graph3_t * graph3, cmph_uint32 curr_edge) +{ + cmph_uint32 i,j=0,vert,edge1,edge2; + for(i=0;i<3;i++){ + vert=graph3->edges[curr_edge].vertices[i]; + edge1=graph3->first_edge[vert]; + edge2=NULL_EDGE; + while(edge1!=curr_edge&&edge1!=NULL_EDGE){ + edge2=edge1; + if(graph3->edges[edge1].vertices[0]==vert){ + j=0; + } else if(graph3->edges[edge1].vertices[1]==vert){ + j=1; + } else + j=2; + edge1=graph3->edges[edge1].next_edges[j]; + }; + if(edge1==NULL_EDGE){ + printf("\nerror remove edge %d dump graph",curr_edge); + bdz_ph_dump_graph(graph3,graph3->nedges,graph3->nedges+graph3->nedges/4); + exit(-1); + }; + + if(edge2!=NULL_EDGE){ + graph3->edges[edge2].next_edges[j] = + graph3->edges[edge1].next_edges[i]; + } else + graph3->first_edge[vert]= + graph3->edges[edge1].next_edges[i]; + graph3->vert_degree[vert]--; + }; + +}; + +static int bdz_ph_generate_queue(cmph_uint32 nedges, cmph_uint32 nvertices, bdz_ph_queue_t queue, bdz_ph_graph3_t* graph3) +{ + cmph_uint32 i,v0,v1,v2; + cmph_uint32 queue_head=0,queue_tail=0; + cmph_uint32 curr_edge; + cmph_uint32 tmp_edge; + cmph_uint8 * marked_edge =malloc((nedges >> 3) + 1); + memset(marked_edge, 0, (nedges >> 3) + 1); + + for(i=0;iedges[i].vertices[0]; + v1=graph3->edges[i].vertices[1]; + v2=graph3->edges[i].vertices[2]; + if(graph3->vert_degree[v0]==1 || + graph3->vert_degree[v1]==1 || + graph3->vert_degree[v2]==1){ + if(!GETBIT(marked_edge,i)) { + queue[queue_head++]=i; + SETBIT(marked_edge,i); + } + }; + }; + while(queue_tail!=queue_head){ + curr_edge=queue[queue_tail++]; + bdz_ph_remove_edge(graph3,curr_edge); + v0=graph3->edges[curr_edge].vertices[0]; + v1=graph3->edges[curr_edge].vertices[1]; + v2=graph3->edges[curr_edge].vertices[2]; + if(graph3->vert_degree[v0]==1 ) { + tmp_edge=graph3->first_edge[v0]; + if(!GETBIT(marked_edge,tmp_edge)) { + queue[queue_head++]=tmp_edge; + SETBIT(marked_edge,tmp_edge); + }; + + }; + if(graph3->vert_degree[v1]==1) { + tmp_edge=graph3->first_edge[v1]; + if(!GETBIT(marked_edge,tmp_edge)){ + queue[queue_head++]=tmp_edge; + SETBIT(marked_edge,tmp_edge); + }; + + }; + if(graph3->vert_degree[v2]==1){ + tmp_edge=graph3->first_edge[v2]; + if(!GETBIT(marked_edge,tmp_edge)){ + queue[queue_head++]=tmp_edge; + SETBIT(marked_edge,tmp_edge); + }; + }; + }; + free(marked_edge); + return queue_head-nedges;/* returns 0 if successful otherwies return negative number*/ +}; + +static int bdz_ph_mapping(cmph_config_t *mph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue); +static void assigning(bdz_ph_config_data_t *bdz_ph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue); +static void bdz_ph_optimization(bdz_ph_config_data_t *bdz_ph); + +bdz_ph_config_data_t *bdz_ph_config_new() +{ + bdz_ph_config_data_t *bdz_ph; + bdz_ph = (bdz_ph_config_data_t *)malloc(sizeof(bdz_ph_config_data_t)); + assert(bdz_ph); + memset(bdz_ph, 0, sizeof(bdz_ph_config_data_t)); + bdz_ph->hashfunc = CMPH_HASH_JENKINS; + bdz_ph->g = NULL; + bdz_ph->hl = NULL; + return bdz_ph; +} + +void bdz_ph_config_destroy(cmph_config_t *mph) +{ + bdz_ph_config_data_t *data = (bdz_ph_config_data_t *)mph->data; + DEBUGP("Destroying algorithm dependent data\n"); + free(data); +} + +void bdz_ph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs) +{ + bdz_ph_config_data_t *bdz_ph = (bdz_ph_config_data_t *)mph->data; + CMPH_HASH *hashptr = hashfuncs; + cmph_uint32 i = 0; + while(*hashptr != CMPH_HASH_COUNT) + { + if (i >= 1) break; //bdz_ph only uses one linear hash function + bdz_ph->hashfunc = *hashptr; + ++i, ++hashptr; + } +} + +cmph_t *bdz_ph_new(cmph_config_t *mph, float c) +{ + cmph_t *mphf = NULL; + bdz_ph_data_t *bdz_phf = NULL; + cmph_uint32 iterations; + bdz_ph_queue_t edges; + bdz_ph_graph3_t graph3; + bdz_ph_config_data_t *bdz_ph = (bdz_ph_config_data_t *)mph->data; + + if (c == 0) c = 1.23; // validating restrictions over parameter c. + DEBUGP("c: %f\n", c); + bdz_ph->m = mph->key_source->nkeys; + bdz_ph->r = ceil((c * mph->key_source->nkeys)/3); + bdz_ph->n = 3*bdz_ph->r; + + + bdz_ph_alloc_graph3(&graph3, bdz_ph->m, bdz_ph->n); + bdz_ph_alloc_queue(&edges,bdz_ph->m); + DEBUGP("Created hypergraph\n"); + + DEBUGP("m (edges): %u n (vertices): %u r: %u c: %f \n", bdz_ph->m, bdz_ph->n, bdz_ph->r, c); + + // Mapping step + iterations = 100; + if (mph->verbosity) + { + fprintf(stderr, "Entering mapping step for mph creation of %u keys with graph sized %u\n", bdz_ph->m, bdz_ph->n); + } + while(1) + { + int ok; + DEBUGP("linear hash function \n"); + bdz_ph->hl = hash_state_new(bdz_ph->hashfunc, 15); + + ok = bdz_ph_mapping(mph, &graph3, edges); + if (!ok) + { + --iterations; + hash_state_destroy(bdz_ph->hl); + bdz_ph->hl = NULL; + DEBUGP("%u iterations remaining\n", iterations); + if (mph->verbosity) + { + fprintf(stderr, "acyclic graph creation failure - %u iterations remaining\n", iterations); + } + if (iterations == 0) break; + } + else break; + } + + if (iterations == 0) + { +// free(bdz_ph->g); + bdz_ph_free_queue(&edges); + bdz_ph_free_graph3(&graph3); + return NULL; + } + bdz_ph_partial_free_graph3(&graph3); + // Assigning step + if (mph->verbosity) + { + fprintf(stderr, "Entering assigning step for mph creation of %u keys with graph sized %u\n", bdz_ph->m, bdz_ph->n); + } + assigning(bdz_ph, &graph3, edges); + + bdz_ph_free_queue(&edges); + bdz_ph_free_graph3(&graph3); + + if (mph->verbosity) + { + fprintf(stderr, "Starting optimization step\n"); + } + + bdz_ph_optimization(bdz_ph); + + mphf = (cmph_t *)malloc(sizeof(cmph_t)); + mphf->algo = mph->algo; + bdz_phf = (bdz_ph_data_t *)malloc(sizeof(bdz_ph_data_t)); + bdz_phf->g = bdz_ph->g; + bdz_ph->g = NULL; //transfer memory ownership + bdz_phf->hl = bdz_ph->hl; + bdz_ph->hl = NULL; //transfer memory ownership + bdz_phf->n = bdz_ph->n; + bdz_phf->m = bdz_ph->m; + bdz_phf->r = bdz_ph->r; + mphf->data = bdz_phf; + mphf->size = bdz_ph->n; + + DEBUGP("Successfully generated minimal perfect hash\n"); + if (mph->verbosity) + { + fprintf(stderr, "Successfully generated minimal perfect hash function\n"); + } + + return mphf; +} + + +static int bdz_ph_mapping(cmph_config_t *mph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue) +{ + cmph_uint32 e; + int cycles = 0; + cmph_uint32 hl[3]; + + bdz_ph_config_data_t *bdz_ph = (bdz_ph_config_data_t *)mph->data; + bdz_ph_init_graph3(graph3, bdz_ph->m, bdz_ph->n); + mph->key_source->rewind(mph->key_source->data); + for (e = 0; e < mph->key_source->nkeys; ++e) + { + cmph_uint32 h0, h1, h2; + cmph_uint32 keylen; + char *key = NULL; + mph->key_source->read(mph->key_source->data, &key, &keylen); + hash_vector(bdz_ph->hl, key, keylen, hl); + h0 = hl[0] % bdz_ph->r; + h1 = hl[1] % bdz_ph->r + bdz_ph->r; + h2 = hl[2] % bdz_ph->r + (bdz_ph->r << 1); + mph->key_source->dispose(mph->key_source->data, key, keylen); + bdz_ph_add_edge(graph3,h0,h1,h2); + } + cycles = bdz_ph_generate_queue(bdz_ph->m, bdz_ph->n, queue, graph3); + return (cycles == 0); +} + +static void assigning(bdz_ph_config_data_t *bdz_ph, bdz_ph_graph3_t* graph3, bdz_ph_queue_t queue) +{ + cmph_uint32 i; + cmph_uint32 nedges=graph3->nedges; + cmph_uint32 curr_edge; + cmph_uint32 v0,v1,v2; + cmph_uint8 * marked_vertices =malloc((bdz_ph->n >> 3) + 1); + bdz_ph->g = (cmph_uint8 *)calloc((bdz_ph->n >> 2)+1, sizeof(cmph_uint8)); + memset(marked_vertices, 0, (bdz_ph->n >> 3) + 1); + //memset(bdz_ph->g, 0xff, (bdz_ph->n >> 2) + 1); + + for(i=nedges-1;i+1>=1;i--){ + curr_edge=queue[i]; + v0=graph3->edges[curr_edge].vertices[0]; + v1=graph3->edges[curr_edge].vertices[1]; + v2=graph3->edges[curr_edge].vertices[2]; + DEBUGP("B:%u %u %u -- %u %u %u\n", v0, v1, v2, GETVALUE(bdz_ph->g, v0), GETVALUE(bdz_ph->g, v1), GETVALUE(bdz_ph->g, v2)); + if(!GETBIT(marked_vertices, v0)){ + if(!GETBIT(marked_vertices,v1)) + { + //SETVALUE(bdz_ph->g, v1, UNASSIGNED); + SETBIT(marked_vertices, v1); + } + if(!GETBIT(marked_vertices,v2)) + { + //SETVALUE(bdz_ph->g, v2, UNASSIGNED); + SETBIT(marked_vertices, v2); + } + SETVALUE0(bdz_ph->g, v0, (6-(GETVALUE(bdz_ph->g, v1) + GETVALUE(bdz_ph->g,v2)))%3); + SETBIT(marked_vertices, v0); + } else if(!GETBIT(marked_vertices, v1)) { + if(!GETBIT(marked_vertices, v2)) + { + //SETVALUE(bdz_ph->g, v2, UNASSIGNED); + SETBIT(marked_vertices, v2); + } + SETVALUE0(bdz_ph->g, v1, (7 - (GETVALUE(bdz_ph->g, v0)+GETVALUE(bdz_ph->g, v2)))%3); + SETBIT(marked_vertices, v1); + }else { + SETVALUE0(bdz_ph->g, v2, (8-(GETVALUE(bdz_ph->g,v0)+GETVALUE(bdz_ph->g, v1)))%3); + SETBIT(marked_vertices, v2); + } + DEBUGP("A:%u %u %u -- %u %u %u\n", v0, v1, v2, GETVALUE(bdz_ph->g, v0), GETVALUE(bdz_ph->g, v1), GETVALUE(bdz_ph->g, v2)); + }; + free(marked_vertices); +} + +static void bdz_ph_optimization(bdz_ph_config_data_t *bdz_ph) +{ + cmph_uint32 i; + cmph_uint8 byte = 0; + cmph_uint8 * new_g = (cmph_uint8 *)calloc((bdz_ph->n/5)+1, sizeof(cmph_uint8)); + cmph_uint8 value; + cmph_uint32 idx; + for(i = 0; i < bdz_ph->n; i++) + { + idx = i/5; + byte = new_g[idx]; + value = GETVALUE(bdz_ph->g, i); + byte += value*pow3_table[i%5]; + new_g[idx] = byte; + } + free(bdz_ph->g); + bdz_ph->g = new_g; +} + + +int bdz_ph_dump(cmph_t *mphf, FILE *fd) +{ + char *buf = NULL; + cmph_uint32 buflen; + bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data; + __cmph_dump(mphf, fd); + + hash_state_dump(data->hl, &buf, &buflen); + DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); + fwrite(&buflen, sizeof(cmph_uint32), 1, fd); + fwrite(buf, buflen, 1, fd); + free(buf); + + fwrite(&(data->n), sizeof(cmph_uint32), 1, fd); + fwrite(&(data->m), sizeof(cmph_uint32), 1, fd); + fwrite(&(data->r), sizeof(cmph_uint32), 1, fd); + + fwrite(data->g, sizeof(cmph_uint8)*((data->n/5)+1), 1, fd); + + #ifdef DEBUG + cmph_uint32 i; + fprintf(stderr, "G: "); + for (i = 0; i < data->n; ++i) fprintf(stderr, "%u ", GETVALUE(data->g, i)); + fprintf(stderr, "\n"); + #endif + return 1; +} + +void bdz_ph_load(FILE *f, cmph_t *mphf) +{ + char *buf = NULL; + cmph_uint32 buflen; + bdz_ph_data_t *bdz_ph = (bdz_ph_data_t *)malloc(sizeof(bdz_ph_data_t)); + + DEBUGP("Loading bdz_ph mphf\n"); + mphf->data = bdz_ph; + + fread(&buflen, sizeof(cmph_uint32), 1, f); + DEBUGP("Hash state has %u bytes\n", buflen); + buf = (char *)malloc(buflen); + fread(buf, buflen, 1, f); + bdz_ph->hl = hash_state_load(buf, buflen); + free(buf); + + + DEBUGP("Reading m and n\n"); + fread(&(bdz_ph->n), sizeof(cmph_uint32), 1, f); + fread(&(bdz_ph->m), sizeof(cmph_uint32), 1, f); + fread(&(bdz_ph->r), sizeof(cmph_uint32), 1, f); + + bdz_ph->g = (cmph_uint8 *)calloc((bdz_ph->n/5)+1, sizeof(cmph_uint8)); + fread(bdz_ph->g, ((bdz_ph->n/5)+1)*sizeof(cmph_uint8), 1, f); + + #ifdef DEBUG + fprintf(stderr, "G: "); + for (i = 0; i < bdz_ph->n; ++i) fprintf(stderr, "%u ", GETVALUE(bdz_ph->g,i)); + fprintf(stderr, "\n"); + #endif + return; +} + + +cmph_uint32 bdz_ph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen) +{ + bdz_ph_data_t *bdz_ph = mphf->data; + cmph_uint32 hl[3]; + cmph_uint8 byte0, byte1, byte2; + cmph_uint32 vertex; + + hash_vector(bdz_ph->hl, key, keylen,hl); + hl[0] = hl[0] % bdz_ph->r; + hl[1] = hl[1] % bdz_ph->r + bdz_ph->r; + hl[2] = hl[2] % bdz_ph->r + (bdz_ph->r << 1); + + byte0 = bdz_ph->g[hl[0]/5]; + byte1 = bdz_ph->g[hl[1]/5]; + byte2 = bdz_ph->g[hl[2]/5]; + + byte0 = lookup_table[hl[0]%5][byte0]; + byte1 = lookup_table[hl[1]%5][byte1]; + byte2 = lookup_table[hl[2]%5][byte2]; + vertex = hl[(byte0 + byte1 + byte2)%3]; + + return vertex; +} + + +void bdz_ph_destroy(cmph_t *mphf) +{ + bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data; + free(data->g); + hash_state_destroy(data->hl); + free(data); + free(mphf); +} diff --git a/src/bdz_ph.h b/src/bdz_ph.h new file mode 100755 index 0000000..fc61e11 --- /dev/null +++ b/src/bdz_ph.h @@ -0,0 +1,18 @@ +#ifndef __CMPH_BDZ_PH_H__ +#define __CMPH_BDZ_PH_H__ + +#include "cmph.h" + +typedef struct __bdz_ph_data_t bdz_ph_data_t; +typedef struct __bdz_ph_config_data_t bdz_ph_config_data_t; + +bdz_ph_config_data_t *bdz_ph_config_new(); +void bdz_ph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); +void bdz_ph_config_destroy(cmph_config_t *mph); +cmph_t *bdz_ph_new(cmph_config_t *mph, float c); + +void bdz_ph_load(FILE *f, cmph_t *mphf); +int bdz_ph_dump(cmph_t *mphf, FILE *f); +void bdz_ph_destroy(cmph_t *mphf); +cmph_uint32 bdz_ph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen); +#endif diff --git a/src/bdz_structs_ph.h b/src/bdz_structs_ph.h new file mode 100755 index 0000000..5874a26 --- /dev/null +++ b/src/bdz_structs_ph.h @@ -0,0 +1,26 @@ +#ifndef __CMPH_BDZ_STRUCTS_PH_H__ +#define __CMPH_BDZ_STRUCTS_PH_H__ + +#include "hash_state.h" + +struct __bdz_ph_data_t +{ + cmph_uint32 m; //edges (words) count + cmph_uint32 n; //vertex count + cmph_uint32 r; //partition vertex count + cmph_uint8 *g; + hash_state_t *hl; // linear hashing +}; + + +struct __bdz_ph_config_data_t +{ + CMPH_HASH hashfunc; + cmph_uint32 m; //edges (words) count + cmph_uint32 n; //vertex count + cmph_uint32 r; //partition vertex count + cmph_uint8 *g; + hash_state_t *hl; // linear hashing +}; + +#endif diff --git a/src/bitbool.h b/src/bitbool.h index ec0250a..c46be05 100644 --- a/src/bitbool.h +++ b/src/bitbool.h @@ -3,8 +3,32 @@ #include "cmph_types.h" extern const cmph_uint8 bitmask[]; + +/** \def GETBIT(array, i) + * \brief get the value of an 1-bit integer stored in an array. + * \param array to get 1-bit integer values from + * \param i is the index in array to get the 1-bit integer value from + * + * GETBIT(array, i) is a macro that gets the value of an 1-bit integer stored in array. + */ #define GETBIT(array, i) ((array[i >> 3] & bitmask[i & 0x00000007]) >> (i & 0x00000007)) + +/** \def SETBIT(array, i) + * \brief set 1 to an 1-bit integer stored in an array. + * \param array to store 1-bit integer values + * \param i is the index in array to set the the bit to 1 + * + * SETBIT(array, i) is a macro that sets 1 to an 1-bit integer stored in an array. + */ #define SETBIT(array, i) (array[i >> 3] |= bitmask[i & 0x00000007]) + +/** \def UNSETBIT(array, i) + * \brief set 0 to an 1-bit integer stored in an array. + * \param array to store 1-bit integer values + * \param i is the index in array to set the the bit to 0 + * + * UNSETBIT(array, i) is a macro that sets 0 to an 1-bit integer stored in an array. + */ #define UNSETBIT(array, i) (array[i >> 3] &= (~(bitmask[i & 0x00000007]))) //#define GETBIT(array, i) (array[(i) / 8] & bitmask[(i) % 8]) @@ -12,7 +36,40 @@ extern const cmph_uint8 bitmask[]; //#define UNSETBIT(array, i) (array[(i) / 8] &= (~(bitmask[(i) % 8]))) extern const cmph_uint8 valuemask[]; -#define SETVALUE(array, i, v) (array[i >> 2] &= ((v << ((i & 0x00000003) << 1)) | valuemask[i & 0x00000003])) + +/** \def SETVALUE1(array, i, v) + * \brief set a value for a 2-bit integer stored in an array initialized with 1s. + * \param array to store 2-bit integer values + * \param i is the index in array to set the value v + * \param v is the value to be set + * + * SETVALUE1(array, i, v) is a macro that set a value for a 2-bit integer stored in an array. + * The array should be initialized with all bits set to 1. For example: + * memset(array, 0xff, arraySize); + */ +#define SETVALUE1(array, i, v) (array[i >> 2] &= ((v << ((i & 0x00000003) << 1)) | valuemask[i & 0x00000003])) + +/** \def SETVALUE0(array, i, v) + * \brief set a value for a 2-bit integer stored in an array initialized with 0s. + * \param array to store 2-bit integer values + * \param i is the index in array to set the value v + * \param v is the value to be set + * + * SETVALUE0(array, i, v) is a macro that set a value for a 2-bit integer stored in an array. + * The array should be initialized with all bits set to 0. For example: + * memset(array, 0, arraySize); + */ +#define SETVALUE0(array, i, v) (array[i >> 2] |= (v << ((i & 0x00000003) << 1))) + + +/** \def GETVALUE(array, i) + * \brief get a value for a 2-bit integer stored in an array. + * \param array to get 2-bit integer values from + * \param i is the index in array to get the value from + * + * GETVALUE(array, i) is a macro that get a value for a 2-bit integer stored in an array. + */ #define GETVALUE(array, i) ((array[i >> 2] >> ((i & 0x00000003) << 1)) & 0x00000003) + #endif diff --git a/src/cmph.c b/src/cmph.c index a4bdbd4..5518c56 100644 --- a/src/cmph.c +++ b/src/cmph.c @@ -6,6 +6,7 @@ #include "brz.h" /* included -- Fabiano */ #include "fch.h" /* included -- Fabiano */ #include "bdz.h" /* included -- Fabiano */ +#include "bdz_ph.h" /* included -- Fabiano */ #include #include @@ -13,7 +14,8 @@ //#define DEBUG #include "debug.h" -const char *cmph_names[] = { "bmz", "bmz8", "chm", "brz", "fch", "bdz", NULL }; /* included -- Fabiano */ +const char *cmph_names[] = {"bmz", "bmz8", "chm", "brz", "fch", "bdz", + "bdz_ph", NULL }; /* included -- Fabiano */ typedef struct { @@ -227,6 +229,9 @@ void cmph_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo) case CMPH_BDZ: bdz_config_destroy(mph); break; + case CMPH_BDZ_PH: + bdz_ph_config_destroy(mph); + break; default: assert(0); } @@ -250,6 +255,9 @@ void cmph_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo) case CMPH_BDZ: mph->data = bdz_config_new(); break; + case CMPH_BDZ_PH: + mph->data = bdz_ph_config_new(); + break; default: assert(0); } @@ -317,6 +325,9 @@ void cmph_config_destroy(cmph_config_t *mph) case CMPH_BDZ: /* included -- Fabiano */ bdz_config_destroy(mph); break; + case CMPH_BDZ_PH: /* included -- Fabiano */ + bdz_ph_config_destroy(mph); + break; default: assert(0); } @@ -350,6 +361,9 @@ void cmph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs) case CMPH_BDZ: /* included -- Fabiano */ bdz_config_set_hashfuncs(mph, hashfuncs); break; + case CMPH_BDZ_PH: /* included -- Fabiano */ + bdz_ph_config_set_hashfuncs(mph, hashfuncs); + break; default: break; } @@ -395,6 +409,10 @@ cmph_t *cmph_new(cmph_config_t *mph) DEBUGP("Creating bdz hash\n"); mphf = bdz_new(mph, c); break; + case CMPH_BDZ_PH: /* included -- Fabiano */ + DEBUGP("Creating bdz_ph hash\n"); + mphf = bdz_ph_new(mph, c); + break; default: assert(0); } @@ -408,15 +426,17 @@ int cmph_dump(cmph_t *mphf, FILE *f) case CMPH_CHM: return chm_dump(mphf, f); case CMPH_BMZ: /* included -- Fabiano */ - return bmz_dump(mphf, f); + return bmz_dump(mphf, f); case CMPH_BMZ8: /* included -- Fabiano */ - return bmz8_dump(mphf, f); + return bmz8_dump(mphf, f); case CMPH_BRZ: /* included -- Fabiano */ - return brz_dump(mphf, f); + return brz_dump(mphf, f); case CMPH_FCH: /* included -- Fabiano */ - return fch_dump(mphf, f); + return fch_dump(mphf, f); case CMPH_BDZ: /* included -- Fabiano */ - return bdz_dump(mphf, f); + return bdz_dump(mphf, f); + case CMPH_BDZ_PH: /* included -- Fabiano */ + return bdz_ph_dump(mphf, f); default: assert(0); } @@ -456,6 +476,10 @@ cmph_t *cmph_load(FILE *f) DEBUGP("Loading bdz algorithm dependent parts\n"); bdz_load(f, mphf); break; + case CMPH_BDZ_PH: /* included -- Fabiano */ + DEBUGP("Loading bdz_ph algorithm dependent parts\n"); + bdz_ph_load(f, mphf); + break; default: assert(0); } @@ -486,6 +510,9 @@ cmph_uint32 cmph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen) case CMPH_BDZ: /* included -- Fabiano */ DEBUGP("bdz algorithm search\n"); return bdz_search(mphf, key, keylen); + case CMPH_BDZ_PH: /* included -- Fabiano */ + DEBUGP("bdz_ph algorithm search\n"); + return bdz_ph_search(mphf, key, keylen); default: assert(0); } @@ -506,19 +533,22 @@ void cmph_destroy(cmph_t *mphf) chm_destroy(mphf); return; case CMPH_BMZ: /* included -- Fabiano */ - bmz_destroy(mphf); + bmz_destroy(mphf); return; case CMPH_BMZ8: /* included -- Fabiano */ - bmz8_destroy(mphf); + bmz8_destroy(mphf); return; case CMPH_BRZ: /* included -- Fabiano */ - brz_destroy(mphf); + brz_destroy(mphf); return; case CMPH_FCH: /* included -- Fabiano */ - fch_destroy(mphf); + fch_destroy(mphf); return; case CMPH_BDZ: /* included -- Fabiano */ - bdz_destroy(mphf); + bdz_destroy(mphf); + return; + case CMPH_BDZ_PH: /* included -- Fabiano */ + bdz_ph_destroy(mphf); return; default: assert(0); diff --git a/src/cmph_types.h b/src/cmph_types.h index 1359adb..73a9d03 100644 --- a/src/cmph_types.h +++ b/src/cmph_types.h @@ -8,7 +8,8 @@ typedef float cmph_float32; typedef enum { CMPH_HASH_JENKINS, CMPH_HASH_COUNT } CMPH_HASH; extern const char *cmph_hash_names[]; -typedef enum { CMPH_BMZ, CMPH_BMZ8, CMPH_CHM, CMPH_BRZ, CMPH_FCH, CMPH_BDZ, CMPH_COUNT } CMPH_ALGO; /* included -- Fabiano */ +typedef enum { CMPH_BMZ, CMPH_BMZ8, CMPH_CHM, CMPH_BRZ, CMPH_FCH, + CMPH_BDZ, CMPH_BDZ_PH, CMPH_COUNT } CMPH_ALGO; /* included -- Fabiano */ extern const char *cmph_names[]; #endif