motiejus/turbonss
1
Fork 0
Code Releases Activity

*** empty log message ***

Browse Source
This commit is contained in:
fc_botelho 2008-04-12 06:17:21 +00:00
parent 44e343a040
commit b8d4392b85
39 changed files with 359 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/file_adapter_ex2.c
View File

@ -22,7 +22,7 @@ int main(int argc, char **argv)
//Find key //Find key
const char *key = "jjjjjjjjjj"; const char *key = "jjjjjjjjjj";
unsigned int id = cmph_search(hash, key, strlen(key)); unsigned int id = cmph_search(hash, key, (cmph_uint32)strlen(key));
fprintf(stderr, "Id:%u\n", id); fprintf(stderr, "Id:%u\n", id);
//Destroy hash //Destroy hash
cmph_destroy(hash); cmph_destroy(hash);

2
examples/struct_vector_adapter_ex3.c
View File

@ -21,7 +21,7 @@ int main(int argc, char **argv)
unsigned int nkeys = 10; unsigned int nkeys = 10;
FILE* mphf_fd = fopen("temp_struct_vector.mph", "w"); FILE* mphf_fd = fopen("temp_struct_vector.mph", "w");
// Source of keys // Source of keys
cmph_io_adapter_t *source = cmph_io_struct_vector_adapter(vector, sizeof(rec_t), sizeof(cmph_uint32), 11, nkeys); cmph_io_adapter_t *source = cmph_io_struct_vector_adapter(vector, (cmph_uint32)sizeof(rec_t), (cmph_uint32)sizeof(cmph_uint32), 11, nkeys);
//Create minimal perfect hash function using the default (chm) algorithm. //Create minimal perfect hash function using the default (chm) algorithm.
cmph_config_t *config = cmph_config_new(source); cmph_config_t *config = cmph_config_new(source);

2
examples/vector_adapter_ex1.c
View File

@ -27,7 +27,7 @@ int main(int argc, char **argv)
hash = cmph_load(mphf_fd); hash = cmph_load(mphf_fd);
while (i < nkeys) { while (i < nkeys) {
const char *key = vector[i]; const char *key = vector[i];
unsigned int id = cmph_search(hash, key, strlen(key)); unsigned int id = cmph_search(hash, key, (cmph_uint32)strlen(key));
fprintf(stderr, "key:%s -- hash:%u\n", key, id); fprintf(stderr, "key:%s -- hash:%u\n", key, id);
i++; i++;
} }

93
src/bdz.c
View File

@ -67,12 +67,12 @@ static void bdz_alloc_graph3(bdz_graph3_t * graph3, cmph_uint32 nedges, cmph_uin
{ {
graph3->edges=malloc(nedges*sizeof(bdz_edge_t)); graph3->edges=malloc(nedges*sizeof(bdz_edge_t));
graph3->first_edge=malloc(nvertices*sizeof(cmph_uint32)); graph3->first_edge=malloc(nvertices*sizeof(cmph_uint32));
graph3->vert_degree=malloc(nvertices); graph3->vert_degree=malloc((size_t)nvertices);
}; };
static void bdz_init_graph3(bdz_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices) static void bdz_init_graph3(bdz_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices)
{ {
memset(graph3->first_edge,0xff,nvertices*sizeof(cmph_uint32)); memset(graph3->first_edge,0xff,nvertices*sizeof(cmph_uint32));
memset(graph3->vert_degree,0,nvertices); memset(graph3->vert_degree,0,(size_t)nvertices);
graph3->nedges=0; graph3->nedges=0;
}; };
static void bdz_free_graph3(bdz_graph3_t *graph3) static void bdz_free_graph3(bdz_graph3_t *graph3)
@ -161,8 +161,8 @@ static int bdz_generate_queue(cmph_uint32 nedges, cmph_uint32 nvertices, bdz_que
cmph_uint32 queue_head=0,queue_tail=0; cmph_uint32 queue_head=0,queue_tail=0;
cmph_uint32 curr_edge; cmph_uint32 curr_edge;
cmph_uint32 tmp_edge; cmph_uint32 tmp_edge;
cmph_uint8 * marked_edge =malloc((nedges >> 3) + 1); cmph_uint8 * marked_edge =malloc((size_t)(nedges >> 3) + 1);
memset(marked_edge, 0, (nedges >> 3) + 1); memset(marked_edge, 0, (size_t)(nedges >> 3) + 1);
for(i=0;i<nedges;i++){ for(i=0;i<nedges;i++){
v0=graph3->edges[i].vertices[0]; v0=graph3->edges[i].vertices[0];
@ -214,7 +214,7 @@ static int bdz_generate_queue(cmph_uint32 nedges, cmph_uint32 nvertices, bdz_que
static int bdz_mapping(cmph_config_t *mph, bdz_graph3_t* graph3, bdz_queue_t queue); static int bdz_mapping(cmph_config_t *mph, bdz_graph3_t* graph3, bdz_queue_t queue);
static void assigning(bdz_config_data_t *bdz, bdz_graph3_t* graph3, bdz_queue_t queue); static void assigning(bdz_config_data_t *bdz, bdz_graph3_t* graph3, bdz_queue_t queue);
static void ranking(bdz_config_data_t *bdz); static void ranking(bdz_config_data_t *bdz);
static cmph_uint32 rank(cmph_uint8 b, cmph_uint32 * ranktable, cmph_uint8 * g, cmph_uint32 vertex); static cmph_uint32 rank(cmph_uint32 b, cmph_uint32 * ranktable, cmph_uint8 * g, cmph_uint32 vertex);
bdz_config_data_t *bdz_config_new() bdz_config_data_t *bdz_config_new()
{ {
@ -239,7 +239,7 @@ void bdz_config_destroy(cmph_config_t *mph)
free(data); free(data);
} }
void bdz_config_set_b(cmph_config_t *mph, cmph_uint8 b) void bdz_config_set_b(cmph_config_t *mph, cmph_uint32 b)
{ {
bdz_config_data_t *bdz = (bdz_config_data_t *)mph->data; bdz_config_data_t *bdz = (bdz_config_data_t *)mph->data;
if (b <= 2 || b > 10) b = 7; // validating restrictions over parameter b. if (b <= 2 || b > 10) b = 7; // validating restrictions over parameter b.
@ -261,7 +261,7 @@ void bdz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
} }
cmph_t *bdz_new(cmph_config_t *mph, float c) cmph_t *bdz_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
bdz_data_t *bdzf = NULL; bdz_data_t *bdzf = NULL;
@ -278,7 +278,7 @@ cmph_t *bdz_new(cmph_config_t *mph, float c)
bdz->k = (1 << bdz->b); bdz->k = (1 << bdz->b);
DEBUGP("b: %u -- k: %u\n", bdz->b, bdz->k); DEBUGP("b: %u -- k: %u\n", bdz->b, bdz->k);
bdz->ranktablesize = bdz->n/bdz->k + 2; bdz->ranktablesize = ceil(bdz->n/(double)bdz->k);
DEBUGP("ranktablesize: %u\n", bdz->ranktablesize); DEBUGP("ranktablesize: %u\n", bdz->ranktablesize);
@ -398,10 +398,11 @@ static void assigning(bdz_config_data_t *bdz, bdz_graph3_t* graph3, bdz_queue_t
cmph_uint32 nedges=graph3->nedges; cmph_uint32 nedges=graph3->nedges;
cmph_uint32 curr_edge; cmph_uint32 curr_edge;
cmph_uint32 v0,v1,v2; cmph_uint32 v0,v1,v2;
cmph_uint8 * marked_vertices =malloc((bdz->n >> 3) + 1); cmph_uint8 * marked_vertices =malloc((size_t)(bdz->n >> 3) + 1);
bdz->g = (cmph_uint8 *)calloc((bdz->n >> 2)+1, sizeof(cmph_uint8)); cmph_uint32 sizeg = ceil(bdz->n/4.0);
memset(marked_vertices, 0, (bdz->n >> 3) + 1); bdz->g = (cmph_uint8 *)calloc((size_t)(sizeg), sizeof(cmph_uint8));
memset(bdz->g, 0xff, (bdz->n >> 2) + 1); memset(marked_vertices, 0, (size_t)(bdz->n >> 3) + 1);
memset(bdz->g, 0xff, (size_t)(sizeg));
for(i=nedges-1;i+1>=1;i--){ for(i=nedges-1;i+1>=1;i--){
curr_edge=queue[i]; curr_edge=queue[i];
@ -442,23 +443,21 @@ static void assigning(bdz_config_data_t *bdz, bdz_graph3_t* graph3, bdz_queue_t
static void ranking(bdz_config_data_t *bdz) static void ranking(bdz_config_data_t *bdz)
{ {
cmph_uint32 i, j, offset = 0, count = 0, size = (bdz->k >> 2), nbytes_total = (bdz->n >> 2)+1, nbytes; cmph_uint32 i, j, offset = 0, count = 0, size = (bdz->k >> 2), nbytes_total = ceil(bdz->n/4.0), nbytes;
bdz->ranktable = (cmph_uint32 *)calloc(bdz->ranktablesize, sizeof(cmph_uint32)); bdz->ranktable = (cmph_uint32 *)calloc((size_t)bdz->ranktablesize, sizeof(cmph_uint32));
// ranktable computation // ranktable computation
bdz->ranktable[0] = 0; bdz->ranktable[0] = 0;
i = 1; i = 1;
while(1) while(1)
{ {
if(i == bdz->ranktablesize) break;
nbytes = size < nbytes_total? size : nbytes_total; nbytes = size < nbytes_total? size : nbytes_total;
for(j = 0; j < nbytes; j++) for(j = 0; j < nbytes; j++)
{ {
count += bdz_lookup_table[*(bdz->g + offset + j)]; count += bdz_lookup_table[*(bdz->g + offset + j)];
} }
if(i == bdz->ranktablesize) fprintf(stderr, "i:%u == bdz->ranktablesize:%u\n", i, bdz->ranktablesize);
assert(i < bdz->ranktablesize);
bdz->ranktable[i] = count; bdz->ranktable[i] = count;
offset += nbytes; offset += nbytes;
if(size >= nbytes_total) break;
nbytes_total -= size; nbytes_total -= size;
i++; i++;
} }
@ -474,21 +473,22 @@ int bdz_dump(cmph_t *mphf, FILE *fd)
hash_state_dump(data->hl, &buf, &buflen); hash_state_dump(data->hl, &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
fwrite(&(data->n), sizeof(cmph_uint32), 1, fd); fwrite(&(data->n), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->m), sizeof(cmph_uint32), 1, fd); fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->r), sizeof(cmph_uint32), 1, fd); fwrite(&(data->r), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(data->g, sizeof(cmph_uint8)*((data->n >> 2) +1), 1, fd); cmph_uint32 sizeg = ceil(data->n/4.0);
fwrite(data->g, sizeof(cmph_uint8)*sizeg, (size_t)1, fd);
fwrite(&(data->k), sizeof(cmph_uint32), 1, fd); fwrite(&(data->k), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->b), sizeof(cmph_uint8), 1, fd); fwrite(&(data->b), sizeof(cmph_uint8), (size_t)1, fd);
fwrite(&(data->ranktablesize), sizeof(cmph_uint32), 1, fd); fwrite(&(data->ranktablesize), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(data->ranktable, sizeof(cmph_uint32)*(data->ranktablesize), 1, fd); fwrite(data->ranktable, sizeof(cmph_uint32)*(data->ranktablesize), (size_t)1, fd);
#ifdef DEBUG #ifdef DEBUG
cmph_uint32 i; cmph_uint32 i;
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
@ -501,34 +501,34 @@ int bdz_dump(cmph_t *mphf, FILE *fd)
void bdz_load(FILE *f, cmph_t *mphf) void bdz_load(FILE *f, cmph_t *mphf)
{ {
char *buf = NULL; char *buf = NULL;
cmph_uint32 buflen; cmph_uint32 buflen, sizeg;
bdz_data_t *bdz = (bdz_data_t *)malloc(sizeof(bdz_data_t)); bdz_data_t *bdz = (bdz_data_t *)malloc(sizeof(bdz_data_t));
DEBUGP("Loading bdz mphf\n"); DEBUGP("Loading bdz mphf\n");
mphf->data = bdz; mphf->data = bdz;
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state has %u bytes\n", buflen); DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
bdz->hl = hash_state_load(buf, buflen); bdz->hl = hash_state_load(buf, buflen);
free(buf); free(buf);
DEBUGP("Reading m and n\n"); DEBUGP("Reading m and n\n");
fread(&(bdz->n), sizeof(cmph_uint32), 1, f); fread(&(bdz->n), sizeof(cmph_uint32), (size_t)1, f);
fread(&(bdz->m), sizeof(cmph_uint32), 1, f); fread(&(bdz->m), sizeof(cmph_uint32), (size_t)1, f);
fread(&(bdz->r), sizeof(cmph_uint32), 1, f); fread(&(bdz->r), sizeof(cmph_uint32), (size_t)1, f);
sizeg = ceil(bdz->n/4.0);
bdz->g = (cmph_uint8 *)calloc((size_t)(sizeg), sizeof(cmph_uint8));
fread(bdz->g, sizeg*sizeof(cmph_uint8), (size_t)1, f);
bdz->g = (cmph_uint8 *)calloc((bdz->n >> 2) + 1, sizeof(cmph_uint8)); fread(&(bdz->k), sizeof(cmph_uint32), (size_t)1, f);
fread(bdz->g, ((bdz->n >> 2) + 1)*sizeof(cmph_uint8), 1, f); fread(&(bdz->b), sizeof(cmph_uint8), (size_t)1, f);
fread(&(bdz->ranktablesize), sizeof(cmph_uint32), (size_t)1, f);
fread(&(bdz->k), sizeof(cmph_uint32), 1, f); bdz->ranktable = (cmph_uint32 *)calloc((size_t)bdz->ranktablesize, sizeof(cmph_uint32));
fread(&(bdz->b), sizeof(cmph_uint8), 1, f); fread(bdz->ranktable, sizeof(cmph_uint32)*(bdz->ranktablesize), (size_t)1, f);
fread(&(bdz->ranktablesize), sizeof(cmph_uint32), 1, f);
bdz->ranktable = (cmph_uint32 *)calloc(bdz->ranktablesize, sizeof(cmph_uint32));
fread(bdz->ranktable, sizeof(cmph_uint32)*(bdz->ranktablesize), 1, f);
#ifdef DEBUG #ifdef DEBUG
cmph_uint32 i = 0; cmph_uint32 i = 0;
@ -553,7 +553,7 @@ cmph_uint32 bdz_search_ph(cmph_t *mphf, const char *key, cmph_uint32 keylen)
return vertex; return vertex;
} }
static inline cmph_uint32 rank(cmph_uint8 b, cmph_uint32 * ranktable, cmph_uint8 * g, cmph_uint32 vertex) static inline cmph_uint32 rank(cmph_uint32 b, cmph_uint32 * ranktable, cmph_uint8 * g, cmph_uint32 vertex)
{ {
register cmph_uint32 index = vertex >> b; register cmph_uint32 index = vertex >> b;
register cmph_uint32 base_rank = ranktable[index]; register cmph_uint32 base_rank = ranktable[index];
@ -661,7 +661,8 @@ void bdz_pack(cmph_t *mphf, void *packed_mphf)
*ptr++ = data->b; *ptr++ = data->b;
// packing g // packing g
memcpy(ptr, data->g, sizeof(cmph_uint8)*((data->n >> 2) +1)); cmph_uint32 sizeg = ceil(data->n/4.0);
memcpy(ptr, data->g, sizeof(cmph_uint8)*sizeg);
} }
/** \fn cmph_uint32 bdz_packed_size(cmph_t *mphf); /** \fn cmph_uint32 bdz_packed_size(cmph_t *mphf);
@ -675,7 +676,7 @@ cmph_uint32 bdz_packed_size(cmph_t *mphf)
CMPH_HASH hl_type = hash_get_type(data->hl); CMPH_HASH hl_type = hash_get_type(data->hl);
return (sizeof(CMPH_ALGO) + hash_state_packed_size(hl_type) + 3*sizeof(cmph_uint32) + sizeof(cmph_uint32)*(data->ranktablesize) + sizeof(cmph_uint8) + sizeof(cmph_uint8)*((data->n >> 2) +1)); return (sizeof(CMPH_ALGO) + hash_state_packed_size(hl_type) + 3*sizeof(cmph_uint32) + sizeof(cmph_uint32)*(data->ranktablesize) + sizeof(cmph_uint8) + sizeof(cmph_uint8)*(ceil(data->n/4.0)));
} }
/** cmph_uint32 bdz_search(void *packed_mphf, const char *key, cmph_uint32 keylen); /** cmph_uint32 bdz_search(void *packed_mphf, const char *key, cmph_uint32 keylen);
@ -690,7 +691,7 @@ cmph_uint32 bdz_search_packed(void *packed_mphf, const char *key, cmph_uint32 ke
register cmph_uint32 vertex; register cmph_uint32 vertex;
register CMPH_HASH hl_type = *(cmph_uint32 *)packed_mphf; register CMPH_HASH hl_type = *(cmph_uint32 *)packed_mphf;
register cmph_uint8 *hl_ptr = (cmph_uint8 *)(packed_mphf + 4); register cmph_uint8 *hl_ptr = (cmph_uint8 *)(packed_mphf) + 4;
register cmph_uint32 *ranktable = (cmph_uint32*)(hl_ptr + hash_state_packed_size(hl_type)); register cmph_uint32 *ranktable = (cmph_uint32*)(hl_ptr + hash_state_packed_size(hl_type));

4
src/bdz.h
View File

@ -9,8 +9,8 @@ typedef struct __bdz_config_data_t bdz_config_data_t;
bdz_config_data_t *bdz_config_new(); bdz_config_data_t *bdz_config_new();
void bdz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void bdz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void bdz_config_destroy(cmph_config_t *mph); void bdz_config_destroy(cmph_config_t *mph);
void bdz_config_set_b(cmph_config_t *mph, cmph_uint8 b); void bdz_config_set_b(cmph_config_t *mph, cmph_uint32 b);
cmph_t *bdz_new(cmph_config_t *mph, float c); cmph_t *bdz_new(cmph_config_t *mph, double c);
void bdz_load(FILE *f, cmph_t *mphf); void bdz_load(FILE *f, cmph_t *mphf);
int bdz_dump(cmph_t *mphf, FILE *f); int bdz_dump(cmph_t *mphf, FILE *f);

65
src/bdz_ph.c
View File

@ -54,12 +54,12 @@ static void bdz_ph_alloc_graph3(bdz_ph_graph3_t * graph3, cmph_uint32 nedges, cm
{ {
graph3->edges=malloc(nedges*sizeof(bdz_ph_edge_t)); graph3->edges=malloc(nedges*sizeof(bdz_ph_edge_t));
graph3->first_edge=malloc(nvertices*sizeof(cmph_uint32)); graph3->first_edge=malloc(nvertices*sizeof(cmph_uint32));
graph3->vert_degree=malloc(nvertices); graph3->vert_degree=malloc((size_t)nvertices);
}; };
static void bdz_ph_init_graph3(bdz_ph_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices) static void bdz_ph_init_graph3(bdz_ph_graph3_t * graph3, cmph_uint32 nedges, cmph_uint32 nvertices)
{ {
memset(graph3->first_edge,0xff,nvertices*sizeof(cmph_uint32)); memset(graph3->first_edge,0xff,nvertices*sizeof(cmph_uint32));
memset(graph3->vert_degree,0,nvertices); memset(graph3->vert_degree,0,(size_t)nvertices);
graph3->nedges=0; graph3->nedges=0;
}; };
static void bdz_ph_free_graph3(bdz_ph_graph3_t *graph3) static void bdz_ph_free_graph3(bdz_ph_graph3_t *graph3)
@ -148,8 +148,8 @@ static int bdz_ph_generate_queue(cmph_uint32 nedges, cmph_uint32 nvertices, bdz_
cmph_uint32 queue_head=0,queue_tail=0; cmph_uint32 queue_head=0,queue_tail=0;
cmph_uint32 curr_edge; cmph_uint32 curr_edge;
cmph_uint32 tmp_edge; cmph_uint32 tmp_edge;
cmph_uint8 * marked_edge =malloc((nedges >> 3) + 1); cmph_uint8 * marked_edge =malloc((size_t)(nedges >> 3) + 1);
memset(marked_edge, 0, (nedges >> 3) + 1); memset(marked_edge, 0, (size_t)(nedges >> 3) + 1);
for(i=0;i<nedges;i++){ for(i=0;i<nedges;i++){
v0=graph3->edges[i].vertices[0]; v0=graph3->edges[i].vertices[0];
@ -234,7 +234,7 @@ void bdz_ph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
} }
cmph_t *bdz_ph_new(cmph_config_t *mph, float c) cmph_t *bdz_ph_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
bdz_ph_data_t *bdz_phf = NULL; bdz_ph_data_t *bdz_phf = NULL;
@ -364,10 +364,11 @@ static void assigning(bdz_ph_config_data_t *bdz_ph, bdz_ph_graph3_t* graph3, bdz
cmph_uint32 nedges=graph3->nedges; cmph_uint32 nedges=graph3->nedges;
cmph_uint32 curr_edge; cmph_uint32 curr_edge;
cmph_uint32 v0,v1,v2; cmph_uint32 v0,v1,v2;
cmph_uint8 * marked_vertices =malloc((bdz_ph->n >> 3) + 1); cmph_uint8 * marked_vertices =malloc((size_t)(bdz_ph->n >> 3) + 1);
bdz_ph->g = (cmph_uint8 *)calloc((bdz_ph->n >> 2)+1, sizeof(cmph_uint8)); cmph_uint32 sizeg = ceil(bdz_ph->n/4.0);
memset(marked_vertices, 0, (bdz_ph->n >> 3) + 1); bdz_ph->g = (cmph_uint8 *)calloc((size_t)sizeg, sizeof(cmph_uint8));
//memset(bdz_ph->g, 0xff, (bdz_ph->n >> 2) + 1); memset(marked_vertices, 0, (size_t)(bdz_ph->n >> 3) + 1);
//memset(bdz_ph->g, 0xff, sizeg);
for(i=nedges-1;i+1>=1;i--){ for(i=nedges-1;i+1>=1;i--){
curr_edge=queue[i]; curr_edge=queue[i];
@ -409,7 +410,8 @@ static void bdz_ph_optimization(bdz_ph_config_data_t *bdz_ph)
{ {
cmph_uint32 i; cmph_uint32 i;
cmph_uint8 byte = 0; cmph_uint8 byte = 0;
cmph_uint8 * new_g = (cmph_uint8 *)calloc((bdz_ph->n/5)+1, sizeof(cmph_uint8)); cmph_uint32 sizeg = ceil(bdz_ph->n/5.0);
cmph_uint8 * new_g = (cmph_uint8 *)calloc((size_t)sizeg, sizeof(cmph_uint8));
cmph_uint8 value; cmph_uint8 value;
cmph_uint32 idx; cmph_uint32 idx;
for(i = 0; i < bdz_ph->n; i++) for(i = 0; i < bdz_ph->n; i++)
@ -429,20 +431,21 @@ int bdz_ph_dump(cmph_t *mphf, FILE *fd)
{ {
char *buf = NULL; char *buf = NULL;
cmph_uint32 buflen; cmph_uint32 buflen;
cmph_uint32 sizeg = 0;
bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data; bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data;
__cmph_dump(mphf, fd); __cmph_dump(mphf, fd);
hash_state_dump(data->hl, &buf, &buflen); hash_state_dump(data->hl, &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
fwrite(&(data->n), sizeof(cmph_uint32), 1, fd); fwrite(&(data->n), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->m), sizeof(cmph_uint32), 1, fd); fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->r), sizeof(cmph_uint32), 1, fd); fwrite(&(data->r), sizeof(cmph_uint32), (size_t)1, fd);
sizeg = ceil(data->n/5.0);
fwrite(data->g, sizeof(cmph_uint8)*((data->n/5)+1), 1, fd); fwrite(data->g, sizeof(cmph_uint8)*sizeg, (size_t)1, fd);
#ifdef DEBUG #ifdef DEBUG
cmph_uint32 i; cmph_uint32 i;
@ -457,26 +460,27 @@ void bdz_ph_load(FILE *f, cmph_t *mphf)
{ {
char *buf = NULL; char *buf = NULL;
cmph_uint32 buflen; cmph_uint32 buflen;
cmph_uint32 sizeg = 0;
bdz_ph_data_t *bdz_ph = (bdz_ph_data_t *)malloc(sizeof(bdz_ph_data_t)); bdz_ph_data_t *bdz_ph = (bdz_ph_data_t *)malloc(sizeof(bdz_ph_data_t));
DEBUGP("Loading bdz_ph mphf\n"); DEBUGP("Loading bdz_ph mphf\n");
mphf->data = bdz_ph; mphf->data = bdz_ph;
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state has %u bytes\n", buflen); DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
bdz_ph->hl = hash_state_load(buf, buflen); bdz_ph->hl = hash_state_load(buf, buflen);
free(buf); free(buf);
DEBUGP("Reading m and n\n"); DEBUGP("Reading m and n\n");
fread(&(bdz_ph->n), sizeof(cmph_uint32), 1, f); fread(&(bdz_ph->n), sizeof(cmph_uint32), (size_t)1, f);
fread(&(bdz_ph->m), sizeof(cmph_uint32), 1, f); fread(&(bdz_ph->m), sizeof(cmph_uint32), (size_t)1, f);
fread(&(bdz_ph->r), sizeof(cmph_uint32), 1, f); fread(&(bdz_ph->r), sizeof(cmph_uint32), (size_t)1, f);
sizeg = ceil(bdz_ph->n/5.0);
bdz_ph->g = (cmph_uint8 *)calloc((bdz_ph->n/5)+1, sizeof(cmph_uint8)); bdz_ph->g = (cmph_uint8 *)calloc((size_t)sizeg, sizeof(cmph_uint8));
fread(bdz_ph->g, ((bdz_ph->n/5)+1)*sizeof(cmph_uint8), 1, f); fread(bdz_ph->g, sizeg*sizeof(cmph_uint8), (size_t)1, f);
/* #ifdef DEBUG /* #ifdef DEBUG
cmph_uint32 i; cmph_uint32 i;
@ -585,7 +589,8 @@ void bdz_ph_pack(cmph_t *mphf, void *packed_mphf)
ptr += sizeof(data->r); ptr += sizeof(data->r);
// packing g // packing g
memcpy(ptr, data->g, sizeof(cmph_uint8)*((data->n/5)+1)); cmph_uint32 sizeg = ceil(data->n/5.0);
memcpy(ptr, data->g, sizeof(cmph_uint8)*sizeg);
} }
/** \fn cmph_uint32 bdz_ph_packed_size(cmph_t *mphf); /** \fn cmph_uint32 bdz_ph_packed_size(cmph_t *mphf);
@ -597,8 +602,8 @@ cmph_uint32 bdz_ph_packed_size(cmph_t *mphf)
{ {
bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data; bdz_ph_data_t *data = (bdz_ph_data_t *)mphf->data;
CMPH_HASH hl_type = hash_get_type(data->hl); CMPH_HASH hl_type = hash_get_type(data->hl);
cmph_uint32 sizeg = ceil(data->n/5.0);
return (sizeof(CMPH_ALGO) + hash_state_packed_size(hl_type) + 2*sizeof(cmph_uint32) + sizeof(cmph_uint8)*((data->n/5)+1)); return (sizeof(CMPH_ALGO) + hash_state_packed_size(hl_type) + 2*sizeof(cmph_uint32) + sizeof(cmph_uint8)*sizeg);
} }
/** cmph_uint32 bdz_ph_search(void *packed_mphf, const char *key, cmph_uint32 keylen); /** cmph_uint32 bdz_ph_search(void *packed_mphf, const char *key, cmph_uint32 keylen);
@ -612,7 +617,7 @@ cmph_uint32 bdz_ph_search_packed(void *packed_mphf, const char *key, cmph_uint32
{ {
register CMPH_HASH hl_type = *(cmph_uint32 *)packed_mphf; register CMPH_HASH hl_type = *(cmph_uint32 *)packed_mphf;
register cmph_uint8 *hl_ptr = (cmph_uint8 *)(packed_mphf + 4); register cmph_uint8 *hl_ptr = (cmph_uint8 *)(packed_mphf) + 4;
register cmph_uint8 * ptr = hl_ptr + hash_state_packed_size(hl_type); register cmph_uint8 * ptr = hl_ptr + hash_state_packed_size(hl_type);

2
src/bdz_ph.h
View File

@ -9,7 +9,7 @@ typedef struct __bdz_ph_config_data_t bdz_ph_config_data_t;
bdz_ph_config_data_t *bdz_ph_config_new(); bdz_ph_config_data_t *bdz_ph_config_new();
void bdz_ph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void bdz_ph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void bdz_ph_config_destroy(cmph_config_t *mph); void bdz_ph_config_destroy(cmph_config_t *mph);
cmph_t *bdz_ph_new(cmph_config_t *mph, float c); cmph_t *bdz_ph_new(cmph_config_t *mph, double c);
void bdz_ph_load(FILE *f, cmph_t *mphf); void bdz_ph_load(FILE *f, cmph_t *mphf);
int bdz_ph_dump(cmph_t *mphf, FILE *f); int bdz_ph_dump(cmph_t *mphf, FILE *f);

42
src/bmz.c
View File

@ -54,7 +54,7 @@ void bmz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
} }
cmph_t *bmz_new(cmph_config_t *mph, float c) cmph_t *bmz_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
bmz_data_t *bmzf = NULL; bmz_data_t *bmzf = NULL;
@ -131,12 +131,12 @@ cmph_t *bmz_new(cmph_config_t *mph, float c)
fprintf(stderr, "\tTraversing critical vertices.\n"); fprintf(stderr, "\tTraversing critical vertices.\n");
} }
DEBUGP("Searching step\n"); DEBUGP("Searching step\n");
visited = (cmph_uint8 *)malloc(bmz->n/8 + 1); visited = (cmph_uint8 *)malloc((size_t)bmz->n/8 + 1);
memset(visited, 0, bmz->n/8 + 1); memset(visited, 0, (size_t)bmz->n/8 + 1);
used_edges = (cmph_uint8 *)malloc(bmz->m/8 + 1); used_edges = (cmph_uint8 *)malloc((size_t)bmz->m/8 + 1);
memset(used_edges, 0, bmz->m/8 + 1); memset(used_edges, 0, (size_t)bmz->m/8 + 1);
free(bmz->g); free(bmz->g);
bmz->g = (cmph_uint32 *)calloc(bmz->n, sizeof(cmph_uint32)); bmz->g = (cmph_uint32 *)calloc((size_t)bmz->n, sizeof(cmph_uint32));
assert(bmz->g); assert(bmz->g);
for (i = 0; i < bmz->n; ++i) // critical nodes for (i = 0; i < bmz->n; ++i) // critical nodes
{ {
@ -450,24 +450,24 @@ int bmz_dump(cmph_t *mphf, FILE *fd)
bmz_data_t *data = (bmz_data_t *)mphf->data; bmz_data_t *data = (bmz_data_t *)mphf->data;
__cmph_dump(mphf, fd); __cmph_dump(mphf, fd);
fwrite(&two, sizeof(cmph_uint32), 1, fd); fwrite(&two, sizeof(cmph_uint32), (size_t)1, fd);
hash_state_dump(data->hashes[0], &buf, &buflen); hash_state_dump(data->hashes[0], &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
hash_state_dump(data->hashes[1], &buf, &buflen); hash_state_dump(data->hashes[1], &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
fwrite(&(data->n), sizeof(cmph_uint32), 1, fd); fwrite(&(data->n), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->m), sizeof(cmph_uint32), 1, fd); fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(data->g, sizeof(cmph_uint32)*(data->n), 1, fd); fwrite(data->g, sizeof(cmph_uint32)*(data->n), (size_t)1, fd);
#ifdef DEBUG #ifdef DEBUG
cmph_uint32 i; cmph_uint32 i;
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
@ -487,28 +487,28 @@ void bmz_load(FILE *f, cmph_t *mphf)
DEBUGP("Loading bmz mphf\n"); DEBUGP("Loading bmz mphf\n");
mphf->data = bmz; mphf->data = bmz;
fread(&nhashes, sizeof(cmph_uint32), 1, f); fread(&nhashes, sizeof(cmph_uint32), (size_t)1, f);
bmz->hashes = (hash_state_t **)malloc(sizeof(hash_state_t *)*(nhashes + 1)); bmz->hashes = (hash_state_t **)malloc(sizeof(hash_state_t *)*(nhashes + 1));
bmz->hashes[nhashes] = NULL; bmz->hashes[nhashes] = NULL;
DEBUGP("Reading %u hashes\n", nhashes); DEBUGP("Reading %u hashes\n", nhashes);
for (i = 0; i < nhashes; ++i) for (i = 0; i < nhashes; ++i)
{ {
hash_state_t *state = NULL; hash_state_t *state = NULL;
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state has %u bytes\n", buflen); DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
state = hash_state_load(buf, buflen); state = hash_state_load(buf, buflen);
bmz->hashes[i] = state; bmz->hashes[i] = state;
free(buf); free(buf);
} }
DEBUGP("Reading m and n\n"); DEBUGP("Reading m and n\n");
fread(&(bmz->n), sizeof(cmph_uint32), 1, f); fread(&(bmz->n), sizeof(cmph_uint32), (size_t)1, f);
fread(&(bmz->m), sizeof(cmph_uint32), 1, f); fread(&(bmz->m), sizeof(cmph_uint32), (size_t)1, f);
bmz->g = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*bmz->n); bmz->g = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*bmz->n);
fread(bmz->g, bmz->n*sizeof(cmph_uint32), 1, f); fread(bmz->g, bmz->n*sizeof(cmph_uint32), (size_t)1, f);
#ifdef DEBUG #ifdef DEBUG
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
for (i = 0; i < bmz->n; ++i) fprintf(stderr, "%u ", bmz->g[i]); for (i = 0; i < bmz->n; ++i) fprintf(stderr, "%u ", bmz->g[i]);

2
src/bmz.h
View File

@ -9,7 +9,7 @@ typedef struct __bmz_config_data_t bmz_config_data_t;
bmz_config_data_t *bmz_config_new(); bmz_config_data_t *bmz_config_new();
void bmz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void bmz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void bmz_config_destroy(cmph_config_t *mph); void bmz_config_destroy(cmph_config_t *mph);
cmph_t *bmz_new(cmph_config_t *mph, float c); cmph_t *bmz_new(cmph_config_t *mph, double c);
void bmz_load(FILE *f, cmph_t *mphf); void bmz_load(FILE *f, cmph_t *mphf);
int bmz_dump(cmph_t *mphf, FILE *f); int bmz_dump(cmph_t *mphf, FILE *f);

54
src/bmz8.c
View File

@ -15,8 +15,8 @@
#include "debug.h" #include "debug.h"
static int bmz8_gen_edges(cmph_config_t *mph); static int bmz8_gen_edges(cmph_config_t *mph);
static cmph_uint8 bmz8_traverse_critical_nodes(bmz8_config_data_t *bmz8, cmph_uint8 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited); static cmph_uint8 bmz8_traverse_critical_nodes(bmz8_config_data_t *bmz8, cmph_uint32 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited);
static cmph_uint8 bmz8_traverse_critical_nodes_heuristic(bmz8_config_data_t *bmz8, cmph_uint8 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited); static cmph_uint8 bmz8_traverse_critical_nodes_heuristic(bmz8_config_data_t *bmz8, cmph_uint32 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited);
static void bmz8_traverse_non_critical_nodes(bmz8_config_data_t *bmz8, cmph_uint8 * used_edges, cmph_uint8 * visited); static void bmz8_traverse_non_critical_nodes(bmz8_config_data_t *bmz8, cmph_uint8 * used_edges, cmph_uint8 * visited);
bmz8_config_data_t *bmz8_config_new() bmz8_config_data_t *bmz8_config_new()
@ -53,7 +53,7 @@ void bmz8_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
} }
cmph_t *bmz8_new(cmph_config_t *mph, float c) cmph_t *bmz8_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
bmz8_data_t *bmz8f = NULL; bmz8_data_t *bmz8f = NULL;
@ -137,12 +137,12 @@ cmph_t *bmz8_new(cmph_config_t *mph, float c)
fprintf(stderr, "\tTraversing critical vertices.\n"); fprintf(stderr, "\tTraversing critical vertices.\n");
} }
DEBUGP("Searching step\n"); DEBUGP("Searching step\n");
visited = (cmph_uint8 *)malloc(bmz8->n/8 + 1); visited = (cmph_uint8 *)malloc((size_t)bmz8->n/8 + 1);
memset(visited, 0, bmz8->n/8 + 1); memset(visited, 0, (size_t)bmz8->n/8 + 1);
used_edges = (cmph_uint8 *)malloc(bmz8->m/8 + 1); used_edges = (cmph_uint8 *)malloc((size_t)bmz8->m/8 + 1);
memset(used_edges, 0, bmz8->m/8 + 1); memset(used_edges, 0, (size_t)bmz8->m/8 + 1);
free(bmz8->g); free(bmz8->g);
bmz8->g = (cmph_uint8 *)calloc(bmz8->n, sizeof(cmph_uint8)); bmz8->g = (cmph_uint8 *)calloc((size_t)bmz8->n, sizeof(cmph_uint8));
assert(bmz8->g); assert(bmz8->g);
for (i = 0; i < bmz8->n; ++i) // critical nodes for (i = 0; i < bmz8->n; ++i) // critical nodes
{ {
@ -196,7 +196,7 @@ cmph_t *bmz8_new(cmph_config_t *mph, float c)
return mphf; return mphf;
} }
static cmph_uint8 bmz8_traverse_critical_nodes(bmz8_config_data_t *bmz8, cmph_uint8 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited) static cmph_uint8 bmz8_traverse_critical_nodes(bmz8_config_data_t *bmz8, cmph_uint32 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited)
{ {
cmph_uint8 next_g; cmph_uint8 next_g;
cmph_uint32 u; /* Auxiliary vertex */ cmph_uint32 u; /* Auxiliary vertex */
@ -263,7 +263,7 @@ static cmph_uint8 bmz8_traverse_critical_nodes(bmz8_config_data_t *bmz8, cmph_ui
return 0; return 0;
} }
static cmph_uint8 bmz8_traverse_critical_nodes_heuristic(bmz8_config_data_t *bmz8, cmph_uint8 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited) static cmph_uint8 bmz8_traverse_critical_nodes_heuristic(bmz8_config_data_t *bmz8, cmph_uint32 v, cmph_uint8 * biggest_g_value, cmph_uint8 * biggest_edge_value, cmph_uint8 * used_edges, cmph_uint8 * visited)
{ {
cmph_uint8 next_g; cmph_uint8 next_g;
cmph_uint32 u; cmph_uint32 u;
@ -360,7 +360,7 @@ static cmph_uint8 bmz8_traverse_critical_nodes_heuristic(bmz8_config_data_t *bmz
return 0; return 0;
} }
static cmph_uint8 next_unused_edge(bmz8_config_data_t *bmz8, cmph_uint8 * used_edges, cmph_uint8 unused_edge_index) static cmph_uint8 next_unused_edge(bmz8_config_data_t *bmz8, cmph_uint8 * used_edges, cmph_uint32 unused_edge_index)
{ {
while(1) while(1)
{ {
@ -371,7 +371,7 @@ static cmph_uint8 next_unused_edge(bmz8_config_data_t *bmz8, cmph_uint8 * used_e
return unused_edge_index; return unused_edge_index;
} }
static void bmz8_traverse(bmz8_config_data_t *bmz8, cmph_uint8 * used_edges, cmph_uint8 v, cmph_uint8 * unused_edge_index, cmph_uint8 * visited) static void bmz8_traverse(bmz8_config_data_t *bmz8, cmph_uint8 * used_edges, cmph_uint32 v, cmph_uint8 * unused_edge_index, cmph_uint8 * visited)
{ {
graph_iterator_t it = graph_neighbors_it(bmz8->graph, v); graph_iterator_t it = graph_neighbors_it(bmz8->graph, v);
cmph_uint32 neighbor = 0; cmph_uint32 neighbor = 0;
@ -460,24 +460,24 @@ int bmz8_dump(cmph_t *mphf, FILE *fd)
bmz8_data_t *data = (bmz8_data_t *)mphf->data; bmz8_data_t *data = (bmz8_data_t *)mphf->data;
__cmph_dump(mphf, fd); __cmph_dump(mphf, fd);
fwrite(&two, sizeof(cmph_uint8), 1, fd); fwrite(&two, sizeof(cmph_uint8), (size_t)1, fd);
hash_state_dump(data->hashes[0], &buf, &buflen); hash_state_dump(data->hashes[0], &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
hash_state_dump(data->hashes[1], &buf, &buflen); hash_state_dump(data->hashes[1], &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
fwrite(&(data->n), sizeof(cmph_uint8), 1, fd); fwrite(&(data->n), sizeof(cmph_uint8), (size_t)1, fd);
fwrite(&(data->m), sizeof(cmph_uint8), 1, fd); fwrite(&(data->m), sizeof(cmph_uint8), (size_t)1, fd);
fwrite(data->g, sizeof(cmph_uint8)*(data->n), 1, fd); fwrite(data->g, sizeof(cmph_uint8)*(data->n), (size_t)1, fd);
#ifdef DEBUG #ifdef DEBUG
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
for (i = 0; i < data->n; ++i) fprintf(stderr, "%u ", data->g[i]); for (i = 0; i < data->n; ++i) fprintf(stderr, "%u ", data->g[i]);
@ -496,28 +496,28 @@ void bmz8_load(FILE *f, cmph_t *mphf)
DEBUGP("Loading bmz8 mphf\n"); DEBUGP("Loading bmz8 mphf\n");
mphf->data = bmz8; mphf->data = bmz8;
fread(&nhashes, sizeof(cmph_uint8), 1, f); fread(&nhashes, sizeof(cmph_uint8), (size_t)1, f);
bmz8->hashes = (hash_state_t **)malloc(sizeof(hash_state_t *)*(nhashes + 1)); bmz8->hashes = (hash_state_t **)malloc(sizeof(hash_state_t *)*(nhashes + 1));
bmz8->hashes[nhashes] = NULL; bmz8->hashes[nhashes] = NULL;
DEBUGP("Reading %u hashes\n", nhashes); DEBUGP("Reading %u hashes\n", nhashes);
for (i = 0; i < nhashes; ++i) for (i = 0; i < nhashes; ++i)
{ {
hash_state_t *state = NULL; hash_state_t *state = NULL;
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state has %u bytes\n", buflen); DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
state = hash_state_load(buf, buflen); state = hash_state_load(buf, buflen);
bmz8->hashes[i] = state; bmz8->hashes[i] = state;
free(buf); free(buf);
} }
DEBUGP("Reading m and n\n"); DEBUGP("Reading m and n\n");
fread(&(bmz8->n), sizeof(cmph_uint8), 1, f); fread(&(bmz8->n), sizeof(cmph_uint8), (size_t)1, f);
fread(&(bmz8->m), sizeof(cmph_uint8), 1, f); fread(&(bmz8->m), sizeof(cmph_uint8), (size_t)1, f);
bmz8->g = (cmph_uint8 *)malloc(sizeof(cmph_uint8)*bmz8->n); bmz8->g = (cmph_uint8 *)malloc(sizeof(cmph_uint8)*bmz8->n);
fread(bmz8->g, bmz8->n*sizeof(cmph_uint8), 1, f); fread(bmz8->g, bmz8->n*sizeof(cmph_uint8), (size_t)1, f);
#ifdef DEBUG #ifdef DEBUG
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
for (i = 0; i < bmz8->n; ++i) fprintf(stderr, "%u ", bmz8->g[i]); for (i = 0; i < bmz8->n; ++i) fprintf(stderr, "%u ", bmz8->g[i]);

2
src/bmz8.h
View File

@ -9,7 +9,7 @@ typedef struct __bmz8_config_data_t bmz8_config_data_t;
bmz8_config_data_t *bmz8_config_new(); bmz8_config_data_t *bmz8_config_new();
void bmz8_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void bmz8_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void bmz8_config_destroy(cmph_config_t *mph); void bmz8_config_destroy(cmph_config_t *mph);
cmph_t *bmz8_new(cmph_config_t *mph, float c); cmph_t *bmz8_new(cmph_config_t *mph, double c);
void bmz8_load(FILE *f, cmph_t *mphf); void bmz8_load(FILE *f, cmph_t *mphf);
int bmz8_dump(cmph_t *mphf, FILE *f); int bmz8_dump(cmph_t *mphf, FILE *f);

128
src/brz.c
View File

@ -21,7 +21,7 @@
static int brz_gen_mphf(cmph_config_t *mph); static int brz_gen_mphf(cmph_config_t *mph);
static cmph_uint32 brz_min_index(cmph_uint32 * vector, cmph_uint32 n); static cmph_uint32 brz_min_index(cmph_uint32 * vector, cmph_uint32 n);
static void brz_destroy_keys_vd(cmph_uint8 ** keys_vd, cmph_uint8 nkeys); static void brz_destroy_keys_vd(cmph_uint8 ** keys_vd, cmph_uint32 nkeys);
static char * brz_copy_partial_fch_mphf(brz_config_data_t *brz, fch_data_t * fchf, cmph_uint32 index, cmph_uint32 *buflen); static char * brz_copy_partial_fch_mphf(brz_config_data_t *brz, fch_data_t * fchf, cmph_uint32 index, cmph_uint32 *buflen);
static char * brz_copy_partial_bmz8_mphf(brz_config_data_t *brz, bmz8_data_t * bmzf, cmph_uint32 index, cmph_uint32 *buflen); static char * brz_copy_partial_bmz8_mphf(brz_config_data_t *brz, bmz8_data_t * bmzf, cmph_uint32 index, cmph_uint32 *buflen);
brz_config_data_t *brz_config_new() brz_config_data_t *brz_config_new()
@ -40,7 +40,7 @@ brz_config_data_t *brz_config_new()
brz->h2 = NULL; brz->h2 = NULL;
brz->h0 = NULL; brz->h0 = NULL;
brz->memory_availability = 1024*1024; brz->memory_availability = 1024*1024;
brz->tmp_dir = (cmph_uint8 *)calloc(10, sizeof(cmph_uint8)); brz->tmp_dir = (cmph_uint8 *)calloc((size_t)10, sizeof(cmph_uint8));
brz->mphf_fd = NULL; brz->mphf_fd = NULL;
strcpy((char *)(brz->tmp_dir), "/var/tmp/"); strcpy((char *)(brz->tmp_dir), "/var/tmp/");
assert(brz); assert(brz);
@ -83,12 +83,12 @@ void brz_config_set_tmp_dir(cmph_config_t *mph, cmph_uint8 *tmp_dir)
free(brz->tmp_dir); free(brz->tmp_dir);
if(tmp_dir[len-1] != '/') if(tmp_dir[len-1] != '/')
{ {
brz->tmp_dir = (cmph_uint8 *)calloc(len+2, sizeof(cmph_uint8)); brz->tmp_dir = (cmph_uint8 *)calloc((size_t)len+2, sizeof(cmph_uint8));
sprintf((char *)(brz->tmp_dir), "%s/", (char *)tmp_dir); sprintf((char *)(brz->tmp_dir), "%s/", (char *)tmp_dir);
} }
else else
{ {
brz->tmp_dir = (cmph_uint8 *)calloc(len+1, sizeof(cmph_uint8)); brz->tmp_dir = (cmph_uint8 *)calloc((size_t)len+1, sizeof(cmph_uint8));
sprintf((char *)(brz->tmp_dir), "%s", (char *)tmp_dir); sprintf((char *)(brz->tmp_dir), "%s", (char *)tmp_dir);
} }
@ -102,7 +102,7 @@ void brz_config_set_mphf_fd(cmph_config_t *mph, FILE *mphf_fd)
assert(brz->mphf_fd); assert(brz->mphf_fd);
} }
void brz_config_set_b(cmph_config_t *mph, cmph_uint8 b) void brz_config_set_b(cmph_config_t *mph, cmph_uint32 b)
{ {
brz_config_data_t *brz = (brz_config_data_t *)mph->data; brz_config_data_t *brz = (brz_config_data_t *)mph->data;
brz->b = b; brz->b = b;
@ -117,7 +117,7 @@ void brz_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo)
} }
} }
cmph_t *brz_new(cmph_config_t *mph, float c) cmph_t *brz_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
brz_data_t *brzf = NULL; brz_data_t *brzf = NULL;
@ -140,9 +140,9 @@ cmph_t *brz_new(cmph_config_t *mph, float c)
brz->c = c; brz->c = c;
brz->m = mph->key_source->nkeys; brz->m = mph->key_source->nkeys;
DEBUGP("m: %u\n", brz->m); DEBUGP("m: %u\n", brz->m);
brz->k = (cmph_uint32)ceil(brz->m/((float)brz->b)); brz->k = (cmph_uint32)ceil(brz->m/((double)brz->b));
DEBUGP("k: %u\n", brz->k); DEBUGP("k: %u\n", brz->k);
brz->size = (cmph_uint8 *) calloc(brz->k, sizeof(cmph_uint8)); brz->size = (cmph_uint8 *) calloc((size_t)brz->k, sizeof(cmph_uint8));
// Clustering the keys by graph id. // Clustering the keys by graph id.
if (mph->verbosity) if (mph->verbosity)
@ -179,7 +179,7 @@ cmph_t *brz_new(cmph_config_t *mph, float c)
} }
DEBUGP("Graphs generated\n"); DEBUGP("Graphs generated\n");
brz->offset = (cmph_uint32 *)calloc(brz->k, sizeof(cmph_uint32)); brz->offset = (cmph_uint32 *)calloc((size_t)brz->k, sizeof(cmph_uint32));
for (i = 1; i < brz->k; ++i) for (i = 1; i < brz->k; ++i)
{ {
brz->offset[i] = brz->size[i-1] + brz->offset[i-1]; brz->offset[i] = brz->size[i-1] + brz->offset[i-1];
@ -220,8 +220,8 @@ static int brz_gen_mphf(cmph_config_t *mph)
brz_config_data_t *brz = (brz_config_data_t *)mph->data; brz_config_data_t *brz = (brz_config_data_t *)mph->data;
cmph_uint32 memory_usage = 0; cmph_uint32 memory_usage = 0;
cmph_uint32 nkeys_in_buffer = 0; cmph_uint32 nkeys_in_buffer = 0;
cmph_uint8 *buffer = (cmph_uint8 *)malloc(brz->memory_availability); cmph_uint8 *buffer = (cmph_uint8 *)malloc((size_t)brz->memory_availability);
cmph_uint32 *buckets_size = (cmph_uint32 *)calloc(brz->k, sizeof(cmph_uint32)); cmph_uint32 *buckets_size = (cmph_uint32 *)calloc((size_t)brz->k, sizeof(cmph_uint32));
cmph_uint32 *keys_index = NULL; cmph_uint32 *keys_index = NULL;
cmph_uint8 **buffer_merge = NULL; cmph_uint8 **buffer_merge = NULL;
cmph_uint32 *buffer_h0 = NULL; cmph_uint32 *buffer_h0 = NULL;
@ -263,7 +263,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
} }
memory_usage = 0; memory_usage = 0;
keys_index = (cmph_uint32 *)calloc(nkeys_in_buffer, sizeof(cmph_uint32)); keys_index = (cmph_uint32 *)calloc((size_t)nkeys_in_buffer, sizeof(cmph_uint32));
for(i = 0; i < nkeys_in_buffer; i++) for(i = 0; i < nkeys_in_buffer; i++)
{ {
memcpy(&keylen1, buffer + memory_usage, sizeof(keylen1)); memcpy(&keylen1, buffer + memory_usage, sizeof(keylen1));
@ -280,7 +280,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
for(i = 0; i < nkeys_in_buffer; i++) for(i = 0; i < nkeys_in_buffer; i++)
{ {
memcpy(&keylen1, buffer + keys_index[i], sizeof(keylen1)); memcpy(&keylen1, buffer + keys_index[i], sizeof(keylen1));
fwrite(buffer + keys_index[i], 1, keylen1 + sizeof(keylen1), tmp_fd); fwrite(buffer + keys_index[i], (size_t)1, keylen1 + sizeof(keylen1), tmp_fd);
} }
nkeys_in_buffer = 0; nkeys_in_buffer = 0;
memory_usage = 0; memory_usage = 0;
@ -290,7 +290,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
fclose(tmp_fd); fclose(tmp_fd);
} }
memcpy(buffer + memory_usage, &keylen, sizeof(keylen)); memcpy(buffer + memory_usage, &keylen, sizeof(keylen));
memcpy(buffer + memory_usage + sizeof(keylen), key, keylen); memcpy(buffer + memory_usage + sizeof(keylen), key, (size_t)keylen);
memory_usage += keylen + sizeof(keylen); memory_usage += keylen + sizeof(keylen);
h0 = hash(brz->h0, key, keylen) % brz->k; h0 = hash(brz->h0, key, keylen) % brz->k;
@ -323,7 +323,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
buckets_size[i] = sum; buckets_size[i] = sum;
} }
memory_usage = 0; memory_usage = 0;
keys_index = (cmph_uint32 *)calloc(nkeys_in_buffer, sizeof(cmph_uint32)); keys_index = (cmph_uint32 *)calloc((size_t)nkeys_in_buffer, sizeof(cmph_uint32));
for(i = 0; i < nkeys_in_buffer; i++) for(i = 0; i < nkeys_in_buffer; i++)
{ {
memcpy(&keylen1, buffer + memory_usage, sizeof(keylen1)); memcpy(&keylen1, buffer + memory_usage, sizeof(keylen1));
@ -340,7 +340,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
for(i = 0; i < nkeys_in_buffer; i++) for(i = 0; i < nkeys_in_buffer; i++)
{ {
memcpy(&keylen1, buffer + keys_index[i], sizeof(keylen1)); memcpy(&keylen1, buffer + keys_index[i], sizeof(keylen1));
fwrite(buffer + keys_index[i], 1, keylen1 + sizeof(keylen1), tmp_fd); fwrite(buffer + keys_index[i], (size_t)1, keylen1 + sizeof(keylen1), tmp_fd);
} }
nkeys_in_buffer = 0; nkeys_in_buffer = 0;
memory_usage = 0; memory_usage = 0;
@ -359,17 +359,17 @@ static int brz_gen_mphf(cmph_config_t *mph)
fprintf(stderr, "\nMPHF generation \n"); fprintf(stderr, "\nMPHF generation \n");
} }
/* Starting to dump to disk the resultant MPHF: __cmph_dump function */ /* Starting to dump to disk the resultant MPHF: __cmph_dump function */
fwrite(cmph_names[CMPH_BRZ], (cmph_uint32)(strlen(cmph_names[CMPH_BRZ]) + 1), 1, brz->mphf_fd); fwrite(cmph_names[CMPH_BRZ], (size_t)(strlen(cmph_names[CMPH_BRZ]) + 1), (size_t)1, brz->mphf_fd);
fwrite(&(brz->m), sizeof(brz->m), 1, brz->mphf_fd); fwrite(&(brz->m), sizeof(brz->m), (size_t)1, brz->mphf_fd);
fwrite(&(brz->c), sizeof(cmph_float32), 1, brz->mphf_fd); fwrite(&(brz->c), sizeof(double), (size_t)1, brz->mphf_fd);
fwrite(&(brz->algo), sizeof(brz->algo), 1, brz->mphf_fd); fwrite(&(brz->algo), sizeof(brz->algo), (size_t)1, brz->mphf_fd);
fwrite(&(brz->k), sizeof(cmph_uint32), 1, brz->mphf_fd); // number of MPHFs fwrite(&(brz->k), sizeof(cmph_uint32), (size_t)1, brz->mphf_fd); // number of MPHFs
fwrite(brz->size, sizeof(cmph_uint8)*(brz->k), 1, brz->mphf_fd); fwrite(brz->size, sizeof(cmph_uint8)*(brz->k), (size_t)1, brz->mphf_fd);
//tmp_fds = (FILE **)calloc(nflushes, sizeof(FILE *)); //tmp_fds = (FILE **)calloc(nflushes, sizeof(FILE *));
buff_manager = buffer_manager_new(brz->memory_availability, nflushes); buff_manager = buffer_manager_new(brz->memory_availability, nflushes);
buffer_merge = (cmph_uint8 **)calloc(nflushes, sizeof(cmph_uint8 *)); buffer_merge = (cmph_uint8 **)calloc((size_t)nflushes, sizeof(cmph_uint8 *));
buffer_h0 = (cmph_uint32 *)calloc(nflushes, sizeof(cmph_uint32)); buffer_h0 = (cmph_uint32 *)calloc((size_t)nflushes, sizeof(cmph_uint32));
memory_usage = 0; memory_usage = 0;
for(i = 0; i < nflushes; i++) for(i = 0; i < nflushes; i++)
@ -386,7 +386,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
key = NULL; //transfer memory ownership key = NULL; //transfer memory ownership
} }
e = 0; e = 0;
keys_vd = (cmph_uint8 **)calloc(MAX_BUCKET_SIZE, sizeof(cmph_uint8 *)); keys_vd = (cmph_uint8 **)calloc((size_t)MAX_BUCKET_SIZE, sizeof(cmph_uint8 *));
nkeys_vd = 0; nkeys_vd = 0;
error = 0; error = 0;
while(e < brz->m) while(e < brz->m)
@ -473,7 +473,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
break; break;
default: assert(0); default: assert(0);
} }
fwrite(bufmphf, buflenmphf, 1, brz->mphf_fd); fwrite(bufmphf, (size_t)buflenmphf, (size_t)1, brz->mphf_fd);
free(bufmphf); free(bufmphf);
bufmphf = NULL; bufmphf = NULL;
cmph_config_destroy(config); cmph_config_destroy(config);
@ -501,7 +501,7 @@ static cmph_uint32 brz_min_index(cmph_uint32 * vector, cmph_uint32 n)
return min_index; return min_index;
} }
static void brz_destroy_keys_vd(cmph_uint8 ** keys_vd, cmph_uint8 nkeys) static void brz_destroy_keys_vd(cmph_uint8 ** keys_vd, cmph_uint32 nkeys)
{ {
cmph_uint8 i; cmph_uint8 i;
for(i = 0; i < nkeys; i++) { free(keys_vd[i]); keys_vd[i] = NULL;} for(i = 0; i < nkeys; i++) { free(keys_vd[i]); keys_vd[i] = NULL;}
@ -519,12 +519,12 @@ static char * brz_copy_partial_fch_mphf(brz_config_data_t *brz, fch_data_t * fch
hash_state_dump(fchf->h1, &bufh1, &buflenh1); hash_state_dump(fchf->h1, &bufh1, &buflenh1);
hash_state_dump(fchf->h2, &bufh2, &buflenh2); hash_state_dump(fchf->h2, &bufh2, &buflenh2);
*buflen = buflenh1 + buflenh2 + n + 2*sizeof(cmph_uint32); *buflen = buflenh1 + buflenh2 + n + 2*sizeof(cmph_uint32);
buf = (char *)malloc(*buflen); buf = (char *)malloc((size_t)(*buflen));
memcpy(buf, &buflenh1, sizeof(cmph_uint32)); memcpy(buf, &buflenh1, sizeof(cmph_uint32));
memcpy(buf+sizeof(cmph_uint32), bufh1, buflenh1); memcpy(buf+sizeof(cmph_uint32), bufh1, (size_t)buflenh1);
memcpy(buf+sizeof(cmph_uint32)+buflenh1, &buflenh2, sizeof(cmph_uint32)); memcpy(buf+sizeof(cmph_uint32)+buflenh1, &buflenh2, sizeof(cmph_uint32));
memcpy(buf+2*sizeof(cmph_uint32)+buflenh1, bufh2, buflenh2); memcpy(buf+2*sizeof(cmph_uint32)+buflenh1, bufh2, (size_t)buflenh2);
for (i = 0; i < n; i++) memcpy(buf+2*sizeof(cmph_uint32)+buflenh1+buflenh2+i,(fchf->g + i), 1); for (i = 0; i < n; i++) memcpy(buf+2*sizeof(cmph_uint32)+buflenh1+buflenh2+i,(fchf->g + i), (size_t)1);
free(bufh1); free(bufh1);
free(bufh2); free(bufh2);
return buf; return buf;
@ -540,12 +540,12 @@ static char * brz_copy_partial_bmz8_mphf(brz_config_data_t *brz, bmz8_data_t * b
hash_state_dump(bmzf->hashes[0], &bufh1, &buflenh1); hash_state_dump(bmzf->hashes[0], &bufh1, &buflenh1);
hash_state_dump(bmzf->hashes[1], &bufh2, &buflenh2); hash_state_dump(bmzf->hashes[1], &bufh2, &buflenh2);
*buflen = buflenh1 + buflenh2 + n + 2*sizeof(cmph_uint32); *buflen = buflenh1 + buflenh2 + n + 2*sizeof(cmph_uint32);
buf = (char *)malloc(*buflen); buf = (char *)malloc((size_t)(*buflen));
memcpy(buf, &buflenh1, sizeof(cmph_uint32)); memcpy(buf, &buflenh1, sizeof(cmph_uint32));
memcpy(buf+sizeof(cmph_uint32), bufh1, buflenh1); memcpy(buf+sizeof(cmph_uint32), bufh1, (size_t)buflenh1);
memcpy(buf+sizeof(cmph_uint32)+buflenh1, &buflenh2, sizeof(cmph_uint32)); memcpy(buf+sizeof(cmph_uint32)+buflenh1, &buflenh2, sizeof(cmph_uint32));
memcpy(buf+2*sizeof(cmph_uint32)+buflenh1, bufh2, buflenh2); memcpy(buf+2*sizeof(cmph_uint32)+buflenh1, bufh2, (size_t)buflenh2);
memcpy(buf+2*sizeof(cmph_uint32)+buflenh1+buflenh2,bmzf->g, n); memcpy(buf+2*sizeof(cmph_uint32)+buflenh1+buflenh2,bmzf->g, (size_t)n);
free(bufh1); free(bufh1);
free(bufh2); free(bufh2);
return buf; return buf;
@ -562,12 +562,12 @@ int brz_dump(cmph_t *mphf, FILE *fd)
// Dumping h0 // Dumping h0
hash_state_dump(data->h0, &buf, &buflen); hash_state_dump(data->h0, &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
// Dumping m and the vector offset. // Dumping m and the vector offset.
fwrite(&(data->m), sizeof(cmph_uint32), 1, fd); fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(data->offset, sizeof(cmph_uint32)*(data->k), 1, fd); fwrite(data->offset, sizeof(cmph_uint32)*(data->k), (size_t)1, fd);
return 1; return 1;
} }
@ -580,30 +580,30 @@ void brz_load(FILE *f, cmph_t *mphf)
DEBUGP("Loading brz mphf\n"); DEBUGP("Loading brz mphf\n");
mphf->data = brz; mphf->data = brz;
fread(&(brz->c), sizeof(cmph_float32), 1, f); fread(&(brz->c), sizeof(double), (size_t)1, f);
fread(&(brz->algo), sizeof(brz->algo), 1, f); // Reading algo. fread(&(brz->algo), sizeof(brz->algo), (size_t)1, f); // Reading algo.
fread(&(brz->k), sizeof(cmph_uint32), 1, f); fread(&(brz->k), sizeof(cmph_uint32), (size_t)1, f);
brz->size = (cmph_uint8 *) malloc(sizeof(cmph_uint8)*brz->k); brz->size = (cmph_uint8 *) malloc(sizeof(cmph_uint8)*brz->k);
fread(brz->size, sizeof(cmph_uint8)*(brz->k), 1, f); fread(brz->size, sizeof(cmph_uint8)*(brz->k), (size_t)1, f);
brz->h1 = (hash_state_t **)malloc(sizeof(hash_state_t *)*brz->k); brz->h1 = (hash_state_t **)malloc(sizeof(hash_state_t *)*brz->k);
brz->h2 = (hash_state_t **)malloc(sizeof(hash_state_t *)*brz->k); brz->h2 = (hash_state_t **)malloc(sizeof(hash_state_t *)*brz->k);
brz->g = (cmph_uint8 **) calloc(brz->k, sizeof(cmph_uint8 *)); brz->g = (cmph_uint8 **) calloc((size_t)brz->k, sizeof(cmph_uint8 *));
DEBUGP("Reading c = %f k = %u algo = %u \n", brz->c, brz->k, brz->algo); DEBUGP("Reading c = %f k = %u algo = %u \n", brz->c, brz->k, brz->algo);
//loading h_i1, h_i2 and g_i. //loading h_i1, h_i2 and g_i.
for(i = 0; i < brz->k; i++) for(i = 0; i < brz->k; i++)
{ {
// h1 // h1
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state 1 has %u bytes\n", buflen); DEBUGP("Hash state 1 has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
brz->h1[i] = hash_state_load(buf, buflen); brz->h1[i] = hash_state_load(buf, buflen);
free(buf); free(buf);
//h2 //h2
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state 2 has %u bytes\n", buflen); DEBUGP("Hash state 2 has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
brz->h2[i] = hash_state_load(buf, buflen); brz->h2[i] = hash_state_load(buf, buflen);
free(buf); free(buf);
switch(brz->algo) switch(brz->algo)
@ -617,21 +617,21 @@ void brz_load(FILE *f, cmph_t *mphf)
default: assert(0); default: assert(0);
} }
DEBUGP("g_i has %u bytes\n", n); DEBUGP("g_i has %u bytes\n", n);
brz->g[i] = (cmph_uint8 *)calloc(n, sizeof(cmph_uint8)); brz->g[i] = (cmph_uint8 *)calloc((size_t)n, sizeof(cmph_uint8));
fread(brz->g[i], sizeof(cmph_uint8)*n, 1, f); fread(brz->g[i], sizeof(cmph_uint8)*n, (size_t)1, f);
} }
//loading h0 //loading h0
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state has %u bytes\n", buflen); DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
brz->h0 = hash_state_load(buf, buflen); brz->h0 = hash_state_load(buf, buflen);
free(buf); free(buf);
//loading c, m, and the vector offset. //loading c, m, and the vector offset.
fread(&(brz->m), sizeof(cmph_uint32), 1, f); fread(&(brz->m), sizeof(cmph_uint32), (size_t)1, f);
brz->offset = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*brz->k); brz->offset = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*brz->k);
fread(brz->offset, sizeof(cmph_uint32)*(brz->k), 1, f); fread(brz->offset, sizeof(cmph_uint32)*(brz->k), (size_t)1, f);
return; return;
} }
@ -665,8 +665,8 @@ static cmph_uint32 brz_fch_search(brz_data_t *brz, const char *key, cmph_uint32
register cmph_uint32 m = brz->size[h0]; register cmph_uint32 m = brz->size[h0];
register cmph_uint32 b = fch_calc_b(brz->c, m); register cmph_uint32 b = fch_calc_b(brz->c, m);
register cmph_float32 p1 = fch_calc_p1(m); register double p1 = fch_calc_p1(m);
register cmph_float32 p2 = fch_calc_p2(b); register double p2 = fch_calc_p2(b);
register cmph_uint32 h1 = hash(brz->h1[h0], key, keylen) % m; register cmph_uint32 h1 = hash(brz->h1[h0], key, keylen) % m;
register cmph_uint32 h2 = hash(brz->h2[h0], key, keylen) % m; register cmph_uint32 h2 = hash(brz->h2[h0], key, keylen) % m;
register cmph_uint8 mphf_bucket = 0; register cmph_uint8 mphf_bucket = 0;
@ -843,7 +843,7 @@ cmph_uint32 brz_packed_size(cmph_t *mphf)
CMPH_HASH h1_type = hash_get_type(data->h1[0]); CMPH_HASH h1_type = hash_get_type(data->h1[0]);
CMPH_HASH h2_type = hash_get_type(data->h2[0]); CMPH_HASH h2_type = hash_get_type(data->h2[0]);
size = (2*sizeof(CMPH_ALGO) + 3*sizeof(CMPH_HASH) + hash_state_packed_size(h0_type) + sizeof(cmph_uint32) + size = (2*sizeof(CMPH_ALGO) + 3*sizeof(CMPH_HASH) + hash_state_packed_size(h0_type) + sizeof(cmph_uint32) +
sizeof(cmph_float32) + sizeof(cmph_uint8)*data->k + sizeof(cmph_uint32)*data->k); sizeof(double) + sizeof(cmph_uint8)*data->k + sizeof(cmph_uint32)*data->k);
// pointers to g_is // pointers to g_is
#if defined (__ia64) || defined (__x86_64__) #if defined (__ia64) || defined (__x86_64__)
size += sizeof(cmph_uint64)*data->k; size += sizeof(cmph_uint64)*data->k;
@ -882,7 +882,7 @@ static cmph_uint32 brz_bmz8_search_packed(cmph_uint32 *packed_mphf, const char *
register cmph_uint32 k = *packed_mphf++; register cmph_uint32 k = *packed_mphf++;
register cmph_float32 c = (cmph_float32)(*packed_mphf); register double c = (double)(*packed_mphf);
packed_mphf++; packed_mphf++;
register CMPH_HASH h1_type = *packed_mphf++; register CMPH_HASH h1_type = *packed_mphf++;
@ -937,7 +937,7 @@ static cmph_uint32 brz_fch_search_packed(cmph_uint32 *packed_mphf, const char *k
register cmph_uint32 k = *packed_mphf++; register cmph_uint32 k = *packed_mphf++;
register cmph_float32 c = (cmph_float32)(*packed_mphf); register double c = (double)(*packed_mphf);
packed_mphf++; packed_mphf++;
register CMPH_HASH h1_type = *packed_mphf++; register CMPH_HASH h1_type = *packed_mphf++;
@ -957,8 +957,8 @@ static cmph_uint32 brz_fch_search_packed(cmph_uint32 *packed_mphf, const char *k
register cmph_uint32 m = size[h0]; register cmph_uint32 m = size[h0];
register cmph_uint32 b = fch_calc_b(c, m); register cmph_uint32 b = fch_calc_b(c, m);
register cmph_float32 p1 = fch_calc_p1(m); register double p1 = fch_calc_p1(m);
register cmph_float32 p2 = fch_calc_p2(b); register double p2 = fch_calc_p2(b);
#if defined (__ia64) || defined (__x86_64__) #if defined (__ia64) || defined (__x86_64__)
register cmph_uint64 * g_is_ptr = (cmph_uint64 *)packed_mphf; register cmph_uint64 * g_is_ptr = (cmph_uint64 *)packed_mphf;

4
src/brz.h
View File

@ -10,11 +10,11 @@ brz_config_data_t *brz_config_new();
void brz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void brz_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void brz_config_set_tmp_dir(cmph_config_t *mph, cmph_uint8 *tmp_dir); void brz_config_set_tmp_dir(cmph_config_t *mph, cmph_uint8 *tmp_dir);
void brz_config_set_mphf_fd(cmph_config_t *mph, FILE *mphf_fd); void brz_config_set_mphf_fd(cmph_config_t *mph, FILE *mphf_fd);
void brz_config_set_b(cmph_config_t *mph, cmph_uint8 b); void brz_config_set_b(cmph_config_t *mph, cmph_uint32 b);
void brz_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo); void brz_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo);
void brz_config_set_memory_availability(cmph_config_t *mph, cmph_uint32 memory_availability); void brz_config_set_memory_availability(cmph_config_t *mph, cmph_uint32 memory_availability);
void brz_config_destroy(cmph_config_t *mph); void brz_config_destroy(cmph_config_t *mph);
cmph_t *brz_new(cmph_config_t *mph, float c); cmph_t *brz_new(cmph_config_t *mph, double c);
void brz_load(FILE *f, cmph_t *mphf); void brz_load(FILE *f, cmph_t *mphf);
int brz_dump(cmph_t *mphf, FILE *f); int brz_dump(cmph_t *mphf, FILE *f);

4
src/brz_structs.h
View File

@ -7,7 +7,7 @@ struct __brz_data_t
{ {
CMPH_ALGO algo; // CMPH algo for generating the MPHFs for the buckets (Just CMPH_FCH and CMPH_BMZ8) CMPH_ALGO algo; // CMPH algo for generating the MPHFs for the buckets (Just CMPH_FCH and CMPH_BMZ8)
cmph_uint32 m; // edges (words) count cmph_uint32 m; // edges (words) count
cmph_float32 c; // constant c double c; // constant c
cmph_uint8 *size; // size[i] stores the number of edges represented by g[i][...]. cmph_uint8 *size; // size[i] stores the number of edges represented by g[i][...].
cmph_uint32 *offset; // offset[i] stores the sum: size[0] + size[1] + ... size[i-1]. cmph_uint32 *offset; // offset[i] stores the sum: size[0] + size[1] + ... size[i-1].
cmph_uint8 **g; // g function. cmph_uint8 **g; // g function.
@ -21,7 +21,7 @@ struct __brz_config_data_t
{ {
CMPH_HASH hashfuncs[3]; CMPH_HASH hashfuncs[3];
CMPH_ALGO algo; // CMPH algo for generating the MPHFs for the buckets (Just CMPH_FCH and CMPH_BMZ8) CMPH_ALGO algo; // CMPH algo for generating the MPHFs for the buckets (Just CMPH_FCH and CMPH_BMZ8)
cmph_float32 c; // constant c double c; // constant c
cmph_uint32 m; // edges (words) count cmph_uint32 m; // edges (words) count
cmph_uint8 *size; // size[i] stores the number of edges represented by g[i][...]. cmph_uint8 *size; // size[i] stores the number of edges represented by g[i][...].
cmph_uint32 *offset; // offset[i] stores the sum: size[0] + size[1] + ... size[i-1]. cmph_uint32 *offset; // offset[i] stores the sum: size[0] + size[1] + ... size[i-1].

12
src/buffer_entry.c
View File

@ -46,8 +46,8 @@ cmph_uint32 buffer_entry_get_capacity(buffer_entry_t * buffer_entry)
void buffer_entry_load(buffer_entry_t * buffer_entry) void buffer_entry_load(buffer_entry_t * buffer_entry)
{ {
free(buffer_entry->buff); free(buffer_entry->buff);
buffer_entry->buff = (cmph_uint8 *)calloc(buffer_entry->capacity, sizeof(cmph_uint8)); buffer_entry->buff = (cmph_uint8 *)calloc((size_t)buffer_entry->capacity, sizeof(cmph_uint8));
buffer_entry->nbytes = fread(buffer_entry->buff, 1, buffer_entry->capacity, buffer_entry->fd); buffer_entry->nbytes = fread(buffer_entry->buff, (size_t)1, (size_t)buffer_entry->capacity, buffer_entry->fd);
if (buffer_entry->nbytes != buffer_entry->capacity) buffer_entry->eof = 1; if (buffer_entry->nbytes != buffer_entry->capacity) buffer_entry->eof = 1;
buffer_entry->pos = 0; buffer_entry->pos = 0;
} }
@ -66,10 +66,10 @@ cmph_uint8 * buffer_entry_read_key(buffer_entry_t * buffer_entry, cmph_uint32 *
{ {
copied_bytes = buffer_entry->nbytes - buffer_entry->pos; copied_bytes = buffer_entry->nbytes - buffer_entry->pos;
lacked_bytes = (buffer_entry->pos + lacked_bytes) - buffer_entry->nbytes; lacked_bytes = (buffer_entry->pos + lacked_bytes) - buffer_entry->nbytes;
if (copied_bytes != 0) memcpy(keylen, buffer_entry->buff + buffer_entry->pos, copied_bytes); if (copied_bytes != 0) memcpy(keylen, buffer_entry->buff + buffer_entry->pos, (size_t)copied_bytes);
buffer_entry_load(buffer_entry); buffer_entry_load(buffer_entry);
} }
memcpy(keylen + copied_bytes, buffer_entry->buff + buffer_entry->pos, lacked_bytes); memcpy(keylen + copied_bytes, buffer_entry->buff + buffer_entry->pos, (size_t)lacked_bytes);
buffer_entry->pos += lacked_bytes; buffer_entry->pos += lacked_bytes;
lacked_bytes = *keylen; lacked_bytes = *keylen;
@ -80,11 +80,11 @@ cmph_uint8 * buffer_entry_read_key(buffer_entry_t * buffer_entry, cmph_uint32 *
copied_bytes = buffer_entry->nbytes - buffer_entry->pos; copied_bytes = buffer_entry->nbytes - buffer_entry->pos;
lacked_bytes = (buffer_entry->pos + lacked_bytes) - buffer_entry->nbytes; lacked_bytes = (buffer_entry->pos + lacked_bytes) - buffer_entry->nbytes;
if (copied_bytes != 0) { if (copied_bytes != 0) {
memcpy(buf + sizeof(*keylen), buffer_entry->buff + buffer_entry->pos, copied_bytes); memcpy(buf + sizeof(*keylen), buffer_entry->buff + buffer_entry->pos, (size_t)copied_bytes);
} }
buffer_entry_load(buffer_entry); buffer_entry_load(buffer_entry);
} }
memcpy(buf+sizeof(*keylen)+copied_bytes, buffer_entry->buff + buffer_entry->pos, lacked_bytes); memcpy(buf+sizeof(*keylen)+copied_bytes, buffer_entry->buff + buffer_entry->pos, (size_t)lacked_bytes);
buffer_entry->pos += lacked_bytes; buffer_entry->pos += lacked_bytes;
return buf; return buf;
} }

4
src/buffer_manage.c
View File

@ -18,8 +18,8 @@ buffer_manage_t * buffer_manage_new(cmph_uint32 memory_avail, cmph_uint32 nentri
buffer_manage_t *buff_manage = (buffer_manage_t *)malloc(sizeof(buffer_manage_t)); buffer_manage_t *buff_manage = (buffer_manage_t *)malloc(sizeof(buffer_manage_t));
assert(buff_manage); assert(buff_manage);
buff_manage->memory_avail = memory_avail; buff_manage->memory_avail = memory_avail;
buff_manage->buffer_entries = (buffer_entry_t **)calloc(nentries, sizeof(buffer_entry_t *)); buff_manage->buffer_entries = (buffer_entry_t **)calloc((size_t)nentries, sizeof(buffer_entry_t *));
buff_manage->memory_avail_list = (cmph_uint32 *)calloc(nentries, sizeof(cmph_uint32)); buff_manage->memory_avail_list = (cmph_uint32 *)calloc((size_t)nentries, sizeof(cmph_uint32));
buff_manage->pos_avail_list = -1; buff_manage->pos_avail_list = -1;
buff_manage->nentries = nentries; buff_manage->nentries = nentries;
memory_avail_entry = buff_manage->memory_avail/buff_manage->nentries + 1; memory_avail_entry = buff_manage->memory_avail/buff_manage->nentries + 1;

4
src/buffer_manager.c
View File

@ -18,8 +18,8 @@ buffer_manager_t * buffer_manager_new(cmph_uint32 memory_avail, cmph_uint32 nent
buffer_manager_t *buff_manager = (buffer_manager_t *)malloc(sizeof(buffer_manager_t)); buffer_manager_t *buff_manager = (buffer_manager_t *)malloc(sizeof(buffer_manager_t));
assert(buff_manager); assert(buff_manager);
buff_manager->memory_avail = memory_avail; buff_manager->memory_avail = memory_avail;
buff_manager->buffer_entries = (buffer_entry_t **)calloc(nentries, sizeof(buffer_entry_t *)); buff_manager->buffer_entries = (buffer_entry_t **)calloc((size_t)nentries, sizeof(buffer_entry_t *));
buff_manager->memory_avail_list = (cmph_uint32 *)calloc(nentries, sizeof(cmph_uint32)); buff_manager->memory_avail_list = (cmph_uint32 *)calloc((size_t)nentries, sizeof(cmph_uint32));
buff_manager->pos_avail_list = -1; buff_manager->pos_avail_list = -1;
buff_manager->nentries = nentries; buff_manager->nentries = nentries;
memory_avail_entry = buff_manager->memory_avail/buff_manager->nentries + 1; memory_avail_entry = buff_manager->memory_avail/buff_manager->nentries + 1;

36
src/chm.c
View File

@ -50,7 +50,7 @@ void chm_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
} }
cmph_t *chm_new(cmph_config_t *mph, float c) cmph_t *chm_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
chm_data_t *chmf = NULL; chm_data_t *chmf = NULL;
@ -107,8 +107,8 @@ cmph_t *chm_new(cmph_config_t *mph, float c)
fprintf(stderr, "Starting assignment step\n"); fprintf(stderr, "Starting assignment step\n");
} }
DEBUGP("Assignment step\n"); DEBUGP("Assignment step\n");
visited = (cmph_uint8 *)malloc(chm->n/8 + 1); visited = (cmph_uint8 *)malloc((size_t)(chm->n/8 + 1));
memset(visited, 0, chm->n/8 + 1); memset(visited, 0, (size_t)(chm->n/8 + 1));
free(chm->g); free(chm->g);
chm->g = (cmph_uint32 *)malloc(chm->n * sizeof(cmph_uint32)); chm->g = (cmph_uint32 *)malloc(chm->n * sizeof(cmph_uint32));
assert(chm->g); assert(chm->g);
@ -206,23 +206,23 @@ int chm_dump(cmph_t *mphf, FILE *fd)
chm_data_t *data = (chm_data_t *)mphf->data; chm_data_t *data = (chm_data_t *)mphf->data;
__cmph_dump(mphf, fd); __cmph_dump(mphf, fd);
fwrite(&two, sizeof(cmph_uint32), 1, fd); fwrite(&two, sizeof(cmph_uint32), (size_t)1, fd);
hash_state_dump(data->hashes[0], &buf, &buflen); hash_state_dump(data->hashes[0], &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
hash_state_dump(data->hashes[1], &buf, &buflen); hash_state_dump(data->hashes[1], &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
fwrite(&(data->n), sizeof(cmph_uint32), 1, fd); fwrite(&(data->n), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->m), sizeof(cmph_uint32), 1, fd); fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(data->g, sizeof(cmph_uint32)*data->n, 1, fd); fwrite(data->g, sizeof(cmph_uint32)*data->n, (size_t)1, fd);
#ifdef DEBUG #ifdef DEBUG
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
for (i = 0; i < data->n; ++i) fprintf(stderr, "%u ", data->g[i]); for (i = 0; i < data->n; ++i) fprintf(stderr, "%u ", data->g[i]);
@ -241,28 +241,28 @@ void chm_load(FILE *f, cmph_t *mphf)
DEBUGP("Loading chm mphf\n"); DEBUGP("Loading chm mphf\n");
mphf->data = chm; mphf->data = chm;
fread(&nhashes, sizeof(cmph_uint32), 1, f); fread(&nhashes, sizeof(cmph_uint32), (size_t)1, f);
chm->hashes = (hash_state_t **)malloc(sizeof(hash_state_t *)*(nhashes + 1)); chm->hashes = (hash_state_t **)malloc(sizeof(hash_state_t *)*(nhashes + 1));
chm->hashes[nhashes] = NULL; chm->hashes[nhashes] = NULL;
DEBUGP("Reading %u hashes\n", nhashes); DEBUGP("Reading %u hashes\n", nhashes);
for (i = 0; i < nhashes; ++i) for (i = 0; i < nhashes; ++i)
{ {
hash_state_t *state = NULL; hash_state_t *state = NULL;
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state has %u bytes\n", buflen); DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
state = hash_state_load(buf, buflen); state = hash_state_load(buf, buflen);
chm->hashes[i] = state; chm->hashes[i] = state;
free(buf); free(buf);
} }
DEBUGP("Reading m and n\n"); DEBUGP("Reading m and n\n");
fread(&(chm->n), sizeof(cmph_uint32), 1, f); fread(&(chm->n), sizeof(cmph_uint32), (size_t)1, f);
fread(&(chm->m), sizeof(cmph_uint32), 1, f); fread(&(chm->m), sizeof(cmph_uint32), (size_t)1, f);
chm->g = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*chm->n); chm->g = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*chm->n);
fread(chm->g, chm->n*sizeof(cmph_uint32), 1, f); fread(chm->g, chm->n*sizeof(cmph_uint32), (size_t)1, f);
#ifdef DEBUG #ifdef DEBUG
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
for (i = 0; i < chm->n; ++i) fprintf(stderr, "%u ", chm->g[i]); for (i = 0; i < chm->n; ++i) fprintf(stderr, "%u ", chm->g[i]);

2
src/chm.h
View File

@ -9,7 +9,7 @@ typedef struct __chm_config_data_t chm_config_data_t;
chm_config_data_t *chm_config_new(); chm_config_data_t *chm_config_new();
void chm_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void chm_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void chm_config_destroy(cmph_config_t *mph); void chm_config_destroy(cmph_config_t *mph);
cmph_t *chm_new(cmph_config_t *mph, float c); cmph_t *chm_new(cmph_config_t *mph, double c);
void chm_load(FILE *f, cmph_t *mphf); void chm_load(FILE *f, cmph_t *mphf);
int chm_dump(cmph_t *mphf, FILE *f); int chm_dump(cmph_t *mphf, FILE *f);

22
src/cmph.c
View File

@ -81,9 +81,11 @@ static int key_byte_vector_read(void *data, char **key, cmph_uint32 *keylen)
{ {
cmph_vector_t *cmph_vector = (cmph_vector_t *)data; cmph_vector_t *cmph_vector = (cmph_vector_t *)data;
cmph_uint8 **keys_vd = (cmph_uint8 **)cmph_vector->vector; cmph_uint8 **keys_vd = (cmph_uint8 **)cmph_vector->vector;
size_t size;
memcpy(keylen, keys_vd[cmph_vector->position], sizeof(*keylen)); memcpy(keylen, keys_vd[cmph_vector->position], sizeof(*keylen));
*key = (char *)malloc(*keylen); size = *keylen;
memcpy(*key, keys_vd[cmph_vector->position] + sizeof(*keylen), *keylen); *key = (char *)malloc(size);
memcpy(*key, keys_vd[cmph_vector->position] + sizeof(*keylen), size);
cmph_vector->position = cmph_vector->position + 1; cmph_vector->position = cmph_vector->position + 1;
return *keylen; return *keylen;
@ -93,9 +95,11 @@ static int key_struct_vector_read(void *data, char **key, cmph_uint32 *keylen)
{ {
cmph_struct_vector_t *cmph_struct_vector = (cmph_struct_vector_t *)data; cmph_struct_vector_t *cmph_struct_vector = (cmph_struct_vector_t *)data;
char *keys_vd = (char *)cmph_struct_vector->vector; char *keys_vd = (char *)cmph_struct_vector->vector;
size_t size;
*keylen = cmph_struct_vector->key_len; *keylen = cmph_struct_vector->key_len;
*key = (char *)malloc(*keylen); size = *keylen;
memcpy(*key, (keys_vd + (cmph_struct_vector->position * cmph_struct_vector->struct_size) + cmph_struct_vector->key_offset), *keylen); *key = (char *)malloc(size);
memcpy(*key, (keys_vd + (cmph_struct_vector->position * cmph_struct_vector->struct_size) + cmph_struct_vector->key_offset), size);
cmph_struct_vector->position = cmph_struct_vector->position + 1; cmph_struct_vector->position = cmph_struct_vector->position + 1;
return *keylen; return *keylen;
} }
@ -104,8 +108,10 @@ static int key_vector_read(void *data, char **key, cmph_uint32 *keylen)
{ {
cmph_vector_t *cmph_vector = (cmph_vector_t *)data; cmph_vector_t *cmph_vector = (cmph_vector_t *)data;
char **keys_vd = (char **)cmph_vector->vector; char **keys_vd = (char **)cmph_vector->vector;
size_t size;
*keylen = strlen(keys_vd[cmph_vector->position]); *keylen = strlen(keys_vd[cmph_vector->position]);
*key = (char *)malloc(*keylen + 1); size = *keylen;
*key = (char *)malloc(size + 1);
strcpy(*key, keys_vd[cmph_vector->position]); strcpy(*key, keys_vd[cmph_vector->position]);
cmph_vector->position = cmph_vector->position + 1; cmph_vector->position = cmph_vector->position + 1;
return *keylen; return *keylen;
@ -365,7 +371,7 @@ void cmph_config_set_mphf_fd(cmph_config_t *mph, FILE *mphf_fd)
} }
} }
void cmph_config_set_b(cmph_config_t *mph, cmph_uint8 b) void cmph_config_set_b(cmph_config_t *mph, cmph_uint32 b)
{ {
if (mph->algo == CMPH_BRZ) if (mph->algo == CMPH_BRZ)
{ {
@ -455,7 +461,7 @@ void cmph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
return; return;
} }
void cmph_config_set_graphsize(cmph_config_t *mph, float c) void cmph_config_set_graphsize(cmph_config_t *mph, double c)
{ {
mph->c = c; mph->c = c;
return; return;
@ -464,7 +470,7 @@ void cmph_config_set_graphsize(cmph_config_t *mph, float c)
cmph_t *cmph_new(cmph_config_t *mph) cmph_t *cmph_new(cmph_config_t *mph)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
float c = mph->c; double c = mph->c;
DEBUGP("Creating mph with algorithm %s\n", cmph_names[mph->algo]); DEBUGP("Creating mph with algorithm %s\n", cmph_names[mph->algo]);
switch (mph->algo) switch (mph->algo)

17
src/cmph.h
View File

@ -37,18 +37,23 @@ void cmph_io_vector_adapter_destroy(cmph_io_adapter_t * key_source);
cmph_io_adapter_t *cmph_io_byte_vector_adapter(cmph_uint8 ** vector, cmph_uint32 nkeys); cmph_io_adapter_t *cmph_io_byte_vector_adapter(cmph_uint8 ** vector, cmph_uint32 nkeys);
void cmph_io_byte_vector_adapter_destroy(cmph_io_adapter_t * key_source); void cmph_io_byte_vector_adapter_destroy(cmph_io_adapter_t * key_source);
cmph_io_adapter_t *cmph_io_struct_vector_adapter(void * vector, cmph_uint32 struct_size, cmph_uint32 key_offset, cmph_uint32 key_len, cmph_uint32 nkeys); cmph_io_adapter_t *cmph_io_struct_vector_adapter(void * vector,
cmph_uint32 struct_size,
cmph_uint32 key_offset,
cmph_uint32 key_len,
cmph_uint32 nkeys);
void cmph_io_struct_vector_adapter_destroy(cmph_io_adapter_t * key_source); void cmph_io_struct_vector_adapter_destroy(cmph_io_adapter_t * key_source);
/** Hash configuration API **/ /** Hash configuration API **/
cmph_config_t *cmph_config_new(cmph_io_adapter_t *key_source); cmph_config_t *cmph_config_new(cmph_io_adapter_t *key_source);
void cmph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void cmph_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void cmph_config_set_verbosity(cmph_config_t *mph, cmph_uint32 verbosity); void cmph_config_set_verbosity(cmph_config_t *mph, cmph_uint32 verbosity);
void cmph_config_set_graphsize(cmph_config_t *mph, float c); void cmph_config_set_graphsize(cmph_config_t *mph, double c);
void cmph_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo); void cmph_config_set_algo(cmph_config_t *mph, CMPH_ALGO algo);
void cmph_config_set_tmp_dir(cmph_config_t *mph, cmph_uint8 *tmp_dir); void cmph_config_set_tmp_dir(cmph_config_t *mph, cmph_uint8 *tmp_dir);
void cmph_config_set_mphf_fd(cmph_config_t *mph, FILE *mphf_fd); void cmph_config_set_mphf_fd(cmph_config_t *mph, FILE *mphf_fd);
void cmph_config_set_b(cmph_config_t *mph, cmph_uint8 b); void cmph_config_set_b(cmph_config_t *mph, cmph_uint32 b);
void cmph_config_set_memory_availability(cmph_config_t *mph, cmph_uint32 memory_availability); void cmph_config_set_memory_availability(cmph_config_t *mph, cmph_uint32 memory_availability);
void cmph_config_destroy(cmph_config_t *mph); void cmph_config_destroy(cmph_config_t *mph);
@ -65,7 +70,8 @@ cmph_t *cmph_new(cmph_config_t *mph);
cmph_uint32 cmph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen); cmph_uint32 cmph_search(cmph_t *mphf, const char *key, cmph_uint32 keylen);
/** cmph_uint32 cmph_search_fingerprint(cmph_t *mphf, const char *key, cmph_uint32 keylen, cmph_uint32 * fingerprint); /** cmph_uint32 cmph_search_fingerprint(cmph_t *mphf, const char *key, cmph_uint32 keylen, cmph_uint32 * fingerprint);
* \brief Computes the mphf value and a fingerprint of 12 bytes (i.e., figerprint should be a prealocated area to fit three 4-byte integers). * \brief Computes the mphf value and a fingerprint of 12 bytes (i.e.,
* \brief figerprint should be a prealocated area to fit three 4-byte integers).
* \param mphf pointer to the resulting function * \param mphf pointer to the resulting function
* \param key is the key to be hashed * \param key is the key to be hashed
* \param keylen is the key legth in bytes * \param keylen is the key legth in bytes
@ -89,7 +95,8 @@ cmph_t *cmph_load(FILE *f);
/** \fn void cmph_pack(cmph_t *mphf, void *packed_mphf); /** \fn void cmph_pack(cmph_t *mphf, void *packed_mphf);
* \brief Support the ability to pack a perfect hash function into a preallocated contiguous memory space pointed by packed_mphf. * \brief Support the ability to pack a perfect hash function into a preallocated contiguous memory space pointed by packed_mphf.
* \param mphf pointer to the resulting mphf * \param mphf pointer to the resulting mphf
* \param packed_mphf pointer to the contiguous memory area used to store the resulting mphf. The size of packed_mphf must be at least cmph_packed_size() * \param packed_mphf pointer to the contiguous memory area used to store the
* \param resulting mphf. The size of packed_mphf must be at least cmph_packed_size()
*/ */
void cmph_pack(cmph_t *mphf, void *packed_mphf); void cmph_pack(cmph_t *mphf, void *packed_mphf);

8
src/cmph_structs.c
View File

@ -24,8 +24,8 @@ void __config_destroy(cmph_config_t *mph)
void __cmph_dump(cmph_t *mphf, FILE *fd) void __cmph_dump(cmph_t *mphf, FILE *fd)
{ {
fwrite(cmph_names[mphf->algo], (cmph_uint32)(strlen(cmph_names[mphf->algo]) + 1), 1, fd); fwrite(cmph_names[mphf->algo], (size_t)(strlen(cmph_names[mphf->algo]) + 1), (size_t)1, fd);
fwrite(&(mphf->size), sizeof(mphf->size), 1, fd); fwrite(&(mphf->size), sizeof(mphf->size), (size_t)1, fd);
} }
cmph_t *__cmph_load(FILE *f) cmph_t *__cmph_load(FILE *f)
{ {
@ -38,7 +38,7 @@ cmph_t *__cmph_load(FILE *f)
DEBUGP("Loading mphf\n"); DEBUGP("Loading mphf\n");
while(1) while(1)
{ {
cmph_uint32 c = fread(ptr, 1, 1, f); cmph_uint32 c = fread(ptr, (size_t)1, (size_t)1, f);
if (c != 1) return NULL; if (c != 1) return NULL;
if (*ptr == 0) break; if (*ptr == 0) break;
++ptr; ++ptr;
@ -57,7 +57,7 @@ cmph_t *__cmph_load(FILE *f)
} }
mphf = (cmph_t *)malloc(sizeof(cmph_t)); mphf = (cmph_t *)malloc(sizeof(cmph_t));
mphf->algo = algo; mphf->algo = algo;
fread(&(mphf->size), sizeof(mphf->size), 1, f); fread(&(mphf->size), sizeof(mphf->size), (size_t)1, f);
mphf->data = NULL; mphf->data = NULL;
DEBUGP("Algorithm is %s and mphf is sized %u\n", cmph_names[algo], mphf->size); DEBUGP("Algorithm is %s and mphf is sized %u\n", cmph_names[algo], mphf->size);

2
src/cmph_structs.h
View File

@ -10,7 +10,7 @@ struct __config_t
CMPH_ALGO algo; CMPH_ALGO algo;
cmph_io_adapter_t *key_source; cmph_io_adapter_t *key_source;
cmph_uint32 verbosity; cmph_uint32 verbosity;
float c; double c;
void *data; // algorithm dependent data void *data; // algorithm dependent data
}; };

1
src/cmph_types.h
View File

@ -4,7 +4,6 @@
typedef unsigned char cmph_uint8; typedef unsigned char cmph_uint8;
typedef unsigned short cmph_uint16; typedef unsigned short cmph_uint16;
typedef unsigned int cmph_uint32; typedef unsigned int cmph_uint32;
typedef float cmph_float32;
#if defined(__ia64) || defined(__x86_64__) #if defined(__ia64) || defined(__x86_64__)
/** \typedef long cmph_int64; /** \typedef long cmph_int64;

72
src/fch.c
View File

@ -55,7 +55,7 @@ void fch_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
} }
cmph_uint32 mixh10h11h12(cmph_uint32 b, cmph_float32 p1, cmph_float32 p2, cmph_uint32 initial_index) cmph_uint32 mixh10h11h12(cmph_uint32 b, double p1, double p2, cmph_uint32 initial_index)
{ {
if (initial_index < p1) initial_index %= (cmph_uint32)p2; /* h11 o h10 */ if (initial_index < p1) initial_index %= (cmph_uint32)p2; /* h11 o h10 */
else { /* h12 o h10 */ else { /* h12 o h10 */
@ -66,17 +66,17 @@ cmph_uint32 mixh10h11h12(cmph_uint32 b, cmph_float32 p1, cmph_float32 p2, cmph_u
} }
cmph_uint32 fch_calc_b(cmph_float32 c, cmph_uint32 m) cmph_uint32 fch_calc_b(double c, cmph_uint32 m)
{ {
return (cmph_uint32)ceil((c*m)/(log(m)/log(2) + 1)); return (cmph_uint32)ceil((c*m)/(log((double)m)/log(2.0) + 1));
} }
cmph_float32 fch_calc_p1(cmph_uint32 m) double fch_calc_p1(cmph_uint32 m)
{ {
return ceil(0.55*m); return ceil(0.55*m);
} }
cmph_float32 fch_calc_p2(cmph_uint32 b) double fch_calc_p2(cmph_uint32 b)
{ {
return ceil(0.3*b); return ceil(0.3*b);
} }
@ -120,13 +120,13 @@ static cmph_uint32 * ordering(fch_buckets_t * buckets)
static cmph_uint8 check_for_collisions_h2(fch_config_data_t *fch, fch_buckets_t * buckets, cmph_uint32 *sorted_indexes) static cmph_uint8 check_for_collisions_h2(fch_config_data_t *fch, fch_buckets_t * buckets, cmph_uint32 *sorted_indexes)
{ {
//cmph_uint32 max_size = fch_buckets_get_max_size(buckets); //cmph_uint32 max_size = fch_buckets_get_max_size(buckets);
cmph_uint8 * hashtable = (cmph_uint8 *)calloc(fch->m, sizeof(cmph_uint8)); cmph_uint8 * hashtable = (cmph_uint8 *)calloc((size_t)fch->m, sizeof(cmph_uint8));
cmph_uint32 nbuckets = fch_buckets_get_nbuckets(buckets); cmph_uint32 nbuckets = fch_buckets_get_nbuckets(buckets);
cmph_uint32 i = 0, index = 0, j =0; cmph_uint32 i = 0, index = 0, j =0;
for (i = 0; i < nbuckets; i++) for (i = 0; i < nbuckets; i++)
{ {
cmph_uint32 nkeys = fch_buckets_get_size(buckets, sorted_indexes[i]); cmph_uint32 nkeys = fch_buckets_get_size(buckets, sorted_indexes[i]);
memset(hashtable, 0, fch->m); memset(hashtable, 0, (size_t)fch->m);
//DEBUGP("bucket %u -- nkeys: %u\n", i, nkeys); //DEBUGP("bucket %u -- nkeys: %u\n", i, nkeys);
for (j = 0; j < nkeys; j++) for (j = 0; j < nkeys; j++)
{ {
@ -157,15 +157,15 @@ static void permut(cmph_uint32 * vector, cmph_uint32 n)
static cmph_uint8 searching(fch_config_data_t *fch, fch_buckets_t *buckets, cmph_uint32 *sorted_indexes) static cmph_uint8 searching(fch_config_data_t *fch, fch_buckets_t *buckets, cmph_uint32 *sorted_indexes)
{ {
cmph_uint32 * random_table = (cmph_uint32 *) calloc(fch->m, sizeof(cmph_uint32)); cmph_uint32 * random_table = (cmph_uint32 *) calloc((size_t)fch->m, sizeof(cmph_uint32));
cmph_uint32 * map_table = (cmph_uint32 *) calloc(fch->m, sizeof(cmph_uint32)); cmph_uint32 * map_table = (cmph_uint32 *) calloc((size_t)fch->m, sizeof(cmph_uint32));
cmph_uint32 iteration_to_generate_h2 = 0; cmph_uint32 iteration_to_generate_h2 = 0;
cmph_uint32 searching_iterations = 0; cmph_uint32 searching_iterations = 0;
cmph_uint8 restart = 0; cmph_uint8 restart = 0;
cmph_uint32 nbuckets = fch_buckets_get_nbuckets(buckets); cmph_uint32 nbuckets = fch_buckets_get_nbuckets(buckets);
cmph_uint32 i, j, z, counter = 0, filled_count = 0; cmph_uint32 i, j, z, counter = 0, filled_count = 0;
if (fch->g) free (fch->g); if (fch->g) free (fch->g);
fch->g = (cmph_uint32 *) calloc(fch->b, sizeof(cmph_uint32)); fch->g = (cmph_uint32 *) calloc((size_t)fch->b, sizeof(cmph_uint32));
//DEBUGP("max bucket size: %u\n", fch_buckets_get_max_size(buckets)); //DEBUGP("max bucket size: %u\n", fch_buckets_get_max_size(buckets));
@ -245,7 +245,7 @@ static cmph_uint8 searching(fch_config_data_t *fch, fch_buckets_t *buckets, cmph
cmph_t *fch_new(cmph_config_t *mph, float c) cmph_t *fch_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
fch_data_t *fchf = NULL; fch_data_t *fchf = NULL;
@ -320,22 +320,22 @@ int fch_dump(cmph_t *mphf, FILE *fd)
hash_state_dump(data->h1, &buf, &buflen); hash_state_dump(data->h1, &buf, &buflen);
//DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); //DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
hash_state_dump(data->h2, &buf, &buflen); hash_state_dump(data->h2, &buf, &buflen);
//DEBUGP("Dumping hash state with %u bytes to disk\n", buflen); //DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), 1, fd); fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, buflen, 1, fd); fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf); free(buf);
fwrite(&(data->m), sizeof(cmph_uint32), 1, fd); fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->c), sizeof(cmph_float32), 1, fd); fwrite(&(data->c), sizeof(double), (size_t)1, fd);
fwrite(&(data->b), sizeof(cmph_uint32), 1, fd); fwrite(&(data->b), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(&(data->p1), sizeof(cmph_float32), 1, fd); fwrite(&(data->p1), sizeof(double), (size_t)1, fd);
fwrite(&(data->p2), sizeof(cmph_float32), 1, fd); fwrite(&(data->p2), sizeof(double), (size_t)1, fd);
fwrite(data->g, sizeof(cmph_uint32)*(data->b), 1, fd); fwrite(data->g, sizeof(cmph_uint32)*(data->b), (size_t)1, fd);
#ifdef DEBUG #ifdef DEBUG
cmph_uint32 i; cmph_uint32 i;
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
@ -355,10 +355,10 @@ void fch_load(FILE *f, cmph_t *mphf)
mphf->data = fch; mphf->data = fch;
//DEBUGP("Reading h1\n"); //DEBUGP("Reading h1\n");
fch->h1 = NULL; fch->h1 = NULL;
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
//DEBUGP("Hash state of h1 has %u bytes\n", buflen); //DEBUGP("Hash state of h1 has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
fch->h1 = hash_state_load(buf, buflen); fch->h1 = hash_state_load(buf, buflen);
free(buf); free(buf);
@ -366,23 +366,23 @@ void fch_load(FILE *f, cmph_t *mphf)
mphf->data = fch; mphf->data = fch;
//DEBUGP("Reading h2\n"); //DEBUGP("Reading h2\n");
fch->h2 = NULL; fch->h2 = NULL;
fread(&buflen, sizeof(cmph_uint32), 1, f); fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
//DEBUGP("Hash state of h2 has %u bytes\n", buflen); //DEBUGP("Hash state of h2 has %u bytes\n", buflen);
buf = (char *)malloc(buflen); buf = (char *)malloc((size_t)buflen);
fread(buf, buflen, 1, f); fread(buf, (size_t)buflen, (size_t)1, f);
fch->h2 = hash_state_load(buf, buflen); fch->h2 = hash_state_load(buf, buflen);
free(buf); free(buf);
//DEBUGP("Reading m and n\n"); //DEBUGP("Reading m and n\n");
fread(&(fch->m), sizeof(cmph_uint32), 1, f); fread(&(fch->m), sizeof(cmph_uint32), (size_t)1, f);
fread(&(fch->c), sizeof(cmph_float32), 1, f); fread(&(fch->c), sizeof(double), (size_t)1, f);
fread(&(fch->b), sizeof(cmph_uint32), 1, f); fread(&(fch->b), sizeof(cmph_uint32), (size_t)1, f);
fread(&(fch->p1), sizeof(cmph_float32), 1, f); fread(&(fch->p1), sizeof(double), (size_t)1, f);
fread(&(fch->p2), sizeof(cmph_float32), 1, f); fread(&(fch->p2), sizeof(double), (size_t)1, f);
fch->g = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*fch->b); fch->g = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*fch->b);
fread(fch->g, fch->b*sizeof(cmph_uint32), 1, f); fread(fch->g, fch->b*sizeof(cmph_uint32), (size_t)1, f);
#ifdef DEBUG #ifdef DEBUG
cmph_uint32 i; cmph_uint32 i;
fprintf(stderr, "G: "); fprintf(stderr, "G: ");
@ -498,7 +498,7 @@ cmph_uint32 fch_packed_size(cmph_t *mphf)
CMPH_HASH h2_type = hash_get_type(data->h2); CMPH_HASH h2_type = hash_get_type(data->h2);
return (sizeof(CMPH_ALGO) + hash_state_packed_size(h1_type) + hash_state_packed_size(h2_type) + return (sizeof(CMPH_ALGO) + hash_state_packed_size(h1_type) + hash_state_packed_size(h2_type) +
4*sizeof(cmph_uint32) + 2*sizeof(cmph_float32) + sizeof(cmph_uint32)*(data->b)); 4*sizeof(cmph_uint32) + 2*sizeof(double) + sizeof(cmph_uint32)*(data->b));
} }
@ -525,10 +525,10 @@ cmph_uint32 fch_search_packed(void *packed_mphf, const char *key, cmph_uint32 ke
register cmph_uint32 b = *g_ptr++; register cmph_uint32 b = *g_ptr++;
register cmph_float32 p1 = (cmph_float32)(*g_ptr); register double p1 = (double)(*g_ptr);
g_ptr++; g_ptr++;
register cmph_float32 p2 = (cmph_float32)(*g_ptr); register double p2 = (double)(*g_ptr);
g_ptr++; g_ptr++;
register cmph_uint32 h1 = hash_packed(h1_ptr, h1_type, key, keylen) % m; register cmph_uint32 h1 = hash_packed(h1_ptr, h1_type, key, keylen) % m;

10
src/fch.h
View File

@ -7,15 +7,15 @@ typedef struct __fch_data_t fch_data_t;
typedef struct __fch_config_data_t fch_config_data_t; typedef struct __fch_config_data_t fch_config_data_t;
/* Parameters calculation */ /* Parameters calculation */
cmph_uint32 fch_calc_b(cmph_float32 c, cmph_uint32 m); cmph_uint32 fch_calc_b(double c, cmph_uint32 m);
cmph_float32 fch_calc_p1(cmph_uint32 m); double fch_calc_p1(cmph_uint32 m);
cmph_float32 fch_calc_p2(cmph_uint32 b); double fch_calc_p2(cmph_uint32 b);
cmph_uint32 mixh10h11h12(cmph_uint32 b, cmph_float32 p1, cmph_float32 p2, cmph_uint32 initial_index); cmph_uint32 mixh10h11h12(cmph_uint32 b, double p1, double p2, cmph_uint32 initial_index);
fch_config_data_t *fch_config_new(); fch_config_data_t *fch_config_new();
void fch_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void fch_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void fch_config_destroy(cmph_config_t *mph); void fch_config_destroy(cmph_config_t *mph);
cmph_t *fch_new(cmph_config_t *mph, float c); cmph_t *fch_new(cmph_config_t *mph, double c);
void fch_load(FILE *f, cmph_t *mphf); void fch_load(FILE *f, cmph_t *mphf);
int fch_dump(cmph_t *mphf, FILE *f); int fch_dump(cmph_t *mphf, FILE *f);

6
src/fch_buckets.c
View File

@ -117,7 +117,7 @@ fch_buckets_t * fch_buckets_new(cmph_uint32 nbuckets)
cmph_uint32 i; cmph_uint32 i;
fch_buckets_t *buckets = (fch_buckets_t *)malloc(sizeof(fch_buckets_t)); fch_buckets_t *buckets = (fch_buckets_t *)malloc(sizeof(fch_buckets_t));
assert(buckets); assert(buckets);
buckets->values = (fch_bucket_t *)calloc(nbuckets, sizeof(fch_bucket_t)); buckets->values = (fch_bucket_t *)calloc((size_t)nbuckets, sizeof(fch_bucket_t));
for (i = 0; i < nbuckets; i++) fch_bucket_new(buckets->values + i); for (i = 0; i < nbuckets; i++) fch_bucket_new(buckets->values + i);
assert(buckets->values); assert(buckets->values);
buckets->nbuckets = nbuckets; buckets->nbuckets = nbuckets;
@ -174,8 +174,8 @@ cmph_uint32 * fch_buckets_get_indexes_sorted_by_size(fch_buckets_t * buckets)
{ {
int i = 0; int i = 0;
cmph_uint32 sum = 0, value; cmph_uint32 sum = 0, value;
cmph_uint32 *nbuckets_size = (cmph_uint32 *) calloc(buckets->max_size + 1, sizeof(cmph_uint32)); cmph_uint32 *nbuckets_size = (cmph_uint32 *) calloc((size_t)buckets->max_size + 1, sizeof(cmph_uint32));
cmph_uint32 * sorted_indexes = (cmph_uint32 *) calloc(buckets->nbuckets, sizeof(cmph_uint32)); cmph_uint32 * sorted_indexes = (cmph_uint32 *) calloc((size_t)buckets->nbuckets, sizeof(cmph_uint32));
// collect how many buckets for each size. // collect how many buckets for each size.
for(i = 0; i < buckets->nbuckets; i++) nbuckets_size[fch_bucket_size(buckets->values + i)] ++; for(i = 0; i < buckets->nbuckets; i++) nbuckets_size[fch_bucket_size(buckets->values + i)] ++;

12
src/fch_structs.h
View File

@ -6,10 +6,10 @@
struct __fch_data_t struct __fch_data_t
{ {
cmph_uint32 m; // words count cmph_uint32 m; // words count
cmph_float32 c; // constant c double c; // constant c
cmph_uint32 b; // parameter b = ceil(c*m/(log(m)/log(2) + 1)). Don't need to be stored cmph_uint32 b; // parameter b = ceil(c*m/(log(m)/log(2) + 1)). Don't need to be stored
cmph_float32 p1; // constant p1 = ceil(0.6*m). Don't need to be stored double p1; // constant p1 = ceil(0.6*m). Don't need to be stored
cmph_float32 p2; // constant p2 = ceil(0.3*b). Don't need to be stored double p2; // constant p2 = ceil(0.3*b). Don't need to be stored
cmph_uint32 *g; // g function. cmph_uint32 *g; // g function.
hash_state_t *h1; // h10 function. hash_state_t *h1; // h10 function.
hash_state_t *h2; // h20 function. hash_state_t *h2; // h20 function.
@ -19,10 +19,10 @@ struct __fch_config_data_t
{ {
CMPH_HASH hashfuncs[2]; CMPH_HASH hashfuncs[2];
cmph_uint32 m; // words count cmph_uint32 m; // words count
cmph_float32 c; // constant c double c; // constant c
cmph_uint32 b; // parameter b = ceil(c*m/(log(m)/log(2) + 1)). Don't need to be stored cmph_uint32 b; // parameter b = ceil(c*m/(log(m)/log(2) + 1)). Don't need to be stored
cmph_float32 p1; // constant p1 = ceil(0.6*m). Don't need to be stored double p1; // constant p1 = ceil(0.6*m). Don't need to be stored
cmph_float32 p2; // constant p2 = ceil(0.3*b). Don't need to be stored double p2; // constant p2 = ceil(0.3*b). Don't need to be stored
cmph_uint32 *g; // g function. cmph_uint32 *g; // g function.
hash_state_t *h1; // h10 function. hash_state_t *h1; // h10 function.
hash_state_t *h2; // h20 function. hash_state_t *h2; // h20 function.

6
src/graph.c
View File

@ -230,7 +230,8 @@ int graph_is_cyclic(graph_t *g)
cmph_uint32 i; cmph_uint32 i;
cmph_uint32 v; cmph_uint32 v;
char *deleted = (char *)malloc((g->nedges*sizeof(char))/8 + 1); char *deleted = (char *)malloc((g->nedges*sizeof(char))/8 + 1);
memset(deleted, 0, g->nedges/8 + 1); size_t deleted_len = g->nedges/8 + 1;
memset(deleted, 0, deleted_len);
DEBUGP("Looking for cycles in graph with %u vertices and %u edges\n", g->nnodes, g->nedges); DEBUGP("Looking for cycles in graph with %u vertices and %u edges\n", g->nnodes, g->nedges);
for (v = 0; v < g->nnodes; ++v) for (v = 0; v < g->nnodes; ++v)
@ -260,7 +261,8 @@ void graph_obtain_critical_nodes(graph_t *g) /* included -- Fabiano*/
cmph_uint32 i; cmph_uint32 i;
cmph_uint32 v; cmph_uint32 v;
char *deleted = (char *)malloc((g->nedges*sizeof(char))/8+1); char *deleted = (char *)malloc((g->nedges*sizeof(char))/8+1);
memset(deleted, 0, g->nedges/8 + 1); size_t deleted_len = g->nedges/8 + 1;
memset(deleted, 0, deleted_len);
free(g->critical_nodes); free(g->critical_nodes);
g->critical_nodes = (cmph_uint8 *)malloc((g->nnodes*sizeof(cmph_uint8))/8 + 1); g->critical_nodes = (cmph_uint8 *)malloc((g->nnodes*sizeof(cmph_uint8))/8 + 1);
g->ncritical_nodes = 0; g->ncritical_nodes = 0;

4
src/hash.c
View File

@ -54,6 +54,7 @@ void hash_vector(hash_state_t *state, const char *key, cmph_uint32 keylen, cmph_
void hash_state_dump(hash_state_t *state, char **buf, cmph_uint32 *buflen) void hash_state_dump(hash_state_t *state, char **buf, cmph_uint32 *buflen)
{ {
char *algobuf; char *algobuf;
size_t len;
switch (state->hashfunc) switch (state->hashfunc)
{ {
case CMPH_HASH_JENKINS: case CMPH_HASH_JENKINS:
@ -66,7 +67,8 @@ void hash_state_dump(hash_state_t *state, char **buf, cmph_uint32 *buflen)
*buf = (char *)malloc(strlen(cmph_hash_names[state->hashfunc]) + 1 + *buflen); *buf = (char *)malloc(strlen(cmph_hash_names[state->hashfunc]) + 1 + *buflen);
memcpy(*buf, cmph_hash_names[state->hashfunc], strlen(cmph_hash_names[state->hashfunc]) + 1); memcpy(*buf, cmph_hash_names[state->hashfunc], strlen(cmph_hash_names[state->hashfunc]) + 1);
DEBUGP("Algobuf is %u\n", *(cmph_uint32 *)algobuf); DEBUGP("Algobuf is %u\n", *(cmph_uint32 *)algobuf);
memcpy(*buf + strlen(cmph_hash_names[state->hashfunc]) + 1, algobuf, *buflen); len = *buflen;
memcpy(*buf + strlen(cmph_hash_names[state->hashfunc]) + 1, algobuf, len);
*buflen = (cmph_uint32)strlen(cmph_hash_names[state->hashfunc]) + 1 + *buflen; *buflen = (cmph_uint32)strlen(cmph_hash_names[state->hashfunc]) + 1 + *buflen;
free(algobuf); free(algobuf);
return; return;

2
src/hashtree.c
View File

@ -46,7 +46,7 @@ void hashtree_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs)
} }
} }
cmph_t *hashtree_new(cmph_config_t *mph, float c) cmph_t *hashtree_new(cmph_config_t *mph, double c)
{ {
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
hashtree_data_t *hashtreef = NULL; hashtree_data_t *hashtreef = NULL;

2
src/hashtree.h
View File

@ -10,7 +10,7 @@ hashtree_config_data_t *hashtree_config_new();
void hashtree_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs); void hashtree_config_set_hashfuncs(cmph_config_t *mph, CMPH_HASH *hashfuncs);
void hashtree_config_set_leaf_algo(cmph_config_t *mph, CMPH_ALGO leaf_algo); void hashtree_config_set_leaf_algo(cmph_config_t *mph, CMPH_ALGO leaf_algo);
void hashtree_config_destroy(cmph_config_t *mph); void hashtree_config_destroy(cmph_config_t *mph);
cmph_t *hashtree_new(cmph_config_t *mph, float c); cmph_t *hashtree_new(cmph_config_t *mph, double c);
void hashtree_load(FILE *f, cmph_t *mphf); void hashtree_load(FILE *f, cmph_t *mphf);
int hashtree_dump(cmph_t *mphf, FILE *f); int hashtree_dump(cmph_t *mphf, FILE *f);

2
src/hashtree_structs.h
View File

@ -6,7 +6,7 @@
struct __hashtree_data_t struct __hashtree_data_t
{ {
cmph_uint32 m; //edges (words) count cmph_uint32 m; //edges (words) count
cmph_float32 c; //constant c double c; //constant c
cmph_uint8 *size; //size[i] stores the number of edges represented by g[i] cmph_uint8 *size; //size[i] stores the number of edges represented by g[i]
cmph_uint32 **g; cmph_uint32 **g;
cmph_uint32 k; //number of components cmph_uint32 k; //number of components

2
src/jenkins_hash.c
View File

@ -220,7 +220,7 @@ void jenkins_hash_vector_(jenkins_state_t *state, const char *k, cmph_uint32 key
void jenkins_state_dump(jenkins_state_t *state, char **buf, cmph_uint32 *buflen) void jenkins_state_dump(jenkins_state_t *state, char **buf, cmph_uint32 *buflen)
{ {
*buflen = sizeof(cmph_uint32); *buflen = sizeof(cmph_uint32);
*buf = (char *)malloc(*buflen); *buf = (char *)malloc(sizeof(cmph_uint32));
if (!*buf) if (!*buf)
{ {
*buflen = UINT_MAX; *buflen = UINT_MAX;

8
src/main.c
View File

@ -57,7 +57,7 @@ void usage_long(const char *prg)
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
char verbosity = 0; cmph_uint32 verbosity = 0;
char generate = 0; char generate = 0;
char *mphf_file = NULL; char *mphf_file = NULL;
FILE *mphf_fd = stdout; FILE *mphf_fd = stdout;
@ -69,7 +69,7 @@ int main(int argc, char **argv)
cmph_uint32 nhashes = 0; cmph_uint32 nhashes = 0;
cmph_uint32 i; cmph_uint32 i;
CMPH_ALGO mph_algo = CMPH_CHM; CMPH_ALGO mph_algo = CMPH_CHM;
float c = 0; double c = 0;
cmph_config_t *config = NULL; cmph_config_t *config = NULL;
cmph_t *mphf = NULL; cmph_t *mphf = NULL;
char * tmp_dir = NULL; char * tmp_dir = NULL;
@ -211,7 +211,7 @@ int main(int argc, char **argv)
{ {
mphf_file = (char *)malloc(strlen(keys_file) + 5); mphf_file = (char *)malloc(strlen(keys_file) + 5);
memcpy(mphf_file, keys_file, strlen(keys_file)); memcpy(mphf_file, keys_file, strlen(keys_file));
memcpy(mphf_file + strlen(keys_file), ".mph\0", 5); memcpy(mphf_file + strlen(keys_file), ".mph\0", (size_t)5);
} }
keys_fd = fopen(keys_file, "r"); keys_fd = fopen(keys_file, "r");
@ -281,7 +281,7 @@ int main(int argc, char **argv)
} }
cmph_uint32 siz = cmph_size(mphf); cmph_uint32 siz = cmph_size(mphf);
hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8)); hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8));
memset(hashtable, 0, siz); memset(hashtable, 0,(size_t) siz);
//check all keys //check all keys
for (i = 0; i < source->nkeys; ++i) for (i = 0; i < source->nkeys; ++i)
{ {

5
src/vqueue.c
View File

@ -10,11 +10,12 @@ struct __vqueue_t
vqueue_t * vqueue_new(cmph_uint32 capacity) vqueue_t * vqueue_new(cmph_uint32 capacity)
{ {
size_t capacity_plus_one = capacity + 1;
vqueue_t *q = (vqueue_t *)malloc(sizeof(vqueue_t)); vqueue_t *q = (vqueue_t *)malloc(sizeof(vqueue_t));
assert(q); assert(q);
q->values = (cmph_uint32 *)calloc(capacity+1, sizeof(cmph_uint32)); q->values = (cmph_uint32 *)calloc(capacity_plus_one, sizeof(cmph_uint32));
q->beg = q->end = 0; q->beg = q->end = 0;
q->capacity = capacity+1; q->capacity = capacity_plus_one;
return q; return q;
} }

4
tests/mphf_fingerprint_tests.c
View File

@ -93,7 +93,7 @@ int main(int argc, char **argv)
{ {
mphf_file = (char *)malloc(strlen(keys_file) + 5); mphf_file = (char *)malloc(strlen(keys_file) + 5);
memcpy(mphf_file, keys_file, strlen(keys_file)); memcpy(mphf_file, keys_file, strlen(keys_file));
memcpy(mphf_file + strlen(keys_file), ".mph\0", 5); memcpy(mphf_file + strlen(keys_file), ".mph\0", (size_t)5);
} }
keys_fd = fopen(keys_file, "r"); keys_fd = fopen(keys_file, "r");
@ -125,7 +125,7 @@ int main(int argc, char **argv)
} }
cmph_uint32 siz = cmph_size(mphf); cmph_uint32 siz = cmph_size(mphf);
hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8)); hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8));
memset(hashtable, 0, siz); memset(hashtable, 0, (size_t)siz);
//check all keys //check all keys
for (i = 0; i < source->nkeys; ++i) for (i = 0; i < source->nkeys; ++i)
{ {

4
tests/mphf_tests.c
View File

@ -92,7 +92,7 @@ int main(int argc, char **argv)
{ {
mphf_file = (char *)malloc(strlen(keys_file) + 5); mphf_file = (char *)malloc(strlen(keys_file) + 5);
memcpy(mphf_file, keys_file, strlen(keys_file)); memcpy(mphf_file, keys_file, strlen(keys_file));
memcpy(mphf_file + strlen(keys_file), ".mph\0", 5); memcpy(mphf_file + strlen(keys_file), ".mph\0", (size_t)5);
} }
keys_fd = fopen(keys_file, "r"); keys_fd = fopen(keys_file, "r");
@ -124,7 +124,7 @@ int main(int argc, char **argv)
} }
cmph_uint32 siz = cmph_size(mphf); cmph_uint32 siz = cmph_size(mphf);
hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8)); hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8));
memset(hashtable, 0, siz); memset(hashtable, 0, (size_t)siz);
//check all keys //check all keys
for (i = 0; i < source->nkeys; ++i) for (i = 0; i < source->nkeys; ++i)
{ {

6
tests/packed_mphf_tests.c
View File

@ -93,7 +93,7 @@ int main(int argc, char **argv)
{ {
mphf_file = (char *)malloc(strlen(keys_file) + 5); mphf_file = (char *)malloc(strlen(keys_file) + 5);
memcpy(mphf_file, keys_file, strlen(keys_file)); memcpy(mphf_file, keys_file, strlen(keys_file));
memcpy(mphf_file + strlen(keys_file), ".mph\0", 5); memcpy(mphf_file + strlen(keys_file), ".mph\0", (size_t)5);
} }
keys_fd = fopen(keys_file, "r"); keys_fd = fopen(keys_file, "r");
@ -125,7 +125,7 @@ int main(int argc, char **argv)
} }
cmph_uint32 siz = cmph_size(mphf); cmph_uint32 siz = cmph_size(mphf);
hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8)); hashtable = (cmph_uint8*)malloc(siz*sizeof(cmph_uint8));
memset(hashtable, 0, siz); memset(hashtable, 0, (size_t)siz);
// packing the function // packing the function
/* Determine how much space is needed to pack the mphf. */ /* Determine how much space is needed to pack the mphf. */
@ -133,7 +133,7 @@ int main(int argc, char **argv)
fprintf(stderr, "packed_size = %u\n", packed_size); fprintf(stderr, "packed_size = %u\n", packed_size);
/* Make sure that we have enough space to pack the mphf. */ /* Make sure that we have enough space to pack the mphf. */
cmph_uint8 * packed_mphf = calloc(packed_size,1); cmph_uint8 * packed_mphf = calloc((size_t)packed_size,(size_t)1);
/* Pack the mphf. */ /* Pack the mphf. */
cmph_pack(mphf, packed_mphf); cmph_pack(mphf, packed_mphf);