motiejus/turbonss
1
Fork 0
Code Releases Activity

select data structure added

Browse Source
This commit is contained in:
fc_botelho
2009-03-14 23:24:05 +00:00
parent 042ff338c3
commit 579487c5cf
18 changed files with 817 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/brz.c
View File

@@ -227,6 +227,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
cmph_uint32 *buffer_h0 = NULL;
cmph_uint32 nflushes = 0;
cmph_uint32 h0;
register cmph_uint32 nbytes;
FILE * tmp_fd = NULL;
buffer_manager_t * buff_manager = NULL;
char *filename = NULL;
@@ -280,7 +281,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
for(i = 0; i < nkeys_in_buffer; i++)
{
memcpy(&keylen1, buffer + keys_index[i], sizeof(keylen1));
fwrite(buffer + keys_index[i], (size_t)1, keylen1 + sizeof(keylen1), tmp_fd);
nbytes = fwrite(buffer + keys_index[i], (size_t)1, keylen1 + sizeof(keylen1), tmp_fd);
}
nkeys_in_buffer = 0;
memory_usage = 0;
@@ -340,7 +341,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
for(i = 0; i < nkeys_in_buffer; i++)
{
memcpy(&keylen1, buffer + keys_index[i], sizeof(keylen1));
fwrite(buffer + keys_index[i], (size_t)1, keylen1 + sizeof(keylen1), tmp_fd);
nbytes = fwrite(buffer + keys_index[i], (size_t)1, keylen1 + sizeof(keylen1), tmp_fd);
}
nkeys_in_buffer = 0;
memory_usage = 0;
@@ -359,12 +360,12 @@ static int brz_gen_mphf(cmph_config_t *mph)
fprintf(stderr, "\nMPHF generation \n");
}
/* Starting to dump to disk the resultant MPHF: __cmph_dump function */
fwrite(cmph_names[CMPH_BRZ], (size_t)(strlen(cmph_names[CMPH_BRZ]) + 1), (size_t)1, brz->mphf_fd);
fwrite(&(brz->m), sizeof(brz->m), (size_t)1, brz->mphf_fd);
fwrite(&(brz->c), sizeof(double), (size_t)1, brz->mphf_fd);
fwrite(&(brz->algo), sizeof(brz->algo), (size_t)1, brz->mphf_fd);
fwrite(&(brz->k), sizeof(cmph_uint32), (size_t)1, brz->mphf_fd); // number of MPHFs
fwrite(brz->size, sizeof(cmph_uint8)*(brz->k), (size_t)1, brz->mphf_fd);
nbytes = fwrite(cmph_names[CMPH_BRZ], (size_t)(strlen(cmph_names[CMPH_BRZ]) + 1), (size_t)1, brz->mphf_fd);
nbytes = fwrite(&(brz->m), sizeof(brz->m), (size_t)1, brz->mphf_fd);
nbytes = fwrite(&(brz->c), sizeof(double), (size_t)1, brz->mphf_fd);
nbytes = fwrite(&(brz->algo), sizeof(brz->algo), (size_t)1, brz->mphf_fd);
nbytes = fwrite(&(brz->k), sizeof(cmph_uint32), (size_t)1, brz->mphf_fd); // number of MPHFs
nbytes = fwrite(brz->size, sizeof(cmph_uint8)*(brz->k), (size_t)1, brz->mphf_fd);
//tmp_fds = (FILE **)calloc(nflushes, sizeof(FILE *));
buff_manager = buffer_manager_new(brz->memory_availability, nflushes);
@@ -473,7 +474,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
break;
default: assert(0);
}
fwrite(bufmphf, (size_t)buflenmphf, (size_t)1, brz->mphf_fd);
nbytes = fwrite(bufmphf, (size_t)buflenmphf, (size_t)1, brz->mphf_fd);
free(bufmphf);
bufmphf = NULL;
cmph_config_destroy(config);
@@ -557,17 +558,18 @@ int brz_dump(cmph_t *mphf, FILE *fd)
brz_data_t *data = (brz_data_t *)mphf->data;
char *buf = NULL;
cmph_uint32 buflen;
register cmph_uint32 nbytes;
DEBUGP("Dumping brzf\n");
// The initial part of the MPHF have already been dumped to disk during construction
// Dumping h0
hash_state_dump(data->h0, &buf, &buflen);
DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
fwrite(buf, (size_t)buflen, (size_t)1, fd);
nbytes = fwrite(&buflen, sizeof(cmph_uint32), (size_t)1, fd);
nbytes = fwrite(buf, (size_t)buflen, (size_t)1, fd);
free(buf);
// Dumping m and the vector offset.
fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
fwrite(data->offset, sizeof(cmph_uint32)*(data->k), (size_t)1, fd);
nbytes = fwrite(&(data->m), sizeof(cmph_uint32), (size_t)1, fd);
nbytes = fwrite(data->offset, sizeof(cmph_uint32)*(data->k), (size_t)1, fd);
return 1;
}
@@ -575,16 +577,17 @@ void brz_load(FILE *f, cmph_t *mphf)
{
char *buf = NULL;
cmph_uint32 buflen;
register cmph_uint32 nbytes;
cmph_uint32 i, n;
brz_data_t *brz = (brz_data_t *)malloc(sizeof(brz_data_t));
DEBUGP("Loading brz mphf\n");
mphf->data = brz;
fread(&(brz->c), sizeof(double), (size_t)1, f);
fread(&(brz->algo), sizeof(brz->algo), (size_t)1, f); // Reading algo.
fread(&(brz->k), sizeof(cmph_uint32), (size_t)1, f);
nbytes = fread(&(brz->c), sizeof(double), (size_t)1, f);
nbytes = fread(&(brz->algo), sizeof(brz->algo), (size_t)1, f); // Reading algo.
nbytes = fread(&(brz->k), sizeof(cmph_uint32), (size_t)1, f);
brz->size = (cmph_uint8 *) malloc(sizeof(cmph_uint8)*brz->k);
fread(brz->size, sizeof(cmph_uint8)*(brz->k), (size_t)1, f);
nbytes = fread(brz->size, sizeof(cmph_uint8)*(brz->k), (size_t)1, f);
brz->h1 = (hash_state_t **)malloc(sizeof(hash_state_t *)*brz->k);
brz->h2 = (hash_state_t **)malloc(sizeof(hash_state_t *)*brz->k);
brz->g = (cmph_uint8 **) calloc((size_t)brz->k, sizeof(cmph_uint8 *));
@@ -593,17 +596,17 @@ void brz_load(FILE *f, cmph_t *mphf)
for(i = 0; i < brz->k; i++)
{
// h1
fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
nbytes = fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state 1 has %u bytes\n", buflen);
buf = (char *)malloc((size_t)buflen);
fread(buf, (size_t)buflen, (size_t)1, f);
nbytes = fread(buf, (size_t)buflen, (size_t)1, f);
brz->h1[i] = hash_state_load(buf, buflen);
free(buf);
//h2
fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
nbytes = fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state 2 has %u bytes\n", buflen);
buf = (char *)malloc((size_t)buflen);
fread(buf, (size_t)buflen, (size_t)1, f);
nbytes = fread(buf, (size_t)buflen, (size_t)1, f);
brz->h2[i] = hash_state_load(buf, buflen);
free(buf);
switch(brz->algo)
@@ -618,20 +621,20 @@ void brz_load(FILE *f, cmph_t *mphf)
}
DEBUGP("g_i has %u bytes\n", n);
brz->g[i] = (cmph_uint8 *)calloc((size_t)n, sizeof(cmph_uint8));
fread(brz->g[i], sizeof(cmph_uint8)*n, (size_t)1, f);
nbytes = fread(brz->g[i], sizeof(cmph_uint8)*n, (size_t)1, f);
}
//loading h0
fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
nbytes = fread(&buflen, sizeof(cmph_uint32), (size_t)1, f);
DEBUGP("Hash state has %u bytes\n", buflen);
buf = (char *)malloc((size_t)buflen);
fread(buf, (size_t)buflen, (size_t)1, f);
nbytes = fread(buf, (size_t)buflen, (size_t)1, f);
brz->h0 = hash_state_load(buf, buflen);
free(buf);
//loading c, m, and the vector offset.
fread(&(brz->m), sizeof(cmph_uint32), (size_t)1, f);
nbytes = fread(&(brz->m), sizeof(cmph_uint32), (size_t)1, f);
brz->offset = (cmph_uint32 *)malloc(sizeof(cmph_uint32)*brz->k);
fread(brz->offset, sizeof(cmph_uint32)*(brz->k), (size_t)1, f);
nbytes = fread(brz->offset, sizeof(cmph_uint32)*(brz->k), (size_t)1, f);
return;
}