Remaining part of the fix for bug 3465649. This one fixes both BRZ and

CHD_PH for small key sets.

examples/Makefile.am

@@ -1,4 +1,4 @@
-noinst_PROGRAMS = vector_adapter_ex1 file_adapter_ex2 struct_vector_adapter_ex3 small_set_test
+noinst_PROGRAMS = vector_adapter_ex1 file_adapter_ex2 struct_vector_adapter_ex3 small_set_ex4
 
 INCLUDES = -I../src/
 
@@ -11,5 +11,5 @@ file_adapter_ex2_SOURCES   = file_adapter_ex2.c
 struct_vector_adapter_ex3_LDADD     = ../src/libcmph.la
 struct_vector_adapter_ex3_SOURCES   = struct_vector_adapter_ex3.c
 
-small_set_test_LDADD     = ../src/libcmph.la
+small_set_ex4_LDADD     = ../src/libcmph.la
-small_set_test_SOURCES   = small_set_test.c
+small_set_ex4_SOURCES   = small_set_ex4.c

examples/small_set_ex4.c

@@ -6,6 +6,8 @@ int test(cmph_uint32* items_to_hash, cmph_uint32 items_len, CMPH_ALGO alg_n)
     cmph_config_t *config;
     cmph_io_adapter_t *source;
     cmph_uint32 i;
+    char filename[256];
+    FILE* mphf_fd = NULL;
 
     printf("%s (%u)\n", cmph_names[alg_n], alg_n);
 
@@ -16,9 +18,21 @@ int test(cmph_uint32* items_to_hash, cmph_uint32 items_len, CMPH_ALGO alg_n)
                                            items_len);
     config = cmph_config_new(source);
     cmph_config_set_algo(config, alg_n);
+    if (alg_n == CMPH_BRZ) {
+        sprintf(filename, "%s_%u.mph", cmph_names[alg_n], items_len);
+        mphf_fd = fopen(filename, "w");
+        cmph_config_set_mphf_fd(config, mphf_fd);
+    }
     hash = cmph_new(config);
     cmph_config_destroy(config);
 
+    if (alg_n == CMPH_BRZ) {
+        cmph_dump(hash, mphf_fd);
+        cmph_destroy(hash);
+        fclose(mphf_fd);
+        mphf_fd = fopen(filename, "r");
+        hash = cmph_load(mphf_fd);
+    }
     printf("packed_size %u\n",cmph_packed_size(hash));
 
     for (i=0; i<items_len; ++i)
@@ -31,6 +45,10 @@ int test(cmph_uint32* items_to_hash, cmph_uint32 items_len, CMPH_ALGO alg_n)
 
     cmph_io_vector_adapter_destroy(source);   
     cmph_destroy(hash);
+
+    if (alg_n == CMPH_BRZ) {
+        fclose(mphf_fd);
+    }
     return 0;
 }
 
@@ -43,6 +61,8 @@ int main (void)
     cmph_uint32 vec2_len = 2;
     cmph_uint32 vec3[] = {2184764, 1882984, 1170551}; // CMPH_CHD_PH, CMPH_CHD (7,8)
     cmph_uint32 vec3_len = 3;
+    cmph_uint32 vec4[] = {2184764}; // CMPH_CHD_PH, CMPH_CHD (7,8)
+    cmph_uint32 vec4_len = 1;
     cmph_uint32 i;
 
     // Testing with vec1
@@ -63,7 +83,7 @@ int main (void)
         test(values, length, i);
     }
 
-    // Testing with vec2
+    // Testing with vec3
     values = (cmph_uint32*)vec3;
     length = vec3_len;
     printf("TESTING VECTOR WITH %u INTEGERS\n", length);
@@ -72,5 +92,14 @@ int main (void)
         test(values, length, i);
     }
 
+    // Testing with vec4
+    values = (cmph_uint32*)vec4;
+    length = vec4_len;
+    printf("TESTING VECTOR WITH %u INTEGERS\n", length);
+    for (i = 0; i < CMPH_COUNT; i++)
+    {
+        test(values, length, i);
+    }
+
     return 0;
 }

src/bmz.c

@@ -70,6 +70,12 @@ cmph_t *bmz_new(cmph_config_t *mph, double c)
 	DEBUGP("c: %f\n", c);
 	bmz->m = mph->key_source->nkeys;
 	bmz->n = (cmph_uint32)ceil(c * mph->key_source->nkeys);
+
+    if (bmz->n < 5) // workaround for small key sets
+    {
+        bmz->n = 5;
+    }
+
 	DEBUGP("m (edges): %u n (vertices): %u c: %f\n", bmz->m, bmz->n, c);
 	bmz->graph = graph_new(bmz->n, bmz->m);
 	DEBUGP("Created graph\n");

src/bmz8.c

@@ -74,6 +74,12 @@ cmph_t *bmz8_new(cmph_config_t *mph, double c)
 	DEBUGP("c: %f\n", c);
 	bmz8->m = (cmph_uint8) mph->key_source->nkeys;
 	bmz8->n = (cmph_uint8) ceil(c * mph->key_source->nkeys);
+
+    if (bmz8->n < 5) // workaround for small key sets
+    {
+        bmz8->n = 5;
+    }
+
 	DEBUGP("m (edges): %u n (vertices): %u c: %f\n", bmz8->m, bmz8->n, c);
 	bmz8->graph = graph_new(bmz8->n, bmz8->m);
 	DEBUGP("Created graph\n");

src/brz.c

@@ -27,9 +27,9 @@ static char * brz_copy_partial_bmz8_mphf(brz_config_data_t *brz, bmz8_data_t * b
 brz_config_data_t *brz_config_new(void)
 {
 	brz_config_data_t *brz = NULL;
-	brz = (brz_config_data_t *)malloc(sizeof(brz_config_data_t));
+    brz = (brz_config_data_t *)malloc(sizeof(brz_config_data_t));
-        if (!brz) return NULL;
+    if (!brz) return NULL;
-	brz->algo = CMPH_FCH;
+    brz->algo = CMPH_FCH;
 	brz->b = 128;
 	brz->hashfuncs[0] = CMPH_HASH_JENKINS;
 	brz->hashfuncs[1] = CMPH_HASH_JENKINS;
@@ -131,6 +131,15 @@ cmph_t *brz_new(cmph_config_t *mph, double c)
 
 	DEBUGP("c: %f\n", c);
 	brz_config_data_t *brz = (brz_config_data_t *)mph->data;
+
+    // Since we keep dumping partial pieces of the MPHF as it gets created
+    // the caller must set the file to store the resulting MPHF before calling
+    // this function.
+    if (brz->mphf_fd == NULL)
+    { 
+        return NULL;
+    }
+
 	switch(brz->algo) // validating restrictions over parameter c.
 	{
 		case CMPH_BMZ8:
@@ -144,6 +153,11 @@ cmph_t *brz_new(cmph_config_t *mph, double c)
 	}
 	brz->c = c;
 	brz->m = mph->key_source->nkeys;
+    if (brz->m < 5)
+    {
+        brz->c = 5;
+    }
+
 	DEBUGP("m: %u\n", brz->m);
         brz->k = (cmph_uint32)ceil(brz->m/((double)brz->b));
 	DEBUGP("k: %u\n", brz->k);
@@ -364,7 +378,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
 	{
 		fprintf(stderr, "\nMPHF generation \n");
 	}
-	/* Starting to dump to disk the resultant MPHF: __cmph_dump function */
+	/* Starting to dump to disk the resulting MPHF: __cmph_dump function */
 	nbytes = fwrite(cmph_names[CMPH_BRZ], (size_t)(strlen(cmph_names[CMPH_BRZ]) + 1), (size_t)1, brz->mphf_fd);
 	nbytes = fwrite(&(brz->m), sizeof(brz->m), (size_t)1, brz->mphf_fd);
 	nbytes = fwrite(&(brz->c), sizeof(double), (size_t)1, brz->mphf_fd);
@@ -442,7 +456,7 @@ static int brz_gen_mphf(cmph_config_t *mph)
 			source = cmph_io_byte_vector_adapter(keys_vd, (cmph_uint32)nkeys_vd);
 			config = cmph_config_new(source);
 			cmph_config_set_algo(config, brz->algo);
-			//cmph_config_set_algo(config, CMPH_BMZ8);
+			cmph_config_set_hashfuncs(config, brz->hashfuncs);
 			cmph_config_set_graphsize(config, brz->c);
 			mphf_tmp = cmph_new(config);
 			if (mphf_tmp == NULL)
@@ -565,7 +579,7 @@ int brz_dump(cmph_t *mphf, FILE *fd)
 	cmph_uint32 buflen;
 	register size_t nbytes;
 	DEBUGP("Dumping brzf\n");
-	// The initial part of the MPHF have already been dumped to disk during construction
+	// The initial part of the MPHF has already been dumped to disk during construction
 	// Dumping h0
         hash_state_dump(data->h0, &buf, &buflen);
         DEBUGP("Dumping hash state with %u bytes to disk\n", buflen);
@@ -731,6 +745,12 @@ void brz_pack(cmph_t *mphf, void *packed_mphf)
 	cmph_uint8 * ptr = (cmph_uint8 *)packed_mphf;
 	cmph_uint32 i,n;
  
+    // This assumes that if one function pointer is NULL, 
+    // all the others will be as well.
+    if (data->h1 == NULL) 
+    {
+        return;
+    }
 	// packing internal algo type
 	memcpy(ptr, &(data->algo), sizeof(data->algo));
 	ptr += sizeof(data->algo);
@@ -821,9 +841,21 @@ cmph_uint32 brz_packed_size(cmph_t *mphf)
 	cmph_uint32 i;
 	cmph_uint32 size = 0;
 	brz_data_t *data = (brz_data_t *)mphf->data;
-	CMPH_HASH h0_type = hash_get_type(data->h0);
+	CMPH_HASH h0_type;
-	CMPH_HASH h1_type = hash_get_type(data->h1[0]);
+	CMPH_HASH h1_type;
-	CMPH_HASH h2_type = hash_get_type(data->h2[0]);
+	CMPH_HASH h2_type;
+
+    // This assumes that if one function pointer is NULL, 
+    // all the others will be as well.
+    if (data->h1 == NULL) 
+    {
+        return 0U;
+    }
+
+	h0_type = hash_get_type(data->h0);
+	h1_type = hash_get_type(data->h1[0]);
+	h2_type = hash_get_type(data->h2[0]);
+
 	size = (cmph_uint32)(2*sizeof(CMPH_ALGO) + 3*sizeof(CMPH_HASH) + hash_state_packed_size(h0_type) + sizeof(cmph_uint32) +
 			sizeof(double) + sizeof(cmph_uint8)*data->k + sizeof(cmph_uint32)*data->k);
 	// pointers to g_is

src/brz.h

@@ -3,6 +3,21 @@
 
 #include "cmph.h"
 
+/*
+ * The BRZ algorithm has been built so to consume the bare minimum 
+ * amount of memory to generate the MPHFs. Thereby we decided
+ * to dump the resulting MPHFs to disk while creating them. Thus,
+ * to use the BRZ algorithm, one has to call brz_config_set_mphf_fd
+ * before calling brz_new. Otherwise we will fail the MPHF creation. 
+ * One side effect of this design decision is that the resulting 
+ * MPHF cannot be used until its dumping process is finalized 
+ * by calling brz_dump and the caller must use brz_load before 
+ * any call to  either one of the following functions is made:
+ *      brz_search
+ *      brz_pack
+ *      brz_packed_size
+ *      brz_search_packed
+ */
 typedef struct __brz_data_t brz_data_t;
 typedef struct __brz_config_data_t brz_config_data_t;
 

src/chd_ph.c

@@ -627,7 +627,8 @@ cmph_t *chd_ph_new(cmph_config_t *mph, double c)
 
 	register double load_factor = c;
 	register cmph_uint8 searching_success = 0;
-	register cmph_uint32 max_probes = 1 << 20; // default value for max_probes
+	register cmph_uint32 max_probes_default = 1 << 20; // default value for max_probes
+	register cmph_uint32 max_probes; 
 	register cmph_uint32 iterations = 100;
 	chd_ph_bucket_t * buckets = NULL;
 	chd_ph_item_t * items = NULL;
@@ -688,7 +689,13 @@ cmph_t *chd_ph_new(cmph_config_t *mph, double c)
 	buckets = chd_ph_bucket_new(chd_ph->nbuckets);
 	items   = (chd_ph_item_t *) calloc(chd_ph->m, sizeof(chd_ph_item_t));
 
-	max_probes = (cmph_uint32)(((log(chd_ph->m)/log(2))/20) * max_probes);
+	max_probes = (cmph_uint32)((log(chd_ph->m)/log(2))/20);
+
+    if (max_probes == 0) {
+        max_probes = max_probes_default;
+    } else {
+        max_probes = max_probes * max_probes_default;
+    }
 
 	if(chd_ph->keys_per_bin == 1)
 		chd_ph->occup_table = (cmph_uint8 *) calloc(((chd_ph->n + 31)/32), sizeof(cmph_uint32));