motiejus/turbonss
1
Fork 0
Code Releases Activity

*** empty log message ***

Browse Source
This commit is contained in:
fc_botelho 2009-04-21 13:01:45 +00:00
parent 1a11b02e71
commit d1fe095a69
3 changed files with 312 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/bitbool.h
View File

@ -38,11 +38,11 @@ static const cmph_uint8 valuemask[] = { 0xfc, 0xf3, 0xcf, 0x3f};
* *
* UNSETBIT(array, i) is a macro that sets 0 to an 1-bit integer stored in an array. * UNSETBIT(array, i) is a macro that sets 0 to an 1-bit integer stored in an array.
*/ */
#define UNSETBIT(array, i) (array[i >> 3] &= (~(bitmask[i & 0x00000007]))) #define UNSETBIT(array, i) (array[i >> 3] ^= ((bitmask[i & 0x00000007])))
//#define GETBIT(array, i) (array[(i) / 8] & bitmask[(i) % 8]) //#define GETBIT(array, i) (array[(i) / 8] & bitmask[(i) % 8])
//#define SETBIT(array, i) (array[(i) / 8] |= bitmask[(i) % 8]) //#define SETBIT(array, i) (array[(i) / 8] |= bitmask[(i) % 8])
//#define UNSETBIT(array, i) (array[(i) / 8] &= (~(bitmask[(i) % 8]))) //#define UNSETBIT(array, i) (array[(i) / 8] ^= ((bitmask[(i) % 8])))
/** \def SETVALUE1(array, i, v) /** \def SETVALUE1(array, i, v)
@ -99,6 +99,15 @@ static const cmph_uint8 valuemask[] = { 0xfc, 0xf3, 0xcf, 0x3f};
*/ */
#define GETBIT32(array, i) (array[i >> 5] & bitmask32[i & 0x0000001f]) #define GETBIT32(array, i) (array[i >> 5] & bitmask32[i & 0x0000001f])
/** \def UNSETBIT32(array, i)
* \brief set 0 to an 1-bit integer stored in an array of 32-bit words.
* \param array to store 1-bit integer values. The entries ar 32-bit words
* \param i is the index in array to set the the bit to 0
*
* UNSETBIT32(array, i) is a macro that sets 0 to an 1-bit integer stored in an array of 32-bit words.
*/
#define UNSETBIT32(array, i) (array[i >> 5] ^= ((bitmask32[i & 0x0000001f])))
#define BITS_TABLE_SIZE(n, bits_length) ((n * bits_length + 31) >> 5) #define BITS_TABLE_SIZE(n, bits_length) ((n * bits_length + 31) >> 5)
static inline void set_bits_value(cmph_uint32 * bits_table, cmph_uint32 index, cmph_uint32 bits_string, static inline void set_bits_value(cmph_uint32 * bits_table, cmph_uint32 index, cmph_uint32 bits_string,

8
src/chd.c
View File

@ -9,7 +9,7 @@
#include "cmph_structs.h" #include "cmph_structs.h"
#include "chd_structs.h" #include "chd_structs.h"
#include "chd.h" #include "chd.h"
#include "bitbool.h"
//#define DEBUG //#define DEBUG
#include "debug.h" #include "debug.h"
@ -78,7 +78,7 @@ cmph_t *chd_new(cmph_config_t *mph, double c)
register cmph_uint32 i, idx, nkeys, nvals, nbins; register cmph_uint32 i, idx, nkeys, nvals, nbins;
cmph_uint32 * vals_table = NULL; cmph_uint32 * vals_table = NULL;
register cmph_uint8 * occup_table = NULL; register cmph_uint32 * occup_table = NULL;
#ifdef CMPH_TIMING #ifdef CMPH_TIMING
double construction_time_begin = 0.0; double construction_time_begin = 0.0;
double construction_time = 0.0; double construction_time = 0.0;
@ -123,11 +123,11 @@ cmph_t *chd_new(cmph_config_t *mph, double c)
nvals = nbins - nkeys; nvals = nbins - nkeys;
vals_table = (cmph_uint32 *)calloc(nvals, sizeof(cmph_uint32)); vals_table = (cmph_uint32 *)calloc(nvals, sizeof(cmph_uint32));
occup_table = chd_ph->occup_table; occup_table = (cmph_uint32 *)chd_ph->occup_table;
for(i = 0, idx = 0; i < nbins; i++) for(i = 0, idx = 0; i < nbins; i++)
{ {
if(occup_table[i] == 0) if(!GETBIT32(occup_table, i))
{ {
vals_table[idx++] = i; vals_table[idx++] = i;
} }

451
src/chd_ph.c
View File

@ -10,6 +10,8 @@
#include "chd_structs_ph.h" #include "chd_structs_ph.h"
#include "chd_ph.h" #include "chd_ph.h"
#include"miller_rabin.h" #include"miller_rabin.h"
#include"bitbool.h"
//#define DEBUG //#define DEBUG
#include "debug.h" #include "debug.h"
@ -19,28 +21,47 @@
#define NO_ELEMENT UINT_MAX #define NO_ELEMENT UINT_MAX
#endif #endif
// struct to represents the buckets items // struct used to represent items at mapping, ordering and searching phases
struct _chd_ph_item_t struct _chd_ph_item_t
{ {
cmph_uint32 f; cmph_uint32 f;
cmph_uint32 h; cmph_uint32 h;
struct _chd_ph_item_t * next;
}; };
typedef struct _chd_ph_item_t chd_ph_item_t; typedef struct _chd_ph_item_t chd_ph_item_t;
// struct to represent the items at mapping phase only.
struct _chd_ph_map_item_t
{
cmph_uint32 f;
cmph_uint32 h;
cmph_uint32 bucket_num;
};
typedef struct _chd_ph_map_item_t chd_ph_map_item_t;
// struct to represent a bucket // struct to represent a bucket
struct _chd_ph_bucket_t struct _chd_ph_bucket_t
{ {
cmph_uint32 size; cmph_uint32 items_list; // offset
chd_ph_item_t * items_list; union
cmph_uint32 next_in_list; {
cmph_uint32 size;
cmph_uint32 bucket_id;
};
}; };
typedef struct _chd_ph_bucket_t chd_ph_bucket_t; typedef struct _chd_ph_bucket_t chd_ph_bucket_t;
struct _chd_ph_sorted_list_t
{
cmph_uint32 buckets_list;
cmph_uint32 size;
};
typedef struct _chd_ph_sorted_list_t chd_ph_sorted_list_t;
static inline chd_ph_bucket_t * chd_ph_bucket_new(cmph_uint32 nbuckets); static inline chd_ph_bucket_t * chd_ph_bucket_new(cmph_uint32 nbuckets);
static inline void chd_ph_bucket_clean(chd_ph_bucket_t * buckets, cmph_uint32 nbuckets); static inline void chd_ph_bucket_clean(chd_ph_bucket_t * buckets, cmph_uint32 nbuckets);
static inline cmph_uint8 chd_ph_bucket_insert(chd_ph_bucket_t * buckets, cmph_uint32 nbuckets, cmph_uint32 g, chd_ph_item_t * item);
static inline void chd_ph_bucket_destroy(chd_ph_bucket_t * buckets); static inline void chd_ph_bucket_destroy(chd_ph_bucket_t * buckets);
chd_ph_bucket_t * chd_ph_bucket_new(cmph_uint32 nbuckets) chd_ph_bucket_t * chd_ph_bucket_new(cmph_uint32 nbuckets)
@ -54,40 +75,31 @@ void chd_ph_bucket_clean(chd_ph_bucket_t * buckets, cmph_uint32 nbuckets)
register cmph_uint32 i = 0; register cmph_uint32 i = 0;
assert(buckets); assert(buckets);
for(i = 0; i < nbuckets; i++) for(i = 0; i < nbuckets; i++)
{
buckets[i].size = 0; buckets[i].size = 0;
buckets[i].items_list = 0;
buckets[i].next_in_list = NO_ELEMENT;
};
} }
cmph_uint8 chd_ph_bucket_insert(chd_ph_bucket_t * buckets,chd_ph_map_item_t * map_items, chd_ph_item_t * items,
cmph_uint8 chd_ph_bucket_insert(chd_ph_bucket_t * buckets, cmph_uint32 nbuckets, cmph_uint32 g, chd_ph_item_t * item) cmph_uint32 nbuckets,cmph_uint32 item_idx)
{ {
chd_ph_item_t * item1, * prior_item1; register cmph_uint32 i = 0;
item1 = buckets[g].items_list; register chd_ph_item_t * tmp_item;
prior_item1 = 0; register chd_ph_map_item_t * tmp_map_item = map_items + item_idx;
while(item1 != 0 && (item1->f < item->f || (item1->f == item->f && item1->h < item->h)) ) register chd_ph_bucket_t * bucket = buckets + tmp_map_item->bucket_num;
tmp_item = items + bucket->items_list;
for(i = 0; i < bucket->size; i++)
{ {
prior_item1 = item1; if(tmp_item->f == tmp_map_item->f && tmp_item->h == tmp_map_item->h)
item1 = item1->next; {
DEBUGP("Item not added\n");
return 0;
};
tmp_item++;
}; };
tmp_item->f = tmp_map_item->f;
if(item1 != 0 && item1->f == item->f && item1->h == item->h) tmp_item->h = tmp_map_item->h;
{ bucket->size++;
DEBUGP("Item not added\n");
return 0;
};
item->next = item1;
if(prior_item1 == 0)
buckets[g].items_list = item;
else
prior_item1->next = item;
buckets[g].size++;
return 1; return 1;
} };
void chd_ph_bucket_destroy(chd_ph_bucket_t * buckets) void chd_ph_bucket_destroy(chd_ph_bucket_t * buckets)
{ {
free(buckets); free(buckets);
@ -96,10 +108,11 @@ void chd_ph_bucket_destroy(chd_ph_bucket_t * buckets)
static inline cmph_uint8 chd_ph_mapping(cmph_config_t *mph, chd_ph_bucket_t * buckets, chd_ph_item_t * items, static inline cmph_uint8 chd_ph_mapping(cmph_config_t *mph, chd_ph_bucket_t * buckets, chd_ph_item_t * items,
cmph_uint32 *max_bucket_size); cmph_uint32 *max_bucket_size);
static inline cmph_uint32 * chd_ph_ordering(chd_ph_bucket_t * buckets, cmph_uint32 nbuckets, cmph_uint32 max_bucket_size); static chd_ph_sorted_list_t * chd_ph_ordering(chd_ph_bucket_t ** _buckets,chd_ph_item_t ** items,
cmph_uint32 nbuckets,cmph_uint32 nitems, cmph_uint32 max_bucket_size);
static inline cmph_uint8 chd_ph_searching(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, cmph_uint32 max_bucket_size, static cmph_uint8 chd_ph_searching(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, chd_ph_item_t *items ,
cmph_uint32 *sorted_lists, cmph_uint32 max_probes, cmph_uint32 * disp_table); cmph_uint32 max_bucket_size, chd_ph_sorted_list_t *sorted_lists, cmph_uint32 max_probes, cmph_uint32 * disp_table);
static inline double chd_ph_space_lower_bound(cmph_uint32 _n, cmph_uint32 _r) static inline double chd_ph_space_lower_bound(cmph_uint32 _n, cmph_uint32 _r)
{ {
@ -207,7 +220,8 @@ cmph_uint8 chd_ph_mapping(cmph_config_t *mph, chd_ph_bucket_t * buckets, chd_ph_
chd_ph_config_data_t *chd_ph = (chd_ph_config_data_t *)mph->data; chd_ph_config_data_t *chd_ph = (chd_ph_config_data_t *)mph->data;
char * key = NULL; char * key = NULL;
cmph_uint32 keylen = 0; cmph_uint32 keylen = 0;
chd_ph_item_t * item; chd_ph_map_item_t * map_item;
chd_ph_map_item_t * map_items = malloc(chd_ph->m*sizeof(chd_ph_map_item_t));
register cmph_uint32 mapping_iterations = 1000; register cmph_uint32 mapping_iterations = 1000;
*max_bucket_size = 0; *max_bucket_size = 0;
while(1) while(1)
@ -225,33 +239,40 @@ cmph_uint8 chd_ph_mapping(cmph_config_t *mph, chd_ph_bucket_t * buckets, chd_ph_
mph->key_source->read(mph->key_source->data, &key, &keylen); mph->key_source->read(mph->key_source->data, &key, &keylen);
hash_vector(chd_ph->hl, key, keylen, hl); hash_vector(chd_ph->hl, key, keylen, hl);
item = (items + i); map_item = (map_items + i);
g = hl[0] % chd_ph->nbuckets; g = hl[0] % chd_ph->nbuckets;
item->f = hl[1] % chd_ph->n; map_item->f = hl[1] % chd_ph->n;
item->h = hl[2] % (chd_ph->n - 1) + 1; map_item->h = hl[2] % (chd_ph->n - 1) + 1;
map_item->bucket_num=g;
mph->key_source->dispose(mph->key_source->data, key, keylen); mph->key_source->dispose(mph->key_source->data, key, keylen);
// if(buckets[g].size == (chd_ph->keys_per_bucket << 2)) // if(buckets[g].size == (chd_ph->keys_per_bucket << 2))
// { // {
// DEBUGP("BUCKET = %u -- SIZE = %u -- MAXIMUM SIZE = %u\n", g, buckets[g].size, (chd_ph->keys_per_bucket << 2)); // DEBUGP("BUCKET = %u -- SIZE = %u -- MAXIMUM SIZE = %u\n", g, buckets[g].size, (chd_ph->keys_per_bucket << 2));
// goto error; // goto error;
// } // }
buckets[g].size++;
if(!chd_ph_bucket_insert(buckets, chd_ph->nbuckets, g, item))
{
break;
}
if(buckets[g].size > *max_bucket_size) if(buckets[g].size > *max_bucket_size)
{ {
*max_bucket_size = buckets[g].size; *max_bucket_size = buckets[g].size;
} }
} }
buckets[0].items_list = 0;
for(i = 1; i < chd_ph->nbuckets; i++)
{
buckets[i].items_list = buckets[i-1].items_list + buckets[i - 1].size;
buckets[i - 1].size = 0;
};
buckets[i - 1].size = 0;
for(i = 0; i < chd_ph->m; i++)
{
map_item = (map_items + i);
if(!chd_ph_bucket_insert(buckets, map_items, items, chd_ph->nbuckets, i))
break;
}
if(i == chd_ph->m) if(i == chd_ph->m)
{ {
free(map_items);
return 1; // SUCCESS return 1; // SUCCESS
} }
@ -261,77 +282,163 @@ cmph_uint8 chd_ph_mapping(cmph_config_t *mph, chd_ph_bucket_t * buckets, chd_ph_
} }
} }
error: error:
free(map_items);
hash_state_destroy(chd_ph->hl); hash_state_destroy(chd_ph->hl);
chd_ph->hl = NULL; chd_ph->hl = NULL;
return 0; // FAILURE return 0; // FAILURE
} }
cmph_uint32 * chd_ph_ordering(chd_ph_bucket_t * buckets, cmph_uint32 nbuckets, cmph_uint32 max_bucket_size) chd_ph_sorted_list_t * chd_ph_ordering(chd_ph_bucket_t ** _buckets, chd_ph_item_t ** _items,
cmph_uint32 nbuckets, cmph_uint32 nitems, cmph_uint32 max_bucket_size)
{ {
cmph_uint32 * sorted_lists = (cmph_uint32 *) calloc(max_bucket_size + 1, sizeof(cmph_uint32)); chd_ph_sorted_list_t * sorted_lists = (chd_ph_sorted_list_t *) calloc(max_bucket_size + 1, sizeof(chd_ph_sorted_list_t));
register cmph_uint32 i, size;
chd_ph_bucket_t * input_buckets = (*_buckets);
chd_ph_bucket_t * output_buckets;
chd_ph_item_t * input_items = (*_items);
chd_ph_item_t * output_items;
register cmph_uint32 i, j, bucket_size, position, position2;
// cmph_uint32 non_empty_buckets;
DEBUGP("MAX BUCKET SIZE = %u\n", max_bucket_size); DEBUGP("MAX BUCKET SIZE = %u\n", max_bucket_size);
for(i = 0; i <= max_bucket_size; i++) // Determine size of each list of buckets
{
sorted_lists[i] = NO_ELEMENT;
}
for(i = 0; i < nbuckets; i++) for(i = 0; i < nbuckets; i++)
{ {
size = buckets[i].size; bucket_size = input_buckets[i].size;
if(size == 0) if(bucket_size == 0)
continue; continue;
buckets[i].next_in_list = sorted_lists[size]; sorted_lists[bucket_size].size++;
sorted_lists[size] = i;
}; };
sorted_lists[1].buckets_list = 0;
// Determine final position of list of buckets into the contiguous array that will store all the buckets
for(i = 2; i <= max_bucket_size; i++)
{
sorted_lists[i].buckets_list = sorted_lists[i-1].buckets_list + sorted_lists[i-1].size;
sorted_lists[i-1].size = 0;
};
sorted_lists[i-1].size = 0;
// Store the buckets in a new array which is sorted by bucket sizes
output_buckets = calloc(nbuckets, sizeof(chd_ph_bucket_t)); // everything is initialized with zero
// non_empty_buckets = nbuckets;
for(i = 0; i < nbuckets; i++)
{
bucket_size = input_buckets[i].size;
if(bucket_size == 0)
{
// non_empty_buckets--;
continue;
};
position = sorted_lists[bucket_size].buckets_list + sorted_lists[bucket_size].size;
output_buckets[position].bucket_id = i;
output_buckets[position].items_list = input_buckets[i].items_list;
sorted_lists[bucket_size].size++;
};
/* for(i = non_empty_buckets; i < nbuckets; i++)
output_buckets[i].size=0;*/
// Return the buckets sorted in new order and free the old buckets sorted in old order
free(input_buckets);
(*_buckets) = output_buckets;
// Store the items according to the new order of buckets.
output_items = (chd_ph_item_t*)calloc(nitems, sizeof(chd_ph_item_t));
position = 0;
i = 0;
for(bucket_size = 1; bucket_size <= max_bucket_size; bucket_size++)
{
for(i = sorted_lists[bucket_size].buckets_list; i < sorted_lists[bucket_size].size + sorted_lists[bucket_size].buckets_list; i++)
{
position2 = output_buckets[i].items_list;
output_buckets[i].items_list = position;
for(j = 0; j < bucket_size; j++)
{
output_items[position].f = input_items[position2].f;
output_items[position].h = input_items[position2].h;
position++;
position2++;
};
};
};
//Return the items sorted in new order and free the old items sorted in old order
free(input_items);
(*_items) = output_items;
return sorted_lists; return sorted_lists;
} };
static inline cmph_uint8 place_bucket_probe(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, cmph_uint32 probe0_num, static inline cmph_uint8 place_bucket_probe(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets,
cmph_uint32 probe1_num, cmph_uint32 bucket_num) chd_ph_item_t *items, cmph_uint32 probe0_num, cmph_uint32 probe1_num,
cmph_uint32 bucket_num, cmph_uint32 size)
{ {
register cmph_uint32 i; register cmph_uint32 i;
register cmph_uint32 size = buckets[bucket_num].size;
register chd_ph_item_t * item; register chd_ph_item_t * item;
register cmph_uint32 position; register cmph_uint32 position;
item = buckets[bucket_num].items_list; item = items + buckets[bucket_num].items_list;
// try place bucket with probe_num // try place bucket with probe_num
for(i = 0; i < size; i++) // placement if(chd_ph->keys_per_bin > 1)
{ {
position = (item->f + ((cmph_uint64)item->h)*probe0_num + probe1_num) % chd_ph->n; for(i = 0; i < size; i++) // placement
if(chd_ph->occup_table[position] >= chd_ph->keys_per_bin)
{ {
break; position = (item->f + ((cmph_uint64)item->h)*probe0_num + probe1_num) % chd_ph->n;
} if(chd_ph->occup_table[position] >= chd_ph->keys_per_bin)
(chd_ph->occup_table[position])++;
item = item->next;
};
if(i != size) // Undo the placement
{
item = buckets[bucket_num].items_list;
while(1)
{
if(i == 0)
{ {
break; break;
} }
position = (item->f + ((cmph_uint64 )item->h) * probe0_num + probe1_num) % chd_ph->n; (chd_ph->occup_table[position])++;
(chd_ph->occup_table[position])--; item++;
item = item->next; };
i--; } else
{
for(i = 0; i < size; i++) // placement
{
position = (item->f + ((cmph_uint64)item->h)*probe0_num + probe1_num) % chd_ph->n;
if(GETBIT32(((cmph_uint32 *)chd_ph->occup_table), position))
{
break;
}
SETBIT32(((cmph_uint32*)chd_ph->occup_table), position);
item++;
};
};
if(i != size) // Undo the placement
{
item = items + buckets[bucket_num].items_list;
if(chd_ph->keys_per_bin > 1)
{
while(1)
{
if(i == 0)
{
break;
}
position = (item->f + ((cmph_uint64 )item->h) * probe0_num + probe1_num) % chd_ph->n;
(chd_ph->occup_table[position])--;
item++;
i--;
};
} else
{
while(1)
{
if(i == 0)
{
break;
}
position = (item->f + ((cmph_uint64 )item->h) * probe0_num + probe1_num) % chd_ph->n;
UNSETBIT32(((cmph_uint32*)chd_ph->occup_table), position);
// ([position/32]^=(1<<(position%32));
item++;
i--;
};
}; };
return 0; return 0;
}; }
return 1; return 1;
}; };
static inline cmph_uint8 place_bucket(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, cmph_uint32 max_probes, static inline cmph_uint8 place_bucket(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, chd_ph_item_t * items, cmph_uint32 max_probes,
cmph_uint32 * disp_table, cmph_uint32 bucket_num) cmph_uint32 * disp_table, cmph_uint32 bucket_num, cmph_uint32 size)
{ {
register cmph_uint32 probe0_num, probe1_num, probe_num; register cmph_uint32 probe0_num, probe1_num, probe_num;
@ -341,9 +448,9 @@ static inline cmph_uint8 place_bucket(chd_ph_config_data_t *chd_ph, chd_ph_bucke
while(1) while(1)
{ {
if(place_bucket_probe(chd_ph, buckets, probe0_num, probe1_num, bucket_num)) if(place_bucket_probe(chd_ph, buckets, items, probe0_num, probe1_num, bucket_num,size))
{ {
disp_table[bucket_num] = probe0_num + probe1_num * chd_ph->n; disp_table[buckets[bucket_num].bucket_id] = probe0_num + probe1_num * chd_ph->n;
return 1; return 1;
} }
probe0_num++; probe0_num++;
@ -361,68 +468,76 @@ static inline cmph_uint8 place_bucket(chd_ph_config_data_t *chd_ph, chd_ph_bucke
return 0; return 0;
}; };
static inline cmph_uint8 place_buckets1(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, cmph_uint32 max_bucket_size, static inline cmph_uint8 place_buckets1(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t * buckets, chd_ph_item_t *items,
cmph_uint32 *sorted_lists, cmph_uint32 max_probes, cmph_uint32 * disp_table) cmph_uint32 max_bucket_size, chd_ph_sorted_list_t *sorted_lists, cmph_uint32 max_probes,
cmph_uint32 * disp_table)
{ {
register cmph_uint32 i = 0; register cmph_uint32 i = 0;
register cmph_uint32 curr_bucket = 0; register cmph_uint32 curr_bucket = 0;
for(i = max_bucket_size; i > 0; i--) for(i = max_bucket_size; i > 0; i--)
{ {
curr_bucket = sorted_lists[i]; curr_bucket = sorted_lists[i].buckets_list;
while(curr_bucket != NO_ELEMENT) while(curr_bucket < sorted_lists[i].size + sorted_lists[i].buckets_list)
{ {
if(!place_bucket(chd_ph, buckets, max_probes, disp_table, curr_bucket)) if(!place_bucket(chd_ph, buckets, items, max_probes, disp_table, curr_bucket, i))
{ {
return 0; return 0;
} }
curr_bucket = buckets[curr_bucket].next_in_list; curr_bucket++;
}; };
}; };
return 1; return 1;
}; };
static inline cmph_uint8 place_buckets2(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, cmph_uint32 max_bucket_size, static inline cmph_uint8 place_buckets2(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, chd_ph_item_t * items,
cmph_uint32 *sorted_lists, cmph_uint32 max_probes, cmph_uint32 * disp_table) cmph_uint32 max_bucket_size, chd_ph_sorted_list_t *sorted_lists, cmph_uint32 max_probes,
cmph_uint32 * disp_table)
{ {
register cmph_uint32 i; register cmph_uint32 i,j, non_placed_bucket;
register cmph_uint32 curr_bucket, prev_bucket; register cmph_uint32 curr_bucket;
register cmph_uint32 probe_num, probe0_num, probe1_num; register cmph_uint32 probe_num, probe0_num, probe1_num;
cmph_uint32 sorted_list_size;
#ifdef DEBUG
cmph_uint32 items_list;
cmph_uint32 bucket_id;
#endif
DEBUGP("USING HEURISTIC TO PLACE BUCKETS\n"); DEBUGP("USING HEURISTIC TO PLACE BUCKETS\n");
for(i = max_bucket_size; i > 0; i--) for(i = max_bucket_size; i > 0; i--)
{ {
probe_num = 0; probe_num = 0;
probe0_num = 0; probe0_num = 0;
probe1_num = 0; probe1_num = 0;
while(sorted_lists[i] != NO_ELEMENT) sorted_list_size = sorted_lists[i].size;
while(sorted_lists[i].size != 0)
{ {
prev_bucket = NO_ELEMENT; curr_bucket = sorted_lists[i].buckets_list;
curr_bucket = sorted_lists[i]; for(j = 0, non_placed_bucket = 0; j < sorted_lists[i].size; j++)
while(curr_bucket != NO_ELEMENT)
{ {
// if bucket is successfully placed remove it from list // if bucket is successfully placed remove it from list
if(place_bucket_probe(chd_ph, buckets, probe0_num, probe1_num, curr_bucket)) if(place_bucket_probe(chd_ph, buckets, items, probe0_num, probe1_num, curr_bucket, i))
{ {
disp_table[curr_bucket] = probe0_num + probe1_num * chd_ph->n; disp_table[buckets[curr_bucket].bucket_id] = probe0_num + probe1_num * chd_ph->n;
// DEBUGP("BUCKET %u PLACED --- DISPLACEMENT = %u\n", curr_bucket, disp_table[curr_bucket]); // DEBUGP("BUCKET %u PLACED --- DISPLACEMENT = %u\n", curr_bucket, disp_table[curr_bucket]);
if(prev_bucket == NO_ELEMENT)
{
sorted_lists[i] = buckets[curr_bucket].next_in_list;
}
else
{
buckets[prev_bucket].next_in_list = buckets[curr_bucket].next_in_list;
}
} }
else else
{ {
// DEBUGP("BUCKET %u NOT PLACED\n", curr_bucket); // DEBUGP("BUCKET %u NOT PLACED\n", curr_bucket);
prev_bucket = curr_bucket; #ifdef DEBUG
items_list = buckets[non_placed_bucket + sorted_lists[i].buckets_list].items_list;
bucket_id = buckets[non_placed_bucket + sorted_lists[i].buckets_list].bucket_id;
#endif
buckets[non_placed_bucket + sorted_lists[i].buckets_list].items_list = buckets[curr_bucket].items_list;
buckets[non_placed_bucket + sorted_lists[i].buckets_list].bucket_id = buckets[curr_bucket].bucket_id;
#ifdef DEBUG
buckets[curr_bucket].items_list=items_list;
buckets[curr_bucket].bucket_id=bucket_id;
#endif
non_placed_bucket++;
} }
curr_bucket = buckets[curr_bucket].next_in_list; curr_bucket++;
}; };
sorted_lists[i].size = non_placed_bucket;
probe0_num++; probe0_num++;
if(probe0_num >= chd_ph->n) if(probe0_num >= chd_ph->n)
{ {
@ -432,58 +547,73 @@ static inline cmph_uint8 place_buckets2(chd_ph_config_data_t *chd_ph, chd_ph_buc
probe_num++; probe_num++;
if(probe_num >= max_probes || probe1_num >= chd_ph->n) if(probe_num >= max_probes || probe1_num >= chd_ph->n)
{ {
sorted_lists[i].size = sorted_list_size;
return 0; return 0;
}; };
}; };
sorted_lists[i].size = sorted_list_size;
}; };
return 1; return 1;
}; };
cmph_uint8 chd_ph_searching(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, cmph_uint32 max_bucket_size, cmph_uint8 chd_ph_searching(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, chd_ph_item_t *items ,
cmph_uint32 *sorted_lists, cmph_uint32 max_probes, cmph_uint32 * disp_table) cmph_uint32 max_bucket_size, chd_ph_sorted_list_t *sorted_lists, cmph_uint32 max_probes,
cmph_uint32 * disp_table)
{ {
if(chd_ph->use_h) if(chd_ph->use_h)
{ {
return place_buckets2(chd_ph, buckets, max_bucket_size, sorted_lists, max_probes, disp_table); return place_buckets2(chd_ph, buckets, items, max_bucket_size, sorted_lists, max_probes, disp_table);
} }
else else
{ {
return place_buckets1(chd_ph, buckets, max_bucket_size, sorted_lists, max_probes, disp_table); return place_buckets1(chd_ph, buckets, items, max_bucket_size, sorted_lists, max_probes, disp_table);
} }
} }
static inline cmph_uint8 chd_ph_check_bin_hashing(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, static inline cmph_uint8 chd_ph_check_bin_hashing(chd_ph_config_data_t *chd_ph, chd_ph_bucket_t *buckets, chd_ph_item_t *items,
cmph_uint32 * disp_table) cmph_uint32 * disp_table, chd_ph_sorted_list_t * sorted_lists,cmph_uint32 max_bucket_size)
{ {
register cmph_uint32 i, j; register cmph_uint32 bucket_size, i, j;
register cmph_uint32 position, probe0_num, probe1_num; register cmph_uint32 position, probe0_num, probe1_num;
register cmph_uint32 m = 0; register cmph_uint32 m = 0;
register chd_ph_item_t * item; register chd_ph_item_t * item;
if(chd_ph->keys_per_bin > 1)
memset(chd_ph->occup_table, 0, chd_ph->n);
else
memset(chd_ph->occup_table, 0, ((chd_ph->n + 31)/32) * sizeof(cmph_uint32));
memset(chd_ph->occup_table, 0, chd_ph->n); for(bucket_size = 1; bucket_size <= max_bucket_size; bucket_size++)
for(i = 0; i < chd_ph->nbuckets; i++) for(i = sorted_lists[bucket_size].buckets_list; i < sorted_lists[bucket_size].size +
{ sorted_lists[bucket_size].buckets_list; i++)
j = buckets[i].size;
item = buckets[i].items_list;
probe0_num = disp_table[i] % chd_ph->n;
probe1_num = disp_table[i] / chd_ph->n;
for(; j > 0; j--)
{ {
if(item == 0) j = bucket_size;
item = items + buckets[i].items_list;
probe0_num = disp_table[buckets[i].bucket_id] % chd_ph->n;
probe1_num = disp_table[buckets[i].bucket_id] / chd_ph->n;
for(; j > 0; j--)
{ {
return 0; m++;
} position = (item->f + ((cmph_uint64 )item->h) * probe0_num + probe1_num) % chd_ph->n;
m++; if(chd_ph->keys_per_bin > 1)
position = (item->f + ((cmph_uint64 )item->h) * probe0_num + probe1_num) % chd_ph->n; {
if(chd_ph->occup_table[position] >= chd_ph->keys_per_bin) if(chd_ph->occup_table[position] >= chd_ph->keys_per_bin)
{ {
return 0; return 0;
} }
(chd_ph->occup_table[position])++; (chd_ph->occup_table[position])++;
item = item->next; }
else
{
if(GETBIT32(((cmph_uint32*)chd_ph->occup_table), position))
{
return 0;
}
SETBIT32(((cmph_uint32*)chd_ph->occup_table), position);
};
item++;
};
}; };
};
DEBUGP("We were able to place m = %u keys\n", m); DEBUGP("We were able to place m = %u keys\n", m);
return 1; return 1;
}; };
@ -503,7 +633,7 @@ cmph_t *chd_ph_new(cmph_config_t *mph, double c)
chd_ph_item_t * items = NULL; chd_ph_item_t * items = NULL;
register cmph_uint8 failure = 0; register cmph_uint8 failure = 0;
cmph_uint32 max_bucket_size = 0; cmph_uint32 max_bucket_size = 0;
cmph_uint32 * sorted_lists = NULL; chd_ph_sorted_list_t * sorted_lists = NULL;
cmph_uint32 * disp_table = NULL; cmph_uint32 * disp_table = NULL;
register double space_lower_bound = 0; register double space_lower_bound = 0;
#ifdef CMPH_TIMING #ifdef CMPH_TIMING
@ -559,7 +689,12 @@ cmph_t *chd_ph_new(cmph_config_t *mph, double c)
items = (chd_ph_item_t *) calloc(chd_ph->m, sizeof(chd_ph_item_t)); items = (chd_ph_item_t *) calloc(chd_ph->m, sizeof(chd_ph_item_t));
max_probes = (cmph_uint32)(((log(chd_ph->m)/log(2))/20) * max_probes); max_probes = (cmph_uint32)(((log(chd_ph->m)/log(2))/20) * max_probes);
chd_ph->occup_table = (cmph_uint8 *) calloc(chd_ph->n, sizeof(cmph_uint8));
if(chd_ph->keys_per_bin == 1)
chd_ph->occup_table = (cmph_uint8 *) calloc(((chd_ph->n + 31)/32), sizeof(cmph_uint32));
else
chd_ph->occup_table = (cmph_uint8 *) calloc(chd_ph->n, sizeof(cmph_uint8));
disp_table = (cmph_uint32 *) calloc(chd_ph->nbuckets, sizeof(cmph_uint32)); disp_table = (cmph_uint32 *) calloc(chd_ph->nbuckets, sizeof(cmph_uint32));
// //
// init_genrand(time(0)); // init_genrand(time(0));
@ -590,19 +725,22 @@ cmph_t *chd_ph_new(cmph_config_t *mph, double c)
{ {
free(sorted_lists); free(sorted_lists);
} }
sorted_lists = chd_ph_ordering(buckets, chd_ph->nbuckets, max_bucket_size);
sorted_lists = chd_ph_ordering(&buckets, &items, chd_ph->nbuckets, chd_ph->m, max_bucket_size);
if (mph->verbosity) if (mph->verbosity)
{ {
fprintf(stderr, "Starting searching step\n"); fprintf(stderr, "Starting searching step\n");
} }
searching_success = chd_ph_searching(chd_ph, buckets, max_bucket_size, sorted_lists, max_probes, disp_table); searching_success = chd_ph_searching(chd_ph, buckets, items, max_bucket_size, sorted_lists, max_probes, disp_table);
if(searching_success) break; if(searching_success) break;
// reset occup_table // reset occup_table
memset(chd_ph->occup_table, 0, chd_ph->n); if(chd_ph->keys_per_bin > 1)
memset(chd_ph->occup_table, 0, chd_ph->n);
else
memset(chd_ph->occup_table, 0, ((chd_ph->n + 31)/32) * sizeof(cmph_uint32));
if(iterations == 0) if(iterations == 0)
{ {
// Cleanup memory // Cleanup memory
@ -617,11 +755,12 @@ cmph_t *chd_ph_new(cmph_config_t *mph, double c)
#ifdef DEBUG #ifdef DEBUG
{ {
if(!chd_ph_check_bin_hashing(chd_ph, buckets, disp_table)) if(!chd_ph_check_bin_hashing(chd_ph, buckets, items, disp_table,sorted_lists,max_bucket_size))
{ {
DEBUGP("Error for bin packing generation"); DEBUGP("Error for bin packing generation");
return NULL; failure = 1;
goto cleanup;
} }
} }
#endif #endif