107#elif defined RUBY_EXPORT
109#include "internal/bits.h"
110#include "internal/hash.h"
111#include "internal/sanitizers.h"
112#include "internal/set_table.h"
113#include "internal/st.h"
114#include "ruby_assert.h"
124#define PREFETCH(addr, write_p) __builtin_prefetch(addr, write_p)
125#define EXPECT(expr, val) __builtin_expect(expr, val)
126#define ATTRIBUTE_UNUSED __attribute__((unused))
128#define PREFETCH(addr, write_p)
129#define EXPECT(expr, val) (expr)
130#define ATTRIBUTE_UNUSED
134typedef st_index_t st_hash_t;
142#define type_numhash st_hashtype_num
148static int st_strcmp(st_data_t, st_data_t);
149static st_index_t strhash(st_data_t);
155static int st_locale_insensitive_strcasecmp_i(st_data_t lhs, st_data_t rhs);
156static st_index_t strcasehash(st_data_t);
158 st_locale_insensitive_strcasecmp_i,
165#define ST_INIT_VAL 0xafafafafafafafaf
166#define ST_INIT_VAL_BYTE 0xafa
173#define malloc ruby_xmalloc
174#define calloc ruby_xcalloc
175#define realloc ruby_xrealloc
176#define free ruby_xfree
179#define EQUAL(tab,x,y) ((x) == (y) || (*(tab)->type->compare)((x),(y)) == 0)
180#define PTR_EQUAL(tab, ptr, hash_val, key_) \
181 ((ptr)->hash == (hash_val) && EQUAL((tab), (key_), (ptr)->key))
185#define DO_PTR_EQUAL_CHECK(tab, ptr, hash_val, key, res, rebuilt_p) \
187 unsigned int _old_rebuilds_num = (tab)->rebuilds_num; \
188 res = PTR_EQUAL(tab, ptr, hash_val, key); \
189 rebuilt_p = _old_rebuilds_num != (tab)->rebuilds_num; \
195 unsigned char entry_power;
199 unsigned char bin_power;
201 unsigned char size_ind;
204 st_index_t bins_words;
208#if SIZEOF_ST_INDEX_T == 8
227 {16, 17, 2, 0x10000},
228 {17, 18, 2, 0x20000},
229 {18, 19, 2, 0x40000},
230 {19, 20, 2, 0x80000},
231 {20, 21, 2, 0x100000},
232 {21, 22, 2, 0x200000},
233 {22, 23, 2, 0x400000},
234 {23, 24, 2, 0x800000},
235 {24, 25, 2, 0x1000000},
236 {25, 26, 2, 0x2000000},
237 {26, 27, 2, 0x4000000},
238 {27, 28, 2, 0x8000000},
239 {28, 29, 2, 0x10000000},
240 {29, 30, 2, 0x20000000},
241 {30, 31, 2, 0x40000000},
242 {31, 32, 2, 0x80000000},
243 {32, 33, 3, 0x200000000},
244 {33, 34, 3, 0x400000000},
245 {34, 35, 3, 0x800000000},
246 {35, 36, 3, 0x1000000000},
247 {36, 37, 3, 0x2000000000},
248 {37, 38, 3, 0x4000000000},
249 {38, 39, 3, 0x8000000000},
250 {39, 40, 3, 0x10000000000},
251 {40, 41, 3, 0x20000000000},
252 {41, 42, 3, 0x40000000000},
253 {42, 43, 3, 0x80000000000},
254 {43, 44, 3, 0x100000000000},
255 {44, 45, 3, 0x200000000000},
256 {45, 46, 3, 0x400000000000},
257 {46, 47, 3, 0x800000000000},
258 {47, 48, 3, 0x1000000000000},
259 {48, 49, 3, 0x2000000000000},
260 {49, 50, 3, 0x4000000000000},
261 {50, 51, 3, 0x8000000000000},
262 {51, 52, 3, 0x10000000000000},
263 {52, 53, 3, 0x20000000000000},
264 {53, 54, 3, 0x40000000000000},
265 {54, 55, 3, 0x80000000000000},
266 {55, 56, 3, 0x100000000000000},
267 {56, 57, 3, 0x200000000000000},
268 {57, 58, 3, 0x400000000000000},
269 {58, 59, 3, 0x800000000000000},
270 {59, 60, 3, 0x1000000000000000},
271 {60, 61, 3, 0x2000000000000000},
272 {61, 62, 3, 0x4000000000000000},
273 {62, 63, 3, 0x8000000000000000},
296 {16, 17, 2, 0x20000},
297 {17, 18, 2, 0x40000},
298 {18, 19, 2, 0x80000},
299 {19, 20, 2, 0x100000},
300 {20, 21, 2, 0x200000},
301 {21, 22, 2, 0x400000},
302 {22, 23, 2, 0x800000},
303 {23, 24, 2, 0x1000000},
304 {24, 25, 2, 0x2000000},
305 {25, 26, 2, 0x4000000},
306 {26, 27, 2, 0x8000000},
307 {27, 28, 2, 0x10000000},
308 {28, 29, 2, 0x20000000},
309 {29, 30, 2, 0x40000000},
310 {30, 31, 2, 0x80000000},
316#define RESERVED_HASH_VAL (~(st_hash_t) 0)
317#define RESERVED_HASH_SUBSTITUTION_VAL ((st_hash_t) 0)
319static inline st_hash_t
320normalize_hash_value(st_hash_t hash)
324 return hash == RESERVED_HASH_VAL ? RESERVED_HASH_SUBSTITUTION_VAL : hash;
328static inline st_hash_t
329do_hash(st_data_t key,
st_table *tab)
331 st_hash_t hash = (st_hash_t)(tab->type->hash)(key);
332 return normalize_hash_value(hash);
336#define MINIMAL_POWER2 2
338#if MINIMAL_POWER2 < 2
339#error "MINIMAL_POWER2 should be >= 2"
344#define MAX_POWER2_FOR_TABLES_WITHOUT_BINS 4
348get_power2(st_index_t size)
350 unsigned int n = ST_INDEX_BITS - nlz_intptr(size);
352 return n < MINIMAL_POWER2 ? MINIMAL_POWER2 : n;
363static inline st_index_t
364get_bin(st_index_t *bins,
int s, st_index_t n)
366 return (s == 0 ? ((
unsigned char *) bins)[n]
367 : s == 1 ? ((unsigned short *) bins)[n]
368 : s == 2 ? ((unsigned int *) bins)[n]
369 : ((st_index_t *) bins)[n]);
375set_bin(st_index_t *bins,
int s, st_index_t n, st_index_t v)
377 if (s == 0) ((
unsigned char *) bins)[n] = (
unsigned char) v;
378 else if (s == 1) ((
unsigned short *) bins)[n] = (
unsigned short) v;
379 else if (s == 2) ((
unsigned int *) bins)[n] = (
unsigned int) v;
380 else ((st_index_t *) bins)[n] = v;
393#define MARK_BIN_EMPTY(tab, i) (set_bin((tab)->bins, get_size_ind(tab), i, EMPTY_BIN))
397#define UNDEFINED_ENTRY_IND (~(st_index_t) 0)
398#define UNDEFINED_BIN_IND (~(st_index_t) 0)
402#define REBUILT_TABLE_ENTRY_IND (~(st_index_t) 1)
403#define REBUILT_TABLE_BIN_IND (~(st_index_t) 1)
408#define MARK_BIN_DELETED(tab, i) \
410 set_bin((tab)->bins, get_size_ind(tab), i, DELETED_BIN); \
415#define EMPTY_BIN_P(b) ((b) == EMPTY_BIN)
416#define DELETED_BIN_P(b) ((b) == DELETED_BIN)
417#define EMPTY_OR_DELETED_BIN_P(b) ((b) <= DELETED_BIN)
421#define IND_EMPTY_BIN_P(tab, i) (EMPTY_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
422#define IND_DELETED_BIN_P(tab, i) (DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
423#define IND_EMPTY_OR_DELETED_BIN_P(tab, i) (EMPTY_OR_DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
427#define MARK_ENTRY_DELETED(e_ptr) ((e_ptr)->hash = RESERVED_HASH_VAL)
428#define DELETED_ENTRY_P(e_ptr) ((e_ptr)->hash == RESERVED_HASH_VAL)
431static inline unsigned int
434 return tab->size_ind;
438static inline st_index_t
441 return ((st_index_t) 1)<<tab->bin_power;
445static inline st_index_t
448 return get_bins_num(tab) - 1;
453static inline st_index_t
454hash_bin(st_hash_t hash_value,
st_table *tab)
456 return hash_value & bins_mask(tab);
460static inline st_index_t
461get_allocated_entries(
const st_table *tab)
463 return ((st_index_t) 1)<<tab->entry_power;
467static inline st_index_t
470 return features[tab->entry_power].bins_words *
sizeof (st_index_t);
477 memset(tab->bins, 0, bins_size(tab));
484 tab->num_entries = 0;
485 tab->entries_start = tab->entries_bound = 0;
486 if (tab->bins != NULL)
487 initialize_bins(tab);
495 int all, total, num, str, strcase;
500static int init_st = 0;
507 char fname[10+
sizeof(long)*3];
509 if (!collision.total)
return;
510 f = fopen((snprintf(fname,
sizeof(fname),
"/tmp/col%ld", (
long)getpid()), fname),
"w");
513 fprintf(f,
"collision: %d / %d (%6.2f)\n", collision.all, collision.total,
514 ((
double)collision.all / (collision.total)) * 100);
515 fprintf(f,
"num: %d, str: %d, strcase: %d\n", collision.num, collision.str, collision.strcase);
521st_init_existing_table_with_size(
st_table *tab,
const struct st_hash_type *type, st_index_t size)
528 const char *e = getenv(
"ST_HASH_LOG");
529 if (!e || !*e) init_st = 1;
538 n = get_power2(size);
545 tab->entry_power = n;
546 tab->bin_power = features[n].bin_power;
547 tab->size_ind = features[n].size_ind;
548 if (n <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
551 tab->bins = (st_index_t *) malloc(bins_size(tab));
553 if (tab->bins == NULL) {
559 tab->entries = (
st_table_entry *) malloc(get_allocated_entries(tab)
562 if (tab->entries == NULL) {
568 tab->rebuilds_num = 0;
576st_init_table_with_size(
const struct st_hash_type *type, st_index_t size)
585 st_init_existing_table_with_size(tab, type, size);
587 if (st_init_existing_table_with_size(tab, type, size) == NULL) {
597st_table_size(
const struct st_table *tbl)
599 return tbl->num_entries;
607 return st_init_table_with_size(type, 0);
613st_init_numtable(
void)
615 return st_init_table(&type_numhash);
620st_init_numtable_with_size(st_index_t size)
622 return st_init_table_with_size(&type_numhash, size);
628st_init_strtable(
void)
630 return st_init_table(&type_strhash);
635st_init_strtable_with_size(st_index_t size)
637 return st_init_table_with_size(&type_strhash, size);
643st_init_strcasetable(
void)
645 return st_init_table(&type_strcasehash);
651st_init_strcasetable_with_size(st_index_t size)
653 return st_init_table_with_size(&type_strcasehash, size);
678 + (tab->bins == NULL ? 0 : bins_size(tab))
683find_table_entry_ind(
st_table *tab, st_hash_t hash_value, st_data_t key);
686find_table_bin_ind(
st_table *tab, st_hash_t hash_value, st_data_t key);
689find_table_bin_ind_direct(
st_table *table, st_hash_t hash_value, st_data_t key);
692find_table_bin_ptr_and_reserve(
st_table *tab, st_hash_t *hash_value,
693 st_data_t key, st_index_t *bin_ind);
700 if (type == &type_numhash) {
703 else if (type == &type_strhash) {
706 else if (type == &type_strcasehash) {
711#define COLLISION (collision_check ? count_collision(tab->type) : (void)0)
712#define FOUND_BIN (collision_check ? collision.total++ : (void)0)
713#define collision_check 0
722#define REBUILD_THRESHOLD 4
724#if REBUILD_THRESHOLD < 2
725#error "REBUILD_THRESHOLD should be >= 2"
728static void rebuild_table_with(
st_table *
const new_tab,
st_table *
const tab);
729static void rebuild_move_table(
st_table *
const new_tab,
st_table *
const tab);
730static void rebuild_cleanup(
st_table *
const tab);
739 if ((2 * tab->num_entries <= get_allocated_entries(tab)
740 && REBUILD_THRESHOLD * tab->num_entries > get_allocated_entries(tab))
741 || tab->num_entries < (1 << MINIMAL_POWER2)) {
743 tab->num_entries = 0;
744 if (tab->bins != NULL)
745 initialize_bins(tab);
746 rebuild_table_with(tab, tab);
753 new_tab = st_init_table_with_size(tab->type,
754 2 * tab->num_entries - 1);
755 rebuild_table_with(new_tab, tab);
756 rebuild_move_table(new_tab, tab);
758 rebuild_cleanup(tab);
765 unsigned int size_ind;
771 new_entries = new_tab->entries;
774 bins = new_tab->bins;
775 size_ind = get_size_ind(new_tab);
776 st_index_t bound = tab->entries_bound;
779 for (i = tab->entries_start; i < bound; i++) {
780 curr_entry_ptr = &entries[i];
781 PREFETCH(entries + i + 1, 0);
782 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
784 if (&new_entries[ni] != curr_entry_ptr)
785 new_entries[ni] = *curr_entry_ptr;
786 if (EXPECT(bins != NULL, 1)) {
787 bin_ind = find_table_bin_ind_direct(new_tab, curr_entry_ptr->hash,
788 curr_entry_ptr->key);
789 set_bin(bins, size_ind, bin_ind, ni + ENTRY_BASE);
791 new_tab->num_entries++;
795 assert(new_tab->num_entries == tab->num_entries);
801 tab->entry_power = new_tab->entry_power;
802 tab->bin_power = new_tab->bin_power;
803 tab->size_ind = new_tab->size_ind;
805 tab->bins = new_tab->bins;
807 tab->entries = new_tab->entries;
814 tab->entries_start = 0;
815 tab->entries_bound = tab->num_entries;
831static inline st_index_t
832secondary_hash(st_index_t ind,
st_table *tab, st_index_t *perturb)
835 ind = (ind << 2) + ind + *perturb + 1;
836 return hash_bin(ind, tab);
843static inline st_index_t
844find_entry(
st_table *tab, st_hash_t hash_value, st_data_t key)
850 bound = tab->entries_bound;
851 entries = tab->entries;
852 for (i = tab->entries_start; i < bound; i++) {
853 DO_PTR_EQUAL_CHECK(tab, &entries[i], hash_value, key, eq_p, rebuilt_p);
854 if (EXPECT(rebuilt_p, 0))
855 return REBUILT_TABLE_ENTRY_IND;
859 return UNDEFINED_ENTRY_IND;
871find_table_entry_ind(
st_table *tab, st_hash_t hash_value, st_data_t key)
875#ifdef QUADRATIC_PROBE
883 ind = hash_bin(hash_value, tab);
884#ifdef QUADRATIC_PROBE
887 perturb = hash_value;
891 bin = get_bin(tab->bins, get_size_ind(tab), ind);
892 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
893 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
894 if (EXPECT(rebuilt_p, 0))
895 return REBUILT_TABLE_ENTRY_IND;
899 else if (EMPTY_BIN_P(bin))
900 return UNDEFINED_ENTRY_IND;
901#ifdef QUADRATIC_PROBE
902 ind = hash_bin(ind + d, tab);
905 ind = secondary_hash(ind, tab, &perturb);
917find_table_bin_ind(
st_table *tab, st_hash_t hash_value, st_data_t key)
921#ifdef QUADRATIC_PROBE
929 ind = hash_bin(hash_value, tab);
930#ifdef QUADRATIC_PROBE
933 perturb = hash_value;
937 bin = get_bin(tab->bins, get_size_ind(tab), ind);
938 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
939 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
940 if (EXPECT(rebuilt_p, 0))
941 return REBUILT_TABLE_BIN_IND;
945 else if (EMPTY_BIN_P(bin))
946 return UNDEFINED_BIN_IND;
947#ifdef QUADRATIC_PROBE
948 ind = hash_bin(ind + d, tab);
951 ind = secondary_hash(ind, tab, &perturb);
962find_table_bin_ind_direct(
st_table *tab, st_hash_t hash_value, st_data_t key)
965#ifdef QUADRATIC_PROBE
972 ind = hash_bin(hash_value, tab);
973#ifdef QUADRATIC_PROBE
976 perturb = hash_value;
980 bin = get_bin(tab->bins, get_size_ind(tab), ind);
981 if (EMPTY_OR_DELETED_BIN_P(bin))
983#ifdef QUADRATIC_PROBE
984 ind = hash_bin(ind + d, tab);
987 ind = secondary_hash(ind, tab, &perturb);
1003find_table_bin_ptr_and_reserve(
st_table *tab, st_hash_t *hash_value,
1004 st_data_t key, st_index_t *bin_ind)
1006 int eq_p, rebuilt_p;
1008 st_hash_t curr_hash_value = *hash_value;
1009#ifdef QUADRATIC_PROBE
1014 st_index_t entry_index;
1015 st_index_t first_deleted_bin_ind;
1018 ind = hash_bin(curr_hash_value, tab);
1019#ifdef QUADRATIC_PROBE
1022 perturb = curr_hash_value;
1025 first_deleted_bin_ind = UNDEFINED_BIN_IND;
1026 entries = tab->entries;
1028 entry_index = get_bin(tab->bins, get_size_ind(tab), ind);
1029 if (EMPTY_BIN_P(entry_index)) {
1031 entry_index = UNDEFINED_ENTRY_IND;
1032 if (first_deleted_bin_ind != UNDEFINED_BIN_IND) {
1034 ind = first_deleted_bin_ind;
1035 MARK_BIN_EMPTY(tab, ind);
1039 else if (! DELETED_BIN_P(entry_index)) {
1040 DO_PTR_EQUAL_CHECK(tab, &entries[entry_index - ENTRY_BASE], curr_hash_value, key, eq_p, rebuilt_p);
1041 if (EXPECT(rebuilt_p, 0))
1042 return REBUILT_TABLE_ENTRY_IND;
1046 else if (first_deleted_bin_ind == UNDEFINED_BIN_IND)
1047 first_deleted_bin_ind = ind;
1048#ifdef QUADRATIC_PROBE
1049 ind = hash_bin(ind + d, tab);
1052 ind = secondary_hash(ind, tab, &perturb);
1063st_lookup(
st_table *tab, st_data_t key, st_data_t *value)
1066 st_hash_t hash = do_hash(key, tab);
1069 if (tab->bins == NULL) {
1070 bin = find_entry(tab, hash, key);
1071 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1073 if (bin == UNDEFINED_ENTRY_IND)
1077 bin = find_table_entry_ind(tab, hash, key);
1078 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1080 if (bin == UNDEFINED_ENTRY_IND)
1085 *value = tab->entries[bin].record;
1092st_get_key(
st_table *tab, st_data_t key, st_data_t *result)
1095 st_hash_t hash = do_hash(key, tab);
1098 if (tab->bins == NULL) {
1099 bin = find_entry(tab, hash, key);
1100 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1102 if (bin == UNDEFINED_ENTRY_IND)
1106 bin = find_table_entry_ind(tab, hash, key);
1107 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1109 if (bin == UNDEFINED_ENTRY_IND)
1114 *result = tab->entries[bin].key;
1120rebuild_table_if_necessary (
st_table *tab)
1122 st_index_t bound = tab->entries_bound;
1124 if (bound == get_allocated_entries(tab))
1132st_insert(
st_table *tab, st_data_t key, st_data_t value)
1137 st_hash_t hash_value;
1141 hash_value = do_hash(key, tab);
1143 rebuild_table_if_necessary(tab);
1144 if (tab->bins == NULL) {
1145 bin = find_entry(tab, hash_value, key);
1146 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1148 new_p = bin == UNDEFINED_ENTRY_IND;
1151 bin_ind = UNDEFINED_BIN_IND;
1154 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1156 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1158 new_p = bin == UNDEFINED_ENTRY_IND;
1162 ind = tab->entries_bound++;
1163 entry = &tab->entries[ind];
1164 entry->hash = hash_value;
1166 entry->record = value;
1167 if (bin_ind != UNDEFINED_BIN_IND)
1168 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1171 tab->entries[bin].record = value;
1178st_add_direct_with_hash(
st_table *tab,
1179 st_data_t key, st_data_t value, st_hash_t hash)
1185 assert(hash != RESERVED_HASH_VAL);
1187 rebuild_table_if_necessary(tab);
1188 ind = tab->entries_bound++;
1189 entry = &tab->entries[ind];
1192 entry->record = value;
1194 if (tab->bins != NULL) {
1195 bin_ind = find_table_bin_ind_direct(tab, hash, key);
1196 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1201rb_st_add_direct_with_hash(
st_table *tab,
1202 st_data_t key, st_data_t value, st_hash_t hash)
1204 st_add_direct_with_hash(tab, key, value, normalize_hash_value(hash));
1210st_add_direct(
st_table *tab, st_data_t key, st_data_t value)
1212 st_hash_t hash_value;
1214 hash_value = do_hash(key, tab);
1215 st_add_direct_with_hash(tab, key, value, hash_value);
1222st_insert2(
st_table *tab, st_data_t key, st_data_t value,
1223 st_data_t (*func)(st_data_t))
1228 st_hash_t hash_value;
1232 hash_value = do_hash(key, tab);
1234 rebuild_table_if_necessary (tab);
1235 if (tab->bins == NULL) {
1236 bin = find_entry(tab, hash_value, key);
1237 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1239 new_p = bin == UNDEFINED_ENTRY_IND;
1242 bin_ind = UNDEFINED_BIN_IND;
1245 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1247 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1249 new_p = bin == UNDEFINED_ENTRY_IND;
1254 ind = tab->entries_bound++;
1255 entry = &tab->entries[ind];
1256 entry->hash = hash_value;
1258 entry->record = value;
1259 if (bin_ind != UNDEFINED_BIN_IND)
1260 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1263 tab->entries[bin].record = value;
1271 *new_tab = *old_tab;
1272 if (old_tab->bins == NULL)
1273 new_tab->bins = NULL;
1275 new_tab->bins = (st_index_t *) malloc(bins_size(old_tab));
1277 if (new_tab->bins == NULL) {
1282 new_tab->entries = (
st_table_entry *) malloc(get_allocated_entries(old_tab)
1285 if (new_tab->entries == NULL) {
1290 get_allocated_entries(old_tab));
1291 if (old_tab->bins != NULL)
1292 MEMCPY(new_tab->bins, old_tab->bins,
char, bins_size(old_tab));
1305 if (new_tab == NULL)
1309 if (st_replace(new_tab, old_tab) == NULL) {
1310 st_free_table(new_tab);
1320update_range_for_deleted(
st_table *tab, st_index_t n)
1324 if (tab->entries_start == n) {
1325 st_index_t start = n + 1;
1326 st_index_t bound = tab->entries_bound;
1328 while (start < bound && DELETED_ENTRY_P(&entries[start])) start++;
1329 tab->entries_start = start;
1338st_general_delete(
st_table *tab, st_data_t *key, st_data_t *value)
1345 hash = do_hash(*key, tab);
1347 if (tab->bins == NULL) {
1348 bin = find_entry(tab, hash, *key);
1349 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1351 if (bin == UNDEFINED_ENTRY_IND) {
1352 if (value != 0) *value = 0;
1357 bin_ind = find_table_bin_ind(tab, hash, *key);
1358 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1360 if (bin_ind == UNDEFINED_BIN_IND) {
1361 if (value != 0) *value = 0;
1364 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1365 MARK_BIN_DELETED(tab, bin_ind);
1367 entry = &tab->entries[bin];
1369 if (value != 0) *value = entry->record;
1370 MARK_ENTRY_DELETED(entry);
1372 update_range_for_deleted(tab, bin);
1377st_delete(
st_table *tab, st_data_t *key, st_data_t *value)
1379 return st_general_delete(tab, key, value);
1388st_delete_safe(
st_table *tab, st_data_t *key, st_data_t *value,
1389 st_data_t never ATTRIBUTE_UNUSED)
1391 return st_general_delete(tab, key, value);
1399st_shift(
st_table *tab, st_data_t *key, st_data_t *value)
1401 st_index_t i, bound;
1406 entries = tab->entries;
1407 bound = tab->entries_bound;
1408 for (i = tab->entries_start; i < bound; i++) {
1409 curr_entry_ptr = &entries[i];
1410 if (! DELETED_ENTRY_P(curr_entry_ptr)) {
1411 st_hash_t entry_hash = curr_entry_ptr->hash;
1412 st_data_t entry_key = curr_entry_ptr->key;
1414 if (value != 0) *value = curr_entry_ptr->record;
1417 if (tab->bins == NULL) {
1418 bin = find_entry(tab, entry_hash, entry_key);
1419 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0)) {
1420 entries = tab->entries;
1423 curr_entry_ptr = &entries[bin];
1426 bin_ind = find_table_bin_ind(tab, entry_hash, entry_key);
1427 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0)) {
1428 entries = tab->entries;
1431 curr_entry_ptr = &entries[get_bin(tab->bins, get_size_ind(tab), bin_ind)
1433 MARK_BIN_DELETED(tab, bin_ind);
1435 MARK_ENTRY_DELETED(curr_entry_ptr);
1437 update_range_for_deleted(tab, i);
1441 if (value != 0) *value = 0;
1447st_cleanup_safe(
st_table *tab ATTRIBUTE_UNUSED,
1448 st_data_t never ATTRIBUTE_UNUSED)
1462st_update(
st_table *tab, st_data_t key,
1463 st_update_callback_func *func, st_data_t arg)
1469 st_data_t value = 0, old_key;
1470 int retval, existing;
1471 st_hash_t hash = do_hash(key, tab);
1474 entries = tab->entries;
1475 if (tab->bins == NULL) {
1476 bin = find_entry(tab, hash, key);
1477 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1479 existing = bin != UNDEFINED_ENTRY_IND;
1480 entry = &entries[bin];
1481 bin_ind = UNDEFINED_BIN_IND;
1484 bin_ind = find_table_bin_ind(tab, hash, key);
1485 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1487 existing = bin_ind != UNDEFINED_BIN_IND;
1489 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1490 entry = &entries[bin];
1495 value = entry->record;
1499 unsigned int rebuilds_num = tab->rebuilds_num;
1501 retval = (*func)(&key, &value, arg, existing);
1505 assert(rebuilds_num == tab->rebuilds_num);
1511 st_add_direct_with_hash(tab, key, value, hash);
1514 if (old_key != key) {
1517 entry->record = value;
1521 if (bin_ind != UNDEFINED_BIN_IND)
1522 MARK_BIN_DELETED(tab, bin_ind);
1523 MARK_ENTRY_DELETED(entry);
1525 update_range_for_deleted(tab, bin);
1541st_general_foreach(
st_table *tab, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg,
1547 enum st_retval retval;
1548 st_index_t i, rebuilds_num;
1551 int error_p, packed_p = tab->bins == NULL;
1553 entries = tab->entries;
1556 for (i = tab->entries_start; i < tab->entries_bound; i++) {
1557 curr_entry_ptr = &entries[i];
1558 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
1560 key = curr_entry_ptr->key;
1561 rebuilds_num = tab->rebuilds_num;
1562 hash = curr_entry_ptr->hash;
1563 retval = (*func)(key, curr_entry_ptr->record, arg, 0);
1565 if (retval == ST_REPLACE && replace) {
1567 value = curr_entry_ptr->record;
1568 retval = (*replace)(&key, &value, arg, TRUE);
1569 curr_entry_ptr->key = key;
1570 curr_entry_ptr->record = value;
1573 if (rebuilds_num != tab->rebuilds_num) {
1575 entries = tab->entries;
1576 packed_p = tab->bins == NULL;
1578 i = find_entry(tab, hash, key);
1579 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
1581 error_p = i == UNDEFINED_ENTRY_IND;
1584 i = find_table_entry_ind(tab, hash, key);
1585 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
1587 error_p = i == UNDEFINED_ENTRY_IND;
1590 if (error_p && check_p) {
1592 retval = (*func)(0, 0, arg, 1);
1595 curr_entry_ptr = &entries[i];
1608 st_data_t key = curr_entry_ptr->key;
1612 bin = find_entry(tab, hash, key);
1613 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1615 if (bin == UNDEFINED_ENTRY_IND)
1619 bin_ind = find_table_bin_ind(tab, hash, key);
1620 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1622 if (bin_ind == UNDEFINED_BIN_IND)
1624 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1625 MARK_BIN_DELETED(tab, bin_ind);
1627 curr_entry_ptr = &entries[bin];
1628 MARK_ENTRY_DELETED(curr_entry_ptr);
1630 update_range_for_deleted(tab, bin);
1639st_foreach_with_replace(
st_table *tab, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg)
1641 return st_general_foreach(tab, func, replace, arg, TRUE);
1645 st_foreach_callback_func *func;
1650apply_functor(st_data_t k, st_data_t v, st_data_t d,
int _)
1652 const struct functor *f = (
void *)d;
1653 return f->func(k, v, f->arg);
1657st_foreach(
st_table *tab, st_foreach_callback_func *func, st_data_t arg)
1659 const struct functor f = { func, arg };
1660 return st_general_foreach(tab, apply_functor, 0, (st_data_t)&f, FALSE);
1665st_foreach_check(
st_table *tab, st_foreach_check_callback_func *func, st_data_t arg,
1666 st_data_t never ATTRIBUTE_UNUSED)
1668 return st_general_foreach(tab, func, 0, arg, TRUE);
1673static inline st_index_t
1674st_general_keys(
st_table *tab, st_data_t *keys, st_index_t size)
1676 st_index_t i, bound;
1677 st_data_t key, *keys_start, *keys_end;
1680 bound = tab->entries_bound;
1682 keys_end = keys + size;
1683 for (i = tab->entries_start; i < bound; i++) {
1684 if (keys == keys_end)
1686 curr_entry_ptr = &entries[i];
1687 key = curr_entry_ptr->key;
1688 if (! DELETED_ENTRY_P(curr_entry_ptr))
1692 return keys - keys_start;
1696st_keys(
st_table *tab, st_data_t *keys, st_index_t size)
1698 return st_general_keys(tab, keys, size);
1703st_keys_check(
st_table *tab, st_data_t *keys, st_index_t size,
1704 st_data_t never ATTRIBUTE_UNUSED)
1706 return st_general_keys(tab, keys, size);
1711static inline st_index_t
1712st_general_values(
st_table *tab, st_data_t *values, st_index_t size)
1714 st_index_t i, bound;
1715 st_data_t *values_start, *values_end;
1718 values_start = values;
1719 values_end = values + size;
1720 bound = tab->entries_bound;
1721 for (i = tab->entries_start; i < bound; i++) {
1722 if (values == values_end)
1724 curr_entry_ptr = &entries[i];
1725 if (! DELETED_ENTRY_P(curr_entry_ptr))
1726 *values++ = curr_entry_ptr->record;
1729 return values - values_start;
1733st_values(
st_table *tab, st_data_t *values, st_index_t size)
1735 return st_general_values(tab, values, size);
1740st_values_check(
st_table *tab, st_data_t *values, st_index_t size,
1741 st_data_t never ATTRIBUTE_UNUSED)
1743 return st_general_values(tab, values, size);
1746#define FNV1_32A_INIT 0x811c9dc5
1751#define FNV_32_PRIME 0x01000193
1754#ifndef UNALIGNED_WORD_ACCESS
1755# if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \
1756 defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || \
1757 defined(__powerpc64__) || defined(__POWERPC__) || defined(__aarch64__) || \
1758 defined(__mc68020__)
1759# define UNALIGNED_WORD_ACCESS 1
1762#ifndef UNALIGNED_WORD_ACCESS
1763# define UNALIGNED_WORD_ACCESS 0
1769#define BIG_CONSTANT(x,y) ((st_index_t)(x)<<32|(st_index_t)(y))
1770#define ROTL(x,n) ((x)<<(n)|(x)>>(SIZEOF_ST_INDEX_T*CHAR_BIT-(n)))
1772#if ST_INDEX_BITS <= 32
1773#define C1 (st_index_t)0xcc9e2d51
1774#define C2 (st_index_t)0x1b873593
1776#define C1 BIG_CONSTANT(0x87c37b91,0x114253d5);
1777#define C2 BIG_CONSTANT(0x4cf5ad43,0x2745937f);
1779NO_SANITIZE(
"unsigned-integer-overflow",
static inline st_index_t murmur_step(st_index_t h, st_index_t k));
1780NO_SANITIZE(
"unsigned-integer-overflow",
static inline st_index_t murmur_finish(st_index_t h));
1781NO_SANITIZE(
"unsigned-integer-overflow",
extern st_index_t st_hash(
const void *ptr,
size_t len, st_index_t h));
1783static inline st_index_t
1784murmur_step(st_index_t h, st_index_t k)
1786#if ST_INDEX_BITS <= 32
1802static inline st_index_t
1803murmur_finish(st_index_t h)
1805#if ST_INDEX_BITS <= 32
1809 const st_index_t c1 = 0x85ebca6b;
1810 const st_index_t c2 = 0xc2b2ae35;
1816 const st_index_t c1 = BIG_CONSTANT(0xbf58476d,0x1ce4e5b9);
1817 const st_index_t c2 = BIG_CONSTANT(0x94d049bb,0x133111eb);
1819#if ST_INDEX_BITS > 64
1836st_hash(
const void *ptr,
size_t len, st_index_t h)
1838 const char *data = ptr;
1842#define data_at(n) (st_index_t)((unsigned char)data[(n)])
1843#define UNALIGNED_ADD_4 UNALIGNED_ADD(2); UNALIGNED_ADD(1); UNALIGNED_ADD(0)
1844#if SIZEOF_ST_INDEX_T > 4
1845#define UNALIGNED_ADD_8 UNALIGNED_ADD(6); UNALIGNED_ADD(5); UNALIGNED_ADD(4); UNALIGNED_ADD(3); UNALIGNED_ADD_4
1846#if SIZEOF_ST_INDEX_T > 8
1847#define UNALIGNED_ADD_16 UNALIGNED_ADD(14); UNALIGNED_ADD(13); UNALIGNED_ADD(12); UNALIGNED_ADD(11); \
1848 UNALIGNED_ADD(10); UNALIGNED_ADD(9); UNALIGNED_ADD(8); UNALIGNED_ADD(7); UNALIGNED_ADD_8
1849#define UNALIGNED_ADD_ALL UNALIGNED_ADD_16
1851#define UNALIGNED_ADD_ALL UNALIGNED_ADD_8
1853#define UNALIGNED_ADD_ALL UNALIGNED_ADD_4
1856 if (
len >=
sizeof(st_index_t)) {
1857#if !UNALIGNED_WORD_ACCESS
1858 int align = (int)((st_data_t)data %
sizeof(st_index_t));
1864#ifdef WORDS_BIGENDIAN
1865# define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1866 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 2)
1868# define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1869 t |= data_at(n) << CHAR_BIT*(n)
1875#ifdef WORDS_BIGENDIAN
1876 t >>= (CHAR_BIT * align) - CHAR_BIT;
1878 t <<= (CHAR_BIT * align);
1881 data +=
sizeof(st_index_t)-align;
1882 len -=
sizeof(st_index_t)-align;
1884 sl = CHAR_BIT * (SIZEOF_ST_INDEX_T-align);
1885 sr = CHAR_BIT * align;
1887 while (
len >=
sizeof(st_index_t)) {
1888 d = *(st_index_t *)data;
1889#ifdef WORDS_BIGENDIAN
1890 t = (t << sr) | (d >> sl);
1892 t = (t >> sr) | (d << sl);
1894 h = murmur_step(h, t);
1896 data +=
sizeof(st_index_t);
1897 len -=
sizeof(st_index_t);
1900 pack =
len < (size_t)align ? (
int)
len : align;
1903#ifdef WORDS_BIGENDIAN
1904# define UNALIGNED_ADD(n) case (n) + 1: \
1905 d |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
1907# define UNALIGNED_ADD(n) case (n) + 1: \
1908 d |= data_at(n) << CHAR_BIT*(n)
1913#ifdef WORDS_BIGENDIAN
1914 t = (t << sr) | (d >> sl);
1916 t = (t >> sr) | (d << sl);
1919 if (
len < (
size_t)align)
goto skip_tail;
1921 h = murmur_step(h, t);
1927#ifdef HAVE_BUILTIN___BUILTIN_ASSUME_ALIGNED
1928#define aligned_data __builtin_assume_aligned(data, sizeof(st_index_t))
1930#define aligned_data data
1934 h = murmur_step(h, *(st_index_t *)aligned_data);
1935 data +=
sizeof(st_index_t);
1936 len -=
sizeof(st_index_t);
1937 }
while (
len >=
sizeof(st_index_t));
1943#if UNALIGNED_WORD_ACCESS && SIZEOF_ST_INDEX_T <= 8 && CHAR_BIT == 8
1945#if SIZEOF_ST_INDEX_T > 4
1946 case 7: t |= data_at(6) << 48;
1947 case 6: t |= data_at(5) << 40;
1948 case 5: t |= data_at(4) << 32;
1950 t |= (st_index_t)*(uint32_t*)aligned_data;
1954 case 3: t |= data_at(2) << 16;
1955 case 2: t |= data_at(1) << 8;
1956 case 1: t |= data_at(0);
1958#ifdef WORDS_BIGENDIAN
1959# define UNALIGNED_ADD(n) case (n) + 1: \
1960 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
1962# define UNALIGNED_ADD(n) case (n) + 1: \
1963 t |= data_at(n) << CHAR_BIT*(n)
1971 h ^= t; h -= ROTL(t, 7);
1977 return murmur_finish(h);
1981st_hash_uint32(st_index_t h, uint32_t i)
1983 return murmur_step(h, i);
1986NO_SANITIZE(
"unsigned-integer-overflow",
extern st_index_t st_hash_uint(st_index_t h, st_index_t i));
1988st_hash_uint(st_index_t h, st_index_t i)
1993#if SIZEOF_ST_INDEX_T*CHAR_BIT > 8*8
1994 h = murmur_step(h, i >> 8*8);
1996 h = murmur_step(h, i);
2001st_hash_end(st_index_t h)
2003 h = murmur_finish(h);
2009rb_st_hash_start(st_index_t h)
2015strhash(st_data_t arg)
2017 register const char *
string = (
const char *)arg;
2018 return st_hash(
string, strlen(
string), FNV1_32A_INIT);
2022st_locale_insensitive_strcasecmp(
const char *s1,
const char *s2)
2029 if (c1 ==
'\0' || c2 ==
'\0') {
2030 if (c1 !=
'\0')
return 1;
2031 if (c2 !=
'\0')
return -1;
2034 if ((
'A' <= c1) && (c1 <=
'Z')) c1 +=
'a' -
'A';
2035 if ((
'A' <= c2) && (c2 <=
'Z')) c2 +=
'a' -
'A';
2046st_locale_insensitive_strncasecmp(
const char *s1,
const char *s2,
size_t n)
2051 for (i = 0; i < n; i++) {
2054 if (c1 ==
'\0' || c2 ==
'\0') {
2055 if (c1 !=
'\0')
return 1;
2056 if (c2 !=
'\0')
return -1;
2059 if ((
'A' <= c1) && (c1 <=
'Z')) c1 +=
'a' -
'A';
2060 if ((
'A' <= c2) && (c2 <=
'Z')) c2 +=
'a' -
'A';
2072st_strcmp(st_data_t lhs, st_data_t rhs)
2074 const char *s1 = (
char *)lhs;
2075 const char *s2 = (
char *)rhs;
2076 return strcmp(s1, s2);
2080st_locale_insensitive_strcasecmp_i(st_data_t lhs, st_data_t rhs)
2082 const char *s1 = (
char *)lhs;
2083 const char *s2 = (
char *)rhs;
2084 return st_locale_insensitive_strcasecmp(s1, s2);
2087NO_SANITIZE(
"unsigned-integer-overflow", PUREFUNC(
static st_index_t strcasehash(st_data_t)));
2089strcasehash(st_data_t arg)
2091 register const char *
string = (
const char *)arg;
2092 register st_index_t hval = FNV1_32A_INIT;
2098 unsigned int c = (
unsigned char)*
string++;
2099 if ((
unsigned int)(c -
'A') <= (
'Z' -
'A')) c +=
'a' -
'A';
2103 hval *= FNV_32_PRIME;
2109st_numcmp(st_data_t x, st_data_t y)
2115st_numhash(st_data_t n)
2117 enum {s1 = 11, s2 = 3};
2118 return (st_index_t)((n>>s1|(n<<s2)) ^ (n>>s2));
2126st_expand_table(
st_table *tab, st_index_t siz)
2131 if (siz <= get_allocated_entries(tab))
2134 tmp = st_init_table_with_size(tab->type, siz);
2135 n = get_allocated_entries(tab);
2140 tab->entry_power = tmp->entry_power;
2141 tab->bin_power = tmp->bin_power;
2142 tab->size_ind = tmp->size_ind;
2143 tab->entries = tmp->entries;
2145 tab->rebuilds_num++;
2154 int eq_p, rebuilt_p;
2161 for (i = tab->entries_start; i < tab->entries_bound; i++) {
2162 p = &tab->entries[i];
2163 if (DELETED_ENTRY_P(p))
2165 for (j = i + 1; j < tab->entries_bound; j++) {
2166 q = &tab->entries[j];
2167 if (DELETED_ENTRY_P(q))
2169 DO_PTR_EQUAL_CHECK(tab, p, q->hash, q->key, eq_p, rebuilt_p);
2170 if (EXPECT(rebuilt_p, 0))
2174 MARK_ENTRY_DELETED(q);
2176 update_range_for_deleted(tab, j);
2188 int eq_p, rebuilt_p;
2190 st_index_t
const n = bins_size(tab);
2191 unsigned int const size_ind = get_size_ind(tab);
2192 st_index_t *bins = realloc(tab->bins, n);
2194 initialize_bins(tab);
2195 for (i = tab->entries_start; i < tab->entries_bound; i++) {
2198#ifdef QUADRATIC_PROBE
2201 st_index_t perturb = p->hash;
2204 if (DELETED_ENTRY_P(p))
2207 ind = hash_bin(p->hash, tab);
2209 st_index_t bin = get_bin(bins, size_ind, ind);
2210 if (EMPTY_OR_DELETED_BIN_P(bin)) {
2212 set_bin(bins, size_ind, ind, i + ENTRY_BASE);
2217 DO_PTR_EQUAL_CHECK(tab, q, p->hash, p->key, eq_p, rebuilt_p);
2218 if (EXPECT(rebuilt_p, 0))
2222 q->record = p->record;
2223 MARK_ENTRY_DELETED(p);
2225 update_range_for_deleted(tab, bin);
2230#ifdef QUADRATIC_PROBE
2231 ind = hash_bin(ind + d, tab);
2234 ind = secondary_hash(ind, tab, &perturb);
2252 if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
2253 rebuilt_p = st_rehash_linear(tab);
2255 rebuilt_p = st_rehash_indexed(tab);
2256 }
while (rebuilt_p);
2260st_stringify(
VALUE key)
2263 rb_hash_key_str(key) : key;
2269 st_data_t k = st_stringify(key);
2271 e.hash = do_hash(k, tab);
2275 tab->entries[tab->entries_bound++] = e;
2286 for (i = 0; i < argc; ) {
2287 st_data_t k = st_stringify(argv[i++]);
2288 st_data_t v = argv[i++];
2289 st_insert(tab, k, v);
2301 for (i = 0; i < argc; ) {
2302 VALUE key = argv[i++];
2303 VALUE val = argv[i++];
2304 st_insert_single(tab, hash, key, val);
2314rb_hash_bulk_insert_into_st_table(
long argc,
const VALUE *argv,
VALUE hash)
2316 st_index_t n, size = argc / 2;
2317 st_table *tab = RHASH_ST_TABLE(hash);
2319 tab = RHASH_TBL_RAW(hash);
2320 n = tab->entries_bound + size;
2321 st_expand_table(tab, n);
2322 if (UNLIKELY(tab->num_entries))
2323 st_insert_generic(tab, argc, argv, hash);
2325 st_insert_single(tab, hash, argv[0], argv[1]);
2326 else if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
2327 st_insert_linear(tab, argc, argv, hash);
2329 st_insert_generic(tab, argc, argv, hash);
2335 st_index_t num = tab->num_entries;
2336 if (REBUILD_THRESHOLD * num <= get_allocated_entries(tab)) {
2338 st_table *new_tab = st_init_table_with_size(tab->type, 2 * num);
2339 rebuild_table_with(new_tab, tab);
2340 rebuild_move_table(new_tab, tab);
2341 rebuild_cleanup(tab);
2349struct set_table_entry {
2355static inline st_hash_t
2356set_do_hash(st_data_t key,
set_table *tab)
2358 st_hash_t hash = (st_hash_t)(tab->type->hash)(key);
2359 return normalize_hash_value(hash);
2363static inline unsigned int
2366 return tab->size_ind;
2370static inline st_index_t
2373 return ((st_index_t) 1)<<tab->bin_power;
2377static inline st_index_t
2380 return set_get_bins_num(tab) - 1;
2385static inline st_index_t
2386set_hash_bin(st_hash_t hash_value,
set_table *tab)
2388 return hash_value & set_bins_mask(tab);
2392static inline st_index_t
2393set_get_allocated_entries(
const set_table *tab)
2395 return ((st_index_t) 1)<<tab->entry_power;
2399static inline st_index_t
2402 return features[tab->entry_power].bins_words *
sizeof (st_index_t);
2409 memset(tab->bins, 0, set_bins_size(tab));
2416 tab->num_entries = 0;
2417 tab->entries_start = tab->entries_bound = 0;
2418 if (tab->bins != NULL)
2419 set_initialize_bins(tab);
2430 const char *e = getenv(
"ST_HASH_LOG");
2431 if (!e || !*e) init_st = 1;
2440 n = get_power2(size);
2443 tab->entry_power = n;
2444 tab->bin_power = features[n].bin_power;
2445 tab->size_ind = features[n].size_ind;
2446 if (n <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
2449 tab->bins = (st_index_t *) malloc(set_bins_size(tab));
2451 tab->entries = (set_table_entry *) malloc(set_get_allocated_entries(tab)
2452 *
sizeof(set_table_entry));
2453 set_make_tab_empty(tab);
2454 tab->rebuilds_num = 0;
2464 if (tab == NULL) tab = malloc(
sizeof(
set_table));
2466 set_init_existing_table_with_size(tab, type, size);
2472set_init_numtable(
void)
2474 return set_init_table_with_size(NULL, &type_numhash, 0);
2478set_init_numtable_with_size(st_index_t size)
2480 return set_init_table_with_size(NULL, &type_numhash, size);
2484set_table_size(
const struct set_table *tbl)
2486 return tbl->num_entries;
2493 set_make_tab_empty(tab);
2494 tab->rebuilds_num++;
2512 + (tab->bins == NULL ? 0 : set_bins_size(tab))
2513 + set_get_allocated_entries(tab) * sizeof(set_table_entry));
2517set_find_table_entry_ind(
set_table *tab, st_hash_t hash_value, st_data_t key);
2520set_find_table_bin_ind(
set_table *tab, st_hash_t hash_value, st_data_t key);
2523set_find_table_bin_ind_direct(
set_table *table, st_hash_t hash_value, st_data_t key);
2526set_find_table_bin_ptr_and_reserve(
set_table *tab, st_hash_t *hash_value,
2527 st_data_t key, st_index_t *bin_ind);
2531static void set_rebuild_cleanup(
set_table *
const tab);
2540 if ((2 * tab->num_entries <= set_get_allocated_entries(tab)
2541 && REBUILD_THRESHOLD * tab->num_entries > set_get_allocated_entries(tab))
2542 || tab->num_entries < (1 << MINIMAL_POWER2)) {
2544 tab->num_entries = 0;
2545 if (tab->bins != NULL)
2546 set_initialize_bins(tab);
2547 set_rebuild_table_with(tab, tab);
2554 new_tab = set_init_table_with_size(NULL, tab->type,
2555 2 * tab->num_entries - 1);
2556 set_rebuild_table_with(new_tab, tab);
2557 set_rebuild_move_table(new_tab, tab);
2559 set_rebuild_cleanup(tab);
2566 unsigned int size_ind;
2567 set_table_entry *new_entries;
2568 set_table_entry *curr_entry_ptr;
2572 new_entries = new_tab->entries;
2575 bins = new_tab->bins;
2576 size_ind = set_get_size_ind(new_tab);
2577 st_index_t bound = tab->entries_bound;
2578 set_table_entry *entries = tab->entries;
2580 for (i = tab->entries_start; i < bound; i++) {
2581 curr_entry_ptr = &entries[i];
2582 PREFETCH(entries + i + 1, 0);
2583 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
2585 if (&new_entries[ni] != curr_entry_ptr)
2586 new_entries[ni] = *curr_entry_ptr;
2587 if (EXPECT(bins != NULL, 1)) {
2588 bin_ind = set_find_table_bin_ind_direct(new_tab, curr_entry_ptr->hash,
2589 curr_entry_ptr->key);
2590 set_bin(bins, size_ind, bin_ind, ni + ENTRY_BASE);
2592 new_tab->num_entries++;
2596 assert(new_tab->num_entries == tab->num_entries);
2602 tab->entry_power = new_tab->entry_power;
2603 tab->bin_power = new_tab->bin_power;
2604 tab->size_ind = new_tab->size_ind;
2606 tab->bins = new_tab->bins;
2608 tab->entries = new_tab->entries;
2613set_rebuild_cleanup(
set_table *
const tab)
2615 tab->entries_start = 0;
2616 tab->entries_bound = tab->num_entries;
2617 tab->rebuilds_num++;
2632static inline st_index_t
2633set_secondary_hash(st_index_t ind,
set_table *tab, st_index_t *perturb)
2636 ind = (ind << 2) + ind + *perturb + 1;
2637 return set_hash_bin(ind, tab);
2644static inline st_index_t
2645set_find_entry(
set_table *tab, st_hash_t hash_value, st_data_t key)
2647 int eq_p, rebuilt_p;
2648 st_index_t i, bound;
2649 set_table_entry *entries;
2651 bound = tab->entries_bound;
2652 entries = tab->entries;
2653 for (i = tab->entries_start; i < bound; i++) {
2654 DO_PTR_EQUAL_CHECK(tab, &entries[i], hash_value, key, eq_p, rebuilt_p);
2655 if (EXPECT(rebuilt_p, 0))
2656 return REBUILT_TABLE_ENTRY_IND;
2660 return UNDEFINED_ENTRY_IND;
2672set_find_table_entry_ind(
set_table *tab, st_hash_t hash_value, st_data_t key)
2674 int eq_p, rebuilt_p;
2676#ifdef QUADRATIC_PROBE
2682 set_table_entry *entries = tab->entries;
2684 ind = set_hash_bin(hash_value, tab);
2685#ifdef QUADRATIC_PROBE
2688 perturb = hash_value;
2691 bin = get_bin(tab->bins, set_get_size_ind(tab), ind);
2692 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
2693 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
2694 if (EXPECT(rebuilt_p, 0))
2695 return REBUILT_TABLE_ENTRY_IND;
2699 else if (EMPTY_BIN_P(bin))
2700 return UNDEFINED_ENTRY_IND;
2701#ifdef QUADRATIC_PROBE
2702 ind = set_hash_bin(ind + d, tab);
2705 ind = set_secondary_hash(ind, tab, &perturb);
2716set_find_table_bin_ind(
set_table *tab, st_hash_t hash_value, st_data_t key)
2718 int eq_p, rebuilt_p;
2720#ifdef QUADRATIC_PROBE
2726 set_table_entry *entries = tab->entries;
2728 ind = set_hash_bin(hash_value, tab);
2729#ifdef QUADRATIC_PROBE
2732 perturb = hash_value;
2735 bin = get_bin(tab->bins, set_get_size_ind(tab), ind);
2736 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
2737 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
2738 if (EXPECT(rebuilt_p, 0))
2739 return REBUILT_TABLE_BIN_IND;
2743 else if (EMPTY_BIN_P(bin))
2744 return UNDEFINED_BIN_IND;
2745#ifdef QUADRATIC_PROBE
2746 ind = set_hash_bin(ind + d, tab);
2749 ind = set_secondary_hash(ind, tab, &perturb);
2759set_find_table_bin_ind_direct(
set_table *tab, st_hash_t hash_value, st_data_t key)
2762#ifdef QUADRATIC_PROBE
2769 ind = set_hash_bin(hash_value, tab);
2770#ifdef QUADRATIC_PROBE
2773 perturb = hash_value;
2776 bin = get_bin(tab->bins, set_get_size_ind(tab), ind);
2777 if (EMPTY_OR_DELETED_BIN_P(bin))
2779#ifdef QUADRATIC_PROBE
2780 ind = set_hash_bin(ind + d, tab);
2783 ind = set_secondary_hash(ind, tab, &perturb);
2790#define MARK_SET_BIN_EMPTY(tab, i) (set_bin((tab)->bins, set_get_size_ind(tab), i, EMPTY_BIN))
2802set_find_table_bin_ptr_and_reserve(
set_table *tab, st_hash_t *hash_value,
2803 st_data_t key, st_index_t *bin_ind)
2805 int eq_p, rebuilt_p;
2807 st_hash_t curr_hash_value = *hash_value;
2808#ifdef QUADRATIC_PROBE
2813 st_index_t entry_index;
2814 st_index_t firset_deleted_bin_ind;
2815 set_table_entry *entries;
2817 ind = set_hash_bin(curr_hash_value, tab);
2818#ifdef QUADRATIC_PROBE
2821 perturb = curr_hash_value;
2823 firset_deleted_bin_ind = UNDEFINED_BIN_IND;
2824 entries = tab->entries;
2826 entry_index = get_bin(tab->bins, set_get_size_ind(tab), ind);
2827 if (EMPTY_BIN_P(entry_index)) {
2829 entry_index = UNDEFINED_ENTRY_IND;
2830 if (firset_deleted_bin_ind != UNDEFINED_BIN_IND) {
2832 ind = firset_deleted_bin_ind;
2833 MARK_SET_BIN_EMPTY(tab, ind);
2837 else if (! DELETED_BIN_P(entry_index)) {
2838 DO_PTR_EQUAL_CHECK(tab, &entries[entry_index - ENTRY_BASE], curr_hash_value, key, eq_p, rebuilt_p);
2839 if (EXPECT(rebuilt_p, 0))
2840 return REBUILT_TABLE_ENTRY_IND;
2844 else if (firset_deleted_bin_ind == UNDEFINED_BIN_IND)
2845 firset_deleted_bin_ind = ind;
2846#ifdef QUADRATIC_PROBE
2847 ind = set_hash_bin(ind + d, tab);
2850 ind = set_secondary_hash(ind, tab, &perturb);
2860set_lookup(
set_table *tab, st_data_t key)
2863 st_hash_t hash = set_do_hash(key, tab);
2866 if (tab->bins == NULL) {
2867 bin = set_find_entry(tab, hash, key);
2868 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2870 if (bin == UNDEFINED_ENTRY_IND)
2874 bin = set_find_table_entry_ind(tab, hash, key);
2875 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2877 if (bin == UNDEFINED_ENTRY_IND)
2886set_rebuild_table_if_necessary (
set_table *tab)
2888 st_index_t bound = tab->entries_bound;
2890 if (bound == set_get_allocated_entries(tab))
2891 set_rebuild_table(tab);
2898set_insert(
set_table *tab, st_data_t key)
2900 set_table_entry *entry;
2903 st_hash_t hash_value;
2907 hash_value = set_do_hash(key, tab);
2909 set_rebuild_table_if_necessary(tab);
2910 if (tab->bins == NULL) {
2911 bin = set_find_entry(tab, hash_value, key);
2912 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2914 new_p = bin == UNDEFINED_ENTRY_IND;
2917 bin_ind = UNDEFINED_BIN_IND;
2920 bin = set_find_table_bin_ptr_and_reserve(tab, &hash_value,
2922 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2924 new_p = bin == UNDEFINED_ENTRY_IND;
2928 ind = tab->entries_bound++;
2929 entry = &tab->entries[ind];
2930 entry->hash = hash_value;
2932 if (bin_ind != UNDEFINED_BIN_IND)
2933 set_bin(tab->bins, set_get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
2943 *new_tab = *old_tab;
2944 if (old_tab->bins == NULL)
2945 new_tab->bins = NULL;
2947 new_tab->bins = (st_index_t *) malloc(set_bins_size(old_tab));
2949 new_tab->entries = (set_table_entry *) malloc(set_get_allocated_entries(old_tab)
2950 *
sizeof(set_table_entry));
2951 MEMCPY(new_tab->entries, old_tab->entries, set_table_entry,
2952 set_get_allocated_entries(old_tab));
2953 if (old_tab->bins != NULL)
2954 MEMCPY(new_tab->bins, old_tab->bins,
char, set_bins_size(old_tab));
2965 if (set_replace(new_tab, old_tab) == NULL) {
2966 set_free_table(new_tab);
2976set_update_range_for_deleted(
set_table *tab, st_index_t n)
2980 if (tab->entries_start == n) {
2981 st_index_t start = n + 1;
2982 st_index_t bound = tab->entries_bound;
2983 set_table_entry *entries = tab->entries;
2984 while (start < bound && DELETED_ENTRY_P(&entries[start])) start++;
2985 tab->entries_start = start;
2992#define MARK_SET_BIN_DELETED(tab, i) \
2994 set_bin((tab)->bins, set_get_size_ind(tab), i, DELETED_BIN); \
3000set_delete(
set_table *tab, st_data_t *key)
3002 set_table_entry *entry;
3007 hash = set_do_hash(*key, tab);
3009 if (tab->bins == NULL) {
3010 bin = set_find_entry(tab, hash, *key);
3011 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
3013 if (bin == UNDEFINED_ENTRY_IND) {
3018 bin_ind = set_find_table_bin_ind(tab, hash, *key);
3019 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
3021 if (bin_ind == UNDEFINED_BIN_IND) {
3024 bin = get_bin(tab->bins, set_get_size_ind(tab), bin_ind) - ENTRY_BASE;
3025 MARK_SET_BIN_DELETED(tab, bin_ind);
3027 entry = &tab->entries[bin];
3029 MARK_ENTRY_DELETED(entry);
3031 set_update_range_for_deleted(tab, bin);
3044set_general_foreach(
set_table *tab, set_foreach_check_callback_func *func,
3045 set_update_callback_func *replace, st_data_t arg,
3050 set_table_entry *entries, *curr_entry_ptr;
3051 enum st_retval retval;
3052 st_index_t i, rebuilds_num;
3055 int error_p, packed_p = tab->bins == NULL;
3057 entries = tab->entries;
3060 for (i = tab->entries_start; i < tab->entries_bound; i++) {
3061 curr_entry_ptr = &entries[i];
3062 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
3064 key = curr_entry_ptr->key;
3065 rebuilds_num = tab->rebuilds_num;
3066 hash = curr_entry_ptr->hash;
3067 retval = (*func)(key, arg, 0);
3069 if (retval == ST_REPLACE && replace) {
3070 retval = (*replace)(&key, arg, TRUE);
3071 curr_entry_ptr->key = key;
3074 if (rebuilds_num != tab->rebuilds_num) {
3076 entries = tab->entries;
3077 packed_p = tab->bins == NULL;
3079 i = set_find_entry(tab, hash, key);
3080 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
3082 error_p = i == UNDEFINED_ENTRY_IND;
3085 i = set_find_table_entry_ind(tab, hash, key);
3086 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
3088 error_p = i == UNDEFINED_ENTRY_IND;
3091 if (error_p && check_p) {
3093 retval = (*func)(0, arg, 1);
3096 curr_entry_ptr = &entries[i];
3109 st_data_t key = curr_entry_ptr->key;
3113 bin = set_find_entry(tab, hash, key);
3114 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
3116 if (bin == UNDEFINED_ENTRY_IND)
3120 bin_ind = set_find_table_bin_ind(tab, hash, key);
3121 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
3123 if (bin_ind == UNDEFINED_BIN_IND)
3125 bin = get_bin(tab->bins, set_get_size_ind(tab), bin_ind) - ENTRY_BASE;
3126 MARK_SET_BIN_DELETED(tab, bin_ind);
3128 curr_entry_ptr = &entries[bin];
3129 MARK_ENTRY_DELETED(curr_entry_ptr);
3131 set_update_range_for_deleted(tab, bin);
3140set_foreach_with_replace(
set_table *tab, set_foreach_check_callback_func *func, set_update_callback_func *replace, st_data_t arg)
3142 return set_general_foreach(tab, func, replace, arg, TRUE);
3146 set_foreach_callback_func *func;
3151set_apply_functor(st_data_t k, st_data_t d,
int _)
3153 const struct set_functor *f = (
void *)d;
3154 return f->func(k, f->arg);
3158set_foreach(
set_table *tab, set_foreach_callback_func *func, st_data_t arg)
3160 const struct set_functor f = { func, arg };
3161 return set_general_foreach(tab, set_apply_functor, NULL, (st_data_t)&f, FALSE);
3166set_foreach_check(
set_table *tab, set_foreach_check_callback_func *func, st_data_t arg,
3167 st_data_t never ATTRIBUTE_UNUSED)
3169 return set_general_foreach(tab, func, NULL, arg, TRUE);
3175set_keys(
set_table *tab, st_data_t *keys, st_index_t size)
3177 st_index_t i, bound;
3178 st_data_t key, *keys_start, *keys_end;
3179 set_table_entry *curr_entry_ptr, *entries = tab->entries;
3181 bound = tab->entries_bound;
3183 keys_end = keys + size;
3184 for (i = tab->entries_start; i < bound; i++) {
3185 if (keys == keys_end)
3187 curr_entry_ptr = &entries[i];
3188 key = curr_entry_ptr->key;
3189 if (! DELETED_ENTRY_P(curr_entry_ptr))
3193 return keys - keys_start;
3199 st_index_t num = tab->num_entries;
3200 if (REBUILD_THRESHOLD * num <= set_get_allocated_entries(tab)) {
3202 set_table *new_tab = set_init_table_with_size(NULL, tab->type, 2 * num);
3203 set_rebuild_table_with(new_tab, tab);
3204 set_rebuild_move_table(new_tab, tab);
3205 set_rebuild_cleanup(tab);
static bool RB_OBJ_FROZEN(VALUE obj)
Checks if an object is frozen.
#define Qundef
Old name of RUBY_Qundef.
VALUE rb_eRuntimeError
RuntimeError exception.
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
VALUE rb_cString
String class.
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
int len
Length of the buffer.
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
VALUE type(ANYARGS)
ANYARGS-ed function type.
#define _(args)
This was a transition path from K&R to ANSI.
uintptr_t VALUE
Type that represents a Ruby object.