diff options
author | bugaevskiy <bugaevskiy@yandex-team.ru> | 2022-02-10 16:46:17 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:17 +0300 |
commit | a6e0145a095c7bb3770d6e07aee301de5c73f96e (patch) | |
tree | 1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /contrib/libs/jemalloc/src/ckh.c | |
parent | c7f68570483e493f4ddaf946de7b3a420ee621b0 (diff) | |
download | ydb-a6e0145a095c7bb3770d6e07aee301de5c73f96e.tar.gz |
Restoring authorship annotation for <bugaevskiy@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/jemalloc/src/ckh.c')
-rw-r--r-- | contrib/libs/jemalloc/src/ckh.c | 264 |
1 files changed, 132 insertions, 132 deletions
diff --git a/contrib/libs/jemalloc/src/ckh.c b/contrib/libs/jemalloc/src/ckh.c index 3c48134b75..1bf6df5a11 100644 --- a/contrib/libs/jemalloc/src/ckh.c +++ b/contrib/libs/jemalloc/src/ckh.c @@ -34,24 +34,24 @@ * respectively. * ******************************************************************************/ -#define JEMALLOC_CKH_C_ -#include "jemalloc/internal/jemalloc_preamble.h" - -#include "jemalloc/internal/ckh.h" - -#include "jemalloc/internal/jemalloc_internal_includes.h" - -#include "jemalloc/internal/assert.h" -#include "jemalloc/internal/hash.h" -#include "jemalloc/internal/malloc_io.h" -#include "jemalloc/internal/prng.h" -#include "jemalloc/internal/util.h" - +#define JEMALLOC_CKH_C_ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/ckh.h" + +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/hash.h" +#include "jemalloc/internal/malloc_io.h" +#include "jemalloc/internal/prng.h" +#include "jemalloc/internal/util.h" + /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static bool ckh_grow(tsd_t *tsd, ckh_t *ckh); -static void ckh_shrink(tsd_t *tsd, ckh_t *ckh); +static bool ckh_grow(tsd_t *tsd, ckh_t *ckh); +static void ckh_shrink(tsd_t *tsd, ckh_t *ckh); /******************************************************************************/ @@ -59,26 +59,26 @@ static void ckh_shrink(tsd_t *tsd, ckh_t *ckh); * Search bucket for key and return the cell number if found; SIZE_T_MAX * otherwise. */ -static size_t -ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) { +static size_t +ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) { ckhc_t *cell; unsigned i; for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; - if (cell->key != NULL && ckh->keycomp(key, cell->key)) { - return (bucket << LG_CKH_BUCKET_CELLS) + i; - } + if (cell->key != NULL && ckh->keycomp(key, cell->key)) { + return (bucket << LG_CKH_BUCKET_CELLS) + i; + } } - return SIZE_T_MAX; + return SIZE_T_MAX; } /* * Search table for key and return cell number if found; SIZE_T_MAX otherwise. */ -static size_t -ckh_isearch(ckh_t *ckh, const void *key) { +static size_t +ckh_isearch(ckh_t *ckh, const void *key) { size_t hashes[2], bucket, cell; assert(ckh != NULL); @@ -88,19 +88,19 @@ ckh_isearch(ckh_t *ckh, const void *key) { /* Search primary bucket. */ bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1); cell = ckh_bucket_search(ckh, bucket, key); - if (cell != SIZE_T_MAX) { - return cell; - } + if (cell != SIZE_T_MAX) { + return cell; + } /* Search secondary bucket. */ bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); cell = ckh_bucket_search(ckh, bucket, key); - return cell; + return cell; } -static bool +static bool ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, - const void *data) { + const void *data) { ckhc_t *cell; unsigned offset, i; @@ -108,8 +108,8 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, * Cycle through the cells in the bucket, starting at a random position. * The randomness avoids worst-case search overhead as buckets fill up. */ - offset = (unsigned)prng_lg_range_u64(&ckh->prng_state, - LG_CKH_BUCKET_CELLS); + offset = (unsigned)prng_lg_range_u64(&ckh->prng_state, + LG_CKH_BUCKET_CELLS); for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; @@ -117,11 +117,11 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, cell->key = key; cell->data = data; ckh->count++; - return false; + return false; } } - return true; + return true; } /* @@ -130,9 +130,9 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, * eviction/relocation procedure until either success or detection of an * eviction/relocation bucket cycle. */ -static bool +static bool ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, - void const **argdata) { + void const **argdata) { const void *key, *data, *tkey, *tdata; ckhc_t *cell; size_t hashes[2], bucket, tbucket; @@ -150,8 +150,8 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, * were an item for which both hashes indicated the same * bucket. */ - i = (unsigned)prng_lg_range_u64(&ckh->prng_state, - LG_CKH_BUCKET_CELLS); + i = (unsigned)prng_lg_range_u64(&ckh->prng_state, + LG_CKH_BUCKET_CELLS); cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; assert(cell->key != NULL); @@ -191,18 +191,18 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, if (tbucket == argbucket) { *argkey = key; *argdata = data; - return true; + return true; } bucket = tbucket; - if (!ckh_try_bucket_insert(ckh, bucket, key, data)) { - return false; - } + if (!ckh_try_bucket_insert(ckh, bucket, key, data)) { + return false; + } } } -static bool -ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) { +static bool +ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) { size_t hashes[2], bucket; const void *key = *argkey; const void *data = *argdata; @@ -211,28 +211,28 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) { /* Try to insert in primary bucket. */ bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1); - if (!ckh_try_bucket_insert(ckh, bucket, key, data)) { - return false; - } + if (!ckh_try_bucket_insert(ckh, bucket, key, data)) { + return false; + } /* Try to insert in secondary bucket. */ bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); - if (!ckh_try_bucket_insert(ckh, bucket, key, data)) { - return false; - } + if (!ckh_try_bucket_insert(ckh, bucket, key, data)) { + return false; + } /* * Try to find a place for this item via iterative eviction/relocation. */ - return ckh_evict_reloc_insert(ckh, bucket, argkey, argdata); + return ckh_evict_reloc_insert(ckh, bucket, argkey, argdata); } /* * Try to rebuild the hash table from scratch by inserting all items from the * old table into the new. */ -static bool -ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) { +static bool +ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) { size_t count, i, nins; const void *key, *data; @@ -244,20 +244,20 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) { data = aTab[i].data; if (ckh_try_insert(ckh, &key, &data)) { ckh->count = count; - return true; + return true; } nins++; } } - return false; + return false; } static bool -ckh_grow(tsd_t *tsd, ckh_t *ckh) { +ckh_grow(tsd_t *tsd, ckh_t *ckh) { bool ret; ckhc_t *tab, *ttab; - unsigned lg_prevbuckets, lg_curcells; + unsigned lg_prevbuckets, lg_curcells; #ifdef CKH_COUNT ckh->ngrows++; @@ -274,14 +274,14 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) { size_t usize; lg_curcells++; - usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 - || usize > SC_LARGE_MAXCLASS)) { + usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } - tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, - true, NULL, true, arena_ichoose(tsd, NULL)); + tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, + true, NULL, true, arena_ichoose(tsd, NULL)); if (tab == NULL) { ret = true; goto label_return; @@ -292,27 +292,27 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) { tab = ttab; ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; - if (!ckh_rebuild(ckh, tab)) { - idalloctm(tsd_tsdn(tsd), tab, NULL, NULL, true, true); + if (!ckh_rebuild(ckh, tab)) { + idalloctm(tsd_tsdn(tsd), tab, NULL, NULL, true, true); break; } /* Rebuilding failed, so back out partially rebuilt table. */ - idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true); + idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true); ckh->tab = tab; ckh->lg_curbuckets = lg_prevbuckets; } ret = false; label_return: - return ret; + return ret; } static void -ckh_shrink(tsd_t *tsd, ckh_t *ckh) { +ckh_shrink(tsd_t *tsd, ckh_t *ckh) { ckhc_t *tab, *ttab; - size_t usize; - unsigned lg_prevbuckets, lg_curcells; + size_t usize; + unsigned lg_prevbuckets, lg_curcells; /* * It is possible (though unlikely, given well behaved hashes) that the @@ -320,12 +320,12 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { */ lg_prevbuckets = ckh->lg_curbuckets; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; - usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { + usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return; - } - tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL, - true, arena_ichoose(tsd, NULL)); + } + tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL, + true, arena_ichoose(tsd, NULL)); if (tab == NULL) { /* * An OOM error isn't worth propagating, since it doesn't @@ -339,8 +339,8 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { tab = ttab; ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; - if (!ckh_rebuild(ckh, tab)) { - idalloctm(tsd_tsdn(tsd), tab, NULL, NULL, true, true); + if (!ckh_rebuild(ckh, tab)) { + idalloctm(tsd_tsdn(tsd), tab, NULL, NULL, true, true); #ifdef CKH_COUNT ckh->nshrinks++; #endif @@ -348,7 +348,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { } /* Rebuilding failed, so back out partially rebuilt table. */ - idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true); + idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true); ckh->tab = tab; ckh->lg_curbuckets = lg_prevbuckets; #ifdef CKH_COUNT @@ -357,8 +357,8 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { } bool -ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, - ckh_keycomp_t *keycomp) { +ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, + ckh_keycomp_t *keycomp) { bool ret; size_t mincells, usize; unsigned lg_mincells; @@ -378,31 +378,31 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh->count = 0; /* - * Find the minimum power of 2 that is large enough to fit minitems + * Find the minimum power of 2 that is large enough to fit minitems * entries. We are using (2+,2) cuckoo hashing, which has an expected * maximum load factor of at least ~0.86, so 0.75 is a conservative load - * factor that will typically allow mincells items to fit without ever + * factor that will typically allow mincells items to fit without ever * growing the table. */ assert(LG_CKH_BUCKET_CELLS > 0); mincells = ((minitems + (3 - (minitems % 3))) / 3) << 2; for (lg_mincells = LG_CKH_BUCKET_CELLS; (ZU(1) << lg_mincells) < mincells; - lg_mincells++) { - /* Do nothing. */ - } + lg_mincells++) { + /* Do nothing. */ + } ckh->lg_minbuckets = lg_mincells - LG_CKH_BUCKET_CELLS; ckh->lg_curbuckets = lg_mincells - LG_CKH_BUCKET_CELLS; ckh->hash = hash; ckh->keycomp = keycomp; - usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); - if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { + usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } - ckh->tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, - NULL, true, arena_ichoose(tsd, NULL)); + ckh->tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, + NULL, true, arena_ichoose(tsd, NULL)); if (ckh->tab == NULL) { ret = true; goto label_return; @@ -410,18 +410,18 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ret = false; label_return: - return ret; + return ret; } void -ckh_delete(tsd_t *tsd, ckh_t *ckh) { +ckh_delete(tsd_t *tsd, ckh_t *ckh) { assert(ckh != NULL); #ifdef CKH_VERBOSE malloc_printf( - "%s(%p): ngrows: %"FMTu64", nshrinks: %"FMTu64"," - " nshrinkfails: %"FMTu64", ninserts: %"FMTu64"," - " nrelocs: %"FMTu64"\n", __func__, ckh, + "%s(%p): ngrows: %"FMTu64", nshrinks: %"FMTu64"," + " nshrinkfails: %"FMTu64", ninserts: %"FMTu64"," + " nrelocs: %"FMTu64"\n", __func__, ckh, (unsigned long long)ckh->ngrows, (unsigned long long)ckh->nshrinks, (unsigned long long)ckh->nshrinkfails, @@ -429,42 +429,42 @@ ckh_delete(tsd_t *tsd, ckh_t *ckh) { (unsigned long long)ckh->nrelocs); #endif - idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true); - if (config_debug) { - memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t)); - } + idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, NULL, true, true); + if (config_debug) { + memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t)); + } } size_t -ckh_count(ckh_t *ckh) { +ckh_count(ckh_t *ckh) { assert(ckh != NULL); - return ckh->count; + return ckh->count; } bool -ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) { +ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) { size_t i, ncells; for (i = *tabind, ncells = (ZU(1) << (ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS)); i < ncells; i++) { if (ckh->tab[i].key != NULL) { - if (key != NULL) { + if (key != NULL) { *key = (void *)ckh->tab[i].key; - } - if (data != NULL) { + } + if (data != NULL) { *data = (void *)ckh->tab[i].data; - } + } *tabind = i + 1; - return false; + return false; } } - return true; + return true; } bool -ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data) { +ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data) { bool ret; assert(ckh != NULL); @@ -475,7 +475,7 @@ ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data) { #endif while (ckh_try_insert(ckh, &key, &data)) { - if (ckh_grow(tsd, ckh)) { + if (ckh_grow(tsd, ckh)) { ret = true; goto label_return; } @@ -483,24 +483,24 @@ ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data) { ret = false; label_return: - return ret; + return ret; } bool -ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, - void **data) { +ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, + void **data) { size_t cell; assert(ckh != NULL); cell = ckh_isearch(ckh, searchkey); if (cell != SIZE_T_MAX) { - if (key != NULL) { + if (key != NULL) { *key = (void *)ckh->tab[cell].key; - } - if (data != NULL) { + } + if (data != NULL) { *data = (void *)ckh->tab[cell].data; - } + } ckh->tab[cell].key = NULL; ckh->tab[cell].data = NULL; /* Not necessary. */ @@ -510,50 +510,50 @@ ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, + LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets > ckh->lg_minbuckets) { /* Ignore error due to OOM. */ - ckh_shrink(tsd, ckh); + ckh_shrink(tsd, ckh); } - return false; + return false; } - return true; + return true; } bool -ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) { +ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) { size_t cell; assert(ckh != NULL); cell = ckh_isearch(ckh, searchkey); if (cell != SIZE_T_MAX) { - if (key != NULL) { + if (key != NULL) { *key = (void *)ckh->tab[cell].key; - } - if (data != NULL) { + } + if (data != NULL) { *data = (void *)ckh->tab[cell].data; - } - return false; + } + return false; } - return true; + return true; } void -ckh_string_hash(const void *key, size_t r_hash[2]) { +ckh_string_hash(const void *key, size_t r_hash[2]) { hash(key, strlen((const char *)key), 0x94122f33U, r_hash); } bool -ckh_string_keycomp(const void *k1, const void *k2) { - assert(k1 != NULL); - assert(k2 != NULL); +ckh_string_keycomp(const void *k1, const void *k2) { + assert(k1 != NULL); + assert(k2 != NULL); - return !strcmp((char *)k1, (char *)k2); + return !strcmp((char *)k1, (char *)k2); } void -ckh_pointer_hash(const void *key, size_t r_hash[2]) { +ckh_pointer_hash(const void *key, size_t r_hash[2]) { union { const void *v; size_t i; @@ -565,6 +565,6 @@ ckh_pointer_hash(const void *key, size_t r_hash[2]) { } bool -ckh_pointer_keycomp(const void *k1, const void *k2) { - return (k1 == k2); +ckh_pointer_keycomp(const void *k1, const void *k2) { + return (k1 == k2); } |