@@ -1129,10 +1129,11 @@ rte_hash_add_key_data(const struct rte_hash *h, const void *key, void *data)
return ret;
}
-/* Search one bucket to find the match key */
+/* Search one bucket to find the match key - uses rw lock */
static inline int32_t
-search_one_bucket(const struct rte_hash *h, const void *key, uint16_t sig,
- void **data, const struct rte_hash_bucket *bkt)
+search_one_bucket_l(const struct rte_hash *h, const void *key,
+ uint16_t sig, void **data,
+ const struct rte_hash_bucket *bkt)
{
int i;
struct rte_hash_key *k, *keys = h->key_store;
@@ -1191,8 +1192,8 @@ search_one_bucket_lf(const struct rte_hash *h, const void *key, uint16_t sig,
}
static inline int32_t
-__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
- hash_sig_t sig, void **data)
+__rte_hash_lookup_with_hash_l(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void **data)
{
uint32_t prim_bucket_idx, sec_bucket_idx;
struct rte_hash_bucket *bkt, *cur_bkt;
@@ -1207,7 +1208,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
/* Check if key is in primary location */
bkt = &h->buckets[prim_bucket_idx];
- ret = search_one_bucket(h, key, short_sig, data, bkt);
+ ret = search_one_bucket_l(h, key, short_sig, data, bkt);
if (ret != -1) {
__hash_rw_reader_unlock(h);
return ret;
@@ -1217,7 +1218,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
/* Check if key is in secondary location */
FOR_EACH_BUCKET(cur_bkt, bkt) {
- ret = search_one_bucket(h, key, short_sig,
+ ret = search_one_bucket_l(h, key, short_sig,
data, cur_bkt);
if (ret != -1) {
__hash_rw_reader_unlock(h);
@@ -1291,6 +1292,16 @@ __rte_hash_lookup_with_hash_lf(const struct rte_hash *h, const void *key,
return -ENOENT;
}
+static inline int32_t
+__rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
+ hash_sig_t sig, void **data)
+{
+ if (h->readwrite_concur_lf_support)
+ return __rte_hash_lookup_with_hash_lf(h, key, sig, data);
+ else
+ return __rte_hash_lookup_with_hash_l(h, key, sig, data);
+}
+
int32_t
rte_hash_lookup_with_hash(const struct rte_hash *h,
const void *key, hash_sig_t sig)
@@ -1592,7 +1603,7 @@ compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches,
#define PREFETCH_OFFSET 4
static inline void
-__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+__rte_hash_lookup_bulk_l(const struct rte_hash *h, const void **keys,
int32_t num_keys, int32_t *positions,
uint64_t *hit_mask, void *data[])
{
@@ -1762,10 +1773,10 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
next_bkt = secondary_bkt[i]->next;
FOR_EACH_BUCKET(cur_bkt, next_bkt) {
if (data != NULL)
- ret = search_one_bucket(h, keys[i],
+ ret = search_one_bucket_l(h, keys[i],
sig[i], &data[i], cur_bkt);
else
- ret = search_one_bucket(h, keys[i],
+ ret = search_one_bucket_l(h, keys[i],
sig[i], NULL, cur_bkt);
if (ret != -1) {
positions[i] = ret;
@@ -2005,6 +2016,19 @@ __rte_hash_lookup_bulk_lf(const struct rte_hash *h, const void **keys,
*hit_mask = hits;
}
+static inline void
+__rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
+ int32_t num_keys, int32_t *positions,
+ uint64_t *hit_mask, void *data[])
+{
+ if (h->readwrite_concur_lf_support)
+ return __rte_hash_lookup_bulk_lf(h, keys, num_keys,
+ positions, hit_mask, data);
+ else
+ return __rte_hash_lookup_bulk_l(h, keys, num_keys,
+ positions, hit_mask, data);
+}
+
int
rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,
uint32_t num_keys, int32_t *positions)
The lock-free algorithm has caused significant lookup performance regression for certain use cases. The regression is attributed to the use of non-relaxed memory orderings. 2 versions of the lookup functions are created. One that uses the RW lock and the one that is lock-free. This restores the performance regression caused for use cases that used RW lock version of the lookup function. Fixes: e605a1d36 ("hash: add lock-free r/w concurrency") Cc: honnappa.nagarahalli@arm.com Suggested-by: Jerin Jacob <jerin.jacob@caviumnetworks.com> Signed-off-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com> Reviewed-by: Ola Liljedahl <ola.liljedahl@arm.com> Reviewed-by: Gavin Hu <gavin.hu@arm.com> --- lib/librte_hash/rte_cuckoo_hash.c | 44 ++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) -- 2.17.1