Merge branch 'rhashtable-inlined-interface' (ebd6af09) · Commits · e / devices / android_kernel_sony_msm8998

include/linux/rhashtable.h

+391 −17

Original line number	Original line	Diff line number	Diff line
	/*		/*
	* Resizable, Scalable, Concurrent Hash Table		* Resizable, Scalable, Concurrent Hash Table
	*		*
			* Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au>
	* Copyright (c) 2014 Thomas Graf <tgraf@suug.ch>		* Copyright (c) 2014 Thomas Graf <tgraf@suug.ch>
	* Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>		* Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
	*		*
	* Based on the following paper by Josh Triplett, Paul E. McKenney
	* and Jonathan Walpole:
	* https://www.usenix.org/legacy/event/atc11/tech/final_files/Triplett.pdf
	*
	* Code partially derived from nft_hash		* Code partially derived from nft_hash
			* Rewritten with rehash code from br_multicast plus single list
			* pointer as suggested by Josh Triplett
	*		*
	* This program is free software; you can redistribute it and/or modify		* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License version 2 as		* it under the terms of the GNU General Public License version 2 as
	@@ -22,6 +21,7 @@
	#include <linux/list_nulls.h>		#include <linux/list_nulls.h>
	#include <linux/workqueue.h>		#include <linux/workqueue.h>
	#include <linux/mutex.h>		#include <linux/mutex.h>
			#include <linux/rcupdate.h>

	/*		/*
	* The end of the chain is marked with a special nulls marks which has		* The end of the chain is marked with a special nulls marks which has
	@@ -42,6 +42,9 @@
	#define RHT_HASH_BITS 27		#define RHT_HASH_BITS 27
	#define RHT_BASE_SHIFT RHT_HASH_BITS		#define RHT_BASE_SHIFT RHT_HASH_BITS

			/* Base bits plus 1 bit for nulls marker */
			#define RHT_HASH_RESERVED_SPACE (RHT_BASE_BITS + 1)

	struct rhash_head {		struct rhash_head {
	struct rhash_head __rcu *next;		struct rhash_head __rcu *next;
	};		};
	@@ -72,8 +75,20 @@ struct bucket_table {
	struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp;		struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp;
	};		};

			/**
			* struct rhashtable_compare_arg - Key for the function rhashtable_compare
			* @ht: Hash table
			* @key: Key to compare against
			*/
			struct rhashtable_compare_arg {
			struct rhashtable *ht;
			const void *key;
			};

	typedef u32 (rht_hashfn_t)(const void data, u32 len, u32 seed);		typedef u32 (rht_hashfn_t)(const void data, u32 len, u32 seed);
	typedef u32 (rht_obj_hashfn_t)(const void data, u32 seed);		typedef u32 (rht_obj_hashfn_t)(const void data, u32 seed);
			typedef int (rht_obj_cmpfn_t)(struct rhashtable_compare_arg arg,
			const void *obj);

	struct rhashtable;		struct rhashtable;

	@@ -89,6 +104,7 @@ struct rhashtable;
	* @locks_mul: Number of bucket locks to allocate per cpu (default: 128)		* @locks_mul: Number of bucket locks to allocate per cpu (default: 128)
	* @hashfn: Function to hash key		* @hashfn: Function to hash key
	* @obj_hashfn: Function to hash object		* @obj_hashfn: Function to hash object
			* @obj_cmpfn: Function to compare key with object
	*/		*/
	struct rhashtable_params {		struct rhashtable_params {
	size_t nelem_hint;		size_t nelem_hint;
	@@ -101,6 +117,7 @@ struct rhashtable_params {
	size_t locks_mul;		size_t locks_mul;
	rht_hashfn_t hashfn;		rht_hashfn_t hashfn;
	rht_obj_hashfn_t obj_hashfn;		rht_obj_hashfn_t obj_hashfn;
			rht_obj_cmpfn_t obj_cmpfn;
	};		};

	/**		/**
	@@ -165,6 +182,83 @@ static inline unsigned long rht_get_nulls_value(const struct rhash_head *ptr)
	return ((unsigned long) ptr) >> 1;		return ((unsigned long) ptr) >> 1;
	}		}

			static inline void rht_obj(const struct rhashtable ht,
			const struct rhash_head *he)
			{
			return (char *)he - ht->p.head_offset;
			}

			static inline unsigned int rht_bucket_index(const struct bucket_table *tbl,
			unsigned int hash)
			{
			return (hash >> RHT_HASH_RESERVED_SPACE) & (tbl->size - 1);
			}

			static inline unsigned int rht_key_hashfn(
			struct rhashtable ht, const struct bucket_table tbl,
			const void *key, const struct rhashtable_params params)
			{
			return rht_bucket_index(tbl, params.hashfn(key, params.key_len ?:
			ht->p.key_len,
			tbl->hash_rnd));
			}

			static inline unsigned int rht_head_hashfn(
			struct rhashtable ht, const struct bucket_table tbl,
			const struct rhash_head *he, const struct rhashtable_params params)
			{
			const char *ptr = rht_obj(ht, he);

			return likely(params.obj_hashfn) ?
			rht_bucket_index(tbl, params.obj_hashfn(ptr, tbl->hash_rnd)) :
			rht_key_hashfn(ht, tbl, ptr + params.key_offset, params);
			}

			/**
			* rht_grow_above_75 - returns true if nelems > 0.75 * table-size
			* @ht: hash table
			* @tbl: current table
			*/
			static inline bool rht_grow_above_75(const struct rhashtable *ht,
			const struct bucket_table *tbl)
			{
			/* Expand table when exceeding 75% load */
			return atomic_read(&ht->nelems) > (tbl->size / 4 * 3) &&
			(!ht->p.max_size \|\| tbl->size < ht->p.max_size);
			}

			/**
			* rht_shrink_below_30 - returns true if nelems < 0.3 * table-size
			* @ht: hash table
			* @tbl: current table
			*/
			static inline bool rht_shrink_below_30(const struct rhashtable *ht,
			const struct bucket_table *tbl)
			{
			/* Shrink table beneath 30% load */
			return atomic_read(&ht->nelems) < (tbl->size * 3 / 10) &&
			tbl->size > ht->p.min_size;
			}

			/* The bucket lock is selected based on the hash and protects mutations
			* on a group of hash buckets.
			*
			* A maximum of tbl->size/2 bucket locks is allocated. This ensures that
			* a single lock always covers both buckets which may both contains
			* entries which link to the same bucket of the old table during resizing.
			* This allows to simplify the locking as locking the bucket in both
			* tables during resize always guarantee protection.
			*
			* IMPORTANT: When holding the bucket lock of both the old and new table
			* during expansions and shrinking, the old bucket lock must always be
			* acquired first.
			*/
			static inline spinlock_t rht_bucket_lock(const struct bucket_table tbl,
			unsigned int hash)
			{
			return &tbl->locks[hash & tbl->locks_mask];
			}

	#ifdef CONFIG_PROVE_LOCKING		#ifdef CONFIG_PROVE_LOCKING
	int lockdep_rht_mutex_is_held(struct rhashtable *ht);		int lockdep_rht_mutex_is_held(struct rhashtable *ht);
	int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash);		int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash);
	@@ -181,24 +275,16 @@ static inline int lockdep_rht_bucket_is_held(const struct bucket_table *tbl,
	}		}
	#endif /* CONFIG_PROVE_LOCKING */		#endif /* CONFIG_PROVE_LOCKING */

	int rhashtable_init(struct rhashtable ht, struct rhashtable_params params);		int rhashtable_init(struct rhashtable *ht,
			const struct rhashtable_params *params);

	void rhashtable_insert(struct rhashtable ht, struct rhash_head node);		int rhashtable_insert_slow(struct rhashtable ht, const void key,
	bool rhashtable_remove(struct rhashtable ht, struct rhash_head node);		struct rhash_head *obj,
			struct bucket_table *old_tbl);

	int rhashtable_expand(struct rhashtable *ht);		int rhashtable_expand(struct rhashtable *ht);
	int rhashtable_shrink(struct rhashtable *ht);		int rhashtable_shrink(struct rhashtable *ht);

	void rhashtable_lookup(struct rhashtable ht, const void *key);
	void rhashtable_lookup_compare(struct rhashtable ht, const void *key,
	bool (compare)(void , void ), void arg);

	bool rhashtable_lookup_insert(struct rhashtable ht, struct rhash_head obj);
	bool rhashtable_lookup_compare_insert(struct rhashtable *ht,
	struct rhash_head *obj,
	bool (compare)(void , void *),
	void *arg);

	int rhashtable_walk_init(struct rhashtable ht, struct rhashtable_iter iter);		int rhashtable_walk_init(struct rhashtable ht, struct rhashtable_iter iter);
	void rhashtable_walk_exit(struct rhashtable_iter *iter);		void rhashtable_walk_exit(struct rhashtable_iter *iter);
	int rhashtable_walk_start(struct rhashtable_iter *iter) __acquires(RCU);		int rhashtable_walk_start(struct rhashtable_iter *iter) __acquires(RCU);
	@@ -355,4 +441,292 @@ void rhashtable_destroy(struct rhashtable *ht);
	rht_for_each_entry_rcu_continue(tpos, pos, (tbl)->buckets[hash],\		rht_for_each_entry_rcu_continue(tpos, pos, (tbl)->buckets[hash],\
	tbl, hash, member)		tbl, hash, member)

			static inline int rhashtable_compare(struct rhashtable_compare_arg *arg,
			const void *obj)
			{
			struct rhashtable *ht = arg->ht;
			const char *ptr = obj;

			return memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len);
			}

			/**
			* rhashtable_lookup_fast - search hash table, inlined version
			* @ht: hash table
			* @key: the pointer to the key
			* @params: hash table parameters
			*
			* Computes the hash value for the key and traverses the bucket chain looking
			* for a entry with an identical key. The first matching entry is returned.
			*
			* Returns the first entry on which the compare function returned true.
			*/
			static inline void *rhashtable_lookup_fast(
			struct rhashtable ht, const void key,
			const struct rhashtable_params params)
			{
			struct rhashtable_compare_arg arg = {
			.ht = ht,
			.key = key,
			};
			const struct bucket_table *tbl;
			struct rhash_head *he;
			unsigned hash;

			rcu_read_lock();

			tbl = rht_dereference_rcu(ht->tbl, ht);
			restart:
			hash = rht_key_hashfn(ht, tbl, key, params);
			rht_for_each_rcu(he, tbl, hash) {
			if (params.obj_cmpfn ?
			params.obj_cmpfn(&arg, rht_obj(ht, he)) :
			rhashtable_compare(&arg, rht_obj(ht, he)))
			continue;
			rcu_read_unlock();
			return rht_obj(ht, he);
			}

			/* Ensure we see any new tables. */
			smp_rmb();

			tbl = rht_dereference_rcu(tbl->future_tbl, ht);
			if (unlikely(tbl))
			goto restart;
			rcu_read_unlock();

			return NULL;
			}

			static inline int __rhashtable_insert_fast(
			struct rhashtable ht, const void key, struct rhash_head *obj,
			const struct rhashtable_params params)
			{
			struct rhashtable_compare_arg arg = {
			.ht = ht,
			.key = key,
			};
			int err = -EEXIST;
			struct bucket_table tbl, new_tbl;
			struct rhash_head *head;
			spinlock_t *lock;
			unsigned hash;

			rcu_read_lock();

			tbl = rht_dereference_rcu(ht->tbl, ht);
			hash = rht_head_hashfn(ht, tbl, obj, params);
			lock = rht_bucket_lock(tbl, hash);

			spin_lock_bh(lock);

			/* Because we have already taken the bucket lock in tbl,
			* if we find that future_tbl is not yet visible then
			* that guarantees all other insertions of the same entry
			* will also grab the bucket lock in tbl because until
			* the rehash completes ht->tbl won't be changed.
			*/
			new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);
			if (unlikely(new_tbl)) {
			err = rhashtable_insert_slow(ht, key, obj, new_tbl);
			goto out;
			}

			if (!key)
			goto skip_lookup;

			rht_for_each(head, tbl, hash) {
			if (unlikely(!(params.obj_cmpfn ?
			params.obj_cmpfn(&arg, rht_obj(ht, head)) :
			rhashtable_compare(&arg, rht_obj(ht, head)))))
			goto out;
			}

			skip_lookup:
			err = 0;

			head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);

			RCU_INIT_POINTER(obj->next, head);

			rcu_assign_pointer(tbl->buckets[hash], obj);

			atomic_inc(&ht->nelems);
			if (rht_grow_above_75(ht, tbl))
			schedule_work(&ht->run_work);

			out:
			spin_unlock_bh(lock);
			rcu_read_unlock();

			return err;
			}

			/**
			* rhashtable_insert_fast - insert object into hash table
			* @ht: hash table
			* @obj: pointer to hash head inside object
			* @params: hash table parameters
			*
			* Will take a per bucket spinlock to protect against mutual mutations
			* on the same bucket. Multiple insertions may occur in parallel unless
			* they map to the same bucket lock.
			*
			* It is safe to call this function from atomic context.
			*
			* Will trigger an automatic deferred table resizing if the size grows
			* beyond the watermark indicated by grow_decision() which can be passed
			* to rhashtable_init().
			*/
			static inline int rhashtable_insert_fast(
			struct rhashtable ht, struct rhash_head obj,
			const struct rhashtable_params params)
			{
			return __rhashtable_insert_fast(ht, NULL, obj, params);
			}

			/**
			* rhashtable_lookup_insert_fast - lookup and insert object into hash table
			* @ht: hash table
			* @obj: pointer to hash head inside object
			* @params: hash table parameters
			*
			* Locks down the bucket chain in both the old and new table if a resize
			* is in progress to ensure that writers can't remove from the old table
			* and can't insert to the new table during the atomic operation of search
			* and insertion. Searches for duplicates in both the old and new table if
			* a resize is in progress.
			*
			* This lookup function may only be used for fixed key hash table (key_len
			* parameter set). It will BUG() if used inappropriately.
			*
			* It is safe to call this function from atomic context.
			*
			* Will trigger an automatic deferred table resizing if the size grows
			* beyond the watermark indicated by grow_decision() which can be passed
			* to rhashtable_init().
			*/
			static inline int rhashtable_lookup_insert_fast(
			struct rhashtable ht, struct rhash_head obj,
			const struct rhashtable_params params)
			{
			const char *key = rht_obj(ht, obj);

			BUG_ON(ht->p.obj_hashfn);

			return __rhashtable_insert_fast(ht, key + ht->p.key_offset, obj,
			params);
			}

			/**
			* rhashtable_lookup_insert_key - search and insert object to hash table
			* with explicit key
			* @ht: hash table
			* @key: key
			* @obj: pointer to hash head inside object
			* @params: hash table parameters
			*
			* Locks down the bucket chain in both the old and new table if a resize
			* is in progress to ensure that writers can't remove from the old table
			* and can't insert to the new table during the atomic operation of search
			* and insertion. Searches for duplicates in both the old and new table if
			* a resize is in progress.
			*
			* Lookups may occur in parallel with hashtable mutations and resizing.
			*
			* Will trigger an automatic deferred table resizing if the size grows
			* beyond the watermark indicated by grow_decision() which can be passed
			* to rhashtable_init().
			*
			* Returns zero on success.
			*/
			static inline int rhashtable_lookup_insert_key(
			struct rhashtable ht, const void key, struct rhash_head *obj,
			const struct rhashtable_params params)
			{
			BUG_ON(!ht->p.obj_hashfn \|\| !key);

			return __rhashtable_insert_fast(ht, key, obj, params);
			}

			static inline int __rhashtable_remove_fast(
			struct rhashtable ht, struct bucket_table tbl,
			struct rhash_head *obj, const struct rhashtable_params params)
			{
			struct rhash_head __rcu **pprev;
			struct rhash_head *he;
			spinlock_t * lock;
			unsigned hash;
			int err = -ENOENT;

			hash = rht_head_hashfn(ht, tbl, obj, params);
			lock = rht_bucket_lock(tbl, hash);

			spin_lock_bh(lock);

			pprev = &tbl->buckets[hash];
			rht_for_each(he, tbl, hash) {
			if (he != obj) {
			pprev = &he->next;
			continue;
			}

			rcu_assign_pointer(*pprev, obj->next);
			err = 0;
			break;
			}

			spin_unlock_bh(lock);

			return err;
			}

			/**
			* rhashtable_remove_fast - remove object from hash table
			* @ht: hash table
			* @obj: pointer to hash head inside object
			* @params: hash table parameters
			*
			* Since the hash chain is single linked, the removal operation needs to
			* walk the bucket chain upon removal. The removal operation is thus
			* considerable slow if the hash table is not correctly sized.
			*
			* Will automatically shrink the table via rhashtable_expand() if the
			* shrink_decision function specified at rhashtable_init() returns true.
			*
			* Returns zero on success, -ENOENT if the entry could not be found.
			*/
			static inline int rhashtable_remove_fast(
			struct rhashtable ht, struct rhash_head obj,
			const struct rhashtable_params params)
			{
			struct bucket_table *tbl;
			int err;

			rcu_read_lock();

			tbl = rht_dereference_rcu(ht->tbl, ht);

			/* Because we have already taken (and released) the bucket
			* lock in old_tbl, if we find that future_tbl is not yet
			* visible then that guarantees the entry to still be in
			* the old tbl if it exists.
			*/
			while ((err = __rhashtable_remove_fast(ht, tbl, obj, params)) &&
			(tbl = rht_dereference_rcu(tbl->future_tbl, ht)))
			;

			if (err)
			goto out;

			atomic_dec(&ht->nelems);
			if (rht_shrink_below_30(ht, tbl))
			schedule_work(&ht->run_work);

			out:
			rcu_read_unlock();

			return err;
			}

	#endif /* _LINUX_RHASHTABLE_H */		#endif /* _LINUX_RHASHTABLE_H */

lib/rhashtable.c

+25 −371

File changed.

Preview size limit exceeded, changes collapsed.

lib/test_rhashtable.c

+19 −14

Original line number	Original line	Diff line number	Diff line
	@@ -38,6 +38,16 @@ struct test_obj {
	struct rhash_head node;		struct rhash_head node;
	};		};

			static const struct rhashtable_params test_rht_params = {
			.nelem_hint = TEST_HT_SIZE,
			.head_offset = offsetof(struct test_obj, node),
			.key_offset = offsetof(struct test_obj, value),
			.key_len = sizeof(int),
			.hashfn = jhash,
			.max_size = 2, /* we expand/shrink manually here */
			.nulls_base = (3U << RHT_BASE_SHIFT),
			};

	static int __init test_rht_lookup(struct rhashtable *ht)		static int __init test_rht_lookup(struct rhashtable *ht)
	{		{
	unsigned int i;		unsigned int i;
	@@ -47,7 +57,7 @@ static int __init test_rht_lookup(struct rhashtable *ht)
	bool expected = !(i % 2);		bool expected = !(i % 2);
	u32 key = i;		u32 key = i;

	obj = rhashtable_lookup(ht, &key);		obj = rhashtable_lookup_fast(ht, &key, test_rht_params);

	if (expected && !obj) {		if (expected && !obj) {
	pr_warn("Test failed: Could not find key %u\n", key);		pr_warn("Test failed: Could not find key %u\n", key);
	@@ -133,7 +143,11 @@ static int __init test_rhashtable(struct rhashtable *ht)
	obj->ptr = TEST_PTR;		obj->ptr = TEST_PTR;
	obj->value = i * 2;		obj->value = i * 2;

	rhashtable_insert(ht, &obj->node);		err = rhashtable_insert_fast(ht, &obj->node, test_rht_params);
			if (err) {
			kfree(obj);
			goto error;
			}
	}		}

	rcu_read_lock();		rcu_read_lock();
	@@ -173,10 +187,10 @@ static int __init test_rhashtable(struct rhashtable *ht)
	for (i = 0; i < TEST_ENTRIES; i++) {		for (i = 0; i < TEST_ENTRIES; i++) {
	u32 key = i * 2;		u32 key = i * 2;

	obj = rhashtable_lookup(ht, &key);		obj = rhashtable_lookup_fast(ht, &key, test_rht_params);
	BUG_ON(!obj);		BUG_ON(!obj);

	rhashtable_remove(ht, &obj->node);		rhashtable_remove_fast(ht, &obj->node, test_rht_params);
	kfree(obj);		kfree(obj);
	}		}

	@@ -195,20 +209,11 @@ static struct rhashtable ht;

	static int __init test_rht_init(void)		static int __init test_rht_init(void)
	{		{
	struct rhashtable_params params = {
	.nelem_hint = TEST_HT_SIZE,
	.head_offset = offsetof(struct test_obj, node),
	.key_offset = offsetof(struct test_obj, value),
	.key_len = sizeof(int),
	.hashfn = jhash,
	.max_size = 2, /* we expand/shrink manually here */
	.nulls_base = (3U << RHT_BASE_SHIFT),
	};
	int err;		int err;

	pr_info("Running resizable hashtable tests...\n");		pr_info("Running resizable hashtable tests...\n");

	err = rhashtable_init(&ht, &params);		err = rhashtable_init(&ht, &test_rht_params);
	if (err < 0) {		if (err < 0) {
	pr_warn("Test failed: Unable to initialize hashtable: %d\n",		pr_warn("Test failed: Unable to initialize hashtable: %d\n",
	err);		err);

net/netfilter/nft_hash.c

+29 −41

Original line number	Original line	Diff line number	Diff line
	@@ -29,6 +29,8 @@ struct nft_hash_elem {
	struct nft_data data[];		struct nft_data data[];
	};		};

			static const struct rhashtable_params nft_hash_params;

	static bool nft_hash_lookup(const struct nft_set *set,		static bool nft_hash_lookup(const struct nft_set *set,
	const struct nft_data *key,		const struct nft_data *key,
	struct nft_data *data)		struct nft_data *data)
	@@ -36,7 +38,7 @@ static bool nft_hash_lookup(const struct nft_set *set,
	struct rhashtable *priv = nft_set_priv(set);		struct rhashtable *priv = nft_set_priv(set);
	const struct nft_hash_elem *he;		const struct nft_hash_elem *he;

	he = rhashtable_lookup(priv, key);		he = rhashtable_lookup_fast(priv, key, nft_hash_params);
	if (he && set->flags & NFT_SET_MAP)		if (he && set->flags & NFT_SET_MAP)
	nft_data_copy(data, he->data);		nft_data_copy(data, he->data);

	@@ -49,6 +51,7 @@ static int nft_hash_insert(const struct nft_set *set,
	struct rhashtable *priv = nft_set_priv(set);		struct rhashtable *priv = nft_set_priv(set);
	struct nft_hash_elem *he;		struct nft_hash_elem *he;
	unsigned int size;		unsigned int size;
			int err;

	if (elem->flags != 0)		if (elem->flags != 0)
	return -EINVAL;		return -EINVAL;
	@@ -65,9 +68,11 @@ static int nft_hash_insert(const struct nft_set *set,
	if (set->flags & NFT_SET_MAP)		if (set->flags & NFT_SET_MAP)
	nft_data_copy(he->data, &elem->data);		nft_data_copy(he->data, &elem->data);

	rhashtable_insert(priv, &he->node);		err = rhashtable_insert_fast(priv, &he->node, nft_hash_params);
			if (err)
			kfree(he);

	return 0;		return err;
	}		}

	static void nft_hash_elem_destroy(const struct nft_set *set,		static void nft_hash_elem_destroy(const struct nft_set *set,
	@@ -84,46 +89,26 @@ static void nft_hash_remove(const struct nft_set *set,
	{		{
	struct rhashtable *priv = nft_set_priv(set);		struct rhashtable *priv = nft_set_priv(set);

	rhashtable_remove(priv, elem->cookie);		rhashtable_remove_fast(priv, elem->cookie, nft_hash_params);
	synchronize_rcu();		synchronize_rcu();
	kfree(elem->cookie);		kfree(elem->cookie);
	}		}

	struct nft_compare_arg {
	const struct nft_set *set;
	struct nft_set_elem *elem;
	};

	static bool nft_hash_compare(void ptr, void arg)
	{
	struct nft_hash_elem *he = ptr;
	struct nft_compare_arg *x = arg;

	if (!nft_data_cmp(&he->key, &x->elem->key, x->set->klen)) {
	x->elem->cookie = he;
	x->elem->flags = 0;
	if (x->set->flags & NFT_SET_MAP)
	nft_data_copy(&x->elem->data, he->data);

	return true;
	}

	return false;
	}

	static int nft_hash_get(const struct nft_set set, struct nft_set_elem elem)		static int nft_hash_get(const struct nft_set set, struct nft_set_elem elem)
	{		{
	struct rhashtable *priv = nft_set_priv(set);		struct rhashtable *priv = nft_set_priv(set);
	struct nft_compare_arg arg = {		struct nft_hash_elem *he;
	.set = set,
	.elem = elem,
	};

	if (rhashtable_lookup_compare(priv, &elem->key,
	&nft_hash_compare, &arg))
	return 0;

			he = rhashtable_lookup_fast(priv, &elem->key, nft_hash_params);
			if (!he)
	return -ENOENT;		return -ENOENT;

			elem->cookie = he;
			elem->flags = 0;
			if (set->flags & NFT_SET_MAP)
			nft_data_copy(&elem->data, he->data);

			return 0;
	}		}

	static void nft_hash_walk(const struct nft_ctx ctx, const struct nft_set set,		static void nft_hash_walk(const struct nft_ctx ctx, const struct nft_set set,
	@@ -181,18 +166,21 @@ static unsigned int nft_hash_privsize(const struct nlattr * const nla[])
	return sizeof(struct rhashtable);		return sizeof(struct rhashtable);
	}		}

			static const struct rhashtable_params nft_hash_params = {
			.head_offset = offsetof(struct nft_hash_elem, node),
			.key_offset = offsetof(struct nft_hash_elem, key),
			.hashfn = jhash,
			};

	static int nft_hash_init(const struct nft_set *set,		static int nft_hash_init(const struct nft_set *set,
	const struct nft_set_desc *desc,		const struct nft_set_desc *desc,
	const struct nlattr * const tb[])		const struct nlattr * const tb[])
	{		{
	struct rhashtable *priv = nft_set_priv(set);		struct rhashtable *priv = nft_set_priv(set);
	struct rhashtable_params params = {		struct rhashtable_params params = nft_hash_params;
	.nelem_hint = desc->size ? : NFT_HASH_ELEMENT_HINT,
	.head_offset = offsetof(struct nft_hash_elem, node),		params.nelem_hint = desc->size ?: NFT_HASH_ELEMENT_HINT;
	.key_offset = offsetof(struct nft_hash_elem, key),		params.key_len = set->klen;
	.key_len = set->klen,
	.hashfn = jhash,
	};

	return rhashtable_init(priv, &params);		return rhashtable_init(priv, &params);
	}		}

net/netlink/af_netlink.c

+56 −32

Original line number	Original line	Diff line number	Diff line
	@@ -116,6 +116,8 @@ static ATOMIC_NOTIFIER_HEAD(netlink_chain);
	static DEFINE_SPINLOCK(netlink_tap_lock);		static DEFINE_SPINLOCK(netlink_tap_lock);
	static struct list_head netlink_tap_all __read_mostly;		static struct list_head netlink_tap_all __read_mostly;

			static const struct rhashtable_params netlink_rhashtable_params;

	static inline u32 netlink_group_mask(u32 group)		static inline u32 netlink_group_mask(u32 group)
	{		{
	return group ? 1 << (group - 1) : 0;		return group ? 1 << (group - 1) : 0;
	@@ -970,41 +972,49 @@ netlink_unlock_table(void)

	struct netlink_compare_arg		struct netlink_compare_arg
	{		{
	struct net *net;		possible_net_t pnet;
	u32 portid;		u32 portid;
			char trailer[];
	};		};

	static bool netlink_compare(void ptr, void arg)		#define netlink_compare_arg_len offsetof(struct netlink_compare_arg, trailer)

			static inline int netlink_compare(struct rhashtable_compare_arg *arg,
			const void *ptr)
	{		{
	struct netlink_compare_arg *x = arg;		const struct netlink_compare_arg *x = arg->key;
	struct sock *sk = ptr;		const struct netlink_sock *nlk = ptr;

	return nlk_sk(sk)->portid == x->portid &&		return nlk->portid != x->portid \|\|
	net_eq(sock_net(sk), x->net);		!net_eq(sock_net(&nlk->sk), read_pnet(&x->pnet));
			}

			static void netlink_compare_arg_init(struct netlink_compare_arg *arg,
			struct net *net, u32 portid)
			{
			memset(arg, 0, sizeof(*arg));
			write_pnet(&arg->pnet, net);
			arg->portid = portid;
	}		}

	static struct sock __netlink_lookup(struct netlink_table table, u32 portid,		static struct sock __netlink_lookup(struct netlink_table table, u32 portid,
	struct net *net)		struct net *net)
	{		{
	struct netlink_compare_arg arg = {		struct netlink_compare_arg arg;
	.net = net,
	.portid = portid,
	};

	return rhashtable_lookup_compare(&table->hash, &portid,		netlink_compare_arg_init(&arg, net, portid);
	&netlink_compare, &arg);		return rhashtable_lookup_fast(&table->hash, &arg,
			netlink_rhashtable_params);
	}		}

	static bool __netlink_insert(struct netlink_table table, struct sock sk)		static int __netlink_insert(struct netlink_table table, struct sock sk)
	{		{
	struct netlink_compare_arg arg = {		struct netlink_compare_arg arg;
	.net = sock_net(sk),
	.portid = nlk_sk(sk)->portid,
	};

	return rhashtable_lookup_compare_insert(&table->hash,		netlink_compare_arg_init(&arg, sock_net(sk), nlk_sk(sk)->portid);
			return rhashtable_lookup_insert_key(&table->hash, &arg,
	&nlk_sk(sk)->node,		&nlk_sk(sk)->node,
	&netlink_compare, &arg);		netlink_rhashtable_params);
	}		}

	static struct sock netlink_lookup(struct net net, int protocol, u32 portid)		static struct sock netlink_lookup(struct net net, int protocol, u32 portid)
	@@ -1066,8 +1076,9 @@ static int netlink_insert(struct sock *sk, u32 portid)
	nlk_sk(sk)->portid = portid;		nlk_sk(sk)->portid = portid;
	sock_hold(sk);		sock_hold(sk);

	err = 0;		err = __netlink_insert(table, sk);
	if (!__netlink_insert(table, sk)) {		if (err) {
			if (err == -EEXIST)
	err = -EADDRINUSE;		err = -EADDRINUSE;
	sock_put(sk);		sock_put(sk);
	}		}
	@@ -1082,7 +1093,8 @@ static void netlink_remove(struct sock *sk)
	struct netlink_table *table;		struct netlink_table *table;

	table = &nl_table[sk->sk_protocol];		table = &nl_table[sk->sk_protocol];
	if (rhashtable_remove(&table->hash, &nlk_sk(sk)->node)) {		if (!rhashtable_remove_fast(&table->hash, &nlk_sk(sk)->node,
			netlink_rhashtable_params)) {
	WARN_ON(atomic_read(&sk->sk_refcnt) == 1);		WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
	__sock_put(sk);		__sock_put(sk);
	}		}
	@@ -3114,18 +3126,29 @@ static struct pernet_operations __net_initdata netlink_net_ops = {
	.exit = netlink_net_exit,		.exit = netlink_net_exit,
	};		};

	static int __init netlink_proto_init(void)		static inline u32 netlink_hash(const void *data, u32 seed)
	{		{
	int i;		const struct netlink_sock *nlk = data;
	int err = proto_register(&netlink_proto, 0);		struct netlink_compare_arg arg;
	struct rhashtable_params ht_params = {
			netlink_compare_arg_init(&arg, sock_net(&nlk->sk), nlk->portid);
			return jhash(&arg, netlink_compare_arg_len, seed);
			}

			static const struct rhashtable_params netlink_rhashtable_params = {
	.head_offset = offsetof(struct netlink_sock, node),		.head_offset = offsetof(struct netlink_sock, node),
	.key_offset = offsetof(struct netlink_sock, portid),		.key_len = netlink_compare_arg_len,
	.key_len = sizeof(u32), /* portid */
	.hashfn = jhash,		.hashfn = jhash,
			.obj_hashfn = netlink_hash,
			.obj_cmpfn = netlink_compare,
	.max_size = 65536,		.max_size = 65536,
	};		};

			static int __init netlink_proto_init(void)
			{
			int i;
			int err = proto_register(&netlink_proto, 0);

	if (err != 0)		if (err != 0)
	goto out;		goto out;

	@@ -3136,7 +3159,8 @@ static int __init netlink_proto_init(void)
	goto panic;		goto panic;

	for (i = 0; i < MAX_LINKS; i++) {		for (i = 0; i < MAX_LINKS; i++) {
	if (rhashtable_init(&nl_table[i].hash, &ht_params) < 0) {		if (rhashtable_init(&nl_table[i].hash,
			&netlink_rhashtable_params) < 0) {
	while (--i > 0)		while (--i > 0)
	rhashtable_destroy(&nl_table[i].hash);		rhashtable_destroy(&nl_table[i].hash);
	kfree(nl_table);		kfree(nl_table);