Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 06d0a11f authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'bpf-Allow-selecting-numa-node-during-map-creation' 



Martin KaFai Lau says:

====================
bpf: Allow selecting numa node during map creation

This series allows user to pick the numa node during map creation.
The first patch has the details
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents bd76b879 ad17d0e6

include/linux/bpf.h

+9 −1
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ struct bpf_map { 
	u32 map_flags;

	u32 pages;

	u32 id;

	int numa_node;

	struct user_struct *user;

	const struct bpf_map_ops *ops;

	struct work_struct work;
@@ -264,7 +265,7 @@ struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref); 
void bpf_map_put_with_uref(struct bpf_map *map);

void bpf_map_put(struct bpf_map *map);

int bpf_map_precharge_memlock(u32 pages);

void *bpf_map_area_alloc(size_t size);

void *bpf_map_area_alloc(size_t size, int numa_node);

void bpf_map_area_free(void *base);



extern int sysctl_unprivileged_bpf_disabled;
@@ -316,6 +317,13 @@ struct net_device *__dev_map_lookup_elem(struct bpf_map *map, u32 key); 
void __dev_map_insert_ctx(struct bpf_map *map, u32 index);

void __dev_map_flush(struct bpf_map *map);



/* Return map's numa specified by userspace */

static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)

{

	return (attr->map_flags & BPF_F_NUMA_NODE) ?

		attr->numa_node : NUMA_NO_NODE;

}



#else

static inline struct bpf_prog *bpf_prog_get(u32 ufd)

{

include/uapi/linux/bpf.h

+9 −1
Original line number Diff line number Diff line
@@ -165,6 +165,7 @@ enum bpf_attach_type { 
#define BPF_NOEXIST	1 /* create new element if it didn't exist */

#define BPF_EXIST	2 /* update existing element */



/* flags for BPF_MAP_CREATE command */

#define BPF_F_NO_PREALLOC	(1U << 0)

/* Instead of having one common LRU list in the

 * BPF_MAP_TYPE_LRU_[PERCPU_]HASH map, use a percpu LRU list
@@ -173,6 +174,8 @@ enum bpf_attach_type { 
 * across different LRU lists.

 */

#define BPF_F_NO_COMMON_LRU	(1U << 1)

/* Specify numa node during map creation */

#define BPF_F_NUMA_NODE		(1U << 2)



union bpf_attr {

	struct { /* anonymous struct used by BPF_MAP_CREATE command */
@@ -180,8 +183,13 @@ union bpf_attr { 
		__u32	key_size;	/* size of key in bytes */

		__u32	value_size;	/* size of value in bytes */

		__u32	max_entries;	/* max number of entries in a map */

		__u32	map_flags;	/* prealloc or not */

		__u32	map_flags;	/* BPF_MAP_CREATE related

					 * flags defined above.

					 */

		__u32	inner_map_fd;	/* fd pointing to the inner map */

		__u32	numa_node;	/* numa node (effective only if

					 * BPF_F_NUMA_NODE is set).

					 */

	};



	struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */

kernel/bpf/arraymap.c

+5 −2
Original line number Diff line number Diff line
@@ -49,13 +49,15 @@ static int bpf_array_alloc_percpu(struct bpf_array *array) 
static struct bpf_map *array_map_alloc(union bpf_attr *attr)

{

	bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;

	int numa_node = bpf_map_attr_numa_node(attr);

	struct bpf_array *array;

	u64 array_size;

	u32 elem_size;



	/* check sanity of attributes */

	if (attr->max_entries == 0 || attr->key_size != 4 ||

	    attr->value_size == 0 || attr->map_flags)

	    attr->value_size == 0 || attr->map_flags & ~BPF_F_NUMA_NODE ||

	    (percpu && numa_node != NUMA_NO_NODE))

		return ERR_PTR(-EINVAL);



	if (attr->value_size > KMALLOC_MAX_SIZE)
@@ -77,7 +79,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) 
		return ERR_PTR(-ENOMEM);



	/* allocate all map elements and zero-initialize them */

	array = bpf_map_area_alloc(array_size);

	array = bpf_map_area_alloc(array_size, numa_node);

	if (!array)

		return ERR_PTR(-ENOMEM);
@@ -87,6 +89,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) 
	array->map.value_size = attr->value_size;

	array->map.max_entries = attr->max_entries;

	array->map.map_flags = attr->map_flags;

	array->map.numa_node = numa_node;

	array->elem_size = elem_size;



	if (!percpu)

kernel/bpf/devmap.c

+6 −3
Original line number Diff line number Diff line
@@ -80,7 +80,7 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 


	/* check sanity of attributes */

	if (attr->max_entries == 0 || attr->key_size != 4 ||

	    attr->value_size != 4 || attr->map_flags)

	    attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE)

		return ERR_PTR(-EINVAL);



	dtab = kzalloc(sizeof(*dtab), GFP_USER);
@@ -93,6 +93,7 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 
	dtab->map.value_size = attr->value_size;

	dtab->map.max_entries = attr->max_entries;

	dtab->map.map_flags = attr->map_flags;

	dtab->map.numa_node = bpf_map_attr_numa_node(attr);



	err = -ENOMEM;
@@ -119,7 +120,8 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) 
		goto free_dtab;



	dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *

					      sizeof(struct bpf_dtab_netdev *));

					      sizeof(struct bpf_dtab_netdev *),

					      dtab->map.numa_node);

	if (!dtab->netdev_map)

		goto free_dtab;
@@ -344,7 +346,8 @@ static int dev_map_update_elem(struct bpf_map *map, void *key, void *value, 
	if (!ifindex) {

		dev = NULL;

	} else {

		dev = kmalloc(sizeof(*dev), GFP_ATOMIC | __GFP_NOWARN);

		dev = kmalloc_node(sizeof(*dev), GFP_ATOMIC | __GFP_NOWARN,

				   map->numa_node);

		if (!dev)

			return -ENOMEM;

kernel/bpf/hashtab.c

+15 −4
Original line number Diff line number Diff line
@@ -18,6 +18,9 @@ 
#include "bpf_lru_list.h"

#include "map_in_map.h"



#define HTAB_CREATE_FLAG_MASK \

	(BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE)



struct bucket {

	struct hlist_nulls_head head;

	raw_spinlock_t lock;
@@ -138,7 +141,8 @@ static int prealloc_init(struct bpf_htab *htab) 
	if (!htab_is_percpu(htab) && !htab_is_lru(htab))

		num_entries += num_possible_cpus();



	htab->elems = bpf_map_area_alloc(htab->elem_size * num_entries);

	htab->elems = bpf_map_area_alloc(htab->elem_size * num_entries,

					 htab->map.numa_node);

	if (!htab->elems)

		return -ENOMEM;
@@ -233,6 +237,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
	 */

	bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU);

	bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC);

	int numa_node = bpf_map_attr_numa_node(attr);

	struct bpf_htab *htab;

	int err, i;

	u64 cost;
@@ -248,7 +253,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
		 */

		return ERR_PTR(-EPERM);



	if (attr->map_flags & ~(BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU))

	if (attr->map_flags & ~HTAB_CREATE_FLAG_MASK)

		/* reserved bits should not be used */

		return ERR_PTR(-EINVAL);
@@ -258,6 +263,9 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
	if (lru && !prealloc)

		return ERR_PTR(-ENOTSUPP);



	if (numa_node != NUMA_NO_NODE && (percpu || percpu_lru))

		return ERR_PTR(-EINVAL);



	htab = kzalloc(sizeof(*htab), GFP_USER);

	if (!htab)

		return ERR_PTR(-ENOMEM);
@@ -268,6 +276,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 
	htab->map.value_size = attr->value_size;

	htab->map.max_entries = attr->max_entries;

	htab->map.map_flags = attr->map_flags;

	htab->map.numa_node = numa_node;



	/* check sanity of attributes.

	 * value_size == 0 may be allowed in the future to use map as a set
@@ -346,7 +355,8 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) 


	err = -ENOMEM;

	htab->buckets = bpf_map_area_alloc(htab->n_buckets *

					   sizeof(struct bucket));

					   sizeof(struct bucket),

					   htab->map.numa_node);

	if (!htab->buckets)

		goto free_htab;
@@ -689,7 +699,8 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, 
				atomic_dec(&htab->count);

				return ERR_PTR(-E2BIG);

			}

		l_new = kmalloc(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN);

		l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN,

				     htab->map.numa_node);

		if (!l_new)

			return ERR_PTR(-ENOMEM);

	}