bpf: expand BPF syscall with program load/unload (09756af4) · Commits · e / devices / android_kernel_xiaomi_sm6125

include/linux/bpf.h

+38 −0

Original line number	Original line	Diff line number	Diff line
	@@ -46,4 +46,42 @@ void bpf_register_map_type(struct bpf_map_type_list *tl);
	void bpf_map_put(struct bpf_map *map);		void bpf_map_put(struct bpf_map *map);
	struct bpf_map *bpf_map_get(struct fd f);		struct bpf_map *bpf_map_get(struct fd f);

			/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
			* to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
			* instructions after verifying
			*/
			struct bpf_func_proto {
			u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
			bool gpl_only;
			};

			struct bpf_verifier_ops {
			/* return eBPF function prototype for verification */
			const struct bpf_func_proto (get_func_proto)(enum bpf_func_id func_id);
			};

			struct bpf_prog_type_list {
			struct list_head list_node;
			struct bpf_verifier_ops *ops;
			enum bpf_prog_type type;
			};

			void bpf_register_prog_type(struct bpf_prog_type_list *tl);

			struct bpf_prog;

			struct bpf_prog_aux {
			atomic_t refcnt;
			bool is_gpl_compatible;
			enum bpf_prog_type prog_type;
			struct bpf_verifier_ops *ops;
			struct bpf_map **used_maps;
			u32 used_map_cnt;
			struct bpf_prog *prog;
			struct work_struct work;
			};

			void bpf_prog_put(struct bpf_prog *prog);
			struct bpf_prog *bpf_prog_get(u32 ufd);

	#endif /* _LINUX_BPF_H */		#endif /* _LINUX_BPF_H */

include/linux/filter.h

+2 −6

Original line number	Original line	Diff line number	Diff line
	@@ -21,6 +21,7 @@
	struct sk_buff;		struct sk_buff;
	struct sock;		struct sock;
	struct seccomp_data;		struct seccomp_data;
			struct bpf_prog_aux;

	/* ArgX, context and stack frame pointer register positions. Note,		/* ArgX, context and stack frame pointer register positions. Note,
	* Arg1, Arg2, Arg3, etc are used as argument mappings of function		* Arg1, Arg2, Arg3, etc are used as argument mappings of function
	@@ -300,17 +301,12 @@ struct bpf_binary_header {
	u8 image[];		u8 image[];
	};		};

	struct bpf_work_struct {
	struct bpf_prog *prog;
	struct work_struct work;
	};

	struct bpf_prog {		struct bpf_prog {
	u16 pages; /* Number of allocated pages */		u16 pages; /* Number of allocated pages */
	bool jited; /* Is our filter JIT'ed? */		bool jited; /* Is our filter JIT'ed? */
	u32 len; /* Number of filter blocks */		u32 len; /* Number of filter blocks */
	struct sock_fprog_kern orig_prog; / Original BPF program */		struct sock_fprog_kern orig_prog; / Original BPF program */
	struct bpf_work_struct work; / Deferred free work struct */		struct bpf_prog_aux aux; / Auxiliary fields */
	unsigned int (bpf_func)(const struct sk_buff skb,		unsigned int (bpf_func)(const struct sk_buff skb,
	const struct bpf_insn *filter);		const struct bpf_insn *filter);
	/* Instructions for interpreter */		/* Instructions for interpreter */

include/uapi/linux/bpf.h

+26 −0

Original line number	Original line	Diff line number	Diff line
	@@ -99,12 +99,23 @@ enum bpf_cmd {
	* returns zero and stores next key or negative error		* returns zero and stores next key or negative error
	*/		*/
	BPF_MAP_GET_NEXT_KEY,		BPF_MAP_GET_NEXT_KEY,

			/* verify and load eBPF program
			* prog_fd = bpf(BPF_PROG_LOAD, union bpf_attr *attr, u32 size)
			* Using attr->prog_type, attr->insns, attr->license
			* returns fd or negative error
			*/
			BPF_PROG_LOAD,
	};		};

	enum bpf_map_type {		enum bpf_map_type {
	BPF_MAP_TYPE_UNSPEC,		BPF_MAP_TYPE_UNSPEC,
	};		};

			enum bpf_prog_type {
			BPF_PROG_TYPE_UNSPEC,
			};

	union bpf_attr {		union bpf_attr {
	struct { /* anonymous struct used by BPF_MAP_CREATE command */		struct { /* anonymous struct used by BPF_MAP_CREATE command */
	__u32 map_type; /* one of enum bpf_map_type */		__u32 map_type; /* one of enum bpf_map_type */
	@@ -121,6 +132,21 @@ union bpf_attr {
	__aligned_u64 next_key;		__aligned_u64 next_key;
	};		};
	};		};

			struct { /* anonymous struct used by BPF_PROG_LOAD command */
			__u32 prog_type; /* one of enum bpf_prog_type */
			__u32 insn_cnt;
			__aligned_u64 insns;
			__aligned_u64 license;
			};
	} __attribute__((aligned(8)));		} __attribute__((aligned(8)));

			/* integer value in 'imm' field of BPF_CALL instruction selects which helper
			* function eBPF program intends to call
			*/
			enum bpf_func_id {
			BPF_FUNC_unspec,
			__BPF_FUNC_MAX_ID,
			};

	#endif /* _UAPI__LINUX_BPF_H__ */		#endif /* _UAPI__LINUX_BPF_H__ */

kernel/bpf/core.c

+15 −14

Original line number	Original line	Diff line number	Diff line
	@@ -27,6 +27,7 @@
	#include <linux/random.h>		#include <linux/random.h>
	#include <linux/moduleloader.h>		#include <linux/moduleloader.h>
	#include <asm/unaligned.h>		#include <asm/unaligned.h>
			#include <linux/bpf.h>

	/* Registers */		/* Registers */
	#define BPF_R0 regs[BPF_REG_0]		#define BPF_R0 regs[BPF_REG_0]
	@@ -71,7 +72,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
	{		{
	gfp_t gfp_flags = GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_ZERO \|		gfp_t gfp_flags = GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_ZERO \|
	gfp_extra_flags;		gfp_extra_flags;
	struct bpf_work_struct *ws;		struct bpf_prog_aux *aux;
	struct bpf_prog *fp;		struct bpf_prog *fp;

	size = round_up(size, PAGE_SIZE);		size = round_up(size, PAGE_SIZE);
	@@ -79,14 +80,14 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
	if (fp == NULL)		if (fp == NULL)
	return NULL;		return NULL;

	ws = kmalloc(sizeof(*ws), GFP_KERNEL \| gfp_extra_flags);		aux = kzalloc(sizeof(*aux), GFP_KERNEL \| gfp_extra_flags);
	if (ws == NULL) {		if (aux == NULL) {
	vfree(fp);		vfree(fp);
	return NULL;		return NULL;
	}		}

	fp->pages = size / PAGE_SIZE;		fp->pages = size / PAGE_SIZE;
	fp->work = ws;		fp->aux = aux;

	return fp;		return fp;
	}		}
	@@ -110,10 +111,10 @@ struct bpf_prog bpf_prog_realloc(struct bpf_prog fp_old, unsigned int size,
	memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);		memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);
	fp->pages = size / PAGE_SIZE;		fp->pages = size / PAGE_SIZE;

	/* We keep fp->work from fp_old around in the new		/* We keep fp->aux from fp_old around in the new
	* reallocated structure.		* reallocated structure.
	*/		*/
	fp_old->work = NULL;		fp_old->aux = NULL;
	__bpf_prog_free(fp_old);		__bpf_prog_free(fp_old);
	}		}

	@@ -123,7 +124,7 @@ EXPORT_SYMBOL_GPL(bpf_prog_realloc);

	void __bpf_prog_free(struct bpf_prog *fp)		void __bpf_prog_free(struct bpf_prog *fp)
	{		{
	kfree(fp->work);		kfree(fp->aux);
	vfree(fp);		vfree(fp);
	}		}
	EXPORT_SYMBOL_GPL(__bpf_prog_free);		EXPORT_SYMBOL_GPL(__bpf_prog_free);
	@@ -638,19 +639,19 @@ EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);

	static void bpf_prog_free_deferred(struct work_struct *work)		static void bpf_prog_free_deferred(struct work_struct *work)
	{		{
	struct bpf_work_struct *ws;		struct bpf_prog_aux *aux;

	ws = container_of(work, struct bpf_work_struct, work);		aux = container_of(work, struct bpf_prog_aux, work);
	bpf_jit_free(ws->prog);		bpf_jit_free(aux->prog);
	}		}

	/* Free internal BPF program */		/* Free internal BPF program */
	void bpf_prog_free(struct bpf_prog *fp)		void bpf_prog_free(struct bpf_prog *fp)
	{		{
	struct bpf_work_struct *ws = fp->work;		struct bpf_prog_aux *aux = fp->aux;

	INIT_WORK(&ws->work, bpf_prog_free_deferred);		INIT_WORK(&aux->work, bpf_prog_free_deferred);
	ws->prog = fp;		aux->prog = fp;
	schedule_work(&ws->work);		schedule_work(&aux->work);
	}		}
	EXPORT_SYMBOL_GPL(bpf_prog_free);		EXPORT_SYMBOL_GPL(bpf_prog_free);

kernel/bpf/syscall.c

+165 −0

Original line number	Original line	Diff line number	Diff line
	@@ -14,6 +14,8 @@
	#include <linux/slab.h>		#include <linux/slab.h>
	#include <linux/anon_inodes.h>		#include <linux/anon_inodes.h>
	#include <linux/file.h>		#include <linux/file.h>
			#include <linux/license.h>
			#include <linux/filter.h>

	static LIST_HEAD(bpf_map_types);		static LIST_HEAD(bpf_map_types);

	@@ -334,6 +336,166 @@ static int map_get_next_key(union bpf_attr *attr)
	return err;		return err;
	}		}

			static LIST_HEAD(bpf_prog_types);

			static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
			{
			struct bpf_prog_type_list *tl;

			list_for_each_entry(tl, &bpf_prog_types, list_node) {
			if (tl->type == type) {
			prog->aux->ops = tl->ops;
			prog->aux->prog_type = type;
			return 0;
			}
			}
			return -EINVAL;
			}

			void bpf_register_prog_type(struct bpf_prog_type_list *tl)
			{
			list_add(&tl->list_node, &bpf_prog_types);
			}

			/* drop refcnt on maps used by eBPF program and free auxilary data */
			static void free_used_maps(struct bpf_prog_aux *aux)
			{
			int i;

			for (i = 0; i < aux->used_map_cnt; i++)
			bpf_map_put(aux->used_maps[i]);

			kfree(aux->used_maps);
			}

			void bpf_prog_put(struct bpf_prog *prog)
			{
			if (atomic_dec_and_test(&prog->aux->refcnt)) {
			free_used_maps(prog->aux);
			bpf_prog_free(prog);
			}
			}

			static int bpf_prog_release(struct inode inode, struct file filp)
			{
			struct bpf_prog *prog = filp->private_data;

			bpf_prog_put(prog);
			return 0;
			}

			static const struct file_operations bpf_prog_fops = {
			.release = bpf_prog_release,
			};

			static struct bpf_prog *get_prog(struct fd f)
			{
			struct bpf_prog *prog;

			if (!f.file)
			return ERR_PTR(-EBADF);

			if (f.file->f_op != &bpf_prog_fops) {
			fdput(f);
			return ERR_PTR(-EINVAL);
			}

			prog = f.file->private_data;

			return prog;
			}

			/* called by sockets/tracing/seccomp before attaching program to an event
			* pairs with bpf_prog_put()
			*/
			struct bpf_prog *bpf_prog_get(u32 ufd)
			{
			struct fd f = fdget(ufd);
			struct bpf_prog *prog;

			prog = get_prog(f);

			if (IS_ERR(prog))
			return prog;

			atomic_inc(&prog->aux->refcnt);
			fdput(f);
			return prog;
			}

			/* last field in 'union bpf_attr' used by this command */
			#define BPF_PROG_LOAD_LAST_FIELD license

			static int bpf_prog_load(union bpf_attr *attr)
			{
			enum bpf_prog_type type = attr->prog_type;
			struct bpf_prog *prog;
			int err;
			char license[128];
			bool is_gpl;

			if (CHECK_ATTR(BPF_PROG_LOAD))
			return -EINVAL;

			/* copy eBPF program license from user space */
			if (strncpy_from_user(license, u64_to_ptr(attr->license),
			sizeof(license) - 1) < 0)
			return -EFAULT;
			license[sizeof(license) - 1] = 0;

			/* eBPF programs must be GPL compatible to use GPL-ed functions */
			is_gpl = license_is_gpl_compatible(license);

			if (attr->insn_cnt >= BPF_MAXINSNS)
			return -EINVAL;

			/* plain bpf_prog allocation */
			prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
			if (!prog)
			return -ENOMEM;

			prog->len = attr->insn_cnt;

			err = -EFAULT;
			if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),
			prog->len * sizeof(struct bpf_insn)) != 0)
			goto free_prog;

			prog->orig_prog = NULL;
			prog->jited = false;

			atomic_set(&prog->aux->refcnt, 1);
			prog->aux->is_gpl_compatible = is_gpl;

			/* find program type: socket_filter vs tracing_filter */
			err = find_prog_type(type, prog);
			if (err < 0)
			goto free_prog;

			/* run eBPF verifier */
			/* err = bpf_check(prog, tb); */

			if (err < 0)
			goto free_used_maps;

			/* eBPF program is ready to be JITed */
			bpf_prog_select_runtime(prog);

			err = anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, O_RDWR \| O_CLOEXEC);

			if (err < 0)
			/* failed to allocate fd */
			goto free_used_maps;

			return err;

			free_used_maps:
			free_used_maps(prog->aux);
			free_prog:
			bpf_prog_free(prog);
			return err;
			}

	SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)		SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
	{		{
	union bpf_attr attr = {};		union bpf_attr attr = {};
	@@ -395,6 +557,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
	case BPF_MAP_GET_NEXT_KEY:		case BPF_MAP_GET_NEXT_KEY:
	err = map_get_next_key(&attr);		err = map_get_next_key(&attr);
	break;		break;
			case BPF_PROG_LOAD:
			err = bpf_prog_load(&attr);
			break;
	default:		default:
	err = -EINVAL;		err = -EINVAL;
	break;		break;