Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 349aa334 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'bpf-helper-cleanups' 



Daniel Borkmann says:

====================
Some BPF helper cleanups

This series contains a couple of misc cleanups and improvements
for BPF helpers. For details please see individual patches. We
let this also sit for a few days with Fengguang's kbuild test
robot, and there were no issues seen (besides one false positive,
see last one for details).
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents bf8d85d4 f3694e00

include/linux/filter.h

+64 −0
Original line number Diff line number Diff line
@@ -314,6 +314,70 @@ struct bpf_prog_aux; 
	bpf_size;						\

})



#define BPF_SIZEOF(type)					\

	({							\

		const int __size = bytes_to_bpf_size(sizeof(type)); \

		BUILD_BUG_ON(__size < 0);			\

		__size;						\

	})



#define BPF_FIELD_SIZEOF(type, field)				\

	({							\

		const int __size = bytes_to_bpf_size(FIELD_SIZEOF(type, field)); \

		BUILD_BUG_ON(__size < 0);			\

		__size;						\

	})



#define __BPF_MAP_0(m, v, ...) v

#define __BPF_MAP_1(m, v, t, a, ...) m(t, a)

#define __BPF_MAP_2(m, v, t, a, ...) m(t, a), __BPF_MAP_1(m, v, __VA_ARGS__)

#define __BPF_MAP_3(m, v, t, a, ...) m(t, a), __BPF_MAP_2(m, v, __VA_ARGS__)

#define __BPF_MAP_4(m, v, t, a, ...) m(t, a), __BPF_MAP_3(m, v, __VA_ARGS__)

#define __BPF_MAP_5(m, v, t, a, ...) m(t, a), __BPF_MAP_4(m, v, __VA_ARGS__)



#define __BPF_REG_0(...) __BPF_PAD(5)

#define __BPF_REG_1(...) __BPF_MAP(1, __VA_ARGS__), __BPF_PAD(4)

#define __BPF_REG_2(...) __BPF_MAP(2, __VA_ARGS__), __BPF_PAD(3)

#define __BPF_REG_3(...) __BPF_MAP(3, __VA_ARGS__), __BPF_PAD(2)

#define __BPF_REG_4(...) __BPF_MAP(4, __VA_ARGS__), __BPF_PAD(1)

#define __BPF_REG_5(...) __BPF_MAP(5, __VA_ARGS__)



#define __BPF_MAP(n, ...) __BPF_MAP_##n(__VA_ARGS__)

#define __BPF_REG(n, ...) __BPF_REG_##n(__VA_ARGS__)



#define __BPF_CAST(t, a)						       \

	(__force t)							       \

	(__force							       \

	 typeof(__builtin_choose_expr(sizeof(t) == sizeof(unsigned long),      \

				      (unsigned long)0, (t)0))) a

#define __BPF_V void

#define __BPF_N



#define __BPF_DECL_ARGS(t, a) t   a

#define __BPF_DECL_REGS(t, a) u64 a



#define __BPF_PAD(n)							       \

	__BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2,       \

		  u64, __ur_3, u64, __ur_4, u64, __ur_5)



#define BPF_CALL_x(x, name, ...)					       \

	static __always_inline						       \

	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__));   \

	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__));	       \

	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__))	       \

	{								       \

		return ____##name(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\

	}								       \

	static __always_inline						       \

	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))



#define BPF_CALL_0(name, ...)	BPF_CALL_x(0, name, __VA_ARGS__)

#define BPF_CALL_1(name, ...)	BPF_CALL_x(1, name, __VA_ARGS__)

#define BPF_CALL_2(name, ...)	BPF_CALL_x(2, name, __VA_ARGS__)

#define BPF_CALL_3(name, ...)	BPF_CALL_x(3, name, __VA_ARGS__)

#define BPF_CALL_4(name, ...)	BPF_CALL_x(4, name, __VA_ARGS__)

#define BPF_CALL_5(name, ...)	BPF_CALL_x(5, name, __VA_ARGS__)



#ifdef CONFIG_COMPAT

/* A struct sock_filter is architecture independent. */

struct compat_sock_fprog {

kernel/bpf/core.c

+1 −1
Original line number Diff line number Diff line
@@ -1018,7 +1018,7 @@ void bpf_user_rnd_init_once(void) 
	prandom_init_once(&bpf_user_rnd_state);

}



u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_0(bpf_user_rnd_u32)

{

	/* Should someone ever have the rather unwise idea to use some

	 * of the registers passed into this function, then note that

kernel/bpf/helpers.c

+15 −37
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
#include <linux/ktime.h>

#include <linux/sched.h>

#include <linux/uidgid.h>

#include <linux/filter.h>



/* If kernel subsystem is allowing eBPF programs to call this function,

 * inside its own verifier_ops->get_func_proto() callback it should return
@@ -26,24 +27,10 @@ 
 * if program is allowed to access maps, so check rcu_read_lock_held in

 * all three functions.

 */

static u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)

{

	/* verifier checked that R1 contains a valid pointer to bpf_map

	 * and R2 points to a program stack and map->key_size bytes were

	 * initialized

	 */

	struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;

	void *key = (void *) (unsigned long) r2;

	void *value;



	WARN_ON_ONCE(!rcu_read_lock_held());



	value = map->ops->map_lookup_elem(map, key);



	/* lookup() returns either pointer to element value or NULL

	 * which is the meaning of PTR_TO_MAP_VALUE_OR_NULL type

	 */

	return (unsigned long) value;

	return (unsigned long) map->ops->map_lookup_elem(map, key);

}



const struct bpf_func_proto bpf_map_lookup_elem_proto = {
@@ -54,15 +41,11 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { 
	.arg2_type	= ARG_PTR_TO_MAP_KEY,

};



static u64 bpf_map_update_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,

	   void *, value, u64, flags)

{

	struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;

	void *key = (void *) (unsigned long) r2;

	void *value = (void *) (unsigned long) r3;



	WARN_ON_ONCE(!rcu_read_lock_held());



	return map->ops->map_update_elem(map, key, value, r4);

	return map->ops->map_update_elem(map, key, value, flags);

}



const struct bpf_func_proto bpf_map_update_elem_proto = {
@@ -75,13 +58,9 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { 
	.arg4_type	= ARG_ANYTHING,

};



static u64 bpf_map_delete_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)

{

	struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;

	void *key = (void *) (unsigned long) r2;



	WARN_ON_ONCE(!rcu_read_lock_held());



	return map->ops->map_delete_elem(map, key);

}
@@ -99,7 +78,7 @@ const struct bpf_func_proto bpf_get_prandom_u32_proto = { 
	.ret_type	= RET_INTEGER,

};



static u64 bpf_get_smp_processor_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_0(bpf_get_smp_processor_id)

{

	return smp_processor_id();

}
@@ -110,7 +89,7 @@ const struct bpf_func_proto bpf_get_smp_processor_id_proto = { 
	.ret_type	= RET_INTEGER,

};



static u64 bpf_ktime_get_ns(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_0(bpf_ktime_get_ns)

{

	/* NMI safe access to clock monotonic */

	return ktime_get_mono_fast_ns();
@@ -122,11 +101,11 @@ const struct bpf_func_proto bpf_ktime_get_ns_proto = { 
	.ret_type	= RET_INTEGER,

};



static u64 bpf_get_current_pid_tgid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_0(bpf_get_current_pid_tgid)

{

	struct task_struct *task = current;



	if (!task)

	if (unlikely(!task))

		return -EINVAL;



	return (u64) task->tgid << 32 | task->pid;
@@ -138,13 +117,13 @@ const struct bpf_func_proto bpf_get_current_pid_tgid_proto = { 
	.ret_type	= RET_INTEGER,

};



static u64 bpf_get_current_uid_gid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_0(bpf_get_current_uid_gid)

{

	struct task_struct *task = current;

	kuid_t uid;

	kgid_t gid;



	if (!task)

	if (unlikely(!task))

		return -EINVAL;



	current_uid_gid(&uid, &gid);
@@ -158,10 +137,9 @@ const struct bpf_func_proto bpf_get_current_uid_gid_proto = { 
	.ret_type	= RET_INTEGER,

};



static u64 bpf_get_current_comm(u64 r1, u64 size, u64 r3, u64 r4, u64 r5)

BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)

{

	struct task_struct *task = current;

	char *buf = (char *) (long) r1;



	if (unlikely(!task))

		goto err_clear;

kernel/bpf/stackmap.c

+2 −3
Original line number Diff line number Diff line
@@ -116,10 +116,9 @@ static struct bpf_map *stack_map_alloc(union bpf_attr *attr) 
	return ERR_PTR(err);

}



u64 bpf_get_stackid(u64 r1, u64 r2, u64 flags, u64 r4, u64 r5)

BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,

	   u64, flags)

{

	struct pt_regs *regs = (struct pt_regs *) (long) r1;

	struct bpf_map *map = (struct bpf_map *) (long) r2;

	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);

	struct perf_callchain_entry *trace;

	struct stack_map_bucket *bucket, *new_bucket, *old_bucket;

kernel/trace/bpf_trace.c

+42 −45
Original line number Diff line number Diff line
@@ -61,11 +61,9 @@ unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx) 
}

EXPORT_SYMBOL_GPL(trace_call_bpf);



static u64 bpf_probe_read(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_3(bpf_probe_read, void *, dst, u32, size, const void *, unsafe_ptr)

{

	void *dst = (void *) (long) r1;

	int ret, size = (int) r2;

	void *unsafe_ptr = (void *) (long) r3;

	int ret;



	ret = probe_kernel_read(dst, unsafe_ptr, size);

	if (unlikely(ret < 0))
@@ -83,12 +81,9 @@ static const struct bpf_func_proto bpf_probe_read_proto = { 
	.arg3_type	= ARG_ANYTHING,

};



static u64 bpf_probe_write_user(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_3(bpf_probe_write_user, void *, unsafe_ptr, const void *, src,

	   u32, size)

{

	void *unsafe_ptr = (void *) (long) r1;

	void *src = (void *) (long) r2;

	int size = (int) r3;



	/*

	 * Ensure we're in user context which is safe for the helper to

	 * run. This helper has no business in a kthread.
@@ -130,9 +125,9 @@ static const struct bpf_func_proto *bpf_get_probe_write_proto(void) 
 * limited trace_printk()

 * only %d %u %x %ld %lu %lx %lld %llu %llx %p %s conversion specifiers allowed

 */

static u64 bpf_trace_printk(u64 r1, u64 fmt_size, u64 r3, u64 r4, u64 r5)

BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1,

	   u64, arg2, u64, arg3)

{

	char *fmt = (char *) (long) r1;

	bool str_seen = false;

	int mod[3] = {};

	int fmt_cnt = 0;
@@ -178,16 +173,16 @@ static u64 bpf_trace_printk(u64 r1, u64 fmt_size, u64 r3, u64 r4, u64 r5) 


				switch (fmt_cnt) {

				case 1:

					unsafe_addr = r3;

					r3 = (long) buf;

					unsafe_addr = arg1;

					arg1 = (long) buf;

					break;

				case 2:

					unsafe_addr = r4;

					r4 = (long) buf;

					unsafe_addr = arg2;

					arg2 = (long) buf;

					break;

				case 3:

					unsafe_addr = r5;

					r5 = (long) buf;

					unsafe_addr = arg3;

					arg3 = (long) buf;

					break;

				}

				buf[0] = 0;
@@ -209,9 +204,9 @@ static u64 bpf_trace_printk(u64 r1, u64 fmt_size, u64 r3, u64 r4, u64 r5) 
	}



	return __trace_printk(1/* fake ip will not be printed */, fmt,

			      mod[0] == 2 ? r3 : mod[0] == 1 ? (long) r3 : (u32) r3,

			      mod[1] == 2 ? r4 : mod[1] == 1 ? (long) r4 : (u32) r4,

			      mod[2] == 2 ? r5 : mod[2] == 1 ? (long) r5 : (u32) r5);

			      mod[0] == 2 ? arg1 : mod[0] == 1 ? (long) arg1 : (u32) arg1,

			      mod[1] == 2 ? arg2 : mod[1] == 1 ? (long) arg2 : (u32) arg2,

			      mod[2] == 2 ? arg3 : mod[2] == 1 ? (long) arg3 : (u32) arg3);

}



static const struct bpf_func_proto bpf_trace_printk_proto = {
@@ -233,9 +228,8 @@ const struct bpf_func_proto *bpf_get_trace_printk_proto(void) 
	return &bpf_trace_printk_proto;

}



static u64 bpf_perf_event_read(u64 r1, u64 flags, u64 r3, u64 r4, u64 r5)

BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)

{

	struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	unsigned int cpu = smp_processor_id();

	u64 index = flags & BPF_F_INDEX_MASK;
@@ -312,11 +306,9 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 
	return 0;

}



static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size)

BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,

	   u64, flags, void *, data, u64, size)

{

	struct pt_regs *regs = (struct pt_regs *)(long) r1;

	struct bpf_map *map  = (struct bpf_map *)(long) r2;

	void *data = (void *)(long) r4;

	struct perf_raw_record raw = {

		.frag = {

			.size = size,
@@ -367,7 +359,7 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 
	return __bpf_perf_event_output(regs, map, flags, &raw);

}



static u64 bpf_get_current_task(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_0(bpf_get_current_task)

{

	return (long) current;

}
@@ -378,16 +370,13 @@ static const struct bpf_func_proto bpf_get_current_task_proto = { 
	.ret_type	= RET_INTEGER,

};



static u64 bpf_current_task_under_cgroup(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx)

{

	struct bpf_map *map = (struct bpf_map *)(long)r1;

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	struct cgroup *cgrp;

	u32 idx = (u32)r2;



	if (unlikely(in_interrupt()))

		return -EINVAL;



	if (unlikely(idx >= array->map.max_entries))

		return -E2BIG;
@@ -481,16 +470,17 @@ static struct bpf_prog_type_list kprobe_tl = { 
	.type	= BPF_PROG_TYPE_KPROBE,

};



static u64 bpf_perf_event_output_tp(u64 r1, u64 r2, u64 index, u64 r4, u64 size)

BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map,

	   u64, flags, void *, data, u64, size)

{

	struct pt_regs *regs = *(struct pt_regs **)tp_buff;



	/*

	 * r1 points to perf tracepoint buffer where first 8 bytes are hidden

	 * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it

	 * from there and call the same bpf_perf_event_output() helper

	 * from there and call the same bpf_perf_event_output() helper inline.

	 */

	u64 ctx = *(long *)(uintptr_t)r1;



	return bpf_perf_event_output(ctx, r2, index, r4, size);

	return ____bpf_perf_event_output(regs, map, flags, data, size);

}



static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
@@ -504,11 +494,18 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { 
	.arg5_type	= ARG_CONST_STACK_SIZE,

};



static u64 bpf_get_stackid_tp(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map,

	   u64, flags)

{

	u64 ctx = *(long *)(uintptr_t)r1;

	struct pt_regs *regs = *(struct pt_regs **)tp_buff;



	return bpf_get_stackid(ctx, r2, r3, r4, r5);

	/*

	 * Same comment as in bpf_perf_event_output_tp(), only that this time

	 * the other helper's function body cannot be inlined due to being

	 * external, thus we need to call raw helper function.

	 */

	return bpf_get_stackid((unsigned long) regs, (unsigned long) map,

			       flags, 0, 0);

}



static const struct bpf_func_proto bpf_get_stackid_proto_tp = {
@@ -583,18 +580,18 @@ static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, int dst_reg, 
	switch (ctx_off) {

	case offsetof(struct bpf_perf_event_data, sample_period):

		BUILD_BUG_ON(FIELD_SIZEOF(struct perf_sample_data, period) != sizeof(u64));

		*insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct bpf_perf_event_data_kern, data)),

				      dst_reg, src_reg,



		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,

						       data), dst_reg, src_reg,

				      offsetof(struct bpf_perf_event_data_kern, data));

		*insn++ = BPF_LDX_MEM(BPF_DW, dst_reg, dst_reg,

				      offsetof(struct perf_sample_data, period));

		break;

	default:

		*insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct bpf_perf_event_data_kern, regs)),

				      dst_reg, src_reg,

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,

						       regs), dst_reg, src_reg,

				      offsetof(struct bpf_perf_event_data_kern, regs));

		*insn++ = BPF_LDX_MEM(bytes_to_bpf_size(sizeof(long)),

				      dst_reg, dst_reg, ctx_off);

		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), dst_reg, dst_reg, ctx_off);

		break;

	}