Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a23244e8 authored by Daniel Borkmann's avatar Daniel Borkmann Committed by Greg Kroah-Hartman
Browse files

bpf: fix corruption on concurrent perf_event_output calls 




[ Upstream commit 283ca526 ]

When tracing and networking programs are both attached in the
system and both use event-output helpers that eventually call
into perf_event_output(), then we could end up in a situation
where the tracing attached program runs in user context while
a cls_bpf program is triggered on that same CPU out of softirq
context.

Since both rely on the same per-cpu perf_sample_data, we could
potentially corrupt it. This can only ever happen in a combination
of the two types; all tracing programs use a bpf_prog_active
counter to bail out in case a program is already running on
that CPU out of a different context. XDP and cls_bpf programs
by themselves don't have this issue as they run in the same
context only. Therefore, split both perf_sample_data so they
cannot be accessed from each other.

Fixes: 20b9d7ac ("bpf: avoid excessive stack usage for perf_sample_data")
Reported-by: default avatarAlexei Starovoitov <ast@fb.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Tested-by: default avatarSong Liu <songliubraving@fb.com>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 2b3ea8ce

kernel/trace/bpf_trace.c

+12 −7
Original line number Original line Diff line number Diff line
@@ -293,14 +293,13 @@ static const struct bpf_func_proto bpf_perf_event_read_proto = { 
	.arg2_type	= ARG_ANYTHING,

	.arg2_type	= ARG_ANYTHING,

};

};





static DEFINE_PER_CPU(struct perf_sample_data, bpf_sd);

static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);





static __always_inline u64

static __always_inline u64

__bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,

__bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,

			u64 flags, struct perf_raw_record *raw)

			u64 flags, struct perf_sample_data *sd)

{

{

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	struct perf_sample_data *sd = this_cpu_ptr(&bpf_sd);

	unsigned int cpu = smp_processor_id();

	unsigned int cpu = smp_processor_id();

	u64 index = flags & BPF_F_INDEX_MASK;

	u64 index = flags & BPF_F_INDEX_MASK;

	struct bpf_event_entry *ee;

	struct bpf_event_entry *ee;
@@ -323,8 +322,6 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 
	if (unlikely(event->oncpu != cpu))

	if (unlikely(event->oncpu != cpu))

		return -EOPNOTSUPP;

		return -EOPNOTSUPP;





	perf_sample_data_init(sd, 0, 0);

	sd->raw = raw;

	perf_event_output(event, sd, regs);

	perf_event_output(event, sd, regs);

	return 0;

	return 0;

}

}
@@ -332,6 +329,7 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 
BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,

BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,

	   u64, flags, void *, data, u64, size)

	   u64, flags, void *, data, u64, size)

{

{

	struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);

	struct perf_raw_record raw = {

	struct perf_raw_record raw = {

		.frag = {

		.frag = {

			.size = size,

			.size = size,
@@ -342,7 +340,10 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, 
	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))

	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))

		return -EINVAL;

		return -EINVAL;





	return __bpf_perf_event_output(regs, map, flags, &raw);

	perf_sample_data_init(sd, 0, 0);

	sd->raw = &raw;



	return __bpf_perf_event_output(regs, map, flags, sd);

}

}





static const struct bpf_func_proto bpf_perf_event_output_proto = {

static const struct bpf_func_proto bpf_perf_event_output_proto = {
@@ -357,10 +358,12 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { 
};

};





static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs);

static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs);

static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd);





u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,

u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,

		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)

		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)

{

{

	struct perf_sample_data *sd = this_cpu_ptr(&bpf_misc_sd);

	struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs);

	struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs);

	struct perf_raw_frag frag = {

	struct perf_raw_frag frag = {

		.copy		= ctx_copy,

		.copy		= ctx_copy,
@@ -378,8 +381,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 
	};

	};





	perf_fetch_caller_regs(regs);

	perf_fetch_caller_regs(regs);

	perf_sample_data_init(sd, 0, 0);

	sd->raw = &raw;





	return __bpf_perf_event_output(regs, map, flags, &raw);

	return __bpf_perf_event_output(regs, map, flags, sd);

}

}





BPF_CALL_0(bpf_get_current_task)

BPF_CALL_0(bpf_get_current_task)