Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1c2a088a authored by Alexei Starovoitov's avatar Alexei Starovoitov Committed by Daniel Borkmann
Browse files

bpf: x64: add JIT support for multi-function programs 



Typical JIT does several passes over bpf instructions to
compute total size and relative offsets of jumps and calls.
With multitple bpf functions calling each other all relative calls
will have invalid offsets intially therefore we need to additional
last pass over the program to emit calls with correct offsets.
For example in case of three bpf functions:
main:
  call foo
  call bpf_map_lookup
  exit
foo:
  call bar
  exit
bar:
  exit

We will call bpf_int_jit_compile() indepedently for main(), foo() and bar()
x64 JIT typically does 4-5 passes to converge.
After these initial passes the image for these 3 functions
will be good except call targets, since start addresses of
foo() and bar() are unknown when we were JITing main()
(note that call bpf_map_lookup will be resolved properly
during initial passes).
Once start addresses of 3 functions are known we patch
call_insn->imm to point to right functions and call
bpf_int_jit_compile() again which needs only one pass.
Additional safety checks are done to make sure this
last pass doesn't produce image that is larger or smaller
than previous pass.

When constant blinding is on it's applied to all functions
at the first pass, since doing it once again at the last
pass can change size of the JITed code.

Tested on x64 and arm64 hw with JIT on/off, blinding on/off.
x64 jits bpf-to-bpf calls correctly while arm64 falls back to interpreter.
All other JITs that support normal BPF_CALL will behave the same way
since bpf-to-bpf call is equivalent to bpf-to-kernel call from
JITs point of view.

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent 60b58afc

arch/x86/net/bpf_jit_comp.c

+44 −3
Original line number Diff line number Diff line
@@ -1109,13 +1109,23 @@ xadd: if (is_imm8(insn->off)) 
	return proglen;

}



struct x64_jit_data {

	struct bpf_binary_header *header;

	int *addrs;

	u8 *image;

	int proglen;

	struct jit_context ctx;

};



struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)

{

	struct bpf_binary_header *header = NULL;

	struct bpf_prog *tmp, *orig_prog = prog;

	struct x64_jit_data *jit_data;

	int proglen, oldproglen = 0;

	struct jit_context ctx = {};

	bool tmp_blinded = false;

	bool extra_pass = false;

	u8 *image = NULL;

	int *addrs;

	int pass;
@@ -1135,10 +1145,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
		prog = tmp;

	}



	jit_data = prog->aux->jit_data;

	if (!jit_data) {

		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);

		if (!jit_data) {

			prog = orig_prog;

			goto out;

		}

		prog->aux->jit_data = jit_data;

	}

	addrs = jit_data->addrs;

	if (addrs) {

		ctx = jit_data->ctx;

		oldproglen = jit_data->proglen;

		image = jit_data->image;

		header = jit_data->header;

		extra_pass = true;

		goto skip_init_addrs;

	}

	addrs = kmalloc(prog->len * sizeof(*addrs), GFP_KERNEL);

	if (!addrs) {

		prog = orig_prog;

		goto out;

		goto out_addrs;

	}



	/* Before first pass, make a rough estimation of addrs[]
@@ -1149,6 +1177,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
		addrs[i] = proglen;

	}

	ctx.cleanup_addr = proglen;

skip_init_addrs:



	/* JITed image shrinks with every pass and the loop iterates

	 * until the image stops shrinking. Very large bpf programs
@@ -1189,7 +1218,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 


	if (image) {

		bpf_flush_icache(header, image + proglen);

		if (!prog->is_func || extra_pass) {

			bpf_jit_binary_lock_ro(header);

		} else {

			jit_data->addrs = addrs;

			jit_data->ctx = ctx;

			jit_data->proglen = proglen;

			jit_data->image = image;

			jit_data->header = header;

		}

		prog->bpf_func = (void *)image;

		prog->jited = 1;

		prog->jited_len = proglen;
@@ -1197,8 +1234,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 
		prog = orig_prog;

	}



	if (!prog->is_func || extra_pass) {

out_addrs:

		kfree(addrs);

		kfree(jit_data);

		prog->aux->jit_data = NULL;

	}

out:

	if (tmp_blinded)

		bpf_jit_prog_release_other(prog, prog == orig_prog ?

include/linux/bpf.h

+3 −0
Original line number Diff line number Diff line
@@ -200,6 +200,9 @@ struct bpf_prog_aux { 
	u32 max_ctx_offset;

	u32 stack_depth;

	u32 id;

	u32 func_cnt;

	struct bpf_prog **func;

	void *jit_data; /* JIT specific data. arch dependent */

	struct latch_tree_node ksym_tnode;

	struct list_head ksym_lnode;

	const struct bpf_prog_ops *ops;

include/linux/bpf_verifier.h

+1 −0
Original line number Diff line number Diff line
@@ -143,6 +143,7 @@ struct bpf_insn_aux_data { 
	union {

		enum bpf_reg_type ptr_type;	/* pointer type for load/store insns */

		struct bpf_map *map_ptr;	/* pointer for call insn into lookup_elem */

		s32 call_imm;			/* saved imm field of call insn */

	};

	int ctx_field_size; /* the ctx field size for load insn, maybe 0 */

	bool seen; /* this insn was processed by the verifier */

include/linux/filter.h

+2 −0
Original line number Diff line number Diff line
@@ -463,6 +463,8 @@ struct bpf_prog { 
				gpl_compatible:1, /* Is filter GPL compatible? */

				cb_access:1,	/* Is control block accessed? */

				dst_needed:1,	/* Do we need dst entry? */

				blinded:1,	/* Was blinded */

				is_func:1,	/* program is a bpf function */

				kprobe_override:1; /* Do we override a kprobe? */

	enum bpf_prog_type	type;		/* Type of BPF program */

	u32			len;		/* Number of filter blocks */

kernel/bpf/core.c

+11 −2
Original line number Diff line number Diff line
@@ -722,7 +722,7 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) 
	struct bpf_insn *insn;

	int i, rewritten;



	if (!bpf_jit_blinding_enabled(prog))

	if (!bpf_jit_blinding_enabled(prog) || prog->blinded)

		return prog;



	clone = bpf_prog_clone_create(prog, GFP_USER);
@@ -764,6 +764,7 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) 
		i        += insn_delta;

	}



	clone->blinded = 1;

	return clone;

}

#endif /* CONFIG_BPF_JIT */
@@ -1629,12 +1630,20 @@ int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array, 
static void bpf_prog_free_deferred(struct work_struct *work)

{

	struct bpf_prog_aux *aux;

	int i;



	aux = container_of(work, struct bpf_prog_aux, work);

	if (bpf_prog_is_dev_bound(aux))

		bpf_prog_offload_destroy(aux->prog);

	for (i = 0; i < aux->func_cnt; i++)

		bpf_jit_free(aux->func[i]);

	if (aux->func_cnt) {

		kfree(aux->func);

		bpf_prog_unlock_free(aux->prog);

	} else {

		bpf_jit_free(aux->prog);

	}

}



/* Free internal BPF program */

void bpf_prog_free(struct bpf_prog *fp)