Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c237ee5e authored by Daniel Borkmann's avatar Daniel Borkmann Committed by David S. Miller
Browse files

bpf: add bpf_patch_insn_single helper 



Move the functionality to patch instructions out of the verifier
code and into the core as the new bpf_patch_insn_single() helper
will be needed later on for blinding as well. No changes in
functionality.

Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 93a73d44

include/linux/filter.h

+3 −0
Original line number Diff line number Diff line
@@ -495,6 +495,9 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 
void bpf_int_jit_compile(struct bpf_prog *fp);

bool bpf_helper_changes_skb_data(void *func);



struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,

				       const struct bpf_insn *patch, u32 len);



#ifdef CONFIG_BPF_JIT

extern int bpf_jit_enable;

kernel/bpf/core.c

+71 −0
Original line number Diff line number Diff line
@@ -136,6 +136,77 @@ void __bpf_prog_free(struct bpf_prog *fp) 
	vfree(fp);

}



static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)

{

	return BPF_CLASS(insn->code) == BPF_JMP  &&

	       /* Call and Exit are both special jumps with no

		* target inside the BPF instruction image.

		*/

	       BPF_OP(insn->code) != BPF_CALL &&

	       BPF_OP(insn->code) != BPF_EXIT;

}



static void bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta)

{

	struct bpf_insn *insn = prog->insnsi;

	u32 i, insn_cnt = prog->len;



	for (i = 0; i < insn_cnt; i++, insn++) {

		if (!bpf_is_jmp_and_has_target(insn))

			continue;



		/* Adjust offset of jmps if we cross boundaries. */

		if (i < pos && i + insn->off + 1 > pos)

			insn->off += delta;

		else if (i > pos + delta && i + insn->off + 1 <= pos + delta)

			insn->off -= delta;

	}

}



struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,

				       const struct bpf_insn *patch, u32 len)

{

	u32 insn_adj_cnt, insn_rest, insn_delta = len - 1;

	struct bpf_prog *prog_adj;



	/* Since our patchlet doesn't expand the image, we're done. */

	if (insn_delta == 0) {

		memcpy(prog->insnsi + off, patch, sizeof(*patch));

		return prog;

	}



	insn_adj_cnt = prog->len + insn_delta;



	/* Several new instructions need to be inserted. Make room

	 * for them. Likely, there's no need for a new allocation as

	 * last page could have large enough tailroom.

	 */

	prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),

				    GFP_USER);

	if (!prog_adj)

		return NULL;



	prog_adj->len = insn_adj_cnt;



	/* Patching happens in 3 steps:

	 *

	 * 1) Move over tail of insnsi from next instruction onwards,

	 *    so we can patch the single target insn with one or more

	 *    new ones (patching is always from 1 to n insns, n > 0).

	 * 2) Inject new instructions at the target location.

	 * 3) Adjust branch offsets if necessary.

	 */

	insn_rest = insn_adj_cnt - off - len;



	memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1,

		sizeof(*patch) * insn_rest);

	memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len);



	bpf_adj_branches(prog_adj, off, insn_delta);



	return prog_adj;

}



#ifdef CONFIG_BPF_JIT

struct bpf_binary_header *

bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,

kernel/bpf/verifier.c

+9 −44
Original line number Diff line number Diff line
@@ -2587,26 +2587,6 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env) 
			insn->src_reg = 0;

}



static void adjust_branches(struct bpf_prog *prog, int pos, int delta)

{

	struct bpf_insn *insn = prog->insnsi;

	int insn_cnt = prog->len;

	int i;



	for (i = 0; i < insn_cnt; i++, insn++) {

		if (BPF_CLASS(insn->code) != BPF_JMP ||

		    BPF_OP(insn->code) == BPF_CALL ||

		    BPF_OP(insn->code) == BPF_EXIT)

			continue;



		/* adjust offset of jmps if necessary */

		if (i < pos && i + insn->off + 1 > pos)

			insn->off += delta;

		else if (i > pos + delta && i + insn->off + 1 <= pos + delta)

			insn->off -= delta;

	}

}



/* convert load instructions that access fields of 'struct __sk_buff'

 * into sequence of instructions that access fields of 'struct sk_buff'

 */
@@ -2616,14 +2596,15 @@ static int convert_ctx_accesses(struct verifier_env *env) 
	int insn_cnt = env->prog->len;

	struct bpf_insn insn_buf[16];

	struct bpf_prog *new_prog;

	u32 cnt;

	int i;

	enum bpf_access_type type;

	int i;



	if (!env->prog->aux->ops->convert_ctx_access)

		return 0;



	for (i = 0; i < insn_cnt; i++, insn++) {

		u32 insn_delta, cnt;



		if (insn->code == (BPF_LDX | BPF_MEM | BPF_W))

			type = BPF_READ;

		else if (insn->code == (BPF_STX | BPF_MEM | BPF_W))
@@ -2645,34 +2626,18 @@ static int convert_ctx_accesses(struct verifier_env *env) 
			return -EINVAL;

		}



		if (cnt == 1) {

			memcpy(insn, insn_buf, sizeof(*insn));

			continue;

		}



		/* several new insns need to be inserted. Make room for them */

		insn_cnt += cnt - 1;

		new_prog = bpf_prog_realloc(env->prog,

					    bpf_prog_size(insn_cnt),

					    GFP_USER);

		new_prog = bpf_patch_insn_single(env->prog, i, insn_buf, cnt);

		if (!new_prog)

			return -ENOMEM;



		new_prog->len = insn_cnt;



		memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1,

			sizeof(*insn) * (insn_cnt - i - cnt));



		/* copy substitute insns in place of load instruction */

		memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);



		/* adjust branches in the whole program */

		adjust_branches(new_prog, i, cnt - 1);

		insn_delta = cnt - 1;



		/* keep walking new program and skip insns we just inserted */

		env->prog = new_prog;

		insn = new_prog->insnsi + i + cnt - 1;

		i += cnt - 1;

		insn      = new_prog->insnsi + i + insn_delta;



		insn_cnt += insn_delta;

		i        += insn_delta;

	}



	return 0;