Commit f696b8f4 authored May 30, 2017 by Alexei Starovoitov Committed by David S. Miller May 31, 2017

bpf: split bpf core interpreter



split __bpf_prog_run() interpreter into stack allocation and execution parts.
The code section shrinks which helps interpreter performance in some cases.
   text	   data	    bss	    dec	    hex	filename
  26350	  10328	    624	  37302	   91b6	kernel/bpf/core.o.before
  25777	  10328	    624	  36729	   8f79	kernel/bpf/core.o.after

Very short programs got slower (due to extra function call):
Before:
test_bpf: #89 ALU64_ADD_K: 1 + 2 = 3 jited:0 7 PASS
test_bpf: #90 ALU64_ADD_K: 3 + 0 = 3 jited:0 8 PASS
test_bpf: #91 ALU64_ADD_K: 1 + 2147483646 = 2147483647 jited:0 7 PASS
test_bpf: #92 ALU64_ADD_K: 4294967294 + 2 = 4294967296 jited:0 11 PASS
test_bpf: #93 ALU64_ADD_K: 2147483646 + -2147483647 = -1 jited:0 7 PASS
After:
test_bpf: #89 ALU64_ADD_K: 1 + 2 = 3 jited:0 11 PASS
test_bpf: #90 ALU64_ADD_K: 3 + 0 = 3 jited:0 11 PASS
test_bpf: #91 ALU64_ADD_K: 1 + 2147483646 = 2147483647 jited:0 11 PASS
test_bpf: #92 ALU64_ADD_K: 4294967294 + 2 = 4294967296 jited:0 14 PASS
test_bpf: #93 ALU64_ADD_K: 2147483646 + -2147483647 = -1 jited:0 10 PASS

Longer programs got faster:
Before:
test_bpf: #266 BPF_MAXINSNS: Ctx heavy transformations jited:0 20286 20513 PASS
test_bpf: #267 BPF_MAXINSNS: Call heavy transformations jited:0 31853 31768 PASS
test_bpf: #268 BPF_MAXINSNS: Jump heavy test jited:0 9815 PASS
test_bpf: #269 BPF_MAXINSNS: Very long jump backwards jited:0 6 PASS
test_bpf: #270 BPF_MAXINSNS: Edge hopping nuthouse jited:0 13959 PASS
test_bpf: #271 BPF_MAXINSNS: Jump, gap, jump, ... jited:0 210 PASS
test_bpf: #272 BPF_MAXINSNS: ld_abs+get_processor_id jited:0 21724 PASS
test_bpf: #273 BPF_MAXINSNS: ld_abs+vlan_push/pop jited:0 19118 PASS
After:
test_bpf: #266 BPF_MAXINSNS: Ctx heavy transformations jited:0 19008 18827 PASS
test_bpf: #267 BPF_MAXINSNS: Call heavy transformations jited:0 29238 28450 PASS
test_bpf: #268 BPF_MAXINSNS: Jump heavy test jited:0 9485 PASS
test_bpf: #269 BPF_MAXINSNS: Very long jump backwards jited:0 12 PASS
test_bpf: #270 BPF_MAXINSNS: Edge hopping nuthouse jited:0 13257 PASS
test_bpf: #271 BPF_MAXINSNS: Jump, gap, jump, ... jited:0 213 PASS
test_bpf: #272 BPF_MAXINSNS: ld_abs+get_processor_id jited:0 19389 PASS
test_bpf: #273 BPF_MAXINSNS: ld_abs+vlan_push/pop jited:0 19583 PASS

For real world production programs the difference is noise.

This patch is first step towards reducing interpreter stack consumption.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: David S. Miller <davem@davemloft.net>

parent 71189fa9

kernel/bpf/core.c

+14 −7

Original line number	Diff line number	Diff line
		@@ -763,10 +763,10 @@ EXPORT_SYMBOL_GPL(__bpf_call_base);
		*
		* Decode and execute eBPF instructions.
		*/
		static unsigned int __bpf_prog_run(void ctx, const struct bpf_insn insn)
		static unsigned int ___bpf_prog_run(u64 regs, const struct bpf_insn insn,
		u64 *stack)
		{
		u64 stack[MAX_BPF_STACK / sizeof(u64)];
		u64 regs[MAX_BPF_REG], tmp;
		u64 tmp;
		static const void *jumptable[256] = {
		[0 ... 255] = &&default_label,
		/* Now overwrite non-defaults ... */
		@@ -874,9 +874,6 @@ static unsigned int __bpf_prog_run(void ctx, const struct bpf_insn insn)
		#define CONT ({ insn++; goto select_insn; })
		#define CONT_JMP ({ insn++; goto select_insn; })

		FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)];
		ARG1 = (u64) (unsigned long) ctx;

		select_insn:
		goto *jumptable[insn->code];

		@@ -1219,7 +1216,17 @@ static unsigned int __bpf_prog_run(void ctx, const struct bpf_insn insn)
		WARN_RATELIMIT(1, "unknown opcode %02x\n", insn->code);
		return 0;
		}
		STACK_FRAME_NON_STANDARD(__bpf_prog_run); /* jump table */
		STACK_FRAME_NON_STANDARD(___bpf_prog_run); /* jump table */

		static unsigned int __bpf_prog_run(void ctx, const struct bpf_insn insn)
		{
		u64 stack[MAX_BPF_STACK / sizeof(u64)];
		u64 regs[MAX_BPF_REG];

		FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)];
		ARG1 = (u64) (unsigned long) ctx;
		return ___bpf_prog_run(regs, insn, stack);
		}

		bool bpf_prog_array_compatible(struct bpf_array *array,
		const struct bpf_prog *fp)