libbpf: add support for bpf_call (48cca7e4) · Commits · e / devices / android_kernel_teracube_emerald

tools/include/uapi/linux/bpf.h

+6 −0

Original line number	Original line	Diff line number	Diff line
	@@ -197,8 +197,14 @@ enum bpf_attach_type {
	*/		*/
	#define BPF_F_STRICT_ALIGNMENT (1U << 0)		#define BPF_F_STRICT_ALIGNMENT (1U << 0)

			/* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */
	#define BPF_PSEUDO_MAP_FD 1		#define BPF_PSEUDO_MAP_FD 1

			/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
			* offset to another bpf function
			*/
			#define BPF_PSEUDO_CALL 1

	/* flags for BPF_MAP_UPDATE_ELEM command */		/* flags for BPF_MAP_UPDATE_ELEM command */
	#define BPF_ANY 0 /* create new element or update existing */		#define BPF_ANY 0 /* create new element or update existing */
	#define BPF_NOEXIST 1 /* create new element if it didn't exist */		#define BPF_NOEXIST 1 /* create new element if it didn't exist */

tools/lib/bpf/bpf.h

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -40,7 +40,7 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
	__u32 map_flags);		__u32 map_flags);

	/* Recommend log buffer size */		/* Recommend log buffer size */
	#define BPF_LOG_BUF_SIZE 65536		#define BPF_LOG_BUF_SIZE (256 * 1024)
	int bpf_load_program_name(enum bpf_prog_type type, const char *name,		int bpf_load_program_name(enum bpf_prog_type type, const char *name,
	const struct bpf_insn *insns,		const struct bpf_insn *insns,
	size_t insns_cnt, const char *license,		size_t insns_cnt, const char *license,

tools/lib/bpf/libbpf.c

+127 −43

Original line number	Original line	Diff line number	Diff line
	@@ -174,12 +174,19 @@ struct bpf_program {
	char *name;		char *name;
	char *section_name;		char *section_name;
	struct bpf_insn *insns;		struct bpf_insn *insns;
	size_t insns_cnt;		size_t insns_cnt, main_prog_cnt;
	enum bpf_prog_type type;		enum bpf_prog_type type;

	struct {		struct reloc_desc {
			enum {
			RELO_LD64,
			RELO_CALL,
			} type;
	int insn_idx;		int insn_idx;
			union {
	int map_idx;		int map_idx;
			int text_off;
			};
	} *reloc_desc;		} *reloc_desc;
	int nr_reloc;		int nr_reloc;

	@@ -234,6 +241,7 @@ struct bpf_object {
	} *reloc;		} *reloc;
	int nr_reloc;		int nr_reloc;
	int maps_shndx;		int maps_shndx;
			int text_shndx;
	} efile;		} efile;
	/*		/*
	* All loaded bpf_object is linked in a list, which is		* All loaded bpf_object is linked in a list, which is
	@@ -375,9 +383,13 @@ bpf_object__init_prog_names(struct bpf_object *obj)
	size_t pi, si;		size_t pi, si;

	for (pi = 0; pi < obj->nr_programs; pi++) {		for (pi = 0; pi < obj->nr_programs; pi++) {
	char *name = NULL;		const char *name = NULL;

	prog = &obj->programs[pi];		prog = &obj->programs[pi];
			if (prog->idx == obj->efile.text_shndx) {
			name = ".text";
			goto skip_search;
			}

	for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;		for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
	si++) {		si++) {
	@@ -405,7 +417,7 @@ bpf_object__init_prog_names(struct bpf_object *obj)
	prog->section_name);		prog->section_name);
	return -EINVAL;		return -EINVAL;
	}		}
			skip_search:
	prog->name = strdup(name);		prog->name = strdup(name);
	if (!prog->name) {		if (!prog->name) {
	pr_warning("failed to allocate memory for prog sym %s\n",		pr_warning("failed to allocate memory for prog sym %s\n",
	@@ -795,6 +807,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
	} else if ((sh.sh_type == SHT_PROGBITS) &&		} else if ((sh.sh_type == SHT_PROGBITS) &&
	(sh.sh_flags & SHF_EXECINSTR) &&		(sh.sh_flags & SHF_EXECINSTR) &&
	(data->d_size > 0)) {		(data->d_size > 0)) {
			if (strcmp(name, ".text") == 0)
			obj->efile.text_shndx = idx;
	err = bpf_object__add_program(obj, data->d_buf,		err = bpf_object__add_program(obj, data->d_buf,
	data->d_size, name, idx);		data->d_size, name, idx);
	if (err) {		if (err) {
	@@ -856,11 +870,14 @@ bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
	}		}

	static int		static int
	bpf_program__collect_reloc(struct bpf_program *prog,		bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr,
	size_t nr_maps, GElf_Shdr *shdr,		Elf_Data data, struct bpf_object obj)
	Elf_Data data, Elf_Data symbols,
	int maps_shndx, struct bpf_map *maps)
	{		{
			Elf_Data *symbols = obj->efile.symbols;
			int text_shndx = obj->efile.text_shndx;
			int maps_shndx = obj->efile.maps_shndx;
			struct bpf_map *maps = obj->maps;
			size_t nr_maps = obj->nr_maps;
	int i, nrels;		int i, nrels;

	pr_debug("collecting relocating info for: '%s'\n",		pr_debug("collecting relocating info for: '%s'\n",
	@@ -893,8 +910,10 @@ bpf_program__collect_reloc(struct bpf_program *prog,
	GELF_R_SYM(rel.r_info));		GELF_R_SYM(rel.r_info));
	return -LIBBPF_ERRNO__FORMAT;		return -LIBBPF_ERRNO__FORMAT;
	}		}
			pr_debug("relo for %ld value %ld name %d\n",
			rel.r_info >> 32, sym.st_value, sym.st_name);

	if (sym.st_shndx != maps_shndx) {		if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
	pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",		pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
	prog->section_name, sym.st_shndx);		prog->section_name, sym.st_shndx);
	return -LIBBPF_ERRNO__RELOC;		return -LIBBPF_ERRNO__RELOC;
	@@ -903,6 +922,17 @@ bpf_program__collect_reloc(struct bpf_program *prog,
	insn_idx = rel.r_offset / sizeof(struct bpf_insn);		insn_idx = rel.r_offset / sizeof(struct bpf_insn);
	pr_debug("relocation: insn_idx=%u\n", insn_idx);		pr_debug("relocation: insn_idx=%u\n", insn_idx);

			if (insns[insn_idx].code == (BPF_JMP \| BPF_CALL)) {
			if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
			pr_warning("incorrect bpf_call opcode\n");
			return -LIBBPF_ERRNO__RELOC;
			}
			prog->reloc_desc[i].type = RELO_CALL;
			prog->reloc_desc[i].insn_idx = insn_idx;
			prog->reloc_desc[i].text_off = sym.st_value;
			continue;
			}

	if (insns[insn_idx].code != (BPF_LD \| BPF_IMM \| BPF_DW)) {		if (insns[insn_idx].code != (BPF_LD \| BPF_IMM \| BPF_DW)) {
	pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",		pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
	insn_idx, insns[insn_idx].code);		insn_idx, insns[insn_idx].code);
	@@ -924,6 +954,7 @@ bpf_program__collect_reloc(struct bpf_program *prog,
	return -LIBBPF_ERRNO__RELOC;		return -LIBBPF_ERRNO__RELOC;
	}		}

			prog->reloc_desc[i].type = RELO_LD64;
	prog->reloc_desc[i].insn_idx = insn_idx;		prog->reloc_desc[i].insn_idx = insn_idx;
	prog->reloc_desc[i].map_idx = map_idx;		prog->reloc_desc[i].map_idx = map_idx;
	}		}
	@@ -962,17 +993,60 @@ bpf_object__create_maps(struct bpf_object *obj)
	return 0;		return 0;
	}		}

			static int
			bpf_program__reloc_text(struct bpf_program prog, struct bpf_object obj,
			struct reloc_desc *relo)
			{
			struct bpf_insn insn, new_insn;
			struct bpf_program *text;
			size_t new_cnt;

			if (relo->type != RELO_CALL)
			return -LIBBPF_ERRNO__RELOC;

			if (prog->idx == obj->efile.text_shndx) {
			pr_warning("relo in .text insn %d into off %d\n",
			relo->insn_idx, relo->text_off);
			return -LIBBPF_ERRNO__RELOC;
			}

			if (prog->main_prog_cnt == 0) {
			text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
			if (!text) {
			pr_warning("no .text section found yet relo into text exist\n");
			return -LIBBPF_ERRNO__RELOC;
			}
			new_cnt = prog->insns_cnt + text->insns_cnt;
			new_insn = realloc(prog->insns, new_cnt * sizeof(*insn));
			if (!new_insn) {
			pr_warning("oom in prog realloc\n");
			return -ENOMEM;
			}
			memcpy(new_insn + prog->insns_cnt, text->insns,
			text->insns_cnt * sizeof(*insn));
			prog->insns = new_insn;
			prog->main_prog_cnt = prog->insns_cnt;
			prog->insns_cnt = new_cnt;
			}
			insn = &prog->insns[relo->insn_idx];
			insn->imm += prog->main_prog_cnt - relo->insn_idx;
			pr_debug("added %zd insn from %s to prog %s\n",
			text->insns_cnt, text->section_name, prog->section_name);
			return 0;
			}

	static int		static int
	bpf_program__relocate(struct bpf_program prog, struct bpf_object obj)		bpf_program__relocate(struct bpf_program prog, struct bpf_object obj)
	{		{
	int i;		int i, err;

	if (!prog \|\| !prog->reloc_desc)		if (!prog \|\| !prog->reloc_desc)
	return 0;		return 0;

	for (i = 0; i < prog->nr_reloc; i++) {		for (i = 0; i < prog->nr_reloc; i++) {
	int insn_idx, map_idx;		if (prog->reloc_desc[i].type == RELO_LD64) {
	struct bpf_insn *insns = prog->insns;		struct bpf_insn *insns = prog->insns;
			int insn_idx, map_idx;

	insn_idx = prog->reloc_desc[i].insn_idx;		insn_idx = prog->reloc_desc[i].insn_idx;
	map_idx = prog->reloc_desc[i].map_idx;		map_idx = prog->reloc_desc[i].map_idx;
	@@ -984,6 +1058,12 @@ bpf_program__relocate(struct bpf_program prog, struct bpf_object obj)
	}		}
	insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;		insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
	insns[insn_idx].imm = obj->maps[map_idx].fd;		insns[insn_idx].imm = obj->maps[map_idx].fd;
			} else {
			err = bpf_program__reloc_text(prog, obj,
			&prog->reloc_desc[i]);
			if (err)
			return err;
			}
	}		}

	zfree(&prog->reloc_desc);		zfree(&prog->reloc_desc);
	@@ -1026,7 +1106,6 @@ static int bpf_object__collect_reloc(struct bpf_object *obj)
	Elf_Data *data = obj->efile.reloc[i].data;		Elf_Data *data = obj->efile.reloc[i].data;
	int idx = shdr->sh_info;		int idx = shdr->sh_info;
	struct bpf_program *prog;		struct bpf_program *prog;
	size_t nr_maps = obj->nr_maps;

	if (shdr->sh_type != SHT_REL) {		if (shdr->sh_type != SHT_REL) {
	pr_warning("internal error at %d\n", __LINE__);		pr_warning("internal error at %d\n", __LINE__);
	@@ -1040,11 +1119,9 @@ static int bpf_object__collect_reloc(struct bpf_object *obj)
	return -LIBBPF_ERRNO__RELOC;		return -LIBBPF_ERRNO__RELOC;
	}		}

	err = bpf_program__collect_reloc(prog, nr_maps,		err = bpf_program__collect_reloc(prog,
	shdr, data,		shdr, data,
	obj->efile.symbols,		obj);
	obj->efile.maps_shndx,
	obj->maps);
	if (err)		if (err)
	return err;		return err;
	}		}
	@@ -1197,6 +1274,8 @@ bpf_object__load_progs(struct bpf_object *obj)
	int err;		int err;

	for (i = 0; i < obj->nr_programs; i++) {		for (i = 0; i < obj->nr_programs; i++) {
			if (obj->programs[i].idx == obj->efile.text_shndx)
			continue;
	err = bpf_program__load(&obj->programs[i],		err = bpf_program__load(&obj->programs[i],
	obj->license,		obj->license,
	obj->kern_version);		obj->kern_version);
	@@ -1859,7 +1938,7 @@ long libbpf_get_error(const void *ptr)
	int bpf_prog_load(const char *file, enum bpf_prog_type type,		int bpf_prog_load(const char *file, enum bpf_prog_type type,
	struct bpf_object *pobj, int prog_fd)		struct bpf_object *pobj, int prog_fd)
	{		{
	struct bpf_program *prog;		struct bpf_program prog, first_prog = NULL;
	struct bpf_object *obj;		struct bpf_object *obj;
	int err;		int err;

	@@ -1867,12 +1946,7 @@ int bpf_prog_load(const char *file, enum bpf_prog_type type,
	if (IS_ERR(obj))		if (IS_ERR(obj))
	return -ENOENT;		return -ENOENT;

	prog = bpf_program__next(NULL, obj);		bpf_object__for_each_program(prog, obj) {
	if (!prog) {
	bpf_object__close(obj);
	return -ENOENT;
	}

	/*		/*
	* If type is not specified, try to guess it based on		* If type is not specified, try to guess it based on
	* section name.		* section name.
	@@ -1886,6 +1960,16 @@ int bpf_prog_load(const char *file, enum bpf_prog_type type,
	}		}

	bpf_program__set_type(prog, type);		bpf_program__set_type(prog, type);
			if (prog->idx != obj->efile.text_shndx && !first_prog)
			first_prog = prog;
			}

			if (!first_prog) {
			pr_warning("object file doesn't contain bpf program\n");
			bpf_object__close(obj);
			return -ENOENT;
			}

	err = bpf_object__load(obj);		err = bpf_object__load(obj);
	if (err) {		if (err) {
	bpf_object__close(obj);		bpf_object__close(obj);
	@@ -1893,6 +1977,6 @@ int bpf_prog_load(const char *file, enum bpf_prog_type type,
	}		}

	*pobj = obj;		*pobj = obj;
	*prog_fd = bpf_program__fd(prog);		*prog_fd = bpf_program__fd(first_prog);
	return 0;		return 0;
	}		}