Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 41a9802f authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'bpf_avoid_clone' 



Alexei Starovoitov says:

====================
bpf: performance improvements

v1->v2: dropped redundant iff_up check in patch 2

At plumbers we discussed different options on how to get rid of skb_clone
from bpf_clone_redirect(), the patch 2 implements the best option.
Patch 1 adds 'integrated exts' to cls_bpf to improve performance by
combining simple actions into bpf classifier.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents f6c53334 27b29f63

include/net/sch_generic.h

+2 −1
Original line number Diff line number Diff line
@@ -251,7 +251,7 @@ struct tcf_proto { 
struct qdisc_skb_cb {

	unsigned int		pkt_len;

	u16			slave_dev_queue_mapping;

	u16			_pad;

	u16			tc_classid;

#define QDISC_CB_PRIV_LEN 20

	unsigned char		data[QDISC_CB_PRIV_LEN];

};
@@ -402,6 +402,7 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, 
			       const struct qdisc_size_table *stab);

bool tcf_destroy(struct tcf_proto *tp, bool force);

void tcf_destroy_chain(struct tcf_proto __rcu **fl);

int skb_do_redirect(struct sk_buff *);



/* Reset all TX qdiscs greater then index of a device.  */

static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i)

include/uapi/linux/bpf.h

+9 −0
Original line number Diff line number Diff line
@@ -272,6 +272,14 @@ enum bpf_func_id { 
	BPF_FUNC_skb_get_tunnel_key,

	BPF_FUNC_skb_set_tunnel_key,

	BPF_FUNC_perf_event_read,	/* u64 bpf_perf_event_read(&map, index) */

	/**

	 * bpf_redirect(ifindex, flags) - redirect to another netdev

	 * @ifindex: ifindex of the net device

	 * @flags: bit 0 - if set, redirect to ingress instead of egress

	 *         other bits - reserved

	 * Return: TC_ACT_REDIRECT

	 */

	BPF_FUNC_redirect,

	__BPF_FUNC_MAX_ID,

};
@@ -293,6 +301,7 @@ struct __sk_buff { 
	__u32 tc_index;

	__u32 cb[5];

	__u32 hash;

	__u32 tc_classid;

};



struct bpf_tunnel_key {

include/uapi/linux/pkt_cls.h

+4 −0
Original line number Diff line number Diff line
@@ -87,6 +87,7 @@ enum { 
#define TC_ACT_STOLEN		4

#define TC_ACT_QUEUED		5

#define TC_ACT_REPEAT		6

#define TC_ACT_REDIRECT		7

#define TC_ACT_JUMP		0x10000000



/* Action type identifiers*/
@@ -373,6 +374,8 @@ enum { 


/* BPF classifier */



#define TCA_BPF_FLAG_ACT_DIRECT		(1 << 0)



enum {

	TCA_BPF_UNSPEC,

	TCA_BPF_ACT,
@@ -382,6 +385,7 @@ enum { 
	TCA_BPF_OPS,

	TCA_BPF_FD,

	TCA_BPF_NAME,

	TCA_BPF_FLAGS,

	__TCA_BPF_MAX,

};

net/core/dev.c

+8 −0
Original line number Diff line number Diff line
@@ -3670,6 +3670,14 @@ static inline struct sk_buff *handle_ing(struct sk_buff *skb, 
	case TC_ACT_QUEUED:

		kfree_skb(skb);

		return NULL;

	case TC_ACT_REDIRECT:

		/* skb_mac_header check was done by cls/act_bpf, so

		 * we can safely push the L2 header back before

		 * redirecting to another netdev

		 */

		__skb_push(skb, skb->mac_len);

		skb_do_redirect(skb);

		return NULL;

	default:

		break;

	}

net/core/filter.c

+58 −0
Original line number Diff line number Diff line
@@ -1427,6 +1427,48 @@ const struct bpf_func_proto bpf_clone_redirect_proto = { 
	.arg3_type      = ARG_ANYTHING,

};



struct redirect_info {

	u32 ifindex;

	u32 flags;

};



static DEFINE_PER_CPU(struct redirect_info, redirect_info);

static u64 bpf_redirect(u64 ifindex, u64 flags, u64 r3, u64 r4, u64 r5)

{

	struct redirect_info *ri = this_cpu_ptr(&redirect_info);



	ri->ifindex = ifindex;

	ri->flags = flags;

	return TC_ACT_REDIRECT;

}



int skb_do_redirect(struct sk_buff *skb)

{

	struct redirect_info *ri = this_cpu_ptr(&redirect_info);

	struct net_device *dev;



	dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->ifindex);

	ri->ifindex = 0;

	if (unlikely(!dev)) {

		kfree_skb(skb);

		return -EINVAL;

	}



	if (BPF_IS_REDIRECT_INGRESS(ri->flags))

		return dev_forward_skb(dev, skb);



	skb->dev = dev;

	return dev_queue_xmit(skb);

}



const struct bpf_func_proto bpf_redirect_proto = {

	.func           = bpf_redirect,

	.gpl_only       = false,

	.ret_type       = RET_INTEGER,

	.arg1_type      = ARG_ANYTHING,

	.arg2_type      = ARG_ANYTHING,

};



static u64 bpf_get_cgroup_classid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)

{

	return task_get_classid((struct sk_buff *) (unsigned long) r1);
@@ -1607,6 +1649,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id) 
		return &bpf_skb_get_tunnel_key_proto;

	case BPF_FUNC_skb_set_tunnel_key:

		return bpf_get_skb_set_tunnel_key_proto();

	case BPF_FUNC_redirect:

		return &bpf_redirect_proto;

	default:

		return sk_filter_func_proto(func_id);

	}
@@ -1632,6 +1676,9 @@ static bool __is_valid_access(int off, int size, enum bpf_access_type type) 
static bool sk_filter_is_valid_access(int off, int size,

				      enum bpf_access_type type)

{

	if (off == offsetof(struct __sk_buff, tc_classid))

		return false;



	if (type == BPF_WRITE) {

		switch (off) {

		case offsetof(struct __sk_buff, cb[0]) ...
@@ -1648,6 +1695,9 @@ static bool sk_filter_is_valid_access(int off, int size, 
static bool tc_cls_act_is_valid_access(int off, int size,

				       enum bpf_access_type type)

{

	if (off == offsetof(struct __sk_buff, tc_classid))

		return type == BPF_WRITE ? true : false;



	if (type == BPF_WRITE) {

		switch (off) {

		case offsetof(struct __sk_buff, mark):
@@ -1760,6 +1810,14 @@ static u32 bpf_net_convert_ctx_access(enum bpf_access_type type, int dst_reg, 
			*insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, ctx_off);

		break;



	case offsetof(struct __sk_buff, tc_classid):

		ctx_off -= offsetof(struct __sk_buff, tc_classid);

		ctx_off += offsetof(struct sk_buff, cb);

		ctx_off += offsetof(struct qdisc_skb_cb, tc_classid);

		WARN_ON(type != BPF_WRITE);

		*insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, ctx_off);

		break;



	case offsetof(struct __sk_buff, tc_index):

#ifdef CONFIG_NET_SCHED

		BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tc_index) != 2);