Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6ee79b6e authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'net-mini_Qdisc' 



Jiri Pirko says:

====================
net: core: introduce mini_Qdisc and eliminate usage of tp->q for clsact fastpath

This patchset's main patch is patch number 2. It carries the
description. Patch 1 is just a dependency.

---
v3->v4:
- rebased to be applicable on top of the current net-next
v2->v3:
- Using head change callback to replace miniq pointer every time tp head
  changes. This eliminates one rcu dereference and makes the claim "without
  added overhead" valid.
v1->v2:
- Use dev instead of skb->dev in sch_handle_egress as pointed out by Daniel
- Fixed synchronize_rcu_bh() in mini_qdisc_disable and commented
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents aa5fbf07 46209401

include/linux/netdevice.h

+6 −3
Original line number Diff line number Diff line
@@ -1559,6 +1559,8 @@ enum netdev_priv_flags { 
 *

 *	@rx_handler:		handler for received packets

 *	@rx_handler_data: 	XXX: need comments on this one

 *	@miniq_ingress:		ingress/clsact qdisc specific data for

 *				ingress processing

 *	@ingress_queue:		XXX: need comments on this one

 *	@broadcast:		hw bcast address

 *
@@ -1576,7 +1578,8 @@ enum netdev_priv_flags { 
 *	@tx_global_lock: 	XXX: need comments on this one

 *

 *	@xps_maps:	XXX: need comments on this one

 *

 *	@miniq_egress:		clsact qdisc specific data for

 *				egress processing

 *	@watchdog_timeo:	Represents the timeout that is used by

 *				the watchdog (see dev_watchdog())

 *	@watchdog_timer:	List of timers
@@ -1795,7 +1798,7 @@ struct net_device { 
	void __rcu		*rx_handler_data;



#ifdef CONFIG_NET_CLS_ACT

	struct tcf_proto __rcu  *ingress_cl_list;

	struct mini_Qdisc __rcu	*miniq_ingress;

#endif

	struct netdev_queue __rcu *ingress_queue;

#ifdef CONFIG_NETFILTER_INGRESS
@@ -1826,7 +1829,7 @@ struct net_device { 
	struct xps_dev_maps __rcu *xps_maps;

#endif

#ifdef CONFIG_NET_CLS_ACT

	struct tcf_proto __rcu  *egress_cl_list;

	struct mini_Qdisc __rcu	*miniq_egress;

#endif



	/* These may be needed for future network-power-down code. */

include/net/pkt_cls.h

+6 −8
Original line number Diff line number Diff line
@@ -26,6 +26,8 @@ enum tcf_block_binder_type { 


struct tcf_block_ext_info {

	enum tcf_block_binder_type binder_type;

	tcf_chain_head_change_t *chain_head_change;

	void *chain_head_change_priv;

};



struct tcf_block_cb;
@@ -37,12 +39,10 @@ struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index, 
void tcf_chain_put(struct tcf_chain *chain);

int tcf_block_get(struct tcf_block **p_block,

		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q);

int tcf_block_get_ext(struct tcf_block **p_block,

		      struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,

int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,

		      struct tcf_block_ext_info *ei);

void tcf_block_put(struct tcf_block *block);

void tcf_block_put_ext(struct tcf_block *block,

		       struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,

void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,

		       struct tcf_block_ext_info *ei);



static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
@@ -82,8 +82,7 @@ int tcf_block_get(struct tcf_block **p_block, 
}



static inline

int tcf_block_get_ext(struct tcf_block **p_block,

		      struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,

int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,

		      struct tcf_block_ext_info *ei)

{

	return 0;
@@ -94,8 +93,7 @@ static inline void tcf_block_put(struct tcf_block *block) 
}



static inline

void tcf_block_put_ext(struct tcf_block *block,

		       struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,

void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,

		       struct tcf_block_ext_info *ei)

{

}

include/net/sch_generic.h

+36 −1
Original line number Diff line number Diff line
@@ -260,9 +260,12 @@ struct qdisc_skb_cb { 
	unsigned char		data[QDISC_CB_PRIV_LEN];

};



typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);



struct tcf_chain {

	struct tcf_proto __rcu *filter_chain;

	struct tcf_proto __rcu **p_filter_chain;

	tcf_chain_head_change_t *chain_head_change;

	void *chain_head_change_priv;

	struct list_head list;

	struct tcf_block *block;

	u32 index; /* chain index */
@@ -901,4 +904,36 @@ static inline void psched_ratecfg_getrate(struct tc_ratespec *res, 
	res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);

}



/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.

 * The fast path only needs to access filter list and to update stats

 */

struct mini_Qdisc {

	struct tcf_proto *filter_list;

	struct gnet_stats_basic_cpu __percpu *cpu_bstats;

	struct gnet_stats_queue	__percpu *cpu_qstats;

	struct rcu_head rcu;

};



static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq,

						const struct sk_buff *skb)

{

	bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb);

}



static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq)

{

	this_cpu_inc(miniq->cpu_qstats->drops);

}



struct mini_Qdisc_pair {

	struct mini_Qdisc miniq1;

	struct mini_Qdisc miniq2;

	struct mini_Qdisc __rcu **p_miniq;

};



void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,

			  struct tcf_proto *tp_head);

void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,

			  struct mini_Qdisc __rcu **p_miniq);



#endif

net/core/dev.c

+11 −10
Original line number Diff line number Diff line
@@ -3274,22 +3274,22 @@ EXPORT_SYMBOL(dev_loopback_xmit); 
static struct sk_buff *

sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)

{

	struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);

	struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);

	struct tcf_result cl_res;



	if (!cl)

	if (!miniq)

		return skb;



	/* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */

	qdisc_bstats_cpu_update(cl->q, skb);

	mini_qdisc_bstats_cpu_update(miniq, skb);



	switch (tcf_classify(skb, cl, &cl_res, false)) {

	switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {

	case TC_ACT_OK:

	case TC_ACT_RECLASSIFY:

		skb->tc_index = TC_H_MIN(cl_res.classid);

		break;

	case TC_ACT_SHOT:

		qdisc_qstats_cpu_drop(cl->q);

		mini_qdisc_qstats_cpu_drop(miniq);

		*ret = NET_XMIT_DROP;

		kfree_skb(skb);

		return NULL;
@@ -4189,7 +4189,7 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 
		   struct net_device *orig_dev)

{

#ifdef CONFIG_NET_CLS_ACT

	struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);

	struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);

	struct tcf_result cl_res;



	/* If there's at least one ingress present somewhere (so
@@ -4197,8 +4197,9 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 
	 * that are not configured with an ingress qdisc will bail

	 * out here.

	 */

	if (!cl)

	if (!miniq)

		return skb;



	if (*pt_prev) {

		*ret = deliver_skb(skb, *pt_prev, orig_dev);

		*pt_prev = NULL;
@@ -4206,15 +4207,15 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 


	qdisc_skb_cb(skb)->pkt_len = skb->len;

	skb->tc_at_ingress = 1;

	qdisc_bstats_cpu_update(cl->q, skb);

	mini_qdisc_bstats_cpu_update(miniq, skb);



	switch (tcf_classify(skb, cl, &cl_res, false)) {

	switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {

	case TC_ACT_OK:

	case TC_ACT_RECLASSIFY:

		skb->tc_index = TC_H_MIN(cl_res.classid);

		break;

	case TC_ACT_SHOT:

		qdisc_qstats_cpu_drop(cl->q);

		mini_qdisc_qstats_cpu_drop(miniq);

		kfree_skb(skb);

		return NULL;

	case TC_ACT_STOLEN:

net/sched/cls_api.c

+32 −22
Original line number Diff line number Diff line
@@ -195,12 +195,19 @@ static struct tcf_chain *tcf_chain_create(struct tcf_block *block, 
	return chain;

}



static void tcf_chain_head_change(struct tcf_chain *chain,

				  struct tcf_proto *tp_head)

{

	if (chain->chain_head_change)

		chain->chain_head_change(tp_head,

					 chain->chain_head_change_priv);

}



static void tcf_chain_flush(struct tcf_chain *chain)

{

	struct tcf_proto *tp;



	if (chain->p_filter_chain)

		RCU_INIT_POINTER(*chain->p_filter_chain, NULL);

	tcf_chain_head_change(chain, NULL);

	while ((tp = rtnl_dereference(chain->filter_chain)) != NULL) {

		RCU_INIT_POINTER(chain->filter_chain, tp->next);

		tcf_chain_put(chain);
@@ -242,13 +249,6 @@ void tcf_chain_put(struct tcf_chain *chain) 
}

EXPORT_SYMBOL(tcf_chain_put);



static void

tcf_chain_filter_chain_ptr_set(struct tcf_chain *chain,

			       struct tcf_proto __rcu **p_filter_chain)

{

	chain->p_filter_chain = p_filter_chain;

}



static void tcf_block_offload_cmd(struct tcf_block *block, struct Qdisc *q,

				  struct tcf_block_ext_info *ei,

				  enum tc_block_command command)
@@ -276,8 +276,7 @@ static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q, 
	tcf_block_offload_cmd(block, q, ei, TC_BLOCK_UNBIND);

}



int tcf_block_get_ext(struct tcf_block **p_block,

		      struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,

int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,

		      struct tcf_block_ext_info *ei)

{

	struct tcf_block *block = kzalloc(sizeof(*block), GFP_KERNEL);
@@ -295,7 +294,9 @@ int tcf_block_get_ext(struct tcf_block **p_block, 
		err = -ENOMEM;

		goto err_chain_create;

	}

	tcf_chain_filter_chain_ptr_set(chain, p_filter_chain);

	WARN_ON(!ei->chain_head_change);

	chain->chain_head_change = ei->chain_head_change;

	chain->chain_head_change_priv = ei->chain_head_change_priv;

	block->net = qdisc_net(q);

	block->q = q;

	tcf_block_offload_bind(block, q, ei);
@@ -308,12 +309,23 @@ int tcf_block_get_ext(struct tcf_block **p_block, 
}

EXPORT_SYMBOL(tcf_block_get_ext);



static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)

{

	struct tcf_proto __rcu **p_filter_chain = priv;



	rcu_assign_pointer(*p_filter_chain, tp_head);

}



int tcf_block_get(struct tcf_block **p_block,

		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q)

{

	struct tcf_block_ext_info ei = {0, };

	struct tcf_block_ext_info ei = {

		.chain_head_change = tcf_chain_head_change_dflt,

		.chain_head_change_priv = p_filter_chain,

	};



	return tcf_block_get_ext(p_block, p_filter_chain, q, &ei);

	WARN_ON(!p_filter_chain);

	return tcf_block_get_ext(p_block, q, &ei);

}

EXPORT_SYMBOL(tcf_block_get);
@@ -334,8 +346,7 @@ static void tcf_block_put_final(struct work_struct *work) 
 * actions should be all removed after flushing. However, filters are now

 * destroyed in tc filter workqueue with RTNL lock, they can not race here.

 */

void tcf_block_put_ext(struct tcf_block *block,

		       struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,

void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,

		       struct tcf_block_ext_info *ei)

{

	struct tcf_chain *chain, *tmp;
@@ -361,7 +372,7 @@ void tcf_block_put(struct tcf_block *block) 


	if (!block)

		return;

	tcf_block_put_ext(block, NULL, block->q, &ei);

	tcf_block_put_ext(block, block->q, &ei);

}



EXPORT_SYMBOL(tcf_block_put);
@@ -537,9 +548,8 @@ static void tcf_chain_tp_insert(struct tcf_chain *chain, 
				struct tcf_chain_info *chain_info,

				struct tcf_proto *tp)

{

	if (chain->p_filter_chain &&

	    *chain_info->pprev == chain->filter_chain)

		rcu_assign_pointer(*chain->p_filter_chain, tp);

	if (*chain_info->pprev == chain->filter_chain)

		tcf_chain_head_change(chain, tp);

	RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain_info));

	rcu_assign_pointer(*chain_info->pprev, tp);

	tcf_chain_hold(chain);
@@ -551,8 +561,8 @@ static void tcf_chain_tp_remove(struct tcf_chain *chain, 
{

	struct tcf_proto *next = rtnl_dereference(chain_info->next);



	if (chain->p_filter_chain && tp == chain->filter_chain)

		RCU_INIT_POINTER(*chain->p_filter_chain, next);

	if (tp == chain->filter_chain)

		tcf_chain_head_change(chain, next);

	RCU_INIT_POINTER(*chain_info->pprev, next);

	tcf_chain_put(chain);

}