Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8b21e230 authored by John Fastabend's avatar John Fastabend Committed by David S. Miller
Browse files

net: sched: rcu'ify cls_rsvp 



Signed-off-by: default avatarJohn Fastabend <john.r.fastabend@intel.com>
Acked-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 8f787cd1

net/sched/cls_rsvp.h

+90 −70
Original line number Original line Diff line number Diff line
@@ -70,22 +70,24 @@ struct rsvp_head { 
	u32			tmap[256/32];

	u32			tmap[256/32];

	u32			hgenerator;

	u32			hgenerator;

	u8			tgenerator;

	u8			tgenerator;

	struct rsvp_session	*ht[256];

	struct rsvp_session __rcu *ht[256];

	struct rcu_head		rcu;

};

};





struct rsvp_session {

struct rsvp_session {

	struct rsvp_session	*next;

	struct rsvp_session __rcu	*next;

	__be32				dst[RSVP_DST_LEN];

	__be32				dst[RSVP_DST_LEN];

	struct tc_rsvp_gpi		dpi;

	struct tc_rsvp_gpi		dpi;

	u8				protocol;

	u8				protocol;

	u8				tunnelid;

	u8				tunnelid;

	/* 16 (src,sport) hash slots, and one wildcard source slot */

	/* 16 (src,sport) hash slots, and one wildcard source slot */

	struct rsvp_filter	*ht[16 + 1];

	struct rsvp_filter __rcu	*ht[16 + 1];

	struct rcu_head			rcu;

};

};









struct rsvp_filter {

struct rsvp_filter {

	struct rsvp_filter	*next;

	struct rsvp_filter __rcu	*next;

	__be32				src[RSVP_DST_LEN];

	__be32				src[RSVP_DST_LEN];

	struct tc_rsvp_gpi		spi;

	struct tc_rsvp_gpi		spi;

	u8				tunnelhdr;

	u8				tunnelhdr;
@@ -95,6 +97,7 @@ struct rsvp_filter { 




	u32				handle;

	u32				handle;

	struct rsvp_session		*sess;

	struct rsvp_session		*sess;

	struct rcu_head			rcu;

};

};





static inline unsigned int hash_dst(__be32 *dst, u8 protocol, u8 tunnelid)

static inline unsigned int hash_dst(__be32 *dst, u8 protocol, u8 tunnelid)
@@ -128,7 +131,7 @@ static inline unsigned int hash_src(__be32 *src) 
static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp,

static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp,

			 struct tcf_result *res)

			 struct tcf_result *res)

{

{

	struct rsvp_session **sht = ((struct rsvp_head *)tp->root)->ht;

	struct rsvp_head *head = rcu_dereference_bh(tp->root);

	struct rsvp_session *s;

	struct rsvp_session *s;

	struct rsvp_filter *f;

	struct rsvp_filter *f;

	unsigned int h1, h2;

	unsigned int h1, h2;
@@ -169,7 +172,8 @@ static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp, 
	h1 = hash_dst(dst, protocol, tunnelid);

	h1 = hash_dst(dst, protocol, tunnelid);

	h2 = hash_src(src);

	h2 = hash_src(src);





	for (s = sht[h1]; s; s = s->next) {

	for (s = rcu_dereference_bh(head->ht[h1]); s;

	     s = rcu_dereference_bh(s->next)) {

		if (dst[RSVP_DST_LEN-1] == s->dst[RSVP_DST_LEN - 1] &&

		if (dst[RSVP_DST_LEN-1] == s->dst[RSVP_DST_LEN - 1] &&

		    protocol == s->protocol &&

		    protocol == s->protocol &&

		    !(s->dpi.mask &

		    !(s->dpi.mask &
@@ -181,7 +185,8 @@ static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp, 
#endif

#endif

		    tunnelid == s->tunnelid) {

		    tunnelid == s->tunnelid) {





			for (f = s->ht[h2]; f; f = f->next) {

			for (f = rcu_dereference_bh(s->ht[h2]); f;

			     f = rcu_dereference_bh(f->next)) {

				if (src[RSVP_DST_LEN-1] == f->src[RSVP_DST_LEN - 1] &&

				if (src[RSVP_DST_LEN-1] == f->src[RSVP_DST_LEN - 1] &&

				    !(f->spi.mask & (*(u32 *)(xprt + f->spi.offset) ^ f->spi.key))

				    !(f->spi.mask & (*(u32 *)(xprt + f->spi.offset) ^ f->spi.key))

#if RSVP_DST_LEN == 4

#if RSVP_DST_LEN == 4
@@ -205,7 +210,8 @@ static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp, 
			}

			}





			/* And wildcard bucket... */

			/* And wildcard bucket... */

			for (f = s->ht[16]; f; f = f->next) {

			for (f = rcu_dereference_bh(s->ht[16]); f;

			     f = rcu_dereference_bh(f->next)) {

				*res = f->res;

				*res = f->res;

				RSVP_APPLY_RESULT();

				RSVP_APPLY_RESULT();

				goto matched;

				goto matched;
@@ -218,7 +224,7 @@ static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp, 




static unsigned long rsvp_get(struct tcf_proto *tp, u32 handle)

static unsigned long rsvp_get(struct tcf_proto *tp, u32 handle)

{

{

	struct rsvp_session **sht = ((struct rsvp_head *)tp->root)->ht;

	struct rsvp_head *head = rtnl_dereference(tp->root);

	struct rsvp_session *s;

	struct rsvp_session *s;

	struct rsvp_filter *f;

	struct rsvp_filter *f;

	unsigned int h1 = handle & 0xFF;

	unsigned int h1 = handle & 0xFF;
@@ -227,8 +233,10 @@ static unsigned long rsvp_get(struct tcf_proto *tp, u32 handle) 
	if (h2 > 16)

	if (h2 > 16)

		return 0;

		return 0;





	for (s = sht[h1]; s; s = s->next) {

	for (s = rtnl_dereference(head->ht[h1]); s;

		for (f = s->ht[h2]; f; f = f->next) {

	     s = rtnl_dereference(s->next)) {

		for (f = rtnl_dereference(s->ht[h2]); f;

		     f = rtnl_dereference(f->next)) {

			if (f->handle == handle)

			if (f->handle == handle)

				return (unsigned long)f;

				return (unsigned long)f;

		}

		}
@@ -246,7 +254,7 @@ static int rsvp_init(struct tcf_proto *tp) 




	data = kzalloc(sizeof(struct rsvp_head), GFP_KERNEL);

	data = kzalloc(sizeof(struct rsvp_head), GFP_KERNEL);

	if (data) {

	if (data) {

		tp->root = data;

		rcu_assign_pointer(tp->root, data);

		return 0;

		return 0;

	}

	}

	return -ENOBUFS;

	return -ENOBUFS;
@@ -257,53 +265,54 @@ rsvp_delete_filter(struct tcf_proto *tp, struct rsvp_filter *f) 
{

{

	tcf_unbind_filter(tp, &f->res);

	tcf_unbind_filter(tp, &f->res);

	tcf_exts_destroy(tp, &f->exts);

	tcf_exts_destroy(tp, &f->exts);

	kfree(f);

	kfree_rcu(f, rcu);

}

}





static void rsvp_destroy(struct tcf_proto *tp)

static void rsvp_destroy(struct tcf_proto *tp)

{

{

	struct rsvp_head *data = xchg(&tp->root, NULL);

	struct rsvp_head *data = rtnl_dereference(tp->root);

	struct rsvp_session **sht;

	int h1, h2;

	int h1, h2;





	if (data == NULL)

	if (data == NULL)

		return;

		return;





	sht = data->ht;

	RCU_INIT_POINTER(tp->root, NULL);





	for (h1 = 0; h1 < 256; h1++) {

	for (h1 = 0; h1 < 256; h1++) {

		struct rsvp_session *s;

		struct rsvp_session *s;





		while ((s = sht[h1]) != NULL) {

		while ((s = rtnl_dereference(data->ht[h1])) != NULL) {

			sht[h1] = s->next;

			RCU_INIT_POINTER(data->ht[h1], s->next);





			for (h2 = 0; h2 <= 16; h2++) {

			for (h2 = 0; h2 <= 16; h2++) {

				struct rsvp_filter *f;

				struct rsvp_filter *f;





				while ((f = s->ht[h2]) != NULL) {

				while ((f = rtnl_dereference(s->ht[h2])) != NULL) {

					s->ht[h2] = f->next;

					rcu_assign_pointer(s->ht[h2], f->next);

					rsvp_delete_filter(tp, f);

					rsvp_delete_filter(tp, f);

				}

				}

			}

			}

			kfree(s);

			kfree_rcu(s, rcu);

		}

		}

	}

	}

	kfree(data);

	kfree_rcu(data, rcu);

}

}





static int rsvp_delete(struct tcf_proto *tp, unsigned long arg)

static int rsvp_delete(struct tcf_proto *tp, unsigned long arg)

{

{

	struct rsvp_filter **fp, *f = (struct rsvp_filter *)arg;

	struct rsvp_head *head = rtnl_dereference(tp->root);

	struct rsvp_filter *nfp, *f = (struct rsvp_filter *)arg;

	struct rsvp_filter __rcu **fp;

	unsigned int h = f->handle;

	unsigned int h = f->handle;

	struct rsvp_session **sp;

	struct rsvp_session __rcu **sp;

	struct rsvp_session *s = f->sess;

	struct rsvp_session *nsp, *s = f->sess;

	int i;

	int i;





	for (fp = &s->ht[(h >> 8) & 0xFF]; *fp; fp = &(*fp)->next) {

	fp = &s->ht[(h >> 8) & 0xFF];

		if (*fp == f) {

	for (nfp = rtnl_dereference(*fp); nfp;

			tcf_tree_lock(tp);

	     fp = &nfp->next, nfp = rtnl_dereference(*fp)) {

			*fp = f->next;

		if (nfp == f) {

			tcf_tree_unlock(tp);

			RCU_INIT_POINTER(*fp, f->next);

			rsvp_delete_filter(tp, f);

			rsvp_delete_filter(tp, f);





			/* Strip tree */

			/* Strip tree */
@@ -313,14 +322,12 @@ static int rsvp_delete(struct tcf_proto *tp, unsigned long arg) 
					return 0;

					return 0;





			/* OK, session has no flows */

			/* OK, session has no flows */

			for (sp = &((struct rsvp_head *)tp->root)->ht[h & 0xFF];

			sp = &head->ht[h & 0xFF];

			     *sp; sp = &(*sp)->next) {

			for (nsp = rtnl_dereference(*sp); nsp;

				if (*sp == s) {

			     sp = &nsp->next, nsp = rtnl_dereference(*sp)) {

					tcf_tree_lock(tp);

				if (nsp == s) {

					*sp = s->next;

					RCU_INIT_POINTER(*sp, s->next);

					tcf_tree_unlock(tp);

					kfree_rcu(s, rcu);



					kfree(s);

					return 0;

					return 0;

				}

				}

			}

			}
@@ -333,7 +340,7 @@ static int rsvp_delete(struct tcf_proto *tp, unsigned long arg) 




static unsigned int gen_handle(struct tcf_proto *tp, unsigned salt)

static unsigned int gen_handle(struct tcf_proto *tp, unsigned salt)

{

{

	struct rsvp_head *data = tp->root;

	struct rsvp_head *data = rtnl_dereference(tp->root);

	int i = 0xFFFF;

	int i = 0xFFFF;





	while (i-- > 0) {

	while (i-- > 0) {
@@ -361,7 +368,7 @@ static int tunnel_bts(struct rsvp_head *data) 




static void tunnel_recycle(struct rsvp_head *data)

static void tunnel_recycle(struct rsvp_head *data)

{

{

	struct rsvp_session **sht = data->ht;

	struct rsvp_session __rcu **sht = data->ht;

	u32 tmap[256/32];

	u32 tmap[256/32];

	int h1, h2;

	int h1, h2;
@@ -369,11 +376,13 @@ static void tunnel_recycle(struct rsvp_head *data) 




	for (h1 = 0; h1 < 256; h1++) {

	for (h1 = 0; h1 < 256; h1++) {

		struct rsvp_session *s;

		struct rsvp_session *s;

		for (s = sht[h1]; s; s = s->next) {

		for (s = rtnl_dereference(sht[h1]); s;

		     s = rtnl_dereference(s->next)) {

			for (h2 = 0; h2 <= 16; h2++) {

			for (h2 = 0; h2 <= 16; h2++) {

				struct rsvp_filter *f;

				struct rsvp_filter *f;





				for (f = s->ht[h2]; f; f = f->next) {

				for (f = rtnl_dereference(s->ht[h2]); f;

				     f = rtnl_dereference(f->next)) {

					if (f->tunnelhdr == 0)

					if (f->tunnelhdr == 0)

						continue;

						continue;

					data->tgenerator = f->res.classid;

					data->tgenerator = f->res.classid;
@@ -417,9 +426,11 @@ static int rsvp_change(struct net *net, struct sk_buff *in_skb, 
		       struct nlattr **tca,

		       struct nlattr **tca,

		       unsigned long *arg, bool ovr)

		       unsigned long *arg, bool ovr)

{

{

	struct rsvp_head *data = tp->root;

	struct rsvp_head *data = rtnl_dereference(tp->root);

	struct rsvp_filter *f, **fp;

	struct rsvp_filter *f, *nfp;

	struct rsvp_session *s, **sp;

	struct rsvp_filter __rcu **fp;

	struct rsvp_session *nsp, *s;

	struct rsvp_session __rcu **sp;

	struct tc_rsvp_pinfo *pinfo = NULL;

	struct tc_rsvp_pinfo *pinfo = NULL;

	struct nlattr *opt = tca[TCA_OPTIONS];

	struct nlattr *opt = tca[TCA_OPTIONS];

	struct nlattr *tb[TCA_RSVP_MAX + 1];

	struct nlattr *tb[TCA_RSVP_MAX + 1];
@@ -499,7 +510,9 @@ static int rsvp_change(struct net *net, struct sk_buff *in_skb, 
			goto errout;

			goto errout;

	}

	}





	for (sp = &data->ht[h1]; (s = *sp) != NULL; sp = &s->next) {

	for (sp = &data->ht[h1];

	     (s = rtnl_dereference(*sp)) != NULL;

	     sp = &s->next) {

		if (dst[RSVP_DST_LEN-1] == s->dst[RSVP_DST_LEN-1] &&

		if (dst[RSVP_DST_LEN-1] == s->dst[RSVP_DST_LEN-1] &&

		    pinfo && pinfo->protocol == s->protocol &&

		    pinfo && pinfo->protocol == s->protocol &&

		    memcmp(&pinfo->dpi, &s->dpi, sizeof(s->dpi)) == 0 &&

		    memcmp(&pinfo->dpi, &s->dpi, sizeof(s->dpi)) == 0 &&
@@ -521,12 +534,16 @@ static int rsvp_change(struct net *net, struct sk_buff *in_skb, 




			tcf_exts_change(tp, &f->exts, &e);

			tcf_exts_change(tp, &f->exts, &e);





			for (fp = &s->ht[h2]; *fp; fp = &(*fp)->next)

			fp = &s->ht[h2];

				if (((*fp)->spi.mask & f->spi.mask) != f->spi.mask)

			for (nfp = rtnl_dereference(*fp); nfp;

			     fp = &nfp->next, nfp = rtnl_dereference(*fp)) {

				__u32 mask = nfp->spi.mask & f->spi.mask;



				if (mask != f->spi.mask)

					break;

					break;

			f->next = *fp;

			}

			wmb();

			RCU_INIT_POINTER(f->next, nfp);

			*fp = f;

			rcu_assign_pointer(*fp, f);





			*arg = (unsigned long)f;

			*arg = (unsigned long)f;

			return 0;

			return 0;
@@ -546,13 +563,14 @@ static int rsvp_change(struct net *net, struct sk_buff *in_skb, 
		s->protocol = pinfo->protocol;

		s->protocol = pinfo->protocol;

		s->tunnelid = pinfo->tunnelid;

		s->tunnelid = pinfo->tunnelid;

	}

	}

	for (sp = &data->ht[h1]; *sp; sp = &(*sp)->next) {

	sp = &data->ht[h1];

		if (((*sp)->dpi.mask&s->dpi.mask) != s->dpi.mask)

	for (nsp = rtnl_dereference(*sp); nsp;

	     sp = &nsp->next, nsp = rtnl_dereference(*sp)) {

		if ((nsp->dpi.mask & s->dpi.mask) != s->dpi.mask)

			break;

			break;

	}

	}

	s->next = *sp;

	RCU_INIT_POINTER(s->next, nsp);

	wmb();

	rcu_assign_pointer(*sp, s);

	*sp = s;





	goto insert;

	goto insert;
@@ -565,7 +583,7 @@ static int rsvp_change(struct net *net, struct sk_buff *in_skb, 




static void rsvp_walk(struct tcf_proto *tp, struct tcf_walker *arg)

static void rsvp_walk(struct tcf_proto *tp, struct tcf_walker *arg)

{

{

	struct rsvp_head *head = tp->root;

	struct rsvp_head *head = rtnl_dereference(tp->root);

	unsigned int h, h1;

	unsigned int h, h1;





	if (arg->stop)

	if (arg->stop)
@@ -574,11 +592,13 @@ static void rsvp_walk(struct tcf_proto *tp, struct tcf_walker *arg) 
	for (h = 0; h < 256; h++) {

	for (h = 0; h < 256; h++) {

		struct rsvp_session *s;

		struct rsvp_session *s;





		for (s = head->ht[h]; s; s = s->next) {

		for (s = rtnl_dereference(head->ht[h]); s;

		     s = rtnl_dereference(s->next)) {

			for (h1 = 0; h1 <= 16; h1++) {

			for (h1 = 0; h1 <= 16; h1++) {

				struct rsvp_filter *f;

				struct rsvp_filter *f;





				for (f = s->ht[h1]; f; f = f->next) {

				for (f = rtnl_dereference(s->ht[h1]); f;

				     f = rtnl_dereference(f->next)) {

					if (arg->count < arg->skip) {

					if (arg->count < arg->skip) {

						arg->count++;

						arg->count++;

						continue;

						continue;