Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a2d7ec58 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

netfilter: use jump_label for nf_hooks 



On configs where CONFIG_JUMP_LABEL=y, we can replace in fast path a
load/compare/conditional jump by a single jump with no dcache reference.

Jump target is modified as soon as nf_hooks[pf][hook] switches from
empty state to non empty states. jump_label state is kept outside of
nf_hooks array so has no cost on cpu caches.

This patch removes the test on CONFIG_NETFILTER_DEBUG : No need to call
nf_hook_slow() at all if nf_hooks[pf][hook] is empty, this didnt give
useful information, but slowed down things a lot.

Signed-off-by: default avatarEric Dumazet <eric.dumazet@gmail.com>
CC: Patrick McHardy <kaber@trash.net>
CC: Pablo Neira Ayuso <pablo@netfilter.org>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 9205fd9c

include/linux/netfilter.h

+21 −5
Original line number Diff line number Diff line
@@ -162,6 +162,24 @@ extern struct ctl_path nf_net_ipv4_netfilter_sysctl_path[]; 


extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];



#if defined(CONFIG_JUMP_LABEL)

#include <linux/jump_label.h>

extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];

static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)

{

	if (__builtin_constant_p(pf) &&

	    __builtin_constant_p(hook))

		return static_branch(&nf_hooks_needed[pf][hook]);



	return !list_empty(&nf_hooks[pf][hook]);

}

#else

static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)

{

	return !list_empty(&nf_hooks[pf][hook]);

}

#endif



int nf_hook_slow(u_int8_t pf, unsigned int hook, struct sk_buff *skb,

		 struct net_device *indev, struct net_device *outdev,

		 int (*okfn)(struct sk_buff *), int thresh);
@@ -179,11 +197,9 @@ static inline int nf_hook_thresh(u_int8_t pf, unsigned int hook, 
				 struct net_device *outdev,

				 int (*okfn)(struct sk_buff *), int thresh)

{

#ifndef CONFIG_NETFILTER_DEBUG

	if (list_empty(&nf_hooks[pf][hook]))

		return 1;

#endif

	if (nf_hooks_active(pf, hook))

		return nf_hook_slow(pf, hook, skb, indev, outdev, okfn, thresh);

	return 1;

}



static inline int nf_hook(u_int8_t pf, unsigned int hook, struct sk_buff *skb,

net/netfilter/core.c

+12 −1
Original line number Diff line number Diff line
@@ -54,6 +54,12 @@ EXPORT_SYMBOL_GPL(nf_unregister_afinfo); 


struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly;

EXPORT_SYMBOL(nf_hooks);



#if defined(CONFIG_JUMP_LABEL)

struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];

EXPORT_SYMBOL(nf_hooks_needed);

#endif



static DEFINE_MUTEX(nf_hook_mutex);



int nf_register_hook(struct nf_hook_ops *reg)
@@ -70,6 +76,9 @@ int nf_register_hook(struct nf_hook_ops *reg) 
	}

	list_add_rcu(&reg->list, elem->list.prev);

	mutex_unlock(&nf_hook_mutex);

#if defined(CONFIG_JUMP_LABEL)

	jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);

#endif

	return 0;

}

EXPORT_SYMBOL(nf_register_hook);
@@ -79,7 +88,9 @@ void nf_unregister_hook(struct nf_hook_ops *reg) 
	mutex_lock(&nf_hook_mutex);

	list_del_rcu(&reg->list);

	mutex_unlock(&nf_hook_mutex);



#if defined(CONFIG_JUMP_LABEL)

	jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);

#endif

	synchronize_net();

}

EXPORT_SYMBOL(nf_unregister_hook);