Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c6cffba4 authored by David S. Miller's avatar David S. Miller
Browse files

ipv4: Fix input route performance regression. 



With the routing cache removal we lost the "noref" code paths on
input, and this can kill some routing workloads.

Reinstate the noref path when we hit a cached route in the FIB
nexthops.

With help from Eric Dumazet.

Reported-by: default avatarAlexander Duyck <alexander.duyck@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 4487e64d

include/net/route.h

+17 −2
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ 
#include <net/inet_sock.h>

#include <linux/in_route.h>

#include <linux/rtnetlink.h>

#include <linux/rcupdate.h>

#include <linux/route.h>

#include <linux/ip.h>

#include <linux/cache.h>
@@ -157,9 +158,23 @@ static inline struct rtable *ip_route_output_gre(struct net *net, struct flowi4 
	return ip_route_output_key(net, fl4);

}



extern int ip_route_input(struct sk_buff *skb, __be32 dst, __be32 src,

extern int ip_route_input_noref(struct sk_buff *skb, __be32 dst, __be32 src,

				u8 tos, struct net_device *devin);



static inline int ip_route_input(struct sk_buff *skb, __be32 dst, __be32 src,

				 u8 tos, struct net_device *devin)

{

	int err;



	rcu_read_lock();

	err = ip_route_input_noref(skb, dst, src, tos, devin);

	if (!err)

		skb_dst_force(skb);

	rcu_read_unlock();



	return err;

}



extern void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,

			     int oif, u32 mark, u8 protocol, int flow_flags);

extern void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu);

net/ipv4/arp.c

+1 −1
Original line number Diff line number Diff line
@@ -827,7 +827,7 @@ static int arp_process(struct sk_buff *skb) 
	}



	if (arp->ar_op == htons(ARPOP_REQUEST) &&

	    ip_route_input(skb, tip, sip, 0, dev) == 0) {

	    ip_route_input_noref(skb, tip, sip, 0, dev) == 0) {



		rt = skb_rtable(skb);

		addr_type = rt->rt_type;

net/ipv4/fib_semantics.c

+2 −2
Original line number Diff line number Diff line
@@ -172,9 +172,9 @@ static void free_fib_info_rcu(struct rcu_head *head) 
		if (nexthop_nh->nh_exceptions)

			free_nh_exceptions(nexthop_nh);

		if (nexthop_nh->nh_rth_output)

			dst_release(&nexthop_nh->nh_rth_output->dst);

			dst_free(&nexthop_nh->nh_rth_output->dst);

		if (nexthop_nh->nh_rth_input)

			dst_release(&nexthop_nh->nh_rth_input->dst);

			dst_free(&nexthop_nh->nh_rth_input->dst);

	} endfor_nexthops(fi);



	release_net(fi->fib_net);

net/ipv4/ip_fragment.c

+2 −2
Original line number Diff line number Diff line
@@ -258,7 +258,7 @@ static void ip_expire(unsigned long arg) 
		/* skb dst is stale, drop it, and perform route lookup again */

		skb_dst_drop(head);

		iph = ip_hdr(head);

		err = ip_route_input(head, iph->daddr, iph->saddr,

		err = ip_route_input_noref(head, iph->daddr, iph->saddr,

					   iph->tos, head->dev);

		if (err)

			goto out_rcu_unlock;

net/ipv4/ip_input.c

+2 −2
Original line number Diff line number Diff line
@@ -339,7 +339,7 @@ static int ip_rcv_finish(struct sk_buff *skb) 
	 *	how the packet travels inside Linux networking.

	 */

	if (!skb_dst(skb)) {

		int err = ip_route_input(skb, iph->daddr, iph->saddr,

		int err = ip_route_input_noref(skb, iph->daddr, iph->saddr,

					       iph->tos, skb->dev);

		if (unlikely(err)) {

			if (err == -EXDEV)