Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit dd28d1a0 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

ipv4: __mkroute_output() speedup 



While doing stress tests with a disabled IP route cache, I found
__mkroute_output() was touching three times in_device atomic refcount.

Use RCU to touch it once to reduce cache line ping pongs.

Before patch

time to perform the test
real	1m42.009s
user	0m12.545s
sys	25m0.726s

Profile :

16109.00 26.4% ip_route_output_slow   vmlinux
 7434.00 12.2% dst_destroy            vmlinux
 3280.00  5.4% fib_rules_lookup       vmlinux
 3252.00  5.3% fib_semantic_match     vmlinux
 2622.00  4.3% fib_table_lookup       vmlinux
 2535.00  4.1% dst_alloc              vmlinux
 1750.00  2.9% _raw_read_lock         vmlinux
 1532.00  2.5% rt_set_nexthop         vmlinux

After patch

real	1m36.503s
user	0m12.977s
sys	23m25.608s

14234.00 22.4% ip_route_output_slow   vmlinux
 8717.00 13.7% dst_destroy            vmlinux
 4052.00  6.4% fib_rules_lookup       vmlinux
 3951.00  6.2% fib_semantic_match     vmlinux
 3191.00  5.0% dst_alloc              vmlinux
 1764.00  2.8% fib_table_lookup       vmlinux
 1692.00  2.7% _raw_read_lock         vmlinux
 1605.00  2.5% rt_set_nexthop         vmlinux

Signed-off-by: default avatarEric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e1a5964f

net/ipv4/route.c

+15 −18
Original line number Diff line number Diff line
@@ -2358,7 +2358,6 @@ static int __mkroute_output(struct rtable **result, 
	struct rtable *rth;

	struct in_device *in_dev;

	u32 tos = RT_FL_TOS(oldflp);

	int err = 0;



	if (ipv4_is_loopback(fl->fl4_src) && !(dev_out->flags & IFF_LOOPBACK))

		return -EINVAL;
@@ -2373,11 +2372,12 @@ static int __mkroute_output(struct rtable **result, 
	if (dev_out->flags & IFF_LOOPBACK)

		flags |= RTCF_LOCAL;



	/* get work reference to inet device */

	in_dev = in_dev_get(dev_out);

	if (!in_dev)

	rcu_read_lock();

	in_dev = __in_dev_get_rcu(dev_out);

	if (!in_dev) {

		rcu_read_unlock();

		return -EINVAL;



	}

	if (res->type == RTN_BROADCAST) {

		flags |= RTCF_BROADCAST | RTCF_LOCAL;

		if (res->fi) {
@@ -2390,8 +2390,8 @@ static int __mkroute_output(struct rtable **result, 
				 oldflp->proto))

			flags &= ~RTCF_LOCAL;

		/* If multicast route do not exist use

		   default one, but do not gateway in this case.

		   Yes, it is hack.

		 * default one, but do not gateway in this case.

		 * Yes, it is hack.

		 */

		if (res->fi && res->prefixlen < 4) {

			fib_info_put(res->fi);
@@ -2402,9 +2402,12 @@ static int __mkroute_output(struct rtable **result, 


	rth = dst_alloc(&ipv4_dst_ops);

	if (!rth) {

		err = -ENOBUFS;

		goto cleanup;

		rcu_read_unlock();

		return -ENOBUFS;

	}

	in_dev_hold(in_dev);

	rcu_read_unlock();

	rth->idev = in_dev;



	atomic_set(&rth->dst.__refcnt, 1);

	rth->dst.flags= DST_HOST;
@@ -2425,7 +2428,6 @@ static int __mkroute_output(struct rtable **result, 
	   cache entry */

	rth->dst.dev	= dev_out;

	dev_hold(dev_out);

	rth->idev	= in_dev_get(dev_out);

	rth->rt_gateway = fl->fl4_dst;

	rth->rt_spec_dst= fl->fl4_src;
@@ -2460,13 +2462,8 @@ static int __mkroute_output(struct rtable **result, 
	rt_set_nexthop(rth, res, 0);



	rth->rt_flags = flags;



	*result = rth;

 cleanup:

	/* release work reference to inet device */

	in_dev_put(in_dev);



	return err;

	return 0;

}



static int ip_mkroute_output(struct rtable **rp,