Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 537c00de authored by Alexander Duyck's avatar Alexander Duyck Committed by David S. Miller
Browse files

net: Add functions netif_reset_xps_queue and netif_set_xps_queue 



This patch adds two functions, netif_reset_xps_queue and
netif_set_xps_queue.  The main idea behind these two functions is to
provide a mechanism through which drivers can update their defaults in
regards to XPS.

Currently no such mechanism exists and as a result we cannot use XPS for
things such as ATR which would require a basic configuration to start in
which the Tx queues are mapped to CPUs via a 1:1 mapping.  With this change
I am making it possible for drivers such as ixgbe to be able to use the XPS
feature by controlling the default configuration.

Signed-off-by: default avatarAlexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 416186fb

include/linux/netdevice.h

+13 −0
Original line number Diff line number Diff line
@@ -2103,6 +2103,19 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index) 
		__netif_schedule(txq->qdisc);

}



#ifdef CONFIG_XPS

extern void netif_reset_xps_queue(struct net_device *dev, u16 index);

extern int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask,

			       u16 index);

#else

static inline int netif_set_xps_queue(struct net_device *dev,

				      struct cpumask *mask,

				      u16 index)

{

	return 0;

}

#endif



/*

 * Returns a Tx hash for the given packet when dev->real_num_tx_queues is used

 * as a distribution range limit for the returned value.

net/core/dev.c

+155 −0
Original line number Diff line number Diff line
@@ -1857,6 +1857,161 @@ static void netif_setup_tc(struct net_device *dev, unsigned int txq) 
	}

}



#ifdef CONFIG_XPS

static DEFINE_MUTEX(xps_map_mutex);

#define xmap_dereference(P)		\

	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))



void netif_reset_xps_queue(struct net_device *dev, u16 index)

{

	struct xps_dev_maps *dev_maps;

	struct xps_map *map;

	int i, pos, nonempty = 0;



	mutex_lock(&xps_map_mutex);

	dev_maps = xmap_dereference(dev->xps_maps);



	if (!dev_maps)

		goto out_no_maps;



	for_each_possible_cpu(i) {

		map = xmap_dereference(dev_maps->cpu_map[i]);

		if (!map)

			continue;



		for (pos = 0; pos < map->len; pos++)

			if (map->queues[pos] == index)

				break;



		if (pos < map->len) {

			if (map->len > 1) {

				map->queues[pos] = map->queues[--map->len];

			} else {

				RCU_INIT_POINTER(dev_maps->cpu_map[i], NULL);

				kfree_rcu(map, rcu);

				map = NULL;

			}

		}

		if (map)

			nonempty = 1;

	}



	if (!nonempty) {

		RCU_INIT_POINTER(dev->xps_maps, NULL);

		kfree_rcu(dev_maps, rcu);

	}



out_no_maps:

	mutex_unlock(&xps_map_mutex);

}



int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask, u16 index)

{

	int i, cpu, pos, map_len, alloc_len, need_set;

	struct xps_map *map, *new_map;

	struct xps_dev_maps *dev_maps, *new_dev_maps;

	int nonempty = 0;

	int numa_node_id = -2;

	int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);



	new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);

	if (!new_dev_maps)

		return -ENOMEM;



	mutex_lock(&xps_map_mutex);



	dev_maps = xmap_dereference(dev->xps_maps);



	for_each_possible_cpu(cpu) {

		map = dev_maps ?

			xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;

		new_map = map;

		if (map) {

			for (pos = 0; pos < map->len; pos++)

				if (map->queues[pos] == index)

					break;

			map_len = map->len;

			alloc_len = map->alloc_len;

		} else

			pos = map_len = alloc_len = 0;



		need_set = cpumask_test_cpu(cpu, mask) && cpu_online(cpu);

#ifdef CONFIG_NUMA

		if (need_set) {

			if (numa_node_id == -2)

				numa_node_id = cpu_to_node(cpu);

			else if (numa_node_id != cpu_to_node(cpu))

				numa_node_id = -1;

		}

#endif

		if (need_set && pos >= map_len) {

			/* Need to add queue to this CPU's map */

			if (map_len >= alloc_len) {

				alloc_len = alloc_len ?

				    2 * alloc_len : XPS_MIN_MAP_ALLOC;

				new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len),

						       GFP_KERNEL,

						       cpu_to_node(cpu));

				if (!new_map)

					goto error;

				new_map->alloc_len = alloc_len;

				for (i = 0; i < map_len; i++)

					new_map->queues[i] = map->queues[i];

				new_map->len = map_len;

			}

			new_map->queues[new_map->len++] = index;

		} else if (!need_set && pos < map_len) {

			/* Need to remove queue from this CPU's map */

			if (map_len > 1)

				new_map->queues[pos] =

				    new_map->queues[--new_map->len];

			else

				new_map = NULL;

		}

		RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], new_map);

	}



	/* Cleanup old maps */

	for_each_possible_cpu(cpu) {

		map = dev_maps ?

			xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;

		if (map && xmap_dereference(new_dev_maps->cpu_map[cpu]) != map)

			kfree_rcu(map, rcu);

		if (new_dev_maps->cpu_map[cpu])

			nonempty = 1;

	}



	if (nonempty) {

		rcu_assign_pointer(dev->xps_maps, new_dev_maps);

	} else {

		kfree(new_dev_maps);

		RCU_INIT_POINTER(dev->xps_maps, NULL);

	}



	if (dev_maps)

		kfree_rcu(dev_maps, rcu);



	netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),

				     (numa_node_id >= 0) ? numa_node_id :

				     NUMA_NO_NODE);



	mutex_unlock(&xps_map_mutex);



	return 0;

error:

	mutex_unlock(&xps_map_mutex);



	if (new_dev_maps)

		for_each_possible_cpu(i)

			kfree(rcu_dereference_protected(

				new_dev_maps->cpu_map[i],

				1));

	kfree(new_dev_maps);

	return -ENOMEM;

}

EXPORT_SYMBOL(netif_set_xps_queue);



#endif

/*

 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues

 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.

net/core/net-sysfs.c

+5 −143
Original line number Diff line number Diff line
@@ -1002,54 +1002,14 @@ static ssize_t show_xps_map(struct netdev_queue *queue, 
	return len;

}



static DEFINE_MUTEX(xps_map_mutex);

#define xmap_dereference(P)		\

	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))



static void xps_queue_release(struct netdev_queue *queue)

{

	struct net_device *dev = queue->dev;

	struct xps_dev_maps *dev_maps;

	struct xps_map *map;

	unsigned long index;

	int i, pos, nonempty = 0;



	index = get_netdev_queue_index(queue);



	mutex_lock(&xps_map_mutex);

	dev_maps = xmap_dereference(dev->xps_maps);



	if (dev_maps) {

		for_each_possible_cpu(i) {

			map = xmap_dereference(dev_maps->cpu_map[i]);

			if (!map)

				continue;



			for (pos = 0; pos < map->len; pos++)

				if (map->queues[pos] == index)

					break;



			if (pos < map->len) {

				if (map->len > 1)

					map->queues[pos] =

					    map->queues[--map->len];

				else {

					RCU_INIT_POINTER(dev_maps->cpu_map[i],

					    NULL);

					kfree_rcu(map, rcu);

					map = NULL;

				}

			}

			if (map)

				nonempty = 1;

		}



		if (!nonempty) {

			RCU_INIT_POINTER(dev->xps_maps, NULL);

			kfree_rcu(dev_maps, rcu);

		}

	}

	mutex_unlock(&xps_map_mutex);

	netif_reset_xps_queue(dev, index);

}



static ssize_t store_xps_map(struct netdev_queue *queue,
@@ -1057,13 +1017,9 @@ static ssize_t store_xps_map(struct netdev_queue *queue, 
		      const char *buf, size_t len)

{

	struct net_device *dev = queue->dev;

	cpumask_var_t mask;

	int err, i, cpu, pos, map_len, alloc_len, need_set;

	unsigned long index;

	struct xps_map *map, *new_map;

	struct xps_dev_maps *dev_maps, *new_dev_maps;

	int nonempty = 0;

	int numa_node_id = -2;

	cpumask_var_t mask;

	int err;



	if (!capable(CAP_NET_ADMIN))

		return -EPERM;
@@ -1079,105 +1035,11 @@ static ssize_t store_xps_map(struct netdev_queue *queue, 
		return err;

	}



	new_dev_maps = kzalloc(max_t(unsigned int,

	    XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES), GFP_KERNEL);

	if (!new_dev_maps) {

		free_cpumask_var(mask);

		return -ENOMEM;

	}



	mutex_lock(&xps_map_mutex);



	dev_maps = xmap_dereference(dev->xps_maps);



	for_each_possible_cpu(cpu) {

		map = dev_maps ?

			xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;

		new_map = map;

		if (map) {

			for (pos = 0; pos < map->len; pos++)

				if (map->queues[pos] == index)

					break;

			map_len = map->len;

			alloc_len = map->alloc_len;

		} else

			pos = map_len = alloc_len = 0;



		need_set = cpumask_test_cpu(cpu, mask) && cpu_online(cpu);

#ifdef CONFIG_NUMA

		if (need_set) {

			if (numa_node_id == -2)

				numa_node_id = cpu_to_node(cpu);

			else if (numa_node_id != cpu_to_node(cpu))

				numa_node_id = -1;

		}

#endif

		if (need_set && pos >= map_len) {

			/* Need to add queue to this CPU's map */

			if (map_len >= alloc_len) {

				alloc_len = alloc_len ?

				    2 * alloc_len : XPS_MIN_MAP_ALLOC;

				new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len),

						       GFP_KERNEL,

						       cpu_to_node(cpu));

				if (!new_map)

					goto error;

				new_map->alloc_len = alloc_len;

				for (i = 0; i < map_len; i++)

					new_map->queues[i] = map->queues[i];

				new_map->len = map_len;

			}

			new_map->queues[new_map->len++] = index;

		} else if (!need_set && pos < map_len) {

			/* Need to remove queue from this CPU's map */

			if (map_len > 1)

				new_map->queues[pos] =

				    new_map->queues[--new_map->len];

			else

				new_map = NULL;

		}

		RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], new_map);

	}



	/* Cleanup old maps */

	for_each_possible_cpu(cpu) {

		map = dev_maps ?

			xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;

		if (map && xmap_dereference(new_dev_maps->cpu_map[cpu]) != map)

			kfree_rcu(map, rcu);

		if (new_dev_maps->cpu_map[cpu])

			nonempty = 1;

	}



	if (nonempty) {

		rcu_assign_pointer(dev->xps_maps, new_dev_maps);

	} else {

		kfree(new_dev_maps);

		RCU_INIT_POINTER(dev->xps_maps, NULL);

	}



	if (dev_maps)

		kfree_rcu(dev_maps, rcu);



	netdev_queue_numa_node_write(queue, (numa_node_id >= 0) ? numa_node_id :

					    NUMA_NO_NODE);



	mutex_unlock(&xps_map_mutex);

	err = netif_set_xps_queue(dev, mask, index);



	free_cpumask_var(mask);

	return len;



error:

	mutex_unlock(&xps_map_mutex);



	if (new_dev_maps)

		for_each_possible_cpu(i)

			kfree(rcu_dereference_protected(

				new_dev_maps->cpu_map[i],

				1));

	kfree(new_dev_maps);

	free_cpumask_var(mask);

	return -ENOMEM;

	return err ? : len;

}



static struct netdev_queue_attribute xps_cpus_attribute =