Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e4917d46 authored by Chopra, Manish's avatar Chopra, Manish Committed by David S. Miller
Browse files

qede: Add aRFS support 



This patch adds support for aRFS for TCP and UDP
protocols with IPv4/IPv6.

Signed-off-by: default avatarManish Chopra <manish.chopra@cavium.com>
Signed-off-by: default avatarYuval Mintz <yuval.mintz@cavium.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent d51e4af5

drivers/net/ethernet/qlogic/qede/qede.h

+21 −1
Original line number Diff line number Diff line
@@ -41,6 +41,9 @@ 
#include <linux/mutex.h>

#include <linux/bpf.h>

#include <linux/io.h>

#ifdef CONFIG_RFS_ACCEL

#include <linux/cpu_rmap.h>

#endif

#include <linux/qed/common_hsi.h>

#include <linux/qed/eth_common.h>

#include <linux/qed/qed_if.h>
@@ -237,6 +240,9 @@ struct qede_dev { 
	u16				vxlan_dst_port;

	u16				geneve_dst_port;



#ifdef CONFIG_RFS_ACCEL

	struct qede_arfs		*arfs;

#endif

	bool				wol_enabled;



	struct qede_rdma_dev		rdma_info;
@@ -439,6 +445,20 @@ struct qede_fastpath { 
#define QEDE_SP_VXLAN_PORT_CONFIG	2

#define QEDE_SP_GENEVE_PORT_CONFIG	3



#ifdef CONFIG_RFS_ACCEL

int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,

		       u16 rxq_index, u32 flow_id);

void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr);

void qede_poll_for_freeing_arfs_filters(struct qede_dev *edev);

void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc);

void qede_free_arfs(struct qede_dev *edev);

int qede_alloc_arfs(struct qede_dev *edev);



#define QEDE_SP_ARFS_CONFIG	4

#define QEDE_SP_TASK_POLL_DELAY	(5 * HZ)

#define QEDE_RFS_MAX_FLTR	256

#endif



struct qede_reload_args {

	void (*func)(struct qede_dev *edev, struct qede_reload_args *args);

	union {

drivers/net/ethernet/qlogic/qede/qede_filter.c

+441 −0
Original line number Diff line number Diff line
@@ -38,6 +38,447 @@ 
#include <linux/qed/qed_if.h>

#include "qede.h"



#ifdef CONFIG_RFS_ACCEL

struct qede_arfs_tuple {

	union {

		__be32 src_ipv4;

		struct in6_addr src_ipv6;

	};

	union {

		__be32 dst_ipv4;

		struct in6_addr dst_ipv6;

	};

	__be16  src_port;

	__be16  dst_port;

	__be16  eth_proto;

	u8      ip_proto;

};



struct qede_arfs_fltr_node {

#define QEDE_FLTR_VALID	 0

	unsigned long state;



	/* pointer to aRFS packet buffer */

	void *data;



	/* dma map address of aRFS packet buffer */

	dma_addr_t mapping;



	/* length of aRFS packet buffer */

	int buf_len;



	/* tuples to hold from aRFS packet buffer */

	struct qede_arfs_tuple tuple;



	u32 flow_id;

	u16 sw_id;

	u16 rxq_id;

	u16 next_rxq_id;

	bool filter_op;

	bool used;

	struct hlist_node node;

};



struct qede_arfs {

#define QEDE_ARFS_POLL_COUNT	100

#define QEDE_RFS_FLW_BITSHIFT	(4)

#define QEDE_RFS_FLW_MASK	((1 << QEDE_RFS_FLW_BITSHIFT) - 1)

	struct hlist_head	arfs_hl_head[1 << QEDE_RFS_FLW_BITSHIFT];



	/* lock for filter list access */

	spinlock_t		arfs_list_lock;

	unsigned long		*arfs_fltr_bmap;

	int			filter_count;

	bool			enable;

};



static void qede_configure_arfs_fltr(struct qede_dev *edev,

				     struct qede_arfs_fltr_node *n,

				     u16 rxq_id, bool add_fltr)

{

	const struct qed_eth_ops *op = edev->ops;



	if (n->used)

		return;



	DP_VERBOSE(edev, NETIF_MSG_RX_STATUS,

		   "%s arfs filter flow_id=%d, sw_id=%d, src_port=%d, dst_port=%d, rxq=%d\n",

		   add_fltr ? "Adding" : "Deleting",

		   n->flow_id, n->sw_id, ntohs(n->tuple.src_port),

		   ntohs(n->tuple.dst_port), rxq_id);



	n->used = true;

	n->filter_op = add_fltr;

	op->ntuple_filter_config(edev->cdev, n, n->mapping, n->buf_len, 0,

				 rxq_id, add_fltr);

}



static void

qede_free_arfs_filter(struct qede_dev *edev,  struct qede_arfs_fltr_node *fltr)

{

	kfree(fltr->data);

	clear_bit(fltr->sw_id, edev->arfs->arfs_fltr_bmap);

	kfree(fltr);

}



void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc)

{

	struct qede_arfs_fltr_node *fltr = filter;

	struct qede_dev *edev = dev;



	if (fw_rc) {

		DP_NOTICE(edev,

			  "Failed arfs filter configuration fw_rc=%d, flow_id=%d, sw_id=%d, src_port=%d, dst_port=%d, rxq=%d\n",

			  fw_rc, fltr->flow_id, fltr->sw_id,

			  ntohs(fltr->tuple.src_port),

			  ntohs(fltr->tuple.dst_port), fltr->rxq_id);



		spin_lock_bh(&edev->arfs->arfs_list_lock);



		fltr->used = false;

		clear_bit(QEDE_FLTR_VALID, &fltr->state);



		spin_unlock_bh(&edev->arfs->arfs_list_lock);

		return;

	}



	spin_lock_bh(&edev->arfs->arfs_list_lock);



	fltr->used = false;



	if (fltr->filter_op) {

		set_bit(QEDE_FLTR_VALID, &fltr->state);

		if (fltr->rxq_id != fltr->next_rxq_id)

			qede_configure_arfs_fltr(edev, fltr, fltr->rxq_id,

						 false);

	} else {

		clear_bit(QEDE_FLTR_VALID, &fltr->state);

		if (fltr->rxq_id != fltr->next_rxq_id) {

			fltr->rxq_id = fltr->next_rxq_id;

			qede_configure_arfs_fltr(edev, fltr,

						 fltr->rxq_id, true);

		}

	}



	spin_unlock_bh(&edev->arfs->arfs_list_lock);

}



/* Should be called while qede_lock is held */

void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr)

{

	int i;



	for (i = 0; i <= QEDE_RFS_FLW_MASK; i++) {

		struct hlist_node *temp;

		struct hlist_head *head;

		struct qede_arfs_fltr_node *fltr;



		head = &edev->arfs->arfs_hl_head[i];



		hlist_for_each_entry_safe(fltr, temp, head, node) {

			bool del = false;



			if (edev->state != QEDE_STATE_OPEN)

				del = true;



			spin_lock_bh(&edev->arfs->arfs_list_lock);



			if ((!test_bit(QEDE_FLTR_VALID, &fltr->state) &&

			     !fltr->used) || free_fltr) {

				hlist_del(&fltr->node);

				dma_unmap_single(&edev->pdev->dev,

						 fltr->mapping,

						 fltr->buf_len, DMA_TO_DEVICE);

				qede_free_arfs_filter(edev, fltr);

				edev->arfs->filter_count--;

			} else {

				if ((rps_may_expire_flow(edev->ndev,

							 fltr->rxq_id,

							 fltr->flow_id,

							 fltr->sw_id) || del) &&

							 !free_fltr)

					qede_configure_arfs_fltr(edev, fltr,

								 fltr->rxq_id,

								 false);

			}



			spin_unlock_bh(&edev->arfs->arfs_list_lock);

		}

	}



	spin_lock_bh(&edev->arfs->arfs_list_lock);



	if (!edev->arfs->filter_count) {

		if (edev->arfs->enable) {

			edev->arfs->enable = false;

			edev->ops->configure_arfs_searcher(edev->cdev, false);

		}

	} else {

		set_bit(QEDE_SP_ARFS_CONFIG, &edev->sp_flags);

		schedule_delayed_work(&edev->sp_task,

				      QEDE_SP_TASK_POLL_DELAY);

	}



	spin_unlock_bh(&edev->arfs->arfs_list_lock);

}



/* This function waits until all aRFS filters get deleted and freed.

 * On timeout it frees all filters forcefully.

 */

void qede_poll_for_freeing_arfs_filters(struct qede_dev *edev)

{

	int count = QEDE_ARFS_POLL_COUNT;



	while (count) {

		qede_process_arfs_filters(edev, false);



		if (!edev->arfs->filter_count)

			break;



		msleep(100);

		count--;

	}



	if (!count) {

		DP_NOTICE(edev, "Timeout in polling for arfs filter free\n");



		/* Something is terribly wrong, free forcefully */

		qede_process_arfs_filters(edev, true);

	}

}



int qede_alloc_arfs(struct qede_dev *edev)

{

	int i;



	edev->arfs = vzalloc(sizeof(*edev->arfs));

	if (!edev->arfs)

		return -ENOMEM;



	spin_lock_init(&edev->arfs->arfs_list_lock);



	for (i = 0; i <= QEDE_RFS_FLW_MASK; i++)

		INIT_HLIST_HEAD(&edev->arfs->arfs_hl_head[i]);



	edev->ndev->rx_cpu_rmap = alloc_irq_cpu_rmap(QEDE_RSS_COUNT(edev));

	if (!edev->ndev->rx_cpu_rmap) {

		vfree(edev->arfs);

		edev->arfs = NULL;

		return -ENOMEM;

	}



	edev->arfs->arfs_fltr_bmap = vzalloc(BITS_TO_LONGS(QEDE_RFS_MAX_FLTR));

	if (!edev->arfs->arfs_fltr_bmap) {

		free_irq_cpu_rmap(edev->ndev->rx_cpu_rmap);

		edev->ndev->rx_cpu_rmap = NULL;

		vfree(edev->arfs);

		edev->arfs = NULL;

		return -ENOMEM;

	}



	return 0;

}



void qede_free_arfs(struct qede_dev *edev)

{

	if (!edev->arfs)

		return;



	if (edev->ndev->rx_cpu_rmap)

		free_irq_cpu_rmap(edev->ndev->rx_cpu_rmap);



	edev->ndev->rx_cpu_rmap = NULL;

	vfree(edev->arfs->arfs_fltr_bmap);

	edev->arfs->arfs_fltr_bmap = NULL;

	vfree(edev->arfs);

	edev->arfs = NULL;

}



static bool qede_compare_ip_addr(struct qede_arfs_fltr_node *tpos,

				 const struct sk_buff *skb)

{

	if (skb->protocol == htons(ETH_P_IP)) {

		if (tpos->tuple.src_ipv4 == ip_hdr(skb)->saddr &&

		    tpos->tuple.dst_ipv4 == ip_hdr(skb)->daddr)

			return true;

		else

			return false;

	} else {

		struct in6_addr *src = &tpos->tuple.src_ipv6;

		u8 size = sizeof(struct in6_addr);



		if (!memcmp(src, &ipv6_hdr(skb)->saddr, size) &&

		    !memcmp(&tpos->tuple.dst_ipv6, &ipv6_hdr(skb)->daddr, size))

			return true;

		else

			return false;

	}

}



static struct qede_arfs_fltr_node *

qede_arfs_htbl_key_search(struct hlist_head *h, const struct sk_buff *skb,

			  __be16 src_port, __be16 dst_port, u8 ip_proto)

{

	struct qede_arfs_fltr_node *tpos;



	hlist_for_each_entry(tpos, h, node)

		if (tpos->tuple.ip_proto == ip_proto &&

		    tpos->tuple.eth_proto == skb->protocol &&

		    qede_compare_ip_addr(tpos, skb) &&

		    tpos->tuple.src_port == src_port &&

		    tpos->tuple.dst_port == dst_port)

			return tpos;



	return NULL;

}



static struct qede_arfs_fltr_node *

qede_alloc_filter(struct qede_dev *edev, int min_hlen)

{

	struct qede_arfs_fltr_node *n;

	int bit_id;



	bit_id = find_first_zero_bit(edev->arfs->arfs_fltr_bmap,

				     QEDE_RFS_MAX_FLTR);



	if (bit_id >= QEDE_RFS_MAX_FLTR)

		return NULL;



	n = kzalloc(sizeof(*n), GFP_ATOMIC);

	if (!n)

		return NULL;



	n->data = kzalloc(min_hlen, GFP_ATOMIC);

	if (!n->data) {

		kfree(n);

		return NULL;

	}



	n->sw_id = (u16)bit_id;

	set_bit(bit_id, edev->arfs->arfs_fltr_bmap);

	return n;

}



int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,

		       u16 rxq_index, u32 flow_id)

{

	struct qede_dev *edev = netdev_priv(dev);

	struct qede_arfs_fltr_node *n;

	int min_hlen, rc, tp_offset;

	struct ethhdr *eth;

	__be16 *ports;

	u16 tbl_idx;

	u8 ip_proto;



	if (skb->encapsulation)

		return -EPROTONOSUPPORT;



	if (skb->protocol != htons(ETH_P_IP) &&

	    skb->protocol != htons(ETH_P_IPV6))

		return -EPROTONOSUPPORT;



	if (skb->protocol == htons(ETH_P_IP)) {

		ip_proto = ip_hdr(skb)->protocol;

		tp_offset = sizeof(struct iphdr);

	} else {

		ip_proto = ipv6_hdr(skb)->nexthdr;

		tp_offset = sizeof(struct ipv6hdr);

	}



	if (ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP)

		return -EPROTONOSUPPORT;



	ports = (__be16 *)(skb->data + tp_offset);

	tbl_idx = skb_get_hash_raw(skb) & QEDE_RFS_FLW_MASK;



	spin_lock_bh(&edev->arfs->arfs_list_lock);



	n = qede_arfs_htbl_key_search(&edev->arfs->arfs_hl_head[tbl_idx],

				      skb, ports[0], ports[1], ip_proto);



	if (n) {

		/* Filter match */

		n->next_rxq_id = rxq_index;



		if (test_bit(QEDE_FLTR_VALID, &n->state)) {

			if (n->rxq_id != rxq_index)

				qede_configure_arfs_fltr(edev, n, n->rxq_id,

							 false);

		} else {

			if (!n->used) {

				n->rxq_id = rxq_index;

				qede_configure_arfs_fltr(edev, n, n->rxq_id,

							 true);

			}

		}



		rc = n->sw_id;

		goto ret_unlock;

	}



	min_hlen = ETH_HLEN + skb_headlen(skb);



	n = qede_alloc_filter(edev, min_hlen);

	if (!n) {

		rc = -ENOMEM;

		goto ret_unlock;

	}



	n->buf_len = min_hlen;

	n->rxq_id = rxq_index;

	n->next_rxq_id = rxq_index;

	n->tuple.src_port = ports[0];

	n->tuple.dst_port = ports[1];

	n->flow_id = flow_id;



	if (skb->protocol == htons(ETH_P_IP)) {

		n->tuple.src_ipv4 = ip_hdr(skb)->saddr;

		n->tuple.dst_ipv4 = ip_hdr(skb)->daddr;

	} else {

		memcpy(&n->tuple.src_ipv6, &ipv6_hdr(skb)->saddr,

		       sizeof(struct in6_addr));

		memcpy(&n->tuple.dst_ipv6, &ipv6_hdr(skb)->daddr,

		       sizeof(struct in6_addr));

	}



	eth = (struct ethhdr *)n->data;

	eth->h_proto = skb->protocol;

	n->tuple.eth_proto = skb->protocol;

	n->tuple.ip_proto = ip_proto;

	memcpy(n->data + ETH_HLEN, skb->data, skb_headlen(skb));



	n->mapping = dma_map_single(&edev->pdev->dev, n->data,

				    n->buf_len, DMA_TO_DEVICE);

	if (dma_mapping_error(&edev->pdev->dev, n->mapping)) {

		DP_NOTICE(edev, "Failed to map DMA memory for arfs\n");

		qede_free_arfs_filter(edev, n);

		rc = -ENOMEM;

		goto ret_unlock;

	}



	INIT_HLIST_NODE(&n->node);

	hlist_add_head(&n->node, &edev->arfs->arfs_hl_head[tbl_idx]);

	edev->arfs->filter_count++;



	if (edev->arfs->filter_count == 1 && !edev->arfs->enable) {

		edev->ops->configure_arfs_searcher(edev->cdev, true);

		edev->arfs->enable = true;

	}



	qede_configure_arfs_fltr(edev, n, n->rxq_id, true);



	spin_unlock_bh(&edev->arfs->arfs_list_lock);



	set_bit(QEDE_SP_ARFS_CONFIG, &edev->sp_flags);

	schedule_delayed_work(&edev->sp_task, 0);

	return n->sw_id;



ret_unlock:

	spin_unlock_bh(&edev->arfs->arfs_list_lock);

	return rc;

}

#endif



void qede_force_mac(void *dev, u8 *mac, bool forced)

{

	struct qede_dev *edev = dev;

drivers/net/ethernet/qlogic/qede/qede_main.c

+46 −4
Original line number Diff line number Diff line
@@ -225,6 +225,9 @@ static struct pci_driver qede_pci_driver = { 


static struct qed_eth_cb_ops qede_ll_ops = {

	{

#ifdef CONFIG_RFS_ACCEL

		.arfs_filter_op = qede_arfs_filter_op,

#endif

		.link_update = qede_link_update,

	},

	.force_mac = qede_force_mac,
@@ -554,6 +557,9 @@ static const struct net_device_ops qede_netdev_ops = { 
	.ndo_udp_tunnel_del = qede_udp_tunnel_del,

	.ndo_features_check = qede_features_check,

	.ndo_xdp = qede_xdp,

#ifdef CONFIG_RFS_ACCEL

	.ndo_rx_flow_steer = qede_rx_flow_steer,

#endif

};



/* -------------------------------------------------------------------------
@@ -603,7 +609,7 @@ static void qede_init_ndev(struct qede_dev *edev) 
{

	struct net_device *ndev = edev->ndev;

	struct pci_dev *pdev = edev->pdev;

	u32 hw_features;

	netdev_features_t hw_features;



	pci_set_drvdata(pdev, ndev);
@@ -629,6 +635,10 @@ static void qede_init_ndev(struct qede_dev *edev) 
	hw_features |= NETIF_F_GSO_GRE | NETIF_F_GSO_UDP_TUNNEL |

		       NETIF_F_TSO_ECN | NETIF_F_GSO_UDP_TUNNEL_CSUM |

		       NETIF_F_GSO_GRE_CSUM;



	if (!IS_VF(edev) && edev->dev_info.common.num_hwfns == 1)

		hw_features |= NETIF_F_NTUPLE;



	ndev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |

				NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO_ECN |

				NETIF_F_TSO6 | NETIF_F_GSO_GRE |
@@ -798,6 +808,12 @@ static void qede_sp_task(struct work_struct *work) 
		qed_ops->tunn_config(cdev, &tunn_params);

	}



#ifdef CONFIG_RFS_ACCEL

	if (test_and_clear_bit(QEDE_SP_ARFS_CONFIG, &edev->sp_flags)) {

		if (edev->state == QEDE_STATE_OPEN)

			qede_process_arfs_filters(edev, false);

	}

#endif

	__qede_unlock(edev);

}
@@ -808,6 +824,9 @@ static void qede_update_pf_params(struct qed_dev *cdev) 
	/* 64 rx + 64 tx + 64 XDP */

	memset(&pf_params, 0, sizeof(struct qed_pf_params));

	pf_params.eth_pf_params.num_cons = (MAX_SB_PER_PF_MIMD - 1) * 3;

#ifdef CONFIG_RFS_ACCEL

	pf_params.eth_pf_params.num_arfs_filters = QEDE_RFS_MAX_FLTR;

#endif

	qed_ops->common->update_pf_params(cdev, &pf_params);

}
@@ -962,9 +981,8 @@ static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode) 


	DP_INFO(edev, "Starting qede_remove\n");



	cancel_delayed_work_sync(&edev->sp_task);



	unregister_netdev(ndev);

	cancel_delayed_work_sync(&edev->sp_task);



	qede_ptp_remove(edev);
@@ -1490,6 +1508,18 @@ static int qede_req_msix_irqs(struct qede_dev *edev) 
	}



	for (i = 0; i < QEDE_QUEUE_CNT(edev); i++) {

#ifdef CONFIG_RFS_ACCEL

		struct qede_fastpath *fp = &edev->fp_array[i];



		if (edev->ndev->rx_cpu_rmap && (fp->type & QEDE_FASTPATH_RX)) {

			rc = irq_cpu_rmap_add(edev->ndev->rx_cpu_rmap,

					      edev->int_info.msix[i].vector);

			if (rc) {

				DP_ERR(edev, "Failed to add CPU rmap\n");

				qede_free_arfs(edev);

			}

		}

#endif

		rc = request_irq(edev->int_info.msix[i].vector,

				 qede_msix_fp_int, 0, edev->fp_array[i].name,

				 &edev->fp_array[i]);
@@ -1871,7 +1901,12 @@ static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode, 


	qede_vlan_mark_nonconfigured(edev);

	edev->ops->fastpath_stop(edev->cdev);



#ifdef CONFIG_RFS_ACCEL

	if (!IS_VF(edev) && edev->dev_info.common.num_hwfns == 1) {

		qede_poll_for_freeing_arfs_filters(edev);

		qede_free_arfs(edev);

	}

#endif

	/* Release the interrupts */

	qede_sync_free_irqs(edev);

	edev->ops->common->set_fp_int(edev->cdev, 0);
@@ -1923,6 +1958,13 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode, 
	if (rc)

		goto err2;



#ifdef CONFIG_RFS_ACCEL

	if (!IS_VF(edev) && edev->dev_info.common.num_hwfns == 1) {

		rc = qede_alloc_arfs(edev);

		if (rc)

			DP_NOTICE(edev, "aRFS memory allocation failed\n");

	}

#endif

	qede_napi_add_enable(edev);

	DP_INFO(edev, "Napi added and enabled\n");