Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3cfef195 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'mlx5-aRFS'



Saeed Mahameed says:

====================
Mellanox 100G mlx5 ethernet aRFS support

This series adds accelerated RFS support for the mlx5e driver.
I have added one patch non-related to aRFS that fixes the rtnl_lock
warning mlx5 driver been getting since b7aade15 ('vxlan: break dependency with netdev drivers')

aRFS support in details:

A direct TIR per RQ is now required in order to have the essential building blocks
for aRFS.  Today the driver has one direct TIR that forwards traffic to RQ[0] (core 0),
and one indirect TIR for RSS indirection table.  For that we've added one direct TIR
per RQ, e.g.: TIR[i] -> RQ[i] (core i).

Publicize Modify flow rule destination and reveal it in flow steering API, to have the
ability to dynamically modify the destination TIR(core) for aRFS rules from the
ethernet driver.

Initializing CPU reverse mapping to notify upper layer on internal receive queue cpu
mappings.

Some design refactoring for mlx5e ethernet driver flow tables and flow steering API.
Now the caller of create_flow_table can choose the level of the flow table, this way
we will create the mlx5e flow tables in a reversed order and connect them as we go,
we create flow table[i+1] before flow table[i] to be able to set flow table[i + 1] as
a destination of flow table[i] once flow table[i] is created.
also we have split the main flow table in the following manner:
    - From before: RX packet had to visit two flow tables until it is delivered to its receive queue:
        RX packet -> vlan filter flow table -> main flow table.
        > vlan filter will check the packet vlan field is allowed.
        > main flow will check if the dest mac is allowed and will check the l3/l4 headers to
        retrieve the RSS hash for steering the packet into its final receive queue.

    - Now main flow table is split into l2 dst mac steering table and ttc (traffic type classifier) table:
        RX packet -> vlan filter -> l2 table -> ttc table
        > vlan filter - same as before
        > L2 filter - filter packets according their destination mac address
        > ttc table - classify packet headers for RSS steering
            - L3/L4 classification rules to steer the packet according to thier headers hash
            - in case of none of the rules applies the packet is steered to RQ[0]

After the above refactoring all left to-do is to create aRFS flow table which will manage
aRFS steering rules to forward traffic to the desired RQ (core) and just connect the ttc
table rules destinations to aRFS flow table.

aRFS flow table in case of a miss will deliver the traffic to the core where the original
ttc hash would have chosen.

TTC table is not initialized and enabled until the user explicitly asks to, i.e. setting the NETIF_F_NTUPLE
to ON.  This way there is no need for ttc table to forward traffic to aRFS table unless required.
When setting back to OFF aRFS flow table is disabled and disconnected.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 4b2523c1 45bf454a
Loading
Loading
Loading
Loading
+2 −1
Original line number Diff line number Diff line
@@ -1438,7 +1438,8 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
	if (!ft) {
		ft = mlx5_create_auto_grouped_flow_table(ns, priority,
							 num_entries,
							 num_groups);
							 num_groups,
							 0);

		if (!IS_ERR(ft)) {
			prio->refcount = 0;
+1 −0
Original line number Diff line number Diff line
@@ -9,3 +9,4 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o eswitch.o \
		en_txrx.o en_clock.o vxlan.o en_tc.o

mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) +=  en_dcbnl.o
mlx5_core-$(CONFIG_RFS_ACCEL) +=  en_arfs.o
+122 −49
Original line number Diff line number Diff line
@@ -48,6 +48,8 @@
#include "mlx5_core.h"
#include "en_stats.h"

#define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)

#define MLX5E_MAX_NUM_TC	8

#define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE                0x6
@@ -385,42 +387,42 @@ enum mlx5e_traffic_types {
	MLX5E_TT_IPV6,
	MLX5E_TT_ANY,
	MLX5E_NUM_TT,
	MLX5E_NUM_INDIR_TIRS = MLX5E_TT_ANY,
};

#define IS_HASHING_TT(tt) (tt != MLX5E_TT_ANY)
enum {
	MLX5E_STATE_ASYNC_EVENTS_ENABLE,
	MLX5E_STATE_OPENED,
	MLX5E_STATE_DESTROYING,
};

enum mlx5e_rqt_ix {
	MLX5E_INDIRECTION_RQT,
	MLX5E_SINGLE_RQ_RQT,
	MLX5E_NUM_RQT,
struct mlx5e_vxlan_db {
	spinlock_t			lock; /* protect vxlan table */
	struct radix_tree_root		tree;
};

struct mlx5e_eth_addr_info {
struct mlx5e_l2_rule {
	u8  addr[ETH_ALEN + 2];
	u32 tt_vec;
	struct mlx5_flow_rule *ft_rule[MLX5E_NUM_TT];
	struct mlx5_flow_rule *rule;
};

#define MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE)

struct mlx5e_eth_addr_db {
	struct hlist_head          netdev_uc[MLX5E_ETH_ADDR_HASH_SIZE];
	struct hlist_head          netdev_mc[MLX5E_ETH_ADDR_HASH_SIZE];
	struct mlx5e_eth_addr_info broadcast;
	struct mlx5e_eth_addr_info allmulti;
	struct mlx5e_eth_addr_info promisc;
	bool                       broadcast_enabled;
	bool                       allmulti_enabled;
	bool                       promisc_enabled;
struct mlx5e_flow_table {
	int num_groups;
	struct mlx5_flow_table *t;
	struct mlx5_flow_group **g;
};

enum {
	MLX5E_STATE_ASYNC_EVENTS_ENABLE,
	MLX5E_STATE_OPENED,
	MLX5E_STATE_DESTROYING,
#define MLX5E_L2_ADDR_HASH_SIZE BIT(BITS_PER_BYTE)

struct mlx5e_tc_table {
	struct mlx5_flow_table		*t;

	struct rhashtable_params        ht_params;
	struct rhashtable               ht;
};

struct mlx5e_vlan_db {
struct mlx5e_vlan_table {
	struct mlx5e_flow_table		ft;
	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
	struct mlx5_flow_rule	*active_vlans_rule[VLAN_N_VID];
	struct mlx5_flow_rule	*untagged_rule;
@@ -428,29 +430,74 @@ struct mlx5e_vlan_db {
	bool          filter_disabled;
};

struct mlx5e_vxlan_db {
	spinlock_t			lock; /* protect vxlan table */
	struct radix_tree_root		tree;
struct mlx5e_l2_table {
	struct mlx5e_flow_table    ft;
	struct hlist_head          netdev_uc[MLX5E_L2_ADDR_HASH_SIZE];
	struct hlist_head          netdev_mc[MLX5E_L2_ADDR_HASH_SIZE];
	struct mlx5e_l2_rule	   broadcast;
	struct mlx5e_l2_rule	   allmulti;
	struct mlx5e_l2_rule	   promisc;
	bool                       broadcast_enabled;
	bool                       allmulti_enabled;
	bool                       promisc_enabled;
};

struct mlx5e_flow_table {
	int num_groups;
	struct mlx5_flow_table		*t;
	struct mlx5_flow_group		**g;
/* L3/L4 traffic type classifier */
struct mlx5e_ttc_table {
	struct mlx5e_flow_table  ft;
	struct mlx5_flow_rule	 *rules[MLX5E_NUM_TT];
};

struct mlx5e_tc_flow_table {
	struct mlx5_flow_table		*t;
#define ARFS_HASH_SHIFT BITS_PER_BYTE
#define ARFS_HASH_SIZE BIT(BITS_PER_BYTE)
struct arfs_table {
	struct mlx5e_flow_table  ft;
	struct mlx5_flow_rule    *default_rule;
	struct hlist_head	 rules_hash[ARFS_HASH_SIZE];
};

	struct rhashtable_params        ht_params;
	struct rhashtable               ht;
enum  arfs_type {
	ARFS_IPV4_TCP,
	ARFS_IPV6_TCP,
	ARFS_IPV4_UDP,
	ARFS_IPV6_UDP,
	ARFS_NUM_TYPES,
};

struct mlx5e_arfs_tables {
	struct arfs_table arfs_tables[ARFS_NUM_TYPES];
	/* Protect aRFS rules list */
	spinlock_t                     arfs_lock;
	struct list_head               rules;
	int                            last_filter_id;
	struct workqueue_struct        *wq;
};

struct mlx5e_flow_tables {
/* NIC prio FTS */
enum {
	MLX5E_VLAN_FT_LEVEL = 0,
	MLX5E_L2_FT_LEVEL,
	MLX5E_TTC_FT_LEVEL,
	MLX5E_ARFS_FT_LEVEL
};

struct mlx5e_flow_steering {
	struct mlx5_flow_namespace      *ns;
	struct mlx5e_tc_flow_table	tc;
	struct mlx5e_flow_table		vlan;
	struct mlx5e_flow_table		main;
	struct mlx5e_tc_table           tc;
	struct mlx5e_vlan_table         vlan;
	struct mlx5e_l2_table           l2;
	struct mlx5e_ttc_table          ttc;
	struct mlx5e_arfs_tables        arfs;
};

struct mlx5e_direct_tir {
	u32              tirn;
	u32              rqtn;
};

enum {
	MLX5E_TC_PRIO = 0,
	MLX5E_NIC_PRIO
};

struct mlx5e_priv {
@@ -470,12 +517,11 @@ struct mlx5e_priv {

	struct mlx5e_channel     **channel;
	u32                        tisn[MLX5E_MAX_NUM_TC];
	u32                        rqtn[MLX5E_NUM_RQT];
	u32                        tirn[MLX5E_NUM_TT];
	u32                        indir_rqtn;
	u32                        indir_tirn[MLX5E_NUM_INDIR_TIRS];
	struct mlx5e_direct_tir    direct_tir[MLX5E_MAX_NUM_CHANNELS];

	struct mlx5e_flow_tables   fts;
	struct mlx5e_eth_addr_db   eth_addr;
	struct mlx5e_vlan_db       vlan;
	struct mlx5e_flow_steering fs;
	struct mlx5e_vxlan_db      vxlan;

	struct mlx5e_params        params;
@@ -557,9 +603,10 @@ struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq);

void mlx5e_update_stats(struct mlx5e_priv *priv);

int mlx5e_create_flow_tables(struct mlx5e_priv *priv);
void mlx5e_destroy_flow_tables(struct mlx5e_priv *priv);
void mlx5e_init_eth_addr(struct mlx5e_priv *priv);
int mlx5e_create_flow_steering(struct mlx5e_priv *priv);
void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv);
void mlx5e_init_l2_addr(struct mlx5e_priv *priv);
void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft);
void mlx5e_set_rx_mode_work(struct work_struct *work);

void mlx5e_fill_hwstamp(struct mlx5e_tstamp *clock, u64 timestamp,
@@ -578,7 +625,7 @@ void mlx5e_disable_vlan_filter(struct mlx5e_priv *priv);

int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd);

int mlx5e_redirect_rqt(struct mlx5e_priv *priv, enum mlx5e_rqt_ix rqt_ix);
int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix);
void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv);

int mlx5e_open_locked(struct net_device *netdev);
@@ -636,6 +683,32 @@ extern const struct dcbnl_rtnl_ops mlx5e_dcbnl_ops;
int mlx5e_dcbnl_ieee_setets_core(struct mlx5e_priv *priv, struct ieee_ets *ets);
#endif

#ifndef CONFIG_RFS_ACCEL
static inline int mlx5e_arfs_create_tables(struct mlx5e_priv *priv)
{
	return 0;
}

static inline void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) {}

static inline int mlx5e_arfs_enable(struct mlx5e_priv *priv)
{
	return -ENOTSUPP;
}

static inline int mlx5e_arfs_disable(struct mlx5e_priv *priv)
{
	return -ENOTSUPP;
}
#else
int mlx5e_arfs_create_tables(struct mlx5e_priv *priv);
void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv);
int mlx5e_arfs_enable(struct mlx5e_priv *priv);
int mlx5e_arfs_disable(struct mlx5e_priv *priv);
int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
			u16 rxq_index, u32 flow_id);
#endif

u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev);

#endif /* __MLX5_EN_H__ */
+749 −0

File added.

Preview size limit exceeded, changes collapsed.

+20 −4
Original line number Diff line number Diff line
@@ -456,6 +456,7 @@ static int mlx5e_set_channels(struct net_device *dev,
	struct mlx5e_priv *priv = netdev_priv(dev);
	int ncv = mlx5e_get_max_num_channels(priv->mdev);
	unsigned int count = ch->combined_count;
	bool arfs_enabled;
	bool was_opened;
	int err = 0;

@@ -484,13 +485,27 @@ static int mlx5e_set_channels(struct net_device *dev,
	if (was_opened)
		mlx5e_close_locked(dev);

	arfs_enabled = dev->features & NETIF_F_NTUPLE;
	if (arfs_enabled)
		mlx5e_arfs_disable(priv);

	priv->params.num_channels = count;
	mlx5e_build_default_indir_rqt(priv->mdev, priv->params.indirection_rqt,
				      MLX5E_INDIR_RQT_SIZE, count);

	if (was_opened)
		err = mlx5e_open_locked(dev);
	if (err)
		goto out;

	if (arfs_enabled) {
		err = mlx5e_arfs_enable(priv);
		if (err)
			netdev_err(dev, "%s: mlx5e_arfs_enable failed: %d\n",
				   __func__, err);
	}

out:
	mutex_unlock(&priv->state_lock);

	return err;
@@ -826,9 +841,8 @@ static void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in, int inlen)
	MLX5_SET(modify_tir_in, in, bitmask.hash, 1);
	mlx5e_build_tir_ctx_hash(tirc, priv);

	for (i = 0; i < MLX5E_NUM_TT; i++)
		if (IS_HASHING_TT(i))
			mlx5_core_modify_tir(mdev, priv->tirn[i], in, inlen);
	for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
		mlx5_core_modify_tir(mdev, priv->indir_tirn[i], in, inlen);
}

static int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir,
@@ -850,9 +864,11 @@ static int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir,
	mutex_lock(&priv->state_lock);

	if (indir) {
		u32 rqtn = priv->indir_rqtn;

		memcpy(priv->params.indirection_rqt, indir,
		       sizeof(priv->params.indirection_rqt));
		mlx5e_redirect_rqt(priv, MLX5E_INDIRECTION_RQT);
		mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
	}

	if (key)
Loading