Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ddae0349 authored by Eugenia Emantayev's avatar Eugenia Emantayev Committed by David S. Miller
Browse files

net/mlx4: Change QP allocation scheme 



When using BF (Blue-Flame), the QPN overrides the VLAN, CV, and SV fields
in the WQE. Thus, BF may only be used for QPNs with bits 6,7 unset.

The current Ethernet driver code reserves a Tx QP range with 256b alignment.

This is wrong because if there are more than 64 Tx QPs in use,
QPNs >= base + 65 will have bits 6/7 set.

This problem is not specific for the Ethernet driver, any entity that
tries to reserve more than 64 BF-enabled QPs should fail. Also, using
ranges is not necessary here and is wasteful.

The new mechanism introduced here will support reservation for
"Eth QPs eligible for BF" for all drivers: bare-metal, multi-PF, and VFs
(when hypervisors support WC in VMs). The flow we use is:

1. In mlx4_en, allocate Tx QPs one by one instead of a range allocation,
   and request "BF enabled QPs" if BF is supported for the function

2. In the ALLOC_RES FW command, change param1 to:
a. param1[23:0]  - number of QPs
b. param1[31-24] - flags controlling QPs reservation

Bit 31 refers to Eth blueflame supported QPs. Those QPs must have
bits 6 and 7 unset in order to be used in Ethernet.

Bits 24-30 of the flags are currently reserved.

When a function tries to allocate a QP, it states the required attributes
for this QP. Those attributes are considered "best-effort". If an attribute,
such as Ethernet BF enabled QP, is a must-have attribute, the function has
to check that attribute is supported before trying to do the allocation.

In a lower layer of the code, mlx4_qp_reserve_range masks out the bits
which are unsupported. If SRIOV is used, the PF validates those attributes
and masks out unsupported attributes as well. In order to notify VFs which
attributes are supported, the VF uses QUERY_FUNC_CAP command. This command's
mailbox is filled by the PF, which notifies which QP allocation attributes
it supports.

Signed-off-by: default avatarEugenia Emantayev <eugenia@mellanox.co.il>
Signed-off-by: default avatarMatan Barak <matanb@mellanox.com>
Signed-off-by: default avatarOr Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 3dca0f42

drivers/infiniband/hw/mlx4/main.c

+1 −1
Original line number Diff line number Diff line
@@ -2227,7 +2227,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) 
		ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;

		err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,

					    MLX4_IB_UC_STEER_QPN_ALIGN,

					    &ibdev->steer_qpn_base);

					    &ibdev->steer_qpn_base, 0);

		if (err)

			goto err_counter;

drivers/infiniband/hw/mlx4/qp.c

+7 −4
Original line number Diff line number Diff line
@@ -802,16 +802,19 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, 
			}

		}

	} else {

		/* Raw packet QPNs must be aligned to 8 bits. If not, the WQE

		 * BlueFlame setup flow wrongly causes VLAN insertion. */

		/* Raw packet QPNs may not have bits 6,7 set in their qp_num;

		 * otherwise, the WQE BlueFlame setup flow wrongly causes

		 * VLAN insertion. */

		if (init_attr->qp_type == IB_QPT_RAW_PACKET)

			err = mlx4_qp_reserve_range(dev->dev, 1, 1 << 8, &qpn);

			err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,

						    init_attr->cap.max_send_wr ?

						    MLX4_RESERVE_ETH_BF_QP : 0);

		else

			if (qp->flags & MLX4_IB_QP_NETIF)

				err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);

			else

				err = mlx4_qp_reserve_range(dev->dev, 1, 1,

							    &qpn);

							    &qpn, 0);

		if (err)

			goto err_proxy;

	}

drivers/net/ethernet/mellanox/mlx4/alloc.c

+37 −6
Original line number Diff line number Diff line
@@ -76,22 +76,53 @@ void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 
	mlx4_bitmap_free_range(bitmap, obj, 1, use_rr);

}



u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)

static unsigned long find_aligned_range(unsigned long *bitmap,

					u32 start, u32 nbits,

					int len, int align, u32 skip_mask)

{

	unsigned long end, i;



again:

	start = ALIGN(start, align);



	while ((start < nbits) && (test_bit(start, bitmap) ||

				   (start & skip_mask)))

		start += align;



	if (start >= nbits)

		return -1;



	end = start+len;

	if (end > nbits)

		return -1;



	for (i = start + 1; i < end; i++) {

		if (test_bit(i, bitmap) || ((u32)i & skip_mask)) {

			start = i + 1;

			goto again;

		}

	}



	return start;

}



u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,

			    int align, u32 skip_mask)

{

	u32 obj;



	if (likely(cnt == 1 && align == 1))

	if (likely(cnt == 1 && align == 1 && !skip_mask))

		return mlx4_bitmap_alloc(bitmap);



	spin_lock(&bitmap->lock);



	obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,

				bitmap->last, cnt, align - 1);

	obj = find_aligned_range(bitmap->table, bitmap->last,

				 bitmap->max, cnt, align, skip_mask);

	if (obj >= bitmap->max) {

		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)

				& bitmap->mask;

		obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,

						0, cnt, align - 1);

		obj = find_aligned_range(bitmap->table, 0, bitmap->max,

					 cnt, align, skip_mask);

	}



	if (obj < bitmap->max) {

drivers/net/ethernet/mellanox/mlx4/en_netdev.c

+1 −9
Original line number Diff line number Diff line
@@ -595,7 +595,7 @@ static int mlx4_en_get_qp(struct mlx4_en_priv *priv) 
		return 0;

	}



	err = mlx4_qp_reserve_range(dev, 1, 1, qpn);

	err = mlx4_qp_reserve_range(dev, 1, 1, qpn, 0);

	en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);

	if (err) {

		en_err(priv, "Failed to reserve qp for mac registration\n");
@@ -1974,15 +1974,8 @@ int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) 
{

	struct mlx4_en_port_profile *prof = priv->prof;

	int i;

	int err;

	int node;



	err = mlx4_qp_reserve_range(priv->mdev->dev, priv->tx_ring_num, 256, &priv->base_tx_qpn);

	if (err) {

		en_err(priv, "failed reserving range for TX rings\n");

		return err;

	}



	/* Create tx Rings */

	for (i = 0; i < priv->tx_ring_num; i++) {

		node = cpu_to_node(i % num_online_cpus());
@@ -1991,7 +1984,6 @@ int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) 
			goto err;



		if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i],

					   priv->base_tx_qpn + i,

					   prof->tx_ring_size, TXBB_SIZE,

					   node, i))

			goto err;

drivers/net/ethernet/mellanox/mlx4/en_rx.c

+2 −2
Original line number Diff line number Diff line
@@ -1131,7 +1131,7 @@ int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) 
	int err;

	u32 qpn;



	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn);

	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0);

	if (err) {

		en_err(priv, "Failed reserving drop qpn\n");

		return err;
@@ -1174,7 +1174,7 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) 
	en_dbg(DRV, priv, "Configuring rss steering\n");

	err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,

				    priv->rx_ring_num,

				    &rss_map->base_qpn);

				    &rss_map->base_qpn, 0);

	if (err) {

		en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);

		return err;