Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7796d2a3 authored by Max Gurtovoy's avatar Max Gurtovoy Committed by Jason Gunthorpe
Browse files

RDMA/mlx5: Refactor MR descriptors allocation 



Improve code readability using static helpers for each memory region
type. Re-use the common logic to get smaller functions that are easy
to maintain and reduce code duplication.

Signed-off-by: default avatarMax Gurtovoy <maxg@mellanox.com>
Signed-off-by: default avatarIsrael Rukshin <israelr@mellanox.com>
Reviewed-by: default avatarSagi Grimberg <sagi@grimberg.me>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent 2563e2f3

drivers/infiniband/hw/mlx5/mr.c

+157 −133
Original line number Original line Diff line number Diff line
@@ -1651,80 +1651,63 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 
	return 0;

	return 0;

}

}





static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,

static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs,

				u32 max_num_sg, u32 max_num_meta_sg,

				   int access_mode, int page_shift)

				int desc_size, int access_mode)

{

{

	struct mlx5_ib_dev *dev = to_mdev(pd->device);

	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);

	int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);

	struct mlx5_ib_mr *mr;

	void *mkc;

	void *mkc;

	u32 *in;

	int err;



	mr = kzalloc(sizeof(*mr), GFP_KERNEL);

	if (!mr)

		return ERR_PTR(-ENOMEM);



	in = kzalloc(inlen, GFP_KERNEL);

	if (!in) {

		err = -ENOMEM;

		goto err_free;

	}





	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);

	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);



	MLX5_SET(mkc, mkc, free, 1);

	MLX5_SET(mkc, mkc, free, 1);

	MLX5_SET(mkc, mkc, translations_octword_size, ndescs);

	if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)

		MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);

	MLX5_SET(mkc, mkc, qpn, 0xffffff);

	MLX5_SET(mkc, mkc, qpn, 0xffffff);

	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);

	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);

	MLX5_SET(mkc, mkc, translations_octword_size, ndescs);

	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);

	MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);

	MLX5_SET(mkc, mkc, umr_en, 1);

	MLX5_SET(mkc, mkc, log_page_size, page_shift);

}



static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,

				  int ndescs, int desc_size, int page_shift,

				  int access_mode, u32 *in, int inlen)

{

	struct mlx5_ib_dev *dev = to_mdev(pd->device);

	int err;





	mr->access_mode = access_mode;

	mr->access_mode = access_mode;

	mr->desc_size = desc_size;

	mr->max_descs = ndescs;





	err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size);

	err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size);

	if (err)

	if (err)

		goto err_free_in;

		return err;

	mr->desc_size = desc_size;

	mr->max_descs = ndescs;





	MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3);

	mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift);

	MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7);

	MLX5_SET(mkc, mkc, umr_en, 1);





	mr->ibmr.pd = pd;

	mr->ibmr.device = pd->device;

	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);

	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);

	if (err)

	if (err)

		goto err_priv_descs;

		goto err_free_descs;





	mr->mmkey.type = MLX5_MKEY_MR;

	mr->mmkey.type = MLX5_MKEY_MR;

	mr->ibmr.lkey = mr->mmkey.key;

	mr->ibmr.lkey = mr->mmkey.key;

	mr->ibmr.rkey = mr->mmkey.key;

	mr->ibmr.rkey = mr->mmkey.key;

	mr->umem = NULL;

	kfree(in);





	return mr;

	return 0;





err_priv_descs:

err_free_descs:

	mlx5_free_priv_descs(mr);

	mlx5_free_priv_descs(mr);

err_free_in:

	return err;

	kfree(in);

err_free:

	kfree(mr);

	return ERR_PTR(err);

}

}





static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,

static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,

					enum ib_mr_type mr_type, u32 max_num_sg,

				u32 max_num_sg, u32 max_num_meta_sg,

					u32 max_num_meta_sg)

				int desc_size, int access_mode)

{

{

	struct mlx5_ib_dev *dev = to_mdev(pd->device);

	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);

	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);

	int ndescs = ALIGN(max_num_sg, 4);

	int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);

	int page_shift = 0;

	struct mlx5_ib_mr *mr;

	struct mlx5_ib_mr *mr;

	void *mkc;

	u32 *in;

	u32 *in;

	int err;

	int err;
@@ -1732,57 +1715,68 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, 
	if (!mr)

	if (!mr)

		return ERR_PTR(-ENOMEM);

		return ERR_PTR(-ENOMEM);





	mr->ibmr.pd = pd;

	mr->ibmr.device = pd->device;



	in = kzalloc(inlen, GFP_KERNEL);

	in = kzalloc(inlen, GFP_KERNEL);

	if (!in) {

	if (!in) {

		err = -ENOMEM;

		err = -ENOMEM;

		goto err_free;

		goto err_free;

	}

	}





	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);

	if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)

	MLX5_SET(mkc, mkc, free, 1);

		page_shift = PAGE_SHIFT;

	MLX5_SET(mkc, mkc, qpn, 0xffffff);

	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);





	if (mr_type == IB_MR_TYPE_MEM_REG) {

	err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift,

		mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;

				     access_mode, in, inlen);

		MLX5_SET(mkc, mkc, translations_octword_size, ndescs);

		MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);

		err = mlx5_alloc_priv_descs(pd->device, mr,

					    ndescs, sizeof(struct mlx5_mtt));

	if (err)

	if (err)

		goto err_free_in;

		goto err_free_in;





		mr->desc_size = sizeof(struct mlx5_mtt);

	mr->umem = NULL;

		mr->max_descs = ndescs;

	kfree(in);

	} else if (mr_type == IB_MR_TYPE_SG_GAPS) {

		mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;

		MLX5_SET(mkc, mkc, translations_octword_size, ndescs);





		err = mlx5_alloc_priv_descs(pd->device, mr,

	return mr;

					    ndescs, sizeof(struct mlx5_klm));



		if (err)

err_free_in:

			goto err_free_in;

	kfree(in);

		mr->desc_size = sizeof(struct mlx5_klm);

err_free:

		mr->max_descs = ndescs;

	kfree(mr);

	} else if (mr_type == IB_MR_TYPE_INTEGRITY) {

	return ERR_PTR(err);

}



static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,

				    int ndescs, u32 *in, int inlen)

{

	return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt),

				      PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in,

				      inlen);

}



static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,

				    int ndescs, u32 *in, int inlen)

{

	return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm),

				      0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);

}



static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,

				      int max_num_sg, int max_num_meta_sg,

				      u32 *in, int inlen)

{

	struct mlx5_ib_dev *dev = to_mdev(pd->device);

	u32 psv_index[2];

	u32 psv_index[2];

	void *mkc;

	int err;





		MLX5_SET(mkc, mkc, bsf_en, 1);

		MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);

		MLX5_SET(mkc, mkc, translations_octword_size, 4);

	mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);

	mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);

		if (!mr->sig) {

	if (!mr->sig)

			err = -ENOMEM;

		return -ENOMEM;

			goto err_free_in;

		}





	/* create mem & wire PSVs */

	/* create mem & wire PSVs */

		err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn,

	err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index);

					   2, psv_index);

	if (err)

	if (err)

		goto err_free_sig;

		goto err_free_sig;





		mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;

	mr->sig->psv_memory.psv_idx = psv_index[0];

	mr->sig->psv_memory.psv_idx = psv_index[0];

	mr->sig->psv_wire.psv_idx = psv_index[1];

	mr->sig->psv_wire.psv_idx = psv_index[1];
@@ -1790,69 +1784,99 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, 
	mr->sig->sig_err_exists = false;

	mr->sig->sig_err_exists = false;

	/* Next UMR, Arm SIGERR */

	/* Next UMR, Arm SIGERR */

	++mr->sig->sigerr_count;

	++mr->sig->sigerr_count;

		mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg,

	mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,

						max_num_meta_sg,

					 sizeof(struct mlx5_klm),

					 sizeof(struct mlx5_klm),

					 MLX5_MKC_ACCESS_MODE_KLMS);

					 MLX5_MKC_ACCESS_MODE_KLMS);

	if (IS_ERR(mr->klm_mr)) {

	if (IS_ERR(mr->klm_mr)) {

		err = PTR_ERR(mr->klm_mr);

		err = PTR_ERR(mr->klm_mr);

		goto err_destroy_psv;

		goto err_destroy_psv;

	}

	}

		mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg,

	mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,

						max_num_meta_sg,

					 sizeof(struct mlx5_mtt),

					 sizeof(struct mlx5_mtt),

					 MLX5_MKC_ACCESS_MODE_MTT);

					 MLX5_MKC_ACCESS_MODE_MTT);

	if (IS_ERR(mr->mtt_mr)) {

	if (IS_ERR(mr->mtt_mr)) {

		err = PTR_ERR(mr->mtt_mr);

		err = PTR_ERR(mr->mtt_mr);

		goto err_free_klm_mr;

		goto err_free_klm_mr;

	}

	}

	} else {

		mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);

		err = -EINVAL;

		goto err_free_in;

	}





	MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3);

	/* Set bsf descriptors for mkey */

	MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7);

	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);

	MLX5_SET(mkc, mkc, umr_en, 1);

	MLX5_SET(mkc, mkc, bsf_en, 1);

	MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);





	mr->ibmr.device = pd->device;

	err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0,

	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);

				     MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);

	if (err)

	if (err)

		goto err_free_pi_mr;

		goto err_free_mtt_mr;





	mr->mmkey.type = MLX5_MKEY_MR;

	return 0;

	mr->ibmr.lkey = mr->mmkey.key;

	mr->ibmr.rkey = mr->mmkey.key;

	mr->umem = NULL;

	kfree(in);



	return &mr->ibmr;





err_free_pi_mr:

err_free_mtt_mr:

	if (mr->mtt_mr) {

	dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);

	dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);

	mr->mtt_mr = NULL;

	mr->mtt_mr = NULL;

	}

err_free_klm_mr:

err_free_klm_mr:

	if (mr->klm_mr) {

	dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr);

	dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr);

	mr->klm_mr = NULL;

	mr->klm_mr = NULL;

	}

err_destroy_psv:

err_destroy_psv:

	if (mr->sig) {

	if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx))

		if (mlx5_core_destroy_psv(dev->mdev,

					  mr->sig->psv_memory.psv_idx))

		mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",

		mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",

			     mr->sig->psv_memory.psv_idx);

			     mr->sig->psv_memory.psv_idx);

		if (mlx5_core_destroy_psv(dev->mdev,

	if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx))

					  mr->sig->psv_wire.psv_idx))

		mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",

		mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",

			     mr->sig->psv_wire.psv_idx);

			     mr->sig->psv_wire.psv_idx);

	}

	mlx5_free_priv_descs(mr);

err_free_sig:

err_free_sig:

	kfree(mr->sig);

	kfree(mr->sig);



	return err;

}



static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,

					enum ib_mr_type mr_type, u32 max_num_sg,

					u32 max_num_meta_sg)

{

	struct mlx5_ib_dev *dev = to_mdev(pd->device);

	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);

	int ndescs = ALIGN(max_num_sg, 4);

	struct mlx5_ib_mr *mr;

	u32 *in;

	int err;



	mr = kzalloc(sizeof(*mr), GFP_KERNEL);

	if (!mr)

		return ERR_PTR(-ENOMEM);



	in = kzalloc(inlen, GFP_KERNEL);

	if (!in) {

		err = -ENOMEM;

		goto err_free;

	}



	mr->ibmr.device = pd->device;

	mr->umem = NULL;



	switch (mr_type) {

	case IB_MR_TYPE_MEM_REG:

		err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen);

		break;

	case IB_MR_TYPE_SG_GAPS:

		err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen);

		break;

	case IB_MR_TYPE_INTEGRITY:

		err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg,

						 max_num_meta_sg, in, inlen);

		break;

	default:

		mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);

		err = -EINVAL;

	}



	if (err)

		goto err_free_in;



	kfree(in);



	return &mr->ibmr;



err_free_in:

err_free_in:

	kfree(in);

	kfree(in);

err_free:

err_free: