Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 61f3d4bf authored by James Smart's avatar James Smart Committed by Martin K. Petersen
Browse files

scsi: lpfc: Fix nvmet RQ resource needs for large block writes. 



Large block writes to the nvme target were failing because the default
number of RQs posted was insufficient.

Expand the NVMET RQs to 2048 RQEs and ensure a minimum of 512 RQEs are
posted, no matter how many MRQs are configured.

Signed-off-by: default avatarDick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: default avatarJames Smart <james.smart@broadcom.com>
Reviewed-by: default avatarHannes Reinecke <hare@suse.de>
Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parent 547077a4

drivers/scsi/lpfc/lpfc_attr.c

+3 −3
Original line number Diff line number Diff line
@@ -60,9 +60,9 @@ 
#define LPFC_MIN_DEVLOSS_TMO	1

#define LPFC_MAX_DEVLOSS_TMO	255



#define LPFC_DEF_MRQ_POST	256

#define LPFC_MIN_MRQ_POST	32

#define LPFC_MAX_MRQ_POST	512

#define LPFC_DEF_MRQ_POST	512

#define LPFC_MIN_MRQ_POST	512

#define LPFC_MAX_MRQ_POST	2048



/*

 * Write key size should be multiple of 4. If write key is changed

drivers/scsi/lpfc/lpfc_init.c

+15 −8
Original line number Diff line number Diff line
@@ -3390,6 +3390,11 @@ lpfc_sli4_nvmet_sgl_update(struct lpfc_hba *phba) 
	 */

	els_xri_cnt = lpfc_sli4_get_els_iocb_cnt(phba);

	nvmet_xri_cnt = phba->cfg_nvmet_mrq * phba->cfg_nvmet_mrq_post;



	/* Ensure we at least meet the minimun for the system */

	if (nvmet_xri_cnt < LPFC_NVMET_RQE_DEF_COUNT)

		nvmet_xri_cnt = LPFC_NVMET_RQE_DEF_COUNT;



	tot_cnt = phba->sli4_hba.max_cfg_param.max_xri - els_xri_cnt;

	if (nvmet_xri_cnt > tot_cnt) {

		phba->cfg_nvmet_mrq_post = tot_cnt / phba->cfg_nvmet_mrq;
@@ -8158,7 +8163,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) 
			/* Create NVMET Receive Queue for header */

			qdesc = lpfc_sli4_queue_alloc(phba,

						      phba->sli4_hba.rq_esize,

						      phba->sli4_hba.rq_ecount);

						      LPFC_NVMET_RQE_DEF_COUNT);

			if (!qdesc) {

				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,

						"3146 Failed allocate "
@@ -8180,7 +8185,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba) 
			/* Create NVMET Receive Queue for data */

			qdesc = lpfc_sli4_queue_alloc(phba,

						      phba->sli4_hba.rq_esize,

						      phba->sli4_hba.rq_ecount);

						      LPFC_NVMET_RQE_DEF_COUNT);

			if (!qdesc) {

				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,

						"3156 Failed allocate "
@@ -8770,9 +8775,6 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba) 
		goto out_destroy;

	}



	lpfc_rq_adjust_repost(phba, phba->sli4_hba.hdr_rq, LPFC_ELS_HBQ);

	lpfc_rq_adjust_repost(phba, phba->sli4_hba.dat_rq, LPFC_ELS_HBQ);



	rc = lpfc_rq_create(phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq,

			    phba->sli4_hba.els_cq, LPFC_USOL);

	if (rc) {
@@ -11096,7 +11098,7 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) 
	struct lpfc_hba   *phba;

	struct lpfc_vport *vport = NULL;

	struct Scsi_Host  *shost = NULL;

	int error, cnt;

	int error, cnt, num;

	uint32_t cfg_mode, intr_mode;



	/* Allocate memory for HBA structure */
@@ -11131,8 +11133,13 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid) 
	}



	cnt = phba->cfg_iocb_cnt * 1024;

	if (phba->nvmet_support)

		cnt += phba->cfg_nvmet_mrq_post * phba->cfg_nvmet_mrq;

	if (phba->nvmet_support) {

		/* Ensure we at least meet the minimun for the system */

		num = (phba->cfg_nvmet_mrq_post * phba->cfg_nvmet_mrq);

		if (num < LPFC_NVMET_RQE_DEF_COUNT)

			num = LPFC_NVMET_RQE_DEF_COUNT;

		cnt += num;

	}



	/* Initialize and populate the iocb list per host */

	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,

drivers/scsi/lpfc/lpfc_nvmet.c

+1 −1
Original line number Diff line number Diff line
@@ -614,9 +614,9 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, 
	lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",

			 ctxp->oxid, rsp->op, rsp->rsplen);



	ctxp->flag |= LPFC_NVMET_IO_INP;

	rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq);

	if (rc == WQE_SUCCESS) {

		ctxp->flag |= LPFC_NVMET_IO_INP;

#ifdef CONFIG_SCSI_LPFC_DEBUG_FS

		if (!phba->ktime_on)

			return 0;

drivers/scsi/lpfc/lpfc_nvmet.h

+1 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 
 ********************************************************************/



#define LPFC_NVMET_DEFAULT_SEGS		(64 + 1)	/* 256K IOs */

#define LPFC_NVMET_RQE_DEF_COUNT	512

#define LPFC_NVMET_SUCCESS_LEN	12



/* Used for NVME Target */

drivers/scsi/lpfc/lpfc_sli.c

+10 −39
Original line number Diff line number Diff line
@@ -479,22 +479,23 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, 
	if (unlikely(!hq) || unlikely(!dq))

		return -ENOMEM;

	put_index = hq->host_index;

	temp_hrqe = hq->qe[hq->host_index].rqe;

	temp_hrqe = hq->qe[put_index].rqe;

	temp_drqe = dq->qe[dq->host_index].rqe;















	if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ)















		return -EINVAL;













	if (hq->host_index != dq->host_index)













	if (put_index != dq->host_index)















		return -EINVAL;















	/* If the host has not yet processed the next entry then we are done */













	if (((hq->host_index + 1) % hq->entry_count) == hq->hba_index)













	if (((put_index + 1) % hq->entry_count) == hq->hba_index)















		return -EBUSY;















	lpfc_sli_pcimem_bcopy(hrqe, temp_hrqe, hq->entry_size);















	lpfc_sli_pcimem_bcopy(drqe, temp_drqe, dq->entry_size);





























	/* Update the host index to point to the next slot */













	hq->host_index = ((hq->host_index + 1) % hq->entry_count);













	hq->host_index = ((put_index + 1) % hq->entry_count);















	dq->host_index = ((dq->host_index + 1) % dq->entry_count);













	hq->RQ_buf_posted++;





























	/* Ring The Header Receive Queue Doorbell */















	if (!(hq->host_index % hq->entry_repost)) {









@@ -512,7 +513,6 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,













		} else {















			return -EINVAL;















		}













		hq->RQ_buf_posted += hq->entry_repost;















		writel(doorbell.word0, hq->db_regaddr);















	}















	return put_index;










@@ -6905,14 +6905,9 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)













			INIT_LIST_HEAD(&rqbp->rqb_buffer_list);















			rqbp->rqb_alloc_buffer = lpfc_sli4_nvmet_alloc;















			rqbp->rqb_free_buffer = lpfc_sli4_nvmet_free;













			rqbp->entry_count = 256;













			rqbp->entry_count = LPFC_NVMET_RQE_DEF_COUNT;















			rqbp->buffer_count = 0;



























			/* Divide by 4 and round down to multiple of 16 */













			rc = (phba->cfg_nvmet_mrq_post >> 2) & 0xfff8;













			phba->sli4_hba.nvmet_mrq_hdr[i]->entry_repost = rc;













			phba->sli4_hba.nvmet_mrq_data[i]->entry_repost = rc;



























			lpfc_post_rq_buffer(















				phba, phba->sli4_hba.nvmet_mrq_hdr[i],















				phba->sli4_hba.nvmet_mrq_data[i],










@@ -14892,34 +14887,6 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq,













	return status;















}



























/**













 * lpfc_rq_adjust_repost - Adjust entry_repost for an RQ













 * @phba: HBA structure that indicates port to create a queue on.













 * @rq:   The queue structure to use for the receive queue.













 * @qno:  The associated HBQ number













 *













 *













 * For SLI4 we need to adjust the RQ repost value based on













 * the number of buffers that are initially posted to the RQ.













 */













void













lpfc_rq_adjust_repost(struct lpfc_hba *phba, struct lpfc_queue *rq, int qno)













{













	uint32_t cnt;

























	/* sanity check on queue memory */













	if (!rq)













		return;













	cnt = lpfc_hbq_defs[qno]->entry_count;

























	/* Recalc repost for RQs based on buffers initially posted */













	cnt = (cnt >> 3);













	if (cnt < LPFC_QUEUE_MIN_REPOST)













		cnt = LPFC_QUEUE_MIN_REPOST;

























	rq->entry_repost = cnt;













}



























/**















 * lpfc_rq_create - Create a Receive Queue on the HBA















 * @phba: HBA structure that indicates port to create a queue on.










@@ -15105,6 +15072,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq,













	hrq->subtype = subtype;















	hrq->host_index = 0;















	hrq->hba_index = 0;













	hrq->entry_repost = LPFC_RQ_REPOST;





























	/* now create the data queue */















	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE,










@@ -15186,6 +15154,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq,













	drq->subtype = subtype;















	drq->host_index = 0;















	drq->hba_index = 0;













	drq->entry_repost = LPFC_RQ_REPOST;





























	/* link the header and data RQs onto the parent cq child list */















	list_add_tail(&hrq->list, &cq->child_list);










@@ -15343,6 +15312,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp,













		hrq->subtype = subtype;















		hrq->host_index = 0;















		hrq->hba_index = 0;













		hrq->entry_repost = LPFC_RQ_REPOST;





























		drq->db_format = LPFC_DB_RING_FORMAT;















		drq->db_regaddr = phba->sli4_hba.RQDBregaddr;









@@ -15351,6 +15321,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp,













		drq->subtype = subtype;















		drq->host_index = 0;















		drq->hba_index = 0;













		drq->entry_repost = LPFC_RQ_REPOST;





























		list_add_tail(&hrq->list, &cq->child_list);















		list_add_tail(&drq->list, &cq->child_list);