[SCSI] be2iscsi: Adding Ring Mode Wrb's V3 (35e66019) · Commits · e / devices / android_kernel_oneplus_sm7250

drivers/scsi/be2iscsi/be_cmds.c

+3 −3

Original line number	Original line	Diff line number	Diff line
	@@ -125,7 +125,7 @@ static void beiscsi_cq_notify(struct beiscsi_hba *phba, u16 qid, bool arm,
	}		}


	int be_process_mcc(struct beiscsi_hba *phba)		int beiscsi_process_mcc(struct beiscsi_hba *phba)
	{		{
	struct be_mcc_compl *compl;		struct be_mcc_compl *compl;
	int num = 0, status = 0;		int num = 0, status = 0;
	@@ -161,7 +161,7 @@ static int be_mcc_wait_compl(struct beiscsi_hba *phba)
	#define mcc_timeout 120000 /* 5s timeout */		#define mcc_timeout 120000 /* 5s timeout */
	int i, status;		int i, status;
	for (i = 0; i < mcc_timeout; i++) {		for (i = 0; i < mcc_timeout; i++) {
	status = be_process_mcc(phba);		status = beiscsi_process_mcc(phba);
	if (status)		if (status)
	return status;		return status;

	@@ -504,7 +504,7 @@ static u32 be_encoded_q_len(int q_len)
	return len_encoded;		return len_encoded;
	}		}

	int be_cmd_mccq_create(struct beiscsi_hba *phba,		int beiscsi_cmd_mccq_create(struct beiscsi_hba *phba,
	struct be_queue_info *mccq,		struct be_queue_info *mccq,
	struct be_queue_info *cq)		struct be_queue_info *cq)
	{		{

drivers/scsi/be2iscsi/be_cmds.h

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -418,7 +418,7 @@ int beiscsi_cmd_cq_create(struct be_ctrl_info *ctrl,

	int beiscsi_cmd_q_destroy(struct be_ctrl_info ctrl, struct be_queue_info q,		int beiscsi_cmd_q_destroy(struct be_ctrl_info ctrl, struct be_queue_info q,
	int type);		int type);
	int be_cmd_mccq_create(struct beiscsi_hba *phba,		int beiscsi_cmd_mccq_create(struct beiscsi_hba *phba,
	struct be_queue_info *mccq,		struct be_queue_info *mccq,
	struct be_queue_info *cq);		struct be_queue_info *cq);

drivers/scsi/be2iscsi/be_main.c

+165 −44

Original line number	Original line	Diff line number	Diff line
	@@ -40,6 +40,7 @@
	static unsigned int be_iopoll_budget = 10;		static unsigned int be_iopoll_budget = 10;
	static unsigned int be_max_phys_size = 64;		static unsigned int be_max_phys_size = 64;
	static unsigned int enable_msix = 1;		static unsigned int enable_msix = 1;
			static unsigned int ring_mode;

	MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);		MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
	MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);		MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
	@@ -670,8 +671,9 @@ static void
	free_wrb_handle(struct beiscsi_hba phba, struct hwi_wrb_context pwrb_context,		free_wrb_handle(struct beiscsi_hba phba, struct hwi_wrb_context pwrb_context,
	struct wrb_handle *pwrb_handle)		struct wrb_handle *pwrb_handle)
	{		{
			if (!ring_mode)
	pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;		pwrb_context->pwrb_handle_base[pwrb_context->free_index] =
			pwrb_handle;
	pwrb_context->wrb_handles_available++;		pwrb_context->wrb_handles_available++;
	if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))		if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
	pwrb_context->free_index = 0;		pwrb_context->free_index = 0;
	@@ -857,6 +859,7 @@ hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
	{		{
	struct hwi_wrb_context *pwrb_context;		struct hwi_wrb_context *pwrb_context;
	struct wrb_handle *pwrb_handle = NULL;		struct wrb_handle *pwrb_handle = NULL;
			struct sgl_handle *psgl_handle = NULL;
	struct hwi_controller *phwi_ctrlr;		struct hwi_controller *phwi_ctrlr;
	struct iscsi_task *task;		struct iscsi_task *task;
	struct beiscsi_io_task *io_task;		struct beiscsi_io_task *io_task;
	@@ -864,6 +867,14 @@ hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
	struct iscsi_session *session = conn->session;		struct iscsi_session *session = conn->session;

	phwi_ctrlr = phba->phwi_ctrlr;		phwi_ctrlr = phba->phwi_ctrlr;
			if (ring_mode) {
			psgl_handle = phba->sgl_hndl_array[((psol->
			dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
			32] & SOL_ICD_INDEX_MASK) >> 6)];
			pwrb_context = &phwi_ctrlr->wrb_context[psgl_handle->cid];
			task = psgl_handle->task;
			pwrb_handle = NULL;
			} else {
	pwrb_context = &phwi_ctrlr->wrb_context[((psol->		pwrb_context = &phwi_ctrlr->wrb_context[((psol->
	dw[offsetof(struct amap_sol_cqe, cid) / 32] &		dw[offsetof(struct amap_sol_cqe, cid) / 32] &
	SOL_CID_MASK) >> 6)];		SOL_CID_MASK) >> 6)];
	@@ -871,6 +882,8 @@ hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
	dw[offsetof(struct amap_sol_cqe, wrb_index) /		dw[offsetof(struct amap_sol_cqe, wrb_index) /
	32] & SOL_WRB_INDEX_MASK) >> 16)];		32] & SOL_WRB_INDEX_MASK) >> 16)];
	task = pwrb_handle->pio_handle;		task = pwrb_handle->pio_handle;
			}

	io_task = task->dd_data;		io_task = task->dd_data;
	spin_lock(&phba->mgmt_sgl_lock);		spin_lock(&phba->mgmt_sgl_lock);
	free_mgmt_sgl_handle(phba, io_task->psgl_handle);		free_mgmt_sgl_handle(phba, io_task->psgl_handle);
	@@ -910,11 +923,19 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
	struct iscsi_wrb *pwrb = NULL;		struct iscsi_wrb *pwrb = NULL;
	struct hwi_controller *phwi_ctrlr;		struct hwi_controller *phwi_ctrlr;
	struct iscsi_task *task;		struct iscsi_task *task;
			struct sgl_handle *psgl_handle = NULL;
	unsigned int type;		unsigned int type;
	struct iscsi_conn *conn = beiscsi_conn->conn;		struct iscsi_conn *conn = beiscsi_conn->conn;
	struct iscsi_session *session = conn->session;		struct iscsi_session *session = conn->session;

	phwi_ctrlr = phba->phwi_ctrlr;		phwi_ctrlr = phba->phwi_ctrlr;
			if (ring_mode) {
			psgl_handle = phba->sgl_hndl_array[((psol->
			dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
			32] & SOL_ICD_INDEX_MASK) >> 6)];
			task = psgl_handle->task;
			type = psgl_handle->type;
			} else {
	pwrb_context = &phwi_ctrlr->		pwrb_context = &phwi_ctrlr->
	wrb_context[((psol->dw[offsetof		wrb_context[((psol->dw[offsetof
	(struct amap_sol_cqe, cid) / 32]		(struct amap_sol_cqe, cid) / 32]
	@@ -926,7 +947,7 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
	pwrb = pwrb_handle->pwrb;		pwrb = pwrb_handle->pwrb;
	type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &		type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
	WRB_TYPE_MASK) >> 28;		WRB_TYPE_MASK) >> 28;
			}
	spin_lock_bh(&session->lock);		spin_lock_bh(&session->lock);
	switch (type) {		switch (type) {
	case HWH_TYPE_IO:		case HWH_TYPE_IO:
	@@ -957,6 +978,14 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
	break;		break;

	default:		default:
			if (ring_mode)
			shost_printk(KERN_WARNING, phba->shost,
			"In hwi_complete_cmd, unknown type = %d"
			"icd_index 0x%x CID 0x%x\n", type,
			((psol->dw[offsetof(struct amap_sol_cqe_ring,
			icd_index) / 32] & SOL_ICD_INDEX_MASK) >> 6),
			psgl_handle->cid);
			else
	shost_printk(KERN_WARNING, phba->shost,		shost_printk(KERN_WARNING, phba->shost,
	"In hwi_complete_cmd, unknown type = %d"		"In hwi_complete_cmd, unknown type = %d"
	"wrb_index 0x%x CID 0x%x\n", type,		"wrb_index 0x%x CID 0x%x\n", type,
	@@ -966,6 +995,7 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
	cid) / 32] & SOL_CID_MASK) >> 6));		cid) / 32] & SOL_CID_MASK) >> 6));
	break;		break;
	}		}

	spin_unlock_bh(&session->lock);		spin_unlock_bh(&session->lock);
	}		}

	@@ -1401,6 +1431,7 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
	unsigned int num_processed = 0;		unsigned int num_processed = 0;
	unsigned int tot_nump = 0;		unsigned int tot_nump = 0;
	struct beiscsi_conn *beiscsi_conn;		struct beiscsi_conn *beiscsi_conn;
			struct sgl_handle *psgl_handle = NULL;
	struct beiscsi_hba *phba;		struct beiscsi_hba *phba;

	cq = pbe_eq->cq;		cq = pbe_eq->cq;
	@@ -1411,6 +1442,20 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
	CQE_VALID_MASK) {		CQE_VALID_MASK) {
	be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));		be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));

			if (ring_mode) {
			psgl_handle = phba->sgl_hndl_array[((sol->
			dw[offsetof(struct amap_sol_cqe_ring,
			icd_index) / 32] & SOL_ICD_INDEX_MASK)
			>> 6)];
			beiscsi_conn = phba->conn_table[psgl_handle->cid];
			if (!beiscsi_conn \|\| !beiscsi_conn->ep) {
			shost_printk(KERN_WARNING, phba->shost,
			"Connection table empty for cid = %d\n",
			psgl_handle->cid);
			return 0;
			}

			} else {
	beiscsi_conn = phba->conn_table[(u32) (sol->		beiscsi_conn = phba->conn_table[(u32) (sol->
	dw[offsetof(struct amap_sol_cqe, cid) / 32] &		dw[offsetof(struct amap_sol_cqe, cid) / 32] &
	SOL_CID_MASK) >> 6];		SOL_CID_MASK) >> 6];
	@@ -1422,6 +1467,7 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
	cid) / 32] & SOL_CID_MASK) >> 6);		cid) / 32] & SOL_CID_MASK) >> 6);
	return 0;		return 0;
	}		}
			}

	if (num_processed >= 32) {		if (num_processed >= 32) {
	hwi_ring_cq_db(phba, cq->id,		hwi_ring_cq_db(phba, cq->id,
	@@ -1465,6 +1511,13 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
	case CMD_CXN_KILLED_ITT_INVALID:		case CMD_CXN_KILLED_ITT_INVALID:
	case CMD_CXN_KILLED_SEQ_OUTOFORDER:		case CMD_CXN_KILLED_SEQ_OUTOFORDER:
	case CMD_CXN_KILLED_INVALID_DATASN_RCVD:		case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
			if (ring_mode) {
			SE_DEBUG(DBG_LVL_1,
			"CQ Error notification for cmd.. "
			"code %d cid 0x%x\n",
			sol->dw[offsetof(struct amap_sol_cqe, code) /
			32] & CQE_CODE_MASK, psgl_handle->cid);
			} else {
	SE_DEBUG(DBG_LVL_1,		SE_DEBUG(DBG_LVL_1,
	"CQ Error notification for cmd.. "		"CQ Error notification for cmd.. "
	"code %d cid 0x%x\n",		"code %d cid 0x%x\n",
	@@ -1472,6 +1525,7 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
	32] & CQE_CODE_MASK,		32] & CQE_CODE_MASK,
	(sol->dw[offsetof(struct amap_sol_cqe, cid) /		(sol->dw[offsetof(struct amap_sol_cqe, cid) /
	32] & SOL_CID_MASK));		32] & SOL_CID_MASK));
			}
	break;		break;
	case UNSOL_DATA_DIGEST_ERROR_NOTIFY:		case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
	SE_DEBUG(DBG_LVL_1,		SE_DEBUG(DBG_LVL_1,
	@@ -1493,23 +1547,37 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
	case CXN_KILLED_OVER_RUN_RESIDUAL:		case CXN_KILLED_OVER_RUN_RESIDUAL:
	case CXN_KILLED_UNDER_RUN_RESIDUAL:		case CXN_KILLED_UNDER_RUN_RESIDUAL:
	case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:		case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
			if (ring_mode) {
			SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
			"0x%x...\n",
			sol->dw[offsetof(struct amap_sol_cqe, code) /
			32] & CQE_CODE_MASK, psgl_handle->cid);
			} else {
	SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "		SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
	"0x%x...\n",		"0x%x...\n",
	sol->dw[offsetof(struct amap_sol_cqe, code) /		sol->dw[offsetof(struct amap_sol_cqe, code) /
	32] & CQE_CODE_MASK,		32] & CQE_CODE_MASK,
	sol->dw[offsetof(struct amap_sol_cqe, cid) /		sol->dw[offsetof(struct amap_sol_cqe, cid) /
	32] & CQE_CID_MASK);		32] & CQE_CID_MASK);
			}
	iscsi_conn_failure(beiscsi_conn->conn,		iscsi_conn_failure(beiscsi_conn->conn,
	ISCSI_ERR_CONN_FAILED);		ISCSI_ERR_CONN_FAILED);
	break;		break;
	case CXN_KILLED_RST_SENT:		case CXN_KILLED_RST_SENT:
	case CXN_KILLED_RST_RCVD:		case CXN_KILLED_RST_RCVD:
			if (ring_mode) {
			SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
			"received/sent on CID 0x%x...\n",
			sol->dw[offsetof(struct amap_sol_cqe, code) /
			32] & CQE_CODE_MASK, psgl_handle->cid);
			} else {
	SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"		SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
	"received/sent on CID 0x%x...\n",		"received/sent on CID 0x%x...\n",
	sol->dw[offsetof(struct amap_sol_cqe, code) /		sol->dw[offsetof(struct amap_sol_cqe, code) /
	32] & CQE_CODE_MASK,		32] & CQE_CODE_MASK,
	sol->dw[offsetof(struct amap_sol_cqe, cid) /		sol->dw[offsetof(struct amap_sol_cqe, cid) /
	32] & CQE_CID_MASK);		32] & CQE_CID_MASK);
			}
	iscsi_conn_failure(beiscsi_conn->conn,		iscsi_conn_failure(beiscsi_conn->conn,
	ISCSI_ERR_CONN_FAILED);		ISCSI_ERR_CONN_FAILED);
	break;		break;
	@@ -2674,7 +2742,7 @@ static int be_mcc_queues_create(struct beiscsi_hba *phba,
	goto mcc_cq_destroy;		goto mcc_cq_destroy;

	/* Ask BE to create MCC queue */		/* Ask BE to create MCC queue */
	if (be_cmd_mccq_create(phba, q, cq))		if (beiscsi_cmd_mccq_create(phba, q, cq))
	goto mcc_q_free;		goto mcc_q_free;

	return 0;		return 0;
	@@ -2735,6 +2803,10 @@ static int hwi_init_port(struct beiscsi_hba *phba)
	goto error;		goto error;
	}		}

			if (phba->fw_config.iscsi_features == 0x1)
			ring_mode = 1;
			else
			ring_mode = 0;
	status = mgmt_get_fw_config(ctrl, phba);		status = mgmt_get_fw_config(ctrl, phba);
	if (status != 0) {		if (status != 0) {
	shost_printk(KERN_ERR, phba->shost,		shost_printk(KERN_ERR, phba->shost,
	@@ -2869,6 +2941,17 @@ static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
	phba->io_sgl_hndl_avbl = 0;		phba->io_sgl_hndl_avbl = 0;
	phba->eh_sgl_hndl_avbl = 0;		phba->eh_sgl_hndl_avbl = 0;

			if (ring_mode) {
			phba->sgl_hndl_array = kzalloc(sizeof(struct sgl_handle )
			phba->params.icds_per_ctrl,
			GFP_KERNEL);
			if (!phba->sgl_hndl_array) {
			shost_printk(KERN_ERR, phba->shost,
			"Mem Alloc Failed. Failing to load\n");
			return -ENOMEM;
			}
			}

	mem_descr_sglh = phba->init_mem;		mem_descr_sglh = phba->init_mem;
	mem_descr_sglh += HWI_MEM_SGLH;		mem_descr_sglh += HWI_MEM_SGLH;
	if (1 == mem_descr_sglh->num_elements) {		if (1 == mem_descr_sglh->num_elements) {
	@@ -2876,6 +2959,8 @@ static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
	phba->params.ios_per_ctrl,		phba->params.ios_per_ctrl,
	GFP_KERNEL);		GFP_KERNEL);
	if (!phba->io_sgl_hndl_base) {		if (!phba->io_sgl_hndl_base) {
			if (ring_mode)
			kfree(phba->sgl_hndl_array);
	shost_printk(KERN_ERR, phba->shost,		shost_printk(KERN_ERR, phba->shost,
	"Mem Alloc Failed. Failing to load\n");		"Mem Alloc Failed. Failing to load\n");
	return -ENOMEM;		return -ENOMEM;
	@@ -3060,6 +3145,8 @@ static int beiscsi_init_port(struct beiscsi_hba *phba)
	if (hba_setup_cid_tbls(phba)) {		if (hba_setup_cid_tbls(phba)) {
	shost_printk(KERN_ERR, phba->shost,		shost_printk(KERN_ERR, phba->shost,
	"Failed in hba_setup_cid_tbls\n");		"Failed in hba_setup_cid_tbls\n");
			if (ring_mode)
			kfree(phba->sgl_hndl_array);
	kfree(phba->io_sgl_hndl_base);		kfree(phba->io_sgl_hndl_base);
	kfree(phba->eh_sgl_hndl_base);		kfree(phba->eh_sgl_hndl_base);
	goto do_cleanup_ctrlr;		goto do_cleanup_ctrlr;
	@@ -3110,6 +3197,8 @@ static void beiscsi_clean_port(struct beiscsi_hba *phba)
	"mgmt_epfw_cleanup FAILED \n");		"mgmt_epfw_cleanup FAILED \n");
	hwi_cleanup(phba);		hwi_cleanup(phba);
	hwi_purge_eq(phba);		hwi_purge_eq(phba);
			if (ring_mode)
			kfree(phba->sgl_hndl_array);
	kfree(phba->io_sgl_hndl_base);		kfree(phba->io_sgl_hndl_base);
	kfree(phba->eh_sgl_hndl_base);		kfree(phba->eh_sgl_hndl_base);
	kfree(phba->cid_array);		kfree(phba->cid_array);
	@@ -3194,6 +3283,7 @@ beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
	be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));		be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));

	doorbell \|= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;		doorbell \|= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
			if (!ring_mode)
	doorbell \|= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)		doorbell \|= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
	<< DB_DEF_PDU_WRB_INDEX_SHIFT;		<< DB_DEF_PDU_WRB_INDEX_SHIFT;
	doorbell \|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;		doorbell \|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
	@@ -3280,6 +3370,13 @@ static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
	itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->		itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
	wrb_index << 16) \| (unsigned int)		wrb_index << 16) \| (unsigned int)
	(io_task->psgl_handle->sgl_index));		(io_task->psgl_handle->sgl_index));
			if (ring_mode) {
			phba->sgl_hndl_array[io_task->psgl_handle->sgl_index -
			phba->fw_config.iscsi_cid_start] =
			io_task->psgl_handle;
			io_task->psgl_handle->task = task;
			io_task->psgl_handle->cid = beiscsi_conn->beiscsi_conn_cid;
			} else
	io_task->pwrb_handle->pio_handle = task;		io_task->pwrb_handle->pio_handle = task;

	io_task->cmd_bhs->iscsi_hdr.itt = itt;		io_task->cmd_bhs->iscsi_hdr.itt = itt;
	@@ -3363,10 +3460,16 @@ static int beiscsi_iotask(struct iscsi_task task, struct scatterlist sg,
	ISCSI_OPCODE_SCSI_DATA_OUT);		ISCSI_OPCODE_SCSI_DATA_OUT);
	AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,		AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
	&io_task->cmd_bhs->iscsi_data_pdu, 1);		&io_task->cmd_bhs->iscsi_data_pdu, 1);
			if (ring_mode)
			io_task->psgl_handle->type = INI_WR_CMD;
			else
	AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
	INI_WR_CMD);		INI_WR_CMD);
	AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);		AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
	} else {		} else {
			if (ring_mode)
			io_task->psgl_handle->type = INI_RD_CMD;
			else
	AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
	INI_RD_CMD);		INI_RD_CMD);
	AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);		AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
	@@ -3393,6 +3496,7 @@ static int beiscsi_iotask(struct iscsi_task task, struct scatterlist sg,
	be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));		be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));

	doorbell \|= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;		doorbell \|= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
			if (!ring_mode)
	doorbell \|= (io_task->pwrb_handle->wrb_index &		doorbell \|= (io_task->pwrb_handle->wrb_index &
	DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;		DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
	doorbell \|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;		doorbell \|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
	@@ -3427,6 +3531,9 @@ static int beiscsi_mtask(struct iscsi_task *task)

	switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {		switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
	case ISCSI_OP_LOGIN:		case ISCSI_OP_LOGIN:
			if (ring_mode)
			io_task->psgl_handle->type = TGT_DM_CMD;
			else
	AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
	TGT_DM_CMD);		TGT_DM_CMD);
	AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);		AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
	@@ -3434,11 +3541,17 @@ static int beiscsi_mtask(struct iscsi_task *task)
	hwi_write_buffer(pwrb, task);		hwi_write_buffer(pwrb, task);
	break;		break;
	case ISCSI_OP_NOOP_OUT:		case ISCSI_OP_NOOP_OUT:
			if (ring_mode)
			io_task->psgl_handle->type = INI_RD_CMD;
			else
	AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
	INI_RD_CMD);		INI_RD_CMD);
	hwi_write_buffer(pwrb, task);		hwi_write_buffer(pwrb, task);
	break;		break;
	case ISCSI_OP_TEXT:		case ISCSI_OP_TEXT:
			if (ring_mode)
			io_task->psgl_handle->type = INI_WR_CMD;
			else
	AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
	INI_WR_CMD);		INI_WR_CMD);
	AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);		AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
	@@ -3462,6 +3575,9 @@ static int beiscsi_mtask(struct iscsi_task *task)
	mgmt_invalidate_icds(phba,		mgmt_invalidate_icds(phba,
	aborted_io_task->psgl_handle->sgl_index,		aborted_io_task->psgl_handle->sgl_index,
	cid);		cid);
			if (ring_mode)
			io_task->psgl_handle->type = INI_TMF_CMD;
			else
	AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
	INI_TMF_CMD);		INI_TMF_CMD);
	AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);		AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
	@@ -3469,6 +3585,9 @@ static int beiscsi_mtask(struct iscsi_task *task)
	break;		break;
	case ISCSI_OP_LOGOUT:		case ISCSI_OP_LOGOUT:
	AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);		AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
			if (ring_mode)
			io_task->psgl_handle->type = HWH_TYPE_LOGOUT;
			else
	AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
	HWH_TYPE_LOGOUT);		HWH_TYPE_LOGOUT);
	hwi_write_buffer(pwrb, task);		hwi_write_buffer(pwrb, task);
	@@ -3487,6 +3606,7 @@ static int beiscsi_mtask(struct iscsi_task *task)
	be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));		be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));

	doorbell \|= cid & DB_WRB_POST_CID_MASK;		doorbell \|= cid & DB_WRB_POST_CID_MASK;
			if (!ring_mode)
	doorbell \|= (io_task->pwrb_handle->wrb_index &		doorbell \|= (io_task->pwrb_handle->wrb_index &
	DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;		DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
	doorbell \|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;		doorbell \|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
	@@ -3797,6 +3917,7 @@ static int __init beiscsi_module_init(void)
	"beiscsi pci driver.\n");		"beiscsi pci driver.\n");
	goto unregister_iscsi_transport;		goto unregister_iscsi_transport;
	}		}
			ring_mode = 0;
	return 0;		return 0;

	unregister_iscsi_transport:		unregister_iscsi_transport: