Merge "msm: ipa: added likely/unlikely branch prediction to the lan dp" (18456e81) · Commits · e / devices / android_kernel_oneplus_sm7250

drivers/platform/msm/gsi/gsi.c

+41 −39

Original line number	Diff line number	Diff line
		@@ -486,12 +486,12 @@ static uint16_t gsi_get_complete_num(struct gsi_ring_ctx *ctx, uint64_t addr1,
		GSIDBG_LOW("gsi base addr 0x%llx end addr 0x%llx\n",
		ctx->base, ctx->end);

		if (addr1 < ctx->base \|\| addr1 >= ctx->end) {
		if (unlikely(addr1 < ctx->base \|\| addr1 >= ctx->end)) {
		GSIERR("address = 0x%llx not in range\n", addr1);
		GSI_ASSERT();
		}

		if (addr2 < ctx->base \|\| addr2 >= ctx->end) {
		if (unlikely(addr2 < ctx->base \|\| addr2 >= ctx->end)) {
		GSIERR("address = 0x%llx not in range\n", addr2);
		GSI_ASSERT();
		}
		@@ -512,14 +512,14 @@ static void gsi_process_chan(struct gsi_xfer_compl_evt *evt,
		uint64_t rp;

		ch_id = evt->chid;
		if (WARN_ON(ch_id >= gsi_ctx->max_ch)) {
		if (unlikely(WARN_ON(ch_id >= gsi_ctx->max_ch))) {
		GSIERR("Unexpected ch %d\n", ch_id);
		return;
		}

		ch_ctx = &gsi_ctx->chan[ch_id];
		if (WARN_ON(ch_ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ch_ctx->props.prot != GSI_CHAN_PROT_GCI))
		if (unlikely(WARN_ON(ch_ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ch_ctx->props.prot != GSI_CHAN_PROT_GCI)))
		return;

		if (evt->type != GSI_XFER_COMPL_TYPE_GCI) {
		@@ -555,7 +555,7 @@ static void gsi_process_chan(struct gsi_xfer_compl_evt *evt,
		notify->bytes_xfered = evt->len;

		if (callback) {
		if (atomic_read(&ch_ctx->poll_mode)) {
		if (unlikely(atomic_read(&ch_ctx->poll_mode))) {
		GSIERR("Calling client callback in polling mode\n");
		WARN_ON(1);
		}
		@@ -633,7 +633,8 @@ static void gsi_handle_ieob(int ee)

		for (i = 0; i < GSI_STTS_REG_BITS; i++) {
		if ((1 << i) & ch & msk) {
		if (i >= gsi_ctx->max_ev \|\| i >= GSI_EVT_RING_MAX) {
		if (unlikely(i >= gsi_ctx->max_ev
		\|\| i >= GSI_EVT_RING_MAX)) {
		GSIERR("invalid event %d\n", i);
		break;
		}
		@@ -646,7 +647,8 @@ static void gsi_handle_ieob(int ee)
		if (ctx->props.intr == GSI_INTR_MSI)
		continue;

		if (ctx->props.intf != GSI_EVT_CHTYPE_GPI_EV) {
		if (unlikely(ctx->props.intf !=
		GSI_EVT_CHTYPE_GPI_EV)) {
		GSIERR("Unexpected irq intf %d\n",
		ctx->props.intf);
		GSI_ASSERT();
		@@ -780,7 +782,7 @@ static void gsi_handle_irq(void)
		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_GENERAL_BMSK)
		gsi_handle_general(ee);

		if (++cnt > GSI_ISR_MAX_ITER) {
		if (unlikely(++cnt > GSI_ISR_MAX_ITER)) {
		/*
		* Max number of spurious interrupts from hardware.
		* Unexpected hardware state.
		@@ -942,17 +944,17 @@ int gsi_complete_clk_grant(unsigned long dev_hdl)
		{
		unsigned long flags;

		if (!gsi_ctx) {
		if (unlikely(!gsi_ctx)) {
		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
		return -GSI_STATUS_NODEV;
		}

		if (!gsi_ctx->per_registered) {
		if (unlikely(!gsi_ctx->per_registered)) {
		GSIERR("no client registered\n");
		return -GSI_STATUS_INVALID_PARAMS;
		}

		if (dev_hdl != (uintptr_t)gsi_ctx) {
		if (unlikely(dev_hdl != (uintptr_t)gsi_ctx)) {
		GSIERR("bad params dev_hdl=0x%lx gsi_ctx=0x%pK\n", dev_hdl,
		gsi_ctx);
		return -GSI_STATUS_INVALID_PARAMS;
		@@ -1871,19 +1873,19 @@ int gsi_ring_evt_ring_db(unsigned long evt_ring_hdl, uint64_t value)
		{
		struct gsi_evt_ctx *ctx;

		if (!gsi_ctx) {
		if (unlikely(!gsi_ctx)) {
		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
		return -GSI_STATUS_NODEV;
		}

		if (evt_ring_hdl >= gsi_ctx->max_ev) {
		if (unlikely(evt_ring_hdl >= gsi_ctx->max_ev)) {
		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
		return -GSI_STATUS_INVALID_PARAMS;
		}

		ctx = &gsi_ctx->evtr[evt_ring_hdl];

		if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
		if (unlikely(ctx->state != GSI_EVT_RING_STATE_ALLOCATED)) {
		GSIERR("bad state %d\n",
		gsi_ctx->evtr[evt_ring_hdl].state);
		return -GSI_STATUS_UNSUPPORTED_OP;
		@@ -1901,19 +1903,19 @@ int gsi_ring_ch_ring_db(unsigned long chan_hdl, uint64_t value)
		struct gsi_chan_ctx *ctx;
		uint32_t val;

		if (!gsi_ctx) {
		if (unlikely(!gsi_ctx)) {
		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
		return -GSI_STATUS_NODEV;
		}

		if (chan_hdl >= gsi_ctx->max_ch) {
		if (unlikely(chan_hdl >= gsi_ctx->max_ch)) {
		GSIERR("bad chan_hdl=%lu\n", chan_hdl);
		return -GSI_STATUS_INVALID_PARAMS;
		}

		ctx = &gsi_ctx->chan[chan_hdl];

		if (ctx->state != GSI_CHAN_STATE_STARTED) {
		if (unlikely(ctx->state != GSI_CHAN_STATE_STARTED)) {
		GSIERR("bad state %d\n", ctx->state);
		return -GSI_STATUS_UNSUPPORTED_OP;
		}
		@@ -3397,13 +3399,13 @@ int __gsi_populate_gci_tre(struct gsi_chan_ctx *ctx,
		uint16_t idx;

		memset(&gci_tre, 0, sizeof(gci_tre));
		if (xfer->addr & 0xFFFFFF0000000000) {
		if (unlikely(xfer->addr & 0xFFFFFF0000000000)) {
		GSIERR("chan_hdl=%u add too large=%llx\n",
		ctx->props.ch_id, xfer->addr);
		return -EINVAL;
		}

		if (xfer->type != GSI_XFER_ELEM_DATA) {
		if (unlikely(xfer->type != GSI_XFER_ELEM_DATA)) {
		GSIERR("chan_hdl=%u bad RE type=%u\n", ctx->props.ch_id,
		xfer->type);
		return -EINVAL;
		@@ -3417,7 +3419,7 @@ int __gsi_populate_gci_tre(struct gsi_chan_ctx *ctx,
		gci_tre.buf_len = xfer->len;
		gci_tre.re_type = GSI_RE_COAL;
		gci_tre.cookie = __gsi_get_gci_cookie(ctx, idx);
		if (gci_tre.cookie > (ctx->ring.max_num_elem + GSI_VEID_MAX))
		if (unlikely(gci_tre.cookie > (ctx->ring.max_num_elem + GSI_VEID_MAX)))
		return -EPERM;

		/* write the TRE to ring */
		@@ -3476,12 +3478,12 @@ int gsi_queue_xfer(unsigned long chan_hdl, uint16_t num_xfers,
		spinlock_t *slock;
		unsigned long flags;

		if (!gsi_ctx) {
		if (unlikely(!gsi_ctx)) {
		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
		return -GSI_STATUS_NODEV;
		}

		if (chan_hdl >= gsi_ctx->max_ch \|\| (num_xfers && !xfer)) {
		if (unlikely(chan_hdl >= gsi_ctx->max_ch \|\| (num_xfers && !xfer))) {
		GSIERR("bad params chan_hdl=%lu num_xfers=%u xfer=%pK\n",
		chan_hdl, num_xfers, xfer);
		return -GSI_STATUS_INVALID_PARAMS;
		@@ -3489,8 +3491,8 @@ int gsi_queue_xfer(unsigned long chan_hdl, uint16_t num_xfers,

		ctx = &gsi_ctx->chan[chan_hdl];

		if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ctx->props.prot != GSI_CHAN_PROT_GCI) {
		if (unlikely(ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ctx->props.prot != GSI_CHAN_PROT_GCI)) {
		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
		return -GSI_STATUS_UNSUPPORTED_OP;
		}
		@@ -3512,7 +3514,7 @@ int gsi_queue_xfer(unsigned long chan_hdl, uint16_t num_xfers,
		*/
		if (ctx->props.prot != GSI_CHAN_PROT_GCI) {
		__gsi_query_channel_free_re(ctx, &free);
		if (num_xfers > free) {
		if (unlikely(num_xfers > free)) {
		GSIERR("chan_hdl=%lu num_xfers=%u free=%u\n",
		chan_hdl, num_xfers, free);
		spin_unlock_irqrestore(slock, flags);
		@@ -3532,7 +3534,7 @@ int gsi_queue_xfer(unsigned long chan_hdl, uint16_t num_xfers,
		gsi_incr_ring_wp(&ctx->ring);
		}

		if (i != num_xfers) {
		if (unlikely(i != num_xfers)) {
		/* reject all the xfers */
		ctx->ring.wp_local = wp_rollback;
		spin_unlock_irqrestore(slock, flags);
		@@ -3609,13 +3611,13 @@ int gsi_poll_n_channel(unsigned long chan_hdl,
		int i;
		unsigned long flags;

		if (!gsi_ctx) {
		if (unlikely(!gsi_ctx)) {
		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
		return -GSI_STATUS_NODEV;
		}

		if (chan_hdl >= gsi_ctx->max_ch \|\| !notify \|\|
		!actual_num \|\| expected_num <= 0) {
		if (unlikely(chan_hdl >= gsi_ctx->max_ch \|\| !notify \|\|
		!actual_num \|\| expected_num <= 0)) {
		GSIERR("bad params chan_hdl=%lu notify=%pK\n",
		chan_hdl, notify);
		GSIERR("actual_num=%pK expected_num=%d\n",
		@@ -3626,13 +3628,13 @@ int gsi_poll_n_channel(unsigned long chan_hdl,
		ctx = &gsi_ctx->chan[chan_hdl];
		ee = gsi_ctx->per.ee;

		if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ctx->props.prot != GSI_CHAN_PROT_GCI) {
		if (unlikely(ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ctx->props.prot != GSI_CHAN_PROT_GCI)) {
		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
		return -GSI_STATUS_UNSUPPORTED_OP;
		}

		if (!ctx->evtr) {
		if (unlikely(!ctx->evtr)) {
		GSIERR("no event ring associated chan_hdl=%lu\n", chan_hdl);
		return -GSI_STATUS_UNSUPPORTED_OP;
		}
		@@ -3690,25 +3692,25 @@ int gsi_config_channel_mode(unsigned long chan_hdl, enum gsi_chan_mode mode)
		unsigned long flags;
		enum gsi_chan_mode chan_mode;

		if (!gsi_ctx) {
		if (unlikely(!gsi_ctx)) {
		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
		return -GSI_STATUS_NODEV;
		}

		if (chan_hdl >= gsi_ctx->max_ch) {
		if (unlikely(chan_hdl >= gsi_ctx->max_ch)) {
		GSIERR("bad params chan_hdl=%lu mode=%u\n", chan_hdl, mode);
		return -GSI_STATUS_INVALID_PARAMS;
		}

		ctx = &gsi_ctx->chan[chan_hdl];

		if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ctx->props.prot != GSI_CHAN_PROT_GCI) {
		if (unlikely(ctx->props.prot != GSI_CHAN_PROT_GPI &&
		ctx->props.prot != GSI_CHAN_PROT_GCI)) {
		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
		return -GSI_STATUS_UNSUPPORTED_OP;
		}

		if (!ctx->evtr \|\| !ctx->evtr->props.exclusive) {
		if (unlikely(!ctx->evtr \|\| !ctx->evtr->props.exclusive)) {
		GSIERR("cannot configure mode on chan_hdl=%lu\n",
		chan_hdl);
		return -GSI_STATUS_UNSUPPORTED_OP;
		@@ -3719,7 +3721,7 @@ int gsi_config_channel_mode(unsigned long chan_hdl, enum gsi_chan_mode mode)
		else
		curr = GSI_CHAN_MODE_CALLBACK;

		if (mode == curr) {
		if (unlikely(mode == curr)) {
		GSIERR("already in requested mode %u chan_hdl=%lu\n",
		curr, chan_hdl);
		return -GSI_STATUS_UNSUPPORTED_OP;

drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c

+8 −8

Original line number	Diff line number	Diff line
		@@ -584,7 +584,7 @@ static netdev_tx_t ecm_ipa_start_xmit
		}

		ret = ipa_pm_activate(ecm_ipa_ctx->pm_hdl);
		if (ret) {
		if (unlikely(ret)) {
		ECM_IPA_DEBUG("Failed to activate PM client\n");
		netif_stop_queue(net);
		goto fail_pm_activate;
		@@ -607,7 +607,7 @@ static netdev_tx_t ecm_ipa_start_xmit
		, skb->protocol);

		ret = ipa_tx_dp(ecm_ipa_ctx->ipa_to_usb_client, skb, NULL);
		if (ret) {
		if (unlikely(ret)) {
		ECM_IPA_ERROR("ipa transmit failed (%d)\n", ret);
		goto fail_tx_packet;
		}
		@@ -642,7 +642,7 @@ static void ecm_ipa_packet_receive_notify
		int result;
		unsigned int packet_len;

		if (!skb) {
		if (unlikely(!skb)) {
		ECM_IPA_ERROR("Bad SKB received from IPA driver\n");
		return;
		}
		@@ -655,7 +655,7 @@ static void ecm_ipa_packet_receive_notify
		return;
		}

		if (evt != IPA_RECEIVE) {
		if (unlikely(evt != IPA_RECEIVE)) {
		ECM_IPA_ERROR("A none IPA_RECEIVE event in ecm_ipa_receive\n");
		return;
		}
		@@ -664,7 +664,7 @@ static void ecm_ipa_packet_receive_notify
		skb->protocol = eth_type_trans(skb, ecm_ipa_ctx->net);

		result = netif_rx(skb);
		if (result)
		if (unlikely(result))
		ECM_IPA_ERROR("fail on netif_rx\n");
		ecm_ipa_ctx->net->stats.rx_packets++;
		ecm_ipa_ctx->net->stats.rx_bytes += packet_len;
		@@ -1129,12 +1129,12 @@ static void ecm_ipa_tx_complete_notify
		struct sk_buff skb = (struct sk_buff )data;
		struct ecm_ipa_dev *ecm_ipa_ctx = priv;

		if (!skb) {
		if (unlikely(!skb)) {
		ECM_IPA_ERROR("Bad SKB received from IPA driver\n");
		return;
		}

		if (!ecm_ipa_ctx) {
		if (unlikely(!ecm_ipa_ctx)) {
		ECM_IPA_ERROR("ecm_ipa_ctx is NULL pointer\n");
		return;
		}
		@@ -1144,7 +1144,7 @@ static void ecm_ipa_tx_complete_notify
		skb->len, skb->protocol,
		atomic_read(&ecm_ipa_ctx->outstanding_pkts));

		if (evt != IPA_WRITE_DONE) {
		if (unlikely(evt != IPA_WRITE_DONE)) {
		ECM_IPA_ERROR("unsupported event on Tx callback\n");
		return;
		}

drivers/platform/msm/ipa/ipa_clients/odu_bridge.c

+5 −5

Original line number	Diff line number	Diff line
		@@ -800,7 +800,7 @@ int odu_bridge_tx_dp(struct sk_buff skb, struct ipa_tx_meta metadata)
		case ODU_BRIDGE_MODE_ROUTER:
		/* Router mode - pass skb to IPA */
		res = ipa_tx_dp(IPA_CLIENT_ODU_PROD, skb, metadata);
		if (res) {
		if (unlikely(res)) {
		ODU_BRIDGE_DBG("tx dp failed %d\n", res);
		goto out;
		}
		@@ -813,7 +813,7 @@ int odu_bridge_tx_dp(struct sk_buff skb, struct ipa_tx_meta metadata)
		ODU_BRIDGE_IS_QMI_ADDR(ipv6hdr->daddr)) {
		ODU_BRIDGE_DBG_LOW("QMI packet\n");
		skb_copied = skb_clone(skb, GFP_KERNEL);
		if (!skb_copied) {
		if (unlikely(!skb_copied)) {
		ODU_BRIDGE_ERR("No memory\n");
		return -ENOMEM;
		}
		@@ -834,13 +834,13 @@ int odu_bridge_tx_dp(struct sk_buff skb, struct ipa_tx_meta metadata)
		ODU_BRIDGE_DBG_LOW(
		"Multicast pkt, send to APPS and IPA\n");
		skb_copied = skb_clone(skb, GFP_KERNEL);
		if (!skb_copied) {
		if (unlikely(!skb_copied)) {
		ODU_BRIDGE_ERR("No memory\n");
		return -ENOMEM;
		}

		res = ipa_tx_dp(IPA_CLIENT_ODU_PROD, skb, metadata);
		if (res) {
		if (unlikely(res)) {
		ODU_BRIDGE_DBG("tx dp failed %d\n", res);
		dev_kfree_skb(skb_copied);
		goto out;
		@@ -855,7 +855,7 @@ int odu_bridge_tx_dp(struct sk_buff skb, struct ipa_tx_meta metadata)
		}

		res = ipa_tx_dp(IPA_CLIENT_ODU_PROD, skb, metadata);
		if (res) {
		if (unlikely(res)) {
		ODU_BRIDGE_DBG("tx dp failed %d\n", res);
		goto out;
		}

drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c

+6 −6

Original line number	Diff line number	Diff line
		@@ -940,7 +940,7 @@ static netdev_tx_t rndis_ipa_start_xmit(struct sk_buff *skb,
		}

		ret = ipa_pm_activate(rndis_ipa_ctx->pm_hdl);
		if (ret) {
		if (unlikely(ret)) {
		RNDIS_IPA_DEBUG("Failed activate PM client\n");
		netif_stop_queue(net);
		goto fail_pm_activate;
		@@ -959,7 +959,7 @@ static netdev_tx_t rndis_ipa_start_xmit(struct sk_buff *skb,
		skb = rndis_encapsulate_skb(skb, rndis_ipa_ctx);
		trace_rndis_tx_dp(skb->protocol);
		ret = ipa_tx_dp(IPA_TO_USB_CLIENT, skb, NULL);
		if (ret) {
		if (unlikely(ret)) {
		RNDIS_IPA_ERROR("ipa transmit failed (%d)\n", ret);
		goto fail_tx_packet;
		}
		@@ -1006,7 +1006,7 @@ static void rndis_ipa_tx_complete_notify(

		ret = 0;
		NULL_CHECK_RETVAL(private);
		if (ret)
		if (unlikely(ret))
		return;

		trace_rndis_status_rcvd(skb->protocol);
		@@ -1120,7 +1120,7 @@ static void rndis_ipa_packet_receive_notify(
		return;
		}

		if (evt != IPA_RECEIVE) {
		if (unlikely(evt != IPA_RECEIVE)) {
		RNDIS_IPA_ERROR("a none IPA_RECEIVE event in driver RX\n");
		return;
		}
		@@ -1140,7 +1140,7 @@ static void rndis_ipa_packet_receive_notify(

		trace_rndis_netif_ni(skb->protocol);
		result = netif_rx_ni(skb);
		if (result)
		if (unlikely(result))
		RNDIS_IPA_ERROR("fail on netif_rx_ni\n");
		rndis_ipa_ctx->net->stats.rx_packets++;
		rndis_ipa_ctx->net->stats.rx_bytes += packet_len;
		@@ -1817,7 +1817,7 @@ static struct sk_buff rndis_encapsulate_skb(struct sk_buff skb,
		if (unlikely(skb_headroom(skb) < sizeof(rndis_template_hdr))) {
		struct sk_buff *new_skb = skb_copy_expand(skb,
		sizeof(rndis_template_hdr), 0, GFP_ATOMIC);
		if (!new_skb) {
		if (unlikely(!new_skb)) {
		RNDIS_IPA_ERROR("no memory for skb expand\n");
		return skb;
		}

drivers/platform/msm/ipa/ipa_v3/ipa_dp.c

+42 −38

Original line number	Diff line number	Diff line
		@@ -344,7 +344,7 @@ int ipa3_send(struct ipa3_sys_context *sys,
		for (i = 0; i < num_desc; i++) {
		tx_pkt = kmem_cache_zalloc(ipa3_ctx->tx_pkt_wrapper_cache,
		GFP_ATOMIC);
		if (!tx_pkt) {
		if (unlikely(!tx_pkt)) {
		IPAERR("failed to alloc tx wrapper\n");
		result = -ENOMEM;
		goto failure;
		@@ -358,8 +358,8 @@ int ipa3_send(struct ipa3_sys_context *sys,

		/* populate tag field */
		if (desc[i].is_tag_status) {
		if (ipa_populate_tag_field(&desc[i], tx_pkt,
		&tag_pyld_ret)) {
		if (unlikely(ipa_populate_tag_field(&desc[i], tx_pkt,
		&tag_pyld_ret))) {
		IPAERR("Failed to populate tag field\n");
		result = -EFAULT;
		goto failure_dma_map;
		@@ -399,7 +399,8 @@ int ipa3_send(struct ipa3_sys_context *sys,
		tx_pkt->no_unmap_dma = true;
		}
		}
		if (dma_mapping_error(ipa3_ctx->pdev, tx_pkt->mem.phys_base)) {
		if (unlikely(dma_mapping_error(ipa3_ctx->pdev,
		tx_pkt->mem.phys_base))) {
		IPAERR("failed to do dma map.\n");
		result = -EFAULT;
		goto failure_dma_map;
		@@ -450,7 +451,7 @@ int ipa3_send(struct ipa3_sys_context *sys,
		IPADBG_LOW("ch:%lu queue xfer\n", sys->ep->gsi_chan_hdl);
		result = gsi_queue_xfer(sys->ep->gsi_chan_hdl, num_desc,
		gsi_xfer, true);
		if (result != GSI_STATUS_SUCCESS) {
		if (unlikely(result != GSI_STATUS_SUCCESS)) {
		IPAERR_RL("GSI xfer failed.\n");
		result = -EFAULT;
		goto failure;
		@@ -1490,7 +1491,7 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		return -EINVAL;
		}

		if (skb->len == 0) {
		if (unlikely(skb->len == 0)) {
		IPAERR("packet size is 0\n");
		return -EINVAL;
		}
		@@ -1506,7 +1507,7 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		*/
		if (IPA_CLIENT_IS_CONS(dst)) {
		src_ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_LAN_PROD);
		if (-1 == src_ep_idx) {
		if (unlikely(-1 == src_ep_idx)) {
		IPAERR("Client %u is not mapped\n",
		IPA_CLIENT_APPS_LAN_PROD);
		goto fail_gen;
		@@ -1514,7 +1515,7 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		dst_ep_idx = ipa3_get_ep_mapping(dst);
		} else {
		src_ep_idx = ipa3_get_ep_mapping(dst);
		if (-1 == src_ep_idx) {
		if (unlikely(-1 == src_ep_idx)) {
		IPAERR("Client %u is not mapped\n", dst);
		goto fail_gen;
		}
		@@ -1526,7 +1527,7 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,

		sys = ipa3_ctx->ep[src_ep_idx].sys;

		if (!sys \|\| !sys->ep->valid) {
		if (unlikely(!sys \|\| !sys->ep->valid)) {
		IPAERR_RL("pipe %d not valid\n", src_ep_idx);
		goto fail_pipe_not_valid;
		}
		@@ -1547,7 +1548,7 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		gsi_ep->prefetch_mode == GSI_FREE_PRE_FETCH)
		max_desc -= gsi_ep->prefetch_threshold;
		if (num_frags + 3 > max_desc) {
		if (skb_linearize(skb)) {
		if (unlikely(skb_linearize(skb))) {
		IPAERR("Failed to linear skb with %d frags\n",
		num_frags);
		goto fail_gen;
		@@ -1561,7 +1562,7 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		* 1 desc for each frag
		*/
		desc = kzalloc(sizeof(desc) (num_frags + 3), GFP_ATOMIC);
		if (!desc) {
		if (unlikely(!desc)) {
		IPAERR("failed to alloc desc array\n");
		goto fail_gen;
		}
		@@ -1623,7 +1624,8 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		desc[skb_idx].callback = NULL;
		}

		if (ipa3_send(sys, num_frags + data_idx, desc, true)) {
		if (unlikely(ipa3_send(sys, num_frags + data_idx,
		desc, true))) {
		IPAERR_RL("fail to send skb %pK num_frags %u SWP\n",
		skb, num_frags);
		goto fail_send;
		@@ -1654,7 +1656,8 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		desc[data_idx].dma_address = meta->dma_address;
		}
		if (num_frags == 0) {
		if (ipa3_send(sys, data_idx + 1, desc, true)) {
		if (unlikely(ipa3_send(sys, data_idx + 1,
		desc, true))) {
		IPAERR("fail to send skb %pK HWP\n", skb);
		goto fail_mem;
		}
		@@ -1673,8 +1676,8 @@ int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
		desc[data_idx+f].user2 = desc[data_idx].user2;
		desc[data_idx].callback = NULL;

		if (ipa3_send(sys, num_frags + data_idx + 1,
		desc, true)) {
		if (unlikely(ipa3_send(sys, num_frags + data_idx + 1,
		desc, true))) {
		IPAERR("fail to send skb %pK num_frags %u\n",
		skb, num_frags);
		goto fail_mem;
		@@ -1729,26 +1732,27 @@ static void ipa3_wq_repl_rx(struct work_struct *work)
		begin:
		while (1) {
		next = (curr + 1) % sys->repl->capacity;
		if (next == atomic_read(&sys->repl->head_idx))
		if (unlikely(next == atomic_read(&sys->repl->head_idx)))
		goto fail_kmem_cache_alloc;

		rx_pkt = kmem_cache_zalloc(ipa3_ctx->rx_pkt_wrapper_cache,
		flag);
		if (!rx_pkt)
		if (unlikely(!rx_pkt))
		goto fail_kmem_cache_alloc;

		INIT_WORK(&rx_pkt->work, ipa3_wq_rx_avail);
		rx_pkt->sys = sys;

		rx_pkt->data.skb = sys->get_skb(sys->rx_buff_sz, flag);
		if (rx_pkt->data.skb == NULL)
		if (unlikely(rx_pkt->data.skb == NULL))
		goto fail_skb_alloc;

		ptr = skb_put(rx_pkt->data.skb, sys->rx_buff_sz);
		rx_pkt->data.dma_addr = dma_map_single(ipa3_ctx->pdev, ptr,
		sys->rx_buff_sz,
		DMA_FROM_DEVICE);
		if (dma_mapping_error(ipa3_ctx->pdev, rx_pkt->data.dma_addr)) {
		if (unlikely(dma_mapping_error(ipa3_ctx->pdev,
		rx_pkt->data.dma_addr))) {
		pr_err_ratelimited("%s dma map fail %pK for %pK sys=%pK\n",
		__func__, (void *)rx_pkt->data.dma_addr,
		ptr, sys);
		@@ -1802,8 +1806,8 @@ static struct ipa3_rx_pkt_wrapper *ipa3_alloc_rx_pkt_page(
		rx_pkt->page_data.dma_addr = dma_map_page(ipa3_ctx->pdev,
		rx_pkt->page_data.page, 0,
		rx_pkt->len, DMA_FROM_DEVICE);
		if (dma_mapping_error(ipa3_ctx->pdev,
		rx_pkt->page_data.dma_addr)) {
		if (unlikely(dma_mapping_error(ipa3_ctx->pdev,
		rx_pkt->page_data.dma_addr))) {
		pr_err_ratelimited("%s dma map fail %pK for %pK\n",
		__func__, (void *)rx_pkt->page_data.dma_addr,
		rx_pkt->page_data.page);
		@@ -1829,7 +1833,7 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys)

		for (curr = 0; curr < sys->repl->capacity; curr++) {
		rx_pkt = ipa3_alloc_rx_pkt_page(GFP_KERNEL, false);
		if (!rx_pkt) {
		if (unlikely(!rx_pkt)) {
		IPAERR("ipa3_alloc_rx_pkt_page fails\n");
		ipa_assert();
		break;
		@@ -1877,7 +1881,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
		rx_pkt = ipa3_alloc_rx_pkt_page(flag, true);
		if (!rx_pkt && flag == GFP_ATOMIC)
		break;
		else if (!rx_pkt)
		else if (unlikely(!rx_pkt))
		goto fail_kmem_cache_alloc;
		rx_pkt->sys = sys;
		}
		@@ -1901,7 +1905,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
		if (idx == IPA_REPL_XFER_MAX) {
		ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
		gsi_xfer_elem_array, false);
		if (ret != GSI_STATUS_SUCCESS) {
		if (unlikely(ret != GSI_STATUS_SUCCESS)) {
		/* we don't expect this will happen */
		IPAERR("failed to provide buffer: %d\n", ret);
		ipa_assert();
		@@ -1913,7 +1917,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
		/* only ring doorbell once here */
		ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
		gsi_xfer_elem_array, true);
		if (ret == GSI_STATUS_SUCCESS) {
		if (likely(ret == GSI_STATUS_SUCCESS)) {
		/* ensure write is done before setting head index */
		mb();
		atomic_set(&sys->repl->head_idx, curr);
		@@ -1971,7 +1975,7 @@ static void ipa3_replenish_wlan_rx_cache(struct ipa3_sys_context *sys)
		ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, 1,
		&gsi_xfer_elem_one, true);

		if (ret) {
		if (unlikely(ret)) {
		IPAERR("failed to provide buffer: %d\n", ret);
		goto fail_provide_rx_buffer;
		}
		@@ -3134,14 +3138,14 @@ static struct sk_buff *handle_skb_completion(struct gsi_chan_xfer_notify
		rx_skb->truesize = rx_pkt->len + sizeof(struct sk_buff);
		}

		if (notify->veid >= GSI_VEID_MAX) {
		if (unlikely(notify->veid >= GSI_VEID_MAX)) {
		WARN_ON(1);
		return NULL;
		}

		/Assesrt when WAN consumer channel receive EOB event/
		if (notify->evt_id == GSI_CHAN_EVT_EOB &&
		sys->ep->client == IPA_CLIENT_APPS_WAN_CONS) {
		if (unlikely(notify->evt_id == GSI_CHAN_EVT_EOB &&
		sys->ep->client == IPA_CLIENT_APPS_WAN_CONS)) {
		IPAERR("EOB event received on WAN consumer channel\n");
		ipa_assert();
		}
		@@ -3265,13 +3269,13 @@ static void ipa3_wq_rx_common(struct ipa3_sys_context *sys,
		struct ipa3_sys_context *coal_sys;
		int ipa_ep_idx;

		if (!notify) {
		if (unlikely(!notify)) {
		IPAERR_RL("gsi_chan_xfer_notify is null\n");
		return;
		}
		rx_skb = handle_skb_completion(notify, true);

		if (rx_skb) {
		if (likely(rx_skb)) {
		sys->pyld_hdlr(rx_skb, sys);

		/* For coalescing, we have 2 transfer rings to replenish */
		@@ -3279,7 +3283,7 @@ static void ipa3_wq_rx_common(struct ipa3_sys_context *sys,
		ipa_ep_idx = ipa3_get_ep_mapping(
		IPA_CLIENT_APPS_WAN_CONS);

		if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
		if (unlikely(ipa_ep_idx == IPA_EP_NOT_ALLOCATED)) {
		IPAERR("Invalid client.\n");
		return;
		}
		@@ -3339,8 +3343,8 @@ static void ipa3_rx_napi_chain(struct ipa3_sys_context *sys,
		*/
		ipa_ep_idx = ipa3_get_ep_mapping(
		IPA_CLIENT_APPS_WAN_CONS);
		if (ipa_ep_idx ==
		IPA_EP_NOT_ALLOCATED) {
		if (unlikely(ipa_ep_idx ==
		IPA_EP_NOT_ALLOCATED)) {
		IPAERR("Invalid client.\n");
		return;
		}
		@@ -3376,8 +3380,8 @@ static void ipa3_rx_napi_chain(struct ipa3_sys_context *sys,
		*/
		ipa_ep_idx = ipa3_get_ep_mapping(
		IPA_CLIENT_APPS_WAN_CONS);
		if (ipa_ep_idx ==
		IPA_EP_NOT_ALLOCATED) {
		if (unlikely(ipa_ep_idx ==
		IPA_EP_NOT_ALLOCATED)) {
		IPAERR("Invalid client.\n");
		return;
		}
		@@ -4165,7 +4169,7 @@ static void ipa_gsi_irq_rx_notify_cb(struct gsi_chan_xfer_notify *notify)
		{
		struct ipa3_sys_context *sys;

		if (!notify) {
		if (unlikely(!notify)) {
		IPAERR("gsi notify is NULL.\n");
		return;
		}
		@@ -4197,7 +4201,7 @@ static void ipa_dma_gsi_irq_rx_notify_cb(struct gsi_chan_xfer_notify *notify)
		{
		struct ipa3_sys_context *sys;

		if (!notify) {
		if (unlikely(!notify)) {
		IPAERR("gsi notify is NULL.\n");
		return;
		}