Merge branch '40GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/jkirsher/next-queue (397df709) · Commits · e / devices / android_kernel_fairphone_FP5

drivers/net/ethernet/intel/i40e/i40e.h

+1 −9

Original line number	Original line	Diff line number	Diff line
	@@ -91,14 +91,6 @@
	#define I40E_QUEUE_WAIT_RETRY_LIMIT 10		#define I40E_QUEUE_WAIT_RETRY_LIMIT 10
	#define I40E_INT_NAME_STR_LEN (IFNAMSIZ + 16)		#define I40E_INT_NAME_STR_LEN (IFNAMSIZ + 16)

	/* Ethtool Private Flags */
	#define I40E_PRIV_FLAGS_MFP_FLAG BIT(0)
	#define I40E_PRIV_FLAGS_LINKPOLL_FLAG BIT(1)
	#define I40E_PRIV_FLAGS_FD_ATR BIT(2)
	#define I40E_PRIV_FLAGS_VEB_STATS BIT(3)
	#define I40E_PRIV_FLAGS_HW_ATR_EVICT BIT(4)
	#define I40E_PRIV_FLAGS_TRUE_PROMISC_SUPPORT BIT(5)

	#define I40E_NVM_VERSION_LO_SHIFT 0		#define I40E_NVM_VERSION_LO_SHIFT 0
	#define I40E_NVM_VERSION_LO_MASK (0xff << I40E_NVM_VERSION_LO_SHIFT)		#define I40E_NVM_VERSION_LO_MASK (0xff << I40E_NVM_VERSION_LO_SHIFT)
	#define I40E_NVM_VERSION_HI_SHIFT 12		#define I40E_NVM_VERSION_HI_SHIFT 12
	@@ -397,7 +389,6 @@ struct i40e_pf {
	#define I40E_FLAG_MSIX_ENABLED BIT_ULL(3)		#define I40E_FLAG_MSIX_ENABLED BIT_ULL(3)
	#define I40E_FLAG_RSS_ENABLED BIT_ULL(6)		#define I40E_FLAG_RSS_ENABLED BIT_ULL(6)
	#define I40E_FLAG_VMDQ_ENABLED BIT_ULL(7)		#define I40E_FLAG_VMDQ_ENABLED BIT_ULL(7)
	#define I40E_FLAG_FDIR_REQUIRES_REINIT BIT_ULL(8)
	#define I40E_FLAG_NEED_LINK_UPDATE BIT_ULL(9)		#define I40E_FLAG_NEED_LINK_UPDATE BIT_ULL(9)
	#define I40E_FLAG_IWARP_ENABLED BIT_ULL(10)		#define I40E_FLAG_IWARP_ENABLED BIT_ULL(10)
	#define I40E_FLAG_CLEAN_ADMINQ BIT_ULL(14)		#define I40E_FLAG_CLEAN_ADMINQ BIT_ULL(14)
	@@ -439,6 +430,7 @@ struct i40e_pf {
	#define I40E_FLAG_TEMP_LINK_POLLING BIT_ULL(55)		#define I40E_FLAG_TEMP_LINK_POLLING BIT_ULL(55)
	#define I40E_FLAG_CLIENT_L2_CHANGE BIT_ULL(56)		#define I40E_FLAG_CLIENT_L2_CHANGE BIT_ULL(56)
	#define I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE BIT_ULL(57)		#define I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE BIT_ULL(57)
			#define I40E_FLAG_LEGACY_RX BIT_ULL(58)

	/* Tracks features that are disabled due to hw limitations.		/* Tracks features that are disabled due to hw limitations.
	* If a bit is set here, it means that the corresponding		* If a bit is set here, it means that the corresponding

drivers/net/ethernet/intel/i40e/i40e_common.c

+6 −2

Original line number	Original line	Diff line number	Diff line
	@@ -4963,7 +4963,9 @@ u32 i40e_read_rx_ctl(struct i40e_hw *hw, u32 reg_addr)
	int retry = 5;		int retry = 5;
	u32 val = 0;		u32 val = 0;

	use_register = (hw->aq.api_maj_ver == 1) && (hw->aq.api_min_ver < 5);		use_register = (((hw->aq.api_maj_ver == 1) &&
			(hw->aq.api_min_ver < 5)) \|\|
			(hw->mac.type == I40E_MAC_X722));
	if (!use_register) {		if (!use_register) {
	do_retry:		do_retry:
	status = i40e_aq_rx_ctl_read_register(hw, reg_addr, &val, NULL);		status = i40e_aq_rx_ctl_read_register(hw, reg_addr, &val, NULL);
	@@ -5022,7 +5024,9 @@ void i40e_write_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val)
	bool use_register;		bool use_register;
	int retry = 5;		int retry = 5;

	use_register = (hw->aq.api_maj_ver == 1) && (hw->aq.api_min_ver < 5);		use_register = (((hw->aq.api_maj_ver == 1) &&
			(hw->aq.api_min_ver < 5)) \|\|
			(hw->mac.type == I40E_MAC_X722));
	if (!use_register) {		if (!use_register) {
	do_retry:		do_retry:
	status = i40e_aq_rx_ctl_write_register(hw, reg_addr,		status = i40e_aq_rx_ctl_write_register(hw, reg_addr,

drivers/net/ethernet/intel/i40e/i40e_ethtool.c

+115 −77

Original line number	Original line	Diff line number	Diff line
	@@ -207,22 +207,37 @@ static const char i40e_gstrings_test[][ETH_GSTRING_LEN] = {

	#define I40E_TEST_LEN (sizeof(i40e_gstrings_test) / ETH_GSTRING_LEN)		#define I40E_TEST_LEN (sizeof(i40e_gstrings_test) / ETH_GSTRING_LEN)

	static const char i40e_priv_flags_strings[][ETH_GSTRING_LEN] = {		struct i40e_priv_flags {
	"MFP",		char flag_string[ETH_GSTRING_LEN];
	"LinkPolling",		u64 flag;
	"flow-director-atr",		bool read_only;
	"veb-stats",
	"hw-atr-eviction",
	};		};

	#define I40E_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_priv_flags_strings)		#define I40E_PRIV_FLAG(_name, _flag, _read_only) { \
			.flag_string = _name, \
			.flag = _flag, \
			.read_only = _read_only, \
			}

			static const struct i40e_priv_flags i40e_gstrings_priv_flags[] = {
			/* NOTE: MFP setting cannot be changed */
			I40E_PRIV_FLAG("MFP", I40E_FLAG_MFP_ENABLED, 1),
			I40E_PRIV_FLAG("LinkPolling", I40E_FLAG_LINK_POLLING_ENABLED, 0),
			I40E_PRIV_FLAG("flow-director-atr", I40E_FLAG_FD_ATR_ENABLED, 0),
			I40E_PRIV_FLAG("veb-stats", I40E_FLAG_VEB_STATS_ENABLED, 0),
			I40E_PRIV_FLAG("hw-atr-eviction", I40E_FLAG_HW_ATR_EVICT_CAPABLE, 0),
			I40E_PRIV_FLAG("legacy-rx", I40E_FLAG_LEGACY_RX, 0),
			};

			#define I40E_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_gstrings_priv_flags)

	/* Private flags with a global effect, restricted to PF 0 */		/* Private flags with a global effect, restricted to PF 0 */
	static const char i40e_gl_priv_flags_strings[][ETH_GSTRING_LEN] = {		static const struct i40e_priv_flags i40e_gl_gstrings_priv_flags[] = {
	"vf-true-promisc-support",		I40E_PRIV_FLAG("vf-true-promisc-support",
			I40E_FLAG_TRUE_PROMISC_SUPPORT, 0),
	};		};

	#define I40E_GL_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_gl_priv_flags_strings)		#define I40E_GL_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_gl_gstrings_priv_flags)

	/**		/**
	* i40e_partition_setting_complaint - generic complaint for MFP restriction		* i40e_partition_setting_complaint - generic complaint for MFP restriction
	@@ -1660,12 +1675,18 @@ static void i40e_get_strings(struct net_device *netdev, u32 stringset,
	/* BUG_ON(p - data != I40E_STATS_LEN * ETH_GSTRING_LEN); */		/* BUG_ON(p - data != I40E_STATS_LEN * ETH_GSTRING_LEN); */
	break;		break;
	case ETH_SS_PRIV_FLAGS:		case ETH_SS_PRIV_FLAGS:
	memcpy(data, i40e_priv_flags_strings,		for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
	I40E_PRIV_FLAGS_STR_LEN * ETH_GSTRING_LEN);		snprintf(p, ETH_GSTRING_LEN, "%s",
	data += I40E_PRIV_FLAGS_STR_LEN * ETH_GSTRING_LEN;		i40e_gstrings_priv_flags[i].flag_string);
	if (pf->hw.pf_id == 0)		p += ETH_GSTRING_LEN;
	memcpy(data, i40e_gl_priv_flags_strings,		}
	I40E_GL_PRIV_FLAGS_STR_LEN * ETH_GSTRING_LEN);		if (pf->hw.pf_id != 0)
			break;
			for (i = 0; i < I40E_GL_PRIV_FLAGS_STR_LEN; i++) {
			snprintf(p, ETH_GSTRING_LEN, "%s",
			i40e_gl_gstrings_priv_flags[i].flag_string);
			p += ETH_GSTRING_LEN;
			}
	break;		break;
	default:		default:
	break;		break;
	@@ -3952,7 +3973,7 @@ static int i40e_set_rxfh(struct net_device netdev, const u32 indir,
	* @dev: network interface device structure		* @dev: network interface device structure
	*		*
	* The get string set count and the string set should be matched for each		* The get string set count and the string set should be matched for each
	* flag returned. Add new strings for each flag to the i40e_priv_flags_strings		* flag returned. Add new strings for each flag to the i40e_gstrings_priv_flags
	* array.		* array.
	*		*
	* Returns a u32 bitmap of flags.		* Returns a u32 bitmap of flags.
	@@ -3962,19 +3983,27 @@ static u32 i40e_get_priv_flags(struct net_device *dev)
	struct i40e_netdev_priv *np = netdev_priv(dev);		struct i40e_netdev_priv *np = netdev_priv(dev);
	struct i40e_vsi *vsi = np->vsi;		struct i40e_vsi *vsi = np->vsi;
	struct i40e_pf *pf = vsi->back;		struct i40e_pf *pf = vsi->back;
	u32 ret_flags = 0;		u32 i, j, ret_flags = 0;

			for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
			const struct i40e_priv_flags *priv_flags;

	ret_flags \|= pf->flags & I40E_FLAG_LINK_POLLING_ENABLED ?		priv_flags = &i40e_gstrings_priv_flags[i];
	I40E_PRIV_FLAGS_LINKPOLL_FLAG : 0;
	ret_flags \|= pf->flags & I40E_FLAG_FD_ATR_ENABLED ?		if (priv_flags->flag & pf->flags)
	I40E_PRIV_FLAGS_FD_ATR : 0;		ret_flags \|= BIT(i);
	ret_flags \|= pf->flags & I40E_FLAG_VEB_STATS_ENABLED ?		}
	I40E_PRIV_FLAGS_VEB_STATS : 0;
	ret_flags \|= pf->hw_disabled_flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE ?		if (pf->hw.pf_id != 0)
	0 : I40E_PRIV_FLAGS_HW_ATR_EVICT;		return ret_flags;
	if (pf->hw.pf_id == 0) {
	ret_flags \|= pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT ?		for (j = 0; j < I40E_GL_PRIV_FLAGS_STR_LEN; j++) {
	I40E_PRIV_FLAGS_TRUE_PROMISC_SUPPORT : 0;		const struct i40e_priv_flags *priv_flags;

			priv_flags = &i40e_gl_gstrings_priv_flags[j];

			if (priv_flags->flag & pf->flags)
			ret_flags \|= BIT(i + j);
	}		}

	return ret_flags;		return ret_flags;
	@@ -3990,54 +4019,66 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
	struct i40e_netdev_priv *np = netdev_priv(dev);		struct i40e_netdev_priv *np = netdev_priv(dev);
	struct i40e_vsi *vsi = np->vsi;		struct i40e_vsi *vsi = np->vsi;
	struct i40e_pf *pf = vsi->back;		struct i40e_pf *pf = vsi->back;
	u16 sw_flags = 0, valid_flags = 0;		u64 changed_flags;
	bool reset_required = false;		u32 i, j;
	bool promisc_change = false;
	int ret;

	/* NOTE: MFP is not settable */		changed_flags = pf->flags;

	if (flags & I40E_PRIV_FLAGS_LINKPOLL_FLAG)		for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
	pf->flags \|= I40E_FLAG_LINK_POLLING_ENABLED;		const struct i40e_priv_flags *priv_flags;

			priv_flags = &i40e_gstrings_priv_flags[i];

			if (priv_flags->read_only)
			continue;

			if (flags & BIT(i))
			pf->flags \|= priv_flags->flag;
			else
			pf->flags &= ~(priv_flags->flag);
			}

			if (pf->hw.pf_id != 0)
			goto flags_complete;

			for (j = 0; j < I40E_GL_PRIV_FLAGS_STR_LEN; j++) {
			const struct i40e_priv_flags *priv_flags;

			priv_flags = &i40e_gl_gstrings_priv_flags[j];

			if (priv_flags->read_only)
			continue;

			if (flags & BIT(i + j))
			pf->flags \|= priv_flags->flag;
	else		else
	pf->flags &= ~I40E_FLAG_LINK_POLLING_ENABLED;		pf->flags &= ~(priv_flags->flag);
			}

			flags_complete:
			/* check for flags that changed */
			changed_flags ^= pf->flags;

	/* allow the user to control the state of the Flow		/* Process any additional changes needed as a result of flag changes.
	* Director ATR (Application Targeted Routing) feature		* The changed_flags value reflects the list of bits that were
	* of the driver		* changed in the code above.
	*/		*/
	if (flags & I40E_PRIV_FLAGS_FD_ATR) {
	pf->flags \|= I40E_FLAG_FD_ATR_ENABLED;
	} else {
	pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
	pf->hw_disabled_flags \|= I40E_FLAG_FD_ATR_ENABLED;

	/* flush current ATR settings */		/* Flush current ATR settings if ATR was disabled */
			if ((changed_flags & I40E_FLAG_FD_ATR_ENABLED) &&
			!(pf->flags & I40E_FLAG_FD_ATR_ENABLED)) {
			pf->hw_disabled_flags \|= I40E_FLAG_FD_ATR_ENABLED;
	set_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);		set_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
	}		}

	if ((flags & I40E_PRIV_FLAGS_VEB_STATS) &&		/* Only allow ATR evict on hardware that is capable of handling it */
	!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED)) {		if (pf->hw_disabled_flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE)
	pf->flags \|= I40E_FLAG_VEB_STATS_ENABLED;		pf->flags &= ~I40E_FLAG_HW_ATR_EVICT_CAPABLE;
	reset_required = true;
	} else if (!(flags & I40E_PRIV_FLAGS_VEB_STATS) &&		if (changed_flags & I40E_FLAG_TRUE_PROMISC_SUPPORT) {
	(pf->flags & I40E_FLAG_VEB_STATS_ENABLED)) {		u16 sw_flags = 0, valid_flags = 0;
	pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;		int ret;
	reset_required = true;
	}

	if (pf->hw.pf_id == 0) {
	if ((flags & I40E_PRIV_FLAGS_TRUE_PROMISC_SUPPORT) &&
	!(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
	pf->flags \|= I40E_FLAG_TRUE_PROMISC_SUPPORT;
	promisc_change = true;
	} else if (!(flags & I40E_PRIV_FLAGS_TRUE_PROMISC_SUPPORT) &&
	(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
	pf->flags &= ~I40E_FLAG_TRUE_PROMISC_SUPPORT;
	promisc_change = true;
	}
	}
	if (promisc_change) {
	if (!(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))		if (!(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
	sw_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;		sw_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
	valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;		valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
	@@ -4053,14 +4094,11 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
	}		}
	}		}

	if ((flags & I40E_PRIV_FLAGS_HW_ATR_EVICT) &&		/* Issue reset to cause things to take effect, as additional bits
	(pf->flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE))		* are added we will need to create a mask of bits requiring reset
	pf->hw_disabled_flags &= ~I40E_FLAG_HW_ATR_EVICT_CAPABLE;		*/
	else		if ((changed_flags & I40E_FLAG_VEB_STATS_ENABLED) \|\|
	pf->hw_disabled_flags \|= I40E_FLAG_HW_ATR_EVICT_CAPABLE;		((changed_flags & I40E_FLAG_LEGACY_RX) && netif_running(dev)))

	/* if needed, issue reset to cause things to take effect */
	if (reset_required)
	i40e_do_reset(pf, BIT(__I40E_PF_RESET_REQUESTED));		i40e_do_reset(pf, BIT(__I40E_PF_RESET_REQUESTED));

	return 0;		return 0;

drivers/net/ethernet/intel/i40e/i40e_main.c

+24 −18

Original line number	Original line	Diff line number	Diff line
	@@ -2995,7 +2995,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)

	ring->rx_buf_len = vsi->rx_buf_len;		ring->rx_buf_len = vsi->rx_buf_len;

	rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;		rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
			BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));

	rx_ctx.base = (ring->dma / 128);		rx_ctx.base = (ring->dma / 128);
	rx_ctx.qlen = ring->count;		rx_ctx.qlen = ring->count;
	@@ -3075,17 +3076,18 @@ static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
	int err = 0;		int err = 0;
	u16 i;		u16 i;

	if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))		if (!vsi->netdev \|\| (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
	vsi->max_frame = vsi->netdev->mtu + ETH_HLEN		vsi->max_frame = I40E_MAX_RXBUFFER;
	+ ETH_FCS_LEN + VLAN_HLEN;
	else
	vsi->max_frame = I40E_RXBUFFER_2048;

	vsi->rx_buf_len = I40E_RXBUFFER_2048;		vsi->rx_buf_len = I40E_RXBUFFER_2048;
			#if (PAGE_SIZE < 8192)
	/* round up for the chip's needs */		} else if (vsi->netdev->mtu <= ETH_DATA_LEN) {
	vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,		vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
	BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));		vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
			#endif
			} else {
			vsi->max_frame = I40E_MAX_RXBUFFER;
			vsi->rx_buf_len = I40E_RXBUFFER_2048;
			}

	/* set up individual rings */		/* set up individual rings */
	for (i = 0; i < vsi->num_queue_pairs && !err; i++)		for (i = 0; i < vsi->num_queue_pairs && !err; i++)
	@@ -4065,6 +4067,12 @@ static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
	}		}
	}		}

			/* Due to HW errata, on Rx disable only, the register can indicate done
			* before it really is. Needs 50ms to be sure
			*/
			if (!enable)
			mdelay(50);

	return ret;		return ret;
	}		}

	@@ -5167,10 +5175,6 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
	(hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {		(hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
	dev_info(&pf->pdev->dev,		dev_info(&pf->pdev->dev,
	"DCBX offload is not supported or is disabled for this PF.\n");		"DCBX offload is not supported or is disabled for this PF.\n");

	if (pf->flags & I40E_FLAG_MFP_ENABLED)
	goto out;

	} else {		} else {
	/* When status is not DISABLED then DCBX in FW */		/* When status is not DISABLED then DCBX in FW */
	pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED \|		pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED \|
	@@ -6519,9 +6523,11 @@ static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
	opcode);		opcode);
	break;		break;
	}		}
	} while (pending && (i++ < pf->adminq_work_limit));		} while (i++ < pf->adminq_work_limit);

			if (i < pf->adminq_work_limit)
	clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);		clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);

	/* re-enable Admin queue interrupt cause */		/* re-enable Admin queue interrupt cause */
	val = rd32(hw, I40E_PFINT_ICR0_ENA);		val = rd32(hw, I40E_PFINT_ICR0_ENA);
	val \|= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;		val \|= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;

drivers/net/ethernet/intel/i40e/i40e_txrx.c

+134 −115

Original line number	Original line	Diff line number	Diff line
	@@ -1294,6 +1294,8 @@ static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
	bi->dma = dma;		bi->dma = dma;
	bi->page = page;		bi->page = page;
	bi->page_offset = 0;		bi->page_offset = 0;

			/* initialize pagecnt_bias to 1 representing we fully own page */
	bi->pagecnt_bias = 1;		bi->pagecnt_bias = 1;

	return true;		return true;
	@@ -1622,8 +1624,6 @@ static inline bool i40e_page_is_reusable(struct page *page)
	* the adapter for another receive		* the adapter for another receive
	*		*
	* @rx_buffer: buffer containing the page		* @rx_buffer: buffer containing the page
	* @page: page address from rx_buffer
	* @truesize: actual size of the buffer in this page
	*		*
	* If page is reusable, rx_buffer->page_offset is adjusted to point to		* If page is reusable, rx_buffer->page_offset is adjusted to point to
	* an unused region in the page.		* an unused region in the page.
	@@ -1646,14 +1646,13 @@ static inline bool i40e_page_is_reusable(struct page *page)
	*		*
	* In either case, if the page is reusable its refcount is increased.		* In either case, if the page is reusable its refcount is increased.
	**/		**/
	static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,		static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer)
	struct page *page,
	const unsigned int truesize)
	{		{
	#if (PAGE_SIZE >= 8192)		#if (PAGE_SIZE >= 8192)
	unsigned int last_offset = PAGE_SIZE - I40E_RXBUFFER_2048;		unsigned int last_offset = PAGE_SIZE - I40E_RXBUFFER_2048;
	#endif		#endif
	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;		unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
			struct page *page = rx_buffer->page;

	/* Is any reuse possible? */		/* Is any reuse possible? */
	if (unlikely(!i40e_page_is_reusable(page)))		if (unlikely(!i40e_page_is_reusable(page)))
	@@ -1661,15 +1660,9 @@ static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,

	#if (PAGE_SIZE < 8192)		#if (PAGE_SIZE < 8192)
	/* if we are only owner of page we can reuse it */		/* if we are only owner of page we can reuse it */
	if (unlikely(page_count(page) != pagecnt_bias))		if (unlikely((page_count(page) - pagecnt_bias) > 1))
	return false;		return false;

	/* flip page offset to other buffer */
	rx_buffer->page_offset ^= truesize;
	#else		#else
	/* move offset up to the next cache line */
	rx_buffer->page_offset += truesize;

	if (rx_buffer->page_offset > last_offset)		if (rx_buffer->page_offset > last_offset)
	return false;		return false;
	#endif		#endif
	@@ -1678,10 +1671,11 @@ static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,
	* the pagecnt_bias and page count so that we fully restock the		* the pagecnt_bias and page count so that we fully restock the
	* number of references the driver holds.		* number of references the driver holds.
	*/		*/
	if (unlikely(pagecnt_bias == 1)) {		if (unlikely(!pagecnt_bias)) {
	page_ref_add(page, USHRT_MAX);		page_ref_add(page, USHRT_MAX);
	rx_buffer->pagecnt_bias = USHRT_MAX;		rx_buffer->pagecnt_bias = USHRT_MAX;
	}		}

	return true;		return true;
	}		}

	@@ -1689,131 +1683,142 @@ static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,
	* i40e_add_rx_frag - Add contents of Rx buffer to sk_buff		* i40e_add_rx_frag - Add contents of Rx buffer to sk_buff
	* @rx_ring: rx descriptor ring to transact packets on		* @rx_ring: rx descriptor ring to transact packets on
	* @rx_buffer: buffer containing page to add		* @rx_buffer: buffer containing page to add
	* @size: packet length from rx_desc
	* @skb: sk_buff to place the data into		* @skb: sk_buff to place the data into
			* @size: packet length from rx_desc
	*		*
	* This function will add the data contained in rx_buffer->page to the skb.		* This function will add the data contained in rx_buffer->page to the skb.
	* This is done either through a direct copy if the data in the buffer is		* It will just attach the page as a frag to the skb.
	* less than the skb header size, otherwise it will just attach the page as
	* a frag to the skb.
	*		*
	* The function will then update the page offset if necessary and return		* The function will then update the page offset.
	* true if the buffer can be reused by the adapter.
	**/		**/
	static bool i40e_add_rx_frag(struct i40e_ring *rx_ring,		static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
	struct i40e_rx_buffer *rx_buffer,		struct i40e_rx_buffer *rx_buffer,
	unsigned int size,		struct sk_buff *skb,
	struct sk_buff *skb)		unsigned int size)
	{		{
	struct page *page = rx_buffer->page;
	unsigned char *va = page_address(page) + rx_buffer->page_offset;
	#if (PAGE_SIZE < 8192)		#if (PAGE_SIZE < 8192)
	unsigned int truesize = I40E_RXBUFFER_2048;		unsigned int truesize = I40E_RXBUFFER_2048;
	#else		#else
	unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);		unsigned int truesize = SKB_DATA_ALIGN(size);
	#endif		#endif
	unsigned int pull_len;

	if (unlikely(skb_is_nonlinear(skb)))
	goto add_tail_frag;

	/* will the data fit in the skb we allocated? if so, just
	* copy it as it is pretty small anyway
	*/
	if (size <= I40E_RX_HDR_SIZE) {
	memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

	/* page is reusable, we can reuse buffer as-is */		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
	if (likely(i40e_page_is_reusable(page)))		rx_buffer->page_offset, size, truesize);
	return true;

	/* this page cannot be reused so discard it */		/* page is being used so we must update the page offset */
	return false;		#if (PAGE_SIZE < 8192)
			rx_buffer->page_offset ^= truesize;
			#else
			rx_buffer->page_offset += truesize;
			#endif
	}		}

	/* we need the header to contain the greater of either		/**
	* ETH_HLEN or 60 bytes if the skb->len is less than		* i40e_get_rx_buffer - Fetch Rx buffer and synchronize data for use
	* 60 for skb_pad.		* @rx_ring: rx descriptor ring to transact packets on
			* @size: size of buffer to add to skb
			*
			* This function will pull an Rx buffer from the ring and synchronize it
			* for use by the CPU.
	*/		*/
	pull_len = eth_get_headlen(va, I40E_RX_HDR_SIZE);		static struct i40e_rx_buffer i40e_get_rx_buffer(struct i40e_ring rx_ring,
			const unsigned int size)
			{
			struct i40e_rx_buffer *rx_buffer;

	/* align pull length to size of long to optimize		rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
	* memcpy performance		prefetchw(rx_buffer->page);
	*/
	memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));

	/* update all of the pointers */		/* we are reusing so sync this buffer for CPU use */
	va += pull_len;		dma_sync_single_range_for_cpu(rx_ring->dev,
	size -= pull_len;		rx_buffer->dma,
			rx_buffer->page_offset,
			size,
			DMA_FROM_DEVICE);

	add_tail_frag:		/* We have pulled a buffer for use, so decrement pagecnt_bias */
	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,		rx_buffer->pagecnt_bias--;
	(unsigned long)va & ~PAGE_MASK, size, truesize);

	return i40e_can_reuse_rx_page(rx_buffer, page, truesize);		return rx_buffer;
	}		}

	/**		/**
	* i40e_fetch_rx_buffer - Allocate skb and populate it		* i40e_construct_skb - Allocate skb and populate it
	* @rx_ring: rx descriptor ring to transact packets on		* @rx_ring: rx descriptor ring to transact packets on
	* @rx_desc: descriptor containing info written by hardware		* @rx_buffer: rx buffer to pull data from
			* @size: size of buffer to add to skb
	*		*
	* This function allocates an skb on the fly, and populates it with the page		* This function allocates an skb. It then populates it with the page
	* data from the current receive descriptor, taking care to set up the skb		* data from the current receive descriptor, taking care to set up the
	* correctly, as well as handling calling the page recycle function if		* skb correctly.
	* necessary.
	*/		*/
	static inline		static struct sk_buff i40e_construct_skb(struct i40e_ring rx_ring,
	struct sk_buff i40e_fetch_rx_buffer(struct i40e_ring rx_ring,		struct i40e_rx_buffer *rx_buffer,
	union i40e_rx_desc *rx_desc,		unsigned int size)
	struct sk_buff *skb)
	{		{
	u64 local_status_error_len =		void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
	le64_to_cpu(rx_desc->wb.qword1.status_error_len);		#if (PAGE_SIZE < 8192)
	unsigned int size =		unsigned int truesize = I40E_RXBUFFER_2048;
	(local_status_error_len & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>		#else
	I40E_RXD_QW1_LENGTH_PBUF_SHIFT;		unsigned int truesize = SKB_DATA_ALIGN(size);
	struct i40e_rx_buffer *rx_buffer;		#endif
	struct page *page;		unsigned int headlen;
			struct sk_buff *skb;
	rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
	page = rx_buffer->page;
	prefetchw(page);

	if (likely(!skb)) {
	void *page_addr = page_address(page) + rx_buffer->page_offset;

	/* prefetch first cache line of first page */		/* prefetch first cache line of first page */
	prefetch(page_addr);		prefetch(va);
	#if L1_CACHE_BYTES < 128		#if L1_CACHE_BYTES < 128
	prefetch(page_addr + L1_CACHE_BYTES);		prefetch(va + L1_CACHE_BYTES);
	#endif		#endif

	/* allocate a skb to store the frags */		/* allocate a skb to store the frags */
	skb = __napi_alloc_skb(&rx_ring->q_vector->napi,		skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
	I40E_RX_HDR_SIZE,		I40E_RX_HDR_SIZE,
	GFP_ATOMIC \| __GFP_NOWARN);		GFP_ATOMIC \| __GFP_NOWARN);
	if (unlikely(!skb)) {		if (unlikely(!skb))
	rx_ring->rx_stats.alloc_buff_failed++;
	return NULL;		return NULL;
	}

	/* we will be copying header into skb->data in		/* Determine available headroom for copy */
	* pskb_may_pull so it is in our interest to prefetch		headlen = size;
	* it now to avoid a possible cache miss		if (headlen > I40E_RX_HDR_SIZE)
	*/		headlen = eth_get_headlen(va, I40E_RX_HDR_SIZE);
	prefetchw(skb->data);
			/* align pull length to size of long to optimize memcpy performance */
			memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));

			/* update all of the pointers */
			size -= headlen;
			if (size) {
			skb_add_rx_frag(skb, 0, rx_buffer->page,
			rx_buffer->page_offset + headlen,
			size, truesize);

			/* buffer is used by skb, update page_offset */
			#if (PAGE_SIZE < 8192)
			rx_buffer->page_offset ^= truesize;
			#else
			rx_buffer->page_offset += truesize;
			#endif
			} else {
			/* buffer is unused, reset bias back to rx_buffer */
			rx_buffer->pagecnt_bias++;
	}		}

	/* we are reusing so sync this buffer for CPU use */		return skb;
	dma_sync_single_range_for_cpu(rx_ring->dev,		}
	rx_buffer->dma,
	rx_buffer->page_offset,
	size,
	DMA_FROM_DEVICE);

	/* pull page into skb */		/**
	if (i40e_add_rx_frag(rx_ring, rx_buffer, size, skb)) {		* i40e_put_rx_buffer - Clean up used buffer and either recycle or free
			* @rx_ring: rx descriptor ring to transact packets on
			* @rx_buffer: rx buffer to pull data from
			*
			* This function will clean up the contents of the rx_buffer. It will
			* either recycle the bufer or unmap it and free the associated resources.
			*/
			static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
			struct i40e_rx_buffer *rx_buffer)
			{
			if (i40e_can_reuse_rx_page(rx_buffer)) {
	/* hand second half of page back to the ring */		/* hand second half of page back to the ring */
	i40e_reuse_rx_page(rx_ring, rx_buffer);		i40e_reuse_rx_page(rx_ring, rx_buffer);
	rx_ring->rx_stats.page_reuse_count++;		rx_ring->rx_stats.page_reuse_count++;
	@@ -1827,8 +1832,6 @@ struct sk_buff i40e_fetch_rx_buffer(struct i40e_ring rx_ring,

	/* clear contents of buffer_info */		/* clear contents of buffer_info */
	rx_buffer->page = NULL;		rx_buffer->page = NULL;

	return skb;
	}		}

	/**		/**
	@@ -1889,7 +1892,9 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
	bool failure = false;		bool failure = false;

	while (likely(total_rx_packets < budget)) {		while (likely(total_rx_packets < budget)) {
			struct i40e_rx_buffer *rx_buffer;
	union i40e_rx_desc *rx_desc;		union i40e_rx_desc *rx_desc;
			unsigned int size;
	u16 vlan_tag;		u16 vlan_tag;
	u8 rx_ptype;		u8 rx_ptype;
	u64 qword;		u64 qword;
	@@ -1906,22 +1911,36 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
	/* status_error_len will always be zero for unused descriptors		/* status_error_len will always be zero for unused descriptors
	* because it's cleared in cleanup, and overlaps with hdr_addr		* because it's cleared in cleanup, and overlaps with hdr_addr
	* which is always zero because packet split isn't used, if the		* which is always zero because packet split isn't used, if the
	* hardware wrote DD then it will be non-zero		* hardware wrote DD then the length will be non-zero
	*/		*/
	if (!i40e_test_staterr(rx_desc,		qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
	BIT(I40E_RX_DESC_STATUS_DD_SHIFT)))		size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
			I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
			if (!size)
	break;		break;

	/* This memory barrier is needed to keep us from reading		/* This memory barrier is needed to keep us from reading
	* any other fields out of the rx_desc until we know the		* any other fields out of the rx_desc until we have
	* DD bit is set.		* verified the descriptor has been written back.
	*/		*/
	dma_rmb();		dma_rmb();

	skb = i40e_fetch_rx_buffer(rx_ring, rx_desc, skb);		rx_buffer = i40e_get_rx_buffer(rx_ring, size);
	if (!skb)
			/* retrieve a buffer from the ring */
			if (skb)
			i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
			else
			skb = i40e_construct_skb(rx_ring, rx_buffer, size);

			/* exit if we failed to retrieve a buffer */
			if (!skb) {
			rx_ring->rx_stats.alloc_buff_failed++;
			rx_buffer->pagecnt_bias++;
	break;		break;
			}

			i40e_put_rx_buffer(rx_ring, rx_buffer);
	cleaned_count++;		cleaned_count++;

	if (i40e_is_non_eop(rx_ring, rx_desc, skb))		if (i40e_is_non_eop(rx_ring, rx_desc, skb))