Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 85740cdf authored by Ben Hutchings's avatar Ben Hutchings
Browse files

sfc: Enable RX DMA scattering where possible 



Enable RX DMA scattering iff an RX buffer large enough for the current
MTU will not fit into a single page and the NIC supports DMA
scattering for kernel-mode RX queues.

On Falcon and Siena, the RX_USR_BUF_SIZE field is used as the DMA
limit for both all RX queues with scatter enabled.  Set it to 1824,
matching what Onload uses now.

Maintain a statistic for frames truncated due to lack of descriptors
(rx_nodesc_trunc).  This is distinct from rx_frm_trunc which may be
incremented when scattering is disabled and implies an over-length
frame.

Whenever an MTU change causes scattering to be turned on or off,
update filters that point to the PF queues, but leave others
unchanged, as VF drivers assume scattering is off.

Add n_frags parameters to various functions, and make them iterate:
- efx_rx_packet()
- efx_recycle_rx_buffers()
- efx_rx_mk_skb()
- efx_rx_deliver()

Make efx_handle_rx_event() responsible for updating
efx_rx_queue::removed_count.

Change the RX pipeline state to a starting ring index and number of
fragments, and make __efx_rx_packet() responsible for clearing it.

Based on earlier versions by David Riddoch and Jon Cooper.

Signed-off-by: default avatarBen Hutchings <bhutchings@solarflare.com>
parent b74e3e8c

drivers/net/ethernet/sfc/efx.c

+29 −5
Original line number Diff line number Diff line
@@ -88,8 +88,6 @@ const char *const efx_reset_type_names[] = { 
	[RESET_TYPE_MC_FAILURE]         = "MC_FAILURE",

};



#define EFX_MAX_MTU (9 * 1024)



/* Reset workqueue. If any NIC has a hardware failure then a reset will be

 * queued onto this work queue. This is not a per-nic work queue, because

 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
@@ -627,9 +625,11 @@ static int efx_probe_channels(struct efx_nic *efx) 
 */

static void efx_start_datapath(struct efx_nic *efx)

{

	bool old_rx_scatter = efx->rx_scatter;

	struct efx_tx_queue *tx_queue;

	struct efx_rx_queue *rx_queue;

	struct efx_channel *channel;

	size_t rx_buf_len;



	/* Calculate the rx buffer allocation parameters required to

	 * support the current MTU, including padding for header
@@ -638,8 +638,32 @@ static void efx_start_datapath(struct efx_nic *efx) 
	efx->rx_dma_len = (efx->type->rx_buffer_hash_size +

			   EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +

			   efx->type->rx_buffer_padding);

	efx->rx_buffer_order = get_order(sizeof(struct efx_rx_page_state) +

	rx_buf_len = (sizeof(struct efx_rx_page_state) +

		      EFX_PAGE_IP_ALIGN + efx->rx_dma_len);

	if (rx_buf_len <= PAGE_SIZE) {

		efx->rx_scatter = false;

		efx->rx_buffer_order = 0;

		if (rx_buf_len <= PAGE_SIZE / 2)

			efx->rx_buffer_truesize = PAGE_SIZE / 2;

		else

			efx->rx_buffer_truesize = PAGE_SIZE;

	} else if (efx->type->can_rx_scatter) {

		BUILD_BUG_ON(sizeof(struct efx_rx_page_state) +

			     EFX_PAGE_IP_ALIGN + EFX_RX_USR_BUF_SIZE >

			     PAGE_SIZE / 2);

		efx->rx_scatter = true;

		efx->rx_dma_len = EFX_RX_USR_BUF_SIZE;

		efx->rx_buffer_order = 0;

		efx->rx_buffer_truesize = PAGE_SIZE / 2;

	} else {

		efx->rx_scatter = false;

		efx->rx_buffer_order = get_order(rx_buf_len);

		efx->rx_buffer_truesize = PAGE_SIZE << efx->rx_buffer_order;

	}



	/* RX filters also have scatter-enabled flags */

	if (efx->rx_scatter != old_rx_scatter)

		efx_filter_update_rx_scatter(efx);



	/* We must keep at least one descriptor in a TX ring empty.

	 * We could avoid this when the queue size does not exactly
@@ -661,7 +685,7 @@ static void efx_start_datapath(struct efx_nic *efx) 
			efx_nic_generate_fill_event(rx_queue);

		}



		WARN_ON(channel->rx_pkt != NULL);

		WARN_ON(channel->rx_pkt_n_frags);

	}



	if (netif_device_present(efx->net_dev))

drivers/net/ethernet/sfc/efx.h

+6 −7
Original line number Diff line number Diff line
@@ -39,16 +39,14 @@ extern void efx_init_rx_queue(struct efx_rx_queue *rx_queue); 
extern void efx_fini_rx_queue(struct efx_rx_queue *rx_queue);

extern void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue);

extern void efx_rx_slow_fill(unsigned long context);

extern void __efx_rx_packet(struct efx_channel *channel,

			    struct efx_rx_buffer *rx_buf);

extern void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index,

extern void __efx_rx_packet(struct efx_channel *channel);

extern void efx_rx_packet(struct efx_rx_queue *rx_queue,

			  unsigned int index, unsigned int n_frags,

			  unsigned int len, u16 flags);

static inline void efx_rx_flush_packet(struct efx_channel *channel)

{

	if (channel->rx_pkt) {

		__efx_rx_packet(channel, channel->rx_pkt);

		channel->rx_pkt = NULL;

	}

	if (channel->rx_pkt_n_frags)

		__efx_rx_packet(channel);

}

extern void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue);
@@ -73,6 +71,7 @@ extern void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue); 
extern int efx_probe_filters(struct efx_nic *efx);

extern void efx_restore_filters(struct efx_nic *efx);

extern void efx_remove_filters(struct efx_nic *efx);

extern void efx_filter_update_rx_scatter(struct efx_nic *efx);

extern s32 efx_filter_insert_filter(struct efx_nic *efx,

				    struct efx_filter_spec *spec,

				    bool replace);

drivers/net/ethernet/sfc/ethtool.c

+3 −1
Original line number Diff line number Diff line
@@ -154,6 +154,7 @@ static const struct efx_ethtool_stat efx_ethtool_stats[] = { 
	EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_tcp_udp_chksum_err),

	EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_mcast_mismatch),

	EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_frm_trunc),

	EFX_ETHTOOL_UINT_CHANNEL_STAT(rx_nodesc_trunc),

};



/* Number of ethtool statistics */
@@ -978,7 +979,8 @@ static int efx_ethtool_set_class_rule(struct efx_nic *efx, 
	     rule->m_ext.data[1]))

		return -EINVAL;



	efx_filter_init_rx(&spec, EFX_FILTER_PRI_MANUAL, 0,

	efx_filter_init_rx(&spec, EFX_FILTER_PRI_MANUAL,

			   efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0,

			   (rule->ring_cookie == RX_CLS_FLOW_DISC) ?

			   0xfff : rule->ring_cookie);

drivers/net/ethernet/sfc/falcon.c

+10 −7
Original line number Diff line number Diff line
@@ -1546,10 +1546,6 @@ static int falcon_probe_nic(struct efx_nic *efx) 


static void falcon_init_rx_cfg(struct efx_nic *efx)

{

	/* Prior to Siena the RX DMA engine will split each frame at

	 * intervals of RX_USR_BUF_SIZE (32-byte units). We set it to

	 * be so large that that never happens. */

	const unsigned huge_buf_size = (3 * 4096) >> 5;

	/* RX control FIFO thresholds (32 entries) */

	const unsigned ctrl_xon_thr = 20;

	const unsigned ctrl_xoff_thr = 25;
@@ -1557,10 +1553,15 @@ static void falcon_init_rx_cfg(struct efx_nic *efx) 


	efx_reado(efx, &reg, FR_AZ_RX_CFG);

	if (efx_nic_rev(efx) <= EFX_REV_FALCON_A1) {

		/* Data FIFO size is 5.5K */

		/* Data FIFO size is 5.5K.  The RX DMA engine only

		 * supports scattering for user-mode queues, but will

		 * split DMA writes at intervals of RX_USR_BUF_SIZE

		 * (32-byte units) even for kernel-mode queues.  We

		 * set it to be so large that that never happens.

		 */

		EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_DESC_PUSH_EN, 0);

		EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_USR_BUF_SIZE,

				    huge_buf_size);

				    (3 * 4096) >> 5);

		EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_MAC_TH, 512 >> 8);

		EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_MAC_TH, 2048 >> 8);

		EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_TX_TH, ctrl_xon_thr);
@@ -1569,7 +1570,7 @@ static void falcon_init_rx_cfg(struct efx_nic *efx) 
		/* Data FIFO size is 80K; register fields moved */

		EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_DESC_PUSH_EN, 0);

		EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_USR_BUF_SIZE,

				    huge_buf_size);

				    EFX_RX_USR_BUF_SIZE >> 5);

		/* Send XON and XOFF at ~3 * max MTU away from empty/full */

		EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_MAC_TH, 27648 >> 8);

		EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_MAC_TH, 54272 >> 8);
@@ -1815,6 +1816,7 @@ const struct efx_nic_type falcon_a1_nic_type = { 
	.evq_rptr_tbl_base = FR_AA_EVQ_RPTR_KER,

	.max_dma_mask = DMA_BIT_MASK(FSF_AZ_TX_KER_BUF_ADDR_WIDTH),

	.rx_buffer_padding = 0x24,

	.can_rx_scatter = false,

	.max_interrupt_mode = EFX_INT_MODE_MSI,

	.phys_addr_channels = 4,

	.timer_period_max =  1 << FRF_AB_TC_TIMER_VAL_WIDTH,
@@ -1865,6 +1867,7 @@ const struct efx_nic_type falcon_b0_nic_type = { 
	.max_dma_mask = DMA_BIT_MASK(FSF_AZ_TX_KER_BUF_ADDR_WIDTH),

	.rx_buffer_hash_size = 0x10,

	.rx_buffer_padding = 0,

	.can_rx_scatter = true,

	.max_interrupt_mode = EFX_INT_MODE_MSIX,

	.phys_addr_channels = 32, /* Hardware limit is 64, but the legacy

				   * interrupt handler only supports 32

drivers/net/ethernet/sfc/filter.c

+71 −3
Original line number Diff line number Diff line
@@ -172,6 +172,25 @@ static void efx_filter_push_rx_config(struct efx_nic *efx) 
			filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,

			!!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags &

			   EFX_FILTER_FLAG_RX_RSS));



		/* There is a single bit to enable RX scatter for all

		 * unmatched packets.  Only set it if scatter is

		 * enabled in both filter specs.

		 */

		EFX_SET_OWORD_FIELD(

			filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q,

			!!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags &

			   table->spec[EFX_FILTER_INDEX_MC_DEF].flags &

			   EFX_FILTER_FLAG_RX_SCATTER));

	} else if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {

		/* We don't expose 'default' filters because unmatched

		 * packets always go to the queue number found in the

		 * RSS table.  But we still need to set the RX scatter

		 * bit here.

		 */

		EFX_SET_OWORD_FIELD(

			filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q,

			efx->rx_scatter);

	}



	efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
@@ -413,13 +432,18 @@ static void efx_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx) 
	struct efx_filter_state *state = efx->filter_state;

	struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_DEF];

	struct efx_filter_spec *spec = &table->spec[filter_idx];

	enum efx_filter_flags flags = 0;



	/* If there's only one channel then disable RSS for non VF

	 * traffic, thereby allowing VFs to use RSS when the PF can't.

	 */

	efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL,

			   efx->n_rx_channels > 1 ? EFX_FILTER_FLAG_RX_RSS : 0,

			   0);

	if (efx->n_rx_channels > 1)

		flags |= EFX_FILTER_FLAG_RX_RSS;



	if (efx->rx_scatter)

		flags |= EFX_FILTER_FLAG_RX_SCATTER;



	efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL, flags, 0);

	spec->type = EFX_FILTER_UC_DEF + filter_idx;

	table->used_bitmap[0] |= 1 << filter_idx;

}
@@ -1101,6 +1125,50 @@ void efx_remove_filters(struct efx_nic *efx) 
	kfree(state);

}



/* Update scatter enable flags for filters pointing to our own RX queues */

void efx_filter_update_rx_scatter(struct efx_nic *efx)

{

	struct efx_filter_state *state = efx->filter_state;

	enum efx_filter_table_id table_id;

	struct efx_filter_table *table;

	efx_oword_t filter;

	unsigned int filter_idx;



	spin_lock_bh(&state->lock);



	for (table_id = EFX_FILTER_TABLE_RX_IP;

	     table_id <= EFX_FILTER_TABLE_RX_DEF;

	     table_id++) {

		table = &state->table[table_id];



		for (filter_idx = 0; filter_idx < table->size; filter_idx++) {

			if (!test_bit(filter_idx, table->used_bitmap) ||

			    table->spec[filter_idx].dmaq_id >=

			    efx->n_rx_channels)

				continue;



			if (efx->rx_scatter)

				table->spec[filter_idx].flags |=

					EFX_FILTER_FLAG_RX_SCATTER;

			else

				table->spec[filter_idx].flags &=

					~EFX_FILTER_FLAG_RX_SCATTER;



			if (table_id == EFX_FILTER_TABLE_RX_DEF)

				/* Pushed by efx_filter_push_rx_config() */

				continue;



			efx_filter_build(&filter, &table->spec[filter_idx]);

			efx_writeo(efx, &filter,

				   table->offset + table->step * filter_idx);

		}

	}



	efx_filter_push_rx_config(efx);



	spin_unlock_bh(&state->lock);

}



#ifdef CONFIG_RFS_ACCEL



int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,