Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2ea32cd6 authored by Doug Ledford's avatar Doug Ledford
Browse files

Merge tag 'mlx5-updates-2018-02-28-2' of... 

Merge tag 'mlx5-updates-2018-02-28-2' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux

 into k.o/wip/dl-for-next

mlx5-updates-2018-02-28-2 (IPSec-2)

This series follows our previous one to lay out the foundations for IPSec
in user-space and extend current kernel netdev IPSec support. As noted in
our previous pull request cover letter "mlx5-updates-2018-02-28-1 (IPSec-1)",
the IPSec mechanism will be supported through our flow steering mechanism.
Therefore, we need to change the initialization order. Furthermore, IPsec
is also supported in both egress and ingress. Since our current flow
steering is egress only, we add an empty (only implemented through FPGA
steering ops) egress namespace to handle that case. We also implement
the required flow steering callbacks and logic in our FPGA driver.

We extend the FPGA support for ESN and modifying a xfrm too. Therefore, we
add support for some new FPGA command interface that supports them. The
other required bits are added too. The new features and requirements are
advertised via cap bits.

Last but not least, we revise our driver's accel_esp API. This API will be
shared between our netdev and IB driver, so we need to have all the required
functionality from both worlds.

Regards,
Aviad and Matan

Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parents 29cf1351 31135eb3

drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.c

+45 −14
Original line number Diff line number Diff line
@@ -37,24 +37,11 @@ 
#include "mlx5_core.h"

#include "fpga/ipsec.h"



void *mlx5_accel_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev,

				   struct mlx5_accel_ipsec_sa *cmd)

{

	if (!MLX5_IPSEC_DEV(mdev))

		return ERR_PTR(-EOPNOTSUPP);



	return mlx5_fpga_ipsec_sa_cmd_exec(mdev, cmd);

}



int mlx5_accel_ipsec_sa_cmd_wait(void *ctx)

{

	return mlx5_fpga_ipsec_sa_cmd_wait(ctx);

}



u32 mlx5_accel_ipsec_device_caps(struct mlx5_core_dev *mdev)

{

	return mlx5_fpga_ipsec_device_caps(mdev);

}

EXPORT_SYMBOL_GPL(mlx5_accel_ipsec_device_caps);



unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev)

{
@@ -67,6 +54,21 @@ int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, 
	return mlx5_fpga_ipsec_counters_read(mdev, counters, count);

}



void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev,

				       struct mlx5_accel_esp_xfrm *xfrm,

				       const __be32 saddr[4],

				       const __be32 daddr[4],

				       const __be32 spi, bool is_ipv6)

{

	return mlx5_fpga_ipsec_create_sa_ctx(mdev, xfrm, saddr, daddr,

					     spi, is_ipv6);

}



void mlx5_accel_esp_free_hw_context(void *context)

{

	mlx5_fpga_ipsec_delete_sa_ctx(context);

}



int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev)

{

	return mlx5_fpga_ipsec_init(mdev);
@@ -76,3 +78,32 @@ void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) 
{

	mlx5_fpga_ipsec_cleanup(mdev);

}



struct mlx5_accel_esp_xfrm *

mlx5_accel_esp_create_xfrm(struct mlx5_core_dev *mdev,

			   const struct mlx5_accel_esp_xfrm_attrs *attrs,

			   u32 flags)

{

	struct mlx5_accel_esp_xfrm *xfrm;



	xfrm = mlx5_fpga_esp_create_xfrm(mdev, attrs, flags);

	if (IS_ERR(xfrm))

		return xfrm;



	xfrm->mdev = mdev;

	return xfrm;

}

EXPORT_SYMBOL_GPL(mlx5_accel_esp_create_xfrm);



void mlx5_accel_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)

{

	mlx5_fpga_esp_destroy_xfrm(xfrm);

}

EXPORT_SYMBOL_GPL(mlx5_accel_esp_destroy_xfrm);



int mlx5_accel_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,

			       const struct mlx5_accel_esp_xfrm_attrs *attrs)

{

	return mlx5_fpga_esp_modify_xfrm(xfrm, attrs);

}

EXPORT_SYMBOL_GPL(mlx5_accel_esp_modify_xfrm);

drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h

+23 −73
Original line number Diff line number Diff line
@@ -35,88 +35,24 @@ 
#define __MLX5_ACCEL_IPSEC_H__



#include <linux/mlx5/driver.h>

#include <linux/mlx5/accel.h>



#ifdef CONFIG_MLX5_ACCEL



enum {

	MLX5_ACCEL_IPSEC_DEVICE = BIT(1),

	MLX5_ACCEL_IPSEC_IPV6 = BIT(2),

	MLX5_ACCEL_IPSEC_ESP = BIT(3),

	MLX5_ACCEL_IPSEC_LSO = BIT(4),

};



#define MLX5_IPSEC_SADB_IP_AH       BIT(7)

#define MLX5_IPSEC_SADB_IP_ESP      BIT(6)

#define MLX5_IPSEC_SADB_SA_VALID    BIT(5)

#define MLX5_IPSEC_SADB_SPI_EN      BIT(4)

#define MLX5_IPSEC_SADB_DIR_SX      BIT(3)

#define MLX5_IPSEC_SADB_IPV6        BIT(2)



enum {

	MLX5_IPSEC_CMD_ADD_SA = 0,

	MLX5_IPSEC_CMD_DEL_SA = 1,

};



enum mlx5_accel_ipsec_enc_mode {

	MLX5_IPSEC_SADB_MODE_NONE = 0,

	MLX5_IPSEC_SADB_MODE_AES_GCM_128_AUTH_128 = 1,

	MLX5_IPSEC_SADB_MODE_AES_GCM_256_AUTH_128 = 3,

};



#define MLX5_IPSEC_DEV(mdev) (mlx5_accel_ipsec_device_caps(mdev) & \

			      MLX5_ACCEL_IPSEC_DEVICE)



struct mlx5_accel_ipsec_sa {

	__be32 cmd;

	u8 key_enc[32];

	u8 key_auth[32];

	__be32 sip[4];

	__be32 dip[4];

	union {

		struct {

			__be32 reserved;

			u8 salt_iv[8];

			__be32 salt;

		} __packed gcm;

		struct {

			u8 salt[16];

		} __packed cbc;

	};

	__be32 spi;

	__be32 sw_sa_handle;

	__be16 tfclen;

	u8 enc_mode;

	u8 sip_masklen;

	u8 dip_masklen;

	u8 flags;

	u8 reserved[2];

} __packed;



/**

 * mlx5_accel_ipsec_sa_cmd_exec - Execute an IPSec SADB command

 * @mdev: mlx5 device

 * @cmd: command to execute

 * May be called from atomic context. Returns context pointer, or error

 * Caller must eventually call mlx5_accel_ipsec_sa_cmd_wait from non-atomic

 * context, to cleanup the context pointer

 */

void *mlx5_accel_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev,

				   struct mlx5_accel_ipsec_sa *cmd);



/**

 * mlx5_accel_ipsec_sa_cmd_wait - Wait for command execution completion

 * @context: Context pointer returned from call to mlx5_accel_ipsec_sa_cmd_exec

 * Sleeps (killable) until command execution is complete.

 * Returns the command result, or -EINTR if killed

 */

int mlx5_accel_ipsec_sa_cmd_wait(void *context);



u32 mlx5_accel_ipsec_device_caps(struct mlx5_core_dev *mdev);

			      MLX5_ACCEL_IPSEC_CAP_DEVICE)



unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev);

int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,

				   unsigned int count);



void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev,

				       struct mlx5_accel_esp_xfrm *xfrm,

				       const __be32 saddr[4],

				       const __be32 daddr[4],

				       const __be32 spi, bool is_ipv6);

void mlx5_accel_esp_free_hw_context(void *context);



int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev);

void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev);
@@ -124,6 +60,20 @@ void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev); 


#define MLX5_IPSEC_DEV(mdev) false



static inline void *

mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev,

				 struct mlx5_accel_esp_xfrm *xfrm,

				 const __be32 saddr[4],

				 const __be32 daddr[4],

				 const __be32 spi, bool is_ipv6)

{

	return NULL;

}



static inline void mlx5_accel_esp_free_hw_context(void *context)

{

}



static inline int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev)

{

	return 0;

drivers/net/ethernet/mellanox/mlx5/core/cq.c

+1 −2
Original line number Diff line number Diff line
@@ -109,8 +109,7 @@ int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 
	cq->cons_index = 0;

	cq->arm_sn     = 0;

	cq->eq         = eq;

	refcount_set(&cq->refcount, 0);

	mlx5_cq_hold(cq);

	refcount_set(&cq->refcount, 1);

	init_completion(&cq->free);

	if (!cq->comp)

		cq->comp = mlx5_add_cq_to_tasklet;

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c

+200 −106
Original line number Diff line number Diff line
@@ -38,17 +38,24 @@ 
#include <linux/module.h>



#include "en.h"

#include "accel/ipsec.h"

#include "en_accel/ipsec.h"

#include "en_accel/ipsec_rxtx.h"



struct mlx5e_ipsec_sa_entry {

	struct hlist_node hlist; /* Item in SADB_RX hashtable */

	unsigned int handle; /* Handle in SADB_RX */

	struct xfrm_state *x;

	struct mlx5e_ipsec *ipsec;

	void *context;

};



static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x)

{

	struct mlx5e_ipsec_sa_entry *sa;



	if (!x)

		return NULL;



	sa = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;

	if (!sa)

		return NULL;



	WARN_ON(sa->x != x);

	return sa;

}



struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec,

					      unsigned int handle)
@@ -105,78 +112,93 @@ static void mlx5e_ipsec_sadb_rx_free(struct mlx5e_ipsec_sa_entry *sa_entry) 
	ida_simple_remove(&ipsec->halloc, sa_entry->handle);

}



static enum mlx5_accel_ipsec_enc_mode mlx5e_ipsec_enc_mode(struct xfrm_state *x)

static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)

{

	unsigned int key_len = (x->aead->alg_key_len + 7) / 8 - 4;

	struct xfrm_replay_state_esn *replay_esn;

	u32 seq_bottom;

	u8 overlap;

	u32 *esn;



	switch (key_len) {

	case 16:

		return MLX5_IPSEC_SADB_MODE_AES_GCM_128_AUTH_128;

	case 32:

		return MLX5_IPSEC_SADB_MODE_AES_GCM_256_AUTH_128;

	default:

		netdev_warn(x->xso.dev, "Bad key len: %d for alg %s\n",

			    key_len, x->aead->alg_name);

		return -1;

	if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) {

		sa_entry->esn_state.trigger = 0;

		return false;

	}



	replay_esn = sa_entry->x->replay_esn;

	seq_bottom = replay_esn->seq - replay_esn->replay_window + 1;

	overlap = sa_entry->esn_state.overlap;



	sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x,

						    htonl(seq_bottom));

	esn = &sa_entry->esn_state.esn;



	sa_entry->esn_state.trigger = 1;

	if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) {

		++(*esn);

		sa_entry->esn_state.overlap = 0;

		return true;

	} else if (unlikely(!overlap &&

			    (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) {

		sa_entry->esn_state.overlap = 1;

		return true;

	}



static void mlx5e_ipsec_build_hw_sa(u32 op, struct mlx5e_ipsec_sa_entry *sa_entry,

				    struct mlx5_accel_ipsec_sa *hw_sa)

	return false;

}



static void

mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,

				   struct mlx5_accel_esp_xfrm_attrs *attrs)

{

	struct xfrm_state *x = sa_entry->x;

	struct aes_gcm_keymat *aes_gcm = &attrs->keymat.aes_gcm;

	struct aead_geniv_ctx *geniv_ctx;

	unsigned int crypto_data_len;

	struct crypto_aead *aead;

	unsigned int key_len;

	unsigned int crypto_data_len, key_len;

	int ivsize;



	memset(hw_sa, 0, sizeof(*hw_sa));

	memset(attrs, 0, sizeof(*attrs));



	if (op == MLX5_IPSEC_CMD_ADD_SA) {

	/* key */

	crypto_data_len = (x->aead->alg_key_len + 7) / 8;

	key_len = crypto_data_len - 4; /* 4 bytes salt at end */



	memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len);

	aes_gcm->key_len = key_len * 8;



	/* salt and seq_iv */

	aead = x->data;

	geniv_ctx = crypto_aead_ctx(aead);

	ivsize = crypto_aead_ivsize(aead);

	memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize);

	memcpy(&aes_gcm->salt, x->aead->alg_key + key_len,

	       sizeof(aes_gcm->salt));



		memcpy(&hw_sa->key_enc, x->aead->alg_key, key_len);

		/* Duplicate 128 bit key twice according to HW layout */

		if (key_len == 16)

			memcpy(&hw_sa->key_enc[16], x->aead->alg_key, key_len);

		memcpy(&hw_sa->gcm.salt_iv, geniv_ctx->salt, ivsize);

		hw_sa->gcm.salt = *((__be32 *)(x->aead->alg_key + key_len));

	}

	/* iv len */

	aes_gcm->icv_len = x->aead->alg_icv_len;



	hw_sa->cmd = htonl(op);

	hw_sa->flags |= MLX5_IPSEC_SADB_SA_VALID | MLX5_IPSEC_SADB_SPI_EN;

	if (x->props.family == AF_INET) {

		hw_sa->sip[3] = x->props.saddr.a4;

		hw_sa->dip[3] = x->id.daddr.a4;

		hw_sa->sip_masklen = 32;

		hw_sa->dip_masklen = 32;

	} else {

		memcpy(hw_sa->sip, x->props.saddr.a6, sizeof(hw_sa->sip));

		memcpy(hw_sa->dip, x->id.daddr.a6, sizeof(hw_sa->dip));

		hw_sa->sip_masklen = 128;

		hw_sa->dip_masklen = 128;

		hw_sa->flags |= MLX5_IPSEC_SADB_IPV6;

	}

	hw_sa->spi = x->id.spi;

	hw_sa->sw_sa_handle = htonl(sa_entry->handle);

	switch (x->id.proto) {

	case IPPROTO_ESP:

		hw_sa->flags |= MLX5_IPSEC_SADB_IP_ESP;

		break;

	case IPPROTO_AH:

		hw_sa->flags |= MLX5_IPSEC_SADB_IP_AH;

		break;

	default:

		break;

	/* esn */

	if (sa_entry->esn_state.trigger) {

		attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;

		attrs->esn = sa_entry->esn_state.esn;

		if (sa_entry->esn_state.overlap)

			attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;

	}

	hw_sa->enc_mode = mlx5e_ipsec_enc_mode(x);

	if (!(x->xso.flags & XFRM_OFFLOAD_INBOUND))

		hw_sa->flags |= MLX5_IPSEC_SADB_DIR_SX;



	/* rx handle */

	attrs->sa_handle = sa_entry->handle;



	/* algo type */

	attrs->keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;



	/* action */

	attrs->action = (!(x->xso.flags & XFRM_OFFLOAD_INBOUND)) ?

			MLX5_ACCEL_ESP_ACTION_ENCRYPT :

			MLX5_ACCEL_ESP_ACTION_DECRYPT;

	/* flags */

	attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ?

			MLX5_ACCEL_ESP_FLAGS_TRANSPORT :

			MLX5_ACCEL_ESP_FLAGS_TUNNEL;

}



static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
@@ -198,7 +220,9 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) 
		netdev_info(netdev, "Cannot offload compressed xfrm states\n");

		return -EINVAL;

	}

	if (x->props.flags & XFRM_STATE_ESN) {

	if (x->props.flags & XFRM_STATE_ESN &&

	    !(mlx5_accel_ipsec_device_caps(priv->mdev) &

	    MLX5_ACCEL_IPSEC_CAP_ESN)) {

		netdev_info(netdev, "Cannot offload ESN xfrm states\n");

		return -EINVAL;

	}
@@ -246,7 +270,8 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) 
		return -EINVAL;

	}

	if (x->props.family == AF_INET6 &&

	    !(mlx5_accel_ipsec_device_caps(priv->mdev) & MLX5_ACCEL_IPSEC_IPV6)) {

	    !(mlx5_accel_ipsec_device_caps(priv->mdev) &

	     MLX5_ACCEL_IPSEC_CAP_IPV6)) {

		netdev_info(netdev, "IPv6 xfrm state offload is not supported by this device\n");

		return -EINVAL;

	}
@@ -257,9 +282,10 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) 
{

	struct mlx5e_ipsec_sa_entry *sa_entry = NULL;

	struct net_device *netdev = x->xso.dev;

	struct mlx5_accel_ipsec_sa hw_sa;

	struct mlx5_accel_esp_xfrm_attrs attrs;

	struct mlx5e_priv *priv;

	void *context;

	__be32 saddr[4] = {0}, daddr[4] = {0}, spi;

	bool is_ipv6 = false;

	int err;



	priv = netdev_priv(netdev);
@@ -286,22 +312,49 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) 
			netdev_info(netdev, "Failed adding to SADB_RX: %d\n", err);

			goto err_entry;

		}

	} else {

		sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?

				mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;

	}



	mlx5e_ipsec_build_hw_sa(MLX5_IPSEC_CMD_ADD_SA, sa_entry, &hw_sa);

	context = mlx5_accel_ipsec_sa_cmd_exec(sa_entry->ipsec->en_priv->mdev, &hw_sa);

	if (IS_ERR(context)) {

		err = PTR_ERR(context);

	/* check esn */

	mlx5e_ipsec_update_esn_state(sa_entry);



	/* create xfrm */

	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);

	sa_entry->xfrm =

		mlx5_accel_esp_create_xfrm(priv->mdev, &attrs,

					   MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA);

	if (IS_ERR(sa_entry->xfrm)) {

		err = PTR_ERR(sa_entry->xfrm);

		goto err_sadb_rx;

	}



	err = mlx5_accel_ipsec_sa_cmd_wait(context);

	if (err)

		goto err_sadb_rx;

	/* create hw context */

	if (x->props.family == AF_INET) {

		saddr[3] = x->props.saddr.a4;

		daddr[3] = x->id.daddr.a4;

	} else {

		memcpy(saddr, x->props.saddr.a6, sizeof(saddr));

		memcpy(daddr, x->id.daddr.a6, sizeof(daddr));

		is_ipv6 = true;

	}

	spi = x->id.spi;

	sa_entry->hw_context =

			mlx5_accel_esp_create_hw_context(priv->mdev,

							 sa_entry->xfrm,

							 saddr, daddr, spi,

							 is_ipv6);

	if (IS_ERR(sa_entry->hw_context)) {

		err = PTR_ERR(sa_entry->hw_context);

		goto err_xfrm;

	}



	x->xso.offload_handle = (unsigned long)sa_entry;

	goto out;



err_xfrm:

	mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);

err_sadb_rx:

	if (x->xso.flags & XFRM_OFFLOAD_INBOUND) {

		mlx5e_ipsec_sadb_rx_del(sa_entry);
@@ -315,43 +368,26 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) 


static void mlx5e_xfrm_del_state(struct xfrm_state *x)

{

	struct mlx5e_ipsec_sa_entry *sa_entry;

	struct mlx5_accel_ipsec_sa hw_sa;

	void *context;

	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);



	if (!x->xso.offload_handle)

	if (!sa_entry)

		return;



	sa_entry = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;

	WARN_ON(sa_entry->x != x);



	if (x->xso.flags & XFRM_OFFLOAD_INBOUND)

		mlx5e_ipsec_sadb_rx_del(sa_entry);



	mlx5e_ipsec_build_hw_sa(MLX5_IPSEC_CMD_DEL_SA, sa_entry, &hw_sa);

	context = mlx5_accel_ipsec_sa_cmd_exec(sa_entry->ipsec->en_priv->mdev, &hw_sa);

	if (IS_ERR(context))

		return;



	sa_entry->context = context;

}



static void mlx5e_xfrm_free_state(struct xfrm_state *x)

{

	struct mlx5e_ipsec_sa_entry *sa_entry;

	int res;

	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);



	if (!x->xso.offload_handle)

	if (!sa_entry)

		return;



	sa_entry = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;

	WARN_ON(sa_entry->x != x);



	res = mlx5_accel_ipsec_sa_cmd_wait(sa_entry->context);

	sa_entry->context = NULL;

	if (res) {

		/* Leftover object will leak */

		return;

	if (sa_entry->hw_context) {

		flush_workqueue(sa_entry->ipsec->wq);

		mlx5_accel_esp_free_hw_context(sa_entry->hw_context);

		mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);

	}



	if (x->xso.flags & XFRM_OFFLOAD_INBOUND)
@@ -378,6 +414,14 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv) 
	ida_init(&ipsec->halloc);

	ipsec->en_priv = priv;

	ipsec->en_priv->ipsec = ipsec;

	ipsec->no_trailer = !!(mlx5_accel_ipsec_device_caps(priv->mdev) &

			       MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER);

	ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,

					    priv->netdev->name);

	if (!ipsec->wq) {

		kfree(ipsec);

		return -ENOMEM;

	}

	netdev_dbg(priv->netdev, "IPSec attached to netdevice\n");

	return 0;

}
@@ -389,6 +433,9 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 
	if (!ipsec)

		return;



	drain_workqueue(ipsec->wq);

	destroy_workqueue(ipsec->wq);



	ida_destroy(&ipsec->halloc);

	kfree(ipsec);

	priv->ipsec = NULL;
@@ -409,11 +456,58 @@ static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 
	return true;

}



struct mlx5e_ipsec_modify_state_work {

	struct work_struct		work;

	struct mlx5_accel_esp_xfrm_attrs attrs;

	struct mlx5e_ipsec_sa_entry	*sa_entry;

};



static void _update_xfrm_state(struct work_struct *work)

{

	int ret;

	struct mlx5e_ipsec_modify_state_work *modify_work =

		container_of(work, struct mlx5e_ipsec_modify_state_work, work);

	struct mlx5e_ipsec_sa_entry *sa_entry = modify_work->sa_entry;



	ret = mlx5_accel_esp_modify_xfrm(sa_entry->xfrm,

					 &modify_work->attrs);

	if (ret)

		netdev_warn(sa_entry->ipsec->en_priv->netdev,

			    "Not an IPSec offload device\n");



	kfree(modify_work);

}



static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)

{

	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);

	struct mlx5e_ipsec_modify_state_work *modify_work;

	bool need_update;



	if (!sa_entry)

		return;



	need_update = mlx5e_ipsec_update_esn_state(sa_entry);

	if (!need_update)

		return;



	modify_work = kzalloc(sizeof(*modify_work), GFP_ATOMIC);

	if (!modify_work)

		return;



	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs);

	modify_work->sa_entry = sa_entry;



	INIT_WORK(&modify_work->work, _update_xfrm_state);

	WARN_ON(!queue_work(sa_entry->ipsec->wq, &modify_work->work));

}



static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {

	.xdo_dev_state_add	= mlx5e_xfrm_add_state,

	.xdo_dev_state_delete	= mlx5e_xfrm_del_state,

	.xdo_dev_state_free	= mlx5e_xfrm_free_state,

	.xdo_dev_offload_ok	= mlx5e_ipsec_offload_ok,

	.xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,

};



void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv)
@@ -424,7 +518,7 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 
	if (!priv->ipsec)

		return;



	if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_ESP) ||

	if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_ESP) ||

	    !MLX5_CAP_ETH(mdev, swp)) {

		mlx5_core_dbg(mdev, "mlx5e: ESP and SWP offload not supported\n");

		return;
@@ -443,7 +537,7 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 
	netdev->features |= NETIF_F_HW_ESP_TX_CSUM;

	netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM;



	if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_LSO) ||

	if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_LSO) ||

	    !MLX5_CAP_ETH(mdev, swp_lso)) {

		mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n");

		return;

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h

+24 −0
Original line number Diff line number Diff line
@@ -40,7 +40,11 @@ 
#include <net/xfrm.h>

#include <linux/idr.h>



#include "accel/ipsec.h"



#define MLX5E_IPSEC_SADB_RX_BITS 10

#define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L



#define MLX5E_METADATA_ETHER_TYPE (0x8CE4)

#define MLX5E_METADATA_ETHER_LEN 8
@@ -77,10 +81,30 @@ struct mlx5e_ipsec_stats { 
struct mlx5e_ipsec {

	struct mlx5e_priv *en_priv;

	DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS);

	bool no_trailer;

	spinlock_t sadb_rx_lock; /* Protects sadb_rx and halloc */

	struct ida halloc;

	struct mlx5e_ipsec_sw_stats sw_stats;

	struct mlx5e_ipsec_stats stats;

	struct workqueue_struct *wq;

};



struct mlx5e_ipsec_esn_state {

	u32 esn;

	u8 trigger: 1;

	u8 overlap: 1;

};



struct mlx5e_ipsec_sa_entry {

	struct hlist_node hlist; /* Item in SADB_RX hashtable */

	struct mlx5e_ipsec_esn_state esn_state;

	unsigned int handle; /* Handle in SADB_RX */

	struct xfrm_state *x;

	struct mlx5e_ipsec *ipsec;

	struct mlx5_accel_esp_xfrm *xfrm;

	void *hw_context;

	void (*set_iv_op)(struct sk_buff *skb, struct xfrm_state *x,

			  struct xfrm_offload *xo);

};



void mlx5e_ipsec_build_inverse_table(void);