Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 20c72bd5 authored by Andy Grover's avatar Andy Grover
Browse files

RDS: Implement masked atomic operations 



Add two CMSGs for masked versions of cswp and fadd. args
struct modified to use a union for different atomic op type's
arguments. Change IB to do masked atomic ops. Atomic op type
in rds_message similarly unionized.

Signed-off-by: default avatarAndy Grover <andy.grover@oracle.com>
parent 59f740a6

include/linux/rds.h

+12 −0
Original line number Diff line number Diff line
@@ -75,6 +75,8 @@ 
#define RDS_CMSG_CONG_UPDATE		5

#define RDS_CMSG_ATOMIC_FADD		6

#define RDS_CMSG_ATOMIC_CSWP		7

#define RDS_CMSG_MASKED_ATOMIC_FADD	8

#define RDS_CMSG_MASKED_ATOMIC_CSWP	9



#define RDS_INFO_FIRST			10000

#define RDS_INFO_COUNTERS		10000
@@ -251,6 +253,16 @@ struct rds_atomic_args { 
		struct {

			uint64_t	add;

		} fadd;

		struct {

			uint64_t	compare;

			uint64_t	swap;

			uint64_t	compare_mask;

			uint64_t	swap_mask;

		} m_cswp;

		struct {

			uint64_t	add;

			uint64_t	nocarry_mask;

		} m_fadd;

	};

	uint64_t	flags;

	uint64_t	user_token;

net/rds/ib_send.c

+9 −5
Original line number Diff line number Diff line
@@ -807,13 +807,17 @@ int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op) 
	send->s_queued = jiffies;



	if (op->op_type == RDS_ATOMIC_TYPE_CSWP) {

		send->s_wr.opcode = IB_WR_ATOMIC_CMP_AND_SWP;

		send->s_wr.wr.atomic.compare_add = op->op_compare;

		send->s_wr.wr.atomic.swap = op->op_swap_add;

		send->s_wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;

		send->s_wr.wr.atomic.compare_add = op->op_m_cswp.compare;

		send->s_wr.wr.atomic.swap = op->op_m_cswp.swap;

		send->s_wr.wr.atomic.compare_add_mask = op->op_m_cswp.compare_mask;

		send->s_wr.wr.atomic.swap_mask = op->op_m_cswp.swap_mask;

	} else { /* FADD */

		send->s_wr.opcode = IB_WR_ATOMIC_FETCH_AND_ADD;

		send->s_wr.wr.atomic.compare_add = op->op_swap_add;

		send->s_wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;

		send->s_wr.wr.atomic.compare_add = op->op_m_fadd.add;

		send->s_wr.wr.atomic.swap = 0;

		send->s_wr.wr.atomic.compare_add_mask = op->op_m_fadd.nocarry_mask;

		send->s_wr.wr.atomic.swap_mask = 0;

	}

	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);

	send->s_wr.num_sge = 1;

net/rds/rdma.c

+27 −6
Original line number Diff line number Diff line
@@ -738,13 +738,34 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm, 


	args = CMSG_DATA(cmsg);



	if (cmsg->cmsg_type == RDS_CMSG_ATOMIC_CSWP) {

		rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;

		rm->atomic.op_swap_add = args->cswp.swap;

		rm->atomic.op_compare = args->cswp.compare;

	} else {

	/* Nonmasked & masked cmsg ops converted to masked hw ops */

	switch (cmsg->cmsg_type) {

	case RDS_CMSG_ATOMIC_FADD:

		rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;

		rm->atomic.op_m_fadd.add = args->fadd.add;

		rm->atomic.op_m_fadd.nocarry_mask = 0;

		break;

	case RDS_CMSG_MASKED_ATOMIC_FADD:

		rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD;

		rm->atomic.op_swap_add = args->fadd.add;

		rm->atomic.op_m_fadd.add = args->m_fadd.add;

		rm->atomic.op_m_fadd.nocarry_mask = args->m_fadd.nocarry_mask;

		break;

	case RDS_CMSG_ATOMIC_CSWP:

		rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;

		rm->atomic.op_m_cswp.compare = args->cswp.compare;

		rm->atomic.op_m_cswp.swap = args->cswp.swap;

		rm->atomic.op_m_cswp.compare_mask = ~0;

		rm->atomic.op_m_cswp.swap_mask = ~0;

		break;

	case RDS_CMSG_MASKED_ATOMIC_CSWP:

		rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP;

		rm->atomic.op_m_cswp.compare = args->m_cswp.compare;

		rm->atomic.op_m_cswp.swap = args->m_cswp.swap;

		rm->atomic.op_m_cswp.compare_mask = args->m_cswp.compare_mask;

		rm->atomic.op_m_cswp.swap_mask = args->m_cswp.swap_mask;

		break;

	default:

		BUG(); /* should never happen */

	}



	rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);

net/rds/rds.h

+12 −2
Original line number Diff line number Diff line
@@ -316,8 +316,18 @@ struct rds_message { 
	struct {

		struct rm_atomic_op {

			int			op_type;

			uint64_t		op_swap_add;

			uint64_t		op_compare;

			union {

				struct {

					uint64_t	compare;

					uint64_t	swap;

					uint64_t	compare_mask;

					uint64_t	swap_mask;

				} op_m_cswp;

				struct {

					uint64_t	add;

					uint64_t	nocarry_mask;

				} op_m_fadd;

			};



			u32			op_rkey;

			u64			op_remote_addr;

net/rds/send.c

+4 −0
Original line number Diff line number Diff line
@@ -843,6 +843,8 @@ static int rds_rm_size(struct msghdr *msg, int data_len) 


		case RDS_CMSG_ATOMIC_CSWP:

		case RDS_CMSG_ATOMIC_FADD:

		case RDS_CMSG_MASKED_ATOMIC_CSWP:

		case RDS_CMSG_MASKED_ATOMIC_FADD:

			cmsg_groups |= 1;

			size += sizeof(struct scatterlist);

			break;
@@ -894,6 +896,8 @@ static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, 
			break;

		case RDS_CMSG_ATOMIC_CSWP:

		case RDS_CMSG_ATOMIC_FADD:

		case RDS_CMSG_MASKED_ATOMIC_CSWP:

		case RDS_CMSG_MASKED_ATOMIC_FADD:

			ret = rds_cmsg_atomic(rs, rm, cmsg);

			break;