Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bf3c5a93 authored by Leon Romanovsky's avatar Leon Romanovsky Committed by Jason Gunthorpe
Browse files

RDMA/nldev: Provide global resource utilization 



Expose through the netlink interface the global per-device utilization of
the supported object types.

Provide both dumpit and doit callbacks.

As an example of possible output from rdmatool for system with 5
mlx5 cards:

$ rdma res
1: mlx5_0: qp 4 cq 5 pd 3
2: mlx5_1: qp 4 cq 5 pd 3
3: mlx5_2: qp 4 cq 5 pd 3
4: mlx5_3: qp 2 cq 3 pd 2
5: mlx5_4: qp 4 cq 5 pd 3

Reviewed-by: default avatarMark Bloch <markb@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Reviewed-by: default avatarSteve Wise <swise@opengridcomputing.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent 9d5f8c20

drivers/infiniband/core/nldev.c

+147 −0
Original line number Diff line number Diff line
@@ -31,6 +31,8 @@ 
 */



#include <linux/module.h>

#include <linux/pid.h>

#include <linux/pid_namespace.h>

#include <net/netlink.h>

#include <rdma/rdma_netlink.h>
@@ -52,6 +54,11 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { 
	[RDMA_NLDEV_ATTR_PORT_STATE]	= { .type = NLA_U8 },

	[RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },

	[RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },

	[RDMA_NLDEV_ATTR_RES_SUMMARY]	= { .type = NLA_NESTED },

	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]	= { .type = NLA_NESTED },

	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME] = { .type = NLA_NUL_STRING,

					     .len = 16 },

	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR] = { .type = NLA_U64 },

};



static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
@@ -134,6 +141,65 @@ static int fill_port_info(struct sk_buff *msg, 
	return 0;

}



static int fill_res_info_entry(struct sk_buff *msg,

			       const char *name, u64 curr)

{

	struct nlattr *entry_attr;



	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY);

	if (!entry_attr)

		return -EMSGSIZE;



	if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))

		goto err;

	if (nla_put_u64_64bit(msg,

			      RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, 0))

		goto err;



	nla_nest_end(msg, entry_attr);

	return 0;



err:

	nla_nest_cancel(msg, entry_attr);

	return -EMSGSIZE;

}



static int fill_res_info(struct sk_buff *msg, struct ib_device *device)

{

	static const char * const names[RDMA_RESTRACK_MAX] = {

		[RDMA_RESTRACK_PD] = "pd",

		[RDMA_RESTRACK_CQ] = "cq",

		[RDMA_RESTRACK_QP] = "qp",

	};



	struct rdma_restrack_root *res = &device->res;

	struct nlattr *table_attr;

	int ret, i, curr;



	if (fill_nldev_handle(msg, device))

		return -EMSGSIZE;



	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY);

	if (!table_attr)

		return -EMSGSIZE;



	for (i = 0; i < RDMA_RESTRACK_MAX; i++) {

		if (!names[i])

			continue;

		curr = rdma_restrack_count(res, i, task_active_pid_ns(current));

		ret = fill_res_info_entry(msg, names[i], curr);

		if (ret)

			goto err;

	}



	nla_nest_end(msg, table_attr);

	return 0;



err:

	nla_nest_cancel(msg, table_attr);

	return ret;

}



static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,

			  struct netlink_ext_ack *extack)

{
@@ -329,6 +395,83 @@ static int nldev_port_get_dumpit(struct sk_buff *skb, 
	return skb->len;

}



static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,

			      struct netlink_ext_ack *extack)

{

	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];

	struct ib_device *device;

	struct sk_buff *msg;

	u32 index;

	int ret;



	ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,

			  nldev_policy, extack);

	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])

		return -EINVAL;



	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);

	device = ib_device_get_by_index(index);

	if (!device)

		return -EINVAL;



	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);

	if (!msg)

		goto err;



	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,

			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),

			0, 0);



	ret = fill_res_info(msg, device);

	if (ret)

		goto err_free;



	nlmsg_end(msg, nlh);

	put_device(&device->dev);

	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);



err_free:

	nlmsg_free(msg);

err:

	put_device(&device->dev);

	return ret;

}



static int _nldev_res_get_dumpit(struct ib_device *device,

				 struct sk_buff *skb,

				 struct netlink_callback *cb,

				 unsigned int idx)

{

	int start = cb->args[0];

	struct nlmsghdr *nlh;



	if (idx < start)

		return 0;



	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,

			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),

			0, NLM_F_MULTI);



	if (fill_res_info(skb, device)) {

		nlmsg_cancel(skb, nlh);

		goto out;

	}



	nlmsg_end(skb, nlh);



	idx++;



out:

	cb->args[0] = idx;

	return skb->len;

}



static int nldev_res_get_dumpit(struct sk_buff *skb,

				struct netlink_callback *cb)

{

	return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb);

}



static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {

	[RDMA_NLDEV_CMD_GET] = {

		.doit = nldev_get_doit,
@@ -338,6 +481,10 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = { 
		.doit = nldev_port_get_doit,

		.dump = nldev_port_get_dumpit,

	},

	[RDMA_NLDEV_CMD_RES_GET] = {

		.doit = nldev_res_get_doit,

		.dump = nldev_res_get_dumpit,

	},

};



void __init nldev_init(void)

include/uapi/rdma/rdma_netlink.h

+7 −0
Original line number Diff line number Diff line
@@ -236,6 +236,8 @@ enum rdma_nldev_command { 
	RDMA_NLDEV_CMD_PORT_NEW,

	RDMA_NLDEV_CMD_PORT_DEL,



	RDMA_NLDEV_CMD_RES_GET, /* can dump */



	RDMA_NLDEV_NUM_OPS

};
@@ -303,6 +305,11 @@ enum rdma_nldev_attr { 


	RDMA_NLDEV_ATTR_DEV_NODE_TYPE,		/* u8 */



	RDMA_NLDEV_ATTR_RES_SUMMARY,		/* nested table */

	RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY,	/* nested table */

	RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME,	/* string */

	RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR,	/* u64 */



	RDMA_NLDEV_ATTR_MAX

};

#endif /* _UAPI_RDMA_NETLINK_H */