Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c7af343b authored by Eran Ben Elisha's avatar Eran Ben Elisha Committed by David S. Miller
Browse files

devlink: Add health report functionality 



Upon error discover, every driver can report it to the devlink health
mechanism via devlink_health_report function, using the appropriate
reporter registered to it. Driver can pass error specific context which
will be delivered to it as part of the dump / recovery callbacks.

Once an error is reported, devlink health will do the following actions:
* A log is being send to the kernel trace events buffer
* Health status and statistics are being updated for the reporter instance
* Object dump is being taken and stored at the reporter instance (as long
  as there is no other dump which is already stored)
* Auto recovery attempt is being done. depends on:
  - Auto Recovery configuration
  - Grace period vs. time since last recover

Signed-off-by: default avatarEran Ben Elisha <eranbe@mellanox.com>
Reviewed-by: default avatarMoshe Shemesh <moshe@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 880ee82f

include/net/devlink.h

+9 −0
Original line number Diff line number Diff line
@@ -641,6 +641,8 @@ devlink_health_reporter_destroy(struct devlink_health_reporter *reporter); 


void *

devlink_health_reporter_priv(struct devlink_health_reporter *reporter);

int devlink_health_report(struct devlink_health_reporter *reporter,

			  const char *msg, void *priv_ctx);

#else



static inline struct devlink *devlink_alloc(const struct devlink_ops *ops,
@@ -979,6 +981,13 @@ devlink_health_reporter_priv(struct devlink_health_reporter *reporter) 
{

	return NULL;

}



static inline int

devlink_health_report(struct devlink_health_reporter *reporter,

		      const char *msg, void *priv_ctx)

{

	return 0;

}

#endif



#endif /* _NET_DEVLINK_H_ */

include/trace/events/devlink.h

+62 −0
Original line number Diff line number Diff line
@@ -46,6 +46,65 @@ TRACE_EVENT(devlink_hwmsg, 
		  (int) __entry->len, __get_dynamic_array(buf), __entry->len)

);



TRACE_EVENT(devlink_health_report,

	TP_PROTO(const struct devlink *devlink, const char *reporter_name,

		 const char *msg),



	TP_ARGS(devlink, reporter_name, msg),



	TP_STRUCT__entry(

		__string(bus_name, devlink->dev->bus->name)

		__string(dev_name, dev_name(devlink->dev))

		__string(driver_name, devlink->dev->driver->name)

		__string(reporter_name, msg)

		__string(msg, msg)

	),



	TP_fast_assign(

		__assign_str(bus_name, devlink->dev->bus->name);

		__assign_str(dev_name, dev_name(devlink->dev));

		__assign_str(driver_name, devlink->dev->driver->name);

		__assign_str(reporter_name, reporter_name);

		__assign_str(msg, msg);

	),



	TP_printk("bus_name=%s dev_name=%s driver_name=%s reporter_name=%s: %s",

		  __get_str(bus_name), __get_str(dev_name),

		  __get_str(driver_name), __get_str(reporter_name),

		  __get_str(msg))

);



TRACE_EVENT(devlink_health_recover_aborted,

	TP_PROTO(const struct devlink *devlink, const char *reporter_name,

		 bool health_state, u64 time_since_last_recover),



	TP_ARGS(devlink, reporter_name, health_state, time_since_last_recover),



	TP_STRUCT__entry(

		__string(bus_name, devlink->dev->bus->name)

		__string(dev_name, dev_name(devlink->dev))

		__string(driver_name, devlink->dev->driver->name)

		__string(reporter_name, reporter_name)

		__field(bool, health_state)

		__field(u64, time_since_last_recover)

	),



	TP_fast_assign(

		__assign_str(bus_name, devlink->dev->bus->name);

		__assign_str(dev_name, dev_name(devlink->dev));

		__assign_str(driver_name, devlink->dev->driver->name);

		__assign_str(reporter_name, reporter_name);

		__entry->health_state = health_state;

		__entry->time_since_last_recover = time_since_last_recover;

	),



	TP_printk("bus_name=%s dev_name=%s driver_name=%s reporter_name=%s: health_state=%d time_since_last_recover = %llu recover aborted",

		  __get_str(bus_name), __get_str(dev_name),

		  __get_str(driver_name), __get_str(reporter_name),

		  __entry->health_state,

		  __entry->time_since_last_recover)

);



#endif /* _TRACE_DEVLINK_H */



/* This part must be outside protection */
@@ -64,6 +123,9 @@ static inline void trace_devlink_hwmsg(const struct devlink *devlink, 
{

}



static inline void trace_devlink_health(const char *msg)

{

}

#endif /* _TRACE_DEVLINK_H */



#endif

net/core/devlink.c

+93 −0
Original line number Diff line number Diff line
@@ -4110,6 +4110,16 @@ struct devlink_health_reporter { 
	u64 graceful_period;

	bool auto_recover;

	u8 health_state;

	u8 dump_avail;

	u64 dump_ts;

	u64 error_count;

	u64 recovery_count;

	u64 last_recovery_ts;

};



enum devlink_health_reporter_state {

	DEVLINK_HEALTH_REPORTER_STATE_HEALTHY,

	DEVLINK_HEALTH_REPORTER_STATE_ERROR,

};



void *
@@ -4224,6 +4234,89 @@ devlink_health_reporter_destroy(struct devlink_health_reporter *reporter) 
}

EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);



static int

devlink_health_reporter_recover(struct devlink_health_reporter *reporter,

				void *priv_ctx)

{

	int err;



	if (!reporter->ops->recover)

		return -EOPNOTSUPP;



	err = reporter->ops->recover(reporter, priv_ctx);

	if (err)

		return err;



	reporter->recovery_count++;

	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;

	reporter->last_recovery_ts = jiffies;



	return 0;

}



static int devlink_health_do_dump(struct devlink_health_reporter *reporter,

				  void *priv_ctx)

{

	int err;



	if (!reporter->ops->dump)

		return 0;



	if (reporter->dump_avail)

		return 0;



	devlink_health_buffers_reset(reporter->dump_buffers_array,

				     DEVLINK_HEALTH_SIZE_TO_BUFFERS(reporter->ops->dump_size));

	err = reporter->ops->dump(reporter, reporter->dump_buffers_array,

				     DEVLINK_HEALTH_BUFFER_SIZE,

				     DEVLINK_HEALTH_SIZE_TO_BUFFERS(reporter->ops->dump_size),

				     priv_ctx);

	if (!err) {

		reporter->dump_avail = true;

		reporter->dump_ts = jiffies;

	}



	return err;

}



int devlink_health_report(struct devlink_health_reporter *reporter,

			  const char *msg, void *priv_ctx)

{

	struct devlink *devlink = reporter->devlink;

	int err = 0;



	/* write a log message of the current error */

	WARN_ON(!msg);

	trace_devlink_health_report(devlink, reporter->ops->name, msg);

	reporter->error_count++;



	/* abort if the previous error wasn't recovered */

	if (reporter->auto_recover &&

	    (reporter->health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||

	     jiffies - reporter->last_recovery_ts <

	     msecs_to_jiffies(reporter->graceful_period))) {

		trace_devlink_health_recover_aborted(devlink,

						     reporter->ops->name,

						     reporter->health_state,

						     jiffies -

						     reporter->last_recovery_ts);

		return -ECANCELED;

	}



	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;



	mutex_lock(&reporter->dump_lock);

	/* store current dump of current error, for later analysis */

	devlink_health_do_dump(reporter, priv_ctx);

	mutex_unlock(&reporter->dump_lock);



	if (reporter->auto_recover)

		err = devlink_health_reporter_recover(reporter, priv_ctx);



	return err;

}

EXPORT_SYMBOL_GPL(devlink_health_report);



static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {

	[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },

	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },