Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 70dd6fdb authored by Feras Daoud's avatar Feras Daoud Committed by Saeed Mahameed
Browse files

net/mlx5: FW tracer, parse traces and kernel tracing support



For each message the driver should do the following:
1- Find the message string in the strings database
2- Count the param number of each message
3- Wait for the param events and accumulate them
4- Calculate the event timestamp using the local event timestamp
and the first timestamp event following it.
5- Print message to trace log

Enable the tracing by:
echo 1 > /sys/kernel/debug/tracing/events/mlx5/mlx5_fw/enable

Read traces by:
cat /sys/kernel/debug/tracing/trace

Signed-off-by: default avatarFeras Daoud <ferasda@mellanox.com>
Signed-off-by: default avatarErez Shitrit <erezsh@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent c71ad41c
Loading
Loading
Loading
Loading
+233 −2
Original line number Diff line number Diff line
@@ -29,8 +29,9 @@
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#define CREATE_TRACE_POINTS
#include "fw_tracer.h"
#include "fw_tracer_tracepoint.h"

static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer)
{
@@ -332,6 +333,109 @@ static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev)
		mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err);
}

static const char *VAL_PARM		= "%llx";
static const char *REPLACE_64_VAL_PARM	= "%x%x";
static const char *PARAM_CHAR		= "%";

static int mlx5_tracer_message_hash(u32 message_id)
{
	return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1);
}

static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer,
							       struct tracer_event *tracer_event)
{
	struct hlist_head *head =
		&tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
	struct tracer_string_format *cur_string;

	cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL);
	if (!cur_string)
		return NULL;

	hlist_add_head(&cur_string->hlist, head);

	return cur_string;
}

static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer,
							   struct tracer_event *tracer_event)
{
	struct tracer_string_format *cur_string;
	u32 str_ptr, offset;
	int i;

	str_ptr = tracer_event->string_event.string_param;

	for (i = 0; i < tracer->str_db.num_string_db; i++) {
		if (str_ptr > tracer->str_db.base_address_out[i] &&
		    str_ptr < tracer->str_db.base_address_out[i] +
		    tracer->str_db.size_out[i]) {
			offset = str_ptr - tracer->str_db.base_address_out[i];
			/* add it to the hash */
			cur_string = mlx5_tracer_message_insert(tracer, tracer_event);
			if (!cur_string)
				return NULL;
			cur_string->string = (char *)(tracer->str_db.buffer[i] +
							offset);
			return cur_string;
		}
	}

	return NULL;
}

static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt)
{
	hlist_del(&str_frmt->hlist);
	kfree(str_frmt);
}

static int mlx5_tracer_get_num_of_params(char *str)
{
	char *substr, *pstr = str;
	int num_of_params = 0;

	/* replace %llx with %x%x */
	substr = strstr(pstr, VAL_PARM);
	while (substr) {
		memcpy(substr, REPLACE_64_VAL_PARM, 4);
		pstr = substr;
		substr = strstr(pstr, VAL_PARM);
	}

	/* count all the % characters */
	substr = strstr(str, PARAM_CHAR);
	while (substr) {
		num_of_params += 1;
		str = substr + 1;
		substr = strstr(str, PARAM_CHAR);
	}

	return num_of_params;
}

static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head,
							     u8 event_id, u32 tmsn)
{
	struct tracer_string_format *message;

	hlist_for_each_entry(message, head, hlist)
		if (message->event_id == event_id && message->tmsn == tmsn)
			return message;

	return NULL;
}

static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer,
							    struct tracer_event *tracer_event)
{
	struct hlist_head *head =
		&tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];

	return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn);
}

static void poll_trace(struct mlx5_fw_tracer *tracer,
		       struct tracer_event *tracer_event, u64 *trace)
{
@@ -396,6 +500,128 @@ static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event)
	return tracer_event.timestamp_event.timestamp;
}

static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer)
{
	struct tracer_string_format *str_frmt;
	struct hlist_node *n;
	int i;

	for (i = 0; i < MESSAGE_HASH_SIZE; i++) {
		hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist)
			mlx5_tracer_clean_message(str_frmt);
	}
}

static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer)
{
	struct tracer_string_format *str_frmt, *tmp_str;

	list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list,
				 list)
		list_del(&str_frmt->list);
}

static void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt,
				    struct mlx5_core_dev *dev,
				    u64 trace_timestamp)
{
	char	tmp[512];

	snprintf(tmp, sizeof(tmp), str_frmt->string,
		 str_frmt->params[0],
		 str_frmt->params[1],
		 str_frmt->params[2],
		 str_frmt->params[3],
		 str_frmt->params[4],
		 str_frmt->params[5],
		 str_frmt->params[6]);

	trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost,
		      str_frmt->event_id, tmp);

	/* remove it from hash */
	mlx5_tracer_clean_message(str_frmt);
}

static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
					   struct tracer_event *tracer_event)
{
	struct tracer_string_format *cur_string;

	if (tracer_event->string_event.tdsn == 0) {
		cur_string = mlx5_tracer_get_string(tracer, tracer_event);
		if (!cur_string)
			return -1;

		cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
		cur_string->last_param_num = 0;
		cur_string->event_id = tracer_event->event_id;
		cur_string->tmsn = tracer_event->string_event.tmsn;
		cur_string->timestamp = tracer_event->string_event.timestamp;
		cur_string->lost = tracer_event->lost_event;
		if (cur_string->num_of_params == 0) /* trace with no params */
			list_add_tail(&cur_string->list, &tracer->ready_strings_list);
	} else {
		cur_string = mlx5_tracer_message_get(tracer, tracer_event);
		if (!cur_string) {
			pr_debug("%s Got string event for unknown string tdsm: %d\n",
				 __func__, tracer_event->string_event.tmsn);
			return -1;
		}
		cur_string->last_param_num += 1;
		if (cur_string->last_param_num > TRACER_MAX_PARAMS) {
			pr_debug("%s Number of params exceeds the max (%d)\n",
				 __func__, TRACER_MAX_PARAMS);
			list_add_tail(&cur_string->list, &tracer->ready_strings_list);
			return 0;
		}
		/* keep the new parameter */
		cur_string->params[cur_string->last_param_num - 1] =
			tracer_event->string_event.string_param;
		if (cur_string->last_param_num == cur_string->num_of_params)
			list_add_tail(&cur_string->list, &tracer->ready_strings_list);
	}

	return 0;
}

static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer,
					       struct tracer_event *tracer_event)
{
	struct tracer_timestamp_event timestamp_event =
						tracer_event->timestamp_event;
	struct tracer_string_format *str_frmt, *tmp_str;
	struct mlx5_core_dev *dev = tracer->dev;
	u64 trace_timestamp;

	list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) {
		list_del(&str_frmt->list);
		if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0))
			trace_timestamp = (timestamp_event.timestamp & MASK_52_7) |
					  (str_frmt->timestamp & MASK_6_0);
		else
			trace_timestamp = ((timestamp_event.timestamp & MASK_52_7) - 1) |
					  (str_frmt->timestamp & MASK_6_0);

		mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp);
	}
}

static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer,
				    struct tracer_event *tracer_event)
{
	if (tracer_event->type == TRACER_EVENT_TYPE_STRING) {
		mlx5_tracer_handle_string_trace(tracer, tracer_event);
	} else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) {
		if (!tracer_event->timestamp_event.unreliable)
			mlx5_tracer_handle_timestamp_trace(tracer, tracer_event);
	} else {
		pr_debug("%s Got unrecognised type %d for parsing, exiting..\n",
			 __func__, tracer_event->type);
	}
	return 0;
}

static void mlx5_fw_tracer_handle_traces(struct work_struct *work)
{
	struct mlx5_fw_tracer *tracer =
@@ -452,8 +678,10 @@ static void mlx5_fw_tracer_handle_traces(struct work_struct *work)
		}

		/* Parse events */
		for (i = 0; i < TRACES_PER_BLOCK ; i++)
		for (i = 0; i < TRACES_PER_BLOCK ; i++) {
			poll_trace(tracer, &tracer_event, &tmp_trace_block[i]);
			mlx5_tracer_handle_trace(tracer, &tracer_event);
		}

		tracer->buff.consumer_index =
			(tracer->buff.consumer_index + 1) & (block_count - 1);
@@ -578,6 +806,7 @@ struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev)

	tracer->dev = dev;

	INIT_LIST_HEAD(&tracer->ready_strings_list);
	INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change);
	INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db);
	INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces);
@@ -675,6 +904,8 @@ void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer)
		return;

	cancel_work_sync(&tracer->read_fw_strings_work);
	mlx5_fw_tracer_clean_ready_list(tracer);
	mlx5_fw_tracer_clean_print_hash(tracer);
	mlx5_fw_tracer_free_strings_db(tracer);
	mlx5_fw_tracer_destroy_log_buf(tracer);
	flush_workqueue(tracer->work_queue);
+22 −0
Original line number Diff line number Diff line
@@ -46,6 +46,13 @@
#define TRACER_BLOCK_SIZE_BYTE 256
#define TRACES_PER_BLOCK 32

#define TRACER_MAX_PARAMS 7
#define MESSAGE_HASH_BITS 6
#define MESSAGE_HASH_SIZE BIT(MESSAGE_HASH_BITS)

#define MASK_52_7 (0x1FFFFFFFFFFF80)
#define MASK_6_0  (0x7F)

struct mlx5_fw_tracer {
	struct mlx5_core_dev *dev;
	bool owner;
@@ -77,6 +84,21 @@ struct mlx5_fw_tracer {

	u64 last_timestamp;
	struct work_struct handle_traces_work;
	struct hlist_head hash[MESSAGE_HASH_SIZE];
	struct list_head ready_strings_list;
};

struct tracer_string_format {
	char *string;
	int params[TRACER_MAX_PARAMS];
	int num_of_params;
	int last_param_num;
	u8 event_id;
	u32 tmsn;
	struct hlist_node hlist;
	struct list_head list;
	u32 timestamp;
	bool lost;
};

enum mlx5_fw_tracer_ownership_state {
+78 −0
Original line number Diff line number Diff line
/*
 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#if !defined(__LIB_TRACER_TRACEPOINT_H__) || defined(TRACE_HEADER_MULTI_READ)
#define __LIB_TRACER_TRACEPOINT_H__

#include <linux/tracepoint.h>
#include "fw_tracer.h"

#undef TRACE_SYSTEM
#define TRACE_SYSTEM mlx5

/* Tracepoint for FWTracer messages: */
TRACE_EVENT(mlx5_fw,
	TP_PROTO(const struct mlx5_fw_tracer *tracer, u64 trace_timestamp,
		 bool lost, u8 event_id, const char *msg),

	TP_ARGS(tracer, trace_timestamp, lost, event_id, msg),

	TP_STRUCT__entry(
		__string(dev_name, dev_name(&tracer->dev->pdev->dev))
		__field(u64, trace_timestamp)
		__field(bool, lost)
		__field(u8, event_id)
		__string(msg, msg)
	),

	TP_fast_assign(
		__assign_str(dev_name, dev_name(&tracer->dev->pdev->dev));
		__entry->trace_timestamp = trace_timestamp;
		__entry->lost = lost;
		__entry->event_id = event_id;
		__assign_str(msg, msg);
	),

	TP_printk("%s [0x%llx] %d [0x%x] %s",
		  __get_str(dev_name),
		  __entry->trace_timestamp,
		  __entry->lost, __entry->event_id,
		  __get_str(msg))
);

#endif

#undef TRACE_INCLUDE_PATH
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_PATH ./diag
#define TRACE_INCLUDE_FILE fw_tracer_tracepoint
#include <trace/define_trace.h>