Merge "Perf: arm64: Add Snapshot of perf tracepoints"

obj-$(CONFIG_MODULES)			+= module.o

obj-$(CONFIG_ARM64_MODULE_PLTS)		+= module-plts.o

obj-$(CONFIG_PERF_EVENTS)		+= perf_regs.o perf_callchain.o

obj-$(CONFIG_HW_PERF_EVENTS)		+= perf_event.o

obj-$(CONFIG_HW_PERF_EVENTS)		+= perf_event.o perf_trace_counters.o \

					   perf_trace_user.o

obj-$(CONFIG_HAVE_HW_BREAKPOINT)	+= hw_breakpoint.o

obj-$(CONFIG_CPU_PM)			+= sleep.o suspend.o

obj-$(CONFIG_CPU_IDLE)			+= cpuidle.o

// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2013-2014, 2017 The Linux Foundation. All rights reserved.

 */

#include <linux/uaccess.h>

#include <linux/debugfs.h>

#include <linux/cpu.h>

#include <linux/tracepoint.h>

#include <trace/events/sched.h>

#define CREATE_TRACE_POINTS

#include "perf_trace_counters.h"

static unsigned int tp_pid_state;

DEFINE_PER_CPU(u32, cntenset_val);

DEFINE_PER_CPU(u32, previous_ccnt);

DEFINE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts);

DEFINE_PER_CPU(u32, old_pid);

DEFINE_PER_CPU(u32, hotplug_flag);

#define USE_CPUHP_STATE CPUHP_AP_ONLINE

static int tracectr_cpu_hotplug_coming_up(unsigned int cpu)

{

	per_cpu(hotplug_flag, cpu) = 1;

	return 0;

}

static void setup_prev_cnts(u32 cpu, u32 cnten_val)

{

	int i;

	if (cnten_val & CC)

		per_cpu(previous_ccnt, cpu) =

			read_sysreg(pmccntr_el0);

	for (i = 0; i < NUM_L1_CTRS; i++) {

		if (cnten_val & (1 << i)) {

			/* Select */

			write_sysreg(i, pmselr_el0);

			isb();

			/* Read value */

			per_cpu(previous_l1_cnts[i], cpu) =

				read_sysreg(pmxevcntr_el0);

		}

	}

}

void tracectr_notifier(void *ignore, bool preempt,

			struct task_struct *prev, struct task_struct *next)

{

	u32 cnten_val;

	int current_pid;

	u32 cpu = task_cpu(next);

	if (tp_pid_state != 1)

		return;

	current_pid = next->pid;

	if (per_cpu(old_pid, cpu) != -1) {

		cnten_val = read_sysreg(pmcntenset_el0);

		per_cpu(cntenset_val, cpu) = cnten_val;

		/* Disable all the counters that were enabled */

		write_sysreg(cnten_val, pmcntenclr_el0);

		if (per_cpu(hotplug_flag, cpu) == 1) {

			per_cpu(hotplug_flag, cpu) = 0;

			setup_prev_cnts(cpu, cnten_val);

		} else {

			trace_sched_switch_with_ctrs(per_cpu(old_pid, cpu),

						     current_pid);

		}

		/* Enable all the counters that were disabled */

		write_sysreg(cnten_val, pmcntenset_el0);

	}

	per_cpu(old_pid, cpu) = current_pid;

}

static void enable_tp_pid(void)

{

	if (tp_pid_state == 0) {

		tp_pid_state = 1;

		register_trace_sched_switch(tracectr_notifier, NULL);

	}

}

static void disable_tp_pid(void)

{

	if (tp_pid_state == 1) {

		tp_pid_state = 0;

		unregister_trace_sched_switch(tracectr_notifier, NULL);

	}

}

static ssize_t read_enabled_perftp_file_bool(struct file *file,

		char __user *user_buf, size_t count, loff_t *ppos)

{

	char buf[2];

	buf[1] = '\n';

	if (tp_pid_state == 0)

		buf[0] = '0';

	else

		buf[0] = '1';

	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);

}

static ssize_t write_enabled_perftp_file_bool(struct file *file,

		const char __user *user_buf, size_t count, loff_t *ppos)

{

	char buf[32];

	size_t buf_size;

	buf[0] = 0;

	buf_size = min(count, (sizeof(buf)-1));

	if (copy_from_user(buf, user_buf, buf_size))

		return -EFAULT;

	switch (buf[0]) {

	case 'y':

	case 'Y':

	case '1':

		enable_tp_pid();

		break;

	case 'n':

	case 'N':

	case '0':

		disable_tp_pid();

		break;

	}

	return count;

}

static const struct file_operations fops_perftp = {

	.read =		read_enabled_perftp_file_bool,

	.write =	write_enabled_perftp_file_bool,

	.llseek =	default_llseek,

};

int __init init_tracecounters(void)

{

	struct dentry *dir;

	struct dentry *file;

	unsigned int value = 1;

	int cpu, rc;

	dir = debugfs_create_dir("perf_debug_tp", NULL);

	if (!dir)

		return -ENOMEM;

	file = debugfs_create_file("enabled", 0660, dir,

		&value, &fops_perftp);

	if (!file) {

		debugfs_remove(dir);

		return -ENOMEM;

	}

	for_each_possible_cpu(cpu)

		per_cpu(old_pid, cpu) = -1;

	rc = cpuhp_setup_state_nocalls(USE_CPUHP_STATE,

		"tracectr_cpu_hotplug",

		tracectr_cpu_hotplug_coming_up,

		NULL);

	return 0;

}

int __exit exit_tracecounters(void)

{

	cpuhp_remove_state_nocalls(USE_CPUHP_STATE);

	return 0;

}

late_initcall(init_tracecounters);

/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * Copyright (c) 2013-2014, 2017, The Linux Foundation. All rights reserved.

 */

#undef TRACE_SYSTEM

#define TRACE_SYSTEM perf_trace_counters

#if !defined(_PERF_TRACE_COUNTERS_H_) || defined(TRACE_HEADER_MULTI_READ)

#define _PERF_TRACE_COUNTERS_H_

/* Ctr index for PMCNTENSET/CLR */

#define CC 0x80000000

#define C0 0x1

#define C1 0x2

#define C2 0x4

#define C3 0x8

#define C4 0x10

#define C5 0x20

#define C_ALL (CC | C0 | C1 | C2 | C3 | C4 | C5)

#define NUM_L1_CTRS 6

#include <linux/sched.h>

#include <linux/cpumask.h>

#include <linux/tracepoint.h>

DECLARE_PER_CPU(u32, cntenset_val);

DECLARE_PER_CPU(u32, previous_ccnt);

DECLARE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts);

TRACE_EVENT(sched_switch_with_ctrs,

		TP_PROTO(pid_t prev, pid_t next),

		TP_ARGS(prev, next),

		TP_STRUCT__entry(

			__field(pid_t,	old_pid)

			__field(pid_t,	new_pid)

			__field(u32, cctr)

			__field(u32, ctr0)

			__field(u32, ctr1)

			__field(u32, ctr2)

			__field(u32, ctr3)

			__field(u32, ctr4)

			__field(u32, ctr5)

		),

		TP_fast_assign(

			u32 cpu = smp_processor_id();

			u32 i;

			u32 cnten_val;

			u32 total_ccnt = 0;

			u32 total_cnt = 0;

			u32 delta_l1_cnts[NUM_L1_CTRS];

			__entry->old_pid	= prev;

			__entry->new_pid	= next;

			cnten_val = per_cpu(cntenset_val, cpu);

			if (cnten_val & CC) {

				/* Read value */

				total_ccnt = read_sysreg(pmccntr_el0);

				__entry->cctr = total_ccnt -

					per_cpu(previous_ccnt, cpu);

				per_cpu(previous_ccnt, cpu) = total_ccnt;

			}

			for (i = 0; i < NUM_L1_CTRS; i++) {

				if (cnten_val & (1 << i)) {

					/* Select */

					write_sysreg(i, pmselr_el0);

					isb();

					/* Read value */

					total_cnt = read_sysreg(pmxevcntr_el0);

					delta_l1_cnts[i] = total_cnt -

					  per_cpu(previous_l1_cnts[i], cpu);

					per_cpu(previous_l1_cnts[i], cpu) =

						total_cnt;

				} else

					delta_l1_cnts[i] = 0;

			}

			__entry->ctr0 = delta_l1_cnts[0];

			__entry->ctr1 = delta_l1_cnts[1];

			__entry->ctr2 = delta_l1_cnts[2];

			__entry->ctr3 = delta_l1_cnts[3];

			__entry->ctr4 = delta_l1_cnts[4];

			__entry->ctr5 = delta_l1_cnts[5];

		),

		TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %u, CTR0: %u, CTR1: %u, CTR2: %u, CTR3: %u, CTR4: %u, CTR5: %u",

				__entry->old_pid, __entry->new_pid,

				__entry->cctr,

				__entry->ctr0, __entry->ctr1,

				__entry->ctr2, __entry->ctr3,

				__entry->ctr4, __entry->ctr5)

);

#endif

#undef TRACE_INCLUDE_PATH

#define TRACE_INCLUDE_PATH ../../arch/arm64/kernel

#define TRACE_INCLUDE_FILE perf_trace_counters

#include <trace/define_trace.h>

// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2014,2017, The Linux Foundation. All rights reserved.

 */

#include <linux/perf_event.h>

#include <linux/types.h>

#include <linux/tracepoint.h>

#include <linux/fs.h>

#include <linux/debugfs.h>

#include <linux/preempt.h>

#include <linux/stat.h>

#include <asm/uaccess.h>

#define CREATE_TRACE_POINTS

#include "perf_trace_user.h"

#undef TRACE_SYSTEM

#define TRACE_SYSTEM perf_trace_counters

#define TRACE_USER_MAX_BUF_SIZE 100

static ssize_t perf_trace_write(struct file *file,

				const char __user *user_string_in,

				size_t len, loff_t *ppos)

{

	u32 cnten_val;

	int rc;

	char buf[TRACE_USER_MAX_BUF_SIZE + 1];

	ssize_t length;

	if (len == 0)

		return 0;

	length = len > TRACE_USER_MAX_BUF_SIZE ? TRACE_USER_MAX_BUF_SIZE : len;

	rc = copy_from_user(buf, user_string_in, length);

	if (rc) {

		pr_err("%s copy_from_user failed, rc=%d\n", __func__, rc);

		return -EFAULT;

	}

	/* Remove any trailing newline and make sure string is terminated */

	if (buf[length - 1] == '\n')

		buf[length - 1] = '\0';

	else

		buf[length] = '\0';

	/*

	 * Disable preemption to ensure that all the performance counter

	 * accesses happen on the same cpu

	 */

	preempt_disable();

	/* stop counters, call the trace function, restart them */

	cnten_val = read_sysreg(pmcntenset_el0);

	/* Disable all the counters that were enabled */

	write_sysreg(cnten_val, pmcntenclr_el0);

	trace_perf_trace_user(buf, cnten_val);

	/* Enable all the counters that were disabled */

	write_sysreg(cnten_val, pmcntenset_el0);

	preempt_enable();

	return length;

}

static const struct file_operations perf_trace_fops = {

	.write = perf_trace_write

};

static int __init init_perf_trace(void)

{

	struct dentry *dir;

	struct dentry *file;

	unsigned int value = 1;

	dir = debugfs_create_dir("msm_perf", NULL);

	if (!dir)

		return -ENOMEM;

	file = debugfs_create_file("trace_marker", 0220, dir,

		&value, &perf_trace_fops);

	if (!file)

		return -ENOMEM;

	return 0;

}

late_initcall(init_perf_trace);

/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * Copyright (c) 2014, 2017, The Linux Foundation. All rights reserved.

 */

#if !defined(_PERF_TRACE_USER_H_) || defined(TRACE_HEADER_MULTI_READ)

#define _PERF_TRACE_USER_H_

#undef TRACE_SYSTEM

#define TRACE_SYSTEM perf_trace_counters

#include <linux/tracepoint.h>

#define CNTENSET_CC    0x80000000

#define NUM_L1_CTRS             6

TRACE_EVENT(perf_trace_user,

	TP_PROTO(char *string, u32 cnten_val),

	TP_ARGS(string, cnten_val),

	TP_STRUCT__entry(

		__field(u32, cctr)

		__field(u32, ctr0)

		__field(u32, ctr1)

		__field(u32, ctr2)

		__field(u32, ctr3)

		__field(u32, ctr4)

		__field(u32, ctr5)

		__string(user_string, string)

		),

	TP_fast_assign(

		u32 cnt;

		u32 l1_cnts[NUM_L1_CTRS];

		int i;

		if (cnten_val & CNTENSET_CC) {

			/* Read value */

			cnt = read_sysreg(pmccntr_el0);

			__entry->cctr = cnt;

		} else

			__entry->cctr = 0;

		for (i = 0; i < NUM_L1_CTRS; i++) {

			if (cnten_val & (1 << i)) {

				/* Select */

				write_sysreg(i, pmselr_el0);

				isb();

				/* Read value */

				cnt = read_sysreg(pmxevcntr_el0);

				l1_cnts[i] = cnt;

			} else {

				l1_cnts[i] = 0;

			}

		}

		__entry->ctr0 = l1_cnts[0];

		__entry->ctr1 = l1_cnts[1];

		__entry->ctr2 = l1_cnts[2];

		__entry->ctr3 = l1_cnts[3];

		__entry->ctr4 = l1_cnts[4];

		__entry->ctr5 = l1_cnts[5];

		__assign_str(user_string, string);

	),

	TP_printk("CCNTR: %u, CTR0: %u, CTR1: %u, CTR2: %u, CTR3: %u, CTR4: %u, CTR5: %u, MSG=%s",

			__entry->cctr,

			__entry->ctr0, __entry->ctr1,

			__entry->ctr2, __entry->ctr3,

			__entry->ctr4, __entry->ctr5,

			__get_str(user_string)

		)

);

#endif

#undef TRACE_INCLUDE_PATH

#define TRACE_INCLUDE_PATH ../../arch/arm64/kernel

#define TRACE_INCLUDE_FILE perf_trace_user

#include <trace/define_trace.h>