Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bcc8fa83 authored by Russell King's avatar Russell King
Browse files

Merge branch 'for-rmk/perf' of... 

Merge branch 'for-rmk/perf' of git://git.kernel.org/pub/scm/linux/kernel/git/will/linux into devel-stable

There's quite a lot here, most of it from Mark Rutland, who has been
working on big.LITTLE PMU support for a while now. His work also brings
us significantly closer to moving the bulk of the CPU PMU driver out
into drivers/, where it can be shared with arm64.

As part of this work, there is a small patch to perf/core, which has
been Acked-by PeterZ and doesn't conflict with tip/perf/core at present.
I've kept that patch on a separate branch, merged in here, so that the
tip guys can pull it too if any unexpected issues crop up.

Please note that there is a conflict with mainline, since we remove
perf_event_cpu.c. The correct resolution is also to remove the file,
since the changes there are already reflected in the rework (and this
resolution is already included in linux-next).
parents 4c9e0f76 74cf0bc7

arch/arm/include/asm/perf_event.h

+7 −0
Original line number Diff line number Diff line
@@ -19,4 +19,11 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs); 
#define perf_misc_flags(regs)	perf_misc_flags(regs)

#endif



#define perf_arch_fetch_caller_regs(regs, __ip) { \

	(regs)->ARM_pc = (__ip); \

	(regs)->ARM_fp = (unsigned long) __builtin_frame_address(0); \

	(regs)->ARM_sp = current_stack_pointer; \

	(regs)->ARM_cpsr = SVC_MODE; \

}



#endif /* __ARM_PERF_EVENT_H__ */

arch/arm/include/asm/pmu.h

+5 −14
Original line number Diff line number Diff line
@@ -24,22 +24,10 @@ 
 *	interrupt and passed the address of the low level handler,

 *	and can be used to implement any platform specific handling

 *	before or after calling it.

 * @runtime_resume: an optional handler which will be called by the

 *	runtime PM framework following a call to pm_runtime_get().

 *	Note that if pm_runtime_get() is called more than once in

 *	succession this handler will only be called once.

 * @runtime_suspend: an optional handler which will be called by the

 *	runtime PM framework following a call to pm_runtime_put().

 *	Note that if pm_runtime_get() is called more than once in

 *	succession this handler will only be called following the

 *	final call to pm_runtime_put() that actually disables the

 *	hardware.

 */

struct arm_pmu_platdata {

	irqreturn_t (*handle_irq)(int irq, void *dev,

				  irq_handler_t pmu_handler);

	int (*runtime_resume)(struct device *dev);

	int (*runtime_suspend)(struct device *dev);

};



#ifdef CONFIG_HW_PERF_EVENTS
@@ -92,6 +80,7 @@ struct pmu_hw_events { 
struct arm_pmu {

	struct pmu	pmu;

	cpumask_t	active_irqs;

	cpumask_t	supported_cpus;

	int		*irq_affinity;

	char		*name;

	irqreturn_t	(*handle_irq)(int irq_num, void *dev);
@@ -122,8 +111,6 @@ struct arm_pmu { 


#define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))



extern const struct dev_pm_ops armpmu_dev_pm_ops;



int armpmu_register(struct arm_pmu *armpmu, int type);



u64 armpmu_event_update(struct perf_event *event);
@@ -158,6 +145,10 @@ struct pmu_probe_info { 
#define XSCALE_PMU_PROBE(_version, _fn) \

	PMU_PROBE(ARM_CPU_IMP_INTEL << 24 | _version, ARM_PMU_XSCALE_MASK, _fn)



int arm_pmu_device_probe(struct platform_device *pdev,

			 const struct of_device_id *of_table,

			 const struct pmu_probe_info *probe_table);



#endif /* CONFIG_HW_PERF_EVENTS */



#endif /* __ARM_PMU_H__ */

arch/arm/kernel/Makefile

+3 −1
Original line number Diff line number Diff line
@@ -70,7 +70,9 @@ obj-$(CONFIG_CPU_PJ4) += pj4-cp0.o 
obj-$(CONFIG_CPU_PJ4B)		+= pj4-cp0.o

obj-$(CONFIG_IWMMXT)		+= iwmmxt.o

obj-$(CONFIG_PERF_EVENTS)	+= perf_regs.o perf_callchain.o

obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o perf_event_cpu.o

obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o \

				   perf_event_xscale.o perf_event_v6.o \

				   perf_event_v7.o

CFLAGS_pj4-cp0.o		:= -marm

AFLAGS_iwmmxt.o			:= -Wa,-mcpu=iwmmxt

obj-$(CONFIG_ARM_CPU_TOPOLOGY)  += topology.o

arch/arm/kernel/perf_event.c

+375 −33
Original line number Diff line number Diff line
@@ -11,12 +11,18 @@ 
 */

#define pr_fmt(fmt) "hw perfevents: " fmt



#include <linux/bitmap.h>

#include <linux/cpumask.h>

#include <linux/export.h>

#include <linux/kernel.h>

#include <linux/of.h>

#include <linux/platform_device.h>

#include <linux/pm_runtime.h>

#include <linux/slab.h>

#include <linux/spinlock.h>

#include <linux/irq.h>

#include <linux/irqdesc.h>



#include <asm/cputype.h>

#include <asm/irq_regs.h>

#include <asm/pmu.h>
@@ -229,6 +235,10 @@ armpmu_add(struct perf_event *event, int flags) 
	int idx;

	int err = 0;



	/* An event following a process won't be stopped earlier */

	if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))

		return -ENOENT;



	perf_pmu_disable(event->pmu);



	/* If we don't have a space for the counter then finish early. */
@@ -344,20 +354,12 @@ static void 
armpmu_release_hardware(struct arm_pmu *armpmu)

{

	armpmu->free_irq(armpmu);

	pm_runtime_put_sync(&armpmu->plat_device->dev);

}



static int

armpmu_reserve_hardware(struct arm_pmu *armpmu)

{

	int err;

	struct platform_device *pmu_device = armpmu->plat_device;



	if (!pmu_device)

		return -ENODEV;



	pm_runtime_get_sync(&pmu_device->dev);

	err = armpmu->request_irq(armpmu, armpmu_dispatch_irq);

	int err = armpmu->request_irq(armpmu, armpmu_dispatch_irq);

	if (err) {

		armpmu_release_hardware(armpmu);

		return err;
@@ -454,6 +456,17 @@ static int armpmu_event_init(struct perf_event *event) 
	int err = 0;

	atomic_t *active_events = &armpmu->active_events;



	/*

	 * Reject CPU-affine events for CPUs that are of a different class to

	 * that which this PMU handles. Process-following events (where

	 * event->cpu == -1) can be migrated between CPUs, and thus we have to

	 * reject them later (in armpmu_add) if they're scheduled on a

	 * different class of CPU.

	 */

	if (event->cpu != -1 &&

		!cpumask_test_cpu(event->cpu, &armpmu->supported_cpus))

		return -ENOENT;



	/* does not support taken branch sampling */

	if (has_branch_stack(event))

		return -EOPNOTSUPP;
@@ -489,6 +502,10 @@ static void armpmu_enable(struct pmu *pmu) 
	struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events);

	int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);



	/* For task-bound events we may be called on other CPUs */

	if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))

		return;



	if (enabled)

		armpmu->start(armpmu);

}
@@ -496,34 +513,25 @@ static void armpmu_enable(struct pmu *pmu) 
static void armpmu_disable(struct pmu *pmu)

{

	struct arm_pmu *armpmu = to_arm_pmu(pmu);

	armpmu->stop(armpmu);

}



#ifdef CONFIG_PM

static int armpmu_runtime_resume(struct device *dev)

{

	struct arm_pmu_platdata *plat = dev_get_platdata(dev);



	if (plat && plat->runtime_resume)

		return plat->runtime_resume(dev);

	/* For task-bound events we may be called on other CPUs */

	if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus))

		return;



	return 0;

	armpmu->stop(armpmu);

}



static int armpmu_runtime_suspend(struct device *dev)

/*

 * In heterogeneous systems, events are specific to a particular

 * microarchitecture, and aren't suitable for another. Thus, only match CPUs of

 * the same microarchitecture.

 */

static int armpmu_filter_match(struct perf_event *event)

{

	struct arm_pmu_platdata *plat = dev_get_platdata(dev);



	if (plat && plat->runtime_suspend)

		return plat->runtime_suspend(dev);



	return 0;

	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);

	unsigned int cpu = smp_processor_id();

	return cpumask_test_cpu(cpu, &armpmu->supported_cpus);

}

#endif



const struct dev_pm_ops armpmu_dev_pm_ops = {

	SET_RUNTIME_PM_OPS(armpmu_runtime_suspend, armpmu_runtime_resume, NULL)

};



static void armpmu_init(struct arm_pmu *armpmu)

{
@@ -539,15 +547,349 @@ static void armpmu_init(struct arm_pmu *armpmu) 
		.start		= armpmu_start,

		.stop		= armpmu_stop,

		.read		= armpmu_read,

		.filter_match	= armpmu_filter_match,

	};

}



int armpmu_register(struct arm_pmu *armpmu, int type)

{

	armpmu_init(armpmu);

	pm_runtime_enable(&armpmu->plat_device->dev);

	pr_info("enabled with %s PMU driver, %d counters available\n",

			armpmu->name, armpmu->num_events);

	return perf_pmu_register(&armpmu->pmu, armpmu->name, type);

}



/* Set at runtime when we know what CPU type we are. */

static struct arm_pmu *__oprofile_cpu_pmu;



/*

 * Despite the names, these two functions are CPU-specific and are used

 * by the OProfile/perf code.

 */

const char *perf_pmu_name(void)

{

	if (!__oprofile_cpu_pmu)

		return NULL;



	return __oprofile_cpu_pmu->name;

}

EXPORT_SYMBOL_GPL(perf_pmu_name);



int perf_num_counters(void)

{

	int max_events = 0;



	if (__oprofile_cpu_pmu != NULL)

		max_events = __oprofile_cpu_pmu->num_events;



	return max_events;

}

EXPORT_SYMBOL_GPL(perf_num_counters);



static void cpu_pmu_enable_percpu_irq(void *data)

{

	int irq = *(int *)data;



	enable_percpu_irq(irq, IRQ_TYPE_NONE);

}



static void cpu_pmu_disable_percpu_irq(void *data)

{

	int irq = *(int *)data;



	disable_percpu_irq(irq);

}



static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu)

{

	int i, irq, irqs;

	struct platform_device *pmu_device = cpu_pmu->plat_device;

	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;



	irqs = min(pmu_device->num_resources, num_possible_cpus());



	irq = platform_get_irq(pmu_device, 0);

	if (irq >= 0 && irq_is_percpu(irq)) {

		on_each_cpu(cpu_pmu_disable_percpu_irq, &irq, 1);

		free_percpu_irq(irq, &hw_events->percpu_pmu);

	} else {

		for (i = 0; i < irqs; ++i) {

			int cpu = i;



			if (cpu_pmu->irq_affinity)

				cpu = cpu_pmu->irq_affinity[i];



			if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs))

				continue;

			irq = platform_get_irq(pmu_device, i);

			if (irq >= 0)

				free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));

		}

	}

}



static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)

{

	int i, err, irq, irqs;

	struct platform_device *pmu_device = cpu_pmu->plat_device;

	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;



	if (!pmu_device)

		return -ENODEV;



	irqs = min(pmu_device->num_resources, num_possible_cpus());

	if (irqs < 1) {

		pr_warn_once("perf/ARM: No irqs for PMU defined, sampling events not supported\n");

		return 0;

	}



	irq = platform_get_irq(pmu_device, 0);

	if (irq >= 0 && irq_is_percpu(irq)) {

		err = request_percpu_irq(irq, handler, "arm-pmu",

					 &hw_events->percpu_pmu);

		if (err) {

			pr_err("unable to request IRQ%d for ARM PMU counters\n",

				irq);

			return err;

		}

		on_each_cpu(cpu_pmu_enable_percpu_irq, &irq, 1);

	} else {

		for (i = 0; i < irqs; ++i) {

			int cpu = i;



			err = 0;

			irq = platform_get_irq(pmu_device, i);

			if (irq < 0)

				continue;



			if (cpu_pmu->irq_affinity)

				cpu = cpu_pmu->irq_affinity[i];



			/*

			 * If we have a single PMU interrupt that we can't shift,

			 * assume that we're running on a uniprocessor machine and

			 * continue. Otherwise, continue without this interrupt.

			 */

			if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {

				pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n",

					irq, cpu);

				continue;

			}



			err = request_irq(irq, handler,

					  IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu",

					  per_cpu_ptr(&hw_events->percpu_pmu, cpu));

			if (err) {

				pr_err("unable to request IRQ%d for ARM PMU counters\n",

					irq);

				return err;

			}



			cpumask_set_cpu(cpu, &cpu_pmu->active_irqs);

		}

	}



	return 0;

}



/*

 * PMU hardware loses all context when a CPU goes offline.

 * When a CPU is hotplugged back in, since some hardware registers are

 * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading

 * junk values out of them.

 */

static int cpu_pmu_notify(struct notifier_block *b, unsigned long action,

			  void *hcpu)

{

	int cpu = (unsigned long)hcpu;

	struct arm_pmu *pmu = container_of(b, struct arm_pmu, hotplug_nb);



	if ((action & ~CPU_TASKS_FROZEN) != CPU_STARTING)

		return NOTIFY_DONE;



	if (!cpumask_test_cpu(cpu, &pmu->supported_cpus))

		return NOTIFY_DONE;



	if (pmu->reset)

		pmu->reset(pmu);

	else

		return NOTIFY_DONE;



	return NOTIFY_OK;

}



static int cpu_pmu_init(struct arm_pmu *cpu_pmu)

{

	int err;

	int cpu;

	struct pmu_hw_events __percpu *cpu_hw_events;



	cpu_hw_events = alloc_percpu(struct pmu_hw_events);

	if (!cpu_hw_events)

		return -ENOMEM;



	cpu_pmu->hotplug_nb.notifier_call = cpu_pmu_notify;

	err = register_cpu_notifier(&cpu_pmu->hotplug_nb);

	if (err)

		goto out_hw_events;



	for_each_possible_cpu(cpu) {

		struct pmu_hw_events *events = per_cpu_ptr(cpu_hw_events, cpu);

		raw_spin_lock_init(&events->pmu_lock);

		events->percpu_pmu = cpu_pmu;

	}



	cpu_pmu->hw_events	= cpu_hw_events;

	cpu_pmu->request_irq	= cpu_pmu_request_irq;

	cpu_pmu->free_irq	= cpu_pmu_free_irq;



	/* Ensure the PMU has sane values out of reset. */

	if (cpu_pmu->reset)

		on_each_cpu_mask(&cpu_pmu->supported_cpus, cpu_pmu->reset,

			 cpu_pmu, 1);



	/* If no interrupts available, set the corresponding capability flag */

	if (!platform_get_irq(cpu_pmu->plat_device, 0))

		cpu_pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;



	return 0;



out_hw_events:

	free_percpu(cpu_hw_events);

	return err;

}



static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)

{

	unregister_cpu_notifier(&cpu_pmu->hotplug_nb);

	free_percpu(cpu_pmu->hw_events);

}



/*

 * CPU PMU identification and probing.

 */

static int probe_current_pmu(struct arm_pmu *pmu,

			     const struct pmu_probe_info *info)

{

	int cpu = get_cpu();

	unsigned int cpuid = read_cpuid_id();

	int ret = -ENODEV;



	pr_info("probing PMU on CPU %d\n", cpu);



	for (; info->init != NULL; info++) {

		if ((cpuid & info->mask) != info->cpuid)

			continue;

		ret = info->init(pmu);

		break;

	}



	put_cpu();

	return ret;

}



static int of_pmu_irq_cfg(struct arm_pmu *pmu)

{

	int i, irq, *irqs;

	struct platform_device *pdev = pmu->plat_device;



	/* Don't bother with PPIs; they're already affine */

	irq = platform_get_irq(pdev, 0);

	if (irq >= 0 && irq_is_percpu(irq))

		return 0;



	irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);

	if (!irqs)

		return -ENOMEM;



	for (i = 0; i < pdev->num_resources; ++i) {

		struct device_node *dn;

		int cpu;



		dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity",

				      i);

		if (!dn) {

			pr_warn("Failed to parse %s/interrupt-affinity[%d]\n",

				of_node_full_name(pdev->dev.of_node), i);

			break;

		}



		for_each_possible_cpu(cpu)

			if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL))

				break;



		of_node_put(dn);

		if (cpu >= nr_cpu_ids) {

			pr_warn("Failed to find logical CPU for %s\n",

				dn->name);

			break;

		}



		irqs[i] = cpu;

		cpumask_set_cpu(cpu, &pmu->supported_cpus);

	}



	if (i == pdev->num_resources) {

		pmu->irq_affinity = irqs;

	} else {

		kfree(irqs);

		cpumask_setall(&pmu->supported_cpus);

	}



	return 0;

}



int arm_pmu_device_probe(struct platform_device *pdev,

			 const struct of_device_id *of_table,

			 const struct pmu_probe_info *probe_table)

{

	const struct of_device_id *of_id;

	const int (*init_fn)(struct arm_pmu *);

	struct device_node *node = pdev->dev.of_node;

	struct arm_pmu *pmu;

	int ret = -ENODEV;



	pmu = kzalloc(sizeof(struct arm_pmu), GFP_KERNEL);

	if (!pmu) {

		pr_info("failed to allocate PMU device!\n");

		return -ENOMEM;

	}



	if (!__oprofile_cpu_pmu)

		__oprofile_cpu_pmu = pmu;



	pmu->plat_device = pdev;



	if (node && (of_id = of_match_node(of_table, pdev->dev.of_node))) {

		init_fn = of_id->data;



		ret = of_pmu_irq_cfg(pmu);

		if (!ret)

			ret = init_fn(pmu);

	} else {

		ret = probe_current_pmu(pmu, probe_table);

		cpumask_setall(&pmu->supported_cpus);

	}



	if (ret) {

		pr_info("failed to probe PMU!\n");

		goto out_free;

	}



	ret = cpu_pmu_init(pmu);

	if (ret)

		goto out_free;



	ret = armpmu_register(pmu, -1);

	if (ret)

		goto out_destroy;



	return 0;



out_destroy:

	cpu_pmu_destroy(pmu);

out_free:

	pr_info("failed to register PMU devices!\n");

	kfree(pmu);

	return ret;

}

arch/arm/kernel/perf_event_cpu.c

deleted100644 → 0
+0 −415
Original line number Diff line number Diff line 
/*

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

 *

 * Copyright (C) 2012 ARM Limited

 *

 * Author: Will Deacon <will.deacon@arm.com>

 */

#define pr_fmt(fmt) "CPU PMU: " fmt



#include <linux/bitmap.h>

#include <linux/export.h>

#include <linux/kernel.h>

#include <linux/of.h>

#include <linux/platform_device.h>

#include <linux/slab.h>

#include <linux/spinlock.h>

#include <linux/irq.h>

#include <linux/irqdesc.h>



#include <asm/cputype.h>

#include <asm/irq_regs.h>

#include <asm/pmu.h>



/* Set at runtime when we know what CPU type we are. */

static struct arm_pmu *cpu_pmu;



/*

 * Despite the names, these two functions are CPU-specific and are used

 * by the OProfile/perf code.

 */

const char *perf_pmu_name(void)

{

	if (!cpu_pmu)

		return NULL;



	return cpu_pmu->name;

}

EXPORT_SYMBOL_GPL(perf_pmu_name);



int perf_num_counters(void)

{

	int max_events = 0;



	if (cpu_pmu != NULL)

		max_events = cpu_pmu->num_events;



	return max_events;

}

EXPORT_SYMBOL_GPL(perf_num_counters);



/* Include the PMU-specific implementations. */

#include "perf_event_xscale.c"

#include "perf_event_v6.c"

#include "perf_event_v7.c"



static void cpu_pmu_enable_percpu_irq(void *data)

{

	int irq = *(int *)data;



	enable_percpu_irq(irq, IRQ_TYPE_NONE);

}



static void cpu_pmu_disable_percpu_irq(void *data)

{

	int irq = *(int *)data;



	disable_percpu_irq(irq);

}



static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu)

{

	int i, irq, irqs;

	struct platform_device *pmu_device = cpu_pmu->plat_device;

	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;



	irqs = min(pmu_device->num_resources, num_possible_cpus());



	irq = platform_get_irq(pmu_device, 0);

	if (irq >= 0 && irq_is_percpu(irq)) {

		on_each_cpu(cpu_pmu_disable_percpu_irq, &irq, 1);

		free_percpu_irq(irq, &hw_events->percpu_pmu);

	} else {

		for (i = 0; i < irqs; ++i) {

			int cpu = i;



			if (cpu_pmu->irq_affinity)

				cpu = cpu_pmu->irq_affinity[i];



			if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs))

				continue;

			irq = platform_get_irq(pmu_device, i);

			if (irq >= 0)

				free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));

		}

	}

}



static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)

{

	int i, err, irq, irqs;

	struct platform_device *pmu_device = cpu_pmu->plat_device;

	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;



	if (!pmu_device)

		return -ENODEV;



	irqs = min(pmu_device->num_resources, num_possible_cpus());

	if (irqs < 1) {

		pr_warn_once("perf/ARM: No irqs for PMU defined, sampling events not supported\n");

		return 0;

	}



	irq = platform_get_irq(pmu_device, 0);

	if (irq >= 0 && irq_is_percpu(irq)) {

		err = request_percpu_irq(irq, handler, "arm-pmu",

					 &hw_events->percpu_pmu);

		if (err) {

			pr_err("unable to request IRQ%d for ARM PMU counters\n",

				irq);

			return err;

		}

		on_each_cpu(cpu_pmu_enable_percpu_irq, &irq, 1);

	} else {

		for (i = 0; i < irqs; ++i) {

			int cpu = i;



			err = 0;

			irq = platform_get_irq(pmu_device, i);

			if (irq < 0)

				continue;



			if (cpu_pmu->irq_affinity)

				cpu = cpu_pmu->irq_affinity[i];



			/*

			 * If we have a single PMU interrupt that we can't shift,

			 * assume that we're running on a uniprocessor machine and

			 * continue. Otherwise, continue without this interrupt.

			 */

			if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {

				pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n",

					irq, cpu);

				continue;

			}



			err = request_irq(irq, handler,

					  IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu",

					  per_cpu_ptr(&hw_events->percpu_pmu, cpu));

			if (err) {

				pr_err("unable to request IRQ%d for ARM PMU counters\n",

					irq);

				return err;

			}



			cpumask_set_cpu(cpu, &cpu_pmu->active_irqs);

		}

	}



	return 0;

}



/*

 * PMU hardware loses all context when a CPU goes offline.

 * When a CPU is hotplugged back in, since some hardware registers are

 * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading

 * junk values out of them.

 */

static int cpu_pmu_notify(struct notifier_block *b, unsigned long action,

			  void *hcpu)

{

	struct arm_pmu *pmu = container_of(b, struct arm_pmu, hotplug_nb);



	if ((action & ~CPU_TASKS_FROZEN) != CPU_STARTING)

		return NOTIFY_DONE;



	if (pmu->reset)

		pmu->reset(pmu);

	else

		return NOTIFY_DONE;



	return NOTIFY_OK;

}



static int cpu_pmu_init(struct arm_pmu *cpu_pmu)

{

	int err;

	int cpu;

	struct pmu_hw_events __percpu *cpu_hw_events;



	cpu_hw_events = alloc_percpu(struct pmu_hw_events);

	if (!cpu_hw_events)

		return -ENOMEM;



	cpu_pmu->hotplug_nb.notifier_call = cpu_pmu_notify;

	err = register_cpu_notifier(&cpu_pmu->hotplug_nb);

	if (err)

		goto out_hw_events;



	for_each_possible_cpu(cpu) {

		struct pmu_hw_events *events = per_cpu_ptr(cpu_hw_events, cpu);

		raw_spin_lock_init(&events->pmu_lock);

		events->percpu_pmu = cpu_pmu;

	}



	cpu_pmu->hw_events	= cpu_hw_events;

	cpu_pmu->request_irq	= cpu_pmu_request_irq;

	cpu_pmu->free_irq	= cpu_pmu_free_irq;



	/* Ensure the PMU has sane values out of reset. */

	if (cpu_pmu->reset)

		on_each_cpu(cpu_pmu->reset, cpu_pmu, 1);



	/* If no interrupts available, set the corresponding capability flag */

	if (!platform_get_irq(cpu_pmu->plat_device, 0))

		cpu_pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;



	return 0;



out_hw_events:

	free_percpu(cpu_hw_events);

	return err;

}



static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)

{

	unregister_cpu_notifier(&cpu_pmu->hotplug_nb);

	free_percpu(cpu_pmu->hw_events);

}



/*

 * PMU platform driver and devicetree bindings.

 */

static const struct of_device_id cpu_pmu_of_device_ids[] = {

	{.compatible = "arm,cortex-a17-pmu",	.data = armv7_a17_pmu_init},

	{.compatible = "arm,cortex-a15-pmu",	.data = armv7_a15_pmu_init},

	{.compatible = "arm,cortex-a12-pmu",	.data = armv7_a12_pmu_init},

	{.compatible = "arm,cortex-a9-pmu",	.data = armv7_a9_pmu_init},

	{.compatible = "arm,cortex-a8-pmu",	.data = armv7_a8_pmu_init},

	{.compatible = "arm,cortex-a7-pmu",	.data = armv7_a7_pmu_init},

	{.compatible = "arm,cortex-a5-pmu",	.data = armv7_a5_pmu_init},

	{.compatible = "arm,arm11mpcore-pmu",	.data = armv6mpcore_pmu_init},

	{.compatible = "arm,arm1176-pmu",	.data = armv6_1176_pmu_init},

	{.compatible = "arm,arm1136-pmu",	.data = armv6_1136_pmu_init},

	{.compatible = "qcom,krait-pmu",	.data = krait_pmu_init},

	{.compatible = "qcom,scorpion-pmu",	.data = scorpion_pmu_init},

	{.compatible = "qcom,scorpion-mp-pmu",	.data = scorpion_mp_pmu_init},

	{},

};



static struct platform_device_id cpu_pmu_plat_device_ids[] = {

	{.name = "arm-pmu"},

	{.name = "armv6-pmu"},

	{.name = "armv7-pmu"},

	{.name = "xscale-pmu"},

	{},

};



static const struct pmu_probe_info pmu_probe_table[] = {

	ARM_PMU_PROBE(ARM_CPU_PART_ARM1136, armv6_1136_pmu_init),

	ARM_PMU_PROBE(ARM_CPU_PART_ARM1156, armv6_1156_pmu_init),

	ARM_PMU_PROBE(ARM_CPU_PART_ARM1176, armv6_1176_pmu_init),

	ARM_PMU_PROBE(ARM_CPU_PART_ARM11MPCORE, armv6mpcore_pmu_init),

	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),

	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),

	XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V1, xscale1pmu_init),

	XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V2, xscale2pmu_init),

	{ /* sentinel value */ }

};



/*

 * CPU PMU identification and probing.

 */

static int probe_current_pmu(struct arm_pmu *pmu)

{

	int cpu = get_cpu();

	unsigned int cpuid = read_cpuid_id();

	int ret = -ENODEV;

	const struct pmu_probe_info *info;



	pr_info("probing PMU on CPU %d\n", cpu);



	for (info = pmu_probe_table; info->init != NULL; info++) {

		if ((cpuid & info->mask) != info->cpuid)

			continue;

		ret = info->init(pmu);

		break;

	}



	put_cpu();

	return ret;

}



static int of_pmu_irq_cfg(struct platform_device *pdev)

{

	int i;

	int *irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);



	if (!irqs)

		return -ENOMEM;



	for (i = 0; i < pdev->num_resources; ++i) {

		struct device_node *dn;

		int cpu;



		dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity",

				      i);

		if (!dn) {

			pr_warn("Failed to parse %s/interrupt-affinity[%d]\n",

				of_node_full_name(dn), i);

			break;

		}



		for_each_possible_cpu(cpu)

			if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL))

				break;



		of_node_put(dn);

		if (cpu >= nr_cpu_ids) {

			pr_warn("Failed to find logical CPU for %s\n",

				dn->name);

			break;

		}



		irqs[i] = cpu;

	}



	if (i == pdev->num_resources)

		cpu_pmu->irq_affinity = irqs;

	else

		kfree(irqs);



	return 0;

}



static int cpu_pmu_device_probe(struct platform_device *pdev)

{

	const struct of_device_id *of_id;

	const int (*init_fn)(struct arm_pmu *);

	struct device_node *node = pdev->dev.of_node;

	struct arm_pmu *pmu;

	int ret = -ENODEV;



	if (cpu_pmu) {

		pr_info("attempt to register multiple PMU devices!\n");

		return -ENOSPC;

	}



	pmu = kzalloc(sizeof(struct arm_pmu), GFP_KERNEL);

	if (!pmu) {

		pr_info("failed to allocate PMU device!\n");

		return -ENOMEM;

	}



	cpu_pmu = pmu;

	cpu_pmu->plat_device = pdev;



	if (node && (of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node))) {

		init_fn = of_id->data;



		ret = of_pmu_irq_cfg(pdev);

		if (!ret)

			ret = init_fn(pmu);

	} else {

		ret = probe_current_pmu(pmu);

	}



	if (ret) {

		pr_info("failed to probe PMU!\n");

		goto out_free;

	}



	ret = cpu_pmu_init(cpu_pmu);

	if (ret)

		goto out_free;



	ret = armpmu_register(cpu_pmu, -1);

	if (ret)

		goto out_destroy;



	return 0;



out_destroy:

	cpu_pmu_destroy(cpu_pmu);

out_free:

	pr_info("failed to register PMU devices!\n");

	kfree(pmu);

	return ret;

}



static struct platform_driver cpu_pmu_driver = {

	.driver		= {

		.name	= "arm-pmu",

		.pm	= &armpmu_dev_pm_ops,

		.of_match_table = cpu_pmu_of_device_ids,

	},

	.probe		= cpu_pmu_device_probe,

	.id_table	= cpu_pmu_plat_device_ids,

};



static int __init register_pmu_driver(void)

{

	return platform_driver_register(&cpu_pmu_driver);

}

device_initcall(register_pmu_driver);