perf counters: add support for group counters (04289bb9) · Commits · e / devices / android_kernel_fairphone_FP3

arch/x86/kernel/cpu/perf_counter.c

+14 −14

Original line number	Original line	Diff line number	Diff line
	@@ -346,18 +346,22 @@ static void perf_save_and_restart(struct perf_counter *counter)
	}		}

	static void		static void
	perf_handle_group(struct perf_counter leader, u64 status, u64 *overflown)		perf_handle_group(struct perf_counter sibling, u64 status, u64 *overflown)
	{		{
	struct perf_counter_context *ctx = leader->ctx;		struct perf_counter counter, group_leader = sibling->group_leader;
	struct perf_counter *counter;
	int bit;		int bit;

	list_for_each_entry(counter, &ctx->counters, list) {		/*
	if (counter->hw_event.record_type != PERF_RECORD_SIMPLE \|\|		* Store the counter's own timestamp first:
	counter == leader)		*/
	continue;		perf_store_irq_data(sibling, sibling->hw_event.type);
			perf_store_irq_data(sibling, atomic64_counter_read(sibling));

	if (counter->active) {		/*
			* Then store sibling timestamps (if any):
			*/
			list_for_each_entry(counter, &group_leader->sibling_list, list_entry) {
			if (!counter->active) {
	/*		/*
	* When counter was not in the overflow mask, we have to		* When counter was not in the overflow mask, we have to
	* read it from hardware. We read it as well, when it		* read it from hardware. We read it as well, when it
	@@ -371,8 +375,8 @@ perf_handle_group(struct perf_counter leader, u64 status, u64 *overflown)
	perf_save_and_restart(counter);		perf_save_and_restart(counter);
	}		}
	}		}
	perf_store_irq_data(leader, counter->hw_event.type);		perf_store_irq_data(sibling, counter->hw_event.type);
	perf_store_irq_data(leader, atomic64_counter_read(counter));		perf_store_irq_data(sibling, atomic64_counter_read(counter));
	}		}
	}		}

	@@ -416,10 +420,6 @@ static void __smp_perf_counter_interrupt(struct pt_regs *regs, int nmi)
	perf_store_irq_data(counter, instruction_pointer(regs));		perf_store_irq_data(counter, instruction_pointer(regs));
	break;		break;
	case PERF_RECORD_GROUP:		case PERF_RECORD_GROUP:
	perf_store_irq_data(counter,
	counter->hw_event.type);
	perf_store_irq_data(counter,
	atomic64_counter_read(counter));
	perf_handle_group(counter, &status, &ack);		perf_handle_group(counter, &status, &ack);
	break;		break;
	}		}

include/linux/perf_counter.h

+6 −2

Original line number	Original line	Diff line number	Diff line
	@@ -117,7 +117,10 @@ struct perf_data {
	* struct perf_counter - performance counter kernel representation:		* struct perf_counter - performance counter kernel representation:
	*/		*/
	struct perf_counter {		struct perf_counter {
	struct list_head list;		struct list_head list_entry;
			struct list_head sibling_list;
			struct perf_counter *group_leader;

	int active;		int active;
	#if BITS_PER_LONG == 64		#if BITS_PER_LONG == 64
	atomic64_t count;		atomic64_t count;
	@@ -158,7 +161,8 @@ struct perf_counter_context {
	* Protect the list of counters:		* Protect the list of counters:
	*/		*/
	spinlock_t lock;		spinlock_t lock;
	struct list_head counters;
			struct list_head counter_list;
	int nr_counters;		int nr_counters;
	int nr_active;		int nr_active;
	struct task_struct *task;		struct task_struct *task;

kernel/perf_counter.c

+216 −66

Original line number	Original line	Diff line number	Diff line
	@@ -10,6 +10,7 @@
	#include <linux/fs.h>		#include <linux/fs.h>
	#include <linux/cpu.h>		#include <linux/cpu.h>
	#include <linux/smp.h>		#include <linux/smp.h>
			#include <linux/file.h>
	#include <linux/poll.h>		#include <linux/poll.h>
	#include <linux/sysfs.h>		#include <linux/sysfs.h>
	#include <linux/ptrace.h>		#include <linux/ptrace.h>
	@@ -55,7 +56,7 @@ void __weak hw_perf_counter_setup(void) { }
	* Read the cached counter in counter safe against cross CPU / NMI		* Read the cached counter in counter safe against cross CPU / NMI
	* modifications. 64 bit version - no complications.		* modifications. 64 bit version - no complications.
	*/		*/
	static inline u64 perf_read_counter_safe(struct perf_counter *counter)		static inline u64 perf_counter_read_safe(struct perf_counter *counter)
	{		{
	return (u64) atomic64_read(&counter->count);		return (u64) atomic64_read(&counter->count);
	}		}
	@@ -66,7 +67,7 @@ static inline u64 perf_read_counter_safe(struct perf_counter *counter)
	* Read the cached counter in counter safe against cross CPU / NMI		* Read the cached counter in counter safe against cross CPU / NMI
	* modifications. 32 bit version.		* modifications. 32 bit version.
	*/		*/
	static u64 perf_read_counter_safe(struct perf_counter *counter)		static u64 perf_counter_read_safe(struct perf_counter *counter)
	{		{
	u32 cntl, cnth;		u32 cntl, cnth;

	@@ -83,13 +84,55 @@ static u64 perf_read_counter_safe(struct perf_counter *counter)

	#endif		#endif

			static void
			list_add_counter(struct perf_counter counter, struct perf_counter_context ctx)
			{
			struct perf_counter *group_leader = counter->group_leader;

			/*
			* Depending on whether it is a standalone or sibling counter,
			* add it straight to the context's counter list, or to the group
			* leader's sibling list:
			*/
			if (counter->group_leader == counter)
			list_add_tail(&counter->list_entry, &ctx->counter_list);
			else
			list_add_tail(&counter->list_entry, &group_leader->sibling_list);
			}

			static void
			list_del_counter(struct perf_counter counter, struct perf_counter_context ctx)
			{
			struct perf_counter sibling, tmp;

			list_del_init(&counter->list_entry);

			if (list_empty(&counter->sibling_list))
			return;

			/*
			* If this was a group counter with sibling counters then
			* upgrade the siblings to singleton counters by adding them
			* to the context list directly:
			*/
			list_for_each_entry_safe(sibling, tmp,
			&counter->sibling_list, list_entry) {

			list_del_init(&sibling->list_entry);
			list_add_tail(&sibling->list_entry, &ctx->counter_list);
			WARN_ON_ONCE(!sibling->group_leader);
			WARN_ON_ONCE(sibling->group_leader == sibling);
			sibling->group_leader = sibling;
			}
			}

	/*		/*
	* Cross CPU call to remove a performance counter		* Cross CPU call to remove a performance counter
	*		*
	* We disable the counter on the hardware level first. After that we		* We disable the counter on the hardware level first. After that we
	* remove it from the context list.		* remove it from the context list.
	*/		*/
	static void __perf_remove_from_context(void *info)		static void __perf_counter_remove_from_context(void *info)
	{		{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);		struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_counter *counter = info;		struct perf_counter *counter = info;
	@@ -119,7 +162,7 @@ static void __perf_remove_from_context(void *info)
	* counters on a global level. NOP for non NMI based counters.		* counters on a global level. NOP for non NMI based counters.
	*/		*/
	hw_perf_disable_all();		hw_perf_disable_all();
	list_del_init(&counter->list);		list_del_counter(counter, ctx);
	hw_perf_enable_all();		hw_perf_enable_all();

	if (!ctx->task) {		if (!ctx->task) {
	@@ -144,7 +187,7 @@ static void __perf_remove_from_context(void *info)
	* CPU counters are removed with a smp call. For task counters we only		* CPU counters are removed with a smp call. For task counters we only
	* call when the task is on a CPU.		* call when the task is on a CPU.
	*/		*/
	static void perf_remove_from_context(struct perf_counter *counter)		static void perf_counter_remove_from_context(struct perf_counter *counter)
	{		{
	struct perf_counter_context *ctx = counter->ctx;		struct perf_counter_context *ctx = counter->ctx;
	struct task_struct *task = ctx->task;		struct task_struct *task = ctx->task;
	@@ -155,32 +198,32 @@ static void perf_remove_from_context(struct perf_counter *counter)
	* the removal is always sucessful.		* the removal is always sucessful.
	*/		*/
	smp_call_function_single(counter->cpu,		smp_call_function_single(counter->cpu,
	__perf_remove_from_context,		__perf_counter_remove_from_context,
	counter, 1);		counter, 1);
	return;		return;
	}		}

	retry:		retry:
	task_oncpu_function_call(task, __perf_remove_from_context,		task_oncpu_function_call(task, __perf_counter_remove_from_context,
	counter);		counter);

	spin_lock_irq(&ctx->lock);		spin_lock_irq(&ctx->lock);
	/*		/*
	* If the context is active we need to retry the smp call.		* If the context is active we need to retry the smp call.
	*/		*/
	if (ctx->nr_active && !list_empty(&counter->list)) {		if (ctx->nr_active && !list_empty(&counter->list_entry)) {
	spin_unlock_irq(&ctx->lock);		spin_unlock_irq(&ctx->lock);
	goto retry;		goto retry;
	}		}

	/*		/*
	* The lock prevents that this context is scheduled in so we		* The lock prevents that this context is scheduled in so we
	* can remove the counter safely, if it the call above did not		* can remove the counter safely, if the call above did not
	* succeed.		* succeed.
	*/		*/
	if (!list_empty(&counter->list)) {		if (!list_empty(&counter->list_entry)) {
	ctx->nr_counters--;		ctx->nr_counters--;
	list_del_init(&counter->list);		list_del_counter(counter, ctx);
	counter->task = NULL;		counter->task = NULL;
	}		}
	spin_unlock_irq(&ctx->lock);		spin_unlock_irq(&ctx->lock);
	@@ -211,7 +254,7 @@ static void __perf_install_in_context(void *info)
	* counters on a global level. NOP for non NMI based counters.		* counters on a global level. NOP for non NMI based counters.
	*/		*/
	hw_perf_disable_all();		hw_perf_disable_all();
	list_add_tail(&counter->list, &ctx->counters);		list_add_counter(counter, ctx);
	hw_perf_enable_all();		hw_perf_enable_all();

	ctx->nr_counters++;		ctx->nr_counters++;
	@@ -268,7 +311,7 @@ perf_install_in_context(struct perf_counter_context *ctx,
	* If the context is active and the counter has not been added		* If the context is active and the counter has not been added
	* we need to retry the smp call.		* we need to retry the smp call.
	*/		*/
	if (ctx->nr_active && list_empty(&counter->list)) {		if (ctx->nr_active && list_empty(&counter->list_entry)) {
	spin_unlock_irq(&ctx->lock);		spin_unlock_irq(&ctx->lock);
	goto retry;		goto retry;
	}		}
	@@ -278,13 +321,45 @@ perf_install_in_context(struct perf_counter_context *ctx,
	* can add the counter safely, if it the call above did not		* can add the counter safely, if it the call above did not
	* succeed.		* succeed.
	*/		*/
	if (list_empty(&counter->list)) {		if (list_empty(&counter->list_entry)) {
	list_add_tail(&counter->list, &ctx->counters);		list_add_counter(counter, ctx);
	ctx->nr_counters++;		ctx->nr_counters++;
	}		}
	spin_unlock_irq(&ctx->lock);		spin_unlock_irq(&ctx->lock);
	}		}

			static void
			counter_sched_out(struct perf_counter *counter,
			struct perf_cpu_context *cpuctx,
			struct perf_counter_context *ctx)
			{
			if (!counter->active)
			return;

			hw_perf_counter_disable(counter);
			counter->active = 0;
			counter->oncpu = -1;

			cpuctx->active_oncpu--;
			ctx->nr_active--;
			}

			static void
			group_sched_out(struct perf_counter *group_counter,
			struct perf_cpu_context *cpuctx,
			struct perf_counter_context *ctx)
			{
			struct perf_counter *counter;

			counter_sched_out(group_counter, cpuctx, ctx);

			/*
			* Schedule out siblings (if any):
			*/
			list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
			counter_sched_out(counter, cpuctx, ctx);
			}

	/*		/*
	* Called from scheduler to remove the counters of the current task,		* Called from scheduler to remove the counters of the current task,
	* with interrupts disabled.		* with interrupts disabled.
	@@ -306,21 +381,48 @@ void perf_counter_task_sched_out(struct task_struct *task, int cpu)
	return;		return;

	spin_lock(&ctx->lock);		spin_lock(&ctx->lock);
	list_for_each_entry(counter, &ctx->counters, list) {		if (ctx->nr_active) {
	if (!ctx->nr_active)		list_for_each_entry(counter, &ctx->counter_list, list_entry)
	break;		group_sched_out(counter, cpuctx, ctx);
	if (counter->active) {
	hw_perf_counter_disable(counter);
	counter->active = 0;
	counter->oncpu = -1;
	ctx->nr_active--;
	cpuctx->active_oncpu--;
	}
	}		}
	spin_unlock(&ctx->lock);		spin_unlock(&ctx->lock);
	cpuctx->task_ctx = NULL;		cpuctx->task_ctx = NULL;
	}		}

			static void
			counter_sched_in(struct perf_counter *counter,
			struct perf_cpu_context *cpuctx,
			struct perf_counter_context *ctx,
			int cpu)
			{
			if (!counter->active)
			return;

			hw_perf_counter_enable(counter);
			counter->active = 1;
			counter->oncpu = cpu; /* TODO: put 'cpu' into cpuctx->cpu */

			cpuctx->active_oncpu++;
			ctx->nr_active++;
			}

			static void
			group_sched_in(struct perf_counter *group_counter,
			struct perf_cpu_context *cpuctx,
			struct perf_counter_context *ctx,
			int cpu)
			{
			struct perf_counter *counter;

			counter_sched_in(group_counter, cpuctx, ctx, cpu);

			/*
			* Schedule in siblings as one group (if any):
			*/
			list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
			counter_sched_in(counter, cpuctx, ctx, cpu);
			}

	/*		/*
	* Called from scheduler to add the counters of the current task		* Called from scheduler to add the counters of the current task
	* with interrupts disabled.		* with interrupts disabled.
	@@ -342,19 +444,21 @@ void perf_counter_task_sched_in(struct task_struct *task, int cpu)
	return;		return;

	spin_lock(&ctx->lock);		spin_lock(&ctx->lock);
	list_for_each_entry(counter, &ctx->counters, list) {		list_for_each_entry(counter, &ctx->counter_list, list_entry) {
	if (ctx->nr_active == cpuctx->max_pertask)		if (ctx->nr_active == cpuctx->max_pertask)
	break;		break;

			/*
			* Listen to the 'cpu' scheduling filter constraint
			* of counters:
			*/
	if (counter->cpu != -1 && counter->cpu != cpu)		if (counter->cpu != -1 && counter->cpu != cpu)
	continue;		continue;

	hw_perf_counter_enable(counter);		group_sched_in(counter, cpuctx, ctx, cpu);
	counter->active = 1;
	counter->oncpu = cpu;
	ctx->nr_active++;
	cpuctx->active_oncpu++;
	}		}
	spin_unlock(&ctx->lock);		spin_unlock(&ctx->lock);

	cpuctx->task_ctx = ctx;		cpuctx->task_ctx = ctx;
	}		}

	@@ -371,12 +475,12 @@ void perf_counter_task_tick(struct task_struct *curr, int cpu)
	spin_lock(&ctx->lock);		spin_lock(&ctx->lock);

	/*		/*
	* Rotate the first entry last:		* Rotate the first entry last (works just fine for group counters too):
	*/		*/
	hw_perf_disable_all();		hw_perf_disable_all();
	list_for_each_entry(counter, &ctx->counters, list) {		list_for_each_entry(counter, &ctx->counter_list, list_entry) {
	list_del(&counter->list);		list_del(&counter->list_entry);
	list_add_tail(&counter->list, &ctx->counters);		list_add_tail(&counter->list_entry, &ctx->counter_list);
	break;		break;
	}		}
	hw_perf_enable_all();		hw_perf_enable_all();
	@@ -387,17 +491,24 @@ void perf_counter_task_tick(struct task_struct *curr, int cpu)
	}		}

	/*		/*
	* Initialize the perf_counter context in task_struct		* Initialize the perf_counter context in a task_struct:
	*/		*/
	void perf_counter_init_task(struct task_struct *task)		static void
			__perf_counter_init_context(struct perf_counter_context *ctx,
			struct task_struct *task)
	{		{
	struct perf_counter_context *ctx = &task->perf_counter_ctx;

	spin_lock_init(&ctx->lock);		spin_lock_init(&ctx->lock);
	INIT_LIST_HEAD(&ctx->counters);		INIT_LIST_HEAD(&ctx->counter_list);
	ctx->nr_counters = 0;		ctx->nr_counters = 0;
	ctx->task = task;		ctx->task = task;
	}		}
			/*
			* Initialize the perf_counter context in task_struct
			*/
			void perf_counter_init_task(struct task_struct *task)
			{
			__perf_counter_init_context(&task->perf_counter_ctx, task);
			}

	/*		/*
	* Cross CPU call to read the hardware counter		* Cross CPU call to read the hardware counter
	@@ -407,7 +518,7 @@ static void __hw_perf_counter_read(void *info)
	hw_perf_counter_read(info);		hw_perf_counter_read(info);
	}		}

	static u64 perf_read_counter(struct perf_counter *counter)		static u64 perf_counter_read(struct perf_counter *counter)
	{		{
	/*		/*
	* If counter is enabled and currently active on a CPU, update the		* If counter is enabled and currently active on a CPU, update the
	@@ -418,7 +529,7 @@ static u64 perf_read_counter(struct perf_counter *counter)
	__hw_perf_counter_read, counter, 1);		__hw_perf_counter_read, counter, 1);
	}		}

	return perf_read_counter_safe(counter);		return perf_counter_read_safe(counter);
	}		}

	/*		/*
	@@ -555,7 +666,7 @@ static int perf_release(struct inode inode, struct file file)

	mutex_lock(&counter->mutex);		mutex_lock(&counter->mutex);

	perf_remove_from_context(counter);		perf_counter_remove_from_context(counter);
	put_context(ctx);		put_context(ctx);

	mutex_unlock(&counter->mutex);		mutex_unlock(&counter->mutex);
	@@ -577,7 +688,7 @@ perf_read_hw(struct perf_counter counter, char __user buf, size_t count)
	return -EINVAL;		return -EINVAL;

	mutex_lock(&counter->mutex);		mutex_lock(&counter->mutex);
	cntval = perf_read_counter(counter);		cntval = perf_counter_read(counter);
	mutex_unlock(&counter->mutex);		mutex_unlock(&counter->mutex);

	return put_user(cntval, (u64 __user *) buf) ? -EFAULT : sizeof(cntval);		return put_user(cntval, (u64 __user *) buf) ? -EFAULT : sizeof(cntval);
	@@ -707,15 +818,25 @@ static const struct file_operations perf_fops = {
	* Allocate and initialize a counter structure		* Allocate and initialize a counter structure
	*/		*/
	static struct perf_counter *		static struct perf_counter *
	perf_counter_alloc(struct perf_counter_hw_event *hw_event, int cpu)		perf_counter_alloc(struct perf_counter_hw_event *hw_event,
			int cpu,
			struct perf_counter *group_leader)
	{		{
	struct perf_counter counter = kzalloc(sizeof(counter), GFP_KERNEL);		struct perf_counter counter = kzalloc(sizeof(counter), GFP_KERNEL);

	if (!counter)		if (!counter)
	return NULL;		return NULL;

			/*
			* Single counters are their own group leaders, with an
			* empty sibling list:
			*/
			if (!group_leader)
			group_leader = counter;

	mutex_init(&counter->mutex);		mutex_init(&counter->mutex);
	INIT_LIST_HEAD(&counter->list);		INIT_LIST_HEAD(&counter->list_entry);
			INIT_LIST_HEAD(&counter->sibling_list);
	init_waitqueue_head(&counter->waitq);		init_waitqueue_head(&counter->waitq);

	counter->irqdata = &counter->data[0];		counter->irqdata = &counter->data[0];
	@@ -723,6 +844,7 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event, int cpu)
	counter->cpu = cpu;		counter->cpu = cpu;
	counter->hw_event = *hw_event;		counter->hw_event = *hw_event;
	counter->wakeup_pending = 0;		counter->wakeup_pending = 0;
			counter->group_leader = group_leader;

	return counter;		return counter;
	}		}
	@@ -743,20 +865,45 @@ asmlinkage int sys_perf_counter_open(
	int group_fd)		int group_fd)

	{		{
	struct perf_counter_context *ctx;		struct perf_counter counter, group_leader;
	struct perf_counter_hw_event hw_event;		struct perf_counter_hw_event hw_event;
	struct perf_counter *counter;		struct perf_counter_context *ctx;
			struct file *group_file = NULL;
			int fput_needed = 0;
	int ret;		int ret;

	if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)		if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
	return -EFAULT;		return -EFAULT;

			/*
			* Look up the group leader:
			*/
			group_leader = NULL;
			if (group_fd != -1) {
			ret = -EINVAL;
			group_file = fget_light(group_fd, &fput_needed);
			if (!group_file)
			goto out_fput;
			if (group_file->f_op != &perf_fops)
			goto out_fput;

			group_leader = group_file->private_data;
			/*
			* Do not allow a recursive hierarchy:
			*/
			if (group_leader->group_leader)
			goto out_fput;
			}

			/*
			* Get the target context (task or percpu):
			*/
	ctx = find_get_context(pid, cpu);		ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))		if (IS_ERR(ctx))
	return PTR_ERR(ctx);		return PTR_ERR(ctx);

	ret = -ENOMEM;		ret = -ENOMEM;
	counter = perf_counter_alloc(&hw_event, cpu);		counter = perf_counter_alloc(&hw_event, cpu, group_leader);
	if (!counter)		if (!counter)
	goto err_put_context;		goto err_put_context;

	@@ -770,11 +917,14 @@ asmlinkage int sys_perf_counter_open(
	if (ret < 0)		if (ret < 0)
	goto err_remove_free_put_context;		goto err_remove_free_put_context;

			out_fput:
			fput_light(group_file, fput_needed);

	return ret;		return ret;

	err_remove_free_put_context:		err_remove_free_put_context:
	mutex_lock(&counter->mutex);		mutex_lock(&counter->mutex);
	perf_remove_from_context(counter);		perf_counter_remove_from_context(counter);
	mutex_unlock(&counter->mutex);		mutex_unlock(&counter->mutex);

	err_free_put_context:		err_free_put_context:
	@@ -783,40 +933,40 @@ asmlinkage int sys_perf_counter_open(
	err_put_context:		err_put_context:
	put_context(ctx);		put_context(ctx);

	return ret;		goto out_fput;
	}		}

	static void __cpuinit perf_init_cpu(int cpu)		static void __cpuinit perf_counter_init_cpu(int cpu)
	{		{
	struct perf_cpu_context *ctx;		struct perf_cpu_context *cpuctx;

	ctx = &per_cpu(perf_cpu_context, cpu);		cpuctx = &per_cpu(perf_cpu_context, cpu);
	spin_lock_init(&ctx->ctx.lock);		__perf_counter_init_context(&cpuctx->ctx, NULL);
	INIT_LIST_HEAD(&ctx->ctx.counters);

	mutex_lock(&perf_resource_mutex);		mutex_lock(&perf_resource_mutex);
	ctx->max_pertask = perf_max_counters - perf_reserved_percpu;		cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
	mutex_unlock(&perf_resource_mutex);		mutex_unlock(&perf_resource_mutex);

	hw_perf_counter_setup();		hw_perf_counter_setup();
	}		}

	#ifdef CONFIG_HOTPLUG_CPU		#ifdef CONFIG_HOTPLUG_CPU
	static void __perf_exit_cpu(void *info)		static void __perf_counter_exit_cpu(void *info)
	{		{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);		struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_counter_context *ctx = &cpuctx->ctx;		struct perf_counter_context *ctx = &cpuctx->ctx;
	struct perf_counter counter, tmp;		struct perf_counter counter, tmp;

	list_for_each_entry_safe(counter, tmp, &ctx->counters, list)		list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
	__perf_remove_from_context(counter);		__perf_counter_remove_from_context(counter);

	}		}
	static void perf_exit_cpu(int cpu)		static void perf_counter_exit_cpu(int cpu)
	{		{
	smp_call_function_single(cpu, __perf_exit_cpu, NULL, 1);		smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
	}		}
	#else		#else
	static inline void perf_exit_cpu(int cpu) { }		static inline void perf_counter_exit_cpu(int cpu) { }
	#endif		#endif

	static int __cpuinit		static int __cpuinit
	@@ -828,12 +978,12 @@ perf_cpu_notify(struct notifier_block self, unsigned long action, void hcpu)

	case CPU_UP_PREPARE:		case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:		case CPU_UP_PREPARE_FROZEN:
	perf_init_cpu(cpu);		perf_counter_init_cpu(cpu);
	break;		break;

	case CPU_DOWN_PREPARE:		case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:		case CPU_DOWN_PREPARE_FROZEN:
	perf_exit_cpu(cpu);		perf_counter_exit_cpu(cpu);
	break;		break;

	default:		default: