perf: Cure task_oncpu_function_call() races

#define TASK_SIZE_OF(tsk)	TASK_SIZE

#endif

/*

 * Call the function if the target task is executing on a CPU right now:

 */

extern void task_oncpu_function_call(struct task_struct *p,

				     void (*func) (void *info), void *info);

#ifdef CONFIG_MM_OWNER

extern void mm_update_next_owner(struct mm_struct *mm);

extern void mm_init_owner(struct mm_struct *mm, struct task_struct *p);

#include <asm/irq_regs.h>

struct remote_function_call {

	struct task_struct *p;

	int (*func)(void *info);

	void *info;

	int ret;

};

static void remote_function(void *data)

{

	struct remote_function_call *tfc = data;

	struct task_struct *p = tfc->p;

	if (p) {

		tfc->ret = -EAGAIN;

		if (task_cpu(p) != smp_processor_id() || !task_curr(p))

			return;

	}

	tfc->ret = tfc->func(tfc->info);

}

/**

 * task_function_call - call a function on the cpu on which a task runs

 * @p:		the task to evaluate

 * @func:	the function to be called

 * @info:	the function call argument

 *

 * Calls the function @func when the task is currently running. This might

 * be on the current CPU, which just calls the function directly

 *

 * returns: @func return value, or

 *	    -ESRCH  - when the process isn't running

 *	    -EAGAIN - when the process moved away

 */

static int

task_function_call(struct task_struct *p, int (*func) (void *info), void *info)

{

	struct remote_function_call data = {

		.p = p,

		.func = func,

		.info = info,

		.ret = -ESRCH, /* No such (running) process */

	};

	if (task_curr(p))

		smp_call_function_single(task_cpu(p), remote_function, &data, 1);

	return data.ret;

}

/**

 * cpu_function_call - call a function on the cpu

 * @func:	the function to be called

 * @info:	the function call argument

 *

 * Calls the function @func on the remote cpu.

 *

 * returns: @func return value or -ENXIO when the cpu is offline

 */

static int cpu_function_call(int cpu, int (*func) (void *info), void *info)

{

	struct remote_function_call data = {

		.p = NULL,

		.func = func,

		.info = info,

		.ret = -ENXIO, /* No such CPU */

	};

	smp_call_function_single(cpu, remote_function, &data, 1);

	return data.ret;

}

enum event_type_t {

	EVENT_FLEXIBLE = 0x1,

	EVENT_PINNED = 0x2,

	raw_spin_lock_irqsave(&ctx->lock, flags);

	--ctx->pin_count;

	raw_spin_unlock_irqrestore(&ctx->lock, flags);

	put_ctx(ctx);

}

/*

 * We disable the event on the hardware level first. After that we

 * remove it from the context list.

 */

static void __perf_event_remove_from_context(void *info)

static int __perf_remove_from_context(void *info)

{

	struct perf_event *event = info;

	struct perf_event_context *ctx = event->ctx;

	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

	/*

	 * If this is a task context, we need to check whether it is

	 * the current task context of this cpu. If not it has been

	 * scheduled out before the smp call arrived.

	 */

	if (ctx->task && cpuctx->task_ctx != ctx)

		return;

	raw_spin_lock(&ctx->lock);

	event_sched_out(event, cpuctx, ctx);

	list_del_event(event, ctx);

	raw_spin_unlock(&ctx->lock);

	return 0;

}

/*

 * Remove the event from a task's (or a CPU's) list of events.

 *

 * Must be called with ctx->mutex held.

 *

 * CPU events are removed with a smp call. For task events we only

 * call when the task is on a CPU.

 *

 * When called from perf_event_exit_task, it's OK because the

 * context has been detached from its task.

 */

static void perf_event_remove_from_context(struct perf_event *event)

static void perf_remove_from_context(struct perf_event *event)

{

	struct perf_event_context *ctx = event->ctx;

	struct task_struct *task = ctx->task;

	lockdep_assert_held(&ctx->mutex);

	if (!task) {

		/*

		 * Per cpu events are removed via an smp call and

		 * the removal is always successful.

		 */

		smp_call_function_single(event->cpu,

					 __perf_event_remove_from_context,

					 event, 1);

		cpu_function_call(event->cpu, __perf_remove_from_context, event);

		return;

	}

retry:

	task_oncpu_function_call(task, __perf_event_remove_from_context,

				 event);

	if (!task_function_call(task, __perf_remove_from_context, event))

		return;

	raw_spin_lock_irq(&ctx->lock);

	/*

	 * If the context is active we need to retry the smp call.

	 * If we failed to find a running task, but find the context active now

	 * that we've acquired the ctx->lock, retry.

	 */

	if (ctx->nr_active && !list_empty(&event->group_entry)) {

	if (ctx->is_active) {

		raw_spin_unlock_irq(&ctx->lock);

		goto retry;

	}

	/*

	 * The lock prevents that this context is scheduled in so we

	 * can remove the event safely, if the call above did not

	 * succeed.

	 * Since the task isn't running, its safe to remove the event, us

	 * holding the ctx->lock ensures the task won't get scheduled in.

	 */

	if (!list_empty(&event->group_entry))

	list_del_event(event, ctx);

	raw_spin_unlock_irq(&ctx->lock);

}

/*

 * Cross CPU call to disable a performance event

 */

static void __perf_event_disable(void *info)

static int __perf_event_disable(void *info)

{

	struct perf_event *event = info;

	struct perf_event_context *ctx = event->ctx;

	/*

	 * If this is a per-task event, need to check whether this

	 * event's task is the current task on this cpu.

	 *

	 * Can trigger due to concurrent perf_event_context_sched_out()

	 * flipping contexts around.

	 */

	if (ctx->task && cpuctx->task_ctx != ctx)

		return;

		return -EINVAL;

	raw_spin_lock(&ctx->lock);

	}

	raw_spin_unlock(&ctx->lock);

	return 0;

}

/*

		/*

		 * Disable the event on the cpu that it's on

		 */

		smp_call_function_single(event->cpu, __perf_event_disable,

					 event, 1);

		cpu_function_call(event->cpu, __perf_event_disable, event);

		return;

	}

retry:

	task_oncpu_function_call(task, __perf_event_disable, event);

	if (!task_function_call(task, __perf_event_disable, event))

		return;

	raw_spin_lock_irq(&ctx->lock);

	/*

	 */

	if (event->state == PERF_EVENT_STATE_ACTIVE) {

		raw_spin_unlock_irq(&ctx->lock);

		/*

		 * Reload the task pointer, it might have been changed by

		 * a concurrent perf_event_context_sched_out().

		 */

		task = ctx->task;

		goto retry;

	}

		update_group_times(event);

		event->state = PERF_EVENT_STATE_OFF;

	}

	raw_spin_unlock_irq(&ctx->lock);

}

	event->tstamp_stopped = tstamp;

}

static void perf_event_context_sched_in(struct perf_event_context *ctx);

/*

 * Cross CPU call to install and enable a performance event

 *

 * Must be called with ctx->mutex held

 */

static void __perf_install_in_context(void *info)

static int  __perf_install_in_context(void *info)

{

	struct perf_event *event = info;

	struct perf_event_context *ctx = event->ctx;

	int err;

	/*

	 * If this is a task context, we need to check whether it is

	 * the current task context of this cpu. If not it has been

	 * scheduled out before the smp call arrived.

	 * Or possibly this is the right context but it isn't

	 * on this cpu because it had no events.

	 * In case we're installing a new context to an already running task,

	 * could also happen before perf_event_task_sched_in() on architectures

	 * which do context switches with IRQs enabled.

	 */

	if (ctx->task && cpuctx->task_ctx != ctx) {

		if (cpuctx->task_ctx || ctx->task != current)

			return;

		cpuctx->task_ctx = ctx;

	}

	if (ctx->task && !cpuctx->task_ctx)

		perf_event_context_sched_in(ctx);

	raw_spin_lock(&ctx->lock);

	ctx->is_active = 1;

unlock:

	raw_spin_unlock(&ctx->lock);

	return 0;

}

/*

 * If the event is attached to a task which is on a CPU we use a smp

 * call to enable it in the task context. The task might have been

 * scheduled away, but we check this in the smp call again.

 *

 * Must be called with ctx->mutex held.

 */

static void

perf_install_in_context(struct perf_event_context *ctx,

{

	struct task_struct *task = ctx->task;

	lockdep_assert_held(&ctx->mutex);

	event->ctx = ctx;

	if (!task) {

		 * Per cpu events are installed via an smp call and

		 * the install is always successful.

		 */

		smp_call_function_single(cpu, __perf_install_in_context,

					 event, 1);

		cpu_function_call(cpu, __perf_install_in_context, event);

		return;

	}

retry:

	task_oncpu_function_call(task, __perf_install_in_context,

				 event);

	if (!task_function_call(task, __perf_install_in_context, event))

		return;

	raw_spin_lock_irq(&ctx->lock);

	/*

	 * we need to retry the smp call.

	 * If we failed to find a running task, but find the context active now

	 * that we've acquired the ctx->lock, retry.

	 */

	if (ctx->is_active && list_empty(&event->group_entry)) {

	if (ctx->is_active) {

		raw_spin_unlock_irq(&ctx->lock);

		goto retry;

	}

	/*

	 * The lock prevents that this context is scheduled in so we

	 * can add the event safely, if it the call above did not

	 * succeed.

	 * Since the task isn't running, its safe to add the event, us holding

	 * the ctx->lock ensures the task won't get scheduled in.

	 */

	if (list_empty(&event->group_entry))

	add_event_to_ctx(event, ctx);

	raw_spin_unlock_irq(&ctx->lock);

}

/*

 * Cross CPU call to enable a performance event

 */

static void __perf_event_enable(void *info)

static int __perf_event_enable(void *info)

{

	struct perf_event *event = info;

	struct perf_event_context *ctx = event->ctx;

	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

	int err;

	/*

	 * If this is a per-task event, need to check whether this

	 * event's task is the current task on this cpu.

	 */

	if (ctx->task && cpuctx->task_ctx != ctx) {

		if (cpuctx->task_ctx || ctx->task != current)

			return;

		cpuctx->task_ctx = ctx;

	}

	if (WARN_ON_ONCE(!ctx->is_active))

		return -EINVAL;

	raw_spin_lock(&ctx->lock);

	ctx->is_active = 1;

	update_context_time(ctx);

	if (event->state >= PERF_EVENT_STATE_INACTIVE)

unlock:

	raw_spin_unlock(&ctx->lock);

	return 0;

}

/*

		/*

		 * Enable the event on the cpu that it's on

		 */

		smp_call_function_single(event->cpu, __perf_event_enable,

					 event, 1);

		cpu_function_call(event->cpu, __perf_event_enable, event);

		return;

	}

		event->state = PERF_EVENT_STATE_OFF;

retry:

	if (!ctx->is_active) {

		__perf_event_mark_enabled(event, ctx);

		goto out;

	}

	raw_spin_unlock_irq(&ctx->lock);

	task_oncpu_function_call(task, __perf_event_enable, event);

	if (!task_function_call(task, __perf_event_enable, event))

		return;

	raw_spin_lock_irq(&ctx->lock);

	 * If the context is active and the event is still off,

	 * we need to retry the cross-call.

	 */

	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)

		goto retry;

	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {

		/*

	 * Since we have the lock this context can't be scheduled

	 * in, so we can change the state safely.

		 * task could have been flipped by a concurrent

		 * perf_event_context_sched_out()

		 */

	if (event->state == PERF_EVENT_STATE_OFF)

		__perf_event_mark_enabled(event, ctx);

		task = ctx->task;

		goto retry;

	}

out:

	raw_spin_unlock_irq(&ctx->lock);

	}

}

void perf_event_context_sched_out(struct task_struct *task, int ctxn,

static void perf_event_context_sched_out(struct task_struct *task, int ctxn,

					 struct task_struct *next)

{

	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];

	cpuctx->task_ctx = ctx;

}

void perf_event_context_sched_in(struct perf_event_context *ctx)

static void perf_event_context_sched_in(struct perf_event_context *ctx)

{

	struct perf_cpu_context *cpuctx;

}

/*

 * Returns a matching context with refcount and pincount.

 */

static struct perf_event_context *

find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)

{

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		ctx = &cpuctx->ctx;

		get_ctx(ctx);

		++ctx->pin_count;

		return ctx;

	}

	ctx = perf_lock_task_context(task, ctxn, &flags);

	if (ctx) {

		unclone_ctx(ctx);

		++ctx->pin_count;

		raw_spin_unlock_irqrestore(&ctx->lock, flags);

	}

			err = -ESRCH;

		else if (task->perf_event_ctxp[ctxn])

			err = -EAGAIN;

		else

		else {

			++ctx->pin_count;

			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);

		}

		mutex_unlock(&task->perf_event_mutex);

		if (unlikely(err)) {

		struct perf_event_context *gctx = group_leader->ctx;

		mutex_lock(&gctx->mutex);

		perf_event_remove_from_context(group_leader);

		perf_remove_from_context(group_leader);

		list_for_each_entry(sibling, &group_leader->sibling_list,

				    group_entry) {

			perf_event_remove_from_context(sibling);

			perf_remove_from_context(sibling);

			put_ctx(gctx);

		}

		mutex_unlock(&gctx->mutex);

	perf_install_in_context(ctx, event, cpu);

	++ctx->generation;

	perf_unpin_context(ctx);

	mutex_unlock(&ctx->mutex);

	event->owner = current;

	return event_fd;

err_context:

	perf_unpin_context(ctx);

	put_ctx(ctx);

err_alloc:

	free_event(event);

	mutex_lock(&ctx->mutex);

	perf_install_in_context(ctx, event, cpu);

	++ctx->generation;

	perf_unpin_context(ctx);

	mutex_unlock(&ctx->mutex);

	return event;

{

	struct perf_event *parent_event;

	perf_event_remove_from_context(child_event);

	perf_remove_from_context(child_event);

	parent_event = child_event->parent;

	/*

	mutex_unlock(&parent_ctx->mutex);

	perf_unpin_context(parent_ctx);

	put_ctx(parent_ctx);

	return ret;

}

	perf_pmu_rotate_stop(ctx->pmu);

	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)

		__perf_event_remove_from_context(event);

		__perf_remove_from_context(event);

	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)

		__perf_event_remove_from_context(event);

		__perf_remove_from_context(event);

}

static void perf_event_exit_cpu_context(int cpu)

EXPORT_SYMBOL_GPL(kick_process);

#endif /* CONFIG_SMP */

/**

 * task_oncpu_function_call - call a function on the cpu on which a task runs

 * @p:		the task to evaluate

 * @func:	the function to be called

 * @info:	the function call argument

 *

 * Calls the function @func when the task is currently running. This might

 * be on the current CPU, which just calls the function directly

 */

void task_oncpu_function_call(struct task_struct *p,

			      void (*func) (void *info), void *info)

{

	int cpu;

	preempt_disable();

	cpu = task_cpu(p);

	if (task_curr(p))

		smp_call_function_single(cpu, func, info, 1);

	preempt_enable();

}

#ifdef CONFIG_SMP

/*

 * ->cpus_allowed is protected by either TASK_WAKING or rq->lock held.

prepare_task_switch(struct rq *rq, struct task_struct *prev,

		    struct task_struct *next)

{

	sched_info_switch(prev, next);

	perf_event_task_sched_out(prev, next);

	fire_sched_out_preempt_notifiers(prev, next);

	prepare_lock_switch(rq, next);

	prepare_arch_switch(next);

	trace_sched_switch(prev, next);

}

/**

	struct mm_struct *mm, *oldmm;

	prepare_task_switch(rq, prev, next);

	trace_sched_switch(prev, next);

	mm = next->mm;

	oldmm = prev->active_mm;

	/*

	rq->skip_clock_update = 0;

	if (likely(prev != next)) {

		sched_info_switch(prev, next);

		perf_event_task_sched_out(prev, next);

		rq->nr_switches++;

		rq->curr = next;

		++*switch_count;