Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

 * enum perf_event_active_state - the states of a event

 */

enum perf_event_active_state {

	PERF_EVENT_STATE_DEAD		= -4,

	PERF_EVENT_STATE_EXIT		= -3,

	PERF_EVENT_STATE_ERROR		= -2,

	PERF_EVENT_STATE_OFF		= -1,

	}

}

extern struct static_key_deferred perf_sched_events;

extern struct static_key_false perf_sched_events;

static __always_inline bool

perf_sw_migrate_enabled(void)

static inline void perf_event_task_sched_in(struct task_struct *prev,

					    struct task_struct *task)

{

	if (static_key_false(&perf_sched_events.key))

	if (static_branch_unlikely(&perf_sched_events))

		__perf_event_task_sched_in(prev, task);

	if (perf_sw_migrate_enabled() && task->sched_migrated) {

{

	perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);

	if (static_key_false(&perf_sched_events.key))

	if (static_branch_unlikely(&perf_sched_events))

		__perf_event_task_sched_out(prev, next);

}

	struct task_struct *p = tfc->p;

	if (p) {

		tfc->ret = -EAGAIN;

		if (task_cpu(p) != smp_processor_id() || !task_curr(p))

		/* -EAGAIN */

		if (task_cpu(p) != smp_processor_id())

			return;

		/*

		 * Now that we're on right CPU with IRQs disabled, we can test

		 * if we hit the right task without races.

		 */

		tfc->ret = -ESRCH; /* No such (running) process */

		if (p != current)

			return;

	}

		.p	= p,

		.func	= func,

		.info	= info,

		.ret	= -ESRCH, /* No such (running) process */

		.ret	= -EAGAIN,

	};

	int ret;

	if (task_curr(p))

		smp_call_function_single(task_cpu(p), remote_function, &data, 1);

	do {

		ret = smp_call_function_single(task_cpu(p), remote_function, &data, 1);

		if (!ret)

			ret = data.ret;

	} while (ret == -EAGAIN);

	return data.ret;

	return ret;

}

/**

 *    rely on ctx->is_active and therefore cannot use event_function_call().

 *    See perf_install_in_context().

 *

 * This is because we need a ctx->lock serialized variable (ctx->is_active)

 * to reliably determine if a particular task/context is scheduled in. The

 * task_curr() use in task_function_call() is racy in that a remote context

 * switch is not a single atomic operation.

 *

 * As is, the situation is 'safe' because we set rq->curr before we do the

 * actual context switch. This means that task_curr() will fail early, but

 * we'll continue spinning on ctx->is_active until we've passed

 * perf_event_task_sched_out().

 *

 * Without this ctx->lock serialized variable we could have race where we find

 * the task (and hence the context) would not be active while in fact they are.

 *

 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.

 */

	 */

	if (ctx->task) {

		if (ctx->task != current) {

			ret = -EAGAIN;

			ret = -ESRCH;

			goto unlock;

		}

		return;

	}

again:

	if (task == TASK_TOMBSTONE)

		return;

again:

	if (!task_function_call(task, event_function, &efs))

		return;

	 * a concurrent perf_event_context_sched_out().

	 */

	task = ctx->task;

	if (task != TASK_TOMBSTONE) {

	if (task == TASK_TOMBSTONE) {

		raw_spin_unlock_irq(&ctx->lock);

		return;

	}

	if (ctx->is_active) {

		raw_spin_unlock_irq(&ctx->lock);

		goto again;

	}

	func(event, NULL, ctx, data);

	}

	raw_spin_unlock_irq(&ctx->lock);

}

enum event_type_t {

	EVENT_FLEXIBLE = 0x1,

	EVENT_PINNED = 0x2,

	EVENT_TIME = 0x4,

	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,

};

 * perf_sched_events : >0 events exist

 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu

 */

struct static_key_deferred perf_sched_events __read_mostly;

static void perf_sched_delayed(struct work_struct *work);

DEFINE_STATIC_KEY_FALSE(perf_sched_events);

static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);

static DEFINE_MUTEX(perf_sched_mutex);

static atomic_t perf_sched_count;

static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);

static DEFINE_PER_CPU(int, perf_sched_cb_usages);

/*

 * Update the total_time_enabled and total_time_running fields for a event.

 * The caller of this function needs to hold the ctx->lock.

 */

static void update_event_times(struct perf_event *event)

{

	struct perf_event_context *ctx = event->ctx;

	u64 run_end;

	lockdep_assert_held(&ctx->lock);

	if (event->state < PERF_EVENT_STATE_INACTIVE ||

	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)

		return;

	/*

	 * in cgroup mode, time_enabled represents

	 * the time the event was enabled AND active

static bool is_orphaned_event(struct perf_event *event)

{

	return event->state == PERF_EVENT_STATE_EXIT;

	return event->state == PERF_EVENT_STATE_DEAD;

}

static inline int pmu_filter_match(struct perf_event *event)

	perf_pmu_disable(event->pmu);

	event->tstamp_stopped = tstamp;

	event->pmu->del(event, 0);

	event->oncpu = -1;

	event->state = PERF_EVENT_STATE_INACTIVE;

	if (event->pending_disable) {

		event->pending_disable = 0;

		event->state = PERF_EVENT_STATE_OFF;

	}

	event->tstamp_stopped = tstamp;

	event->pmu->del(event, 0);

	event->oncpu = -1;

	if (!is_software_event(event))

		cpuctx->active_oncpu--;

}

#define DETACH_GROUP	0x01UL

#define DETACH_STATE	0x02UL

/*

 * Cross CPU call to remove a performance event

	if (flags & DETACH_GROUP)

		perf_group_detach(event);

	list_del_event(event, ctx);

	if (flags & DETACH_STATE)

		event->state = PERF_EVENT_STATE_EXIT;

	if (!ctx->nr_events && ctx->is_active) {

		ctx->is_active = 0;

	event->tstamp_stopped = tstamp;

}

static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,

			       struct perf_event_context *ctx);

static void ctx_sched_out(struct perf_event_context *ctx,

			  struct perf_cpu_context *cpuctx,

			  enum event_type_t event_type);

static void

ctx_sched_in(struct perf_event_context *ctx,

	     struct perf_cpu_context *cpuctx,

	     enum event_type_t event_type,

	     struct task_struct *task);

static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,

			       struct perf_event_context *ctx)

{

	if (!cpuctx->task_ctx)

		return;

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))

		return;

	ctx_sched_out(ctx, cpuctx, EVENT_ALL);

}

static void perf_event_sched_in(struct perf_cpu_context *cpuctx,

				struct perf_event_context *ctx,

				struct task_struct *task)

/*

 * Cross CPU call to install and enable a performance event

 *

 * Must be called with ctx->mutex held

 * Very similar to remote_function() + event_function() but cannot assume that

 * things like ctx->is_active and cpuctx->task_ctx are set.

 */

static int  __perf_install_in_context(void *info)

{

	struct perf_event_context *ctx = info;

	struct perf_event *event = info;

	struct perf_event_context *ctx = event->ctx;

	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

	struct perf_event_context *task_ctx = cpuctx->task_ctx;

	bool activate = true;

	int ret = 0;

	raw_spin_lock(&cpuctx->ctx.lock);

	if (ctx->task) {

		raw_spin_lock(&ctx->lock);

		/*

		 * If we hit the 'wrong' task, we've since scheduled and

		 * everything should be sorted, nothing to do!

		 */

		task_ctx = ctx;

		if (ctx->task != current)

		/* If we're on the wrong CPU, try again */

		if (task_cpu(ctx->task) != smp_processor_id()) {

			ret = -ESRCH;

			goto unlock;

		}

		/*

		 * If task_ctx is set, it had better be to us.

		 * If we're on the right CPU, see if the task we target is

		 * current, if not we don't have to activate the ctx, a future

		 * context switch will do that for us.

		 */

		WARN_ON_ONCE(cpuctx->task_ctx != ctx && cpuctx->task_ctx);

		if (ctx->task != current)

			activate = false;

		else

			WARN_ON_ONCE(cpuctx->task_ctx && cpuctx->task_ctx != ctx);

	} else if (task_ctx) {

		raw_spin_lock(&task_ctx->lock);

	}

	if (activate) {

		ctx_sched_out(ctx, cpuctx, EVENT_TIME);

		add_event_to_ctx(event, ctx);

		ctx_resched(cpuctx, task_ctx);

	} else {

		add_event_to_ctx(event, ctx);

	}

unlock:

	perf_ctx_unlock(cpuctx, task_ctx);

	return 0;

	return ret;

}

/*

 * Attach a performance event to a context

 * Attach a performance event to a context.

 *

 * Very similar to event_function_call, see comment there.

 */

static void

perf_install_in_context(struct perf_event_context *ctx,

			struct perf_event *event,

			int cpu)

{

	struct task_struct *task = NULL;

	struct task_struct *task = READ_ONCE(ctx->task);

	lockdep_assert_held(&ctx->mutex);

	if (event->cpu != -1)

		event->cpu = cpu;

	if (!task) {

		cpu_function_call(cpu, __perf_install_in_context, event);

		return;

	}

	/*

	 * Should not happen, we validate the ctx is still alive before calling.

	 */

	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))

		return;

	/*

	 * Installing events is tricky because we cannot rely on ctx->is_active

	 * to be set in case this is the nr_events 0 -> 1 transition.

	 *

	 * So what we do is we add the event to the list here, which will allow

	 * a future context switch to DTRT and then send a racy IPI. If the IPI

	 * fails to hit the right task, this means a context switch must have

	 * happened and that will have taken care of business.

	 */

again:

	/*

	 * Cannot use task_function_call() because we need to run on the task's

	 * CPU regardless of whether its current or not.

	 */

	if (!cpu_function_call(task_cpu(task), __perf_install_in_context, event))

		return;

	raw_spin_lock_irq(&ctx->lock);

	task = ctx->task;

	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {

		/*

	 * Worse, we cannot even rely on the ctx actually existing anymore. If

	 * between find_get_context() and perf_install_in_context() the task

	 * went through perf_event_exit_task() its dead and we should not be

	 * adding new events.

		 * Cannot happen because we already checked above (which also

		 * cannot happen), and we hold ctx->mutex, which serializes us

		 * against perf_event_exit_task_context().

		 */

	if (task == TASK_TOMBSTONE) {

		raw_spin_unlock_irq(&ctx->lock);

		return;

	}

	update_context_time(ctx);

	raw_spin_unlock_irq(&ctx->lock);

	/*

	 * Update cgrp time only if current cgrp matches event->cgrp.

	 * Must be done before calling add_event_to_ctx().

	 * Since !ctx->is_active doesn't mean anything, we must IPI

	 * unconditionally.

	 */

	update_cgrp_time_from_event(event);

	add_event_to_ctx(event, ctx);

	raw_spin_unlock_irq(&ctx->lock);

	if (task)

		task_function_call(task, __perf_install_in_context, ctx);

	else

		cpu_function_call(cpu, __perf_install_in_context, ctx);

	goto again;

}

/*

	    event->state <= PERF_EVENT_STATE_ERROR)

		return;

	update_context_time(ctx);

	if (ctx->is_active)

		ctx_sched_out(ctx, cpuctx, EVENT_TIME);

	__perf_event_mark_enabled(event);

	if (!ctx->is_active)

		return;

	if (!event_filter_match(event)) {

		if (is_cgroup_event(event)) {

			perf_cgroup_set_timestamp(current, ctx); // XXX ?

		if (is_cgroup_event(event))

			perf_cgroup_defer_enabled(event);

		}

		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);

		return;

	}

	 * If the event is in a group and isn't the group leader,

	 * then don't put it on unless the group is on.

	 */

	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)

	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {

		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);

		return;

	}

	task_ctx = cpuctx->task_ctx;

	if (ctx->task)

	}

	ctx->is_active &= ~event_type;

	if (!(ctx->is_active & EVENT_ALL))

		ctx->is_active = 0;

	if (ctx->task) {

		WARN_ON_ONCE(cpuctx->task_ctx != ctx);

		if (!ctx->is_active)

			cpuctx->task_ctx = NULL;

	}

	is_active ^= ctx->is_active; /* changed bits */

	if (is_active & EVENT_TIME) {

		/* update (and stop) ctx time */

		update_context_time(ctx);

		update_cgrp_time_from_cpuctx(cpuctx);

	if (!ctx->nr_active)

	}

	if (!ctx->nr_active || !(is_active & EVENT_ALL))

		return;

	perf_pmu_disable(ctx->pmu);

	if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {

	if (is_active & EVENT_PINNED) {

		list_for_each_entry(event, &ctx->pinned_groups, group_entry)

			group_sched_out(event, cpuctx, ctx);

	}

	if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {

	if (is_active & EVENT_FLEXIBLE) {

		list_for_each_entry(event, &ctx->flexible_groups, group_entry)

			group_sched_out(event, cpuctx, ctx);

	}

		perf_cgroup_sched_out(task, next);

}

static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,

			       struct perf_event_context *ctx)

{

	if (!cpuctx->task_ctx)

		return;

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))

		return;

	ctx_sched_out(ctx, cpuctx, EVENT_ALL);

}

/*

 * Called with IRQs disabled

 */

	if (likely(!ctx->nr_events))

		return;

	ctx->is_active |= event_type;

	ctx->is_active |= (event_type | EVENT_TIME);

	if (ctx->task) {

		if (!is_active)

			cpuctx->task_ctx = ctx;

			WARN_ON_ONCE(cpuctx->task_ctx != ctx);

	}

	is_active ^= ctx->is_active; /* changed bits */

	if (is_active & EVENT_TIME) {

		/* start ctx time */

		now = perf_clock();

		ctx->timestamp = now;

		perf_cgroup_set_timestamp(task, ctx);

	}

	/*

	 * First go through the list and put on any pinned groups

	 * in order to give them the best chance of going on.

	 */

	if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))

	if (is_active & EVENT_PINNED)

		ctx_pinned_sched_in(ctx, cpuctx);

	/* Then walk through the lower prio flexible groups */

	if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))

	if (is_active & EVENT_FLEXIBLE)

		ctx_flexible_sched_in(ctx, cpuctx);

}

	cpuctx = __get_cpu_context(ctx);

	perf_ctx_lock(cpuctx, ctx);

	ctx_sched_out(ctx, cpuctx, EVENT_TIME);

	list_for_each_entry(event, &ctx->event_list, event_entry)

		enabled |= event_enable_on_exec(event, ctx);

	if (has_branch_stack(event))

		dec = true;

	if (dec)

		static_key_slow_dec_deferred(&perf_sched_events);

	if (dec) {

		if (!atomic_add_unless(&perf_sched_count, -1, 1))

			schedule_delayed_work(&perf_sched_work, HZ);

	}

	unaccount_event_cpu(event, event->cpu);

}

static void perf_sched_delayed(struct work_struct *work)

{

	mutex_lock(&perf_sched_mutex);

	if (atomic_dec_and_test(&perf_sched_count))

		static_branch_disable(&perf_sched_events);

	mutex_unlock(&perf_sched_mutex);

}

/*

 * The following implement mutual exclusion of events on "exclusive" pmus

 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled

 */

int perf_event_release_kernel(struct perf_event *event)

{

	struct perf_event_context *ctx;

	struct perf_event_context *ctx = event->ctx;

	struct perf_event *child, *tmp;

	/*

	 * If we got here through err_file: fput(event_file); we will not have

	 * attached to a context yet.

	 */

	if (!ctx) {

		WARN_ON_ONCE(event->attach_state &

				(PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));

		goto no_ctx;

	}

	if (!is_kernel_event(event))

		perf_remove_from_owner(event);

	ctx = perf_event_ctx_lock(event);

	WARN_ON_ONCE(ctx->parent_ctx);

	perf_remove_from_context(event, DETACH_GROUP | DETACH_STATE);

	perf_event_ctx_unlock(event, ctx);

	perf_remove_from_context(event, DETACH_GROUP);

	raw_spin_lock_irq(&ctx->lock);

	/*

	 * At this point we must have event->state == PERF_EVENT_STATE_EXIT,

	 * either from the above perf_remove_from_context() or through

	 * perf_event_exit_event().

	 * Mark this even as STATE_DEAD, there is no external reference to it

	 * anymore.

	 *

	 * Therefore, anybody acquiring event->child_mutex after the below

	 * loop _must_ also see this, most importantly inherit_event() which

	 * will avoid placing more children on the list.

	 * Anybody acquiring event->child_mutex after the below loop _must_

	 * also see this, most importantly inherit_event() which will avoid

	 * placing more children on the list.

	 *

	 * Thus this guarantees that we will in fact observe and kill _ALL_

	 * child events.

	 */

	WARN_ON_ONCE(event->state != PERF_EVENT_STATE_EXIT);

	event->state = PERF_EVENT_STATE_DEAD;

	raw_spin_unlock_irq(&ctx->lock);

	perf_event_ctx_unlock(event, ctx);

again:

	mutex_lock(&event->child_mutex);

	}

	mutex_unlock(&event->child_mutex);

	/* Must be the last reference */

	put_event(event);

no_ctx:

	put_event(event); /* Must be the 'last' reference */

	return 0;

}

EXPORT_SYMBOL_GPL(perf_event_release_kernel);

{

	bool no_children;

	if (event->state != PERF_EVENT_STATE_EXIT)

	if (event->state > PERF_EVENT_STATE_EXIT)

		return false;

	mutex_lock(&event->child_mutex);

	if (is_cgroup_event(event))

		inc = true;

	if (inc)

		static_key_slow_inc(&perf_sched_events.key);

	if (inc) {

		if (atomic_inc_not_zero(&perf_sched_count))

			goto enabled;

		mutex_lock(&perf_sched_mutex);

		if (!atomic_read(&perf_sched_count)) {

			static_branch_enable(&perf_sched_events);

			/*

			 * Guarantee that all CPUs observe they key change and

			 * call the perf scheduling hooks before proceeding to

			 * install events that need them.

			 */

			synchronize_sched();

		}

		/*

		 * Now that we have waited for the sync_sched(), allow further

		 * increments to by-pass the mutex.

		 */

		atomic_inc(&perf_sched_count);

		mutex_unlock(&perf_sched_mutex);

	}

enabled:

	account_event_cpu(event, event->cpu);

}

	if (move_group) {

		gctx = group_leader->ctx;

		mutex_lock_double(&gctx->mutex, &ctx->mutex);

		if (gctx->task == TASK_TOMBSTONE) {

			err = -ESRCH;

			goto err_locked;

		}

	} else {

		mutex_lock(&ctx->mutex);

	}