Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5a3126d4 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

perf: Fix the perf context switch optimization 



Currently we only optimize the context switch between two
contexts that have the same parent; this forgoes the
optimization between parent and child context, even though these
contexts could be equivalent too.

Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Adrian Hunter <adrian.hunter@intel.com>
Cc: Shishkin, Alexander <alexander.shishkin@intel.com>
Link: http://lkml.kernel.org/r/20131007164257.GH3081@twins.programming.kicks-ass.net


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent e00b12e6

kernel/events/core.c

+46 −18
Original line number Original line Diff line number Diff line
@@ -899,6 +899,7 @@ static void unclone_ctx(struct perf_event_context *ctx) 
		put_ctx(ctx->parent_ctx);

		put_ctx(ctx->parent_ctx);

		ctx->parent_ctx = NULL;

		ctx->parent_ctx = NULL;

	}

	}

	ctx->generation++;

}

}





static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)

static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
@@ -1136,6 +1137,8 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx) 
	ctx->nr_events++;

	ctx->nr_events++;

	if (event->attr.inherit_stat)

	if (event->attr.inherit_stat)

		ctx->nr_stat++;

		ctx->nr_stat++;



	ctx->generation++;

}

}





/*

/*
@@ -1313,6 +1316,8 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) 
	 */

	 */

	if (event->state > PERF_EVENT_STATE_OFF)

	if (event->state > PERF_EVENT_STATE_OFF)

		event->state = PERF_EVENT_STATE_OFF;

		event->state = PERF_EVENT_STATE_OFF;



	ctx->generation++;

}

}





static void perf_group_detach(struct perf_event *event)

static void perf_group_detach(struct perf_event *event)
@@ -2149,22 +2154,38 @@ static void ctx_sched_out(struct perf_event_context *ctx, 
}

}





/*

/*

 * Test whether two contexts are equivalent, i.e. whether they

 * Test whether two contexts are equivalent, i.e. whether they have both been

 * have both been cloned from the same version of the same context

 * cloned from the same version of the same context.

 * and they both have the same number of enabled events.

 *

 * If the number of enabled events is the same, then the set

 * Equivalence is measured using a generation number in the context that is

 * of enabled events should be the same, because these are both

 * incremented on each modification to it; see unclone_ctx(), list_add_event()

 * inherited contexts, therefore we can't access individual events

 * and list_del_event().

 * in them directly with an fd; we can only enable/disable all

 * events via prctl, or enable/disable all events in a family

 * via ioctl, which will have the same effect on both contexts.

 */

 */

static int context_equiv(struct perf_event_context *ctx1,

static int context_equiv(struct perf_event_context *ctx1,

			 struct perf_event_context *ctx2)

			 struct perf_event_context *ctx2)

{

{

	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx

	/* Pinning disables the swap optimization */

		&& ctx1->parent_gen == ctx2->parent_gen

	if (ctx1->pin_count || ctx2->pin_count)

		&& !ctx1->pin_count && !ctx2->pin_count;

		return 0;



	/* If ctx1 is the parent of ctx2 */

	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)

		return 1;



	/* If ctx2 is the parent of ctx1 */

	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)

		return 1;



	/*

	 * If ctx1 and ctx2 have the same parent; we flatten the parent

	 * hierarchy, see perf_event_init_context().

	 */

	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&

			ctx1->parent_gen == ctx2->parent_gen)

		return 1;



	/* Unmatched */

	return 0;

}

}





static void __perf_event_sync_stat(struct perf_event *event,

static void __perf_event_sync_stat(struct perf_event *event,
@@ -2247,7 +2268,7 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 
{

{

	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];

	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];

	struct perf_event_context *next_ctx;

	struct perf_event_context *next_ctx;

	struct perf_event_context *parent;

	struct perf_event_context *parent, *next_parent;

	struct perf_cpu_context *cpuctx;

	struct perf_cpu_context *cpuctx;

	int do_switch = 1;

	int do_switch = 1;
@@ -2259,10 +2280,18 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 
		return;

		return;





	rcu_read_lock();

	rcu_read_lock();

	parent = rcu_dereference(ctx->parent_ctx);

	next_ctx = next->perf_event_ctxp[ctxn];

	next_ctx = next->perf_event_ctxp[ctxn];

	if (parent && next_ctx &&

	if (!next_ctx)

	    rcu_dereference(next_ctx->parent_ctx) == parent) {

		goto unlock;



	parent = rcu_dereference(ctx->parent_ctx);

	next_parent = rcu_dereference(next_ctx->parent_ctx);



	/* If neither context have a parent context; they cannot be clones. */

	if (!parent && !next_parent)

		goto unlock;



	if (next_parent == ctx || next_ctx == parent || next_parent == parent) {

		/*

		/*

		 * Looks like the two contexts are clones, so we might be

		 * Looks like the two contexts are clones, so we might be

		 * able to optimize the context switch.  We lock both

		 * able to optimize the context switch.  We lock both
@@ -2290,6 +2319,7 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 
		raw_spin_unlock(&next_ctx->lock);

		raw_spin_unlock(&next_ctx->lock);

		raw_spin_unlock(&ctx->lock);

		raw_spin_unlock(&ctx->lock);

	}

	}

unlock:

	rcu_read_unlock();

	rcu_read_unlock();





	if (do_switch) {

	if (do_switch) {
@@ -7136,7 +7166,6 @@ SYSCALL_DEFINE5(perf_event_open, 
	}

	}





	perf_install_in_context(ctx, event, event->cpu);

	perf_install_in_context(ctx, event, event->cpu);

	++ctx->generation;

	perf_unpin_context(ctx);

	perf_unpin_context(ctx);

	mutex_unlock(&ctx->mutex);

	mutex_unlock(&ctx->mutex);
@@ -7219,7 +7248,6 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 
	WARN_ON_ONCE(ctx->parent_ctx);

	WARN_ON_ONCE(ctx->parent_ctx);

	mutex_lock(&ctx->mutex);

	mutex_lock(&ctx->mutex);

	perf_install_in_context(ctx, event, cpu);

	perf_install_in_context(ctx, event, cpu);

	++ctx->generation;

	perf_unpin_context(ctx);

	perf_unpin_context(ctx);

	mutex_unlock(&ctx->mutex);

	mutex_unlock(&ctx->mutex);