Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b76834bc authored by Christoph Lameter's avatar Christoph Lameter Committed by Tejun Heo
Browse files

kprobes: Use this_cpu_ops 



Use this_cpu ops in various places to optimize per cpu data access.

Cc: Jason Baron <jbaron@redhat.com>
Cc: Namhyung Kim <namhyung@gmail.com>
Acked-by: default avatarH. Peter Anvin <hpa@zytor.com>
Signed-off-by: default avatarChristoph Lameter <cl@linux.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 4a6f4fe8

arch/x86/kernel/kprobes.c

+7 −7
Original line number Diff line number Diff line
@@ -403,7 +403,7 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 


static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)

{

	__get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;

	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);

	kcb->kprobe_status = kcb->prev_kprobe.status;

	kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;

	kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
@@ -412,7 +412,7 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 
static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,

				struct kprobe_ctlblk *kcb)

{

	__get_cpu_var(current_kprobe) = p;

	__this_cpu_write(current_kprobe, p);

	kcb->kprobe_saved_flags = kcb->kprobe_old_flags

		= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));

	if (is_IF_modifier(p->ainsn.insn))
@@ -586,7 +586,7 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) 
		preempt_enable_no_resched();

		return 1;

	} else if (kprobe_running()) {

		p = __get_cpu_var(current_kprobe);

		p = __this_cpu_read(current_kprobe);

		if (p->break_handler && p->break_handler(p, regs)) {

			setup_singlestep(p, regs, kcb, 0);

			return 1;
@@ -759,11 +759,11 @@ static __used __kprobes void *trampoline_handler(struct pt_regs *regs) 


		orig_ret_address = (unsigned long)ri->ret_addr;

		if (ri->rp && ri->rp->handler) {

			__get_cpu_var(current_kprobe) = &ri->rp->kp;

			__this_cpu_write(current_kprobe, &ri->rp->kp);

			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;

			ri->ret_addr = correct_ret_addr;

			ri->rp->handler(ri, regs);

			__get_cpu_var(current_kprobe) = NULL;

			__this_cpu_write(current_kprobe, NULL);

		}



		recycle_rp_inst(ri, &empty_rp);
@@ -1198,10 +1198,10 @@ static void __kprobes optimized_callback(struct optimized_kprobe *op, 
		regs->ip = (unsigned long)op->kp.addr + INT3_SIZE;

		regs->orig_ax = ~0UL;



		__get_cpu_var(current_kprobe) = &op->kp;

		__this_cpu_write(current_kprobe, &op->kp);

		kcb->kprobe_status = KPROBE_HIT_ACTIVE;

		opt_pre_handler(&op->kp, regs);

		__get_cpu_var(current_kprobe) = NULL;

		__this_cpu_write(current_kprobe, NULL);

	}

	preempt_enable_no_resched();

}

include/linux/kprobes.h

+2 −2
Original line number Diff line number Diff line
@@ -303,12 +303,12 @@ struct hlist_head * kretprobe_inst_table_head(struct task_struct *tsk); 
/* kprobe_running() will just return the current_kprobe on this CPU */

static inline struct kprobe *kprobe_running(void)

{

	return (__get_cpu_var(current_kprobe));

	return (__this_cpu_read(current_kprobe));

}



static inline void reset_current_kprobe(void)

{

	__get_cpu_var(current_kprobe) = NULL;

	__this_cpu_write(current_kprobe, NULL);

}



static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void)

kernel/kprobes.c

+4 −4
Original line number Diff line number Diff line
@@ -317,12 +317,12 @@ void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty) 
/* We have preemption disabled.. so it is safe to use __ versions */

static inline void set_kprobe_instance(struct kprobe *kp)

{

	__get_cpu_var(kprobe_instance) = kp;

	__this_cpu_write(kprobe_instance, kp);

}



static inline void reset_kprobe_instance(void)

{

	__get_cpu_var(kprobe_instance) = NULL;

	__this_cpu_write(kprobe_instance, NULL);

}



/*
@@ -775,7 +775,7 @@ static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 
static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,

					int trapnr)

{

	struct kprobe *cur = __get_cpu_var(kprobe_instance);

	struct kprobe *cur = __this_cpu_read(kprobe_instance);



	/*

	 * if we faulted "during" the execution of a user specified
@@ -790,7 +790,7 @@ static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 


static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)

{

	struct kprobe *cur = __get_cpu_var(kprobe_instance);

	struct kprobe *cur = __this_cpu_read(kprobe_instance);

	int ret = 0;



	if (cur && cur->break_handler) {