Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 00cc2e07 authored by Catalin Marinas's avatar Catalin Marinas
Browse files

Merge Will Deacon's for-next/perf branch into for-next/core 

* will/for-next/perf:
  selftests: arm64: add test for unaligned/inexact watchpoint handling
  arm64: Allow hw watchpoint of length 3,5,6 and 7
  arm64: hw_breakpoint: Handle inexact watchpoint addresses
  arm64: Allow hw watchpoint at varied offset from base address
  hw_breakpoint: Allow watchpoint of length 3,5,6 and 7
parents 1650ac49 f43365ee

arch/arm64/include/asm/hw_breakpoint.h

+5 −1
Original line number Diff line number Diff line
@@ -77,7 +77,11 @@ static inline void decode_ctrl_reg(u32 reg, 
/* Lengths */

#define ARM_BREAKPOINT_LEN_1	0x1

#define ARM_BREAKPOINT_LEN_2	0x3

#define ARM_BREAKPOINT_LEN_3	0x7

#define ARM_BREAKPOINT_LEN_4	0xf

#define ARM_BREAKPOINT_LEN_5	0x1f

#define ARM_BREAKPOINT_LEN_6	0x3f

#define ARM_BREAKPOINT_LEN_7	0x7f

#define ARM_BREAKPOINT_LEN_8	0xff



/* Kernel stepping */
@@ -119,7 +123,7 @@ struct perf_event; 
struct pmu;



extern int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,

				  int *gen_len, int *gen_type);

				  int *gen_len, int *gen_type, int *offset);

extern int arch_check_bp_in_kernelspace(struct perf_event *bp);

extern int arch_validate_hwbkpt_settings(struct perf_event *bp);

extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused,

arch/arm64/kernel/hw_breakpoint.c

+115 −38
Original line number Diff line number Diff line
@@ -317,9 +317,21 @@ static int get_hbp_len(u8 hbp_len) 
	case ARM_BREAKPOINT_LEN_2:

		len_in_bytes = 2;

		break;

	case ARM_BREAKPOINT_LEN_3:

		len_in_bytes = 3;

		break;

	case ARM_BREAKPOINT_LEN_4:

		len_in_bytes = 4;

		break;

	case ARM_BREAKPOINT_LEN_5:

		len_in_bytes = 5;

		break;

	case ARM_BREAKPOINT_LEN_6:

		len_in_bytes = 6;

		break;

	case ARM_BREAKPOINT_LEN_7:

		len_in_bytes = 7;

		break;

	case ARM_BREAKPOINT_LEN_8:

		len_in_bytes = 8;

		break;
@@ -349,7 +361,7 @@ int arch_check_bp_in_kernelspace(struct perf_event *bp) 
 * to generic breakpoint descriptions.

 */

int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,

			   int *gen_len, int *gen_type)

			   int *gen_len, int *gen_type, int *offset)

{

	/* Type */

	switch (ctrl.type) {
@@ -369,17 +381,33 @@ int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, 
		return -EINVAL;

	}



	if (!ctrl.len)

		return -EINVAL;

	*offset = __ffs(ctrl.len);



	/* Len */

	switch (ctrl.len) {

	switch (ctrl.len >> *offset) {

	case ARM_BREAKPOINT_LEN_1:

		*gen_len = HW_BREAKPOINT_LEN_1;

		break;

	case ARM_BREAKPOINT_LEN_2:

		*gen_len = HW_BREAKPOINT_LEN_2;

		break;

	case ARM_BREAKPOINT_LEN_3:

		*gen_len = HW_BREAKPOINT_LEN_3;

		break;

	case ARM_BREAKPOINT_LEN_4:

		*gen_len = HW_BREAKPOINT_LEN_4;

		break;

	case ARM_BREAKPOINT_LEN_5:

		*gen_len = HW_BREAKPOINT_LEN_5;

		break;

	case ARM_BREAKPOINT_LEN_6:

		*gen_len = HW_BREAKPOINT_LEN_6;

		break;

	case ARM_BREAKPOINT_LEN_7:

		*gen_len = HW_BREAKPOINT_LEN_7;

		break;

	case ARM_BREAKPOINT_LEN_8:

		*gen_len = HW_BREAKPOINT_LEN_8;

		break;
@@ -423,9 +451,21 @@ static int arch_build_bp_info(struct perf_event *bp) 
	case HW_BREAKPOINT_LEN_2:

		info->ctrl.len = ARM_BREAKPOINT_LEN_2;

		break;

	case HW_BREAKPOINT_LEN_3:

		info->ctrl.len = ARM_BREAKPOINT_LEN_3;

		break;

	case HW_BREAKPOINT_LEN_4:

		info->ctrl.len = ARM_BREAKPOINT_LEN_4;

		break;

	case HW_BREAKPOINT_LEN_5:

		info->ctrl.len = ARM_BREAKPOINT_LEN_5;

		break;

	case HW_BREAKPOINT_LEN_6:

		info->ctrl.len = ARM_BREAKPOINT_LEN_6;

		break;

	case HW_BREAKPOINT_LEN_7:

		info->ctrl.len = ARM_BREAKPOINT_LEN_7;

		break;

	case HW_BREAKPOINT_LEN_8:

		info->ctrl.len = ARM_BREAKPOINT_LEN_8;

		break;
@@ -517,18 +557,17 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) 
		default:

			return -EINVAL;

		}



		info->address &= ~alignment_mask;

		info->ctrl.len <<= offset;

	} else {

		if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE)

			alignment_mask = 0x3;

		else

			alignment_mask = 0x7;

		if (info->address & alignment_mask)

			return -EINVAL;

		offset = info->address & alignment_mask;

	}



	info->address &= ~alignment_mask;

	info->ctrl.len <<= offset;



	/*

	 * Disallow per-task kernel breakpoints since these would

	 * complicate the stepping code.
@@ -661,12 +700,47 @@ static int breakpoint_handler(unsigned long unused, unsigned int esr, 
}

NOKPROBE_SYMBOL(breakpoint_handler);



/*

 * Arm64 hardware does not always report a watchpoint hit address that matches

 * one of the watchpoints set. It can also report an address "near" the

 * watchpoint if a single instruction access both watched and unwatched

 * addresses. There is no straight-forward way, short of disassembling the

 * offending instruction, to map that address back to the watchpoint. This

 * function computes the distance of the memory access from the watchpoint as a

 * heuristic for the likelyhood that a given access triggered the watchpoint.

 *

 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint

 * exception" of ARMv8 Architecture Reference Manual for details.

 *

 * The function returns the distance of the address from the bytes watched by

 * the watchpoint. In case of an exact match, it returns 0.

 */

static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,

					struct arch_hw_breakpoint_ctrl *ctrl)

{

	u64 wp_low, wp_high;

	u32 lens, lene;



	lens = __ffs(ctrl->len);

	lene = __fls(ctrl->len);



	wp_low = val + lens;

	wp_high = val + lene;

	if (addr < wp_low)

		return wp_low - addr;

	else if (addr > wp_high)

		return addr - wp_high;

	else

		return 0;

}



static int watchpoint_handler(unsigned long addr, unsigned int esr,

			      struct pt_regs *regs)

{

	int i, step = 0, *kernel_step, access;

	int i, step = 0, *kernel_step, access, closest_match = 0;

	u64 min_dist = -1, dist;

	u32 ctrl_reg;

	u64 val, alignment_mask;

	u64 val;

	struct perf_event *wp, **slots;

	struct debug_info *debug_info;

	struct arch_hw_breakpoint *info;
@@ -675,35 +749,15 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr, 
	slots = this_cpu_ptr(wp_on_reg);

	debug_info = &current->thread.debug;



	for (i = 0; i < core_num_wrps; ++i) {

	/*

	 * Find all watchpoints that match the reported address. If no exact

	 * match is found. Attribute the hit to the closest watchpoint.

	 */

	rcu_read_lock();



	for (i = 0; i < core_num_wrps; ++i) {

		wp = slots[i];



		if (wp == NULL)

			goto unlock;



		info = counter_arch_bp(wp);

		/* AArch32 watchpoints are either 4 or 8 bytes aligned. */

		if (is_compat_task()) {

			if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)

				alignment_mask = 0x7;

			else

				alignment_mask = 0x3;

		} else {

			alignment_mask = 0x7;

		}



		/* Check if the watchpoint value matches. */

		val = read_wb_reg(AARCH64_DBG_REG_WVR, i);

		if (val != (addr & ~alignment_mask))

			goto unlock;



		/* Possible match, check the byte address select to confirm. */

		ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);

		decode_ctrl_reg(ctrl_reg, &ctrl);

		if (!((1 << (addr & alignment_mask)) & ctrl.len))

			goto unlock;

			continue;



		/*

		 * Check that the access type matches.
@@ -712,18 +766,41 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr, 
		access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :

			 HW_BREAKPOINT_R;

		if (!(access & hw_breakpoint_type(wp)))

			goto unlock;

			continue;



		/* Check if the watchpoint value and byte select match. */

		val = read_wb_reg(AARCH64_DBG_REG_WVR, i);

		ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);

		decode_ctrl_reg(ctrl_reg, &ctrl);

		dist = get_distance_from_watchpoint(addr, val, &ctrl);

		if (dist < min_dist) {

			min_dist = dist;

			closest_match = i;

		}

		/* Is this an exact match? */

		if (dist != 0)

			continue;



		info = counter_arch_bp(wp);

		info->trigger = addr;

		perf_bp_event(wp, regs);



		/* Do we need to handle the stepping? */

		if (is_default_overflow_handler(wp))

			step = 1;

	}

	if (min_dist > 0 && min_dist != -1) {

		/* No exact match found. */

		wp = slots[closest_match];

		info = counter_arch_bp(wp);

		info->trigger = addr;

		perf_bp_event(wp, regs);



unlock:

		rcu_read_unlock();

		/* Do we need to handle the stepping? */

		if (is_default_overflow_handler(wp))

			step = 1;

	}

	rcu_read_unlock();



	if (!step)

		return 0;

arch/arm64/kernel/ptrace.c

+4 −3
Original line number Diff line number Diff line
@@ -327,13 +327,13 @@ static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type, 
				     struct arch_hw_breakpoint_ctrl ctrl,

				     struct perf_event_attr *attr)

{

	int err, len, type, disabled = !ctrl.enabled;

	int err, len, type, offset, disabled = !ctrl.enabled;



	attr->disabled = disabled;

	if (disabled)

		return 0;



	err = arch_bp_generic_fields(ctrl, &len, &type);

	err = arch_bp_generic_fields(ctrl, &len, &type, &offset);

	if (err)

		return err;
@@ -352,6 +352,7 @@ static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type, 


	attr->bp_len	= len;

	attr->bp_type	= type;

	attr->bp_addr	+= offset;



	return 0;

}
@@ -404,7 +405,7 @@ static int ptrace_hbp_get_addr(unsigned int note_type, 
	if (IS_ERR(bp))

		return PTR_ERR(bp);



	*addr = bp ? bp->attr.bp_addr : 0;

	*addr = bp ? counter_arch_bp(bp)->address : 0;

	return 0;

}

include/uapi/linux/hw_breakpoint.h

+4 −0
Original line number Diff line number Diff line
@@ -4,7 +4,11 @@ 
enum {

	HW_BREAKPOINT_LEN_1 = 1,

	HW_BREAKPOINT_LEN_2 = 2,

	HW_BREAKPOINT_LEN_3 = 3,

	HW_BREAKPOINT_LEN_4 = 4,

	HW_BREAKPOINT_LEN_5 = 5,

	HW_BREAKPOINT_LEN_6 = 6,

	HW_BREAKPOINT_LEN_7 = 7,

	HW_BREAKPOINT_LEN_8 = 8,

};

tools/include/uapi/linux/hw_breakpoint.h

+4 −0
Original line number Diff line number Diff line
@@ -4,7 +4,11 @@ 
enum {

	HW_BREAKPOINT_LEN_1 = 1,

	HW_BREAKPOINT_LEN_2 = 2,

	HW_BREAKPOINT_LEN_3 = 3,

	HW_BREAKPOINT_LEN_4 = 4,

	HW_BREAKPOINT_LEN_5 = 5,

	HW_BREAKPOINT_LEN_6 = 6,

	HW_BREAKPOINT_LEN_7 = 7,

	HW_BREAKPOINT_LEN_8 = 8,

};