edac: cortex_arm64_edac: modify sbe detection to use perf events (262b8c58) · Commits · e / devices / android_kernel_xiaomi_markw

Documentation/devicetree/bindings/cache/arm64_cache_erp.txt

+0 −3

Original line number	Diff line number	Diff line
		@@ -13,9 +13,6 @@ Required properties:
		"cci-irq" - CCI error interrupt. If this property is present, having
		the 'cci' reg-base defined using the 'reg' property is
		recommended.
		"sbe-irq" - Single-bit Error interrupt. Single-bit errors interrupts
		are generated by the pmu overflow interrupt, so this essentially
		is just the arm64 pmu interrupt. This is a percpu interrupt.
		At least one irq entry is required.

		Optional properties:

arch/arm64/include/asm/perf_event.h

+0 −17

Original line number	Diff line number	Diff line
		@@ -17,28 +17,11 @@
		#ifndef __ASM_PERF_EVENT_H
		#define __ASM_PERF_EVENT_H

		#include <linux/irqreturn.h>
		#include <linux/spinlock_types.h>

		#ifdef CONFIG_HW_PERF_EVENTS
		struct pt_regs;
		extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
		extern unsigned long perf_misc_flags(struct pt_regs *regs);
		irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev);
		void arm64_pmu_irq_handled_externally(void);
		void arm64_pmu_lock(raw_spinlock_t lock, unsigned long flags);
		void arm64_pmu_unlock(raw_spinlock_t lock, unsigned long flags);
		#define perf_misc_flags(regs) perf_misc_flags(regs)
		#else
		static inline irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
		{
		return IRQ_HANDLED;
		}
		static inline void arm64_pmu_irq_handled_externally(void) { }
		static inline void arm64_pmu_lock(raw_spinlock_t lock, unsigned long flags)
		{ }
		static inline void arm64_pmu_unlock(raw_spinlock_t lock, unsigned long flags)
		{ }
		#endif

		#endif

arch/arm64/kernel/perf_event.c

+15 −57

Original line number	Diff line number	Diff line
		@@ -53,40 +53,9 @@ static DEFINE_PER_CPU(u32, armv8_pm_pmuserenr);
		static DEFINE_PER_CPU(u32, hotplug_down);

		#define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
		static struct pmu_hw_events *armpmu_get_cpu_events(void);

		/* Set at runtime when we know what CPU type we are. */
		static struct arm_pmu *cpu_pmu;
		static int msm_pmu_use_irq = 1;

		void arm64_pmu_irq_handled_externally(void)
		{
		msm_pmu_use_irq = 0;
		}

		void arm64_pmu_lock(raw_spinlock_t lock, unsigned long flags)
		{
		struct pmu_hw_events *events_cpu;

		if (lock) {
		raw_spin_lock_irqsave(lock, *flags);
		} else {
		events_cpu = armpmu_get_cpu_events();
		raw_spin_lock_irqsave(&events_cpu->pmu_lock, *flags);
		}
		}

		void arm64_pmu_unlock(raw_spinlock_t lock, unsigned long flags)
		{
		struct pmu_hw_events *events_cpu;

		if (lock) {
		raw_spin_unlock_irqrestore(lock, *flags);
		} else {
		events_cpu = armpmu_get_cpu_events();
		raw_spin_unlock_irqrestore(&events_cpu->pmu_lock, *flags);
		}
		}

		int
		armpmu_get_max_events(void)
		@@ -1013,7 +982,7 @@ static void armv8pmu_enable_event(struct hw_perf_event *hwc, int idx)
		* Enable counter and interrupt, and set the counter to count
		* the event that we're interested in.
		*/
		arm64_pmu_lock(&events->pmu_lock, &flags);
		raw_spin_lock_irqsave(&events->pmu_lock, flags);

		/*
		* Disable counter
		@@ -1040,7 +1009,7 @@ static void armv8pmu_enable_event(struct hw_perf_event *hwc, int idx)
		*/
		armv8pmu_enable_counter(idx);

		arm64_pmu_unlock(&events->pmu_lock, &flags);
		raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
		}

		static void armv8pmu_disable_event(struct hw_perf_event *hwc, int idx)
		@@ -1051,7 +1020,7 @@ static void armv8pmu_disable_event(struct hw_perf_event *hwc, int idx)
		/*
		* Disable counter and interrupt
		*/
		arm64_pmu_lock(&events->pmu_lock, &flags);
		raw_spin_lock_irqsave(&events->pmu_lock, flags);

		/*
		* Disable counter
		@@ -1063,7 +1032,7 @@ static void armv8pmu_disable_event(struct hw_perf_event *hwc, int idx)
		*/
		armv8pmu_disable_intens(idx);

		arm64_pmu_unlock(&events->pmu_lock, &flags);
		raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
		}

		static int armv8pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
		@@ -1084,11 +1053,6 @@ static int armv8pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
		return -ENODEV;
		}

		if (!msm_pmu_use_irq) {
		pr_info("EDAC driver requests for the PMU interrupt\n");
		goto out;
		}

		if (irq_is_percpu(irq)) {
		err = request_percpu_irq(irq, handler,
		"arm-pmu", &cpu_hw_events);
		@@ -1131,7 +1095,6 @@ static int armv8pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
		}
		}

		out:
		return 0;
		}

		@@ -1156,10 +1119,8 @@ static void armv8pmu_free_irq(struct arm_pmu *cpu_pmu)
		cpu_pmu->pmu_state = ARM_PMU_STATE_GOING_DOWN;

		if (irq_is_percpu(irq)) {
		if (msm_pmu_use_irq) {
		on_each_cpu(armpmu_disable_percpu_irq, &irq, 1);
		free_percpu_irq(irq, &cpu_hw_events);
		}
		} else {
		for (i = 0; i < irqs; ++i) {
		if (!cpumask_test_and_clear_cpu(i,
		@@ -1173,7 +1134,7 @@ static void armv8pmu_free_irq(struct arm_pmu *cpu_pmu)
		cpu_pmu->pmu_state = ARM_PMU_STATE_OFF;
		}

		irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
		static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
		{
		u32 pmovsr;
		struct perf_sample_data data;
		@@ -1240,10 +1201,10 @@ static void armv8pmu_start(void)
		unsigned long flags;
		struct pmu_hw_events *events = cpu_pmu->get_hw_events();

		arm64_pmu_lock(&events->pmu_lock, &flags);
		raw_spin_lock_irqsave(&events->pmu_lock, flags);
		/* Enable all counters */
		armv8pmu_pmcr_write(armv8pmu_pmcr_read() \| ARMV8_PMCR_E);
		arm64_pmu_unlock(&events->pmu_lock, &flags);
		raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
		}

		static void armv8pmu_stop(void)
		@@ -1251,10 +1212,10 @@ static void armv8pmu_stop(void)
		unsigned long flags;
		struct pmu_hw_events *events = cpu_pmu->get_hw_events();

		arm64_pmu_lock(&events->pmu_lock, &flags);
		raw_spin_lock_irqsave(&events->pmu_lock, flags);
		/* Disable all counters */
		armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMCR_E);
		arm64_pmu_unlock(&events->pmu_lock, &flags);
		raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
		}

		static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
		@@ -1392,9 +1353,6 @@ static u32 __init armv8pmu_read_num_pmnc_events(void)
		/* Read the nb of CNTx counters supported from PMNC */
		nb_cnt = (armv8pmu_pmcr_read() >> ARMV8_PMCR_N_SHIFT) & ARMV8_PMCR_N_MASK;

		#ifdef CONFIG_EDAC_CORTEX_ARM64
		nb_cnt -= 1;
		#endif
		/* Add the CPU cycles counter and return */
		return nb_cnt + 1;
		}
		@@ -1537,8 +1495,8 @@ static int __cpuinit cpu_pmu_notify(struct notifier_block *b,
		smp_call_function_single(cpu,
		armpmu_hotplug_disable, cpu_pmu, 1);
		/* Disarm the PMU IRQ before disappearing. */
		if (msm_pmu_use_irq && cpu_pmu->percpu_irq) {
		irq = cpu_pmu->percpu_irq;
		if (cpu_pmu->plat_device) {
		irq = platform_get_irq(cpu_pmu->plat_device, 0);
		smp_call_function_single(cpu,
		armpmu_disable_percpu_irq, &irq, 1);
		}
		@@ -1554,8 +1512,8 @@ static int __cpuinit cpu_pmu_notify(struct notifier_block *b,
		cpu_pmu->restore_pm_registers(hcpu);
		if (cpu_pmu->pmu_state == ARM_PMU_STATE_RUNNING) {
		/* Arm the PMU IRQ before appearing. */
		if (msm_pmu_use_irq && cpu_pmu->percpu_irq) {
		irq = cpu_pmu->percpu_irq;
		if (cpu_pmu->plat_device) {
		irq = platform_get_irq(cpu_pmu->plat_device, 0);
		armpmu_enable_percpu_irq(&irq);
		}
		if (cpu_has_active_perf(cpu)) {

drivers/edac/cortex_arm64_edac.c

+54 −189

Original line number	Diff line number	Diff line
		@@ -116,12 +116,11 @@ const char *err_name[] = {
		struct erp_drvdata {
		struct edac_device_ctl_info *edev_ctl;
		void __iomem *cci_base;
		u32 mem_perf_counter;
		struct notifier_block nb_pm;
		struct notifier_block nb_cpu;
		struct notifier_block nb_panic;
		struct work_struct work;
		unsigned int sbe_irq;
		struct perf_event *memerr_counters[NR_CPUS];
		};

		static struct erp_drvdata *panic_handler_drvdata;
		@@ -131,15 +130,7 @@ struct erp_local_data {
		enum error_type err;
		};

		/* Reserving the last PMU counter for Memory Event for EDAC */
		#define ARMV8_PMCR_N_SHIFT 11 /* Number of counters supported */
		#define ARMV8_PMCR_N_MASK 0x1f
		#define ARMV8PMU_EVTYPE_NSH BIT(27)
		#define ARMV8_PMCR_E (1 << 0) /* Enable all counters */
		#define MEM_ERROR_EVENT 0x1A
		#define MAX_COUNTER_VALUE 0xFFFFFFFF

		static inline void sbe_enable_event(void *info);

		struct errors_edac {
		const char * const msg;
		@@ -614,140 +605,19 @@ static irqreturn_t arm64_cci_handler(int irq, void *drvdata)
		return IRQ_HANDLED;
		}

		static inline u32 armv8pmu_pmcr_read(void)
		{
		u32 val;

		asm volatile("mrs %0, pmcr_el0" : "=r" (val));
		return val;
		}

		static inline u32 arm64_pmu_get_last_counter(void)
		{
		u32 pmcr, cntr;

		asm volatile("mrs %0, pmcr_el0" : "=r" (pmcr));
		cntr = (pmcr >> ARMV8_PMCR_N_SHIFT) & ARMV8_PMCR_N_MASK;

		return cntr-1;
		}

		static inline void arm64pmu_select_mem_counter(u32 cntr)
		{
		asm volatile("msr pmselr_el0, %0" : : "r" (cntr));
		isb();
		}

		static inline u32 arm64pmu_getreset_flags(u32 cntr)
		{
		u32 value;
		u32 write_val;

		/* Read */
		asm volatile("mrs %0, pmovsclr_el0" : "=r" (value));

		/* Write to clear flags */
		write_val = value & BIT(cntr);
		asm volatile("msr pmovsclr_el0, %0" : : "r" (write_val));

		return value;
		}

		static inline int arm64pmu_mem_counter_has_overflowed(u32 pmnc, u32 cntr)
		{
		int ret = pmnc & BIT(cntr);
		return ret;
		}

		static inline void arm64pmu_disable_mem_counter(u32 cntr)
		{
		asm volatile("msr pmcntenclr_el0, %0" : : "r" (BIT(cntr)));
		}

		static inline void arm64pmu_enable_mem_counter(u32 cntr)
		{
		asm volatile("msr pmcntenset_el0, %0" : : "r" (BIT(cntr)));
		}

		static inline void arm64pmu_write_mem_counter(u32 value, u32 cntr)
		{
		arm64pmu_select_mem_counter(cntr);
		asm volatile("msr pmxevcntr_el0, %0" : : "r" (value));
		}

		static inline void arm64pmu_set_mem_evtype(u32 cntr)
		{
		u32 val = ARMV8PMU_EVTYPE_NSH \| MEM_ERROR_EVENT;

		arm64pmu_select_mem_counter(cntr);
		asm volatile("msr pmxevtyper_el0, %0" : : "r" (val));
		}

		static inline void arm64pmu_enable_mem_irq(u32 cntr)
		{
		asm volatile("msr pmintenset_el1, %0" : : "r" (BIT(cntr)));
		}

		static inline void arm64pmu_pmcr_enable(void)
		{
		u32 val = armv8pmu_pmcr_read();

		val \|= ARMV8_PMCR_E;
		asm volatile("msr pmcr_el0, %0" : : "r" (val));
		isb();
		}

		/*
		* This function follows the sequence to enable
		* the memory event counter. Steps done here include
		* programming the counter for memory error perf event,
		* setting the initial counter value and enabling the interrupt
		* associated with the counter.
		*/
		static inline void sbe_enable_event(void *info)
		static void arm64_sbe_handler(struct perf_event *event,
		struct perf_sample_data *data,
		struct pt_regs *regs)
		{
		struct erp_drvdata *drv = info;
		unsigned long flags;
		u32 cntr = drv->mem_perf_counter;

		arm64_pmu_lock(NULL, &flags);
		arm64pmu_disable_mem_counter(cntr);
		arm64pmu_set_mem_evtype(cntr);
		arm64pmu_enable_mem_irq(cntr);
		arm64pmu_write_mem_counter(MAX_COUNTER_VALUE, cntr);
		arm64pmu_enable_mem_counter(cntr);
		arm64pmu_pmcr_enable();
		arm64_pmu_unlock(NULL, &flags);
		}

		static irqreturn_t arm64_sbe_handler(int irq, void *drvdata)
		{
		u32 pmovsr, cntr;
		struct erp_local_data errdata;
		unsigned long flags;
		int overflow = 0, ret = IRQ_HANDLED;
		int cpu = raw_smp_processor_id();

		errdata.drv = ((struct erp_drvdata *)drvdata);

		cntr = errdata.drv->mem_perf_counter;
		arm64_pmu_lock(NULL, &flags);
		pmovsr = arm64pmu_getreset_flags(cntr);
		arm64_pmu_unlock(NULL, &flags);
		overflow = arm64pmu_mem_counter_has_overflowed(pmovsr, cntr);

		if (overflow) {
		errdata.drv = event->overflow_handler_context;
		errdata.err = SBE;
		edac_printk(KERN_CRIT, EDAC_CPU, "ARM64 CPU ERP: Single-bit error interrupt received on CPU %d!\n",
		cpu);
		WARN_ON(!panic_on_ce);
		arm64_erp_local_handler(&errdata);
		sbe_enable_event(errdata.drv);
		} else {
		ret = armv8pmu_handle_irq(irq, NULL);
		}

		return ret;
		}

		static int request_erp_irq(struct platform_device pdev, const char propname,
		@@ -780,20 +650,6 @@ static int request_erp_irq(struct platform_device pdev, const char propname,
		return 0;
		}

		static void arm64_enable_pmu_irq(void *data)
		{
		unsigned int irq = (unsigned int )data;

		enable_percpu_irq(irq, IRQ_TYPE_NONE);
		}

		static void arm64_disable_pmu_irq(void *data)
		{
		unsigned int irq = (unsigned int )data;

		disable_percpu_irq(irq);
		}

		static void check_sbe_event(struct erp_drvdata *drv)
		{
		unsigned int partnum = read_cpuid_part_number();
		@@ -818,6 +674,39 @@ static void check_sbe_event(struct erp_drvdata *drv)
		spin_unlock_irqrestore(&local_handler_lock, flags);
		}

		#ifdef CONFIG_EDAC_CORTEX_ARM64_DBE_IRQ_ONLY
		static void create_sbe_counter(int cpu, void *info)
		{ }
		#else
		static void create_sbe_counter(int cpu, void *info)
		{
		struct erp_drvdata *drv = info;
		struct perf_event *event = drv->memerr_counters[cpu];
		struct perf_event_attr attr = {
		.pinned = 1,
		.disabled = 0, /* 0 will enable the counter upon creation */
		.sample_period = 1, /* 1 will set the counter to max int */
		.type = PERF_TYPE_RAW,
		.config = MEM_ERROR_EVENT,
		.size = sizeof(struct perf_event_attr),
		};

		if (event)
		return;

		/* Fails if cpu is not online */
		event = perf_event_create_kernel_counter(&attr, cpu, NULL,
		arm64_sbe_handler,
		drv);
		if (IS_ERR(event)) {
		pr_err("PERF Event creation failed on cpu %d ptr_err %ld\n",
		cpu, PTR_ERR(event));
		return;
		}
		drv->memerr_counters[cpu] = event;
		}
		#endif

		static int arm64_pmu_cpu_pm_notify(struct notifier_block *self,
		unsigned long action, void *v)
		{
		@@ -826,7 +715,6 @@ static int arm64_pmu_cpu_pm_notify(struct notifier_block *self,
		switch (action) {
		case CPU_PM_EXIT:
		check_sbe_event(drv);
		sbe_enable_event(drv);
		break;
		}

		@@ -838,13 +726,11 @@ static int arm64_edac_pmu_cpu_notify(struct notifier_block *self,
		{
		struct erp_drvdata *drv = container_of(self, struct erp_drvdata,
		nb_cpu);
		unsigned long cpu = (unsigned long)hcpu;

		switch (action & ~CPU_TASKS_FROZEN) {
		case CPU_STARTING:
		sbe_enable_event(drv);
		arm64_enable_pmu_irq(&drv->sbe_irq);
		break;
		case CPU_DYING:
		arm64_disable_pmu_irq(&drv->sbe_irq);
		case CPU_ONLINE:
		create_sbe_counter(cpu, drv);
		break;
		};

		@@ -894,10 +780,8 @@ static int arm64_cpu_erp_probe(struct platform_device *pdev)
		struct resource *r;
		int cpu;
		u32 poll_msec;
		struct erp_drvdata * __percpu *drv_cpu =
		alloc_percpu(struct erp_drvdata *);

		int rc, sbe_irq, fail = 0;
		int rc, fail = 0;

		drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);

		@@ -958,41 +842,22 @@ static int arm64_cpu_erp_probe(struct platform_device *pdev)
		arm64_cci_handler, drv))
		fail++;

		if (!drv_cpu)
		goto out_irq;

		sbe_irq = platform_get_irq_byname(pdev, "sbe-irq");
		if (sbe_irq < 0 \|\| IS_ENABLED(CONFIG_EDAC_CORTEX_ARM64_DBE_IRQ_ONLY)) {
		pr_err("ARM64 CPU ERP: Could not find sbe-irq IRQ property. Proceeding anyway.\n");
		fail++;
		goto out_irq;
		}

		for_each_possible_cpu(cpu)
		*per_cpu_ptr(drv_cpu, cpu) = drv;

		rc = request_percpu_irq(sbe_irq, arm64_sbe_handler,
		"ARM64 Single-Bit Error PMU IRQ",
		drv_cpu);
		if (rc) {
		pr_err("ARM64 CPU ERP: Failed to request IRQ %d: %d. Proceeding anyway.\n",
		sbe_irq, rc);
		if (IS_ENABLED(CONFIG_EDAC_CORTEX_ARM64_DBE_IRQ_ONLY)) {
		pr_err("ARM64 CPU ERP: SBE detection is disabled.\n");
		goto out_irq;
		}
		drv->sbe_irq = sbe_irq;

		drv->nb_pm.notifier_call = arm64_pmu_cpu_pm_notify;
		drv->mem_perf_counter = arm64_pmu_get_last_counter();
		cpu_pm_register_notifier(&(drv->nb_pm));
		drv->nb_panic.notifier_call = arm64_erp_panic_notify;
		atomic_notifier_chain_register(&panic_notifier_list,
		&drv->nb_panic);
		arm64_pmu_irq_handled_externally();

		drv->nb_cpu.notifier_call = arm64_edac_pmu_cpu_notify;
		register_cpu_notifier(&drv->nb_cpu);
		on_each_cpu(sbe_enable_event, drv, 1);
		on_each_cpu(arm64_enable_pmu_irq, &sbe_irq, 1);
		get_online_cpus();
		for_each_online_cpu(cpu)
		create_sbe_counter(cpu, drv);
		put_online_cpus();

		out_irq:
		if (fail == of_irq_count(dev->of_node)) {