powerpc: Add NMI IPI infrastructure (ddd703ca) · Commits · e / devices / android_kernel_xiaomi_nabu

arch/powerpc/Kconfig

+5 −0

Original line number	Original line	Diff line number	Diff line
	@@ -80,6 +80,11 @@ config NR_IRQS
	/proc/interrupts. If you configure your system to have too few,		/proc/interrupts. If you configure your system to have too few,
	drivers will fail to load or worse - handle with care.		drivers will fail to load or worse - handle with care.

			config NMI_IPI
			bool
			depends on SMP && (DEBUGGER \|\| KEXEC_CORE)
			default y

	config STACKTRACE_SUPPORT		config STACKTRACE_SUPPORT
	bool		bool
	default y		default y

arch/powerpc/include/asm/smp.h

+11 −3

Original line number	Original line	Diff line number	Diff line
	@@ -115,11 +115,19 @@ extern int cpu_to_core_id(int cpu);
	#define PPC_MSG_CALL_FUNCTION 0		#define PPC_MSG_CALL_FUNCTION 0
	#define PPC_MSG_RESCHEDULE 1		#define PPC_MSG_RESCHEDULE 1
	#define PPC_MSG_TICK_BROADCAST 2		#define PPC_MSG_TICK_BROADCAST 2
	#define PPC_MSG_DEBUGGER_BREAK 3		#define PPC_MSG_NMI_IPI 3

	/* This is only used by the powernv kernel */		/* This is only used by the powernv kernel */
	#define PPC_MSG_RM_HOST_ACTION 4		#define PPC_MSG_RM_HOST_ACTION 4

			#define NMI_IPI_ALL_OTHERS -2

			#ifdef CONFIG_NMI_IPI
			extern int smp_handle_nmi_ipi(struct pt_regs *regs);
			#else
			static inline int smp_handle_nmi_ipi(struct pt_regs *regs) { return 0; }
			#endif

	/* for irq controllers that have dedicated ipis per message (4) */		/* for irq controllers that have dedicated ipis per message (4) */
	extern int smp_request_message_ipi(int virq, int message);		extern int smp_request_message_ipi(int virq, int message);
	extern const char *smp_ipi_name[];		extern const char *smp_ipi_name[];

arch/powerpc/kernel/smp.c

+207 −36

Original line number	Original line	Diff line number	Diff line
	@@ -87,8 +87,6 @@ volatile unsigned int cpu_callin_map[NR_CPUS];

	int smt_enabled_at_boot = 1;		int smt_enabled_at_boot = 1;

	static void (crash_ipi_function_ptr)(struct pt_regs ) = NULL;

	/*		/*
	* Returns 1 if the specified cpu should be brought up during boot.		* Returns 1 if the specified cpu should be brought up during boot.
	* Used to inhibit booting threads if they've been disabled or		* Used to inhibit booting threads if they've been disabled or
	@@ -159,32 +157,33 @@ static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
	return IRQ_HANDLED;		return IRQ_HANDLED;
	}		}

	static irqreturn_t debug_ipi_action(int irq, void *data)		#ifdef CONFIG_NMI_IPI
			static irqreturn_t nmi_ipi_action(int irq, void *data)
	{		{
	if (crash_ipi_function_ptr) {		smp_handle_nmi_ipi(get_irq_regs());
	crash_ipi_function_ptr(get_irq_regs());
	return IRQ_HANDLED;
	}

	#ifdef CONFIG_DEBUGGER
	debugger_ipi(get_irq_regs());
	#endif /* CONFIG_DEBUGGER */

	return IRQ_HANDLED;		return IRQ_HANDLED;
	}		}
			#endif

	static irq_handler_t smp_ipi_action[] = {		static irq_handler_t smp_ipi_action[] = {
	[PPC_MSG_CALL_FUNCTION] = call_function_action,		[PPC_MSG_CALL_FUNCTION] = call_function_action,
	[PPC_MSG_RESCHEDULE] = reschedule_action,		[PPC_MSG_RESCHEDULE] = reschedule_action,
	[PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,		[PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,
	[PPC_MSG_DEBUGGER_BREAK] = debug_ipi_action,		#ifdef CONFIG_NMI_IPI
			[PPC_MSG_NMI_IPI] = nmi_ipi_action,
			#endif
	};		};

			/*
			* The NMI IPI is a fallback and not truly non-maskable. It is simpler
			* than going through the call function infrastructure, and strongly
			* serialized, so it is more appropriate for debugging.
			*/
	const char *smp_ipi_name[] = {		const char *smp_ipi_name[] = {
	[PPC_MSG_CALL_FUNCTION] = "ipi call function",		[PPC_MSG_CALL_FUNCTION] = "ipi call function",
	[PPC_MSG_RESCHEDULE] = "ipi reschedule",		[PPC_MSG_RESCHEDULE] = "ipi reschedule",
	[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",		[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
	[PPC_MSG_DEBUGGER_BREAK] = "ipi debugger",		[PPC_MSG_NMI_IPI] = "nmi ipi",
	};		};

	/* optional function to request ipi, for controllers with >= 4 ipis */		/* optional function to request ipi, for controllers with >= 4 ipis */
	@@ -192,14 +191,13 @@ int smp_request_message_ipi(int virq, int msg)
	{		{
	int err;		int err;

	if (msg < 0 \|\| msg > PPC_MSG_DEBUGGER_BREAK) {		if (msg < 0 \|\| msg > PPC_MSG_NMI_IPI)
	return -EINVAL;		return -EINVAL;
	}		#ifndef CONFIG_NMI_IPI
	#if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC_CORE)		if (msg == PPC_MSG_NMI_IPI)
	if (msg == PPC_MSG_DEBUGGER_BREAK) {
	return 1;		return 1;
	}
	#endif		#endif

	err = request_irq(virq, smp_ipi_action[msg],		err = request_irq(virq, smp_ipi_action[msg],
	IRQF_PERCPU \| IRQF_NO_THREAD \| IRQF_NO_SUSPEND,		IRQF_PERCPU \| IRQF_NO_THREAD \| IRQF_NO_SUSPEND,
	smp_ipi_name[msg], NULL);		smp_ipi_name[msg], NULL);
	@@ -277,8 +275,10 @@ irqreturn_t smp_ipi_demux_relaxed(void)
	scheduler_ipi();		scheduler_ipi();
	if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))		if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
	tick_broadcast_ipi_handler();		tick_broadcast_ipi_handler();
	if (all & IPI_MESSAGE(PPC_MSG_DEBUGGER_BREAK))		#ifdef CONFIG_NMI_IPI
	debug_ipi_action(0, NULL);		if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))
			nmi_ipi_action(0, NULL);
			#endif
	} while (info->messages);		} while (info->messages);

	return IRQ_HANDLED;		return IRQ_HANDLED;
	@@ -315,6 +315,184 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask)
	do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);		do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
	}		}

			#ifdef CONFIG_NMI_IPI

			/*
			* "NMI IPI" system.
			*
			* NMI IPIs may not be recoverable, so should not be used as ongoing part of
			* a running system. They can be used for crash, debug, halt/reboot, etc.
			*
			* NMI IPIs are globally single threaded. No more than one in progress at
			* any time.
			*
			* The IPI call waits with interrupts disabled until all targets enter the
			* NMI handler, then the call returns.
			*
			* No new NMI can be initiated until targets exit the handler.
			*
			* The IPI call may time out without all targets entering the NMI handler.
			* In that case, there is some logic to recover (and ignore subsequent
			* NMI interrupts that may eventually be raised), but the platform interrupt
			* handler may not be able to distinguish this from other exception causes,
			* which may cause a crash.
			*/

			static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0);
			static struct cpumask nmi_ipi_pending_mask;
			static int nmi_ipi_busy_count = 0;
			static void (nmi_ipi_function)(struct pt_regs ) = NULL;

			static void nmi_ipi_lock_start(unsigned long *flags)
			{
			raw_local_irq_save(*flags);
			hard_irq_disable();
			while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
			raw_local_irq_restore(*flags);
			cpu_relax();
			raw_local_irq_save(*flags);
			hard_irq_disable();
			}
			}

			static void nmi_ipi_lock(void)
			{
			while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
			cpu_relax();
			}

			static void nmi_ipi_unlock(void)
			{
			smp_mb();
			WARN_ON(atomic_read(&__nmi_ipi_lock) != 1);
			atomic_set(&__nmi_ipi_lock, 0);
			}

			static void nmi_ipi_unlock_end(unsigned long *flags)
			{
			nmi_ipi_unlock();
			raw_local_irq_restore(*flags);
			}

			/*
			* Platform NMI handler calls this to ack
			*/
			int smp_handle_nmi_ipi(struct pt_regs *regs)
			{
			void (fn)(struct pt_regs );
			unsigned long flags;
			int me = raw_smp_processor_id();
			int ret = 0;

			/*
			* Unexpected NMIs are possible here because the interrupt may not
			* be able to distinguish NMI IPIs from other types of NMIs, or
			* because the caller may have timed out.
			*/
			nmi_ipi_lock_start(&flags);
			if (!nmi_ipi_busy_count)
			goto out;
			if (!cpumask_test_cpu(me, &nmi_ipi_pending_mask))
			goto out;

			fn = nmi_ipi_function;
			if (!fn)
			goto out;

			cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
			nmi_ipi_busy_count++;
			nmi_ipi_unlock();

			ret = 1;

			fn(regs);

			nmi_ipi_lock();
			nmi_ipi_busy_count--;
			out:
			nmi_ipi_unlock_end(&flags);

			return ret;
			}

			static void do_smp_send_nmi_ipi(int cpu)
			{
			if (cpu >= 0) {
			do_message_pass(cpu, PPC_MSG_NMI_IPI);
			} else {
			int c;

			for_each_online_cpu(c) {
			if (c == raw_smp_processor_id())
			continue;
			do_message_pass(c, PPC_MSG_NMI_IPI);
			}
			}
			}

			/*
			* - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS.
			* - fn is the target callback function.
			* - delay_us > 0 is the delay before giving up waiting for targets to
			* enter the handler, == 0 specifies indefinite delay.
			*/
			static int smp_send_nmi_ipi(int cpu, void (fn)(struct pt_regs ), u64 delay_us)
			{
			unsigned long flags;
			int me = raw_smp_processor_id();
			int ret = 1;

			BUG_ON(cpu == me);
			BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS);

			if (unlikely(!smp_ops))
			return 0;

			/* Take the nmi_ipi_busy count/lock with interrupts hard disabled */
			nmi_ipi_lock_start(&flags);
			while (nmi_ipi_busy_count) {
			nmi_ipi_unlock_end(&flags);
			cpu_relax();
			nmi_ipi_lock_start(&flags);
			}

			nmi_ipi_function = fn;

			if (cpu < 0) {
			/* ALL_OTHERS */
			cpumask_copy(&nmi_ipi_pending_mask, cpu_online_mask);
			cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
			} else {
			/* cpumask starts clear */
			cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);
			}
			nmi_ipi_busy_count++;
			nmi_ipi_unlock();

			do_smp_send_nmi_ipi(cpu);

			while (!cpumask_empty(&nmi_ipi_pending_mask)) {
			udelay(1);
			if (delay_us) {
			delay_us--;
			if (!delay_us)
			break;
			}
			}

			nmi_ipi_lock();
			if (!cpumask_empty(&nmi_ipi_pending_mask)) {
			/* Could not gather all CPUs */
			ret = 0;
			cpumask_clear(&nmi_ipi_pending_mask);
			}
			nmi_ipi_busy_count--;
			nmi_ipi_unlock_end(&flags);

			return ret;
			}
			#endif /* CONFIG_NMI_IPI */

	#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST		#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
	void tick_broadcast(const struct cpumask *mask)		void tick_broadcast(const struct cpumask *mask)
	{		{
	@@ -325,29 +503,22 @@ void tick_broadcast(const struct cpumask *mask)
	}		}
	#endif		#endif

	#if defined(CONFIG_DEBUGGER) \|\| defined(CONFIG_KEXEC_CORE)		#ifdef CONFIG_DEBUGGER
	void smp_send_debugger_break(void)		void debugger_ipi_callback(struct pt_regs *regs)
	{		{
	int cpu;		debugger_ipi(regs);
	int me = raw_smp_processor_id();		}

	if (unlikely(!smp_ops))
	return;

	for_each_online_cpu(cpu)		void smp_send_debugger_break(void)
	if (cpu != me)		{
	do_message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);		smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, debugger_ipi_callback, 1000000);
	}		}
	#endif		#endif

	#ifdef CONFIG_KEXEC_CORE		#ifdef CONFIG_KEXEC_CORE
	void crash_send_ipi(void (crash_ipi_callback)(struct pt_regs ))		void crash_send_ipi(void (crash_ipi_callback)(struct pt_regs ))
	{		{
	crash_ipi_function_ptr = crash_ipi_callback;		smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000);
	if (crash_ipi_callback) {
	mb();
	smp_send_debugger_break();
	}
	}		}
	#endif		#endif

arch/powerpc/platforms/cell/interrupt.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -211,7 +211,7 @@ void iic_request_IPIs(void)
	iic_request_ipi(PPC_MSG_CALL_FUNCTION);		iic_request_ipi(PPC_MSG_CALL_FUNCTION);
	iic_request_ipi(PPC_MSG_RESCHEDULE);		iic_request_ipi(PPC_MSG_RESCHEDULE);
	iic_request_ipi(PPC_MSG_TICK_BROADCAST);		iic_request_ipi(PPC_MSG_TICK_BROADCAST);
	iic_request_ipi(PPC_MSG_DEBUGGER_BREAK);		iic_request_ipi(PPC_MSG_NMI_IPI);
	}		}

	#endif /* CONFIG_SMP */		#endif /* CONFIG_SMP */

arch/powerpc/platforms/ps3/smp.c

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -77,7 +77,7 @@ static void __init ps3_smp_probe(void)
	BUILD_BUG_ON(PPC_MSG_CALL_FUNCTION != 0);		BUILD_BUG_ON(PPC_MSG_CALL_FUNCTION != 0);
	BUILD_BUG_ON(PPC_MSG_RESCHEDULE != 1);		BUILD_BUG_ON(PPC_MSG_RESCHEDULE != 1);
	BUILD_BUG_ON(PPC_MSG_TICK_BROADCAST != 2);		BUILD_BUG_ON(PPC_MSG_TICK_BROADCAST != 2);
	BUILD_BUG_ON(PPC_MSG_DEBUGGER_BREAK != 3);		BUILD_BUG_ON(PPC_MSG_NMI_IPI != 3);

	for (i = 0; i < MSG_COUNT; i++) {		for (i = 0; i < MSG_COUNT; i++) {
	result = ps3_event_receive_port_setup(cpu, &virqs[i]);		result = ps3_event_receive_port_setup(cpu, &virqs[i]);
	@@ -96,7 +96,7 @@ static void __init ps3_smp_probe(void)
	ps3_register_ipi_irq(cpu, virqs[i]);		ps3_register_ipi_irq(cpu, virqs[i]);
	}		}

	ps3_register_ipi_debug_brk(cpu, virqs[PPC_MSG_DEBUGGER_BREAK]);		ps3_register_ipi_debug_brk(cpu, virqs[PPC_MSG_NMI_IPI]);

	DBG(" <- %s:%d: (%d)\n", __func__, __LINE__, cpu);		DBG(" <- %s:%d: (%d)\n", __func__, __LINE__, cpu);
	}		}