kernel: use lockless list for smp_call_function_single (6897fc22) · Commits · e / devices / android_kernel_xiaomi_markw

include/linux/blkdev.h

+1 −4

Original line number	Original line	Diff line number	Diff line
	@@ -95,10 +95,7 @@ enum rq_cmd_type_bits {
	* as well!		* as well!
	*/		*/
	struct request {		struct request {
	union {
	struct list_head queuelist;		struct list_head queuelist;
	struct llist_node ll_list;
	};
	union {		union {
	struct call_single_data csd;		struct call_single_data csd;
	struct work_struct mq_flush_data;		struct work_struct mq_flush_data;

include/linux/smp.h

+5 −1

Original line number	Original line	Diff line number	Diff line
	@@ -11,12 +11,16 @@
	#include <linux/list.h>		#include <linux/list.h>
	#include <linux/cpumask.h>		#include <linux/cpumask.h>
	#include <linux/init.h>		#include <linux/init.h>
			#include <linux/llist.h>

	extern void cpu_idle(void);		extern void cpu_idle(void);

	typedef void (smp_call_func_t)(void info);		typedef void (smp_call_func_t)(void info);
	struct call_single_data {		struct call_single_data {
			union {
	struct list_head list;		struct list_head list;
			struct llist_node llist;
			};
	smp_call_func_t func;		smp_call_func_t func;
	void *info;		void *info;
	u16 flags;		u16 flags;

kernel/smp.c

+13 −38

Original line number	Original line	Diff line number	Diff line
	@@ -28,12 +28,7 @@ struct call_function_data {

	static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);		static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);

	struct call_single_queue {		static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
	struct list_head list;
	raw_spinlock_t lock;
	};

	static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);

	static int		static int
	hotplug_cfd(struct notifier_block nfb, unsigned long action, void hcpu)		hotplug_cfd(struct notifier_block nfb, unsigned long action, void hcpu)
	@@ -85,12 +80,8 @@ void __init call_function_init(void)
	void cpu = (void )(long)smp_processor_id();		void cpu = (void )(long)smp_processor_id();
	int i;		int i;

	for_each_possible_cpu(i) {		for_each_possible_cpu(i)
	struct call_single_queue *q = &per_cpu(call_single_queue, i);		init_llist_head(&per_cpu(call_single_queue, i));

	raw_spin_lock_init(&q->lock);
	INIT_LIST_HEAD(&q->list);
	}

	hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);		hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
	register_cpu_notifier(&hotplug_cfd_notifier);		register_cpu_notifier(&hotplug_cfd_notifier);
	@@ -141,18 +132,9 @@ static void csd_unlock(struct call_single_data *csd)
	*/		*/
	static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)		static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
	{		{
	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
	unsigned long flags;
	int ipi;

	if (wait)		if (wait)
	csd->flags \|= CSD_FLAG_WAIT;		csd->flags \|= CSD_FLAG_WAIT;

	raw_spin_lock_irqsave(&dst->lock, flags);
	ipi = list_empty(&dst->list);
	list_add_tail(&csd->list, &dst->list);
	raw_spin_unlock_irqrestore(&dst->lock, flags);

	/*		/*
	* The list addition should be visible before sending the IPI		* The list addition should be visible before sending the IPI
	* handler locks the list to pull the entry off it because of		* handler locks the list to pull the entry off it because of
	@@ -164,7 +146,7 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
	* locking and barrier primitives. Generic code isn't really		* locking and barrier primitives. Generic code isn't really
	* equipped to do the right thing...		* equipped to do the right thing...
	*/		*/
	if (ipi)		if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
	arch_send_call_function_single_ipi(cpu);		arch_send_call_function_single_ipi(cpu);

	if (wait)		if (wait)
	@@ -177,27 +159,26 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
	*/		*/
	void generic_smp_call_function_single_interrupt(void)		void generic_smp_call_function_single_interrupt(void)
	{		{
	struct call_single_queue *q = &__get_cpu_var(call_single_queue);		struct llist_node entry, next;
	LIST_HEAD(list);

	/*		/*
	* Shouldn't receive this interrupt on a cpu that is not yet online.		* Shouldn't receive this interrupt on a cpu that is not yet online.
	*/		*/
	WARN_ON_ONCE(!cpu_online(smp_processor_id()));		WARN_ON_ONCE(!cpu_online(smp_processor_id()));

	raw_spin_lock(&q->lock);		entry = llist_del_all(&__get_cpu_var(call_single_queue));
	list_replace_init(&q->list, &list);		entry = llist_reverse_order(entry);
	raw_spin_unlock(&q->lock);

	while (!list_empty(&list)) {		while (entry) {
	struct call_single_data *csd;		struct call_single_data *csd;

	csd = list_entry(list.next, struct call_single_data, list);		next = entry->next;
	list_del(&csd->list);

			csd = llist_entry(entry, struct call_single_data, llist);
	csd->func(csd->info);		csd->func(csd->info);

	csd_unlock(csd);		csd_unlock(csd);

			entry = next;
	}		}
	}		}

	@@ -411,17 +392,11 @@ void smp_call_function_many(const struct cpumask *mask,

	for_each_cpu(cpu, cfd->cpumask) {		for_each_cpu(cpu, cfd->cpumask) {
	struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);		struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);
	struct call_single_queue *dst =
	&per_cpu(call_single_queue, cpu);
	unsigned long flags;

	csd_lock(csd);		csd_lock(csd);
	csd->func = func;		csd->func = func;
	csd->info = info;		csd->info = info;
			llist_add(&csd->llist, &per_cpu(call_single_queue, cpu));
	raw_spin_lock_irqsave(&dst->lock, flags);
	list_add_tail(&csd->list, &dst->list);
	raw_spin_unlock_irqrestore(&dst->lock, flags);
	}		}

	/* Send a message to all CPUs in the map */		/* Send a message to all CPUs in the map */