Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 207205a2 authored by Eric Dumazet's avatar Eric Dumazet Committed by Linus Torvalds
Browse files

kthread: NUMA aware kthread_create_on_node() 



All kthreads being created from a single helper task, they all use memory
from a single node for their kernel stack and task struct.

This patch suite creates kthread_create_on_node(), adding a 'cpu' parameter
to parameters already used by kthread_create().

This parameter serves in allocating memory for the new kthread on its
memory node if possible.

Signed-off-by: default avatarEric Dumazet <eric.dumazet@gmail.com>
Acked-by: default avatarDavid S. Miller <davem@davemloft.net>
Reviewed-by: default avatarAndi Kleen <ak@linux.intel.com>
Acked-by: default avatarRusty Russell <rusty@rustcorp.com.au>
Cc: Tejun Heo <tj@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: David Howells <dhowells@redhat.com>
Cc: <linux-arch@vger.kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b6a84016

include/linux/kthread.h

+10 −4
Original line number Original line Diff line number Diff line
@@ -4,10 +4,15 @@ 
#include <linux/err.h>

#include <linux/err.h>

#include <linux/sched.h>

#include <linux/sched.h>





struct task_struct *kthread_create(int (*threadfn)(void *data),

struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),

					   void *data,

					   void *data,

					   int node,

					   const char namefmt[], ...)

					   const char namefmt[], ...)

	__attribute__((format(printf, 3, 4)));

	__attribute__((format(printf, 4, 5)));



#define kthread_create(threadfn, data, namefmt, arg...) \

	kthread_create_on_node(threadfn, data, -1, namefmt, ##arg)







/**

/**

 * kthread_run - create and wake a thread.

 * kthread_run - create and wake a thread.
@@ -34,6 +39,7 @@ void *kthread_data(struct task_struct *k); 




int kthreadd(void *unused);

int kthreadd(void *unused);

extern struct task_struct *kthreadd_task;

extern struct task_struct *kthreadd_task;

extern int tsk_fork_get_node(struct task_struct *tsk);





/*

/*

 * Simple work processor based on kthread.

 * Simple work processor based on kthread.

include/linux/sched.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -1471,6 +1471,7 @@ struct task_struct { 
#ifdef CONFIG_NUMA

#ifdef CONFIG_NUMA

	struct mempolicy *mempolicy;	/* Protected by alloc_lock */

	struct mempolicy *mempolicy;	/* Protected by alloc_lock */

	short il_next;

	short il_next;

	short pref_node_fork;

#endif

#endif

	atomic_t fs_excl;	/* holding fs exclusive resources */

	atomic_t fs_excl;	/* holding fs exclusive resources */

	struct rcu_head rcu;

	struct rcu_head rcu;

kernel/fork.c

+2 −1
Original line number Original line Diff line number Diff line
@@ -40,6 +40,7 @@ 
#include <linux/tracehook.h>

#include <linux/tracehook.h>

#include <linux/futex.h>

#include <linux/futex.h>

#include <linux/compat.h>

#include <linux/compat.h>

#include <linux/kthread.h>

#include <linux/task_io_accounting_ops.h>

#include <linux/task_io_accounting_ops.h>

#include <linux/rcupdate.h>

#include <linux/rcupdate.h>

#include <linux/ptrace.h>

#include <linux/ptrace.h>
@@ -254,7 +255,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) 
	struct task_struct *tsk;

	struct task_struct *tsk;

	struct thread_info *ti;

	struct thread_info *ti;

	unsigned long *stackend;

	unsigned long *stackend;

	int node = numa_node_id();

	int node = tsk_fork_get_node(orig);

	int err;

	int err;





	prepare_to_copy(orig);

	prepare_to_copy(orig);

kernel/kthread.c

+25 −6
Original line number Original line Diff line number Diff line
@@ -27,6 +27,7 @@ struct kthread_create_info 
	/* Information passed to kthread() from kthreadd. */

	/* Information passed to kthread() from kthreadd. */

	int (*threadfn)(void *data);

	int (*threadfn)(void *data);

	void *data;

	void *data;

	int node;





	/* Result passed back to kthread_create() from kthreadd. */

	/* Result passed back to kthread_create() from kthreadd. */

	struct task_struct *result;

	struct task_struct *result;
@@ -98,10 +99,23 @@ static int kthread(void *_create) 
	do_exit(ret);

	do_exit(ret);

}

}





/* called from do_fork() to get node information for about to be created task */

int tsk_fork_get_node(struct task_struct *tsk)

{

#ifdef CONFIG_NUMA

	if (tsk == kthreadd_task)

		return tsk->pref_node_fork;

#endif

	return numa_node_id();

}



static void create_kthread(struct kthread_create_info *create)

static void create_kthread(struct kthread_create_info *create)

{

{

	int pid;

	int pid;





#ifdef CONFIG_NUMA

	current->pref_node_fork = create->node;

#endif

	/* We want our own signal handler (we take no signals by default). */

	/* We want our own signal handler (we take no signals by default). */

	pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);

	pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);

	if (pid < 0) {

	if (pid < 0) {
@@ -111,15 +125,18 @@ static void create_kthread(struct kthread_create_info *create) 
}

}





/**

/**

 * kthread_create - create a kthread.

 * kthread_create_on_node - create a kthread.

 * @threadfn: the function to run until signal_pending(current).

 * @threadfn: the function to run until signal_pending(current).

 * @data: data ptr for @threadfn.

 * @data: data ptr for @threadfn.

 * @node: memory node number.

 * @namefmt: printf-style name for the thread.

 * @namefmt: printf-style name for the thread.

 *

 *

 * Description: This helper function creates and names a kernel

 * Description: This helper function creates and names a kernel

 * thread.  The thread will be stopped: use wake_up_process() to start

 * thread.  The thread will be stopped: use wake_up_process() to start

 * it.  See also kthread_run().

 * it.  See also kthread_run().

 *

 *

 * If thread is going to be bound on a particular cpu, give its node

 * in @node, to get NUMA affinity for kthread stack, or else give -1.

 * When woken, the thread will run @threadfn() with @data as its

 * When woken, the thread will run @threadfn() with @data as its

 * argument. @threadfn() can either call do_exit() directly if it is a

 * argument. @threadfn() can either call do_exit() directly if it is a

 * standalone thread for which noone will call kthread_stop(), or

 * standalone thread for which noone will call kthread_stop(), or
@@ -129,8 +146,9 @@ static void create_kthread(struct kthread_create_info *create) 
 *

 *

 * Returns a task_struct or ERR_PTR(-ENOMEM).

 * Returns a task_struct or ERR_PTR(-ENOMEM).

 */

 */

struct task_struct *kthread_create(int (*threadfn)(void *data),

struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),

					   void *data,

					   void *data,

					   int node,

					   const char namefmt[],

					   const char namefmt[],

					   ...)

					   ...)

{

{
@@ -138,6 +156,7 @@ struct task_struct *kthread_create(int (*threadfn)(void *data), 




	create.threadfn = threadfn;

	create.threadfn = threadfn;

	create.data = data;

	create.data = data;

	create.node = node;

	init_completion(&create.done);

	init_completion(&create.done);





	spin_lock(&kthread_create_lock);

	spin_lock(&kthread_create_lock);
@@ -164,7 +183,7 @@ struct task_struct *kthread_create(int (*threadfn)(void *data), 
	}

	}

	return create.result;

	return create.result;

}

}

EXPORT_SYMBOL(kthread_create);

EXPORT_SYMBOL(kthread_create_on_node);





/**

/**

 * kthread_bind - bind a just-created kthread to a cpu.

 * kthread_bind - bind a just-created kthread to a cpu.