Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 88aa7cc6 authored by Yang Shi's avatar Yang Shi Committed by Linus Torvalds
Browse files

mm: introduce arg_lock to protect arg_start|end and env_start|end in mm_struct 

mmap_sem is on the hot path of kernel, and it very contended, but it is
abused too.  It is used to protect arg_start|end and evn_start|end when
reading /proc/$PID/cmdline and /proc/$PID/environ, but it doesn't make
sense since those proc files just expect to read 4 values atomically and
not related to VM, they could be set to arbitrary values by C/R.

And, the mmap_sem contention may cause unexpected issue like below:

INFO: task ps:14018 blocked for more than 120 seconds.
       Tainted: G            E 4.9.79-009.ali3000.alios7.x86_64 #1
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this
message.
 ps              D    0 14018      1 0x00000004
 Call Trace:
   schedule+0x36/0x80
   rwsem_down_read_failed+0xf0/0x150
   call_rwsem_down_read_failed+0x18/0x30
   down_read+0x20/0x40
   proc_pid_cmdline_read+0xd9/0x4e0
   __vfs_read+0x37/0x150
   vfs_read+0x96/0x130
   SyS_read+0x55/0xc0
   entry_SYSCALL_64_fastpath+0x1a/0xc5

Both Alexey Dobriyan and Michal Hocko suggested to use dedicated lock
for them to mitigate the abuse of mmap_sem.

So, introduce a new spinlock in mm_struct to protect the concurrent
access to arg_start|end, env_start|end and others, as well as replace
write map_sem to read to protect the race condition between prctl and
sys_brk which might break check_data_rlimit(), and makes prctl more
friendly to other VM operations.

This patch just eliminates the abuse of mmap_sem, but it can't resolve
the above hung task warning completely since the later
access_remote_vm() call needs acquire mmap_sem.  The mmap_sem
scalability issue will be solved in the future.

[yang.shi@linux.alibaba.com: add comment about mmap_sem and arg_lock]
  Link: http://lkml.kernel.org/r/1524077799-80690-1-git-send-email-yang.shi@linux.alibaba.com
Link: http://lkml.kernel.org/r/1523730291-109696-1-git-send-email-yang.shi@linux.alibaba.com


Signed-off-by: default avatarYang Shi <yang.shi@linux.alibaba.com>
Reviewed-by: default avatarCyrill Gorcunov <gorcunov@openvz.org>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mateusz Guzik <mguzik@redhat.com>
Cc: Kirill Tkhai <ktkhai@virtuozzo.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 05fec35e

fs/proc/base.c

+4 −4
Original line number Diff line number Diff line
@@ -239,12 +239,12 @@ static ssize_t proc_pid_cmdline_read(struct file *file, char __user *buf, 
		goto out_mmput;

	}



	down_read(&mm->mmap_sem);

	spin_lock(&mm->arg_lock);

	arg_start = mm->arg_start;

	arg_end = mm->arg_end;

	env_start = mm->env_start;

	env_end = mm->env_end;

	up_read(&mm->mmap_sem);

	spin_unlock(&mm->arg_lock);



	BUG_ON(arg_start > arg_end);

	BUG_ON(env_start > env_end);
@@ -927,10 +927,10 @@ static ssize_t environ_read(struct file *file, char __user *buf, 
	if (!mmget_not_zero(mm))

		goto free;



	down_read(&mm->mmap_sem);

	spin_lock(&mm->arg_lock);

	env_start = mm->env_start;

	env_end = mm->env_end;

	up_read(&mm->mmap_sem);

	spin_unlock(&mm->arg_lock);



	while (count > 0) {

		size_t this_len, max_len;

include/linux/mm_types.h

+2 −0
Original line number Diff line number Diff line
@@ -413,6 +413,8 @@ struct mm_struct { 
	unsigned long exec_vm;		/* VM_EXEC & ~VM_WRITE & ~VM_STACK */

	unsigned long stack_vm;		/* VM_STACK */

	unsigned long def_flags;



	spinlock_t arg_lock; /* protect the below fields */

	unsigned long start_code, end_code, start_data, end_data;

	unsigned long start_brk, brk, start_stack;

	unsigned long arg_start, arg_end, env_start, env_end;

kernel/fork.c

+1 −0
Original line number Diff line number Diff line
@@ -899,6 +899,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, 
	mm->pinned_vm = 0;

	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));

	spin_lock_init(&mm->page_table_lock);

	spin_lock_init(&mm->arg_lock);

	mm_init_cpumask(mm);

	mm_init_aio(mm);

	mm_init_owner(mm, p);

kernel/sys.c

+8 −2
Original line number Diff line number Diff line
@@ -2018,7 +2018,11 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data 
			return error;

	}



	down_write(&mm->mmap_sem);

	/*

	 * arg_lock protects concurent updates but we still need mmap_sem for

	 * read to exclude races with sys_brk.

	 */

	down_read(&mm->mmap_sem);



	/*

	 * We don't validate if these members are pointing to
@@ -2032,6 +2036,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data 
	 *    to any problem in kernel itself

	 */



	spin_lock(&mm->arg_lock);

	mm->start_code	= prctl_map.start_code;

	mm->end_code	= prctl_map.end_code;

	mm->start_data	= prctl_map.start_data;
@@ -2043,6 +2048,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data 
	mm->arg_end	= prctl_map.arg_end;

	mm->env_start	= prctl_map.env_start;

	mm->env_end	= prctl_map.env_end;

	spin_unlock(&mm->arg_lock);



	/*

	 * Note this update of @saved_auxv is lockless thus
@@ -2055,7 +2061,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data 
	if (prctl_map.auxv_size)

		memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv));



	up_write(&mm->mmap_sem);

	up_read(&mm->mmap_sem);

	return 0;

}

#endif /* CONFIG_CHECKPOINT_RESTORE */

mm/init-mm.c

+1 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ struct mm_struct init_mm = { 
	.mm_count	= ATOMIC_INIT(1),

	.mmap_sem	= __RWSEM_INITIALIZER(init_mm.mmap_sem),

	.page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),

	.arg_lock	=  __SPIN_LOCK_UNLOCKED(init_mm.arg_lock),

	.mmlist		= LIST_HEAD_INIT(init_mm.mmlist),

	.user_ns	= &init_user_ns,

	INIT_MM_CONTEXT(init_mm)