Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 90f31d0e authored by Konstantin Khlebnikov's avatar Konstantin Khlebnikov Committed by Linus Torvalds
Browse files

mm: rcu-protected get_mm_exe_file() 



This patch removes mm->mmap_sem from mm->exe_file read side.
Also it kills dup_mm_exe_file() and moves exe_file duplication into
dup_mmap() where both mmap_sems are locked.

[akpm@linux-foundation.org: fix comment typo]
Signed-off-by: default avatarKonstantin Khlebnikov <khlebnikov@yandex-team.ru>
Cc: Davidlohr Bueso <dbueso@suse.de>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: "Paul E. McKenney" <paulmck@us.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 0ec62afe

fs/file.c

+1 −2
Original line number Original line Diff line number Diff line
@@ -638,8 +638,7 @@ static struct file *__fget(unsigned int fd, fmode_t mask) 
	file = fcheck_files(files, fd);

	file = fcheck_files(files, fd);

	if (file) {

	if (file) {

		/* File object ref couldn't be taken */

		/* File object ref couldn't be taken */

		if ((file->f_mode & mask) ||

		if ((file->f_mode & mask) || !get_file_rcu(file))

		    !atomic_long_inc_not_zero(&file->f_count))

			file = NULL;

			file = NULL;

	}

	}

	rcu_read_unlock();

	rcu_read_unlock();

include/linux/fs.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -870,6 +870,7 @@ static inline struct file *get_file(struct file *f) 
	atomic_long_inc(&f->f_count);

	atomic_long_inc(&f->f_count);

	return f;

	return f;

}

}

#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count)

#define fput_atomic(x)	atomic_long_add_unless(&(x)->f_count, -1, 1)

#define fput_atomic(x)	atomic_long_add_unless(&(x)->f_count, -1, 1)

#define file_count(x)	atomic_long_read(&(x)->f_count)

#define file_count(x)	atomic_long_read(&(x)->f_count)

include/linux/mm_types.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -429,7 +429,7 @@ struct mm_struct { 
#endif

#endif





	/* store ref to file /proc/<pid>/exe symlink points to */

	/* store ref to file /proc/<pid>/exe symlink points to */

	struct file *exe_file;

	struct file __rcu *exe_file;

#ifdef CONFIG_MMU_NOTIFIER

#ifdef CONFIG_MMU_NOTIFIER

	struct mmu_notifier_mm *mmu_notifier_mm;

	struct mmu_notifier_mm *mmu_notifier_mm;

#endif

#endif

kernel/fork.c

+37 −19
Original line number Original line Diff line number Diff line
@@ -403,6 +403,9 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) 
	 */

	 */

	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);

	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);





	/* No ordering required: file already has been exposed. */

	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));



	mm->total_vm = oldmm->total_vm;

	mm->total_vm = oldmm->total_vm;

	mm->shared_vm = oldmm->shared_vm;

	mm->shared_vm = oldmm->shared_vm;

	mm->exec_vm = oldmm->exec_vm;

	mm->exec_vm = oldmm->exec_vm;
@@ -528,7 +531,13 @@ static inline void mm_free_pgd(struct mm_struct *mm) 
	pgd_free(mm, mm->pgd);

	pgd_free(mm, mm->pgd);

}

}

#else

#else

#define dup_mmap(mm, oldmm)	(0)

static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)

{

	down_write(&oldmm->mmap_sem);

	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));

	up_write(&oldmm->mmap_sem);

	return 0;

}

#define mm_alloc_pgd(mm)	(0)

#define mm_alloc_pgd(mm)	(0)

#define mm_free_pgd(mm)

#define mm_free_pgd(mm)

#endif /* CONFIG_MMU */

#endif /* CONFIG_MMU */
@@ -697,35 +706,46 @@ void mmput(struct mm_struct *mm) 
}

}

EXPORT_SYMBOL_GPL(mmput);

EXPORT_SYMBOL_GPL(mmput);





/**

 * set_mm_exe_file - change a reference to the mm's executable file

 *

 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).

 *

 * Main users are mmput(), sys_execve() and sys_prctl(PR_SET_MM_MAP/EXE_FILE).

 * Callers prevent concurrent invocations: in mmput() nobody alive left,

 * in execve task is single-threaded, prctl holds mmap_sem exclusively.

 */

void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)

void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)

{

{

	struct file *old_exe_file = rcu_dereference_protected(mm->exe_file,

			!atomic_read(&mm->mm_users) || current->in_execve ||

			lockdep_is_held(&mm->mmap_sem));



	if (new_exe_file)

	if (new_exe_file)

		get_file(new_exe_file);

		get_file(new_exe_file);

	if (mm->exe_file)

	rcu_assign_pointer(mm->exe_file, new_exe_file);

		fput(mm->exe_file);

	if (old_exe_file)

	mm->exe_file = new_exe_file;

		fput(old_exe_file);

}

}





/**

 * get_mm_exe_file - acquire a reference to the mm's executable file

 *

 * Returns %NULL if mm has no associated executable file.

 * User must release file via fput().

 */

struct file *get_mm_exe_file(struct mm_struct *mm)

struct file *get_mm_exe_file(struct mm_struct *mm)

{

{

	struct file *exe_file;

	struct file *exe_file;





	/* We need mmap_sem to protect against races with removal of exe_file */

	rcu_read_lock();

	down_read(&mm->mmap_sem);

	exe_file = rcu_dereference(mm->exe_file);

	exe_file = mm->exe_file;

	if (exe_file && !get_file_rcu(exe_file))

	if (exe_file)

		exe_file = NULL;

		get_file(exe_file);

	rcu_read_unlock();

	up_read(&mm->mmap_sem);

	return exe_file;

	return exe_file;

}

}





static void dup_mm_exe_file(struct mm_struct *oldmm, struct mm_struct *newmm)

{

	/* It's safe to write the exe_file pointer without exe_file_lock because

	 * this is called during fork when the task is not yet in /proc */

	newmm->exe_file = get_mm_exe_file(oldmm);

}



/**

/**

 * get_task_mm - acquire a reference to the task's mm

 * get_task_mm - acquire a reference to the task's mm

 *

 *
@@ -887,8 +907,6 @@ static struct mm_struct *dup_mm(struct task_struct *tsk) 
	if (!mm_init(mm, tsk))

	if (!mm_init(mm, tsk))

		goto fail_nomem;

		goto fail_nomem;





	dup_mm_exe_file(oldmm, mm);



	err = dup_mmap(mm, oldmm);

	err = dup_mmap(mm, oldmm);

	if (err)

	if (err)

		goto free_pt;

		goto free_pt;