Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2c5b7e1b authored by Andrea Arcangeli's avatar Andrea Arcangeli Committed by Linus Torvalds
Browse files

userfaultfd: avoid missing wakeups during refile in userfaultfd_read 



During the refile in userfaultfd_read both waitqueues could look empty to
the lockless wake_userfault().  Use a seqcount to prevent this false
negative that could leave an userfault blocked.

Signed-off-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 230c92a8

fs/userfaultfd.c

+24 −2
Original line number Diff line number Diff line
@@ -45,6 +45,8 @@ struct userfaultfd_ctx { 
	wait_queue_head_t fault_wqh;

	/* waitqueue head for the pseudo fd to wakeup poll/read */

	wait_queue_head_t fd_wqh;

	/* a refile sequence protected by fault_pending_wqh lock */

	struct seqcount refile_seq;

	/* pseudo fd refcounting */

	atomic_t refcount;

	/* userfaultfd syscall flags */
@@ -546,6 +548,15 @@ static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, 
		spin_lock(&ctx->fault_pending_wqh.lock);

		uwq = find_userfault(ctx);

		if (uwq) {

			/*

			 * Use a seqcount to repeat the lockless check

			 * in wake_userfault() to avoid missing

			 * wakeups because during the refile both

			 * waitqueue could become empty if this is the

			 * only userfault.

			 */

			write_seqcount_begin(&ctx->refile_seq);



			/*

			 * The fault_pending_wqh.lock prevents the uwq

			 * to disappear from under us.
@@ -570,6 +581,8 @@ static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, 
			list_del(&uwq->wq.task_list);

			__add_wait_queue(&ctx->fault_wqh, &uwq->wq);



			write_seqcount_end(&ctx->refile_seq);



			/* careful to always initialize msg if ret == 0 */

			*msg = uwq->msg;

			spin_unlock(&ctx->fault_pending_wqh.lock);
@@ -647,6 +660,9 @@ static void __wake_userfault(struct userfaultfd_ctx *ctx, 
static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,

					   struct userfaultfd_wake_range *range)

{

	unsigned seq;

	bool need_wakeup;



	/*

	 * To be sure waitqueue_active() is not reordered by the CPU

	 * before the pagetable update, use an explicit SMP memory
@@ -662,8 +678,13 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, 
	 * userfaults yet. So we take the spinlock only when we're

	 * sure we've userfaults to wake.

	 */

	if (waitqueue_active(&ctx->fault_pending_wqh) ||

	    waitqueue_active(&ctx->fault_wqh))

	do {

		seq = read_seqcount_begin(&ctx->refile_seq);

		need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||

			waitqueue_active(&ctx->fault_wqh);

		cond_resched();

	} while (read_seqcount_retry(&ctx->refile_seq, seq));

	if (need_wakeup)

		__wake_userfault(ctx, range);

}
@@ -1219,6 +1240,7 @@ static void init_once_userfaultfd_ctx(void *mem) 
	init_waitqueue_head(&ctx->fault_pending_wqh);

	init_waitqueue_head(&ctx->fault_wqh);

	init_waitqueue_head(&ctx->fd_wqh);

	seqcount_init(&ctx->refile_seq);

}



/**