Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cc44ca84 authored by Oleg Nesterov's avatar Oleg Nesterov Committed by Paul E. McKenney
Browse files

rcu: Create rcu_sync infrastructure 



The rcu_sync infrastructure can be thought of as infrastructure to be
used to implement reader-writer primitives having extremely lightweight
readers during times when there are no writers.  The first use is in
the percpu_rwsem used by the VFS subsystem.

This infrastructure is functionally equivalent to

        struct rcu_sync_struct {
                atomic_t counter;
        };

	/* Check possibility of fast-path read-side operations. */
        static inline bool rcu_sync_is_idle(struct rcu_sync_struct *rss)
        {
                return atomic_read(&rss->counter) == 0;
        }

	/* Tell readers to use slowpaths. */
        static inline void rcu_sync_enter(struct rcu_sync_struct *rss)
        {
                atomic_inc(&rss->counter);
                synchronize_sched();
        }

	/* Allow readers to once again use fastpaths. */
        static inline void rcu_sync_exit(struct rcu_sync_struct *rss)
        {
                synchronize_sched();
                atomic_dec(&rss->counter);
        }

The main difference is that it records the state and only calls
synchronize_sched() if required.  At least some of the calls to
synchronize_sched() will be optimized away when rcu_sync_enter() and
rcu_sync_exit() are invoked repeatedly in quick succession.

Signed-off-by: default avatarOleg Nesterov <oleg@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: default avatarJosh Triplett <josh@joshtriplett.org>
parent 3836f533

include/linux/rcu_sync.h

0 → 100644
+94 −0
Original line number Original line Diff line number Diff line 
/*

 * RCU-based infrastructure for lightweight reader-writer locking

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, you can access it online at

 * http://www.gnu.org/licenses/gpl-2.0.html.

 *

 * Copyright (c) 2015, Red Hat, Inc.

 *

 * Author: Oleg Nesterov <oleg@redhat.com>

 */



#ifndef _LINUX_RCU_SYNC_H_

#define _LINUX_RCU_SYNC_H_



#include <linux/wait.h>

#include <linux/rcupdate.h>



/* Structure to mediate between updaters and fastpath-using readers.  */

struct rcu_sync {

	int			gp_state;

	int			gp_count;

	wait_queue_head_t	gp_wait;



	int			cb_state;

	struct rcu_head		cb_head;



	void (*sync)(void);

	void (*call)(struct rcu_head *, void (*)(struct rcu_head *));

};



#define ___RCU_SYNC_INIT(name)						\

	.gp_state = 0,							\

	.gp_count = 0,							\

	.gp_wait = __WAIT_QUEUE_HEAD_INITIALIZER(name.gp_wait),		\

	.cb_state = 0



#define __RCU_SCHED_SYNC_INIT(name) {					\

	___RCU_SYNC_INIT(name),						\

	.sync = synchronize_sched,					\

	.call = call_rcu_sched,						\

}



#define __RCU_BH_SYNC_INIT(name) {					\

	___RCU_SYNC_INIT(name),						\

	.sync = synchronize_rcu_bh,					\

	.call = call_rcu_bh,						\

}



#define __RCU_SYNC_INIT(name) {						\

	___RCU_SYNC_INIT(name),						\

	.sync = synchronize_rcu,					\

	.call = call_rcu,						\

}



#define DEFINE_RCU_SCHED_SYNC(name)					\

	struct rcu_sync name = __RCU_SCHED_SYNC_INIT(name)



#define DEFINE_RCU_BH_SYNC(name)					\

	struct rcu_sync name = __RCU_BH_SYNC_INIT(name)



#define DEFINE_RCU_SYNC(name)						\

	struct rcu_sync name = __RCU_SYNC_INIT(name)



/**

 * rcu_sync_is_idle() - Are readers permitted to use their fastpaths?

 * @rsp: Pointer to rcu_sync structure to use for synchronization

 *

 * Returns true if readers are permitted to use their fastpaths.

 * Must be invoked within an RCU read-side critical section whose

 * flavor matches that of the rcu_sync struture.

 */

static inline bool rcu_sync_is_idle(struct rcu_sync *rsp)

{

	return !rsp->gp_state; /* GP_IDLE */

}



enum rcu_sync_type { RCU_SYNC, RCU_SCHED_SYNC, RCU_BH_SYNC };



extern void rcu_sync_init(struct rcu_sync *, enum rcu_sync_type);

extern void rcu_sync_enter(struct rcu_sync *);

extern void rcu_sync_exit(struct rcu_sync *);



#endif /* _LINUX_RCU_SYNC_H_ */

kernel/rcu/Makefile

+1 −1
Original line number Original line Diff line number Diff line 
obj-y += update.o

obj-y += update.o sync.o

obj-$(CONFIG_SRCU) += srcu.o

obj-$(CONFIG_SRCU) += srcu.o

obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o

obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o

obj-$(CONFIG_TREE_RCU) += tree.o

obj-$(CONFIG_TREE_RCU) += tree.o

kernel/rcu/sync.c

0 → 100644
+175 −0
Original line number Original line Diff line number Diff line 
/*

 * RCU-based infrastructure for lightweight reader-writer locking

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, you can access it online at

 * http://www.gnu.org/licenses/gpl-2.0.html.

 *

 * Copyright (c) 2015, Red Hat, Inc.

 *

 * Author: Oleg Nesterov <oleg@redhat.com>

 */



#include <linux/rcu_sync.h>

#include <linux/sched.h>



enum { GP_IDLE = 0, GP_PENDING, GP_PASSED };

enum { CB_IDLE = 0, CB_PENDING, CB_REPLAY };



#define	rss_lock	gp_wait.lock



/**

 * rcu_sync_init() - Initialize an rcu_sync structure

 * @rsp: Pointer to rcu_sync structure to be initialized

 * @type: Flavor of RCU with which to synchronize rcu_sync structure

 */

void rcu_sync_init(struct rcu_sync *rsp, enum rcu_sync_type type)

{

	memset(rsp, 0, sizeof(*rsp));

	init_waitqueue_head(&rsp->gp_wait);



	switch (type) {

	case RCU_SYNC:

		rsp->sync = synchronize_rcu;

		rsp->call = call_rcu;

		break;



	case RCU_SCHED_SYNC:

		rsp->sync = synchronize_sched;

		rsp->call = call_rcu_sched;

		break;



	case RCU_BH_SYNC:

		rsp->sync = synchronize_rcu_bh;

		rsp->call = call_rcu_bh;

		break;

	}

}



/**

 * rcu_sync_enter() - Force readers onto slowpath

 * @rsp: Pointer to rcu_sync structure to use for synchronization

 *

 * This function is used by updaters who need readers to make use of

 * a slowpath during the update.  After this function returns, all

 * subsequent calls to rcu_sync_is_idle() will return false, which

 * tells readers to stay off their fastpaths.  A later call to

 * rcu_sync_exit() re-enables reader slowpaths.

 *

 * When called in isolation, rcu_sync_enter() must wait for a grace

 * period, however, closely spaced calls to rcu_sync_enter() can

 * optimize away the grace-period wait via a state machine implemented

 * by rcu_sync_enter(), rcu_sync_exit(), and rcu_sync_func().

 */

void rcu_sync_enter(struct rcu_sync *rsp)

{

	bool need_wait, need_sync;



	spin_lock_irq(&rsp->rss_lock);

	need_wait = rsp->gp_count++;

	need_sync = rsp->gp_state == GP_IDLE;

	if (need_sync)

		rsp->gp_state = GP_PENDING;

	spin_unlock_irq(&rsp->rss_lock);



	BUG_ON(need_wait && need_sync);



	if (need_sync) {

		rsp->sync();

		rsp->gp_state = GP_PASSED;

		wake_up_all(&rsp->gp_wait);

	} else if (need_wait) {

		wait_event(rsp->gp_wait, rsp->gp_state == GP_PASSED);

	} else {

		/*

		 * Possible when there's a pending CB from a rcu_sync_exit().

		 * Nobody has yet been allowed the 'fast' path and thus we can

		 * avoid doing any sync(). The callback will get 'dropped'.

		 */

		BUG_ON(rsp->gp_state != GP_PASSED);

	}

}



/**

 * rcu_sync_func() - Callback function managing reader access to fastpath

 * @rsp: Pointer to rcu_sync structure to use for synchronization

 *

 * This function is passed to one of the call_rcu() functions by

 * rcu_sync_exit(), so that it is invoked after a grace period following the

 * that invocation of rcu_sync_exit().  It takes action based on events that

 * have taken place in the meantime, so that closely spaced rcu_sync_enter()

 * and rcu_sync_exit() pairs need not wait for a grace period.

 *

 * If another rcu_sync_enter() is invoked before the grace period

 * ended, reset state to allow the next rcu_sync_exit() to let the

 * readers back onto their fastpaths (after a grace period).  If both

 * another rcu_sync_enter() and its matching rcu_sync_exit() are invoked

 * before the grace period ended, re-invoke call_rcu() on behalf of that

 * rcu_sync_exit().  Otherwise, set all state back to idle so that readers

 * can again use their fastpaths.

 */

static void rcu_sync_func(struct rcu_head *rcu)

{

	struct rcu_sync *rsp = container_of(rcu, struct rcu_sync, cb_head);

	unsigned long flags;



	BUG_ON(rsp->gp_state != GP_PASSED);

	BUG_ON(rsp->cb_state == CB_IDLE);



	spin_lock_irqsave(&rsp->rss_lock, flags);

	if (rsp->gp_count) {

		/*

		 * A new rcu_sync_begin() has happened; drop the callback.

		 */

		rsp->cb_state = CB_IDLE;

	} else if (rsp->cb_state == CB_REPLAY) {

		/*

		 * A new rcu_sync_exit() has happened; requeue the callback

		 * to catch a later GP.

		 */

		rsp->cb_state = CB_PENDING;

		rsp->call(&rsp->cb_head, rcu_sync_func);

	} else {

		/*

		 * We're at least a GP after rcu_sync_exit(); eveybody will now

		 * have observed the write side critical section. Let 'em rip!.

		 */

		rsp->cb_state = CB_IDLE;

		rsp->gp_state = GP_IDLE;

	}

	spin_unlock_irqrestore(&rsp->rss_lock, flags);

}



/**

 * rcu_sync_exit() - Allow readers back onto fast patch after grace period

 * @rsp: Pointer to rcu_sync structure to use for synchronization

 *

 * This function is used by updaters who have completed, and can therefore

 * now allow readers to make use of their fastpaths after a grace period

 * has elapsed.  After this grace period has completed, all subsequent

 * calls to rcu_sync_is_idle() will return true, which tells readers that

 * they can once again use their fastpaths.

 */

void rcu_sync_exit(struct rcu_sync *rsp)

{

	spin_lock_irq(&rsp->rss_lock);

	if (!--rsp->gp_count) {

		if (rsp->cb_state == CB_IDLE) {

			rsp->cb_state = CB_PENDING;

			rsp->call(&rsp->cb_head, rcu_sync_func);

		} else if (rsp->cb_state == CB_PENDING) {

			rsp->cb_state = CB_REPLAY;

		}

	}

	spin_unlock_irq(&rsp->rss_lock);

}