Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2e8c28c2 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Move rcu_seq_start() and friends to rcu.h 



This commit moves rcu_seq_start(), rcu_seq_end(), rcu_seq_snap(),
and rcu_seq_done() from kernel/rcu/tree.c to kernel/rcu/rcu.h.
This will allow SRCU to use these functions, which in turn will
allow SRCU to move from a single global callback queue to a
per-CPU callback queue.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parent bdcabf4c

kernel/rcu/rcu.h

+40 −0
Original line number Diff line number Diff line
@@ -56,6 +56,46 @@ 
#define DYNTICK_TASK_EXIT_IDLE	   (DYNTICK_TASK_NEST_VALUE + \

				    DYNTICK_TASK_FLAG)





/*

 * Grace-period counter management.

 */



/* Adjust sequence number for start of update-side operation. */

static inline void rcu_seq_start(unsigned long *sp)

{

	WRITE_ONCE(*sp, *sp + 1);

	smp_mb(); /* Ensure update-side operation after counter increment. */

	WARN_ON_ONCE(!(*sp & 0x1));

}



/* Adjust sequence number for end of update-side operation. */

static inline void rcu_seq_end(unsigned long *sp)

{

	smp_mb(); /* Ensure update-side operation before counter increment. */

	WRITE_ONCE(*sp, *sp + 1);

	WARN_ON_ONCE(*sp & 0x1);

}



/* Take a snapshot of the update side's sequence number. */

static inline unsigned long rcu_seq_snap(unsigned long *sp)

{

	unsigned long s;



	s = (READ_ONCE(*sp) + 3) & ~0x1;

	smp_mb(); /* Above access must not bleed into critical section. */

	return s;

}



/*

 * Given a snapshot from rcu_seq_snap(), determine whether or not a

 * full update-side operation has occurred.

 */

static inline bool rcu_seq_done(unsigned long *sp, unsigned long s)

{

	return ULONG_CMP_GE(READ_ONCE(*sp), s);

}



/*

 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally

 * by call_rcu() and rcu callback execution, and are therefore not part of the

kernel/rcu/tree.c

+0 −35
Original line number Diff line number Diff line
@@ -3409,41 +3409,6 @@ void cond_synchronize_sched(unsigned long oldstate) 
}

EXPORT_SYMBOL_GPL(cond_synchronize_sched);



/* Adjust sequence number for start of update-side operation. */

static void rcu_seq_start(unsigned long *sp)

{

	WRITE_ONCE(*sp, *sp + 1);

	smp_mb(); /* Ensure update-side operation after counter increment. */

	WARN_ON_ONCE(!(*sp & 0x1));

}



/* Adjust sequence number for end of update-side operation. */

static void rcu_seq_end(unsigned long *sp)

{

	smp_mb(); /* Ensure update-side operation before counter increment. */

	WRITE_ONCE(*sp, *sp + 1);

	WARN_ON_ONCE(*sp & 0x1);

}



/* Take a snapshot of the update side's sequence number. */

static unsigned long rcu_seq_snap(unsigned long *sp)

{

	unsigned long s;



	s = (READ_ONCE(*sp) + 3) & ~0x1;

	smp_mb(); /* Above access must not bleed into critical section. */

	return s;

}



/*

 * Given a snapshot from rcu_seq_snap(), determine whether or not a

 * full update-side operation has occurred.

 */

static bool rcu_seq_done(unsigned long *sp, unsigned long s)

{

	return ULONG_CMP_GE(READ_ONCE(*sp), s);

}



/*

 * Check to see if there is any immediate RCU-related work to be done

 * by the current CPU, for the specified type of RCU, returning 1 if so.