Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7d0ae808 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu: Convert ACCESS_ONCE() to READ_ONCE() and WRITE_ONCE() 



This commit moves from the old ACCESS_ONCE() API to the new READ_ONCE()
and WRITE_ONCE() APIs.

Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck:  Updated to include kernel/torture.c as suggested by Jason Low. ]
parent 030bbdbf

include/linux/rculist.h

+3 −3
Original line number Diff line number Diff line
@@ -29,8 +29,8 @@ 
 */

static inline void INIT_LIST_HEAD_RCU(struct list_head *list)

{

	ACCESS_ONCE(list->next) = list;

	ACCESS_ONCE(list->prev) = list;

	WRITE_ONCE(list->next, list);

	WRITE_ONCE(list->prev, list);

}



/*
@@ -288,7 +288,7 @@ static inline void list_splice_init_rcu(struct list_head *list, 
#define list_first_or_null_rcu(ptr, type, member) \

({ \

	struct list_head *__ptr = (ptr); \

	struct list_head *__next = ACCESS_ONCE(__ptr->next); \

	struct list_head *__next = READ_ONCE(__ptr->next); \

	likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \

})

include/linux/rcupdate.h

+8 −8
Original line number Diff line number Diff line
@@ -364,8 +364,8 @@ extern struct srcu_struct tasks_rcu_exit_srcu; 
#define rcu_note_voluntary_context_switch(t) \

	do { \

		rcu_all_qs(); \

		if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \

			ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \

		if (READ_ONCE((t)->rcu_tasks_holdout)) \

			WRITE_ONCE((t)->rcu_tasks_holdout, false); \

	} while (0)

#else /* #ifdef CONFIG_TASKS_RCU */

#define TASKS_RCU(x) do { } while (0)
@@ -609,7 +609,7 @@ static inline void rcu_preempt_sleep_check(void) 


#define __rcu_access_pointer(p, space) \

({ \

	typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \

	typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \

	rcu_dereference_sparse(p, space); \

	((typeof(*p) __force __kernel *)(_________p1)); \

})
@@ -630,7 +630,7 @@ static inline void rcu_preempt_sleep_check(void) 


#define __rcu_access_index(p, space) \

({ \

	typeof(p) _________p1 = ACCESS_ONCE(p); \

	typeof(p) _________p1 = READ_ONCE(p); \

	rcu_dereference_sparse(p, space); \

	(_________p1); \

})
@@ -659,7 +659,7 @@ static inline void rcu_preempt_sleep_check(void) 
 */

#define lockless_dereference(p) \

({ \

	typeof(p) _________p1 = ACCESS_ONCE(p); \

	typeof(p) _________p1 = READ_ONCE(p); \

	smp_read_barrier_depends(); /* Dependency order vs. p above. */ \

	(_________p1); \

})
@@ -702,7 +702,7 @@ static inline void rcu_preempt_sleep_check(void) 
 * @p: The pointer to read

 *

 * Return the value of the specified RCU-protected pointer, but omit the

 * smp_read_barrier_depends() and keep the ACCESS_ONCE().  This is useful

 * smp_read_barrier_depends() and keep the READ_ONCE().  This is useful

 * when the value of this pointer is accessed, but the pointer is not

 * dereferenced, for example, when testing an RCU-protected pointer against

 * NULL.  Although rcu_access_pointer() may also be used in cases where
@@ -791,7 +791,7 @@ static inline void rcu_preempt_sleep_check(void) 
 * @p: The index to read

 *

 * Return the value of the specified RCU-protected index, but omit the

 * smp_read_barrier_depends() and keep the ACCESS_ONCE().  This is useful

 * smp_read_barrier_depends() and keep the READ_ONCE().  This is useful

 * when the value of this index is accessed, but the index is not

 * dereferenced, for example, when testing an RCU-protected index against

 * -1.  Although rcu_access_index() may also be used in cases where
@@ -827,7 +827,7 @@ static inline void rcu_preempt_sleep_check(void) 
 * @c: The conditions under which the dereference will take place

 *

 * Return the value of the specified RCU-protected pointer, but omit

 * both the smp_read_barrier_depends() and the ACCESS_ONCE().  This

 * both the smp_read_barrier_depends() and the READ_ONCE().  This

 * is useful in cases where update-side locks prevent the value of the

 * pointer from changing.  Please note that this primitive does -not-

 * prevent the compiler from repeating this reference or combining it

kernel/rcu/rcutorture.c

+1 −1
Original line number Diff line number Diff line
@@ -1413,7 +1413,7 @@ static int rcu_torture_barrier_cbs(void *arg) 
	do {

		wait_event(barrier_cbs_wq[myid],

			   (newphase =

			    ACCESS_ONCE(barrier_phase)) != lastphase ||

			    READ_ONCE(barrier_phase)) != lastphase ||

			   torture_must_stop());

		lastphase = newphase;

		smp_mb(); /* ensure barrier_phase load before ->call(). */

kernel/rcu/srcu.c

+5 −5
Original line number Diff line number Diff line
@@ -151,7 +151,7 @@ static unsigned long srcu_readers_seq_idx(struct srcu_struct *sp, int idx) 
	unsigned long t;



	for_each_possible_cpu(cpu) {

		t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]);

		t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]);

		sum += t;

	}

	return sum;
@@ -168,7 +168,7 @@ static unsigned long srcu_readers_active_idx(struct srcu_struct *sp, int idx) 
	unsigned long t;



	for_each_possible_cpu(cpu) {

		t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]);

		t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]);

		sum += t;

	}

	return sum;
@@ -265,8 +265,8 @@ static int srcu_readers_active(struct srcu_struct *sp) 
	unsigned long sum = 0;



	for_each_possible_cpu(cpu) {

		sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]);

		sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]);

		sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]);

		sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]);

	}

	return sum;

}
@@ -296,7 +296,7 @@ int __srcu_read_lock(struct srcu_struct *sp) 
{

	int idx;



	idx = ACCESS_ONCE(sp->completed) & 0x1;

	idx = READ_ONCE(sp->completed) & 0x1;

	preempt_disable();

	__this_cpu_inc(sp->per_cpu_ref->c[idx]);

	smp_mb(); /* B */  /* Avoid leaking the critical section. */

kernel/rcu/tiny_plugin.h

+7 −5
Original line number Diff line number Diff line
@@ -144,16 +144,17 @@ static void check_cpu_stall(struct rcu_ctrlblk *rcp) 
		return;

	rcp->ticks_this_gp++;

	j = jiffies;

	js = ACCESS_ONCE(rcp->jiffies_stall);

	js = READ_ONCE(rcp->jiffies_stall);

	if (rcp->rcucblist && ULONG_CMP_GE(j, js)) {

		pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld)\n",

		       rcp->name, rcp->ticks_this_gp, DYNTICK_TASK_EXIT_IDLE,

		       jiffies - rcp->gp_start, rcp->qlen);

		dump_stack();

		ACCESS_ONCE(rcp->jiffies_stall) = jiffies +

			3 * rcu_jiffies_till_stall_check() + 3;

		WRITE_ONCE(rcp->jiffies_stall,

			   jiffies + 3 * rcu_jiffies_till_stall_check() + 3);

	} else if (ULONG_CMP_GE(j, js)) {

		ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check();

		WRITE_ONCE(rcp->jiffies_stall,

			   jiffies + rcu_jiffies_till_stall_check());

	}

}
@@ -161,7 +162,8 @@ static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp) 
{

	rcp->ticks_this_gp = 0;

	rcp->gp_start = jiffies;

	ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check();

	WRITE_ONCE(rcp->jiffies_stall,

		   jiffies + rcu_jiffies_till_stall_check());

}



static void check_cpu_stalls(void)