Preempt-RCU: reorganize RCU code into rcuclassic.c and rcupdate.c (01c1c660) · Commits · e / devices / android_kernel_sony_msm8998

include/linux/rcuclassic.h

0 → 100644

+161 −0

Original line number	Diff line number	Diff line
		/*
		* Read-Copy Update mechanism for mutual exclusion (classic version)
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation; either version 2 of the License, or
		* (at your option) any later version.
		*
		* This program is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with this program; if not, write to the Free Software
		* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
		*
		* Copyright IBM Corporation, 2001
		*
		* Author: Dipankar Sarma <dipankar@in.ibm.com>
		*
		* Based on the original work by Paul McKenney <paulmck@us.ibm.com>
		* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
		* Papers:
		* http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
		* http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
		*
		* For detailed explanation of Read-Copy Update mechanism see -
		* Documentation/RCU
		*
		*/

		#ifndef __LINUX_RCUCLASSIC_H
		#define __LINUX_RCUCLASSIC_H

		#ifdef __KERNEL__

		#include <linux/cache.h>
		#include <linux/spinlock.h>
		#include <linux/threads.h>
		#include <linux/percpu.h>
		#include <linux/cpumask.h>
		#include <linux/seqlock.h>


		/* Global control variables for rcupdate callback mechanism. */
		struct rcu_ctrlblk {
		long cur; /* Current batch number. */
		long completed; /* Number of the last completed batch */
		int next_pending; /* Is the next batch already waiting? */

		int signaled;

		spinlock_t lock ____cacheline_internodealigned_in_smp;
		cpumask_t cpumask; /* CPUs that need to switch in order */
		/* for current batch to proceed. */
		} ____cacheline_internodealigned_in_smp;

		/* Is batch a before batch b ? */
		static inline int rcu_batch_before(long a, long b)
		{
		return (a - b) < 0;
		}

		/* Is batch a after batch b ? */
		static inline int rcu_batch_after(long a, long b)
		{
		return (a - b) > 0;
		}

		/*
		* Per-CPU data for Read-Copy UPdate.
		* nxtlist - new callbacks are added here
		* curlist - current batch for which quiescent cycle started if any
		*/
		struct rcu_data {
		/* 1) quiescent state handling : */
		long quiescbatch; /* Batch # for grace period */
		int passed_quiesc; /* User-mode/idle loop etc. */
		int qs_pending; /* core waits for quiesc state */

		/* 2) batch handling */
		long batch; /* Batch # for current RCU batch */
		struct rcu_head *nxtlist;
		struct rcu_head **nxttail;
		long qlen; /* # of queued callbacks */
		struct rcu_head *curlist;
		struct rcu_head **curtail;
		struct rcu_head *donelist;
		struct rcu_head **donetail;
		long blimit; /* Upper limit on a processed batch */
		int cpu;
		struct rcu_head barrier;
		};

		DECLARE_PER_CPU(struct rcu_data, rcu_data);
		DECLARE_PER_CPU(struct rcu_data, rcu_bh_data);

		/*
		* Increment the quiescent state counter.
		* The counter is a bit degenerated: We do not need to know
		* how many quiescent states passed, just if there was at least
		* one since the start of the grace period. Thus just a flag.
		*/
		static inline void rcu_qsctr_inc(int cpu)
		{
		struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
		rdp->passed_quiesc = 1;
		}
		static inline void rcu_bh_qsctr_inc(int cpu)
		{
		struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
		rdp->passed_quiesc = 1;
		}

		extern int rcu_pending(int cpu);
		extern int rcu_needs_cpu(int cpu);

		#ifdef CONFIG_DEBUG_LOCK_ALLOC
		extern struct lockdep_map rcu_lock_map;
		# define rcu_read_acquire() \
		lock_acquire(&rcu_lock_map, 0, 0, 2, 1, _THIS_IP_)
		# define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_)
		#else
		# define rcu_read_acquire() do { } while (0)
		# define rcu_read_release() do { } while (0)
		#endif

		#define __rcu_read_lock() \
		do { \
		preempt_disable(); \
		__acquire(RCU); \
		rcu_read_acquire(); \
		} while (0)
		#define __rcu_read_unlock() \
		do { \
		rcu_read_release(); \
		__release(RCU); \
		preempt_enable(); \
		} while (0)
		#define __rcu_read_lock_bh() \
		do { \
		local_bh_disable(); \
		__acquire(RCU_BH); \
		rcu_read_acquire(); \
		} while (0)
		#define __rcu_read_unlock_bh() \
		do { \
		rcu_read_release(); \
		__release(RCU_BH); \
		local_bh_enable(); \
		} while (0)

		#define __synchronize_sched() synchronize_rcu()

		extern void __rcu_init(void);
		extern void rcu_check_callbacks(int cpu, int user);
		extern void rcu_restart_cpu(int cpu);

		#endif /* __KERNEL__ */
		#endif /* __LINUX_RCUCLASSIC_H */

include/linux/rcupdate.h

+49 −119

Original line number	Diff line number	Diff line
		@@ -15,7 +15,7 @@
		* along with this program; if not, write to the Free Software
		* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
		*
		* Copyright (C) IBM Corporation, 2001
		* Copyright IBM Corporation, 2001
		*
		* Author: Dipankar Sarma <dipankar@in.ibm.com>
		*
		@@ -53,96 +53,14 @@ struct rcu_head {
		void (func)(struct rcu_head head);
		};

		#include <linux/rcuclassic.h>

		#define RCU_HEAD_INIT { .next = NULL, .func = NULL }
		#define RCU_HEAD(head) struct rcu_head head = RCU_HEAD_INIT
		#define INIT_RCU_HEAD(ptr) do { \
		(ptr)->next = NULL; (ptr)->func = NULL; \
		} while (0)



		/* Global control variables for rcupdate callback mechanism. */
		struct rcu_ctrlblk {
		long cur; /* Current batch number. */
		long completed; /* Number of the last completed batch */
		int next_pending; /* Is the next batch already waiting? */

		int signaled;

		spinlock_t lock ____cacheline_internodealigned_in_smp;
		cpumask_t cpumask; /* CPUs that need to switch in order */
		/* for current batch to proceed. */
		} ____cacheline_internodealigned_in_smp;

		/* Is batch a before batch b ? */
		static inline int rcu_batch_before(long a, long b)
		{
		return (a - b) < 0;
		}

		/* Is batch a after batch b ? */
		static inline int rcu_batch_after(long a, long b)
		{
		return (a - b) > 0;
		}

		/*
		* Per-CPU data for Read-Copy UPdate.
		* nxtlist - new callbacks are added here
		* curlist - current batch for which quiescent cycle started if any
		*/
		struct rcu_data {
		/* 1) quiescent state handling : */
		long quiescbatch; /* Batch # for grace period */
		int passed_quiesc; /* User-mode/idle loop etc. */
		int qs_pending; /* core waits for quiesc state */

		/* 2) batch handling */
		long batch; /* Batch # for current RCU batch */
		struct rcu_head *nxtlist;
		struct rcu_head **nxttail;
		long qlen; /* # of queued callbacks */
		struct rcu_head *curlist;
		struct rcu_head **curtail;
		struct rcu_head *donelist;
		struct rcu_head **donetail;
		long blimit; /* Upper limit on a processed batch */
		int cpu;
		struct rcu_head barrier;
		};

		DECLARE_PER_CPU(struct rcu_data, rcu_data);
		DECLARE_PER_CPU(struct rcu_data, rcu_bh_data);

		/*
		* Increment the quiescent state counter.
		* The counter is a bit degenerated: We do not need to know
		* how many quiescent states passed, just if there was at least
		* one since the start of the grace period. Thus just a flag.
		*/
		static inline void rcu_qsctr_inc(int cpu)
		{
		struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
		rdp->passed_quiesc = 1;
		}
		static inline void rcu_bh_qsctr_inc(int cpu)
		{
		struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
		rdp->passed_quiesc = 1;
		}

		extern int rcu_pending(int cpu);
		extern int rcu_needs_cpu(int cpu);

		#ifdef CONFIG_DEBUG_LOCK_ALLOC
		extern struct lockdep_map rcu_lock_map;
		# define rcu_read_acquire() lock_acquire(&rcu_lock_map, 0, 0, 2, 1, _THIS_IP_)
		# define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_)
		#else
		# define rcu_read_acquire() do { } while (0)
		# define rcu_read_release() do { } while (0)
		#endif

		/**
		* rcu_read_lock - mark the beginning of an RCU read-side critical section.
		*
		@@ -172,24 +90,13 @@ extern struct lockdep_map rcu_lock_map;
		*
		* It is illegal to block while in an RCU read-side critical section.
		*/
		#define rcu_read_lock() \
		do { \
		preempt_disable(); \
		__acquire(RCU); \
		rcu_read_acquire(); \
		} while(0)
		#define rcu_read_lock() __rcu_read_lock()

		/**
		* rcu_read_unlock - marks the end of an RCU read-side critical section.
		*
		* See rcu_read_lock() for more information.
		*/
		#define rcu_read_unlock() \
		do { \
		rcu_read_release(); \
		__release(RCU); \
		preempt_enable(); \
		} while(0)

		/*
		* So where is rcu_write_lock()? It does not exist, as there is no
		@@ -200,6 +107,7 @@ extern struct lockdep_map rcu_lock_map;
		* used as well. RCU does not care how the writers keep out of each
		* others' way, as long as they do so.
		*/
		#define rcu_read_unlock() __rcu_read_unlock()

		/**
		* rcu_read_lock_bh - mark the beginning of a softirq-only RCU critical section
		@@ -212,24 +120,14 @@ extern struct lockdep_map rcu_lock_map;
		* can use just rcu_read_lock().
		*
		*/
		#define rcu_read_lock_bh() \
		do { \
		local_bh_disable(); \
		__acquire(RCU_BH); \
		rcu_read_acquire(); \
		} while(0)
		#define rcu_read_lock_bh() __rcu_read_lock_bh()

		/*
		* rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
		*
		* See rcu_read_lock_bh() for more information.
		*/
		#define rcu_read_unlock_bh() \
		do { \
		rcu_read_release(); \
		__release(RCU_BH); \
		local_bh_enable(); \
		} while(0)
		#define rcu_read_unlock_bh() __rcu_read_unlock_bh()

		/*
		* Prevent the compiler from merging or refetching accesses. The compiler
		@@ -293,21 +191,53 @@ extern struct lockdep_map rcu_lock_map;
		* In "classic RCU", these two guarantees happen to be one and
		* the same, but can differ in realtime RCU implementations.
		*/
		#define synchronize_sched() synchronize_rcu()
		#define synchronize_sched() __synchronize_sched()

		/**
		* call_rcu - Queue an RCU callback for invocation after a grace period.
		* @head: structure to be used for queueing the RCU updates.
		* @func: actual update function to be invoked after the grace period
		*
		* The update function will be invoked some time after a full grace
		* period elapses, in other words after all currently executing RCU
		* read-side critical sections have completed. RCU read-side critical
		* sections are delimited by rcu_read_lock() and rcu_read_unlock(),
		* and may be nested.
		*/
		extern void call_rcu(struct rcu_head *head,
		void (func)(struct rcu_head head));

		/**
		* call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
		* @head: structure to be used for queueing the RCU updates.
		* @func: actual update function to be invoked after the grace period
		*
		* The update function will be invoked some time after a full grace
		* period elapses, in other words after all currently executing RCU
		* read-side critical sections have completed. call_rcu_bh() assumes
		* that the read-side critical sections end on completion of a softirq
		* handler. This means that read-side critical sections in process
		* context must not be interrupted by softirqs. This interface is to be
		* used when most of the read-side critical sections are in softirq context.
		* RCU read-side critical sections are delimited by :
		* - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
		* OR
		* - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
		* These may be nested.
		*/
		extern void call_rcu_bh(struct rcu_head *head,
		void (func)(struct rcu_head head));

		/* Exported common interfaces */
		extern void synchronize_rcu(void);
		extern void rcu_barrier(void);

		/* Internal to kernel */
		extern void rcu_init(void);
		extern void rcu_check_callbacks(int cpu, int user);
		extern void rcu_restart_cpu(int cpu);

		extern long rcu_batches_completed(void);
		extern long rcu_batches_completed_bh(void);

		/* Exported interfaces */
		extern void FASTCALL(call_rcu(struct rcu_head *head,
		void (func)(struct rcu_head head)));
		extern void FASTCALL(call_rcu_bh(struct rcu_head *head,
		void (func)(struct rcu_head head)));
		extern void synchronize_rcu(void);
		extern void rcu_barrier(void);

		#endif /* __KERNEL__ */
		#endif /* __LINUX_RCUPDATE_H */

kernel/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -6,7 +6,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \
		exit.o itimer.o time.o softirq.o resource.o \
		sysctl.o capability.o ptrace.o timer.o user.o user_namespace.o \
		signal.o sys.o kmod.o workqueue.o pid.o \
		rcupdate.o extable.o params.o posix-timers.o \
		rcupdate.o rcuclassic.o extable.o params.o posix-timers.o \
		kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
		hrtimer.o rwsem.o latency.o nsproxy.o srcu.o \
		utsname.o notifier.o

kernel/rcuclassic.c

0 → 100644

+576 −0

File added.

Preview size limit exceeded, changes collapsed.

kernel/rcupdate.c

+25 −550

File changed.

Preview size limit exceeded, changes collapsed.