locking/rwlocks: Introduce 'qrwlocks' - fair, queued rwlocks (70af2f8a) · Commits · e / devices / android_kernel_oneplus_sm8150

include/asm-generic/qrwlock.h

0 → 100644

+166 −0

Original line number	Diff line number	Diff line
		/*
		* Queue read/write lock
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation; either version 2 of the License, or
		* (at your option) any later version.
		*
		* This program is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
		*
		* Authors: Waiman Long <waiman.long@hp.com>
		*/
		#ifndef __ASM_GENERIC_QRWLOCK_H
		#define __ASM_GENERIC_QRWLOCK_H

		#include <linux/atomic.h>
		#include <asm/barrier.h>
		#include <asm/processor.h>

		#include <asm-generic/qrwlock_types.h>

		/*
		* Writer states & reader shift and bias
		*/
		#define _QW_WAITING 1 /* A writer is waiting */
		#define _QW_LOCKED 0xff /* A writer holds the lock */
		#define _QW_WMASK 0xff /* Writer mask */
		#define _QR_SHIFT 8 /* Reader count shift */
		#define _QR_BIAS (1U << _QR_SHIFT)

		/*
		* External function declarations
		*/
		extern void queue_read_lock_slowpath(struct qrwlock *lock);
		extern void queue_write_lock_slowpath(struct qrwlock *lock);

		/**
		* queue_read_can_lock- would read_trylock() succeed?
		* @lock: Pointer to queue rwlock structure
		*/
		static inline int queue_read_can_lock(struct qrwlock *lock)
		{
		return !(atomic_read(&lock->cnts) & _QW_WMASK);
		}

		/**
		* queue_write_can_lock- would write_trylock() succeed?
		* @lock: Pointer to queue rwlock structure
		*/
		static inline int queue_write_can_lock(struct qrwlock *lock)
		{
		return !atomic_read(&lock->cnts);
		}

		/**
		* queue_read_trylock - try to acquire read lock of a queue rwlock
		* @lock : Pointer to queue rwlock structure
		* Return: 1 if lock acquired, 0 if failed
		*/
		static inline int queue_read_trylock(struct qrwlock *lock)
		{
		u32 cnts;

		cnts = atomic_read(&lock->cnts);
		if (likely(!(cnts & _QW_WMASK))) {
		cnts = (u32)atomic_add_return(_QR_BIAS, &lock->cnts);
		if (likely(!(cnts & _QW_WMASK)))
		return 1;
		atomic_sub(_QR_BIAS, &lock->cnts);
		}
		return 0;
		}

		/**
		* queue_write_trylock - try to acquire write lock of a queue rwlock
		* @lock : Pointer to queue rwlock structure
		* Return: 1 if lock acquired, 0 if failed
		*/
		static inline int queue_write_trylock(struct qrwlock *lock)
		{
		u32 cnts;

		cnts = atomic_read(&lock->cnts);
		if (unlikely(cnts))
		return 0;

		return likely(atomic_cmpxchg(&lock->cnts,
		cnts, cnts \| _QW_LOCKED) == cnts);
		}
		/**
		* queue_read_lock - acquire read lock of a queue rwlock
		* @lock: Pointer to queue rwlock structure
		*/
		static inline void queue_read_lock(struct qrwlock *lock)
		{
		u32 cnts;

		cnts = atomic_add_return(_QR_BIAS, &lock->cnts);
		if (likely(!(cnts & _QW_WMASK)))
		return;

		/* The slowpath will decrement the reader count, if necessary. */
		queue_read_lock_slowpath(lock);
		}

		/**
		* queue_write_lock - acquire write lock of a queue rwlock
		* @lock : Pointer to queue rwlock structure
		*/
		static inline void queue_write_lock(struct qrwlock *lock)
		{
		/* Optimize for the unfair lock case where the fair flag is 0. */
		if (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0)
		return;

		queue_write_lock_slowpath(lock);
		}

		/**
		* queue_read_unlock - release read lock of a queue rwlock
		* @lock : Pointer to queue rwlock structure
		*/
		static inline void queue_read_unlock(struct qrwlock *lock)
		{
		/*
		* Atomically decrement the reader count
		*/
		smp_mb__before_atomic();
		atomic_sub(_QR_BIAS, &lock->cnts);
		}

		#ifndef queue_write_unlock
		/**
		* queue_write_unlock - release write lock of a queue rwlock
		* @lock : Pointer to queue rwlock structure
		*/
		static inline void queue_write_unlock(struct qrwlock *lock)
		{
		/*
		* If the writer field is atomic, it can be cleared directly.
		* Otherwise, an atomic subtraction will be used to clear it.
		*/
		smp_mb__before_atomic();
		atomic_sub(_QW_LOCKED, &lock->cnts);
		}
		#endif

		/*
		* Remapping rwlock architecture specific functions to the corresponding
		* queue rwlock functions.
		*/
		#define arch_read_can_lock(l) queue_read_can_lock(l)
		#define arch_write_can_lock(l) queue_write_can_lock(l)
		#define arch_read_lock(l) queue_read_lock(l)
		#define arch_write_lock(l) queue_write_lock(l)
		#define arch_read_trylock(l) queue_read_trylock(l)
		#define arch_write_trylock(l) queue_write_trylock(l)
		#define arch_read_unlock(l) queue_read_unlock(l)
		#define arch_write_unlock(l) queue_write_unlock(l)

		#endif /* __ASM_GENERIC_QRWLOCK_H */

include/asm-generic/qrwlock_types.h

0 → 100644

+21 −0

Original line number	Diff line number	Diff line
		#ifndef __ASM_GENERIC_QRWLOCK_TYPES_H
		#define __ASM_GENERIC_QRWLOCK_TYPES_H

		#include <linux/types.h>
		#include <asm/spinlock_types.h>

		/*
		* The queue read/write lock data structure
		*/

		typedef struct qrwlock {
		atomic_t cnts;
		arch_spinlock_t lock;
		} arch_rwlock_t;

		#define __ARCH_RW_LOCK_UNLOCKED { \
		.cnts = ATOMIC_INIT(0), \
		.lock = __ARCH_SPIN_LOCK_UNLOCKED, \
		}

		#endif /* __ASM_GENERIC_QRWLOCK_TYPES_H */

kernel/Kconfig.locks

+7 −0

Original line number	Diff line number	Diff line
		@@ -223,3 +223,10 @@ endif
		config MUTEX_SPIN_ON_OWNER
		def_bool y
		depends on SMP && !DEBUG_MUTEXES

		config ARCH_USE_QUEUE_RWLOCK
		bool

		config QUEUE_RWLOCK
		def_bool y if ARCH_USE_QUEUE_RWLOCK
		depends on SMP

kernel/locking/Makefile

+1 −0

Original line number	Diff line number	Diff line
		@@ -24,4 +24,5 @@ obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
		obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o
		obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o
		obj-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o
		obj-$(CONFIG_QUEUE_RWLOCK) += qrwlock.o
		obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o

kernel/locking/qrwlock.c

0 → 100644

+133 −0

Original line number	Diff line number	Diff line
		/*
		* Queue read/write lock
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation; either version 2 of the License, or
		* (at your option) any later version.
		*
		* This program is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
		*
		* Authors: Waiman Long <waiman.long@hp.com>
		*/
		#include <linux/smp.h>
		#include <linux/bug.h>
		#include <linux/cpumask.h>
		#include <linux/percpu.h>
		#include <linux/hardirq.h>
		#include <linux/mutex.h>
		#include <asm/qrwlock.h>

		/**
		* rspin_until_writer_unlock - inc reader count & spin until writer is gone
		* @lock : Pointer to queue rwlock structure
		* @writer: Current queue rwlock writer status byte
		*
		* In interrupt context or at the head of the queue, the reader will just
		* increment the reader count & wait until the writer releases the lock.
		*/
		static __always_inline void
		rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
		{
		while ((cnts & _QW_WMASK) == _QW_LOCKED) {
		arch_mutex_cpu_relax();
		cnts = smp_load_acquire((u32 *)&lock->cnts);
		}
		}

		/**
		* queue_read_lock_slowpath - acquire read lock of a queue rwlock
		* @lock: Pointer to queue rwlock structure
		*/
		void queue_read_lock_slowpath(struct qrwlock *lock)
		{
		u32 cnts;

		/*
		* Readers come here when they cannot get the lock without waiting
		*/
		if (unlikely(in_interrupt())) {
		/*
		* Readers in interrupt context will spin until the lock is
		* available without waiting in the queue.
		*/
		cnts = smp_load_acquire((u32 *)&lock->cnts);
		rspin_until_writer_unlock(lock, cnts);
		return;
		}
		atomic_sub(_QR_BIAS, &lock->cnts);

		/*
		* Put the reader into the wait queue
		*/
		arch_spin_lock(&lock->lock);

		/*
		* At the head of the wait queue now, wait until the writer state
		* goes to 0 and then try to increment the reader count and get
		* the lock. It is possible that an incoming writer may steal the
		* lock in the interim, so it is necessary to check the writer byte
		* to make sure that the write lock isn't taken.
		*/
		while (atomic_read(&lock->cnts) & _QW_WMASK)
		arch_mutex_cpu_relax();

		cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS;
		rspin_until_writer_unlock(lock, cnts);

		/*
		* Signal the next one in queue to become queue head
		*/
		arch_spin_unlock(&lock->lock);
		}
		EXPORT_SYMBOL(queue_read_lock_slowpath);

		/**
		* queue_write_lock_slowpath - acquire write lock of a queue rwlock
		* @lock : Pointer to queue rwlock structure
		*/
		void queue_write_lock_slowpath(struct qrwlock *lock)
		{
		u32 cnts;

		/* Put the writer into the wait queue */
		arch_spin_lock(&lock->lock);

		/* Try to acquire the lock directly if no reader is present */
		if (!atomic_read(&lock->cnts) &&
		(atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0))
		goto unlock;

		/*
		* Set the waiting flag to notify readers that a writer is pending,
		* or wait for a previous writer to go away.
		*/
		for (;;) {
		cnts = atomic_read(&lock->cnts);
		if (!(cnts & _QW_WMASK) &&
		(atomic_cmpxchg(&lock->cnts, cnts,
		cnts \| _QW_WAITING) == cnts))
		break;

		arch_mutex_cpu_relax();
		}

		/* When no more readers, set the locked flag */
		for (;;) {
		cnts = atomic_read(&lock->cnts);
		if ((cnts == _QW_WAITING) &&
		(atomic_cmpxchg(&lock->cnts, _QW_WAITING,
		_QW_LOCKED) == _QW_WAITING))
		break;

		arch_mutex_cpu_relax();
		}
		unlock:
		arch_spin_unlock(&lock->lock);
		}
		EXPORT_SYMBOL(queue_write_lock_slowpath);