Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4b65f455 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull locking fixes from Thomas Gleixner:
 "Two fixes to address shortcomings of the rwsem/percpu-rwsem lock
  debugging code which emits false positive warnings when the rwsem is
  anonymously locked and unlocked"

* 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  locking/percpu-rwsem: Annotate rwsem ownership transfer by setting RWSEM_OWNER_UNKNOWN
  locking/rwsem: Add a new RWSEM_ANONYMOUSLY_OWNED flag
parents 056ad121 5a817641

include/linux/percpu-rwsem.h

+5 −1
Original line number Original line Diff line number Diff line
@@ -133,7 +133,7 @@ static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, 
	lock_release(&sem->rw_sem.dep_map, 1, ip);

	lock_release(&sem->rw_sem.dep_map, 1, ip);

#ifdef CONFIG_RWSEM_SPIN_ON_OWNER

#ifdef CONFIG_RWSEM_SPIN_ON_OWNER

	if (!read)

	if (!read)

		sem->rw_sem.owner = NULL;

		sem->rw_sem.owner = RWSEM_OWNER_UNKNOWN;

#endif

#endif

}

}
@@ -141,6 +141,10 @@ static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, 
					bool read, unsigned long ip)

					bool read, unsigned long ip)

{

{

	lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);

	lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);

#ifdef CONFIG_RWSEM_SPIN_ON_OWNER

	if (!read)

		sem->rw_sem.owner = current;

#endif

}

}





#endif
 
#endif

include/linux/rwsem.h

+6 −0
Original line number Original line Diff line number Diff line
@@ -44,6 +44,12 @@ struct rw_semaphore { 
#endif

#endif

};

};





/*

 * Setting bit 0 of the owner field with other non-zero bits will indicate

 * that the rwsem is writer-owned with an unknown owner.

 */

#define RWSEM_OWNER_UNKNOWN	((struct task_struct *)-1L)



extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_down_read_failed_killable(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);

extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);

kernel/locking/rwsem-xadd.c

+9 −10
Original line number Original line Diff line number Diff line
@@ -352,16 +352,15 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) 
	struct task_struct *owner;

	struct task_struct *owner;

	bool ret = true;

	bool ret = true;





	BUILD_BUG_ON(!rwsem_has_anonymous_owner(RWSEM_OWNER_UNKNOWN));



	if (need_resched())

	if (need_resched())

		return false;

		return false;





	rcu_read_lock();

	rcu_read_lock();

	owner = READ_ONCE(sem->owner);

	owner = READ_ONCE(sem->owner);

	if (!rwsem_owner_is_writer(owner)) {

	if (!owner || !is_rwsem_owner_spinnable(owner)) {

		/*

		ret = !owner;	/* !owner is spinnable */

		 * Don't spin if the rwsem is readers owned.

		 */

		ret = !rwsem_owner_is_reader(owner);

		goto done;

		goto done;

	}

	}
@@ -382,11 +381,11 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem) 
{

{

	struct task_struct *owner = READ_ONCE(sem->owner);

	struct task_struct *owner = READ_ONCE(sem->owner);





	if (!rwsem_owner_is_writer(owner))

	if (!is_rwsem_owner_spinnable(owner))

		goto out;

		return false;





	rcu_read_lock();

	rcu_read_lock();

	while (sem->owner == owner) {

	while (owner && (READ_ONCE(sem->owner) == owner)) {

		/*

		/*

		 * Ensure we emit the owner->on_cpu, dereference _after_

		 * Ensure we emit the owner->on_cpu, dereference _after_

		 * checking sem->owner still matches owner, if that fails,

		 * checking sem->owner still matches owner, if that fails,
@@ -408,12 +407,12 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem) 
		cpu_relax();

		cpu_relax();

	}

	}

	rcu_read_unlock();

	rcu_read_unlock();

out:



	/*

	/*

	 * If there is a new owner or the owner is not set, we continue

	 * If there is a new owner or the owner is not set, we continue

	 * spinning.

	 * spinning.

	 */

	 */

	return !rwsem_owner_is_reader(READ_ONCE(sem->owner));

	return is_rwsem_owner_spinnable(READ_ONCE(sem->owner));

}

}





static bool rwsem_optimistic_spin(struct rw_semaphore *sem)

static bool rwsem_optimistic_spin(struct rw_semaphore *sem)

kernel/locking/rwsem.c

+0 −2
Original line number Original line Diff line number Diff line
@@ -221,5 +221,3 @@ void up_read_non_owner(struct rw_semaphore *sem) 
EXPORT_SYMBOL(up_read_non_owner);

EXPORT_SYMBOL(up_read_non_owner);





#endif
 
#endif

kernel/locking/rwsem.h

+21 −9
Original line number Original line Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/* SPDX-License-Identifier: GPL-2.0 */

/*

/*

 * The owner field of the rw_semaphore structure will be set to

 * The owner field of the rw_semaphore structure will be set to

 * RWSEM_READ_OWNED when a reader grabs the lock. A writer will clear

 * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear

 * the owner field when it unlocks. A reader, on the other hand, will

 * the owner field when it unlocks. A reader, on the other hand, will

 * not touch the owner field when it unlocks.

 * not touch the owner field when it unlocks.

 *

 *

 * In essence, the owner field now has the following 3 states:

 * In essence, the owner field now has the following 4 states:

 *  1) 0

 *  1) 0

 *     - lock is free or the owner hasn't set the field yet

 *     - lock is free or the owner hasn't set the field yet

 *  2) RWSEM_READER_OWNED

 *  2) RWSEM_READER_OWNED

 *     - lock is currently or previously owned by readers (lock is free

 *     - lock is currently or previously owned by readers (lock is free

 *       or not set by owner yet)

 *       or not set by owner yet)

 *  3) Other non-zero value

 *  3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well

 *     - a writer owns the lock

 *     - lock is owned by an anonymous writer, so spinning on the lock

 *       owner should be disabled.

 *  4) Other non-zero value

 *     - a writer owns the lock and other writers can spin on the lock owner.

 */

 */

#define RWSEM_READER_OWNED	((struct task_struct *)1UL)

#define RWSEM_ANONYMOUSLY_OWNED	(1UL << 0)

#define RWSEM_READER_OWNED	((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED)





#ifdef CONFIG_DEBUG_RWSEMS

#ifdef CONFIG_DEBUG_RWSEMS

# define DEBUG_RWSEMS_WARN_ON(c)	DEBUG_LOCKS_WARN_ON(c)

# define DEBUG_RWSEMS_WARN_ON(c)	DEBUG_LOCKS_WARN_ON(c)
@@ -51,14 +55,22 @@ static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) 
		WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);

		WRITE_ONCE(sem->owner, RWSEM_READER_OWNED);

}

}





static inline bool rwsem_owner_is_writer(struct task_struct *owner)

/*

 * Return true if the a rwsem waiter can spin on the rwsem's owner

 * and steal the lock, i.e. the lock is not anonymously owned.

 * N.B. !owner is considered spinnable.

 */

static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)

{

{

	return owner && owner != RWSEM_READER_OWNED;

	return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED);

}

}





static inline bool rwsem_owner_is_reader(struct task_struct *owner)

/*

 * Return true if rwsem is owned by an anonymous writer or readers.

 */

static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)

{

{

	return owner == RWSEM_READER_OWNED;

	return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;

}

}

#else

#else

static inline void rwsem_set_owner(struct rw_semaphore *sem)

static inline void rwsem_set_owner(struct rw_semaphore *sem)