locking/lockdep: Free lock classes that are no longer in use

#define LOCKSTAT_POINTS		4

/*

 * The lock-class itself:

 * The lock-class itself. The order of the structure members matters.

 * reinit_class() zeroes the key member and all subsequent members.

 */

struct lock_class {

	/*

	struct hlist_node		hash_entry;

	/*

	 * global list of all lock-classes:

	 * Entry in all_lock_classes when in use. Entry in free_lock_classes

	 * when not in use. Instances that are being freed are on one of the

	 * zapped_classes lists.

	 */

	struct list_head		lock_entry;

	unsigned long			contention_point[LOCKSTAT_POINTS];

	unsigned long			contending_point[LOCKSTAT_POINTS];

#endif

};

} __no_randomize_layout;

#ifdef CONFIG_LOCK_STAT

struct lock_time {

#include <linux/random.h>

#include <linux/jhash.h>

#include <linux/nmi.h>

#include <linux/rcupdate.h>

#include <asm/sections.h>

/*

 * All data structures here are protected by the global debug_lock.

 *

 * Mutex key structs only get allocated, once during bootup, and never

 * get freed - this significantly simplifies the debugging code.

 * nr_lock_classes is the number of elements of lock_classes[] that is

 * in use.

 */

unsigned long nr_lock_classes;

#ifndef CONFIG_DEBUG_LOCKDEP

#endif

/*

 * We keep a global list of all lock classes. The list only grows,

 * never shrinks. The list is only accessed with the lockdep

 * spinlock lock held.

 * We keep a global list of all lock classes. The list is only accessed with

 * the lockdep spinlock lock held. free_lock_classes is a list with free

 * elements. These elements are linked together by the lock_entry member in

 * struct lock_class.

 */

LIST_HEAD(all_lock_classes);

static LIST_HEAD(free_lock_classes);

/**

 * struct pending_free - information about data structures about to be freed

 * @zapped: Head of a list with struct lock_class elements.

 */

struct pending_free {

	struct list_head zapped;

};

/**

 * struct delayed_free - data structures used for delayed freeing

 *

 * A data structure for delayed freeing of data structures that may be

 * accessed by RCU readers at the time these were freed.

 *

 * @rcu_head:  Used to schedule an RCU callback for freeing data structures.

 * @index:     Index of @pf to which freed data structures are added.

 * @scheduled: Whether or not an RCU callback has been scheduled.

 * @pf:        Array with information about data structures about to be freed.

 */

static struct delayed_free {

	struct rcu_head		rcu_head;

	int			index;

	int			scheduled;

	struct pending_free	pf[2];

} delayed_free;

/*

 * The lockdep classes are in a hash-table as well, for fast lookup:

}

/*

 * Initialize the lock_classes[] array elements.

 * Initialize the lock_classes[] array elements, the free_lock_classes list

 * and also the delayed_free structure.

 */

static void init_data_structures_once(void)

{

	initialization_happened = true;

	init_rcu_head(&delayed_free.rcu_head);

	INIT_LIST_HEAD(&delayed_free.pf[0].zapped);

	INIT_LIST_HEAD(&delayed_free.pf[1].zapped);

	for (i = 0; i < ARRAY_SIZE(lock_classes); i++) {

		list_add_tail(&lock_classes[i].lock_entry, &free_lock_classes);

		INIT_LIST_HEAD(&lock_classes[i].locks_after);

		INIT_LIST_HEAD(&lock_classes[i].locks_before);

	}

	init_data_structures_once();

	/*

	 * Allocate a new key from the static array, and add it to

	 * the hash:

	 */

	if (nr_lock_classes >= MAX_LOCKDEP_KEYS) {

	/* Allocate a new lock class and add it to the hash. */

	class = list_first_entry_or_null(&free_lock_classes, typeof(*class),

					 lock_entry);

	if (!class) {

		if (!debug_locks_off_graph_unlock()) {

			return NULL;

		}

		dump_stack();

		return NULL;

	}

	class = lock_classes + nr_lock_classes++;

	nr_lock_classes++;

	debug_atomic_inc(nr_unused_locks);

	class->key = key;

	class->name = lock->name;

	 */

	hlist_add_head_rcu(&class->hash_entry, hash_head);

	/*

	 * Add it to the global list of classes:

	 * Remove the class from the free list and add it to the global list

	 * of classes.

	 */

	list_add_tail(&class->lock_entry, &all_lock_classes);

	list_move_tail(&class->lock_entry, &all_lock_classes);

	if (verbose(class)) {

		graph_unlock();

	struct lock_list this;

	int ret;

	if (!hlock_class(prev)->key || !hlock_class(next)->key) {

		/*

		 * The warning statements below may trigger a use-after-free

		 * of the class name. It is better to trigger a use-after free

		 * and to have the class name most of the time instead of not

		 * having the class name available.

		 */

		WARN_ONCE(!debug_locks_silent && !hlock_class(prev)->key,

			  "Detected use-after-free of lock class %px/%s\n",

			  hlock_class(prev),

			  hlock_class(prev)->name);

		WARN_ONCE(!debug_locks_silent && !hlock_class(next)->key,

			  "Detected use-after-free of lock class %px/%s\n",

			  hlock_class(next),

			  hlock_class(next)->name);

		return 2;

	}

	/*

	 * Prove that the new <prev> -> <next> dependency would not

	 * create a circular dependency in the graph. (We do this by

}

/*

 * Look up a dependency chain.

 * Look up a dependency chain. Must be called with either the graph lock or

 * the RCU read lock held.

 */

static inline struct lock_chain *lookup_chain_cache(u64 chain_key)

{

	struct hlist_head *hash_head = chainhashentry(chain_key);

	struct lock_chain *chain;

	/*

	 * We can walk it lock-free, because entries only get added

	 * to the hash:

	 */

	hlist_for_each_entry_rcu(chain, hash_head, entry) {

		if (chain->chain_key == chain_key) {

		if (READ_ONCE(chain->chain_key) == chain_key) {

			debug_atomic_inc(chain_lookup_hits);

			return chain;

		}

	if (nest_lock && !__lock_is_held(nest_lock, -1))

		return print_lock_nested_lock_not_held(curr, hlock, ip);

	if (!debug_locks_silent) {

		WARN_ON_ONCE(depth && !hlock_class(hlock - 1)->key);

		WARN_ON_ONCE(!hlock_class(hlock)->key);

	}

	if (!validate_chain(curr, lock, hlock, chain_head, chain_key))

		return 0;

	raw_local_irq_restore(flags);

}

/* Remove a class from a lock chain. Must be called with the graph lock held. */

static void remove_class_from_lock_chain(struct lock_chain *chain,

					 struct lock_class *class)

{

#ifdef CONFIG_PROVE_LOCKING

	struct lock_chain *new_chain;

	u64 chain_key;

	int i;

	for (i = chain->base; i < chain->base + chain->depth; i++) {

		if (chain_hlocks[i] != class - lock_classes)

			continue;

		/* The code below leaks one chain_hlock[] entry. */

		if (--chain->depth > 0)

			memmove(&chain_hlocks[i], &chain_hlocks[i + 1],

				(chain->base + chain->depth - i) *

				sizeof(chain_hlocks[0]));

		/*

		 * Each lock class occurs at most once in a lock chain so once

		 * we found a match we can break out of this loop.

		 */

		goto recalc;

	}

	/* Since the chain has not been modified, return. */

	return;

recalc:

	chain_key = 0;

	for (i = chain->base; i < chain->base + chain->depth; i++)

		chain_key = iterate_chain_key(chain_key, chain_hlocks[i] + 1);

	if (chain->depth && chain->chain_key == chain_key)

		return;

	/* Overwrite the chain key for concurrent RCU readers. */

	WRITE_ONCE(chain->chain_key, chain_key);

	/*

	 * Note: calling hlist_del_rcu() from inside a

	 * hlist_for_each_entry_rcu() loop is safe.

	 */

	hlist_del_rcu(&chain->entry);

	if (chain->depth == 0)

		return;

	/*

	 * If the modified lock chain matches an existing lock chain, drop

	 * the modified lock chain.

	 */

	if (lookup_chain_cache(chain_key))

		return;

	if (WARN_ON_ONCE(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) {

		debug_locks_off();

		return;

	}

	/*

	 * Leak *chain because it is not safe to reinsert it before an RCU

	 * grace period has expired.

	 */

	new_chain = lock_chains + nr_lock_chains++;

	*new_chain = *chain;

	hlist_add_head_rcu(&new_chain->entry, chainhashentry(chain_key));

#endif

}

/* Must be called with the graph lock held. */

static void remove_class_from_lock_chains(struct lock_class *class)

{

	struct lock_chain *chain;

	struct hlist_head *head;

	int i;

	for (i = 0; i < ARRAY_SIZE(chainhash_table); i++) {

		head = chainhash_table + i;

		hlist_for_each_entry_rcu(chain, head, entry) {

			remove_class_from_lock_chain(chain, class);

		}

	}

}

/*

 * Remove all references to a lock class. The caller must hold the graph lock.

 */

static void zap_class(struct lock_class *class)

static void zap_class(struct pending_free *pf, struct lock_class *class)

{

	struct lock_list *entry;

	int i;

	WARN_ON_ONCE(!class->key);

	/*

	 * Remove all dependencies this lock is

	 * involved in:

		WRITE_ONCE(entry->class, NULL);

		WRITE_ONCE(entry->links_to, NULL);

	}

	/*

	 * Unhash the class and remove it from the all_lock_classes list:

	 */

	if (list_empty(&class->locks_after) &&

	    list_empty(&class->locks_before)) {

		list_move_tail(&class->lock_entry, &pf->zapped);

		hlist_del_rcu(&class->hash_entry);

	list_del(&class->lock_entry);

		WRITE_ONCE(class->key, NULL);

		WRITE_ONCE(class->name, NULL);

		nr_lock_classes--;

	} else {

		WARN_ONCE(true, "%s() failed for class %s\n", __func__,

			  class->name);

	}

	RCU_INIT_POINTER(class->key, NULL);

	RCU_INIT_POINTER(class->name, NULL);

	remove_class_from_lock_chains(class);

}

static void reinit_class(struct lock_class *class)

{

	void *const p = class;

	const unsigned int offset = offsetof(struct lock_class, key);

	WARN_ON_ONCE(!class->lock_entry.next);

	WARN_ON_ONCE(!list_empty(&class->locks_after));

	WARN_ON_ONCE(!list_empty(&class->locks_before));

	memset(p + offset, 0, sizeof(*class) - offset);

	WARN_ON_ONCE(!class->lock_entry.next);

	WARN_ON_ONCE(!list_empty(&class->locks_after));

	WARN_ON_ONCE(!list_empty(&class->locks_before));

}

static inline int within(const void *addr, void *start, unsigned long size)

	return addr >= start && addr < start + size;

}

static void __lockdep_free_key_range(void *start, unsigned long size)

static bool inside_selftest(void)

{

	return current == lockdep_selftest_task_struct;

}

/* The caller must hold the graph lock. */

static struct pending_free *get_pending_free(void)

{

	return delayed_free.pf + delayed_free.index;

}

static void free_zapped_rcu(struct rcu_head *cb);

/*

 * Schedule an RCU callback if no RCU callback is pending. Must be called with

 * the graph lock held.

 */

static void call_rcu_zapped(struct pending_free *pf)

{

	WARN_ON_ONCE(inside_selftest());

	if (list_empty(&pf->zapped))

		return;

	if (delayed_free.scheduled)

		return;

	delayed_free.scheduled = true;

	WARN_ON_ONCE(delayed_free.pf + delayed_free.index != pf);

	delayed_free.index ^= 1;

	call_rcu(&delayed_free.rcu_head, free_zapped_rcu);

}

/* The caller must hold the graph lock. May be called from RCU context. */

static void __free_zapped_classes(struct pending_free *pf)

{

	struct lock_class *class;

	list_for_each_entry(class, &pf->zapped, lock_entry)

		reinit_class(class);

	list_splice_init(&pf->zapped, &free_lock_classes);

}

static void free_zapped_rcu(struct rcu_head *ch)

{

	struct pending_free *pf;

	unsigned long flags;

	if (WARN_ON_ONCE(ch != &delayed_free.rcu_head))

		return;

	raw_local_irq_save(flags);

	if (!graph_lock())

		goto out_irq;

	/* closed head */

	pf = delayed_free.pf + (delayed_free.index ^ 1);

	__free_zapped_classes(pf);

	delayed_free.scheduled = false;

	/*

	 * If there's anything on the open list, close and start a new callback.

	 */

	call_rcu_zapped(delayed_free.pf + delayed_free.index);

	graph_unlock();

out_irq:

	raw_local_irq_restore(flags);

}

/*

 * Remove all lock classes from the class hash table and from the

 * all_lock_classes list whose key or name is in the address range [start,

 * start + size). Move these lock classes to the zapped_classes list. Must

 * be called with the graph lock held.

 */

static void __lockdep_free_key_range(struct pending_free *pf, void *start,

				     unsigned long size)

{

	struct lock_class *class;

	struct hlist_head *head;

			if (!within(class->key, start, size) &&

			    !within(class->name, start, size))

				continue;

			zap_class(class);

			zap_class(pf, class);

		}

	}

}

 * guaranteed nobody will look up these exact classes -- they're properly dead

 * but still allocated.

 */

void lockdep_free_key_range(void *start, unsigned long size)

static void lockdep_free_key_range_reg(void *start, unsigned long size)

{

	struct pending_free *pf;

	unsigned long flags;

	int locked;

	raw_local_irq_save(flags);

	locked = graph_lock();

	__lockdep_free_key_range(start, size);

	if (locked)

	if (!locked)

		goto out_irq;

	pf = get_pending_free();

	__lockdep_free_key_range(pf, start, size);

	call_rcu_zapped(pf);

	graph_unlock();

out_irq:

	raw_local_irq_restore(flags);

	/*

	 * before continuing to free the memory they refer to.

	 */

	synchronize_rcu();

}

/*

	 * XXX at this point we could return the resources to the pool;

	 * instead we leak them. We would need to change to bitmap allocators

	 * instead of the linear allocators we have now.

 * Free all lockdep keys in the range [start, start+size). Does not sleep.

 * Ignores debug_locks. Must only be used by the lockdep selftests.

 */

static void lockdep_free_key_range_imm(void *start, unsigned long size)

{

	struct pending_free *pf = delayed_free.pf;

	unsigned long flags;

	init_data_structures_once();

	raw_local_irq_save(flags);

	arch_spin_lock(&lockdep_lock);

	__lockdep_free_key_range(pf, start, size);

	__free_zapped_classes(pf);

	arch_spin_unlock(&lockdep_lock);

	raw_local_irq_restore(flags);

}

void lockdep_free_key_range(void *start, unsigned long size)

{

	init_data_structures_once();

	if (inside_selftest())

		lockdep_free_key_range_imm(start, size);

	else

		lockdep_free_key_range_reg(start, size);

}

/*

}

/* The caller must hold the graph lock. Does not sleep. */

static void __lockdep_reset_lock(struct lockdep_map *lock)

static void __lockdep_reset_lock(struct pending_free *pf,

				 struct lockdep_map *lock)

{

	struct lock_class *class;

	int j;

		 */

		class = look_up_lock_class(lock, j);

		if (class)

			zap_class(class);

			zap_class(pf, class);

	}

	/*

	 * Debug check: in the end all mapped classes should

		debug_locks_off();

}

void lockdep_reset_lock(struct lockdep_map *lock)

/*

 * Remove all information lockdep has about a lock if debug_locks == 1. Free

 * released data structures from RCU context.

 */

static void lockdep_reset_lock_reg(struct lockdep_map *lock)

{

	struct pending_free *pf;

	unsigned long flags;

	int locked;

	init_data_structures_once();

	raw_local_irq_save(flags);

	locked = graph_lock();

	__lockdep_reset_lock(lock);

	if (locked)

	if (!locked)

		goto out_irq;

	pf = get_pending_free();

	__lockdep_reset_lock(pf, lock);

	call_rcu_zapped(pf);

	graph_unlock();

out_irq:

	raw_local_irq_restore(flags);

}

/*

 * Reset a lock. Does not sleep. Ignores debug_locks. Must only be used by the

 * lockdep selftests.

 */

static void lockdep_reset_lock_imm(struct lockdep_map *lock)

{

	struct pending_free *pf = delayed_free.pf;

	unsigned long flags;

	raw_local_irq_save(flags);

	arch_spin_lock(&lockdep_lock);

	__lockdep_reset_lock(pf, lock);

	__free_zapped_classes(pf);

	arch_spin_unlock(&lockdep_lock);

	raw_local_irq_restore(flags);

}

void lockdep_reset_lock(struct lockdep_map *lock)

{

	init_data_structures_once();

	if (inside_selftest())

		lockdep_reset_lock_imm(lock);

	else

		lockdep_reset_lock_reg(lock);

}

void __init lockdep_init(void)

{

	printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n");

	       (sizeof(lock_classes) +

		sizeof(classhash_table) +

		sizeof(list_entries) +

		sizeof(chainhash_table)

		sizeof(chainhash_table) +

		sizeof(delayed_free)

#ifdef CONFIG_PROVE_LOCKING

		+ sizeof(lock_cq)

		+ sizeof(lock_chains)