dcache: let the dentry count go down to zero without taking d_lock (360f5479) · Commits · e / devices / android_kernel_oneplus_sm7250

fs/dcache.c

+114 −4

Original line number	Original line	Diff line number	Diff line
	@@ -508,7 +508,7 @@ static void __dentry_kill(struct dentry *dentry)
	* dentry_iput drops the locks, at which point nobody (except		* dentry_iput drops the locks, at which point nobody (except
	* transient RCU lookups) can reach this dentry.		* transient RCU lookups) can reach this dentry.
	*/		*/
	BUG_ON((int)dentry->d_lockref.count > 0);		BUG_ON(dentry->d_lockref.count > 0);
	this_cpu_dec(nr_dentry);		this_cpu_dec(nr_dentry);
	if (dentry->d_op && dentry->d_op->d_release)		if (dentry->d_op && dentry->d_op->d_release)
	dentry->d_op->d_release(dentry);		dentry->d_op->d_release(dentry);
	@@ -561,7 +561,7 @@ static inline struct dentry lock_parent(struct dentry dentry)
	struct dentry *parent = dentry->d_parent;		struct dentry *parent = dentry->d_parent;
	if (IS_ROOT(dentry))		if (IS_ROOT(dentry))
	return NULL;		return NULL;
	if (unlikely((int)dentry->d_lockref.count < 0))		if (unlikely(dentry->d_lockref.count < 0))
	return NULL;		return NULL;
	if (likely(spin_trylock(&parent->d_lock)))		if (likely(spin_trylock(&parent->d_lock)))
	return parent;		return parent;
	@@ -590,6 +590,110 @@ static inline struct dentry lock_parent(struct dentry dentry)
	return parent;		return parent;
	}		}

			/*
			* Try to do a lockless dput(), and return whether that was successful.
			*
			* If unsuccessful, we return false, having already taken the dentry lock.
			*
			* The caller needs to hold the RCU read lock, so that the dentry is
			* guaranteed to stay around even if the refcount goes down to zero!
			*/
			static inline bool fast_dput(struct dentry *dentry)
			{
			int ret;
			unsigned int d_flags;

			/*
			* If we have a d_op->d_delete() operation, we sould not
			* let the dentry count go to zero, so use "put__or_lock".
			*/
			if (unlikely(dentry->d_flags & DCACHE_OP_DELETE))
			return lockref_put_or_lock(&dentry->d_lockref);

			/*
			* .. otherwise, we can try to just decrement the
			* lockref optimistically.
			*/
			ret = lockref_put_return(&dentry->d_lockref);

			/*
			* If the lockref_put_return() failed due to the lock being held
			* by somebody else, the fast path has failed. We will need to
			* get the lock, and then check the count again.
			*/
			if (unlikely(ret < 0)) {
			spin_lock(&dentry->d_lock);
			if (dentry->d_lockref.count > 1) {
			dentry->d_lockref.count--;
			spin_unlock(&dentry->d_lock);
			return 1;
			}
			return 0;
			}

			/*
			* If we weren't the last ref, we're done.
			*/
			if (ret)
			return 1;

			/*
			* Careful, careful. The reference count went down
			* to zero, but we don't hold the dentry lock, so
			* somebody else could get it again, and do another
			* dput(), and we need to not race with that.
			*
			* However, there is a very special and common case
			* where we don't care, because there is nothing to
			* do: the dentry is still hashed, it does not have
			* a 'delete' op, and it's referenced and already on
			* the LRU list.
			*
			* NOTE! Since we aren't locked, these values are
			* not "stable". However, it is sufficient that at
			* some point after we dropped the reference the
			* dentry was hashed and the flags had the proper
			* value. Other dentry users may have re-gotten
			* a reference to the dentry and change that, but
			* our work is done - we can leave the dentry
			* around with a zero refcount.
			*/
			smp_rmb();
			d_flags = ACCESS_ONCE(dentry->d_flags);
			d_flags &= DCACHE_REFERENCED \| DCACHE_LRU_LIST;

			/* Nothing to do? Dropping the reference was all we needed? */
			if (d_flags == (DCACHE_REFERENCED \| DCACHE_LRU_LIST) && !d_unhashed(dentry))
			return 1;

			/*
			* Not the fast normal case? Get the lock. We've already decremented
			* the refcount, but we'll need to re-check the situation after
			* getting the lock.
			*/
			spin_lock(&dentry->d_lock);

			/*
			* Did somebody else grab a reference to it in the meantime, and
			* we're no longer the last user after all? Alternatively, somebody
			* else could have killed it and marked it dead. Either way, we
			* don't need to do anything else.
			*/
			if (dentry->d_lockref.count) {
			spin_unlock(&dentry->d_lock);
			return 1;
			}

			/*
			* Re-get the reference we optimistically dropped. We hold the
			* lock, and we just tested that it was zero, so we can just
			* set it to 1.
			*/
			dentry->d_lockref.count = 1;
			return 0;
			}


	/*		/*
	* This is dput		* This is dput
	*		*
	@@ -622,8 +726,14 @@ void dput(struct dentry *dentry)
	return;		return;

	repeat:		repeat:
	if (lockref_put_or_lock(&dentry->d_lockref))		rcu_read_lock();
			if (likely(fast_dput(dentry))) {
			rcu_read_unlock();
	return;		return;
			}

			/* Slow case: now with the dentry lock held */
			rcu_read_unlock();

	/* Unreachable? Get rid of it */		/* Unreachable? Get rid of it */
	if (unlikely(d_unhashed(dentry)))		if (unlikely(d_unhashed(dentry)))
	@@ -810,7 +920,7 @@ static void shrink_dentry_list(struct list_head *list)
	* We found an inuse dentry which was not removed from		* We found an inuse dentry which was not removed from
	* the LRU because of laziness during lookup. Do not free it.		* the LRU because of laziness during lookup. Do not free it.
	*/		*/
	if ((int)dentry->d_lockref.count > 0) {		if (dentry->d_lockref.count > 0) {
	spin_unlock(&dentry->d_lock);		spin_unlock(&dentry->d_lock);
	if (parent)		if (parent)
	spin_unlock(&parent->d_lock);		spin_unlock(&parent->d_lock);

include/linux/lockref.h

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -28,12 +28,13 @@ struct lockref {
	#endif		#endif
	struct {		struct {
	spinlock_t lock;		spinlock_t lock;
	unsigned int count;		int count;
	};		};
	};		};
	};		};

	extern void lockref_get(struct lockref *);		extern void lockref_get(struct lockref *);
			extern int lockref_put_return(struct lockref *);
	extern int lockref_get_not_zero(struct lockref *);		extern int lockref_get_not_zero(struct lockref *);
	extern int lockref_get_or_lock(struct lockref *);		extern int lockref_get_or_lock(struct lockref *);
	extern int lockref_put_or_lock(struct lockref *);		extern int lockref_put_or_lock(struct lockref *);

lib/lockref.c

+28 −8

Original line number	Original line	Diff line number	Diff line
	@@ -60,7 +60,7 @@ void lockref_get(struct lockref *lockref)
	EXPORT_SYMBOL(lockref_get);		EXPORT_SYMBOL(lockref_get);

	/**		/**
	* lockref_get_not_zero - Increments count unless the count is 0		* lockref_get_not_zero - Increments count unless the count is 0 or dead
	* @lockref: pointer to lockref structure		* @lockref: pointer to lockref structure
	* Return: 1 if count updated successfully or 0 if count was zero		* Return: 1 if count updated successfully or 0 if count was zero
	*/		*/
	@@ -70,7 +70,7 @@ int lockref_get_not_zero(struct lockref *lockref)

	CMPXCHG_LOOP(		CMPXCHG_LOOP(
	new.count++;		new.count++;
	if (!old.count)		if (old.count <= 0)
	return 0;		return 0;
	,		,
	return 1;		return 1;
	@@ -78,7 +78,7 @@ int lockref_get_not_zero(struct lockref *lockref)

	spin_lock(&lockref->lock);		spin_lock(&lockref->lock);
	retval = 0;		retval = 0;
	if (lockref->count) {		if (lockref->count > 0) {
	lockref->count++;		lockref->count++;
	retval = 1;		retval = 1;
	}		}
	@@ -88,7 +88,7 @@ int lockref_get_not_zero(struct lockref *lockref)
	EXPORT_SYMBOL(lockref_get_not_zero);		EXPORT_SYMBOL(lockref_get_not_zero);

	/**		/**
	* lockref_get_or_lock - Increments count unless the count is 0		* lockref_get_or_lock - Increments count unless the count is 0 or dead
	* @lockref: pointer to lockref structure		* @lockref: pointer to lockref structure
	* Return: 1 if count updated successfully or 0 if count was zero		* Return: 1 if count updated successfully or 0 if count was zero
	* and we got the lock instead.		* and we got the lock instead.
	@@ -97,14 +97,14 @@ int lockref_get_or_lock(struct lockref *lockref)
	{		{
	CMPXCHG_LOOP(		CMPXCHG_LOOP(
	new.count++;		new.count++;
	if (!old.count)		if (old.count <= 0)
	break;		break;
	,		,
	return 1;		return 1;
	);		);

	spin_lock(&lockref->lock);		spin_lock(&lockref->lock);
	if (!lockref->count)		if (lockref->count <= 0)
	return 0;		return 0;
	lockref->count++;		lockref->count++;
	spin_unlock(&lockref->lock);		spin_unlock(&lockref->lock);
	@@ -112,6 +112,26 @@ int lockref_get_or_lock(struct lockref *lockref)
	}		}
	EXPORT_SYMBOL(lockref_get_or_lock);		EXPORT_SYMBOL(lockref_get_or_lock);

			/**
			* lockref_put_return - Decrement reference count if possible
			* @lockref: pointer to lockref structure
			*
			* Decrement the reference count and return the new value.
			* If the lockref was dead or locked, return an error.
			*/
			int lockref_put_return(struct lockref *lockref)
			{
			CMPXCHG_LOOP(
			new.count--;
			if (old.count <= 0)
			return -1;
			,
			return new.count;
			);
			return -1;
			}
			EXPORT_SYMBOL(lockref_put_return);

	/**		/**
	* lockref_put_or_lock - decrements count unless count <= 1 before decrement		* lockref_put_or_lock - decrements count unless count <= 1 before decrement
	* @lockref: pointer to lockref structure		* @lockref: pointer to lockref structure
	@@ -158,7 +178,7 @@ int lockref_get_not_dead(struct lockref *lockref)

	CMPXCHG_LOOP(		CMPXCHG_LOOP(
	new.count++;		new.count++;
	if ((int)old.count < 0)		if (old.count < 0)
	return 0;		return 0;
	,		,
	return 1;		return 1;
	@@ -166,7 +186,7 @@ int lockref_get_not_dead(struct lockref *lockref)

	spin_lock(&lockref->lock);		spin_lock(&lockref->lock);
	retval = 0;		retval = 0;
	if ((int) lockref->count >= 0) {		if (lockref->count >= 0) {
	lockref->count++;		lockref->count++;
	retval = 1;		retval = 1;
	}		}