Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c1e2ee2d authored by Andrew Bresticker's avatar Andrew Bresticker Committed by Linus Torvalds
Browse files

memcg: replace ss->id_lock with a rwlock 



While back-porting Johannes Weiner's patch "mm: memcg-aware global
reclaim" for an internal effort, we noticed a significant performance
regression during page-reclaim heavy workloads due to high contention of
the ss->id_lock.  This lock protects idr map, and serializes calls to
idr_get_next() in css_get_next() (which is used during the memcg hierarchy
walk).

Since idr_get_next() is just doing a look up, we need only serialize it
with respect to idr_remove()/idr_get_new().  By making the ss->id_lock a
rwlock, contention is greatly reduced and performance improves.

Tested: cat a 256m file from a ramdisk in a 128m container 50 times on
each core (one file + container per core) in parallel on a NUMA machine.
Result is the time for the test to complete in 1 of the containers.
Both kernels included Johannes' memcg-aware global reclaim patches.

Before rwlock patch: 1710.778s
After rwlock patch: 152.227s

Signed-off-by: default avatarAndrew Bresticker <abrestic@google.com>
Cc: Paul Menage <menage@gmail.com>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Acked-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Ying Han <yinghan@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 080d676d

include/linux/cgroup.h

+1 −1
Original line number Diff line number Diff line
@@ -516,7 +516,7 @@ struct cgroup_subsys { 
	struct list_head sibling;

	/* used when use_id == true */

	struct idr idr;

	spinlock_t id_lock;

	rwlock_t id_lock;



	/* should be defined only by modular subsystems */

	struct module *module;

kernel/cgroup.c

+9 −9
Original line number Diff line number Diff line
@@ -4883,9 +4883,9 @@ void free_css_id(struct cgroup_subsys *ss, struct cgroup_subsys_state *css) 


	rcu_assign_pointer(id->css, NULL);

	rcu_assign_pointer(css->id, NULL);

	spin_lock(&ss->id_lock);

	write_lock(&ss->id_lock);

	idr_remove(&ss->idr, id->id);

	spin_unlock(&ss->id_lock);

	write_unlock(&ss->id_lock);

	kfree_rcu(id, rcu_head);

}

EXPORT_SYMBOL_GPL(free_css_id);
@@ -4911,10 +4911,10 @@ static struct css_id *get_new_cssid(struct cgroup_subsys *ss, int depth) 
		error = -ENOMEM;

		goto err_out;

	}

	spin_lock(&ss->id_lock);

	write_lock(&ss->id_lock);

	/* Don't use 0. allocates an ID of 1-65535 */

	error = idr_get_new_above(&ss->idr, newid, 1, &myid);

	spin_unlock(&ss->id_lock);

	write_unlock(&ss->id_lock);



	/* Returns error when there are no free spaces for new ID.*/

	if (error) {
@@ -4929,9 +4929,9 @@ static struct css_id *get_new_cssid(struct cgroup_subsys *ss, int depth) 
	return newid;

remove_idr:

	error = -ENOSPC;

	spin_lock(&ss->id_lock);

	write_lock(&ss->id_lock);

	idr_remove(&ss->idr, myid);

	spin_unlock(&ss->id_lock);

	write_unlock(&ss->id_lock);

err_out:

	kfree(newid);

	return ERR_PTR(error);
@@ -4943,7 +4943,7 @@ static int __init_or_module cgroup_init_idr(struct cgroup_subsys *ss, 
{

	struct css_id *newid;



	spin_lock_init(&ss->id_lock);

	rwlock_init(&ss->id_lock);

	idr_init(&ss->idr);



	newid = get_new_cssid(ss, 0);
@@ -5038,9 +5038,9 @@ css_get_next(struct cgroup_subsys *ss, int id, 
		 * scan next entry from bitmap(tree), tmpid is updated after

		 * idr_get_next().

		 */

		spin_lock(&ss->id_lock);

		read_lock(&ss->id_lock);

		tmp = idr_get_next(&ss->idr, &tmpid);

		spin_unlock(&ss->id_lock);

		read_unlock(&ss->id_lock);



		if (!tmp)

			break;