bcache: Move sector allocator to alloc.c (2599b53b) · Commits · e / devices / android_kernel_fairphone_FP4

drivers/md/bcache/alloc.c

+180 −0

Original line number	Diff line number	Diff line
		@@ -487,8 +487,188 @@ int bch_bucket_alloc_set(struct cache_set *c, unsigned watermark,
		return ret;
		}

		/* Sector allocator */

		struct open_bucket {
		struct list_head list;
		unsigned last_write_point;
		unsigned sectors_free;
		BKEY_PADDED(key);
		};

		/*
		* We keep multiple buckets open for writes, and try to segregate different
		* write streams for better cache utilization: first we look for a bucket where
		* the last write to it was sequential with the current write, and failing that
		* we look for a bucket that was last used by the same task.
		*
		* The ideas is if you've got multiple tasks pulling data into the cache at the
		* same time, you'll get better cache utilization if you try to segregate their
		* data and preserve locality.
		*
		* For example, say you've starting Firefox at the same time you're copying a
		* bunch of files. Firefox will likely end up being fairly hot and stay in the
		* cache awhile, but the data you copied might not be; if you wrote all that
		* data to the same buckets it'd get invalidated at the same time.
		*
		* Both of those tasks will be doing fairly random IO so we can't rely on
		* detecting sequential IO to segregate their data, but going off of the task
		* should be a sane heuristic.
		*/
		static struct open_bucket pick_data_bucket(struct cache_set c,
		const struct bkey *search,
		unsigned write_point,
		struct bkey *alloc)
		{
		struct open_bucket ret, ret_task = NULL;

		list_for_each_entry_reverse(ret, &c->data_buckets, list)
		if (!bkey_cmp(&ret->key, search))
		goto found;
		else if (ret->last_write_point == write_point)
		ret_task = ret;

		ret = ret_task ?: list_first_entry(&c->data_buckets,
		struct open_bucket, list);
		found:
		if (!ret->sectors_free && KEY_PTRS(alloc)) {
		ret->sectors_free = c->sb.bucket_size;
		bkey_copy(&ret->key, alloc);
		bkey_init(alloc);
		}

		if (!ret->sectors_free)
		ret = NULL;

		return ret;
		}

		/*
		* Allocates some space in the cache to write to, and k to point to the newly
		* allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
		* end of the newly allocated space).
		*
		* May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
		* sectors were actually allocated.
		*
		* If s->writeback is true, will not fail.
		*/
		bool bch_alloc_sectors(struct cache_set c, struct bkey k, unsigned sectors,
		unsigned write_point, unsigned write_prio, bool wait)
		{
		struct open_bucket *b;
		BKEY_PADDED(key) alloc;
		unsigned i;

		/*
		* We might have to allocate a new bucket, which we can't do with a
		* spinlock held. So if we have to allocate, we drop the lock, allocate
		* and then retry. KEY_PTRS() indicates whether alloc points to
		* allocated bucket(s).
		*/

		bkey_init(&alloc.key);
		spin_lock(&c->data_bucket_lock);

		while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
		unsigned watermark = write_prio
		? WATERMARK_MOVINGGC
		: WATERMARK_NONE;

		spin_unlock(&c->data_bucket_lock);

		if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, wait))
		return false;

		spin_lock(&c->data_bucket_lock);
		}

		/*
		* If we had to allocate, we might race and not need to allocate the
		* second time we call find_data_bucket(). If we allocated a bucket but
		* didn't use it, drop the refcount bch_bucket_alloc_set() took:
		*/
		if (KEY_PTRS(&alloc.key))
		__bkey_put(c, &alloc.key);

		for (i = 0; i < KEY_PTRS(&b->key); i++)
		EBUG_ON(ptr_stale(c, &b->key, i));

		/* Set up the pointer to the space we're allocating: */

		for (i = 0; i < KEY_PTRS(&b->key); i++)
		k->ptr[i] = b->key.ptr[i];

		sectors = min(sectors, b->sectors_free);

		SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
		SET_KEY_SIZE(k, sectors);
		SET_KEY_PTRS(k, KEY_PTRS(&b->key));

		/*
		* Move b to the end of the lru, and keep track of what this bucket was
		* last used for:
		*/
		list_move_tail(&b->list, &c->data_buckets);
		bkey_copy_key(&b->key, k);
		b->last_write_point = write_point;

		b->sectors_free -= sectors;

		for (i = 0; i < KEY_PTRS(&b->key); i++) {
		SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);

		atomic_long_add(sectors,
		&PTR_CACHE(c, &b->key, i)->sectors_written);
		}

		if (b->sectors_free < c->sb.block_size)
		b->sectors_free = 0;

		/*
		* k takes refcounts on the buckets it points to until it's inserted
		* into the btree, but if we're done with this bucket we just transfer
		* get_data_bucket()'s refcount.
		*/
		if (b->sectors_free)
		for (i = 0; i < KEY_PTRS(&b->key); i++)
		atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);

		spin_unlock(&c->data_bucket_lock);
		return true;
		}

		/* Init */

		void bch_open_buckets_free(struct cache_set *c)
		{
		struct open_bucket *b;

		while (!list_empty(&c->data_buckets)) {
		b = list_first_entry(&c->data_buckets,
		struct open_bucket, list);
		list_del(&b->list);
		kfree(b);
		}
		}

		int bch_open_buckets_alloc(struct cache_set *c)
		{
		int i;

		spin_lock_init(&c->data_bucket_lock);

		for (i = 0; i < 6; i++) {
		struct open_bucket b = kzalloc(sizeof(b), GFP_KERNEL);
		if (!b)
		return -ENOMEM;

		list_add(&b->list, &c->data_buckets);
		}

		return 0;
		}

		int bch_cache_allocator_start(struct cache *ca)
		{
		struct task_struct *k = kthread_run(bch_allocator_thread,

drivers/md/bcache/bcache.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -1170,6 +1170,8 @@ int __bch_bucket_alloc_set(struct cache_set *, unsigned,
		struct bkey *, int, bool);
		int bch_bucket_alloc_set(struct cache_set *, unsigned,
		struct bkey *, int, bool);
		bool bch_alloc_sectors(struct cache_set , struct bkey , unsigned,
		unsigned, unsigned, bool);

		__printf(2, 3)
		bool bch_cache_set_error(struct cache_set , const char , ...);
		@@ -1210,6 +1212,8 @@ struct cache_set bch_cache_set_alloc(struct cache_sb );
		void bch_btree_cache_free(struct cache_set *);
		int bch_btree_cache_alloc(struct cache_set *);
		void bch_moving_init_cache_set(struct cache_set *);
		int bch_open_buckets_alloc(struct cache_set *);
		void bch_open_buckets_free(struct cache_set *);

		int bch_cache_allocator_start(struct cache *ca);
		int bch_cache_allocator_init(struct cache *ca);

drivers/md/bcache/request.c

+4 −182

Original line number	Diff line number	Diff line
		@@ -255,186 +255,6 @@ static void bch_data_insert_keys(struct closure *cl)
		closure_return(cl);
		}

		struct open_bucket {
		struct list_head list;
		struct task_struct *last;
		unsigned sectors_free;
		BKEY_PADDED(key);
		};

		void bch_open_buckets_free(struct cache_set *c)
		{
		struct open_bucket *b;

		while (!list_empty(&c->data_buckets)) {
		b = list_first_entry(&c->data_buckets,
		struct open_bucket, list);
		list_del(&b->list);
		kfree(b);
		}
		}

		int bch_open_buckets_alloc(struct cache_set *c)
		{
		int i;

		spin_lock_init(&c->data_bucket_lock);

		for (i = 0; i < 6; i++) {
		struct open_bucket b = kzalloc(sizeof(b), GFP_KERNEL);
		if (!b)
		return -ENOMEM;

		list_add(&b->list, &c->data_buckets);
		}

		return 0;
		}

		/*
		* We keep multiple buckets open for writes, and try to segregate different
		* write streams for better cache utilization: first we look for a bucket where
		* the last write to it was sequential with the current write, and failing that
		* we look for a bucket that was last used by the same task.
		*
		* The ideas is if you've got multiple tasks pulling data into the cache at the
		* same time, you'll get better cache utilization if you try to segregate their
		* data and preserve locality.
		*
		* For example, say you've starting Firefox at the same time you're copying a
		* bunch of files. Firefox will likely end up being fairly hot and stay in the
		* cache awhile, but the data you copied might not be; if you wrote all that
		* data to the same buckets it'd get invalidated at the same time.
		*
		* Both of those tasks will be doing fairly random IO so we can't rely on
		* detecting sequential IO to segregate their data, but going off of the task
		* should be a sane heuristic.
		*/
		static struct open_bucket pick_data_bucket(struct cache_set c,
		const struct bkey *search,
		struct task_struct *task,
		struct bkey *alloc)
		{
		struct open_bucket ret, ret_task = NULL;

		list_for_each_entry_reverse(ret, &c->data_buckets, list)
		if (!bkey_cmp(&ret->key, search))
		goto found;
		else if (ret->last == task)
		ret_task = ret;

		ret = ret_task ?: list_first_entry(&c->data_buckets,
		struct open_bucket, list);
		found:
		if (!ret->sectors_free && KEY_PTRS(alloc)) {
		ret->sectors_free = c->sb.bucket_size;
		bkey_copy(&ret->key, alloc);
		bkey_init(alloc);
		}

		if (!ret->sectors_free)
		ret = NULL;

		return ret;
		}

		/*
		* Allocates some space in the cache to write to, and k to point to the newly
		* allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
		* end of the newly allocated space).
		*
		* May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
		* sectors were actually allocated.
		*
		* If s->writeback is true, will not fail.
		*/
		static bool bch_alloc_sectors(struct data_insert_op *op,
		struct bkey *k, unsigned sectors)
		{
		struct cache_set *c = op->c;
		struct open_bucket *b;
		BKEY_PADDED(key) alloc;
		unsigned i;

		/*
		* We might have to allocate a new bucket, which we can't do with a
		* spinlock held. So if we have to allocate, we drop the lock, allocate
		* and then retry. KEY_PTRS() indicates whether alloc points to
		* allocated bucket(s).
		*/

		bkey_init(&alloc.key);
		spin_lock(&c->data_bucket_lock);

		while (!(b = pick_data_bucket(c, k, op->task, &alloc.key))) {
		unsigned watermark = op->write_prio
		? WATERMARK_MOVINGGC
		: WATERMARK_NONE;

		spin_unlock(&c->data_bucket_lock);

		if (bch_bucket_alloc_set(c, watermark, &alloc.key,
		1, op->writeback))
		return false;

		spin_lock(&c->data_bucket_lock);
		}

		/*
		* If we had to allocate, we might race and not need to allocate the
		* second time we call find_data_bucket(). If we allocated a bucket but
		* didn't use it, drop the refcount bch_bucket_alloc_set() took:
		*/
		if (KEY_PTRS(&alloc.key))
		__bkey_put(c, &alloc.key);

		for (i = 0; i < KEY_PTRS(&b->key); i++)
		EBUG_ON(ptr_stale(c, &b->key, i));

		/* Set up the pointer to the space we're allocating: */

		for (i = 0; i < KEY_PTRS(&b->key); i++)
		k->ptr[i] = b->key.ptr[i];

		sectors = min(sectors, b->sectors_free);

		SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
		SET_KEY_SIZE(k, sectors);
		SET_KEY_PTRS(k, KEY_PTRS(&b->key));

		/*
		* Move b to the end of the lru, and keep track of what this bucket was
		* last used for:
		*/
		list_move_tail(&b->list, &c->data_buckets);
		bkey_copy_key(&b->key, k);
		b->last = op->task;

		b->sectors_free -= sectors;

		for (i = 0; i < KEY_PTRS(&b->key); i++) {
		SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);

		atomic_long_add(sectors,
		&PTR_CACHE(c, &b->key, i)->sectors_written);
		}

		if (b->sectors_free < c->sb.block_size)
		b->sectors_free = 0;

		/*
		* k takes refcounts on the buckets it points to until it's inserted
		* into the btree, but if we're done with this bucket we just transfer
		* get_data_bucket()'s refcount.
		*/
		if (b->sectors_free)
		for (i = 0; i < KEY_PTRS(&b->key); i++)
		atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);

		spin_unlock(&c->data_bucket_lock);
		return true;
		}

		static void bch_data_invalidate(struct closure *cl)
		{
		struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
		@@ -545,7 +365,9 @@ static void bch_data_insert_start(struct closure *cl)
		SET_KEY_INODE(k, op->inode);
		SET_KEY_OFFSET(k, bio->bi_sector);

		if (!bch_alloc_sectors(op, k, bio_sectors(bio)))
		if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),
		op->write_point, op->write_prio,
		op->writeback))
		goto err;

		n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split);
		@@ -968,7 +790,7 @@ static struct search search_alloc(struct bio bio, struct bcache_device *d)
		s->iop.c = d->c;
		s->d = d;
		s->op.lock = -1;
		s->iop.task = current;
		s->iop.write_point = hash_long((unsigned long) current, 16);
		s->orig_bio = bio;
		s->write = (bio->bi_rw & REQ_WRITE) != 0;
		s->iop.flush_journal = (bio->bi_rw & (REQ_FLUSH\|REQ_FUA)) != 0;

drivers/md/bcache/request.h

+1 −4

Original line number	Diff line number	Diff line
		@@ -6,10 +6,10 @@
		struct data_insert_op {
		struct closure cl;
		struct cache_set *c;
		struct task_struct *task;
		struct bio *bio;

		unsigned inode;
		uint16_t write_point;
		uint16_t write_prio;
		short error;

		@@ -31,9 +31,6 @@ struct data_insert_op {
		unsigned bch_get_congested(struct cache_set *);
		void bch_data_insert(struct closure *cl);

		void bch_open_buckets_free(struct cache_set *);
		int bch_open_buckets_alloc(struct cache_set *);

		void bch_cached_dev_request_init(struct cached_dev *dc);
		void bch_flash_dev_request_init(struct bcache_device *d);