dm cache: add cache block invalidation support

E.g.

   dmsetup message my_cache 0 sequential_threshold 1024

Invalidation is removing an entry from the cache without writing it

back.  Cache blocks can be invalidated via the invalidate_cblocks

message, which takes an arbitrary number of cblock ranges.

   invalidate_cblocks [<cblock>|<cblock begin>-<cblock end>]*

E.g.

   dmsetup message my_cache 0 invalidate_cblocks 2345 3456-4567 5678-6789

Examples

========

The test suite can be found here:

https://github.com/jthornber/thinp-test-suite

https://github.com/jthornber/device-mapper-test-suite

dmsetup create my_cache --table '0 41943040 cache /dev/mapper/metadata \

	/dev/mapper/ssd /dev/mapper/origin 512 1 writeback default 0'

	atomic_t discard_count;

};

/*

 * Defines a range of cblocks, begin to (end - 1) are in the range.  end is

 * the one-past-the-end value.

 */

struct cblock_range {

	dm_cblock_t begin;

	dm_cblock_t end;

};

struct invalidation_request {

	struct list_head list;

	struct cblock_range *cblocks;

	atomic_t complete;

	int err;

	wait_queue_head_t result_wait;

};

struct cache {

	struct dm_target *ti;

	struct dm_target_callbacks callbacks;

	bool need_tick_bio:1;

	bool sized:1;

	bool invalidate:1;

	bool commit_requested:1;

	bool loaded_mappings:1;

	bool loaded_discards:1;

	struct cache_features features;

	struct cache_stats stats;

	/*

	 * Invalidation fields.

	 */

	spinlock_t invalidation_lock;

	struct list_head invalidation_requests;

};

struct per_bio_data {

	bool demote:1;

	bool promote:1;

	bool requeue_holder:1;

	bool invalidate:1;

	struct dm_bio_prison_cell *old_ocell;

	struct dm_bio_prison_cell *new_ocell;

			list_add_tail(&mg->list, &cache->quiesced_migrations);

			spin_unlock_irqrestore(&cache->lock, flags);

		} else

		} else {

			if (mg->invalidate)

				policy_remove_mapping(cache->policy, mg->old_oblock);

			cleanup_migration(mg);

		}

	} else {

		if (mg->requeue_holder)

	mg->demote = false;

	mg->promote = true;

	mg->requeue_holder = true;

	mg->invalidate = false;

	mg->cache = cache;

	mg->new_oblock = oblock;

	mg->cblock = cblock;

	mg->demote = false;

	mg->promote = false;

	mg->requeue_holder = true;

	mg->invalidate = false;

	mg->cache = cache;

	mg->old_oblock = oblock;

	mg->cblock = cblock;

	mg->demote = true;

	mg->promote = true;

	mg->requeue_holder = true;

	mg->invalidate = false;

	mg->cache = cache;

	mg->old_oblock = old_oblock;

	mg->new_oblock = new_oblock;

	mg->demote = true;

	mg->promote = false;

	mg->requeue_holder = true;

	mg->invalidate = true;

	mg->cache = cache;

	mg->old_oblock = oblock;

	mg->cblock = cblock;

	prealloc_free_structs(cache, &structs);

}

/*----------------------------------------------------------------

 * Invalidations.

 * Dropping something from the cache *without* writing back.

 *--------------------------------------------------------------*/

static void process_invalidation_request(struct cache *cache, struct invalidation_request *req)

{

	int r = 0;

	uint64_t begin = from_cblock(req->cblocks->begin);

	uint64_t end = from_cblock(req->cblocks->end);

	while (begin != end) {

		r = policy_remove_cblock(cache->policy, to_cblock(begin));

		if (!r) {

			r = dm_cache_remove_mapping(cache->cmd, to_cblock(begin));

			if (r)

				break;

		} else if (r == -ENODATA) {

			/* harmless, already unmapped */

			r = 0;

		} else {

			DMERR("policy_remove_cblock failed");

			break;

		}

		begin++;

        }

	cache->commit_requested = true;

	req->err = r;

	atomic_set(&req->complete, 1);

	wake_up(&req->result_wait);

}

static void process_invalidation_requests(struct cache *cache)

{

	struct list_head list;

	struct invalidation_request *req, *tmp;

	INIT_LIST_HEAD(&list);

	spin_lock(&cache->invalidation_lock);

	list_splice_init(&cache->invalidation_requests, &list);

	spin_unlock(&cache->invalidation_lock);

	list_for_each_entry_safe (req, tmp, &list, list)

		process_invalidation_request(cache, req);

}

/*----------------------------------------------------------------

 * Main worker loop

 *--------------------------------------------------------------*/

			!bio_list_empty(&cache->deferred_writethrough_bios) ||

			!list_empty(&cache->quiesced_migrations) ||

			!list_empty(&cache->completed_migrations) ||

			!list_empty(&cache->need_commit_migrations);

			!list_empty(&cache->need_commit_migrations) ||

			cache->invalidate;

}

static void do_worker(struct work_struct *ws)

			writeback_some_dirty_blocks(cache);

			process_deferred_writethrough_bios(cache);

			process_deferred_bios(cache);

			process_invalidation_requests(cache);

		}

		process_migrations(cache, &cache->quiesced_migrations, issue_copy);

	cache->need_tick_bio = true;

	cache->sized = false;

	cache->invalidate = false;

	cache->commit_requested = false;

	cache->loaded_mappings = false;

	cache->loaded_discards = false;

	atomic_set(&cache->stats.commit_count, 0);

	atomic_set(&cache->stats.discard_count, 0);

	spin_lock_init(&cache->invalidation_lock);

	INIT_LIST_HEAD(&cache->invalidation_requests);

	*result = cache;

	return 0;

}

/*

 * Supports <key> <value>.

 * A cache block range can take two forms:

 *

 * i) A single cblock, eg. '3456'

 * ii) A begin and end cblock with dots between, eg. 123-234

 */

static int parse_cblock_range(struct cache *cache, const char *str,

			      struct cblock_range *result)

{

	char dummy;

	uint64_t b, e;

	int r;

	/*

	 * Try and parse form (ii) first.

	 */

	r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);

	if (r < 0)

		return r;

	if (r == 2) {

		result->begin = to_cblock(b);

		result->end = to_cblock(e);

		return 0;

	}

	/*

	 * That didn't work, try form (i).

	 */

	r = sscanf(str, "%llu%c", &b, &dummy);

	if (r < 0)

		return r;

	if (r == 1) {

		result->begin = to_cblock(b);

		result->end = to_cblock(from_cblock(result->begin) + 1u);

		return 0;

	}

	DMERR("invalid cblock range '%s'", str);

	return -EINVAL;

}

static int validate_cblock_range(struct cache *cache, struct cblock_range *range)

{

	uint64_t b = from_cblock(range->begin);

	uint64_t e = from_cblock(range->end);

	uint64_t n = from_cblock(cache->cache_size);

	if (b >= n) {

		DMERR("begin cblock out of range: %llu >= %llu", b, n);

		return -EINVAL;

	}

	if (e > n) {

		DMERR("end cblock out of range: %llu > %llu", e, n);

		return -EINVAL;

	}

	if (b >= e) {

		DMERR("invalid cblock range: %llu >= %llu", b, e);

		return -EINVAL;

	}

	return 0;

}

static int request_invalidation(struct cache *cache, struct cblock_range *range)

{

	struct invalidation_request req;

	INIT_LIST_HEAD(&req.list);

	req.cblocks = range;

	atomic_set(&req.complete, 0);

	req.err = 0;

	init_waitqueue_head(&req.result_wait);

	spin_lock(&cache->invalidation_lock);

	list_add(&req.list, &cache->invalidation_requests);

	spin_unlock(&cache->invalidation_lock);

	wake_worker(cache);

	wait_event(req.result_wait, atomic_read(&req.complete));

	return req.err;

}

static int process_invalidate_cblocks_message(struct cache *cache, unsigned count,

					      const char **cblock_ranges)

{

	int r = 0;

	unsigned i;

	struct cblock_range range;

	if (!passthrough_mode(&cache->features)) {

		DMERR("cache has to be in passthrough mode for invalidation");

		return -EPERM;

	}

	for (i = 0; i < count; i++) {

		r = parse_cblock_range(cache, cblock_ranges[i], &range);

		if (r)

			break;

		r = validate_cblock_range(cache, &range);

		if (r)

			break;

		/*

		 * Pass begin and end origin blocks to the worker and wake it.

		 */

		r = request_invalidation(cache, &range);

		if (r)

			break;

	}

	return r;

}

/*

 * Supports

 *	"<key> <value>"

 * and

 *     "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*

 *

 * The key migration_threshold is supported by the cache target core.

 */

{

	struct cache *cache = ti->private;

	if (!argc)

		return -EINVAL;

	if (!strcmp(argv[0], "invalidate_cblocks"))

		return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);

	if (argc != 2)

		return -EINVAL;