block: maple: Fix some logic, import former/latter request logic from SIO, and... (dc36c510) · Commits · e / devices / android_kernel_xiaomi_markw

block/maple-iosched.c

+248 −166

Original line number	Original line	Diff line number	Diff line
	/*		/*
	* Maple I/O Scheduler		* Maple I/O Scheduler
	* Based on Zen and TripNDroid.		* Based on Zen and SIO.
	*		*
	* Copyright (C) 2012 Brandon Berhent <bbedward@gmail.com>		* Copyright (C) 2016 Joe Maples <joe@frap129.org>
	* (C) 2014 LoungeKatt <twistedumbrella@gmail.com>		* (C) 2012 Brandon Berhent <bbedward@gmail.com
	* 2015 Fixes to stop crashing on 3.10 by Matthew Alex <matthewalex@outlook.com>		* (C) 2012 Miguel Boton <mboton@gmail.com>
	* (C) 2016 Joe Maples <joe@frap129.org>
	*		*
	* Maple uses Zen's first come first serve style algorithm with seperated read/write		* Maple uses a first come first serve style algorithm with seperated read/write
	* expiry to allow for read biases. By prioritizing reads, simple tasks should improve		* handling to allow for read biases. By prioritizing reads, simple tasks should improve
	* in performance.		* in performance. Maple also uses hooks for the powersuspend driver to increase
			* expirations when power is suspended to decrease workload.
	*/		*/
	#include <linux/blkdev.h>		#include <linux/blkdev.h>
	#include <linux/elevator.h>		#include <linux/elevator.h>
	#include <linux/bio.h>		#include <linux/bio.h>
	#include <linux/module.h>		#include <linux/module.h>
	#include <linux/slab.h>
	#include <linux/init.h>		#include <linux/init.h>
			#include <linux/slab.h>
	#ifdef CONFIG_POWERSUSPEND		#ifdef CONFIG_POWERSUSPEND
	#include <linux/powersuspend.h>		#include <linux/powersuspend.h>
	#endif		#endif

	#define MAPLE_IOSCHED_PATCHLEVEL (3)		#define MAPLE_IOSCHED_PATCHLEVEL (4)

	enum maple_sync { ASYNC, SYNC };		enum { ASYNC, SYNC };

	static const int sync_read_expire = 150; /* max time before a read sync is submitted. */		/* Tunables */
	static const int sync_write_expire = 150; /* max time before a write sync is submitted. */		static const int sync_read_expire = 100; /* max time before a read sync is submitted. */
	static const int async_read_expire = 900; /* ditto for read async, these limits are SOFT! */		static const int sync_write_expire = 350; /* max time before a write sync is submitted. */
	static const int async_write_expire = 900; /* ditto for write async, these limits are SOFT! */		static const int async_read_expire = 200; /* ditto for read async, these limits are SOFT! */
			static const int async_write_expire = 500; /* ditto for write async, these limits are SOFT! */
	static const int fifo_batch = 16; /* # of sequential requests treated as one by the above parameters. */		static const int fifo_batch = 16; /* # of sequential requests treated as one by the above parameters. */
	static const int writes_starved = 3; /* max times reads can starve a write */		static const int writes_starved = 3; /* max times reads can starve a write */
	static const int sleep_latency_multiple = 3; /* multple for expire time when device is asleep */		static const int sleep_latency_multiple = 5; /* multple for expire time when device is asleep */

			/* Elevator data */
	struct maple_data {		struct maple_data {
	/* Runtime Data */		/* Request queues */
	/* Requests are only present on fifo_list */
	struct list_head fifo_list[2][2];		struct list_head fifo_list[2][2];

	unsigned int batching; /* number of sequential requests made */		/* Attributes */
			unsigned int batched;
	unsigned int starved;		unsigned int starved;

	/* tunables */		/* Settings */
	int fifo_expire[2][2];		int fifo_expire[2][2];
	int fifo_batch;		int fifo_batch;
	int writes_starved;		int writes_starved;
	@@ -53,15 +55,13 @@ maple_get_data(struct request_queue *q) {
	return q->elevator->elevator_data;		return q->elevator->elevator_data;
	}		}

	static void maple_dispatch(struct maple_data , struct request );

	static void		static void
	maple_merged_requests(struct request_queue q, struct request rq,		maple_merged_requests(struct request_queue q, struct request rq,
	struct request *next)		struct request *next)
	{		{
	/*		/*
	* if next expires before rq, assign its expire time to arq		* If next expires before rq, assign its expire time to rq
	* and move into next position (next will be deleted) in fifo		* and move into next position (next will be deleted) in fifo.
	*/		*/
	if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist)) {		if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist)) {
	if (time_before(rq_fifo_time(next), rq_fifo_time(rq))) {		if (time_before(rq_fifo_time(next), rq_fifo_time(rq))) {
	@@ -70,16 +70,21 @@ maple_merged_requests(struct request_queue q, struct request rq,
	}		}
	}		}

	/* next request is gone */		/* Delete next request */
	rq_fifo_clear(next);		rq_fifo_clear(next);
	}		}

	static void maple_add_request(struct request_queue q, struct request rq)		static void
			maple_add_request(struct request_queue q, struct request rq)
	{		{
	struct maple_data *mdata = maple_get_data(q);		struct maple_data *mdata = maple_get_data(q);
	const int sync = rq_is_sync(rq);		const int sync = rq_is_sync(rq);
	const int dir = rq_data_dir(rq);		const int dir = rq_data_dir(rq);

			/*
			* Add request to the proper fifo list and set its
			* expire time.
			*/
	#ifdef CONFIG_POWERSUSPEND		#ifdef CONFIG_POWERSUSPEND
	/* inrease expiration when device is asleep */		/* inrease expiration when device is asleep */
	unsigned int fifo_expire_suspended = mdata->fifo_expire[sync][dir] * sleep_latency_multiple;		unsigned int fifo_expire_suspended = mdata->fifo_expire[sync][dir] * sleep_latency_multiple;
	@@ -98,114 +103,175 @@ static void maple_add_request(struct request_queue q, struct request rq)
	#endif		#endif
	}		}

	static void maple_dispatch(struct maple_data mdata, struct request rq)
	{
	/* Remove request from list and dispatch it */
	rq_fifo_clear(rq);
	elv_dispatch_add_tail(rq->q, rq);

	/* Increment # of sequential requests */
	mdata->batching++;

	if (rq_data_dir(rq))
	mdata->starved = 0;
	else
	mdata->starved++;
	}

	/*
	* get the first expired request in direction ddir
	*/
	static struct request *		static struct request *
	maple_expired_request(struct maple_data *mdata, int ddir, int rqtype)		maple_expired_request(struct maple_data *mdata, int sync, int data_dir)
	{		{
			struct list_head *list = &mdata->fifo_list[sync][data_dir];
	struct request *rq;		struct request *rq;

	if (list_empty(&mdata->fifo_list[ddir][rqtype]))		if (list_empty(list))
	return NULL;		return NULL;

	rq = rq_entry_fifo(mdata->fifo_list[ddir][rqtype].next);		/* Retrieve request */
	if (time_after(jiffies, rq_fifo_time(rq)))		rq = rq_entry_fifo(list->next);

			/* Request has expired */
			if (time_after_eq(jiffies, rq_fifo_time(rq)))
	return rq;		return rq;

	return NULL;		return NULL;
	}		}

	/*
	* maple_check_fifo returns 0 if there are no expired requests on the fifo,
	* otherwise it returns the next expired request
	*/
	static struct request *		static struct request *
	maple_check_fifo(struct maple_data *mdata)		maple_choose_expired_request(struct maple_data *mdata)
	{		{
			/* Reset (non-expired-)batch-counter */
			mdata->batched = 0;

	struct request *rq_sync_read = maple_expired_request(mdata, SYNC, READ);		struct request *rq_sync_read = maple_expired_request(mdata, SYNC, READ);
	struct request *rq_sync_write = maple_expired_request(mdata, SYNC, WRITE);		struct request *rq_sync_write = maple_expired_request(mdata, SYNC, WRITE);
	struct request *rq_async_read = maple_expired_request(mdata, ASYNC, READ);		struct request *rq_async_read = maple_expired_request(mdata, ASYNC, READ);
	struct request *rq_async_write = maple_expired_request(mdata, ASYNC, WRITE);		struct request *rq_async_write = maple_expired_request(mdata, ASYNC, WRITE);

			/*
			* Check expired requests.
			* Asynchronous requests have priority over synchronous.
			* Read requests have priority over write.
			*/

	if (rq_async_read && rq_sync_read) {		if (rq_async_read && rq_sync_read) {
	if (time_after(rq_fifo_time(rq_async_read), rq_fifo_time(rq_async_read)))		if (time_after(rq_fifo_time(rq_sync_read), rq_fifo_time(rq_async_read)))
	return rq_sync_read;		return rq_async_read;
	} else if (rq_sync_read) {
	return rq_sync_read;
	} else if (rq_async_read) {		} else if (rq_async_read) {
	return rq_async_read;		return rq_async_read;
			} else if (rq_sync_read) {
			return rq_sync_read;
	}		}

	if (rq_async_write && rq_sync_write) {		if (rq_async_write && rq_sync_write) {
	if (time_after(rq_fifo_time(rq_async_write), rq_fifo_time(rq_sync_write)))		if (time_after(rq_fifo_time(rq_sync_write), rq_fifo_time(rq_async_write)))
	return rq_sync_write;		return rq_async_write;
	} else if (rq_sync_write) {
	return rq_sync_write;
	} else if (rq_async_write) {		} else if (rq_async_write) {
	return rq_async_write;		return rq_async_write;
			} else if (rq_sync_write) {
			return rq_sync_write;
	}		}

	return 0;		return NULL;
	}		}

	static struct request *		static struct request *
	maple_choose_request(struct maple_data *mdata, int rqtype)		maple_choose_request(struct maple_data *mdata, int data_dir)
	{		{
			/* Increase (non-expired-)batch-counter */
			mdata->batched++;

			struct list_head *sync = mdata->fifo_list[SYNC];
			struct list_head *async = mdata->fifo_list[ASYNC];

	/*		/*
	* Retrieve request from available fifo list.		* Retrieve request from available fifo list.
	* Synchronous requests have priority over asynchronous.		* Synchronous requests have priority over asynchronous.
	* Read requests have priority over writes.		* Read requests have priority over write.
	*/		*/
	if (!list_empty(&mdata->fifo_list[SYNC][rqtype]))		if (!list_empty(&sync[data_dir]))
	return rq_entry_fifo(mdata->fifo_list[SYNC][rqtype].next);		return rq_entry_fifo(sync[data_dir].next);
	if (!list_empty(&mdata->fifo_list[ASYNC][rqtype]))		if (!list_empty(&async[data_dir]))
	return rq_entry_fifo(mdata->fifo_list[ASYNC][rqtype].next);		return rq_entry_fifo(async[data_dir].next);

			if (!list_empty(&sync[!data_dir]))
			return rq_entry_fifo(sync[!data_dir].next);
			if (!list_empty(&async[!data_dir]))
			return rq_entry_fifo(async[!data_dir].next);

	return NULL;		return NULL;
	}		}

	static int maple_dispatch_requests(struct request_queue *q, int force)		static inline void
			maple_dispatch_request(struct maple_data mdata, struct request rq)
			{
			/*
			* Remove the request from the fifo list
			* and dispatch it.
			*/
			rq_fifo_clear(rq);
			elv_dispatch_add_tail(rq->q, rq);

			if (rq_data_dir(rq)) {
			mdata->starved = 0;
			} else {
			if (!list_empty(&mdata->fifo_list[SYNC][WRITE]) \|\|
			!list_empty(&mdata->fifo_list[ASYNC][WRITE]))
			mdata->starved++;
			}
			}

			static int
			maple_dispatch_requests(struct request_queue *q, int force)
	{		{
	struct maple_data *mdata = maple_get_data(q);		struct maple_data *mdata = maple_get_data(q);
	struct request *rq = NULL;		struct request *rq = NULL;
	int readwrite = READ;		int data_dir = READ;

	/* Check for and issue expired requests */		/*
	if (mdata->batching > mdata->fifo_batch) {		* Retrieve any expired request after a batch of
	mdata->batching = 0;		* sequential requests.
	rq = maple_check_fifo(mdata);		*/
	}		if (mdata->batched >= mdata->fifo_batch)
			rq = maple_choose_expired_request(mdata);

			/* Retrieve request */
	if (!rq) {		if (!rq) {
			#ifdef CONFIG_POWERSUSPEND
			/* Treat writes fairly while suspended, otherwise allow them to be starved */
			if (!power_suspended && mdata->starved >= mdata->writes_starved)
			data_dir = WRITE;
			else if (power_suspended && mdata->starved >= 1)
			data_dir = WRITE;
			#else
	if (mdata->starved >= mdata->writes_starved)		if (mdata->starved >= mdata->writes_starved)
	readwrite = WRITE;		data_dir = WRITE;
			#endif

	rq = maple_choose_request(mdata, readwrite);		rq = maple_choose_request(mdata, data_dir);
	if (!rq)		if (!rq)
	return 0;		return 0;
	}		}

	maple_dispatch(mdata, rq);		/* Dispatch request */
			maple_dispatch_request(mdata, rq);

	return 1;		return 1;
	}		}

			static struct request *
			maple_former_request(struct request_queue q, struct request rq)
			{
			struct maple_data *mdata = maple_get_data(q);
			const int sync = rq_is_sync(rq);
			const int data_dir = rq_data_dir(rq);

			if (rq->queuelist.prev == &mdata->fifo_list[sync][data_dir])
			return NULL;

			/* Return former request */
			return list_entry(rq->queuelist.prev, struct request, queuelist);
			}

			static struct request *
			maple_latter_request(struct request_queue q, struct request rq)
			{
			struct maple_data *mdata = maple_get_data(q);
			const int sync = rq_is_sync(rq);
			const int data_dir = rq_data_dir(rq);

			if (rq->queuelist.next == &mdata->fifo_list[sync][data_dir])
			return NULL;

			/* Return latter request */
			return list_entry(rq->queuelist.next, struct request, queuelist);
			}

	static int maple_init_queue(struct request_queue q, struct elevator_type e)		static int maple_init_queue(struct request_queue q, struct elevator_type e)
	{		{
	struct maple_data *mdata;		struct maple_data *mdata;
	@@ -215,6 +281,7 @@ static int maple_init_queue(struct request_queue q, struct elevator_type e)
	if (!eq)		if (!eq)
	return -ENOMEM;		return -ENOMEM;

			/* Allocate structure */
	mdata = kmalloc_node(sizeof(*mdata), GFP_KERNEL, q->node);		mdata = kmalloc_node(sizeof(*mdata), GFP_KERNEL, q->node);
	if (!mdata) {		if (!mdata) {
	kobject_put(&eq->kobj);		kobject_put(&eq->kobj);
	@@ -222,14 +289,14 @@ static int maple_init_queue(struct request_queue q, struct elevator_type e)
	}		}
	eq->elevator_data = mdata;		eq->elevator_data = mdata;

	spin_lock_irq(q->queue_lock);		/* Initialize fifo lists */
	q->elevator = eq;
	spin_unlock_irq(q->queue_lock);

	INIT_LIST_HEAD(&mdata->fifo_list[SYNC][READ]);		INIT_LIST_HEAD(&mdata->fifo_list[SYNC][READ]);
	INIT_LIST_HEAD(&mdata->fifo_list[SYNC][WRITE]);		INIT_LIST_HEAD(&mdata->fifo_list[SYNC][WRITE]);
	INIT_LIST_HEAD(&mdata->fifo_list[ASYNC][READ]);		INIT_LIST_HEAD(&mdata->fifo_list[ASYNC][READ]);
	INIT_LIST_HEAD(&mdata->fifo_list[ASYNC][WRITE]);		INIT_LIST_HEAD(&mdata->fifo_list[ASYNC][WRITE]);

			/* Initialize data */
			mdata->batched = 0;
	mdata->fifo_expire[SYNC][READ] = sync_read_expire;		mdata->fifo_expire[SYNC][READ] = sync_read_expire;
	mdata->fifo_expire[SYNC][WRITE] = sync_write_expire;		mdata->fifo_expire[SYNC][WRITE] = sync_write_expire;
	mdata->fifo_expire[ASYNC][READ] = async_read_expire;		mdata->fifo_expire[ASYNC][READ] = async_read_expire;
	@@ -237,10 +304,15 @@ static int maple_init_queue(struct request_queue q, struct elevator_type e)
	mdata->fifo_batch = fifo_batch;		mdata->fifo_batch = fifo_batch;
	mdata->writes_starved = writes_starved;		mdata->writes_starved = writes_starved;
	mdata->sleep_latency_multiple = sleep_latency_multiple;		mdata->sleep_latency_multiple = sleep_latency_multiple;

			spin_lock_irq(q->queue_lock);
			q->elevator = eq;
			spin_unlock_irq(q->queue_lock);
	return 0;		return 0;
	}		}

	static void maple_exit_queue(struct elevator_queue *e)		static void
			maple_exit_queue(struct elevator_queue *e)
	{		{
	struct maple_data *mdata = e->elevator_data;		struct maple_data *mdata = e->elevator_data;

	@@ -248,10 +320,15 @@ static void maple_exit_queue(struct elevator_queue *e)
	BUG_ON(!list_empty(&mdata->fifo_list[SYNC][WRITE]));		BUG_ON(!list_empty(&mdata->fifo_list[SYNC][WRITE]));
	BUG_ON(!list_empty(&mdata->fifo_list[ASYNC][READ]));		BUG_ON(!list_empty(&mdata->fifo_list[ASYNC][READ]));
	BUG_ON(!list_empty(&mdata->fifo_list[ASYNC][WRITE]));		BUG_ON(!list_empty(&mdata->fifo_list[ASYNC][WRITE]));

			/* Free structure */
	kfree(mdata);		kfree(mdata);
	}		}

	/* Sysfs */		/*
			* sysfs code
			*/

	static ssize_t		static ssize_t
	maple_var_show(int var, char *page)		maple_var_show(int var, char *page)
	{		{
	@@ -261,7 +338,9 @@ maple_var_show(int var, char *page)
	static ssize_t		static ssize_t
	maple_var_store(int var, const char page, size_t count)		maple_var_store(int var, const char page, size_t count)
	{		{
	*var = simple_strtol(page, NULL, 10);		char p = (char ) page;

			*var = simple_strtol(p, &p, 10);
	return count;		return count;
	}		}

	@@ -303,9 +382,9 @@ STORE_FUNCTION(maple_sync_read_expire_store, &mdata->fifo_expire[SYNC][READ], 0,
	STORE_FUNCTION(maple_sync_write_expire_store, &mdata->fifo_expire[SYNC][WRITE], 0, INT_MAX, 1);		STORE_FUNCTION(maple_sync_write_expire_store, &mdata->fifo_expire[SYNC][WRITE], 0, INT_MAX, 1);
	STORE_FUNCTION(maple_async_read_expire_store, &mdata->fifo_expire[ASYNC][READ], 0, INT_MAX, 1);		STORE_FUNCTION(maple_async_read_expire_store, &mdata->fifo_expire[ASYNC][READ], 0, INT_MAX, 1);
	STORE_FUNCTION(maple_async_write_expire_store, &mdata->fifo_expire[ASYNC][WRITE], 0, INT_MAX, 1);		STORE_FUNCTION(maple_async_write_expire_store, &mdata->fifo_expire[ASYNC][WRITE], 0, INT_MAX, 1);
	STORE_FUNCTION(maple_fifo_batch_store, &mdata->fifo_batch, 0, INT_MAX, 0);		STORE_FUNCTION(maple_fifo_batch_store, &mdata->fifo_batch, 1, INT_MAX, 0);
	STORE_FUNCTION(maple_writes_starved_store, &mdata->writes_starved, 0, INT_MAX, 0);		STORE_FUNCTION(maple_writes_starved_store, &mdata->writes_starved, 1, INT_MAX, 0);
	STORE_FUNCTION(maple_sleep_latency_multiple_store, &mdata->sleep_latency_multiple, 0, INT_MAX, 0);		STORE_FUNCTION(maple_sleep_latency_multiple_store, &mdata->sleep_latency_multiple, 1, INT_MAX, 0);
	#undef STORE_FUNCTION		#undef STORE_FUNCTION

	#define DD_ATTR(name) \		#define DD_ATTR(name) \
	@@ -328,11 +407,12 @@ static struct elevator_type iosched_maple = {
	.elevator_merge_req_fn = maple_merged_requests,		.elevator_merge_req_fn = maple_merged_requests,
	.elevator_dispatch_fn = maple_dispatch_requests,		.elevator_dispatch_fn = maple_dispatch_requests,
	.elevator_add_req_fn = maple_add_request,		.elevator_add_req_fn = maple_add_request,
	.elevator_former_req_fn = elv_rb_former_request,		.elevator_former_req_fn = maple_former_request,
	.elevator_latter_req_fn = elv_rb_latter_request,		.elevator_latter_req_fn = maple_latter_request,
	.elevator_init_fn = maple_init_queue,		.elevator_init_fn = maple_init_queue,
	.elevator_exit_fn = maple_exit_queue,		.elevator_exit_fn = maple_exit_queue,
	},		},

	.elevator_attrs = maple_attrs,		.elevator_attrs = maple_attrs,
	.elevator_name = "maple",		.elevator_name = "maple",
	.elevator_owner = THIS_MODULE,		.elevator_owner = THIS_MODULE,
	@@ -340,21 +420,23 @@ static struct elevator_type iosched_maple = {

	static int __init maple_init(void)		static int __init maple_init(void)
	{		{
	return elv_register(&iosched_maple);		/* Register elevator */
			elv_register(&iosched_maple);

			return 0;
	}		}

	static void __exit maple_exit(void)		static void __exit maple_exit(void)
	{		{
			/* Unregister elevator */
	elv_unregister(&iosched_maple);		elv_unregister(&iosched_maple);
	}		}

	module_init(maple_init);		module_init(maple_init);
	module_exit(maple_exit);		module_exit(maple_exit);

	MODULE_AUTHOR("Brandon Berhent");
	MODULE_AUTHOR("TripNRaVeR");
	MODULE_AUTHOR("Joe Maples");		MODULE_AUTHOR("Joe Maples");
	MODULE_LICENSE("GPL");		MODULE_LICENSE("GPL");
	MODULE_DESCRIPTION("Maple I/O Scheduler");		MODULE_DESCRIPTION("Maple I/O Scheduler");
	MODULE_VERSION("1.0");		MODULE_VERSION("Testing");