cfq-iosched: Convert from jiffies to nanoseconds (9a7f38c4) · Commits · e / devices / android_kernel_teracube_2e

block/cfq-iosched.c

+136 −137

Original line number	Original line	Diff line number	Diff line
	@@ -10,7 +10,7 @@
	#include <linux/slab.h>		#include <linux/slab.h>
	#include <linux/blkdev.h>		#include <linux/blkdev.h>
	#include <linux/elevator.h>		#include <linux/elevator.h>
	#include <linux/jiffies.h>		#include <linux/ktime.h>
	#include <linux/rbtree.h>		#include <linux/rbtree.h>
	#include <linux/ioprio.h>		#include <linux/ioprio.h>
	#include <linux/blktrace_api.h>		#include <linux/blktrace_api.h>
	@@ -22,28 +22,28 @@
	*/		*/
	/* max queue in one round of service */		/* max queue in one round of service */
	static const int cfq_quantum = 8;		static const int cfq_quantum = 8;
	static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 };		static const u64 cfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 };
	/* maximum backwards seek, in KiB */		/* maximum backwards seek, in KiB */
	static const int cfq_back_max = 16 * 1024;		static const int cfq_back_max = 16 * 1024;
	/* penalty of a backwards seek */		/* penalty of a backwards seek */
	static const int cfq_back_penalty = 2;		static const int cfq_back_penalty = 2;
	static const int cfq_slice_sync = HZ / 10;		static const u64 cfq_slice_sync = NSEC_PER_SEC / 10;
	static int cfq_slice_async = HZ / 25;		static u64 cfq_slice_async = NSEC_PER_SEC / 25;
	static const int cfq_slice_async_rq = 2;		static const int cfq_slice_async_rq = 2;
	static int cfq_slice_idle = HZ / 125;		static u64 cfq_slice_idle = NSEC_PER_SEC / 125;
	static int cfq_group_idle = HZ / 125;		static u64 cfq_group_idle = NSEC_PER_SEC / 125;
	static const int cfq_target_latency = HZ * 3/10; /* 300 ms */		static const u64 cfq_target_latency = (u64)NSEC_PER_SEC * 3/10; /* 300 ms */
	static const int cfq_hist_divisor = 4;		static const int cfq_hist_divisor = 4;

	/*		/*
	* offset from end of service tree		* offset from end of service tree
	*/		*/
	#define CFQ_IDLE_DELAY (HZ / 5)		#define CFQ_IDLE_DELAY (NSEC_PER_SEC / 5)

	/*		/*
	* below this threshold, we consider thinktime immediate		* below this threshold, we consider thinktime immediate
	*/		*/
	#define CFQ_MIN_TT (2)		#define CFQ_MIN_TT (2 * NSEC_PER_SEC / HZ)

	#define CFQ_SLICE_SCALE (5)		#define CFQ_SLICE_SCALE (5)
	#define CFQ_HW_QUEUE_MIN (5)		#define CFQ_HW_QUEUE_MIN (5)
	@@ -73,11 +73,11 @@ static struct kmem_cache *cfq_pool;
	#define CFQ_WEIGHT_LEGACY_MAX 1000		#define CFQ_WEIGHT_LEGACY_MAX 1000

	struct cfq_ttime {		struct cfq_ttime {
	unsigned long last_end_request;		u64 last_end_request;

	unsigned long ttime_total;		u64 ttime_total;
			u64 ttime_mean;
	unsigned long ttime_samples;		unsigned long ttime_samples;
	unsigned long ttime_mean;
	};		};

	/*		/*
	@@ -94,7 +94,7 @@ struct cfq_rb_root {
	struct cfq_ttime ttime;		struct cfq_ttime ttime;
	};		};
	#define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, \		#define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, \
	.ttime = {.last_end_request = jiffies,},}		.ttime = {.last_end_request = ktime_get_ns(),},}

	/*		/*
	* Per process-grouping structure		* Per process-grouping structure
	@@ -109,7 +109,7 @@ struct cfq_queue {
	/* service_tree member */		/* service_tree member */
	struct rb_node rb_node;		struct rb_node rb_node;
	/* service_tree key */		/* service_tree key */
	unsigned long rb_key;		u64 rb_key;
	/* prio tree member */		/* prio tree member */
	struct rb_node p_node;		struct rb_node p_node;
	/* prio tree root we belong to, if any */		/* prio tree root we belong to, if any */
	@@ -126,13 +126,13 @@ struct cfq_queue {
	struct list_head fifo;		struct list_head fifo;

	/* time when queue got scheduled in to dispatch first request. */		/* time when queue got scheduled in to dispatch first request. */
	unsigned long dispatch_start;		u64 dispatch_start;
	unsigned int allocated_slice;		u64 allocated_slice;
	unsigned int slice_dispatch;		u64 slice_dispatch;
	/* time when first request from queue completed and slice started. */		/* time when first request from queue completed and slice started. */
	unsigned long slice_start;		u64 slice_start;
	unsigned long slice_end;		u64 slice_end;
	long slice_resid;		u64 slice_resid;

	/* pending priority requests */		/* pending priority requests */
	int prio_pending;		int prio_pending;
	@@ -290,7 +290,7 @@ struct cfq_group {
	struct cfq_rb_root service_trees[2][3];		struct cfq_rb_root service_trees[2][3];
	struct cfq_rb_root service_tree_idle;		struct cfq_rb_root service_tree_idle;

	unsigned long saved_wl_slice;		u64 saved_wl_slice;
	enum wl_type_t saved_wl_type;		enum wl_type_t saved_wl_type;
	enum wl_class_t saved_wl_class;		enum wl_class_t saved_wl_class;

	@@ -329,7 +329,7 @@ struct cfq_data {
	*/		*/
	enum wl_class_t serving_wl_class;		enum wl_class_t serving_wl_class;
	enum wl_type_t serving_wl_type;		enum wl_type_t serving_wl_type;
	unsigned long workload_expires;		u64 workload_expires;
	struct cfq_group *serving_group;		struct cfq_group *serving_group;

	/*		/*
	@@ -374,22 +374,22 @@ struct cfq_data {
	* tunables, see top of file		* tunables, see top of file
	*/		*/
	unsigned int cfq_quantum;		unsigned int cfq_quantum;
	unsigned int cfq_fifo_expire[2];
	unsigned int cfq_back_penalty;		unsigned int cfq_back_penalty;
	unsigned int cfq_back_max;		unsigned int cfq_back_max;
	unsigned int cfq_slice[2];
	unsigned int cfq_slice_async_rq;		unsigned int cfq_slice_async_rq;
	unsigned int cfq_slice_idle;
	unsigned int cfq_group_idle;
	unsigned int cfq_latency;		unsigned int cfq_latency;
	unsigned int cfq_target_latency;		u64 cfq_fifo_expire[2];
			u64 cfq_slice[2];
			u64 cfq_slice_idle;
			u64 cfq_group_idle;
			u64 cfq_target_latency;

	/*		/*
	* Fallback dummy cfqq for extreme OOM conditions		* Fallback dummy cfqq for extreme OOM conditions
	*/		*/
	struct cfq_queue oom_cfqq;		struct cfq_queue oom_cfqq;

	unsigned long last_delayed_sync;		u64 last_delayed_sync;
	};		};

	static struct cfq_group cfq_get_next_cfqg(struct cfq_data cfqd);		static struct cfq_group cfq_get_next_cfqg(struct cfq_data cfqd);
	@@ -676,7 +676,7 @@ static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
	}		}

	static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,		static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
	unsigned long time, unsigned long unaccounted_time)		uint64_t time, unsigned long unaccounted_time)
	{		{
	blkg_stat_add(&cfqg->stats.time, time);		blkg_stat_add(&cfqg->stats.time, time);
	#ifdef CONFIG_DEBUG_BLK_CGROUP		#ifdef CONFIG_DEBUG_BLK_CGROUP
	@@ -788,7 +788,7 @@ static inline void cfqg_put(struct cfq_group *cfqg) { }
	static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,		static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
	struct cfq_group *curr_cfqg, int op, int op_flags) { }		struct cfq_group *curr_cfqg, int op, int op_flags) { }
	static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,		static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
	unsigned long time, unsigned long unaccounted_time) { }		uint64_t time, unsigned long unaccounted_time) { }
	static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op,		static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op,
	int op_flags) { }		int op_flags) { }
	static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op,		static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op,
	@@ -815,7 +815,7 @@ static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
	static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd,		static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd,
	struct cfq_ttime *ttime, bool group_idle)		struct cfq_ttime *ttime, bool group_idle)
	{		{
	unsigned long slice;		u64 slice;
	if (!sample_valid(ttime->ttime_samples))		if (!sample_valid(ttime->ttime_samples))
	return false;		return false;
	if (group_idle)		if (group_idle)
	@@ -938,17 +938,18 @@ static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
	* if a queue is marked sync and has sync io queued. A sync queue with async		* if a queue is marked sync and has sync io queued. A sync queue with async
	* io only, should not get full sync slice length.		* io only, should not get full sync slice length.
	*/		*/
	static inline int cfq_prio_slice(struct cfq_data *cfqd, bool sync,		static inline u64 cfq_prio_slice(struct cfq_data *cfqd, bool sync,
	unsigned short prio)		unsigned short prio)
	{		{
	const int base_slice = cfqd->cfq_slice[sync];		u64 base_slice = cfqd->cfq_slice[sync];
			u64 slice = div_u64(base_slice, CFQ_SLICE_SCALE);

	WARN_ON(prio >= IOPRIO_BE_NR);		WARN_ON(prio >= IOPRIO_BE_NR);

	return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - prio));		return base_slice + (slice * (4 - prio));
	}		}

	static inline int		static inline u64
	cfq_prio_to_slice(struct cfq_data cfqd, struct cfq_queue cfqq)		cfq_prio_to_slice(struct cfq_data cfqd, struct cfq_queue cfqq)
	{		{
	return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio);		return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio);
	@@ -966,15 +967,14 @@ cfq_prio_to_slice(struct cfq_data cfqd, struct cfq_queue cfqq)
	*		*
	* The result is also in fixed point w/ CFQ_SERVICE_SHIFT.		* The result is also in fixed point w/ CFQ_SERVICE_SHIFT.
	*/		*/
	static inline u64 cfqg_scale_charge(unsigned long charge,		static inline u64 cfqg_scale_charge(u64 charge,
	unsigned int vfraction)		unsigned int vfraction)
	{		{
	u64 c = charge << CFQ_SERVICE_SHIFT; /* make it fixed point */		u64 c = charge << CFQ_SERVICE_SHIFT; /* make it fixed point */

	/* charge / vfraction */		/* charge / vfraction */
	c <<= CFQ_SERVICE_SHIFT;		c <<= CFQ_SERVICE_SHIFT;
	do_div(c, vfraction);		return div_u64(c, vfraction);
	return c;
	}		}

	static inline u64 max_vdisktime(u64 min_vdisktime, u64 vdisktime)		static inline u64 max_vdisktime(u64 min_vdisktime, u64 vdisktime)
	@@ -1027,16 +1027,16 @@ static inline unsigned cfq_group_get_avg_queues(struct cfq_data *cfqd,
	return cfqg->busy_queues_avg[rt];		return cfqg->busy_queues_avg[rt];
	}		}

	static inline unsigned		static inline u64
	cfq_group_slice(struct cfq_data cfqd, struct cfq_group cfqg)		cfq_group_slice(struct cfq_data cfqd, struct cfq_group cfqg)
	{		{
	return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT;		return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT;
	}		}

	static inline unsigned		static inline u64
	cfq_scaled_cfqq_slice(struct cfq_data cfqd, struct cfq_queue cfqq)		cfq_scaled_cfqq_slice(struct cfq_data cfqd, struct cfq_queue cfqq)
	{		{
	unsigned slice = cfq_prio_to_slice(cfqd, cfqq);		u64 slice = cfq_prio_to_slice(cfqd, cfqq);
	if (cfqd->cfq_latency) {		if (cfqd->cfq_latency) {
	/*		/*
	* interested queues (we consider only the ones with the same		* interested queues (we consider only the ones with the same
	@@ -1044,20 +1044,22 @@ cfq_scaled_cfqq_slice(struct cfq_data cfqd, struct cfq_queue cfqq)
	*/		*/
	unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg,		unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg,
	cfq_class_rt(cfqq));		cfq_class_rt(cfqq));
	unsigned sync_slice = cfqd->cfq_slice[1];		u64 sync_slice = cfqd->cfq_slice[1];
	unsigned expect_latency = sync_slice * iq;		u64 expect_latency = sync_slice * iq;
	unsigned group_slice = cfq_group_slice(cfqd, cfqq->cfqg);		u64 group_slice = cfq_group_slice(cfqd, cfqq->cfqg);

	if (expect_latency > group_slice) {		if (expect_latency > group_slice) {
	unsigned base_low_slice = 2 * cfqd->cfq_slice_idle;		u64 base_low_slice = 2 * cfqd->cfq_slice_idle;
			u64 low_slice;

	/* scale low_slice according to IO priority		/* scale low_slice according to IO priority
	* and sync vs async */		* and sync vs async */
	unsigned low_slice =		low_slice = div64_u64(base_low_slice*slice, sync_slice);
	min(slice, base_low_slice * slice / sync_slice);		low_slice = min(slice, low_slice);
	/* the adapted slice value is scaled to fit all iqs		/* the adapted slice value is scaled to fit all iqs
	* into the target latency */		* into the target latency */
	slice = max(slice * group_slice / expect_latency,		slice = div64_u64(slice*group_slice, expect_latency);
	low_slice);		slice = max(slice, low_slice);
	}		}
	}		}
	return slice;		return slice;
	@@ -1066,12 +1068,13 @@ cfq_scaled_cfqq_slice(struct cfq_data cfqd, struct cfq_queue cfqq)
	static inline void		static inline void
	cfq_set_prio_slice(struct cfq_data cfqd, struct cfq_queue cfqq)		cfq_set_prio_slice(struct cfq_data cfqd, struct cfq_queue cfqq)
	{		{
	unsigned slice = cfq_scaled_cfqq_slice(cfqd, cfqq);		u64 slice = cfq_scaled_cfqq_slice(cfqd, cfqq);
			u64 now = ktime_get_ns();

	cfqq->slice_start = jiffies;		cfqq->slice_start = now;
	cfqq->slice_end = jiffies + slice;		cfqq->slice_end = now + slice;
	cfqq->allocated_slice = slice;		cfqq->allocated_slice = slice;
	cfq_log_cfqq(cfqd, cfqq, "set_slice=%lu", cfqq->slice_end - jiffies);		cfq_log_cfqq(cfqd, cfqq, "set_slice=%llu", cfqq->slice_end - now);
	}		}

	/*		/*
	@@ -1083,7 +1086,7 @@ static inline bool cfq_slice_used(struct cfq_queue *cfqq)
	{		{
	if (cfq_cfqq_slice_new(cfqq))		if (cfq_cfqq_slice_new(cfqq))
	return false;		return false;
	if (time_before(jiffies, cfqq->slice_end))		if (ktime_get_ns() < cfqq->slice_end)
	return false;		return false;

	return true;		return true;
	@@ -1249,7 +1252,7 @@ cfq_find_next_rq(struct cfq_data cfqd, struct cfq_queue cfqq,
	return cfq_choose_req(cfqd, next, prev, blk_rq_pos(last));		return cfq_choose_req(cfqd, next, prev, blk_rq_pos(last));
	}		}

	static unsigned long cfq_slice_offset(struct cfq_data *cfqd,		static u64 cfq_slice_offset(struct cfq_data *cfqd,
	struct cfq_queue *cfqq)		struct cfq_queue *cfqq)
	{		{
	/*		/*
	@@ -1443,31 +1446,31 @@ cfq_group_notify_queue_del(struct cfq_data cfqd, struct cfq_group cfqg)
	cfqg_stats_update_dequeue(cfqg);		cfqg_stats_update_dequeue(cfqg);
	}		}

	static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq,		static inline u64 cfq_cfqq_slice_usage(struct cfq_queue *cfqq,
	unsigned int *unaccounted_time)		u64 *unaccounted_time)
	{		{
	unsigned int slice_used;		u64 slice_used;
			u64 now = ktime_get_ns();

	/*		/*
	* Queue got expired before even a single request completed or		* Queue got expired before even a single request completed or
	* got expired immediately after first request completion.		* got expired immediately after first request completion.
	*/		*/
	if (!cfqq->slice_start \|\| cfqq->slice_start == jiffies) {		if (!cfqq->slice_start \|\| cfqq->slice_start == now) {
	/*		/*
	* Also charge the seek time incurred to the group, otherwise		* Also charge the seek time incurred to the group, otherwise
	* if there are mutiple queues in the group, each can dispatch		* if there are mutiple queues in the group, each can dispatch
	* a single request on seeky media and cause lots of seek time		* a single request on seeky media and cause lots of seek time
	* and group will never know it.		* and group will never know it.
	*/		*/
	slice_used = max_t(unsigned, (jiffies - cfqq->dispatch_start),		slice_used = max_t(u64, (now - cfqq->dispatch_start), 1);
	1);
	} else {		} else {
	slice_used = jiffies - cfqq->slice_start;		slice_used = now - cfqq->slice_start;
	if (slice_used > cfqq->allocated_slice) {		if (slice_used > cfqq->allocated_slice) {
	*unaccounted_time = slice_used - cfqq->allocated_slice;		*unaccounted_time = slice_used - cfqq->allocated_slice;
	slice_used = cfqq->allocated_slice;		slice_used = cfqq->allocated_slice;
	}		}
	if (time_after(cfqq->slice_start, cfqq->dispatch_start))		if (cfqq->slice_start > cfqq->dispatch_start)
	*unaccounted_time += cfqq->slice_start -		*unaccounted_time += cfqq->slice_start -
	cfqq->dispatch_start;		cfqq->dispatch_start;
	}		}
	@@ -1479,10 +1482,11 @@ static void cfq_group_served(struct cfq_data cfqd, struct cfq_group cfqg,
	struct cfq_queue *cfqq)		struct cfq_queue *cfqq)
	{		{
	struct cfq_rb_root *st = &cfqd->grp_service_tree;		struct cfq_rb_root *st = &cfqd->grp_service_tree;
	unsigned int used_sl, charge, unaccounted_sl = 0;		u64 used_sl, charge, unaccounted_sl = 0;
	int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg)		int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg)
	- cfqg->service_tree_idle.count;		- cfqg->service_tree_idle.count;
	unsigned int vfr;		unsigned int vfr;
			u64 now = ktime_get_ns();

	BUG_ON(nr_sync < 0);		BUG_ON(nr_sync < 0);
	used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl);		used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl);
	@@ -1504,9 +1508,8 @@ static void cfq_group_served(struct cfq_data cfqd, struct cfq_group cfqg,
	cfq_group_service_tree_add(st, cfqg);		cfq_group_service_tree_add(st, cfqg);

	/* This group is being expired. Save the context */		/* This group is being expired. Save the context */
	if (time_after(cfqd->workload_expires, jiffies)) {		if (cfqd->workload_expires > now) {
	cfqg->saved_wl_slice = cfqd->workload_expires		cfqg->saved_wl_slice = cfqd->workload_expires - now;
	- jiffies;
	cfqg->saved_wl_type = cfqd->serving_wl_type;		cfqg->saved_wl_type = cfqd->serving_wl_type;
	cfqg->saved_wl_class = cfqd->serving_wl_class;		cfqg->saved_wl_class = cfqd->serving_wl_class;
	} else		} else
	@@ -1515,7 +1518,7 @@ static void cfq_group_served(struct cfq_data cfqd, struct cfq_group cfqg,
	cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,		cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,
	st->min_vdisktime);		st->min_vdisktime);
	cfq_log_cfqq(cfqq->cfqd, cfqq,		cfq_log_cfqq(cfqq->cfqd, cfqq,
	"sl_used=%u disp=%u charge=%u iops=%u sect=%lu",		"sl_used=%llu disp=%llu charge=%llu iops=%u sect=%lu",
	used_sl, cfqq->slice_dispatch, charge,		used_sl, cfqq->slice_dispatch, charge,
	iops_mode(cfqd), cfqq->nr_sectors);		iops_mode(cfqd), cfqq->nr_sectors);
	cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl);		cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl);
	@@ -1538,7 +1541,7 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg)
	*st = CFQ_RB_ROOT;		*st = CFQ_RB_ROOT;
	RB_CLEAR_NODE(&cfqg->rb_node);		RB_CLEAR_NODE(&cfqg->rb_node);

	cfqg->ttime.last_end_request = jiffies;		cfqg->ttime.last_end_request = ktime_get_ns();
	}		}

	#ifdef CONFIG_CFQ_GROUP_IOSCHED		#ifdef CONFIG_CFQ_GROUP_IOSCHED
	@@ -2221,10 +2224,11 @@ static void cfq_service_tree_add(struct cfq_data cfqd, struct cfq_queue cfqq,
	{		{
	struct rb_node *p, parent;		struct rb_node *p, parent;
	struct cfq_queue *__cfqq;		struct cfq_queue *__cfqq;
	unsigned long rb_key;		u64 rb_key;
	struct cfq_rb_root *st;		struct cfq_rb_root *st;
	int left;		int left;
	int new_cfqq = 1;		int new_cfqq = 1;
			u64 now = ktime_get_ns();

	st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq));		st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq));
	if (cfq_class_idle(cfqq)) {		if (cfq_class_idle(cfqq)) {
	@@ -2234,7 +2238,7 @@ static void cfq_service_tree_add(struct cfq_data cfqd, struct cfq_queue cfqq,
	__cfqq = rb_entry(parent, struct cfq_queue, rb_node);		__cfqq = rb_entry(parent, struct cfq_queue, rb_node);
	rb_key += __cfqq->rb_key;		rb_key += __cfqq->rb_key;
	} else		} else
	rb_key += jiffies;		rb_key += now;
	} else if (!add_front) {		} else if (!add_front) {
	/*		/*
	* Get our rb key offset. Subtract any residual slice		* Get our rb key offset. Subtract any residual slice
	@@ -2242,13 +2246,13 @@ static void cfq_service_tree_add(struct cfq_data cfqd, struct cfq_queue cfqq,
	* count indicates slice overrun, and this should position		* count indicates slice overrun, and this should position
	* the next service time further away in the tree.		* the next service time further away in the tree.
	*/		*/
	rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies;		rb_key = cfq_slice_offset(cfqd, cfqq) + now;
	rb_key -= cfqq->slice_resid;		rb_key -= cfqq->slice_resid;
	cfqq->slice_resid = 0;		cfqq->slice_resid = 0;
	} else {		} else {
	rb_key = -HZ;		rb_key = -NSEC_PER_SEC;
	__cfqq = cfq_rb_first(st);		__cfqq = cfq_rb_first(st);
	rb_key += __cfqq ? __cfqq->rb_key : jiffies;		rb_key += __cfqq ? __cfqq->rb_key : now;
	}		}

	if (!RB_EMPTY_NODE(&cfqq->rb_node)) {		if (!RB_EMPTY_NODE(&cfqq->rb_node)) {
	@@ -2274,7 +2278,7 @@ static void cfq_service_tree_add(struct cfq_data cfqd, struct cfq_queue cfqq,
	/*		/*
	* sort by key, that represents service time.		* sort by key, that represents service time.
	*/		*/
	if (time_before(rb_key, __cfqq->rb_key))		if (rb_key < __cfqq->rb_key)
	p = &parent->rb_left;		p = &parent->rb_left;
	else {		else {
	p = &parent->rb_right;		p = &parent->rb_right;
	@@ -2574,7 +2578,7 @@ cfq_merged_requests(struct request_queue q, struct request rq,
	* reposition in fifo if next is older than rq		* reposition in fifo if next is older than rq
	*/		*/
	if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&		if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&
	time_before(next->fifo_time, rq->fifo_time) &&		next->fifo_time < rq->fifo_time &&
	cfqq == RQ_CFQQ(next)) {		cfqq == RQ_CFQQ(next)) {
	list_move(&rq->queuelist, &next->queuelist);		list_move(&rq->queuelist, &next->queuelist);
	rq->fifo_time = next->fifo_time;		rq->fifo_time = next->fifo_time;
	@@ -2635,7 +2639,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd,
	cfqd->serving_wl_class, cfqd->serving_wl_type);		cfqd->serving_wl_class, cfqd->serving_wl_type);
	cfqg_stats_update_avg_queue_size(cfqq->cfqg);		cfqg_stats_update_avg_queue_size(cfqq->cfqg);
	cfqq->slice_start = 0;		cfqq->slice_start = 0;
	cfqq->dispatch_start = jiffies;		cfqq->dispatch_start = ktime_get_ns();
	cfqq->allocated_slice = 0;		cfqq->allocated_slice = 0;
	cfqq->slice_end = 0;		cfqq->slice_end = 0;
	cfqq->slice_dispatch = 0;		cfqq->slice_dispatch = 0;
	@@ -2684,8 +2688,8 @@ __cfq_slice_expired(struct cfq_data cfqd, struct cfq_queue cfqq,
	if (cfq_cfqq_slice_new(cfqq))		if (cfq_cfqq_slice_new(cfqq))
	cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq);		cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq);
	else		else
	cfqq->slice_resid = cfqq->slice_end - jiffies;		cfqq->slice_resid = cfqq->slice_end - ktime_get_ns();
	cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid);		cfq_log_cfqq(cfqd, cfqq, "resid=%llu", cfqq->slice_resid);
	}		}

	cfq_group_served(cfqd, cfqq->cfqg, cfqq);		cfq_group_served(cfqd, cfqq->cfqg, cfqq);
	@@ -2919,7 +2923,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
	struct cfq_queue *cfqq = cfqd->active_queue;		struct cfq_queue *cfqq = cfqd->active_queue;
	struct cfq_rb_root *st = cfqq->service_tree;		struct cfq_rb_root *st = cfqq->service_tree;
	struct cfq_io_cq *cic;		struct cfq_io_cq *cic;
	unsigned long sl, group_idle = 0;		u64 sl, group_idle = 0;
			u64 now = ktime_get_ns();

	/*		/*
	* SSD device without seek penalty, disable idling. But only do so		* SSD device without seek penalty, disable idling. But only do so
	@@ -2962,8 +2967,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
	* time slice.		* time slice.
	*/		*/
	if (sample_valid(cic->ttime.ttime_samples) &&		if (sample_valid(cic->ttime.ttime_samples) &&
	(cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) {		(cfqq->slice_end - now < cic->ttime.ttime_mean)) {
	cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%lu",		cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%llu",
	cic->ttime.ttime_mean);		cic->ttime.ttime_mean);
	return;		return;
	}		}
	@@ -2984,9 +2989,9 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
	else		else
	sl = cfqd->cfq_slice_idle;		sl = cfqd->cfq_slice_idle;

	mod_timer(&cfqd->idle_slice_timer, jiffies + sl);		mod_timer(&cfqd->idle_slice_timer, now + sl);
	cfqg_stats_set_start_idle_time(cfqq->cfqg);		cfqg_stats_set_start_idle_time(cfqq->cfqg);
	cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl,		cfq_log_cfqq(cfqd, cfqq, "arm_idle: %llu group_idle: %d", sl,
	group_idle ? 1 : 0);		group_idle ? 1 : 0);
	}		}

	@@ -3026,7 +3031,7 @@ static struct request cfq_check_fifo(struct cfq_queue cfqq)
	return NULL;		return NULL;

	rq = rq_entry_fifo(cfqq->fifo.next);		rq = rq_entry_fifo(cfqq->fifo.next);
	if (time_before(jiffies, rq->fifo_time))		if (ktime_get_ns() < rq->fifo_time)
	rq = NULL;		rq = NULL;

	cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq);		cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq);
	@@ -3104,14 +3109,14 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd,
	struct cfq_queue *queue;		struct cfq_queue *queue;
	int i;		int i;
	bool key_valid = false;		bool key_valid = false;
	unsigned long lowest_key = 0;		u64 lowest_key = 0;
	enum wl_type_t cur_best = SYNC_NOIDLE_WORKLOAD;		enum wl_type_t cur_best = SYNC_NOIDLE_WORKLOAD;

	for (i = 0; i <= SYNC_WORKLOAD; ++i) {		for (i = 0; i <= SYNC_WORKLOAD; ++i) {
	/* select the one with lowest rb_key */		/* select the one with lowest rb_key */
	queue = cfq_rb_first(st_for(cfqg, wl_class, i));		queue = cfq_rb_first(st_for(cfqg, wl_class, i));
	if (queue &&		if (queue &&
	(!key_valid \|\| time_before(queue->rb_key, lowest_key))) {		(!key_valid \|\| queue->rb_key < lowest_key)) {
	lowest_key = queue->rb_key;		lowest_key = queue->rb_key;
	cur_best = i;		cur_best = i;
	key_valid = true;		key_valid = true;
	@@ -3124,11 +3129,12 @@ static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd,
	static void		static void
	choose_wl_class_and_type(struct cfq_data cfqd, struct cfq_group cfqg)		choose_wl_class_and_type(struct cfq_data cfqd, struct cfq_group cfqg)
	{		{
	unsigned slice;		u64 slice;
	unsigned count;		unsigned count;
	struct cfq_rb_root *st;		struct cfq_rb_root *st;
	unsigned group_slice;		u64 group_slice;
	enum wl_class_t original_class = cfqd->serving_wl_class;		enum wl_class_t original_class = cfqd->serving_wl_class;
			u64 now = ktime_get_ns();

	/* Choose next priority. RT > BE > IDLE */		/* Choose next priority. RT > BE > IDLE */
	if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg))		if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg))
	@@ -3137,7 +3143,7 @@ choose_wl_class_and_type(struct cfq_data cfqd, struct cfq_group cfqg)
	cfqd->serving_wl_class = BE_WORKLOAD;		cfqd->serving_wl_class = BE_WORKLOAD;
	else {		else {
	cfqd->serving_wl_class = IDLE_WORKLOAD;		cfqd->serving_wl_class = IDLE_WORKLOAD;
	cfqd->workload_expires = jiffies + 1;		cfqd->workload_expires = now + jiffies_to_nsecs(1);
	return;		return;
	}		}

	@@ -3155,7 +3161,7 @@ choose_wl_class_and_type(struct cfq_data cfqd, struct cfq_group cfqg)
	/*		/*
	* check workload expiration, and that we still have other queues ready		* check workload expiration, and that we still have other queues ready
	*/		*/
	if (count && !time_after(jiffies, cfqd->workload_expires))		if (count && !(now > cfqd->workload_expires))
	return;		return;

	new_workload:		new_workload:
	@@ -3172,13 +3178,13 @@ choose_wl_class_and_type(struct cfq_data cfqd, struct cfq_group cfqg)
	*/		*/
	group_slice = cfq_group_slice(cfqd, cfqg);		group_slice = cfq_group_slice(cfqd, cfqg);

	slice = group_slice * count /		slice = div_u64(group_slice * count,
	max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class],		max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class],
	cfq_group_busy_queues_wl(cfqd->serving_wl_class, cfqd,		cfq_group_busy_queues_wl(cfqd->serving_wl_class, cfqd,
	cfqg));		cfqg)));

	if (cfqd->serving_wl_type == ASYNC_WORKLOAD) {		if (cfqd->serving_wl_type == ASYNC_WORKLOAD) {
	unsigned int tmp;		u64 tmp;

	/*		/*
	* Async queues are currently system wide. Just taking		* Async queues are currently system wide. Just taking
	@@ -3189,19 +3195,19 @@ choose_wl_class_and_type(struct cfq_data cfqd, struct cfq_group cfqg)
	*/		*/
	tmp = cfqd->cfq_target_latency *		tmp = cfqd->cfq_target_latency *
	cfqg_busy_async_queues(cfqd, cfqg);		cfqg_busy_async_queues(cfqd, cfqg);
	tmp = tmp/cfqd->busy_queues;		tmp = div_u64(tmp, cfqd->busy_queues);
	slice = min_t(unsigned, slice, tmp);		slice = min_t(u64, slice, tmp);

	/* async workload slice is scaled down according to		/* async workload slice is scaled down according to
	* the sync/async slice ratio. */		* the sync/async slice ratio. */
	slice = slice * cfqd->cfq_slice[0] / cfqd->cfq_slice[1];		slice = div64_u64(slice*cfqd->cfq_slice[0], cfqd->cfq_slice[1]);
	} else		} else
	/* sync workload slice is at least 2 * cfq_slice_idle */		/* sync workload slice is at least 2 * cfq_slice_idle */
	slice = max(slice, 2 * cfqd->cfq_slice_idle);		slice = max(slice, 2 * cfqd->cfq_slice_idle);

	slice = max_t(unsigned, slice, CFQ_MIN_TT);		slice = max_t(u64, slice, CFQ_MIN_TT);
	cfq_log(cfqd, "workload slice:%d", slice);		cfq_log(cfqd, "workload slice:%llu", slice);
	cfqd->workload_expires = jiffies + slice;		cfqd->workload_expires = now + slice;
	}		}

	static struct cfq_group cfq_get_next_cfqg(struct cfq_data cfqd)		static struct cfq_group cfq_get_next_cfqg(struct cfq_data cfqd)
	@@ -3219,16 +3225,17 @@ static struct cfq_group cfq_get_next_cfqg(struct cfq_data cfqd)
	static void cfq_choose_cfqg(struct cfq_data *cfqd)		static void cfq_choose_cfqg(struct cfq_data *cfqd)
	{		{
	struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd);		struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd);
			u64 now = ktime_get_ns();

	cfqd->serving_group = cfqg;		cfqd->serving_group = cfqg;

	/* Restore the workload type data */		/* Restore the workload type data */
	if (cfqg->saved_wl_slice) {		if (cfqg->saved_wl_slice) {
	cfqd->workload_expires = jiffies + cfqg->saved_wl_slice;		cfqd->workload_expires = now + cfqg->saved_wl_slice;
	cfqd->serving_wl_type = cfqg->saved_wl_type;		cfqd->serving_wl_type = cfqg->saved_wl_type;
	cfqd->serving_wl_class = cfqg->saved_wl_class;		cfqd->serving_wl_class = cfqg->saved_wl_class;
	} else		} else
	cfqd->workload_expires = jiffies - 1;		cfqd->workload_expires = now - 1;

	choose_wl_class_and_type(cfqd, cfqg);		choose_wl_class_and_type(cfqd, cfqg);
	}		}
	@@ -3240,6 +3247,7 @@ static void cfq_choose_cfqg(struct cfq_data *cfqd)
	static struct cfq_queue cfq_select_queue(struct cfq_data cfqd)		static struct cfq_queue cfq_select_queue(struct cfq_data cfqd)
	{		{
	struct cfq_queue cfqq, new_cfqq = NULL;		struct cfq_queue cfqq, new_cfqq = NULL;
			u64 now = ktime_get_ns();

	cfqq = cfqd->active_queue;		cfqq = cfqd->active_queue;
	if (!cfqq)		if (!cfqq)
	@@ -3311,7 +3319,7 @@ static struct cfq_queue cfq_select_queue(struct cfq_data cfqd)
	**/		**/
	if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) &&		if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) &&
	(cfq_cfqq_slice_new(cfqq) \|\|		(cfq_cfqq_slice_new(cfqq) \|\|
	(cfqq->slice_end - jiffies > jiffies - cfqq->slice_start))) {		(cfqq->slice_end - now > now - cfqq->slice_start))) {
	cfq_clear_cfqq_deep(cfqq);		cfq_clear_cfqq_deep(cfqq);
	cfq_clear_cfqq_idle_window(cfqq);		cfq_clear_cfqq_idle_window(cfqq);
	}		}
	@@ -3389,11 +3397,12 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd)
	static inline bool cfq_slice_used_soon(struct cfq_data *cfqd,		static inline bool cfq_slice_used_soon(struct cfq_data *cfqd,
	struct cfq_queue *cfqq)		struct cfq_queue *cfqq)
	{		{
			u64 now = ktime_get_ns();

	/* the queue hasn't finished any request, can't estimate */		/* the queue hasn't finished any request, can't estimate */
	if (cfq_cfqq_slice_new(cfqq))		if (cfq_cfqq_slice_new(cfqq))
	return true;		return true;
	if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched,		if (now + cfqd->cfq_slice_idle * cfqq->dispatched > cfqq->slice_end)
	cfqq->slice_end))
	return true;		return true;

	return false;		return false;
	@@ -3468,10 +3477,10 @@ static bool cfq_may_dispatch(struct cfq_data cfqd, struct cfq_queue cfqq)
	* based on the last sync IO we serviced		* based on the last sync IO we serviced
	*/		*/
	if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) {		if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) {
	unsigned long last_sync = jiffies - cfqd->last_delayed_sync;		u64 last_sync = ktime_get_ns() - cfqd->last_delayed_sync;
	unsigned int depth;		unsigned int depth;

	depth = last_sync / cfqd->cfq_slice[1];		depth = div64_u64(last_sync, cfqd->cfq_slice[1]);
	if (!depth && !cfqq->dispatched)		if (!depth && !cfqq->dispatched)
	depth = 1;		depth = 1;
	if (depth < max_dispatch)		if (depth < max_dispatch)
	@@ -3554,7 +3563,7 @@ static int cfq_dispatch_requests(struct request_queue *q, int force)
	if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) &&		if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) &&
	cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) \|\|		cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) \|\|
	cfq_class_idle(cfqq))) {		cfq_class_idle(cfqq))) {
	cfqq->slice_end = jiffies + 1;		cfqq->slice_end = ktime_get_ns() + 1;
	cfq_slice_expired(cfqd, 0);		cfq_slice_expired(cfqd, 0);
	}		}

	@@ -3632,7 +3641,7 @@ static void cfq_init_icq(struct io_cq *icq)
	{		{
	struct cfq_io_cq *cic = icq_to_cic(icq);		struct cfq_io_cq *cic = icq_to_cic(icq);

	cic->ttime.last_end_request = jiffies;		cic->ttime.last_end_request = ktime_get_ns();
	}		}

	static void cfq_exit_icq(struct io_cq *icq)		static void cfq_exit_icq(struct io_cq *icq)
	@@ -3853,14 +3862,15 @@ cfq_get_queue(struct cfq_data cfqd, bool is_sync, struct cfq_io_cq cic,
	}		}

	static void		static void
	__cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle)		__cfq_update_io_thinktime(struct cfq_ttime *ttime, u64 slice_idle)
	{		{
	unsigned long elapsed = jiffies - ttime->last_end_request;		u64 elapsed = ktime_get_ns() - ttime->last_end_request;
	elapsed = min(elapsed, 2UL * slice_idle);		elapsed = min(elapsed, 2UL * slice_idle);

	ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8;		ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8;
	ttime->ttime_total = (7ttime->ttime_total + 256elapsed) / 8;		ttime->ttime_total = div_u64(7ttime->ttime_total + 256elapsed, 8);
	ttime->ttime_mean = (ttime->ttime_total + 128) / ttime->ttime_samples;		ttime->ttime_mean = div64_ul(ttime->ttime_total + 128,
			ttime->ttime_samples);
	}		}

	static void		static void
	@@ -4113,7 +4123,7 @@ static void cfq_insert_request(struct request_queue q, struct request rq)
	cfq_log_cfqq(cfqd, cfqq, "insert_request");		cfq_log_cfqq(cfqd, cfqq, "insert_request");
	cfq_init_prio_data(cfqq, RQ_CIC(rq));		cfq_init_prio_data(cfqq, RQ_CIC(rq));

	rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)];		rq->fifo_time = ktime_get_ns() + cfqd->cfq_fifo_expire[rq_is_sync(rq)];
	list_add_tail(&rq->queuelist, &cfqq->fifo);		list_add_tail(&rq->queuelist, &cfqq->fifo);
	cfq_add_rq_rb(rq);		cfq_add_rq_rb(rq);
	cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq),		cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq),
	@@ -4161,6 +4171,7 @@ static void cfq_update_hw_tag(struct cfq_data *cfqd)
	static bool cfq_should_wait_busy(struct cfq_data cfqd, struct cfq_queue cfqq)		static bool cfq_should_wait_busy(struct cfq_data cfqd, struct cfq_queue cfqq)
	{		{
	struct cfq_io_cq *cic = cfqd->active_cic;		struct cfq_io_cq *cic = cfqd->active_cic;
			u64 now = ktime_get_ns();

	/* If the queue already has requests, don't wait */		/* If the queue already has requests, don't wait */
	if (!RB_EMPTY_ROOT(&cfqq->sort_list))		if (!RB_EMPTY_ROOT(&cfqq->sort_list))
	@@ -4179,7 +4190,7 @@ static bool cfq_should_wait_busy(struct cfq_data cfqd, struct cfq_queue cfqq)

	/* if slice left is less than think time, wait busy */		/* if slice left is less than think time, wait busy */
	if (cic && sample_valid(cic->ttime.ttime_samples)		if (cic && sample_valid(cic->ttime.ttime_samples)
	&& (cfqq->slice_end - jiffies < cic->ttime.ttime_mean))		&& (cfqq->slice_end - now < cic->ttime.ttime_mean))
	return true;		return true;

	/*		/*
	@@ -4189,7 +4200,7 @@ static bool cfq_should_wait_busy(struct cfq_data cfqd, struct cfq_queue cfqq)
	* case where think time is less than a jiffy, mark the queue wait		* case where think time is less than a jiffy, mark the queue wait
	* busy if only 1 jiffy is left in the slice.		* busy if only 1 jiffy is left in the slice.
	*/		*/
	if (cfqq->slice_end - jiffies == 1)		if (cfqq->slice_end - now <= jiffies_to_nsecs(1))
	return true;		return true;

	return false;		return false;
	@@ -4200,9 +4211,8 @@ static void cfq_completed_request(struct request_queue q, struct request rq)
	struct cfq_queue *cfqq = RQ_CFQQ(rq);		struct cfq_queue *cfqq = RQ_CFQQ(rq);
	struct cfq_data *cfqd = cfqq->cfqd;		struct cfq_data *cfqd = cfqq->cfqd;
	const int sync = rq_is_sync(rq);		const int sync = rq_is_sync(rq);
	unsigned long now;		u64 now = ktime_get_ns();

	now = jiffies;
	cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d",		cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d",
	!!(rq->cmd_flags & REQ_NOIDLE));		!!(rq->cmd_flags & REQ_NOIDLE));

	@@ -4231,7 +4241,7 @@ static void cfq_completed_request(struct request_queue q, struct request rq)
	cfqq_type(cfqq));		cfqq_type(cfqq));

	st->ttime.last_end_request = now;		st->ttime.last_end_request = now;
	if (!time_after(rq->start_time + cfqd->cfq_fifo_expire[1], now))		if (!(rq->start_time + cfqd->cfq_fifo_expire[1] > now))
	cfqd->last_delayed_sync = now;		cfqd->last_delayed_sync = now;
	}		}

	@@ -4256,10 +4266,10 @@ static void cfq_completed_request(struct request_queue q, struct request rq)
	* the queue.		* the queue.
	*/		*/
	if (cfq_should_wait_busy(cfqd, cfqq)) {		if (cfq_should_wait_busy(cfqd, cfqq)) {
	unsigned long extend_sl = cfqd->cfq_slice_idle;		u64 extend_sl = cfqd->cfq_slice_idle;
	if (!cfqd->cfq_slice_idle)		if (!cfqd->cfq_slice_idle)
	extend_sl = cfqd->cfq_group_idle;		extend_sl = cfqd->cfq_group_idle;
	cfqq->slice_end = jiffies + extend_sl;		cfqq->slice_end = now + extend_sl;
	cfq_mark_cfqq_wait_busy(cfqq);		cfq_mark_cfqq_wait_busy(cfqq);
	cfq_log_cfqq(cfqd, cfqq, "will busy wait");		cfq_log_cfqq(cfqd, cfqq, "will busy wait");
	}		}
	@@ -4618,7 +4628,7 @@ static int cfq_init_queue(struct request_queue q, struct elevator_type e)
	* we optimistically start assuming sync ops weren't delayed in last		* we optimistically start assuming sync ops weren't delayed in last
	* second, in order to have larger depth for async operations.		* second, in order to have larger depth for async operations.
	*/		*/
	cfqd->last_delayed_sync = jiffies - HZ;		cfqd->last_delayed_sync = ktime_get_ns() - NSEC_PER_SEC;
	return 0;		return 0;

	out_free:		out_free:
	@@ -4661,9 +4671,9 @@ cfq_var_store(unsigned int var, const char page, size_t count)
	static ssize_t __FUNC(struct elevator_queue e, char page) \		static ssize_t __FUNC(struct elevator_queue e, char page) \
	{ \		{ \
	struct cfq_data *cfqd = e->elevator_data; \		struct cfq_data *cfqd = e->elevator_data; \
	unsigned int __data = __VAR; \		u64 __data = __VAR; \
	if (__CONV) \		if (__CONV) \
	__data = jiffies_to_msecs(__data); \		__data = div_u64(__data, NSEC_PER_MSEC); \
	return cfq_var_show(__data, (page)); \		return cfq_var_show(__data, (page)); \
	}		}
	SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0);		SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0);
	@@ -4691,7 +4701,7 @@ static ssize_t __FUNC(struct elevator_queue e, const char page, size_t count)
	else if (__data > (MAX)) \		else if (__data > (MAX)) \
	__data = (MAX); \		__data = (MAX); \
	if (__CONV) \		if (__CONV) \
	*(__PTR) = msecs_to_jiffies(__data); \		(__PTR) = (u64)__data NSEC_PER_MSEC; \
	else \		else \
	*(__PTR) = __data; \		*(__PTR) = __data; \
	return ret; \		return ret; \
	@@ -4785,18 +4795,7 @@ static int __init cfq_init(void)
	{		{
	int ret;		int ret;

	/*
	* could be 0 on HZ < 1000 setups
	*/
	if (!cfq_slice_async)
	cfq_slice_async = 1;
	if (!cfq_slice_idle)
	cfq_slice_idle = 1;

	#ifdef CONFIG_CFQ_GROUP_IOSCHED		#ifdef CONFIG_CFQ_GROUP_IOSCHED
	if (!cfq_group_idle)
	cfq_group_idle = 1;

	ret = blkcg_policy_register(&blkcg_policy_cfq);		ret = blkcg_policy_register(&blkcg_policy_cfq);
	if (ret)		if (ret)
	return ret;		return ret;