[gentoo-commits] proj/linux-patches:4.7 commit in: /

["Mike Pagano" <mpagano@gentoo.org>]

commit:     b0e8d6a9cbc65c09037bd00a8ee549053007b79e
Author:     Mike Pagano <mpagano <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 22 14:48:04 2016 +0000
Commit:     Mike Pagano <mpagano <AT> gentoo <DOT> org>
CommitDate: Mon Aug 22 14:48:04 2016 +0000
URL:        https://gitweb.gentoo.org/proj/linux-patches.git/commit/?id=b0e8d6a9

Update for BFQ patchset to v8r2. See bug #591828

 0000_README                                        |   2 +-
 ...-BFQ-v7r11-for-4.7.0-into-BFQ-v8r2-for-4.patch1 | 568 +++++++++++++--------
 2 files changed, 350 insertions(+), 220 deletions(-)

diff --git a/0000_README b/0000_README
index 23c35b0..d857e6a 100644
--- a/0000_README
+++ b/0000_README
@@ -91,7 +91,7 @@ Patch:  5003_block-bfq-add-Early-Queue-Merge-EQM-to-BFQ-v7r11-for-4.7.patch
 From:   http://algo.ing.unimo.it/people/paolo/disk_sched/
 Desc:   BFQ v7r11 patch 3 for 4.7: Early Queue Merge (EQM)
 
-Patch:  5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8-for-4.patch2
+Patch:  5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8r2-for-4.patch1
 From:   http://algo.ing.unimo.it/people/paolo/disk_sched/
 Desc:   BFQ v7r11 patch 4 for 4.7: Early Queue Merge (EQM)
 

diff --git a/5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8-for-4.patch1 b/5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8r2-for-4.patch1
similarity index 94%
rename from 5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8-for-4.patch1
rename to 5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8r2-for-4.patch1
index 372f093..cbc051f 100644
--- a/5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8-for-4.patch1
+++ b/5004_blkck-bfq-turn-BFQ-v7r11-for-4.7.0-into-BFQ-v8r2-for-4.patch1
@@ -1,16 +1,16 @@
-From 21d90fdc7488cd7c28f47b5ba759e62c697c0382 Mon Sep 17 00:00:00 2001
+From 0061399c3c07fb8d119c0d581b613b870e63b165 Mon Sep 17 00:00:00 2001
 From: Paolo Valente <paolo.valente@linaro.org>
 Date: Tue, 17 May 2016 08:28:04 +0200
-Subject: [PATCH 4/4] block, bfq: turn BFQ-v7r11 for 4.7.0 into BFQ-v8 for
+Subject: [PATCH 4/4] block, bfq: turn BFQ-v7r11 for 4.7.0 into BFQ-v8r2 for
  4.7.0
 
 ---
  block/Kconfig.iosched |    2 +-
- block/bfq-cgroup.c    |  448 +++++----
- block/bfq-iosched.c   | 2581 +++++++++++++++++++++++++++++--------------------
- block/bfq-sched.c     |  432 +++++++--
- block/bfq.h           |  697 +++++++------
- 5 files changed, 2433 insertions(+), 1727 deletions(-)
+ block/bfq-cgroup.c    |  480 +++++----
+ block/bfq-iosched.c   | 2601 +++++++++++++++++++++++++++++--------------------
+ block/bfq-sched.c     |  441 +++++++--
+ block/bfq.h           |  708 +++++++-------
+ 5 files changed, 2483 insertions(+), 1749 deletions(-)
 
 diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched
 index f78cd1a..6d92579 100644
@@ -26,7 +26,7 @@ index f78cd1a..6d92579 100644
  	---help---
  	  Enable hierarchical scheduling in BFQ, using the blkio controller.
 diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
-index 5ee99ec..bc01663 100644
+index 5ee99ec..c83d90c 100644
 --- a/block/bfq-cgroup.c
 +++ b/block/bfq-cgroup.c
 @@ -162,7 +162,6 @@ static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)
@@ -228,7 +228,7 @@ index 5ee99ec..bc01663 100644
  
  	entity->orig_weight = entity->weight = entity->new_weight = d->weight;
  	entity->my_sched_data = &bfqg->sched_data;
-@@ -445,45 +426,28 @@ static void bfq_pd_free(struct blkg_policy_data *pd)
+@@ -445,70 +426,53 @@ static void bfq_pd_free(struct blkg_policy_data *pd)
  	struct bfq_group *bfqg = pd_to_bfqg(pd);
  
  	bfqg_stats_exit(&bfqg->stats);
@@ -262,30 +262,70 @@ index 5ee99ec..bc01663 100644
 +					struct bfq_group *parent)
  {
 -	struct blkg_rwstat a, b;
--
++	struct bfq_entity *entity;
++
++	BUG_ON(!parent);
++	BUG_ON(!bfqg);
++	BUG_ON(bfqg == parent);
+ 
 -	a = blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off);
 -	b = blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq,
 -				      off + dead_stats_off_delta);
 -	blkg_rwstat_add_aux(&a, &b);
 -	return a;
--}
-+	struct bfq_entity *entity;
++	entity = &bfqg->entity;
++	entity->parent = parent->my_entity;
++	entity->sched_data = &parent->sched_data;
+ }
  
 -static void bfq_pd_reset_stats(struct blkg_policy_data *pd)
--{
++static struct bfq_group *bfq_lookup_bfqg(struct bfq_data *bfqd,
++					 struct blkcg *blkcg)
+ {
 -	struct bfq_group *bfqg = pd_to_bfqg(pd);
-+	BUG_ON(!parent);
-+	BUG_ON(!bfqg);
-+	BUG_ON(bfqg == parent);
++	struct blkcg_gq *blkg;
  
 -	bfqg_stats_reset(&bfqg->stats);
 -	bfqg_stats_reset(&bfqg->dead_stats);
-+	entity = &bfqg->entity;
-+	entity->parent = parent->my_entity;
-+	entity->sched_data = &parent->sched_data;
++	blkg = blkg_lookup(blkcg, bfqd->queue);
++	if (likely(blkg))
++		return blkg_to_bfqg(blkg);
++	return NULL;
  }
  
- static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
+-static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
+-					      struct blkcg *blkcg)
++static struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
++					    struct blkcg *blkcg)
+ {
+-	struct request_queue *q = bfqd->queue;
+-	struct bfq_group *bfqg = NULL, *parent;
+-	struct bfq_entity *entity = NULL;
++	struct bfq_group *bfqg, *parent;
++	struct bfq_entity *entity;
+ 
+ 	assert_spin_locked(bfqd->queue->queue_lock);
+ 
+-	/* avoid lookup for the common case where there's no blkcg */
+-	if (blkcg == &blkcg_root) {
+-		bfqg = bfqd->root_group;
+-	} else {
+-		struct blkcg_gq *blkg;
+-
+-		blkg = blkg_lookup_create(blkcg, q);
+-		if (!IS_ERR(blkg))
+-			bfqg = blkg_to_bfqg(blkg);
+-		else /* fallback to root_group */
+-			bfqg = bfqd->root_group;
+-	}
++	bfqg = bfq_lookup_bfqg(bfqd, blkcg);
+ 
+-	BUG_ON(!bfqg);
++	if (unlikely(!bfqg))
++		return NULL;
+ 
+ 	/*
+ 	 * Update chain of bfq_groups as we might be handling a leaf group
 @@ -531,13 +495,18 @@ static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
  	return bfqg;
  }
@@ -315,11 +355,11 @@ index 5ee99ec..bc01663 100644
 +			  struct bfq_group *bfqg)
  {
 -	int busy, resume;
-+	struct bfq_entity *entity = &bfqq->entity;
- 
+-
 -	busy = bfq_bfqq_busy(bfqq);
 -	resume = !RB_EMPTY_ROOT(&bfqq->sort_list);
--
++	struct bfq_entity *entity = &bfqq->entity;
+ 
 -	BUG_ON(resume && !entity->on_st);
 -	BUG_ON(busy && !resume && entity->on_st &&
 +	BUG_ON(!bfq_bfqq_busy(bfqq) && !RB_EMPTY_ROOT(&bfqq->sort_list));
@@ -383,6 +423,15 @@ index 5ee99ec..bc01663 100644
  }
  
  /**
+@@ -613,7 +599,7 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
+ 
+ 	lockdep_assert_held(bfqd->queue->queue_lock);
+ 
+-	bfqg = bfq_find_alloc_group(bfqd, blkcg);
++	bfqg = bfq_find_set_group(bfqd, blkcg);
+ 	if (async_bfqq) {
+ 		entity = &async_bfqq->entity;
+ 
 @@ -621,7 +607,8 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
  			bic_set_bfqq(bic, NULL, 0);
  			bfq_log_bfqq(bfqd, async_bfqq,
@@ -650,8 +699,9 @@ index 5ee99ec..bc01663 100644
  		return NULL;
  
 -        return blkg_to_bfqg(bfqd->queue->root_blkg);
--}
--
++	return blkg_to_bfqg(bfqd->queue->root_blkg);
+ }
+ 
 -static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp)
 -{
 -        struct bfq_group_data *bgd;
@@ -660,9 +710,8 @@ index 5ee99ec..bc01663 100644
 -        if (!bgd)
 -                return NULL;
 -        return &bgd->pd;
-+	return blkg_to_bfqg(bfqd->queue->root_blkg);
- }
- 
+-}
+-
 -static void bfq_cpd_free(struct blkcg_policy_data *cpd)
 -{
 -        kfree(cpd_to_bfqgd(cpd));
@@ -798,7 +847,7 @@ index 5ee99ec..bc01663 100644
  
  static void bfq_init_entity(struct bfq_entity *entity,
  			    struct bfq_group *bfqg)
-@@ -1146,29 +1145,22 @@ bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
+@@ -1146,27 +1145,20 @@ bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
  	return bfqd->root_group;
  }
  
@@ -815,26 +864,24 @@ index 5ee99ec..bc01663 100644
  }
  
 -static void bfq_disconnect_groups(struct bfq_data *bfqd)
--{
--	bfq_put_async_queues(bfqd, bfqd->root_group);
--}
--
- static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
-                                               struct blkcg *blkcg)
++static struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
++                                              struct blkcg *blkcg)
  {
- 	return bfqd->root_group;
+-	bfq_put_async_queues(bfqd, bfqd->root_group);
++	return bfqd->root_group;
  }
  
+-static struct bfq_group *bfq_find_alloc_group(struct bfq_data *bfqd,
+-                                              struct blkcg *blkcg)
 +static struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
-+{
+ {
+-	return bfqd->root_group;
 +	return bfqq->bfqd->root_group;
-+}
-+
+ }
+ 
  static struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
- {
- 	struct bfq_group *bfqg;
 diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
-index d1f648d..5469442 100644
+index d1f648d..5bff378 100644
 --- a/block/bfq-iosched.c
 +++ b/block/bfq-iosched.c
 @@ -7,25 +7,26 @@
@@ -1321,7 +1368,7 @@ index d1f648d..5469442 100644
  	}
  
  	/*
-@@ -856,25 +875,492 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -856,25 +875,497 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	 * queue. Then we add bfqq to the burst.
  	 */
  	bfq_add_to_burst(bfqd, bfqq);
@@ -1529,12 +1576,15 @@ index d1f648d..5469442 100644
 +{
 +	if (old_wr_coeff == 1 && wr_or_deserves_wr) {
 +		/* start a weight-raising period */
-+		bfqq->wr_coeff = bfqd->bfq_wr_coeff;
-+		if (interactive) /* update wr duration */
++		if (interactive) {
++			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
 +			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
-+		else
++		} else {
++			bfqq->wr_coeff = bfqd->bfq_wr_coeff *
++				BFQ_SOFTRT_WEIGHT_FACTOR;
 +			bfqq->wr_cur_max_time =
 +				bfqd->bfq_wr_rt_max_time;
++		}
 +		/*
 +		 * If needed, further reduce budget to make sure it is
 +		 * close to bfqq's backlog, so as to reduce the
@@ -1614,6 +1664,8 @@ index d1f648d..5469442 100644
 +			bfqq->last_wr_start_finish = jiffies;
 +			bfqq->wr_cur_max_time =
 +				bfqd->bfq_wr_rt_max_time;
++			bfqq->wr_coeff = bfqd->bfq_wr_coeff *
++				BFQ_SOFTRT_WEIGHT_FACTOR;
 +			bfq_log_bfqq(bfqd, bfqq,
 +				     "switching to soft_rt wr, or "
 +				     " just moving forward duration");
@@ -1773,7 +1825,7 @@ index d1f648d..5469442 100644
 +	 * function bfq_bfqq_update_budg_for_activation).
 +	 */
 +	if (bfqd->in_service_queue && bfqq_wants_to_preempt &&
-+	    bfqd->in_service_queue->wr_coeff == 1 &&
++	    bfqd->in_service_queue->wr_coeff < bfqq->wr_coeff &&
 +	    next_queue_may_preempt(bfqd)) {
 +		struct bfq_queue *in_serv =
 +			bfqd->in_service_queue;
@@ -1818,7 +1870,7 @@ index d1f648d..5469442 100644
  	 */
  	prev = bfqq->next_rq;
  	next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position);
-@@ -887,160 +1373,10 @@ static void bfq_add_request(struct request *rq)
+@@ -887,160 +1378,10 @@ static void bfq_add_request(struct request *rq)
  	if (prev != bfqq->next_rq)
  		bfq_pos_tree_add_move(bfqd, bfqq);
  
@@ -1983,7 +2035,7 @@ index d1f648d..5469442 100644
  		if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) &&
  		    time_is_before_jiffies(
  				bfqq->last_wr_start_finish +
-@@ -1049,16 +1385,43 @@ add_bfqq_busy:
+@@ -1049,16 +1390,43 @@ add_bfqq_busy:
  			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
  
  			bfqd->wr_busy_queues++;
@@ -2031,7 +2083,7 @@ index d1f648d..5469442 100644
  	if (bfqd->low_latency &&
  		(old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive))
  		bfqq->last_wr_start_finish = jiffies;
-@@ -1106,6 +1469,9 @@ static void bfq_remove_request(struct request *rq)
+@@ -1106,6 +1474,9 @@ static void bfq_remove_request(struct request *rq)
  	struct bfq_data *bfqd = bfqq->bfqd;
  	const int sync = rq_is_sync(rq);
  
@@ -2041,7 +2093,7 @@ index d1f648d..5469442 100644
  	if (bfqq->next_rq == rq) {
  		bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq);
  		bfq_updated_next_req(bfqd, bfqq);
-@@ -1119,8 +1485,25 @@ static void bfq_remove_request(struct request *rq)
+@@ -1119,8 +1490,25 @@ static void bfq_remove_request(struct request *rq)
  	elv_rb_del(&bfqq->sort_list, rq);
  
  	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
@@ -2068,7 +2120,7 @@ index d1f648d..5469442 100644
  		/*
  		 * Remove queue from request-position tree as it is empty.
  		 */
-@@ -1134,9 +1517,7 @@ static void bfq_remove_request(struct request *rq)
+@@ -1134,9 +1522,7 @@ static void bfq_remove_request(struct request *rq)
  		BUG_ON(bfqq->meta_pending == 0);
  		bfqq->meta_pending--;
  	}
@@ -2078,7 +2130,7 @@ index d1f648d..5469442 100644
  }
  
  static int bfq_merge(struct request_queue *q, struct request **req,
-@@ -1221,21 +1602,25 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq,
+@@ -1221,21 +1607,25 @@ static void bfq_merged_requests(struct request_queue *q, struct request *rq,
  		bfqq->next_rq = rq;
  
  	bfq_remove_request(next);
@@ -2107,7 +2159,7 @@ index d1f648d..5469442 100644
  }
  
  static void bfq_end_wr_async_queues(struct bfq_data *bfqd,
-@@ -1278,7 +1663,7 @@ static int bfq_rq_close_to_sector(void *io_struct, bool request,
+@@ -1278,7 +1668,7 @@ static int bfq_rq_close_to_sector(void *io_struct, bool request,
  				  sector_t sector)
  {
  	return abs(bfq_io_struct_pos(io_struct, request) - sector) <=
@@ -2116,7 +2168,7 @@ index d1f648d..5469442 100644
  }
  
  static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd,
-@@ -1400,7 +1785,7 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
+@@ -1400,7 +1790,7 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
  	 * throughput.
  	 */
  	bfqq->new_bfqq = new_bfqq;
@@ -2125,7 +2177,7 @@ index d1f648d..5469442 100644
  	return new_bfqq;
  }
  
-@@ -1431,9 +1816,23 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
+@@ -1431,9 +1821,23 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
  }
  
  /*
@@ -2152,7 +2204,7 @@ index d1f648d..5469442 100644
   * structure otherwise.
   *
   * The OOM queue is not allowed to participate to cooperation: in fact, since
-@@ -1442,6 +1841,18 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
+@@ -1442,6 +1846,18 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
   * handle merging with the OOM queue would be quite complex and expensive
   * to maintain. Besides, in such a critical condition as an out of memory,
   * the benefits of queue merging may be little relevant, or even negligible.
@@ -2171,7 +2223,7 @@ index d1f648d..5469442 100644
   */
  static struct bfq_queue *
  bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
-@@ -1451,16 +1862,32 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -1451,16 +1867,32 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  
  	if (bfqq->new_bfqq)
  		return bfqq->new_bfqq;
@@ -2207,7 +2259,7 @@ index d1f648d..5469442 100644
  	    unlikely(in_service_bfqq == &bfqd->oom_bfqq))
  		goto check_scheduled;
  
-@@ -1482,7 +1909,15 @@ check_scheduled:
+@@ -1482,7 +1914,15 @@ check_scheduled:
  
  	BUG_ON(new_bfqq && bfqq->entity.parent != new_bfqq->entity.parent);
  
@@ -2224,7 +2276,7 @@ index d1f648d..5469442 100644
  	    bfq_may_be_close_cooperator(bfqq, new_bfqq))
  		return bfq_setup_merge(bfqq, new_bfqq);
  
-@@ -1498,46 +1933,11 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
+@@ -1498,46 +1938,11 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
  	 */
  	if (!bfqq->bic)
  		return;
@@ -2272,7 +2324,7 @@ index d1f648d..5469442 100644
  }
  
  static void bfq_get_bic_reference(struct bfq_queue *bfqq)
-@@ -1562,6 +1962,40 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
+@@ -1562,6 +1967,40 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
  	if (bfq_bfqq_IO_bound(bfqq))
  		bfq_mark_bfqq_IO_bound(new_bfqq);
  	bfq_clear_bfqq_IO_bound(bfqq);
@@ -2313,7 +2365,7 @@ index d1f648d..5469442 100644
  	/*
  	 * Grab a reference to the bic, to prevent it from being destroyed
  	 * before being possibly touched by a bfq_split_bfqq().
-@@ -1588,18 +2022,6 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
+@@ -1588,18 +2027,6 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
  	bfq_put_queue(bfqq);
  }
  
@@ -2332,7 +2384,7 @@ index d1f648d..5469442 100644
  static int bfq_allow_merge(struct request_queue *q, struct request *rq,
  			   struct bio *bio)
  {
-@@ -1637,30 +2059,86 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq,
+@@ -1637,30 +2064,86 @@ static int bfq_allow_merge(struct request_queue *q, struct request *rq,
  			 * to decide whether bio and rq can be merged.
  			 */
  			bfqq = new_bfqq;
@@ -2425,7 +2477,7 @@ index d1f648d..5469442 100644
  
  	bfqd->in_service_queue = bfqq;
  }
-@@ -1676,31 +2154,6 @@ static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd)
+@@ -1676,31 +2159,6 @@ static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd)
  	return bfqq;
  }
  
@@ -2457,12 +2509,10 @@ index d1f648d..5469442 100644
  static void bfq_arm_slice_timer(struct bfq_data *bfqd)
  {
  	struct bfq_queue *bfqq = bfqd->in_service_queue;
-@@ -1723,64 +2176,36 @@ static void bfq_arm_slice_timer(struct bfq_data *bfqd)
- 	 *
- 	 * To prevent processes with (partly) seeky workloads from
+@@ -1725,62 +2183,34 @@ static void bfq_arm_slice_timer(struct bfq_data *bfqd)
  	 * being too ill-treated, grant them a small fraction of the
--	 * assigned budget before reducing the waiting time to
--	 * BFQ_MIN_TT. This happened to help reduce latency.
+ 	 * assigned budget before reducing the waiting time to
+ 	 * BFQ_MIN_TT. This happened to help reduce latency.
 -	 */
 -	sl = bfqd->bfq_slice_idle;
 -	/*
@@ -2507,8 +2557,6 @@ index d1f648d..5469442 100644
 -	bfq_clear_bfqq_budget_new(bfqq);
 -	bfqq->budget_timeout = jiffies +
 -		bfqd->bfq_timeout[bfq_bfqq_sync(bfqq)] * timeout_coeff;
-+	 * assigned budget before reducing the waiting time to
-+	 * BFQ_MIN_TT. This happened to help reduce latency.
 +	 */
 +	sl = bfqd->bfq_slice_idle;
 +	/*
@@ -2545,7 +2593,7 @@ index d1f648d..5469442 100644
  	struct bfq_queue *bfqq = RQ_BFQQ(rq);
  
  	/*
-@@ -1794,15 +2219,9 @@ static void bfq_dispatch_insert(struct request_queue *q, struct request *rq)
+@@ -1794,15 +2224,9 @@ static void bfq_dispatch_insert(struct request_queue *q, struct request *rq)
  	 * incrementing bfqq->dispatched.
  	 */
  	bfqq->dispatched++;
@@ -2562,7 +2610,7 @@ index d1f648d..5469442 100644
  }
  
  /*
-@@ -1822,18 +2241,12 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq)
+@@ -1822,18 +2246,12 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq)
  
  	rq = rq_entry_fifo(bfqq->fifo.next);
  
@@ -2582,7 +2630,7 @@ index d1f648d..5469442 100644
  static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  {
  	BUG_ON(bfqq != bfqd->in_service_queue);
-@@ -1850,12 +2263,15 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -1850,12 +2268,15 @@ static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		bfq_mark_bfqq_split_coop(bfqq);
  
  	if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
@@ -2604,7 +2652,7 @@ index d1f648d..5469442 100644
  		bfq_del_bfqq_busy(bfqd, bfqq, 1);
  	} else {
  		bfq_activate_bfqq(bfqd, bfqq);
-@@ -1882,10 +2298,19 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1882,10 +2303,19 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  	struct request *next_rq;
  	int budget, min_budget;
  
@@ -2626,7 +2674,7 @@ index d1f648d..5469442 100644
  
  	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d",
  		bfqq->entity.budget, bfq_bfqq_budget_left(bfqq));
-@@ -1894,7 +2319,7 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1894,7 +2324,7 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  	bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d",
  		bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue));
  
@@ -2635,7 +2683,7 @@ index d1f648d..5469442 100644
  		switch (reason) {
  		/*
  		 * Caveat: in all the following cases we trade latency
-@@ -1936,14 +2361,10 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1936,14 +2366,10 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  			break;
  		case BFQ_BFQQ_BUDGET_TIMEOUT:
  			/*
@@ -2654,7 +2702,7 @@ index d1f648d..5469442 100644
  			 */
  			budget = min(budget * 2, bfqd->bfq_max_budget);
  			break;
-@@ -1960,17 +2381,49 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1960,17 +2386,49 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  			budget = min(budget * 4, bfqd->bfq_max_budget);
  			break;
  		case BFQ_BFQQ_NO_MORE_REQUESTS:
@@ -2711,7 +2759,7 @@ index d1f648d..5469442 100644
  		 */
  		budget = bfqd->bfq_max_budget;
  
-@@ -1981,65 +2434,105 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
+@@ -1981,65 +2439,105 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd,
  		bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget);
  
  	/*
@@ -2846,7 +2894,7 @@ index d1f648d..5469442 100644
  
  	/*
  	 * Calculate the bandwidth for the last slice.  We use a 64 bit
-@@ -2048,32 +2541,51 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -2048,32 +2546,51 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	 * and to avoid overflows.
  	 */
  	bw = (u64)bfqq->entity.service << BFQ_RATE_SHIFT;
@@ -2914,7 +2962,7 @@ index d1f648d..5469442 100644
  		}
  
  		update |= bfqd->peak_rate_samples == BFQ_PEAK_RATE_SAMPLES - 1;
-@@ -2086,9 +2598,8 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -2086,9 +2603,8 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  			int dev_type = blk_queue_nonrot(bfqd->queue);
  			if (bfqd->bfq_user_max_budget == 0) {
  				bfqd->bfq_max_budget =
@@ -2926,7 +2974,7 @@ index d1f648d..5469442 100644
  					bfqd->bfq_max_budget);
  			}
  			if (bfqd->device_speed == BFQ_BFQD_FAST &&
-@@ -2102,38 +2613,35 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -2102,38 +2618,35 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  				bfqd->RT_prod = R_fast[dev_type] *
  						T_fast[dev_type];
  			}
@@ -2990,7 +3038,7 @@ index d1f648d..5469442 100644
  }
  
  /*
-@@ -2191,6 +2699,15 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -2191,6 +2704,15 @@ static bool bfq_update_peak_rate(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd,
  						struct bfq_queue *bfqq)
  {
@@ -3006,7 +3054,7 @@ index d1f648d..5469442 100644
  	return max(bfqq->last_idle_bklogged +
  		   HZ * bfqq->service_from_backlogged /
  		   bfqd->bfq_wr_max_softrt_rate,
-@@ -2198,13 +2715,21 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd,
+@@ -2198,13 +2720,21 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd,
  }
  
  /*
@@ -3033,7 +3081,7 @@ index d1f648d..5469442 100644
  }
  
  /**
-@@ -2214,28 +2739,24 @@ static unsigned long bfq_infinity_from_now(unsigned long now)
+@@ -2214,28 +2744,24 @@ static unsigned long bfq_infinity_from_now(unsigned long now)
   * @compensate: if true, compensate for the time spent idling.
   * @reason: the reason causing the expiration.
   *
@@ -3079,7 +3127,7 @@ index d1f648d..5469442 100644
   */
  static void bfq_bfqq_expire(struct bfq_data *bfqd,
  			    struct bfq_queue *bfqq,
-@@ -2243,40 +2764,51 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2243,40 +2769,53 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  			    enum bfqq_expiration reason)
  {
  	bool slow;
@@ -3120,13 +3168,6 @@ index d1f648d..5469442 100644
 +	bfqq->service_from_backlogged += entity->service;
  
 -	bfqq->service_from_backlogged += bfqq->entity.service;
--
--	if (BFQQ_SEEKY(bfqq) && reason == BFQ_BFQQ_BUDGET_TIMEOUT &&
--	    !bfq_bfqq_constantly_seeky(bfqq)) {
--		bfq_mark_bfqq_constantly_seeky(bfqq);
--		if (!blk_queue_nonrot(bfqd->queue))
--			bfqd->const_seeky_busy_in_flight_queues++;
--	}
 +	/*
 +	 * As above explained, charge slow (typically seeky) and
 +	 * timed-out queues with the time and not the service
@@ -3148,13 +3189,21 @@ index d1f648d..5469442 100644
 +	      bfq_bfqq_budget_left(bfqq) >=  entity->budget / 3)))
 +		bfq_bfqq_charge_time(bfqd, bfqq, delta);
  
+-	if (BFQQ_SEEKY(bfqq) && reason == BFQ_BFQQ_BUDGET_TIMEOUT &&
+-	    !bfq_bfqq_constantly_seeky(bfqq)) {
+-		bfq_mark_bfqq_constantly_seeky(bfqq);
+-		if (!blk_queue_nonrot(bfqd->queue))
+-			bfqd->const_seeky_busy_in_flight_queues++;
+-	}
++	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
+ 
  	if (reason == BFQ_BFQQ_TOO_IDLE &&
 -	    bfqq->entity.service <= 2 * bfqq->entity.budget / 10 )
 +	    entity->service <= 2 * entity->budget / 10 )
  		bfq_clear_bfqq_IO_bound(bfqq);
  
  	if (bfqd->low_latency && bfqq->wr_coeff == 1)
-@@ -2285,19 +2817,23 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2285,19 +2824,23 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  	if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 &&
  	    RB_EMPTY_ROOT(&bfqq->sort_list)) {
  		/*
@@ -3186,7 +3235,7 @@ index d1f648d..5469442 100644
  			/*
  			 * The application is still waiting for the
  			 * completion of one or more requests:
-@@ -2314,7 +2850,7 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2314,7 +2857,7 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  			 *    happened to be in the past.
  			 */
  			bfqq->soft_rt_next_start =
@@ -3195,7 +3244,7 @@ index d1f648d..5469442 100644
  			/*
  			 * Schedule an update of soft_rt_next_start to when
  			 * the task may be discovered to be isochronous.
-@@ -2324,8 +2860,9 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2324,15 +2867,27 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
  	}
  
  	bfq_log_bfqq(bfqd, bfqq,
@@ -3207,9 +3256,12 @@ index d1f648d..5469442 100644
  
  	/*
  	 * Increase, decrease or leave budget unchanged according to
-@@ -2333,6 +2870,14 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+ 	 * reason.
  	 */
++	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
  	__bfq_bfqq_recalc_budget(bfqd, bfqq, reason);
++	BUG_ON(bfqq->next_rq == NULL &&
++	       bfqq->entity.budget < bfqq->entity.service);
  	__bfq_bfqq_expire(bfqd, bfqq);
 +
 +	BUG_ON(!bfq_bfqq_busy(bfqq) && reason == BFQ_BFQQ_BUDGET_EXHAUSTED &&
@@ -3222,7 +3274,7 @@ index d1f648d..5469442 100644
  }
  
  /*
-@@ -2342,20 +2887,17 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
+@@ -2342,20 +2897,17 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd,
   */
  static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq)
  {
@@ -3251,7 +3303,7 @@ index d1f648d..5469442 100644
  static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq)
  {
  	bfq_log_bfqq(bfqq->bfqd, bfqq,
-@@ -2397,10 +2939,12 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2397,10 +2949,12 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  {
  	struct bfq_data *bfqd = bfqq->bfqd;
  	bool idling_boosts_thr, idling_boosts_thr_without_issues,
@@ -3265,7 +3317,7 @@ index d1f648d..5469442 100644
  	/*
  	 * The next variable takes into account the cases where idling
  	 * boosts the throughput.
-@@ -2422,7 +2966,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2422,7 +2976,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 */
  	idling_boosts_thr = !bfqd->hw_tag ||
  		(!blk_queue_nonrot(bfqd->queue) && bfq_bfqq_IO_bound(bfqq) &&
@@ -3274,7 +3326,7 @@ index d1f648d..5469442 100644
  
  	/*
  	 * The value of the next variable,
-@@ -2463,74 +3007,27 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2463,74 +3017,27 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  		bfqd->wr_busy_queues == 0;
  
  	/*
@@ -3299,7 +3351,8 @@ index d1f648d..5469442 100644
 -	 * it. And, beneficially, this would imply that throughput
 -	 * would always be boosted also with random I/O on NCQ-capable
 -	 * HDDs.
--	 *
++	 * guarantees.
+ 	 *
 -	 * But we must be careful on this point, to avoid an unfair
 -	 * treatment for bfqq. In fact, because of the same above
 -	 * assignments, idling_boosts_thr_without_issues is, on the
@@ -3313,8 +3366,7 @@ index d1f648d..5469442 100644
 -	 * latter might then get a low share of the device throughput,
 -	 * simply because the former would get many requests served
 -	 * after being set as in service, while the latter would not.
-+	 * guarantees.
- 	 *
+-	 *
 -	 * To address this issue, we start by setting to true a
 -	 * sentinel variable, on_hdd_and_not_all_queues_seeky, if the
 -	 * device is rotational and not all queues with pending or
@@ -3368,7 +3420,7 @@ index d1f648d..5469442 100644
  	 * (i)  each of these processes must get the same throughput as
  	 *      the others;
  	 * (ii) all these processes have the same I/O pattern
-@@ -2552,26 +3049,53 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2552,26 +3059,53 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * words, only if sub-condition (i) holds, then idling is
  	 * allowed, and the device tends to be prevented from queueing
  	 * many requests, possibly of several processes. The reason
@@ -3442,7 +3494,7 @@ index d1f648d..5469442 100644
  	 *
  	 * According to the above considerations, the next variable is
  	 * true (only) if sub-condition (i) holds. To compute the
-@@ -2579,7 +3103,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2579,7 +3113,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * the function bfq_symmetric_scenario(), but also check
  	 * whether bfqq is being weight-raised, because
  	 * bfq_symmetric_scenario() does not take into account also
@@ -3451,7 +3503,7 @@ index d1f648d..5469442 100644
  	 * bfq_weights_tree_add()).
  	 *
  	 * As a side note, it is worth considering that the above
-@@ -2601,17 +3125,16 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2601,17 +3135,16 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * bfqq. Such a case is when bfqq became active in a burst of
  	 * queue activations. Queues that became active during a large
  	 * burst benefit only from throughput, as discussed in the
@@ -3474,7 +3526,7 @@ index d1f648d..5469442 100644
  
  	/*
  	 * We have now all the components we need to compute the return
-@@ -2621,6 +3144,14 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2621,6 +3154,14 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
  	 * 2) idling either boosts the throughput (without issues), or
  	 *    is necessary to preserve service guarantees.
  	 */
@@ -3489,7 +3541,7 @@ index d1f648d..5469442 100644
  	return bfq_bfqq_sync(bfqq) &&
  		(idling_boosts_thr_without_issues ||
  		 idling_needed_for_service_guarantees);
-@@ -2632,7 +3163,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
+@@ -2632,7 +3173,7 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
   * 1) the queue must remain in service and cannot be expired, and
   * 2) the device must be idled to wait for the possible arrival of a new
   *    request for the queue.
@@ -3498,7 +3550,7 @@ index d1f648d..5469442 100644
   * why performing device idling is the best choice to boost the throughput
   * and preserve service guarantees when bfq_bfqq_may_idle itself
   * returns true.
-@@ -2698,9 +3229,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
+@@ -2698,9 +3239,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
  				 */
  				bfq_clear_bfqq_wait_request(bfqq);
  				del_timer(&bfqd->idle_slice_timer);
@@ -3508,7 +3560,7 @@ index d1f648d..5469442 100644
  			}
  			goto keep_queue;
  		}
-@@ -2745,14 +3274,11 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -2745,14 +3284,11 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  			bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change");
  
  		/*
@@ -3526,7 +3578,11 @@ index d1f648d..5469442 100644
  		    time_is_before_jiffies(bfqq->last_wr_start_finish +
  					   bfqq->wr_cur_max_time)) {
  			bfqq->last_wr_start_finish = jiffies;
-@@ -2814,10 +3340,25 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
+@@ -2811,13 +3347,29 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
+ 		 */
+ 		if (!bfqd->rq_in_driver)
+ 			bfq_schedule_dispatch(bfqd);
++		BUG_ON(bfqq->entity.budget < bfqq->entity.service);
  		goto expire;
  	}
  
@@ -3552,7 +3608,7 @@ index d1f648d..5469442 100644
  	bfq_update_wr_data(bfqd, bfqq);
  
  	bfq_log_bfqq(bfqd, bfqq,
-@@ -2833,9 +3374,7 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
+@@ -2833,9 +3385,7 @@ static int bfq_dispatch_request(struct bfq_data *bfqd,
  		bfqd->in_service_bic = RQ_BIC(rq);
  	}
  
@@ -3563,7 +3619,7 @@ index d1f648d..5469442 100644
  		goto expire;
  
  	return dispatched;
-@@ -2881,8 +3420,8 @@ static int bfq_forced_dispatch(struct bfq_data *bfqd)
+@@ -2881,8 +3431,8 @@ static int bfq_forced_dispatch(struct bfq_data *bfqd)
  		st = bfq_entity_service_tree(&bfqq->entity);
  
  		dispatched += __bfq_forced_dispatch_bfqq(bfqq);
@@ -3573,7 +3629,7 @@ index d1f648d..5469442 100644
  		bfq_forget_idle(st);
  	}
  
-@@ -2895,9 +3434,9 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+@@ -2895,9 +3445,9 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
  {
  	struct bfq_data *bfqd = q->elevator->elevator_data;
  	struct bfq_queue *bfqq;
@@ -3584,7 +3640,7 @@ index d1f648d..5469442 100644
  	if (bfqd->busy_queues == 0)
  		return 0;
  
-@@ -2908,21 +3447,7 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+@@ -2908,21 +3458,7 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
  	if (!bfqq)
  		return 0;
  
@@ -3607,15 +3663,21 @@ index d1f648d..5469442 100644
  
  	bfq_clear_bfqq_wait_request(bfqq);
  	BUG_ON(timer_pending(&bfqd->idle_slice_timer));
-@@ -2933,6 +3458,7 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+@@ -2933,6 +3469,8 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
  	bfq_log_bfqq(bfqd, bfqq, "dispatched %s request",
  			bfq_bfqq_sync(bfqq) ? "sync" : "async");
  
-+	BUG_ON(bfqq->entity.budget < bfqq->entity.service);
++	BUG_ON(bfqq->next_rq == NULL &&
++	       bfqq->entity.budget < bfqq->entity.service);
  	return 1;
  }
  
-@@ -2949,11 +3475,11 @@ static void bfq_put_queue(struct bfq_queue *bfqq)
+@@ -2944,23 +3482,22 @@ static int bfq_dispatch_requests(struct request_queue *q, int force)
+  */
+ static void bfq_put_queue(struct bfq_queue *bfqq)
+ {
+-	struct bfq_data *bfqd = bfqq->bfqd;
+ #ifdef CONFIG_BFQ_GROUP_IOSCHED
  	struct bfq_group *bfqg = bfqq_group(bfqq);
  #endif
  
@@ -3625,13 +3687,30 @@ index d1f648d..5469442 100644
 -	bfq_log_bfqq(bfqd, bfqq, "put_queue: %p %d", bfqq,
 -		     atomic_read(&bfqq->ref));
 -	if (!atomic_dec_and_test(&bfqq->ref))
-+	bfq_log_bfqq(bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref);
++	bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref);
 +	bfqq->ref--;
 +	if (bfqq->ref)
  		return;
  
  	BUG_ON(rb_first(&bfqq->sort_list));
-@@ -3007,8 +3533,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+ 	BUG_ON(bfqq->allocated[READ] + bfqq->allocated[WRITE] != 0);
+ 	BUG_ON(bfqq->entity.tree);
+ 	BUG_ON(bfq_bfqq_busy(bfqq));
+-	BUG_ON(bfqd->in_service_queue == bfqq);
++	BUG_ON(bfqq->bfqd->in_service_queue == bfqq);
+ 
+ 	if (bfq_bfqq_sync(bfqq))
+ 		/*
+@@ -2973,7 +3510,7 @@ static void bfq_put_queue(struct bfq_queue *bfqq)
+ 		 */
+ 		hlist_del_init(&bfqq->burst_list_node);
+ 
+-	bfq_log_bfqq(bfqd, bfqq, "put_queue: %p freed", bfqq);
++	bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p freed", bfqq);
+ 
+ 	kmem_cache_free(bfq_pool, bfqq);
+ #ifdef CONFIG_BFQ_GROUP_IOSCHED
+@@ -3007,8 +3544,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  		bfq_schedule_dispatch(bfqd);
  	}
  
@@ -3641,7 +3720,7 @@ index d1f648d..5469442 100644
  
  	bfq_put_cooperator(bfqq);
  
-@@ -3019,26 +3544,7 @@ static void bfq_init_icq(struct io_cq *icq)
+@@ -3019,26 +3555,7 @@ static void bfq_init_icq(struct io_cq *icq)
  {
  	struct bfq_io_cq *bic = icq_to_bic(icq);
  
@@ -3669,7 +3748,7 @@ index d1f648d..5469442 100644
  }
  
  static void bfq_exit_icq(struct io_cq *icq)
-@@ -3046,21 +3552,21 @@ static void bfq_exit_icq(struct io_cq *icq)
+@@ -3046,21 +3563,21 @@ static void bfq_exit_icq(struct io_cq *icq)
  	struct bfq_io_cq *bic = icq_to_bic(icq);
  	struct bfq_data *bfqd = bic_to_bfqd(bic);
  
@@ -3698,7 +3777,7 @@ index d1f648d..5469442 100644
  	}
  }
  
-@@ -3068,7 +3574,8 @@ static void bfq_exit_icq(struct io_cq *icq)
+@@ -3068,7 +3585,8 @@ static void bfq_exit_icq(struct io_cq *icq)
   * Update the entity prio values; note that the new values will not
   * be used until the next (re)activation.
   */
@@ -3708,7 +3787,7 @@ index d1f648d..5469442 100644
  {
  	struct task_struct *tsk = current;
  	int ioprio_class;
-@@ -3100,7 +3607,7 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *b
+@@ -3100,7 +3618,7 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *b
  		break;
  	}
  
@@ -3717,7 +3796,7 @@ index d1f648d..5469442 100644
  		printk(KERN_CRIT "bfq_set_next_ioprio_data: new_ioprio %d\n",
  				 bfqq->new_ioprio);
  		BUG();
-@@ -3108,45 +3615,40 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *b
+@@ -3108,45 +3626,40 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *b
  
  	bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio);
  	bfqq->entity.prio_changed = 1;
@@ -3777,7 +3856,7 @@ index d1f648d..5469442 100644
  }
  
  static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
-@@ -3155,8 +3657,9 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3155,8 +3668,9 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	RB_CLEAR_NODE(&bfqq->entity.rb_node);
  	INIT_LIST_HEAD(&bfqq->fifo);
  	INIT_HLIST_NODE(&bfqq->burst_list_node);
@@ -3788,7 +3867,7 @@ index d1f648d..5469442 100644
  	bfqq->bfqd = bfqd;
  
  	if (bic)
-@@ -3166,6 +3669,7 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3166,6 +3680,7 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  		if (!bfq_class_idle(bfqq))
  			bfq_mark_bfqq_idle_window(bfqq);
  		bfq_mark_bfqq_sync(bfqq);
@@ -3796,7 +3875,7 @@ index d1f648d..5469442 100644
  	} else
  		bfq_clear_bfqq_sync(bfqq);
  	bfq_mark_bfqq_IO_bound(bfqq);
-@@ -3175,72 +3679,17 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3175,72 +3690,17 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	bfqq->pid = pid;
  
  	bfqq->wr_coeff = 1;
@@ -3865,17 +3944,17 @@ index d1f648d..5469442 100644
 -
 -	if (new_bfqq)
 -		kmem_cache_free(bfq_pool, new_bfqq);
+-
+-	rcu_read_unlock();
 +	bfqq->soft_rt_next_start = bfq_greatest_from_now();
  
--	rcu_read_unlock();
--
 -	return bfqq;
 +	/* first request is almost certainly seeky */
 +	bfqq->seek_history = 1;
  }
  
  static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
-@@ -3263,44 +3712,56 @@ static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
+@@ -3263,44 +3723,60 @@ static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
  }
  
  static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
@@ -3895,13 +3974,17 @@ index d1f648d..5469442 100644
 -		struct blkcg *blkcg;
 -		struct bfq_group *bfqg;
 +	rcu_read_lock();
++
++	bfqg = bfq_find_set_group(bfqd,bio_blkcg(bio));
++	if (!bfqg) {
++		bfqq = &bfqd->oom_bfqq;
++		goto out;
++	}
  
 -		rcu_read_lock();
 -		blkcg = bio_blkcg(bio);
 -		rcu_read_unlock();
 -		bfqg = bfq_find_alloc_group(bfqd, blkcg);
-+	bfqg = bfq_find_alloc_group(bfqd,bio_blkcg(bio));
-+
 +	if (!is_sync) {
  		async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class,
  						  ioprio);
@@ -3950,7 +4033,7 @@ index d1f648d..5469442 100644
  	return bfqq;
  }
  
-@@ -3316,37 +3777,21 @@ static void bfq_update_io_thinktime(struct bfq_data *bfqd,
+@@ -3316,37 +3792,21 @@ static void bfq_update_io_thinktime(struct bfq_data *bfqd,
  				bic->ttime.ttime_samples;
  }
  
@@ -4001,7 +4084,7 @@ index d1f648d..5469442 100644
  }
  
  /*
-@@ -3364,7 +3809,8 @@ static void bfq_update_idle_window(struct bfq_data *bfqd,
+@@ -3364,7 +3824,8 @@ static void bfq_update_idle_window(struct bfq_data *bfqd,
  		return;
  
  	/* Idle window just restored, statistics are meaningless. */
@@ -4011,7 +4094,7 @@ index d1f648d..5469442 100644
  		return;
  
  	enable_idle = bfq_bfqq_idle_window(bfqq);
-@@ -3404,22 +3850,13 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3404,22 +3865,13 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  
  	bfq_update_io_thinktime(bfqd, bic);
  	bfq_update_io_seektime(bfqd, bfqq, rq);
@@ -4036,7 +4119,7 @@ index d1f648d..5469442 100644
  
  	bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
  
-@@ -3433,14 +3870,15 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3433,14 +3885,15 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  		 * is small and the queue is not to be expired, then
  		 * just exit.
  		 *
@@ -4060,7 +4143,7 @@ index d1f648d..5469442 100644
  		 */
  		if (small_req && !budget_timeout)
  			return;
-@@ -3453,9 +3891,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -3453,9 +3906,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  		 */
  		bfq_clear_bfqq_wait_request(bfqq);
  		del_timer(&bfqd->idle_slice_timer);
@@ -4070,7 +4153,7 @@ index d1f648d..5469442 100644
  
  		/*
  		 * The queue is not empty, because a new request just
-@@ -3499,27 +3935,19 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
+@@ -3499,27 +3950,19 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
  			 */
  			new_bfqq->allocated[rq_data_dir(rq)]++;
  			bfqq->allocated[rq_data_dir(rq)]--;
@@ -4101,7 +4184,7 @@ index d1f648d..5469442 100644
  	rq->fifo_time = jiffies + bfqd->bfq_fifo_expire[rq_is_sync(rq)];
  	list_add_tail(&rq->queuelist, &bfqq->fifo);
  
-@@ -3528,8 +3956,8 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
+@@ -3528,8 +3971,8 @@ static void bfq_insert_request(struct request_queue *q, struct request *rq)
  
  static void bfq_update_hw_tag(struct bfq_data *bfqd)
  {
@@ -4112,9 +4195,16 @@ index d1f648d..5469442 100644
  
  	if (bfqd->hw_tag == 1)
  		return;
-@@ -3560,43 +3988,41 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
- 	bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left (%d)",
- 		     blk_rq_sectors(rq), sync);
+@@ -3555,48 +3998,45 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
+ {
+ 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
+ 	struct bfq_data *bfqd = bfqq->bfqd;
+-	bool sync = bfq_bfqq_sync(bfqq);
+ 
+-	bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left (%d)",
+-		     blk_rq_sectors(rq), sync);
++	bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left",
++		     blk_rq_sectors(rq));
  
 +	assert_spin_locked(bfqd->queue->queue_lock);
  	bfq_update_hw_tag(bfqd);
@@ -4174,7 +4264,7 @@ index d1f648d..5469442 100644
  	 */
  	if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 &&
  	    RB_EMPTY_ROOT(&bfqq->sort_list))
-@@ -3608,10 +4034,7 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
+@@ -3608,10 +4048,7 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq)
  	 * or if we want to idle in case it has no pending requests.
  	 */
  	if (bfqd->in_service_queue == bfqq) {
@@ -4186,7 +4276,7 @@ index d1f648d..5469442 100644
  			bfq_arm_slice_timer(bfqd);
  			goto out;
  		} else if (bfq_may_expire_for_budg_timeout(bfqq))
-@@ -3682,14 +4105,14 @@ static void bfq_put_request(struct request *rq)
+@@ -3682,14 +4119,14 @@ static void bfq_put_request(struct request *rq)
  		rq->elv.priv[1] = NULL;
  
  		bfq_log_bfqq(bfqq->bfqd, bfqq, "put_request %p, %d",
@@ -4203,7 +4293,7 @@ index d1f648d..5469442 100644
   */
  static struct bfq_queue *
  bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)
-@@ -3727,11 +4150,8 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
+@@ -3727,11 +4164,8 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  	unsigned long flags;
  	bool split = false;
  
@@ -4216,7 +4306,7 @@ index d1f648d..5469442 100644
  
  	if (!bic)
  		goto queue_fail;
-@@ -3741,23 +4161,47 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
+@@ -3741,23 +4175,47 @@ static int bfq_set_request(struct request_queue *q, struct request *rq,
  new_queue:
  	bfqq = bic_to_bfqq(bic, is_sync);
  	if (!bfqq || bfqq == &bfqd->oom_bfqq) {
@@ -4271,7 +4361,7 @@ index d1f648d..5469442 100644
  			bfqq = bfq_split_bfqq(bic, bfqq);
  			split = true;
  			if (!bfqq)
-@@ -3766,9 +4210,8 @@ new_queue:
+@@ -3766,9 +4224,8 @@ new_queue:
  	}
  
  	bfqq->allocated[rw]++;
@@ -4283,7 +4373,7 @@ index d1f648d..5469442 100644
  
  	rq->elv.priv[0] = bic;
  	rq->elv.priv[1] = bfqq;
-@@ -3783,7 +4226,6 @@ new_queue:
+@@ -3783,7 +4240,6 @@ new_queue:
  	if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) {
  		bfqq->bic = bic;
  		if (split) {
@@ -4291,7 +4381,7 @@ index d1f648d..5469442 100644
  			/*
  			 * If the queue has just been split from a shared
  			 * queue, restore the idle window and the possible
-@@ -3793,6 +4235,9 @@ new_queue:
+@@ -3793,6 +4249,9 @@ new_queue:
  		}
  	}
  
@@ -4301,7 +4391,7 @@ index d1f648d..5469442 100644
  	spin_unlock_irqrestore(q->queue_lock, flags);
  
  	return 0;
-@@ -3872,6 +4317,7 @@ static void bfq_shutdown_timer_wq(struct bfq_data *bfqd)
+@@ -3872,6 +4331,7 @@ static void bfq_shutdown_timer_wq(struct bfq_data *bfqd)
  	cancel_work_sync(&bfqd->unplug_work);
  }
  
@@ -4309,7 +4399,7 @@ index d1f648d..5469442 100644
  static void __bfq_put_async_bfqq(struct bfq_data *bfqd,
  					struct bfq_queue **bfqq_ptr)
  {
-@@ -3880,9 +4326,9 @@ static void __bfq_put_async_bfqq(struct bfq_data *bfqd,
+@@ -3880,9 +4340,9 @@ static void __bfq_put_async_bfqq(struct bfq_data *bfqd,
  
  	bfq_log(bfqd, "put_async_bfqq: %p", bfqq);
  	if (bfqq) {
@@ -4321,7 +4411,7 @@ index d1f648d..5469442 100644
  		bfq_put_queue(bfqq);
  		*bfqq_ptr = NULL;
  	}
-@@ -3904,6 +4350,7 @@ static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg)
+@@ -3904,6 +4364,7 @@ static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg)
  
  	__bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq);
  }
@@ -4329,7 +4419,7 @@ index d1f648d..5469442 100644
  
  static void bfq_exit_queue(struct elevator_queue *e)
  {
-@@ -3923,8 +4370,6 @@ static void bfq_exit_queue(struct elevator_queue *e)
+@@ -3923,8 +4384,6 @@ static void bfq_exit_queue(struct elevator_queue *e)
  
  	bfq_shutdown_timer_wq(bfqd);
  
@@ -4338,7 +4428,7 @@ index d1f648d..5469442 100644
  	BUG_ON(timer_pending(&bfqd->idle_slice_timer));
  
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
-@@ -3973,11 +4418,14 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -3973,11 +4432,14 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  	 * will not attempt to free it.
  	 */
  	bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, NULL, 1, 0);
@@ -4354,7 +4444,7 @@ index d1f648d..5469442 100644
  	/*
  	 * Trigger weight initialization, according to ioprio, at the
  	 * oom_bfqq's first activation. The oom_bfqq's ioprio and ioprio
-@@ -3996,9 +4444,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -3996,9 +4458,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  		goto out_free;
  	bfq_init_root_group(bfqd->root_group, bfqd);
  	bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group);
@@ -4364,7 +4454,7 @@ index d1f648d..5469442 100644
  
  	init_timer(&bfqd->idle_slice_timer);
  	bfqd->idle_slice_timer.function = bfq_idle_slice_timer;
-@@ -4023,20 +4468,19 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -4023,20 +4482,19 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  	bfqd->bfq_back_penalty = bfq_back_penalty;
  	bfqd->bfq_slice_idle = bfq_slice_idle;
  	bfqd->bfq_class_idle_last_service = 0;
@@ -4392,7 +4482,7 @@ index d1f648d..5469442 100644
  	bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300);
  	bfqd->bfq_wr_max_time = 0;
  	bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000);
-@@ -4048,16 +4492,15 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
+@@ -4048,16 +4506,15 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
  					      * video.
  					      */
  	bfqd->wr_busy_queues = 0;
@@ -4413,7 +4503,7 @@ index d1f648d..5469442 100644
  	bfqd->device_speed = BFQ_BFQD_FAST;
  
  	return 0;
-@@ -4161,10 +4604,8 @@ SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0);
+@@ -4161,10 +4618,8 @@ SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0);
  SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0);
  SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 1);
  SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0);
@@ -4426,7 +4516,7 @@ index d1f648d..5469442 100644
  SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0);
  SHOW_FUNCTION(bfq_wr_coeff_show, bfqd->bfq_wr_coeff, 0);
  SHOW_FUNCTION(bfq_wr_rt_max_time_show, bfqd->bfq_wr_rt_max_time, 1);
-@@ -4199,10 +4640,6 @@ STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0);
+@@ -4199,10 +4654,6 @@ STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0);
  STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1,
  		INT_MAX, 0);
  STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 1);
@@ -4437,7 +4527,7 @@ index d1f648d..5469442 100644
  STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0);
  STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1);
  STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX,
-@@ -4224,10 +4661,8 @@ static ssize_t bfq_weights_store(struct elevator_queue *e,
+@@ -4224,10 +4675,8 @@ static ssize_t bfq_weights_store(struct elevator_queue *e,
  
  static unsigned long bfq_estimated_max_budget(struct bfq_data *bfqd)
  {
@@ -4449,7 +4539,7 @@ index d1f648d..5469442 100644
  	else
  		return bfq_default_max_budget;
  }
-@@ -4252,6 +4687,10 @@ static ssize_t bfq_max_budget_store(struct elevator_queue *e,
+@@ -4252,6 +4701,10 @@ static ssize_t bfq_max_budget_store(struct elevator_queue *e,
  	return ret;
  }
  
@@ -4460,7 +4550,7 @@ index d1f648d..5469442 100644
  static ssize_t bfq_timeout_sync_store(struct elevator_queue *e,
  				      const char *page, size_t count)
  {
-@@ -4264,13 +4703,31 @@ static ssize_t bfq_timeout_sync_store(struct elevator_queue *e,
+@@ -4264,13 +4717,31 @@ static ssize_t bfq_timeout_sync_store(struct elevator_queue *e,
  	else if (__data > INT_MAX)
  		__data = INT_MAX;
  
@@ -4493,7 +4583,7 @@ index d1f648d..5469442 100644
  static ssize_t bfq_low_latency_store(struct elevator_queue *e,
  				     const char *page, size_t count)
  {
-@@ -4297,9 +4754,8 @@ static struct elv_fs_entry bfq_attrs[] = {
+@@ -4297,9 +4768,8 @@ static struct elv_fs_entry bfq_attrs[] = {
  	BFQ_ATTR(back_seek_penalty),
  	BFQ_ATTR(slice_idle),
  	BFQ_ATTR(max_budget),
@@ -4504,7 +4594,7 @@ index d1f648d..5469442 100644
  	BFQ_ATTR(low_latency),
  	BFQ_ATTR(wr_coeff),
  	BFQ_ATTR(wr_max_time),
-@@ -4342,9 +4798,28 @@ static struct elevator_type iosched_bfq = {
+@@ -4342,9 +4812,28 @@ static struct elevator_type iosched_bfq = {
  	.elevator_owner =	THIS_MODULE,
  };
  
@@ -4529,11 +4619,11 @@ index d1f648d..5469442 100644
  static int __init bfq_init(void)
  {
  	int ret;
-+	char msg[50] = "BFQ I/O-scheduler: v8";
++	char msg[50] = "BFQ I/O-scheduler: v8r2";
  
  	/*
  	 * Can be 0 on HZ < 1000 setups.
-@@ -4352,9 +4827,6 @@ static int __init bfq_init(void)
+@@ -4352,9 +4841,6 @@ static int __init bfq_init(void)
  	if (bfq_slice_idle == 0)
  		bfq_slice_idle = 1;
  
@@ -4543,7 +4633,7 @@ index d1f648d..5469442 100644
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
  	ret = blkcg_policy_register(&blkcg_policy_bfq);
  	if (ret)
-@@ -4370,23 +4842,34 @@ static int __init bfq_init(void)
+@@ -4370,23 +4856,34 @@ static int __init bfq_init(void)
  	 * installed on the reference devices (see the comments before the
  	 * definitions of the two arrays).
  	 */
@@ -4588,7 +4678,7 @@ index d1f648d..5469442 100644
  	return 0;
  
 diff --git a/block/bfq-sched.c b/block/bfq-sched.c
-index a64fec1..e54b149 100644
+index a64fec1..7d73b9d 100644
 --- a/block/bfq-sched.c
 +++ b/block/bfq-sched.c
 @@ -7,9 +7,11 @@
@@ -4741,6 +4831,15 @@ index a64fec1..e54b149 100644
  		bfq_put_queue(bfqq);
  	}
  }
+@@ -602,7 +627,7 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
+ 
+ 	if (entity->prio_changed) {
+ 		struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
+-		unsigned short prev_weight, new_weight;
++		unsigned int prev_weight, new_weight;
+ 		struct bfq_data *bfqd = NULL;
+ 		struct rb_root *root;
+ #ifdef CONFIG_BFQ_GROUP_IOSCHED
 @@ -628,12 +653,14 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
  		if (entity->new_weight != entity->orig_weight) {
  			if (entity->new_weight < BFQ_MIN_WEIGHT ||
@@ -4760,7 +4859,21 @@ index a64fec1..e54b149 100644
  			if (bfqq)
  				bfqq->ioprio =
  				  bfq_weight_to_ioprio(entity->orig_weight);
-@@ -708,7 +735,7 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
+@@ -662,6 +689,13 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
+ 		 * associated with its new weight.
+ 		 */
+ 		if (prev_weight != new_weight) {
++			if (bfqq)
++				bfq_log_bfqq(bfqq->bfqd, bfqq,
++					     "weight changed %d %d(%d %d)",
++					     prev_weight, new_weight,
++					     entity->orig_weight,
++					     bfqq->wr_coeff);
++
+ 			root = bfqq ? &bfqd->queue_weights_tree :
+ 				      &bfqd->group_weights_tree;
+ 			bfq_weights_tree_remove(bfqd, entity, root);
+@@ -708,7 +742,7 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
  		st = bfq_entity_service_tree(entity);
  
  		entity->service += served;
@@ -4769,7 +4882,7 @@ index a64fec1..e54b149 100644
  		BUG_ON(st->wsum == 0);
  
  		st->vtime += bfq_delta(served, st->wsum);
-@@ -717,31 +744,69 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
+@@ -717,31 +751,69 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
  	bfqg_stats_set_start_empty_time(bfqq_group(bfqq));
  #endif
@@ -4826,18 +4939,18 @@ index a64fec1..e54b149 100644
 +
 +	if (tot_serv_to_charge < entity->service)
 +		tot_serv_to_charge = entity->service;
-+
+ 
+-	bfq_log_bfqq(bfqq->bfqd, bfqq, "charge_full_budget");
 +	bfq_log_bfqq(bfqq->bfqd, bfqq,
 +		     "charge_time: %lu/%u ms, %d/%d/%d sectors",
 +		     time_ms, timeout_ms, entity->service,
 +		     tot_serv_to_charge, entity->budget);
  
--	bfq_log_bfqq(bfqq->bfqd, bfqq, "charge_full_budget");
+-	bfq_bfqq_served(bfqq, entity->budget - entity->service);
 +	/* Increase budget to avoid inconsistencies */
 +	if (tot_serv_to_charge > entity->budget)
 +		entity->budget = tot_serv_to_charge;
- 
--	bfq_bfqq_served(bfqq, entity->budget - entity->service);
++
 +	bfq_bfqq_served(bfqq,
 +			max_t(int, 0, tot_serv_to_charge - entity->service));
  }
@@ -4849,7 +4962,7 @@ index a64fec1..e54b149 100644
   *
   * Called whenever an entity is activated, i.e., it is not active and one
   * of its children receives a new request, or has to be reactivated due to
-@@ -749,11 +814,16 @@ static void bfq_bfqq_charge_full_budget(struct bfq_queue *bfqq)
+@@ -749,11 +821,16 @@ static void bfq_bfqq_charge_full_budget(struct bfq_queue *bfqq)
   * service received if @entity is active) of the queue to calculate its
   * timestamps.
   */
@@ -4867,7 +4980,7 @@ index a64fec1..e54b149 100644
  	if (entity == sd->in_service_entity) {
  		BUG_ON(entity->tree);
  		/*
-@@ -771,45 +841,133 @@ static void __bfq_activate_entity(struct bfq_entity *entity)
+@@ -771,45 +848,133 @@ static void __bfq_activate_entity(struct bfq_entity *entity)
  		 * old start time.
  		 */
  		bfq_active_extract(st, entity);
@@ -4889,16 +5002,16 @@ index a64fec1..e54b149 100644
 -		st->wsum += entity->weight;
 -		bfq_get_entity(entity);
 +		unsigned long long min_vstart;
-+
+ 
+-		BUG_ON(entity->on_st);
+-		entity->on_st = 1;
 +		/* See comments on bfq_fqq_update_budg_for_activation */
 +		if (non_blocking_wait_rq && bfq_gt(st->vtime, entity->finish)) {
 +			backshifted = true;
 +			min_vstart = entity->finish;
 +		} else
 +			min_vstart = st->vtime;
- 
--		BUG_ON(entity->on_st);
--		entity->on_st = 1;
++
 +		if (entity->tree == &st->idle) {
 +			/*
 +			 * Must be on the idle tree, bfq_idle_extract() will
@@ -5021,7 +5134,7 @@ index a64fec1..e54b149 100644
  
  		sd = entity->sched_data;
  		if (!bfq_update_next_in_service(sd))
-@@ -890,23 +1048,24 @@ static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
+@@ -890,23 +1055,24 @@ static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
  
  		if (!__bfq_deactivate_entity(entity, requeue))
  			/*
@@ -5054,7 +5167,7 @@ index a64fec1..e54b149 100644
  		 */
  		requeue = 1;
  	}
-@@ -916,9 +1075,23 @@ static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
+@@ -916,9 +1082,23 @@ static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue)
  update:
  	entity = parent;
  	for_each_entity(entity) {
@@ -5079,7 +5192,7 @@ index a64fec1..e54b149 100644
  		if (!bfq_update_next_in_service(sd))
  			break;
  	}
-@@ -997,10 +1170,11 @@ left:
+@@ -997,10 +1177,11 @@ left:
   * Update the virtual time in @st and return the first eligible entity
   * it contains.
   */
@@ -5093,7 +5206,7 @@ index a64fec1..e54b149 100644
  
  	if (RB_EMPTY_ROOT(&st->active))
  		return NULL;
-@@ -1009,6 +1183,24 @@ static struct bfq_entity *__bfq_lookup_next_entity(struct bfq_service_tree *st,
+@@ -1009,6 +1190,24 @@ static struct bfq_entity *__bfq_lookup_next_entity(struct bfq_service_tree *st,
  	entity = bfq_first_active_entity(st);
  	BUG_ON(bfq_gt(entity->start, st->vtime));
  
@@ -5118,7 +5231,7 @@ index a64fec1..e54b149 100644
  	/*
  	 * If the chosen entity does not match with the sched_data's
  	 * next_in_service and we are forcedly serving the IDLE priority
-@@ -1045,10 +1237,28 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
+@@ -1045,10 +1244,28 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
  	BUG_ON(sd->in_service_entity);
  
  	if (bfqd &&
@@ -5148,7 +5261,7 @@ index a64fec1..e54b149 100644
  			i = BFQ_IOPRIO_CLASSES - 1;
  			bfqd->bfq_class_idle_last_service = jiffies;
  			sd->next_in_service = entity;
-@@ -1057,6 +1267,24 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
+@@ -1057,6 +1274,24 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
  	for (; i < BFQ_IOPRIO_CLASSES; i++) {
  		entity = __bfq_lookup_next_entity(st + i, false);
  		if (entity) {
@@ -5173,7 +5286,7 @@ index a64fec1..e54b149 100644
  			if (extract) {
  				bfq_check_next_in_service(sd, entity);
  				bfq_active_extract(st + i, entity);
-@@ -1070,6 +1298,13 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
+@@ -1070,6 +1305,13 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd,
  	return entity;
  }
  
@@ -5187,7 +5300,7 @@ index a64fec1..e54b149 100644
  /*
   * Get next queue for service.
   */
-@@ -1086,7 +1321,36 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd)
+@@ -1086,7 +1328,36 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd)
  
  	sd = &bfqd->root_group->sched_data;
  	for (; sd ; sd = entity->my_sched_data) {
@@ -5224,7 +5337,7 @@ index a64fec1..e54b149 100644
  		BUG_ON(!entity);
  		entity->service = 0;
  	}
-@@ -1113,9 +1377,7 @@ static void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -1113,9 +1384,7 @@ static void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  {
  	struct bfq_entity *entity = &bfqq->entity;
  
@@ -5235,7 +5348,7 @@ index a64fec1..e54b149 100644
  	bfq_deactivate_entity(entity, requeue);
  }
  
-@@ -1123,12 +1385,11 @@ static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -1123,12 +1392,11 @@ static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  {
  	struct bfq_entity *entity = &bfqq->entity;
  
@@ -5250,7 +5363,7 @@ index a64fec1..e54b149 100644
  
  /*
   * Called when the bfqq no longer has requests pending, remove it from
-@@ -1139,6 +1400,7 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -1139,6 +1407,7 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  {
  	BUG_ON(!bfq_bfqq_busy(bfqq));
  	BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list));
@@ -5258,7 +5371,7 @@ index a64fec1..e54b149 100644
  
  	bfq_log_bfqq(bfqd, bfqq, "del from busy");
  
-@@ -1147,27 +1409,20 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+@@ -1147,27 +1416,20 @@ static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  	BUG_ON(bfqd->busy_queues == 0);
  	bfqd->busy_queues--;
  
@@ -5292,7 +5405,7 @@ index a64fec1..e54b149 100644
  }
  
  /*
-@@ -1185,16 +1440,11 @@ static void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+@@ -1185,16 +1447,11 @@ static void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq)
  	bfq_mark_bfqq_busy(bfqq);
  	bfqd->busy_queues++;
  
@@ -5312,17 +5425,17 @@ index a64fec1..e54b149 100644
  		bfqd->wr_busy_queues++;
  }
 diff --git a/block/bfq.h b/block/bfq.h
-index f73c942..b8ad02a 100644
+index f73c942..c6ba099 100644
 --- a/block/bfq.h
 +++ b/block/bfq.h
 @@ -1,5 +1,5 @@
  /*
 - * BFQ-v7r11 for 4.5.0: data structures and common functions prototypes.
-+ * BFQ-v8 for 4.7.0: data structures and common functions prototypes.
++ * BFQ-v8r2 for 4.7.0: data structures and common functions prototypes.
   *
   * Based on ideas and code from CFQ:
   * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
-@@ -28,7 +28,7 @@
+@@ -28,20 +28,21 @@
  
  #define BFQ_DEFAULT_QUEUE_IOPRIO	4
  
@@ -5331,7 +5444,14 @@ index f73c942..b8ad02a 100644
  #define BFQ_DEFAULT_GRP_IOPRIO	0
  #define BFQ_DEFAULT_GRP_CLASS	IOPRIO_CLASS_BE
  
-@@ -36,12 +36,6 @@ struct bfq_entity;
++/*
++ * Soft real-time applications are extremely more latency sensitive
++ * than interactive ones. Over-raise the weight of the former to
++ * privilege them against the latter.
++ */
++#define BFQ_SOFTRT_WEIGHT_FACTOR	100
++
+ struct bfq_entity;
  
  /**
   * struct bfq_service_tree - per ioprio_class service tree.
@@ -5344,7 +5464,7 @@ index f73c942..b8ad02a 100644
   *
   * Each service tree represents a B-WF2Q+ scheduler on its own.  Each
   * ioprio_class has its own independent scheduler, and so its own
-@@ -49,27 +43,28 @@ struct bfq_entity;
+@@ -49,27 +50,28 @@ struct bfq_entity;
   * of the containing bfqd.
   */
  struct bfq_service_tree {
@@ -5383,7 +5503,7 @@ index f73c942..b8ad02a 100644
   *
   * The supported ioprio_classes are the same as in CFQ, in descending
   * priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE.
-@@ -79,48 +74,29 @@ struct bfq_service_tree {
+@@ -79,48 +81,29 @@ struct bfq_service_tree {
   * All the fields are protected by the queue lock of the containing bfqd.
   */
  struct bfq_sched_data {
@@ -5406,7 +5526,7 @@ index f73c942..b8ad02a 100644
  struct bfq_weight_counter {
 -	short int weight;
 -	unsigned int num_active;
-+	short int weight; /* weight of the entities this counter refers to */
++	unsigned int weight; /* weight of the entities this counter refers to */
 +	unsigned int num_active; /* nr of active entities with this weight */
 +	/*
 +	 * Weights tree member (see bfq_data's @queue_weights_tree and
@@ -5441,7 +5561,7 @@ index f73c942..b8ad02a 100644
   *
   * A bfq_entity is used to represent either a bfq_queue (leaf node in the
   * cgroup hierarchy) or a bfq_group into the upper level scheduler.  Each
-@@ -147,27 +123,52 @@ struct bfq_weight_counter {
+@@ -147,27 +130,52 @@ struct bfq_weight_counter {
   * containing bfqd.
   */
  struct bfq_entity {
@@ -5472,17 +5592,18 @@ index f73c942..b8ad02a 100644
  
 -	int service, budget;
 -	unsigned short weight, new_weight;
+-	unsigned short orig_weight;
 +	/* amount of service received during the last service slot */
 +	int service;
 +
 +	/* budget, used also to calculate F_i: F_i = S_i + @budget / @weight */
 +	int budget;
 +
-+	unsigned short weight; 	/* weight of the queue */
-+	unsigned short new_weight; /* next weight if a change is in progress */
++	unsigned int weight; 	/* weight of the queue */
++	unsigned int new_weight; /* next weight if a change is in progress */
 +
 +	/* original weight, used to implement weight boosting */
- 	unsigned short orig_weight;
++	unsigned int orig_weight;
  
 +	/* parent entity, for hierarchical scheduling */
  	struct bfq_entity *parent;
@@ -5499,7 +5620,7 @@ index f73c942..b8ad02a 100644
  	int prio_changed;
  };
  
-@@ -175,56 +176,6 @@ struct bfq_group;
+@@ -175,56 +183,6 @@ struct bfq_group;
  
  /**
   * struct bfq_queue - leaf schedulable entity.
@@ -5556,7 +5677,7 @@ index f73c942..b8ad02a 100644
   *
   * A bfq_queue is a leaf request queue; it can be associated with an
   * io_context or more, if it  is  async or shared  between  cooperating
-@@ -235,117 +186,163 @@ struct bfq_group;
+@@ -235,117 +193,163 @@ struct bfq_group;
   * All the fields are protected by the queue lock of the containing bfqd.
   */
  struct bfq_queue {
@@ -5767,7 +5888,7 @@ index f73c942..b8ad02a 100644
  };
  
  enum bfq_device_speed {
-@@ -354,224 +351,216 @@ enum bfq_device_speed {
+@@ -354,224 +358,216 @@ enum bfq_device_speed {
  };
  
  /**
@@ -6128,7 +6249,7 @@ index f73c942..b8ad02a 100644
  	BFQ_BFQQ_FLAG_IO_bound,		/*
  					 * bfqq has timed-out at least once
  					 * having consumed at most 2/10 of
-@@ -581,17 +570,12 @@ enum bfqq_state_flags {
+@@ -581,17 +577,12 @@ enum bfqq_state_flags {
  					 * bfqq activated in a large burst,
  					 * see comments to bfq_handle_burst.
  					 */
@@ -6147,7 +6268,7 @@ index f73c942..b8ad02a 100644
  };
  
  #define BFQ_BFQQ_FNS(name)						\
-@@ -608,25 +592,53 @@ static int bfq_bfqq_##name(const struct bfq_queue *bfqq)		\
+@@ -608,25 +599,53 @@ static int bfq_bfqq_##name(const struct bfq_queue *bfqq)		\
  	return ((bfqq)->flags & (1 << BFQ_BFQQ_FLAG_##name)) != 0;	\
  }
  
@@ -6206,7 +6327,7 @@ index f73c942..b8ad02a 100644
  
  #define bfq_log(bfqd, fmt, args...) \
  	blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args)
-@@ -640,15 +652,12 @@ enum bfqq_expiration {
+@@ -640,15 +659,12 @@ enum bfqq_expiration {
  	BFQ_BFQQ_BUDGET_TIMEOUT,	/* budget took too long to be used */
  	BFQ_BFQQ_BUDGET_EXHAUSTED,	/* budget consumed */
  	BFQ_BFQQ_NO_MORE_REQUESTS,	/* the queue has no more requests */
@@ -6224,7 +6345,7 @@ index f73c942..b8ad02a 100644
  	/* number of ios merged */
  	struct blkg_rwstat		merged;
  	/* total time spent on device in ns, may not be accurate w/ queueing */
-@@ -657,12 +666,8 @@ struct bfqg_stats {
+@@ -657,12 +673,8 @@ struct bfqg_stats {
  	struct blkg_rwstat		wait_time;
  	/* number of IOs queued up */
  	struct blkg_rwstat		queued;
@@ -6237,7 +6358,7 @@ index f73c942..b8ad02a 100644
  	/* sum of number of ios queued across all samples */
  	struct blkg_stat		avg_queue_size_sum;
  	/* count of samples taken for average */
-@@ -680,8 +685,10 @@ struct bfqg_stats {
+@@ -680,8 +692,10 @@ struct bfqg_stats {
  	uint64_t			start_idle_time;
  	uint64_t			start_empty_time;
  	uint16_t			flags;
@@ -6248,7 +6369,16 @@ index f73c942..b8ad02a 100644
  /*
   * struct bfq_group_data - per-blkcg storage for the blkio subsystem.
   *
-@@ -712,7 +719,7 @@ struct bfq_group_data {
+@@ -692,7 +706,7 @@ struct bfq_group_data {
+ 	/* must be the first member */
+ 	struct blkcg_policy_data pd;
+ 
+-	unsigned short weight;
++	unsigned int weight;
+ };
+ 
+ /**
+@@ -712,7 +726,7 @@ struct bfq_group_data {
   *                   unused for the root group. Used to know whether there
   *                   are groups with more than one active @bfq_entity
   *                   (see the comments to the function
@@ -6257,7 +6387,7 @@ index f73c942..b8ad02a 100644
   * @rq_pos_tree: rbtree sorted by next_request position, used when
   *               determining if two or more queues have interleaving
   *               requests (see bfq_find_close_cooperator()).
-@@ -745,7 +752,6 @@ struct bfq_group {
+@@ -745,7 +759,6 @@ struct bfq_group {
  	struct rb_root rq_pos_tree;
  
  	struct bfqg_stats stats;
@@ -6265,7 +6395,7 @@ index f73c942..b8ad02a 100644
  };
  
  #else
-@@ -767,11 +773,25 @@ bfq_entity_service_tree(struct bfq_entity *entity)
+@@ -767,11 +780,25 @@ bfq_entity_service_tree(struct bfq_entity *entity)
  	struct bfq_sched_data *sched_data = entity->sched_data;
  	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
  	unsigned int idx = bfqq ? bfqq->ioprio_class - 1 :
@@ -6292,7 +6422,7 @@ index f73c942..b8ad02a 100644
  	return sched_data->service_tree + idx;
  }
  
-@@ -791,47 +811,6 @@ static struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)
+@@ -791,47 +818,6 @@ static struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)
  	return bic->icq.q->elevator->elevator_data;
  }
  
@@ -6340,7 +6470,7 @@ index f73c942..b8ad02a 100644
  #ifdef CONFIG_BFQ_GROUP_IOSCHED
  
  static struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq)
-@@ -857,11 +836,13 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio);
+@@ -857,11 +843,13 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio);
  static void bfq_put_queue(struct bfq_queue *bfqq);
  static void bfq_dispatch_insert(struct request_queue *q, struct request *rq);
  static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,