From: Jeff Moyer <jmoyer@redhat.com>
Date: Tue, 3 Nov 2009 11:36:44 -0500
Subject: [block] cfq: separate merged cfqqs if they stop cooperating
Message-id: 1257266206-24003-11-git-send-email-jmoyer@redhat.com
O-Subject: [PATCH 10/12] cfq: break apart merged cfqqs if they stop cooperating
Bugzilla: 456181 448130 427709
RH-Acked-by: Josef Bacik <josef@redhat.com>
RH-Acked-by: Rik van Riel <riel@redhat.com>
RH-Acked-by: Vivek Goyal <vgoyal@redhat.com>

commit e6c5bc737ab71e4af6025ef7d150f5a26ae5f146
Author: Jeff Moyer <jmoyer@redhat.com>
Date:   Fri Oct 23 17:14:52 2009 -0400

    cfq: break apart merged cfqqs if they stop cooperating

    cfq_queues are merged if they are issuing requests within the mean seek
    distance of one another.  This patch detects when the coopearting stops and
    breaks the queues back up.

    Signed-off-by: Jeff Moyer <jmoyer@redhat.com>
    Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 69c906f..08275c1 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -27,6 +27,12 @@ static int cfq_slice_async = HZ / 25;
 static const int cfq_slice_async_rq = 2;
 static int cfq_slice_idle = HZ / 125;
 
+/*
+ * Allow merged cfqqs to perform this amount of seeky I/O before
+ * deciding to break the queues up again.
+ */
+#define CFQQ_COOP_TOUT		(HZ)
+
 #define CFQ_IDLE_GRACE		(HZ / 10)
 #define CFQ_SLICE_SCALE		(5)
 
@@ -211,6 +217,7 @@ struct cfq_queue {
 	u64 seek_total;
 	sector_t seek_mean;
 	sector_t last_request_pos;
+	unsigned long seeky_start;
 
 	/* various state flags, see below */
 	unsigned int flags;
@@ -1080,6 +1087,11 @@ static struct cfq_queue *cfq_close_cooperator(struct cfq_data *cfqd,
 {
 	struct cfq_queue *cfqq;
 
+	if (!cfq_cfqq_sync(cur_cfqq))
+		return NULL;
+	if (sample_valid(cur_cfqq->seek_samples) && CFQQ_SEEKY(cur_cfqq))
+		return NULL;
+
 	/*
 	 * We should notice if some of the queues are cooperating, eg
 	 * working closely on the same area of the disk.  In that case,
@@ -1094,6 +1106,8 @@ static struct cfq_queue *cfq_close_cooperator(struct cfq_data *cfqd,
 	 */
 	if (!cfq_cfqq_sync(cfqq))
 		return NULL;
+	if (sample_valid(cfqq->seek_samples) && CFQQ_SEEKY(cfqq))
+		return NULL;
 
 	return cfqq;
 }
@@ -1226,7 +1240,7 @@ static int cfqq_process_refs(struct cfq_queue *cfqq)
 
 static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq)
 {
-	int process_refs;
+	int process_refs, new_process_refs;
 	struct cfq_queue *__cfqq;
 
 	/* Avoid a circular list and skip interim queue merges */
@@ -1244,8 +1258,17 @@ static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq)
 	if (process_refs == 0)
 		return;
 
-	cfqq->new_cfqq = new_cfqq;
-	atomic_add(process_refs, &new_cfqq->ref);
+	/*
+	 * Merge in the direction of the lesser amount of work.
+	 */
+	new_process_refs = cfqq_process_refs(new_cfqq);
+	if (new_process_refs >= process_refs) {
+		cfqq->new_cfqq = new_cfqq;
+		atomic_add(process_refs, &new_cfqq->ref);
+	} else {
+		new_cfqq->new_cfqq = cfqq;
+		atomic_add(new_process_refs, &cfqq->ref);
+	}
 }
 
 /*
@@ -1935,6 +1958,19 @@ cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 	total = cfqq->seek_total + (cfqq->seek_samples/2);
 	do_div(total, cfqq->seek_samples);
 	cfqq->seek_mean = (sector_t)total;
+
+	/*
+	 * If this cfqq is shared between multiple processes, check to
+	 * make sure tha those processes are still issuing I/O within
+	 * the mean seek distance.  If not, it may be time to break the
+	 * queues apart again.
+	 */
+	if (cfq_cfqq_coop(cfqq)) {
+		if (CFQQ_SEEKY(cfqq) && !cfqq->seeky_start)
+			cfqq->seeky_start = jiffies;
+		else if (!CFQQ_SEEKY(cfqq))
+			cfqq->seeky_start = 0;
+	}
 }
 
 /*
@@ -2297,6 +2333,32 @@ cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_context *cic,
 	return cic->cfqq[SYNC];
 }
 
+static int should_split_cfqq(struct cfq_queue *cfqq)
+{
+	if (cfqq->seeky_start &&
+	    time_after(jiffies, cfqq->seeky_start + CFQQ_COOP_TOUT))
+		return 1;
+	return 0;
+}
+
+/*
+ * Returns NULL if a new cfqq should be allocated, or the old cfqq if this
+ * was the last process referring to said cfqq.
+ */
+static struct cfq_queue *
+split_cfqq(struct cfq_io_context *cic, struct cfq_queue *cfqq)
+{
+	if (cfqq_process_refs(cfqq) == 1) {
+		cfqq->seeky_start = 0;
+		cfq_clear_cfqq_coop(cfqq);
+		return cfqq;
+	}
+
+	cic->cfqq[SYNC] = NULL;
+	cfq_put_queue(cfqq);
+	return NULL;
+}
+
 /*
  * Allocate cfq data structures associated with this request.
  */
@@ -2320,6 +2382,7 @@ cfq_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
 
 	spin_lock_irqsave(q->queue_lock, flags);
 
+new_queue:
 	if (!cic)
 		goto queue_fail;
 
@@ -2329,10 +2392,18 @@ cfq_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
 			goto queue_fail;
 
 		cic->cfqq[is_sync] = cfqq;
-	} else if (cic->cfqq[is_sync]->new_cfqq)
-		cfqq = cfq_merge_cfqqs(cfqd, cic, cic->cfqq[is_sync]);
-	else
+	} else {
+		/*
+		 * If the queue was seeky for too long, break it apart.
+		 */
 		cfqq = cic->cfqq[is_sync];
+		if (cfq_cfqq_coop(cfqq) && should_split_cfqq(cfqq)) {
+			cfqq = split_cfqq(cic, cfqq);
+			if (!cfqq)
+				goto new_queue;
+		} else if (cfqq->new_cfqq)
+			cfqq = cfq_merge_cfqqs(cfqd, cic, cfqq);
+	}
 
 	cfqq->allocated[rw]++;
 	cfq_clear_cfqq_must_alloc(cfqq);