From: Benjamin Marzinski <bmarzins@redhat.com>
Date: Fri, 10 Jul 2009 15:22:15 -0500
Subject: [gfs2] don't put unlikely reclaim glocks on reclaim list
Message-id: 20090710202215.GS3172@ether.msp.redhat.com
O-Subject: [RHEL-5.4 PATCH] BZ#504335 gfs2: Don't put unlikely reclaim candidates on the reclaim list.
Bugzilla: 504335
RH-Acked-by: Steven Whitehouse <swhiteho@redhat.com>
RH-Acked-by: Bob Peterson <rpeterso@redhat.com>

GFS2 was placing far to many glocks on the reclaim list that were not
good candidates for freeing up from cache.  These locks would sit there
and repeatedly get scanned to see if they could be reclaimed, wasting a
lot of time when there was memory pressure. This patch does more checks
on the locks to see if they are actually likely to be removable from
cache.

This has been tested on the IBM BIGI testbed, where the performance
problem was noticed.  It does not completely remove the performance
problem, but it does improve it.

Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>

diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index 875a3af..84736cd 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -209,15 +209,45 @@ void gfs2_glock_hold(struct gfs2_glock *gl)
 }
 
 /**
+ * demote_ok - Check to see if it's ok to unlock a glock
+ * @gl: the glock
+ *
+ * Returns: 1 if it's ok
+ */
+
+static int demote_ok(const struct gfs2_glock *gl)
+{
+	const struct gfs2_glock_operations *glops = gl->gl_ops;
+
+	if (gl->gl_state == LM_ST_UNLOCKED)
+		return 0;
+	if (!list_empty(&gl->gl_holders))
+		return 0;
+	if (glops->go_demote_ok)
+		return glops->go_demote_ok(gl);
+	return 1;
+}
+
+/**
  * gfs2_glock_schedule_for_reclaim - Add a glock to the reclaim list
  * @gl: the glock
  *
  */
 
-static void gfs2_glock_schedule_for_reclaim(struct gfs2_glock *gl)
+static void gfs2_glock_schedule_for_reclaim(struct gfs2_glock *gl,
+					    int need_lock)
 {
+	int may_reclaim;
+	if (need_lock)
+		spin_lock(&gl->gl_spin);
+	may_reclaim = (demote_ok(gl) &&
+		       (atomic_read(&gl->gl_ref) == 1 ||
+			(gl->gl_name.ln_type == LM_TYPE_INODE &&
+			 atomic_read(&gl->gl_ref) <= 2)));
+	if (need_lock)
+		spin_unlock(&gl->gl_spin);
 	spin_lock(&lru_lock);
-	if (list_empty(&gl->gl_lru) && gl->gl_state != LM_ST_UNLOCKED) {
+	if (list_empty(&gl->gl_lru) && may_reclaim) {
 		list_add_tail(&gl->gl_lru, &lru_list);
 		atomic_inc(&lru_count);
 	}
@@ -225,6 +255,21 @@ static void gfs2_glock_schedule_for_reclaim(struct gfs2_glock *gl)
 }
 
 /**
+ * gfs2_glock_put_nolock() - Decrement reference count on glock
+ * @gl: The glock to put
+ *
+ * This function should only be used if the caller has its own reference
+ * to the glock, in addition to the one it is dropping.
+ */
+
+static void gfs2_glock_put_nolock(struct gfs2_glock *gl)
+{
+	if (atomic_dec_and_test(&gl->gl_ref))
+		GLOCK_BUG_ON(gl, 1);
+	gfs2_glock_schedule_for_reclaim(gl, 0);
+}
+
+/**
  * gfs2_glock_put() - Decrement reference count on glock
  * @gl: The glock to put
  *
@@ -251,9 +296,7 @@ int gfs2_glock_put(struct gfs2_glock *gl)
 		rv = 1;
 		goto out;
 	}
-	/* 1 for being hashed, 1 for having state != LM_ST_UNLOCKED */
-	if (atomic_read(&gl->gl_ref) == 2)
-		gfs2_glock_schedule_for_reclaim(gl);
+	gfs2_glock_schedule_for_reclaim(gl, 1);
 	write_unlock(gl_lock_addr(gl->gl_hash));
 out:
 	return rv;
@@ -452,7 +495,7 @@ static void state_change(struct gfs2_glock *gl, unsigned int new_state)
 		if (held2)
 			gfs2_glock_hold(gl);
 		else
-			gfs2_glock_put(gl);
+			gfs2_glock_put_nolock(gl);
 	}
 
 	gl->gl_state = new_state;
@@ -683,7 +726,7 @@ __acquires(&gl->gl_spin)
 out_sched:
 	gfs2_glock_hold(gl);
 	if (queue_delayed_work(glock_workqueue, &gl->gl_work, 0) == 0)
-		gfs2_glock_put(gl);
+		gfs2_glock_put_nolock(gl);
 out:
 	clear_bit(GLF_LOCK, &gl->gl_flags);
 }
@@ -1394,26 +1437,6 @@ void gfs2_glock_cb(void *cb_data, unsigned int type, void *data)
 	}
 }
 
-/**
- * demote_ok - Check to see if it's ok to unlock a glock
- * @gl: the glock
- *
- * Returns: 1 if it's ok
- */
-
-static int demote_ok(const struct gfs2_glock *gl)
-{
-	const struct gfs2_glock_operations *glops = gl->gl_ops;
-
-	if (gl->gl_state == LM_ST_UNLOCKED)
-		return 0;
-	if (!list_empty(&gl->gl_holders))
-		return 0;
-	if (glops->go_demote_ok)
-		return glops->go_demote_ok(gl);
-	return 1;
-}
-
 
 static int gfs2_shrink_glock_memory(int nr, gfp_t gfp_mask)
 {