From: mchristi@redhat.com <mchristi@redhat.com>
Date: Sun, 21 Jun 2009 12:50:17 -0500
Subject: [scsi] cxgb3i: use kref to track ddp, support page sizes
Message-id: 1245606617-16673-1-git-send-email-mchristi@redhat.com
O-Subject: [PATCH] RHEL 5.4: bug fix update for cxgb3i driver
Bugzilla: 506151
RH-Acked-by: Tomas Henzl <thenzl@redhat.com>

From: Mike Christie <mchristi@redhat.com>

This is for BZ 506151.

The patch fixes these two bugs:
1. cxgb3i -- use kref to track ddp usage

   The iscsi ddp functionality could be used by multiple iscsi entities.
   This patch add a refcnt to keep track of it, so we would not release
   the ddp table pre-maturely. Without this patch, the ddp could be
   disabled in the middle of a scsi read transaction, and the data then
   would have to be copied into the scsi buffer.

2. cxgb3i -- suppot of different kernel page sizes

   The default kernel pages supported are 4K, 8K, 16K, and 64K.
   If PAGE_SIZE is not one of the defaults, re-calculate ddp page table
   entries (so that we would reprogrammed the h/w when initializing
ddp).
   Without this patch, the ddp functionality would be disabled if the
   PAGE_SIZE is not one of the defaults.

   (NOTE: this fix is not needed for RHEL. It is only included to remain
    in sync with the upstream driver).

The patches are in the scsi maintainer's tree
http://git.kernel.org/?p=linux/kernel/git/jejb/scsi-misc-2.6.git;a=commit;h=9194c6264040d71f851236437a392594b26e5b91
and here
http://git.kernel.org/?p=linux/kernel/git/jejb/scsi-misc-2.6.git;a=commit;h=295ab1b54393aec064533fbc5b483844736ccbf0

I have tested here by running the iscsi regression tests while pulling
cables and stopping/starting interfaces.

diff --git a/drivers/scsi/cxgb3i/cxgb3i_ddp.c b/drivers/scsi/cxgb3i/cxgb3i_ddp.c
index 756dfd5..d0765eb 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_ddp.c
+++ b/drivers/scsi/cxgb3i/cxgb3i_ddp.c
@@ -206,6 +206,31 @@ int cxgb3i_ddp_find_page_index(unsigned long pgsz)
 	return DDP_PGIDX_MAX;
 }
 
+/**
+ * cxgb3i_ddp_adjust_page_table - adjust page table with PAGE_SIZE
+ * return the ddp page index, if no match is found return DDP_PGIDX_MAX.
+ */
+int cxgb3i_ddp_adjust_page_table(void)
+{
+	int i;
+	unsigned int base_order, order;
+
+	if (PAGE_SIZE < (1UL << ddp_page_shift[0])) {
+		ddp_log_info("PAGE_SIZE 0x%lx too small, min. 0x%lx.\n",
+				PAGE_SIZE, 1UL << ddp_page_shift[0]);
+		return -EINVAL;
+	}
+
+	base_order = get_order(1UL << ddp_page_shift[0]);
+	order = get_order(1 << PAGE_SHIFT);
+	for (i = 0; i < DDP_PGIDX_MAX; i++) {
+		/* first is the kernel page size, then just doubling the size */
+		ddp_page_order[i] = order - base_order + i;
+		ddp_page_shift[i] = PAGE_SHIFT + i;
+	}
+	return 0;
+}
+
 static inline void ddp_gl_unmap(struct pci_dev *pdev,
 				struct cxgb3i_gather_list *gl)
 {
@@ -595,30 +620,40 @@ int cxgb3i_adapter_ddp_info(struct t3cdev *tdev,
  * release all the resource held by the ddp pagepod manager for a given
  * adapter if needed
  */
-void cxgb3i_ddp_cleanup(struct t3cdev *tdev)
+
+static void ddp_cleanup(struct kref *kref)
 {
+	struct cxgb3i_ddp_info *ddp = container_of(kref,
+						struct cxgb3i_ddp_info,
+						refcnt);
 	int i = 0;
+
+	ddp_log_info("kref release ddp 0x%p, t3dev 0x%p.\n", ddp, ddp->tdev);
+
+	ddp->tdev->ulp_iscsi = NULL;
+	while (i < ddp->nppods) {
+		struct cxgb3i_gather_list *gl = ddp->gl_map[i];
+		if (gl) {
+			int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
+					>> PPOD_PAGES_SHIFT;
+			ddp_log_info("t3dev 0x%p, ddp %d + %d.\n",
+					ddp->tdev, i, npods);
+			kfree(gl);
+			ddp_free_gl_skb(ddp, i, npods);
+			i += npods;
+		} else
+			i++;
+	}
+	cxgb3i_free_big_mem(ddp);
+}
+
+void cxgb3i_ddp_cleanup(struct t3cdev *tdev)
+{
 	struct cxgb3i_ddp_info *ddp = (struct cxgb3i_ddp_info *)tdev->ulp_iscsi;
 
 	ddp_log_info("t3dev 0x%p, release ddp 0x%p.\n", tdev, ddp);
-
-	if (ddp) {
-		tdev->ulp_iscsi = NULL;
-		while (i < ddp->nppods) {
-			struct cxgb3i_gather_list *gl = ddp->gl_map[i];
-			if (gl) {
-				int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
-						>> PPOD_PAGES_SHIFT;
-				ddp_log_info("t3dev 0x%p, ddp %d + %d.\n",
-						tdev, i, npods);
-				kfree(gl);
-				ddp_free_gl_skb(ddp, i, npods);
-				i += npods;
-			} else
-				i++;
-		}
-		cxgb3i_free_big_mem(ddp);
-	}
+	if (ddp)
+		kref_put(&ddp->refcnt, ddp_cleanup);
 }
 
 /**
@@ -628,12 +663,13 @@ void cxgb3i_ddp_cleanup(struct t3cdev *tdev)
  */
 static void ddp_init(struct t3cdev *tdev)
 {
-	struct cxgb3i_ddp_info *ddp;
+	struct cxgb3i_ddp_info *ddp = tdev->ulp_iscsi;
 	struct ulp_iscsi_info uinfo;
 	unsigned int ppmax, bits;
 	int i, err;
 
-	if (tdev->ulp_iscsi) {
+	if (ddp) {
+		kref_get(&ddp->refcnt);
 		ddp_log_warn("t3dev 0x%p, ddp 0x%p already set up.\n",
 				tdev, tdev->ulp_iscsi);
 		return;
@@ -667,6 +703,7 @@ static void ddp_init(struct t3cdev *tdev)
 					  ppmax *
 					  sizeof(struct cxgb3i_gather_list *));
 	spin_lock_init(&ddp->map_lock);
+	kref_init(&ddp->refcnt);
 
 	ddp->tdev = tdev;
 	ddp->pdev = uinfo.pdev;
@@ -712,6 +749,17 @@ void cxgb3i_ddp_init(struct t3cdev *tdev)
 {
 	if (page_idx == DDP_PGIDX_MAX) {
 		page_idx = cxgb3i_ddp_find_page_index(PAGE_SIZE);
+
+		if (page_idx == DDP_PGIDX_MAX) {
+			ddp_log_info("system PAGE_SIZE %lu, update hw.\n",
+					PAGE_SIZE);
+			if (cxgb3i_ddp_adjust_page_table() < 0) {
+				ddp_log_info("PAGE_SIZE %lu, ddp disabled.\n",
+						PAGE_SIZE);
+				return;
+			}
+			page_idx = cxgb3i_ddp_find_page_index(PAGE_SIZE);
+		}
 		ddp_log_info("system PAGE_SIZE %lu, ddp idx %u.\n",
 				PAGE_SIZE, page_idx);
 	}
diff --git a/drivers/scsi/cxgb3i/cxgb3i_ddp.h b/drivers/scsi/cxgb3i/cxgb3i_ddp.h
index 0d296de..87dd56b 100644
--- a/drivers/scsi/cxgb3i/cxgb3i_ddp.h
+++ b/drivers/scsi/cxgb3i/cxgb3i_ddp.h
@@ -54,6 +54,7 @@ struct cxgb3i_gather_list {
  * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload
  *
  * @list:	list head to link elements
+ * @refcnt:	ref. count
  * @tdev:	pointer to t3cdev used by cxgb3 driver
  * @max_txsz:	max tx packet size for ddp
  * @max_rxsz:	max rx packet size for ddp
@@ -70,6 +71,7 @@ struct cxgb3i_gather_list {
  */
 struct cxgb3i_ddp_info {
 	struct list_head list;
+	struct kref refcnt;
 	struct t3cdev *tdev;
 	struct pci_dev *pdev;
 	unsigned int max_txsz;