From: Andy Gospodarek <gospo@redhat.com>
Date: Wed, 26 Nov 2008 11:29:31 -0500
Subject: [net] cxgb3: eeh, lro, and multiqueue fixes
Message-id: 20081126162930.GE13699@gospo.rdu.redhat.com
O-Subject: [RHEL5.3 PATCH] cxgb3: eeh, lro, and multiqueue fixes
Bugzilla: 441959
RH-Acked-by: John W. Linville <linville@redhat.com>

This patch includes 3 patches from upstream that Divy at Chelsio thinks
are quite important for RHEL5.3.  They are:

    commit 0ca41c0413a4d9ca58767d53d23accea9aa1cdef
    Author: Divy Le Ray <divy@chelsio.com>
    Date:   Thu Sep 25 14:05:28 2008 +0000

        [2.6.28,1/1] cxgb3 - fix race in EEH

        A SGE queue set timer might access registers while in EEH recovery,
        triggering an EEH error loop. Stop all timers early in EEH process.

    commit 004f23b9d3874efc81d2d1cf18fd0fe48dc2f26f
    Author: Divy Le Ray <divy@chelsio.com>
    Date:   Thu Sep 4 22:34:13 2008 -0700

        cxgb3 - remove duplicate tests in lro

        The generic lro code checks TCP flags/options.
        Remove duplicate tests done in the driver.

    commit 8c26376112fb4b8dfea42069b602c03d53366052
    Author: Divy Le Ray <divy@chelsio.com>
    Date:   Wed Oct 8 17:37:33 2008 -0700

        cxgb3: Allocate multiqueues at init time

        Allocate a queue set per core, up to the maximum of available qsets.
        Share the queue sets on multi port adapters.
        Rename MSI-X interrupt vectors ethX-N, N being the queue set number.

        Signed-off-by: Divy Le Ray <divy@chelsio.com>
        Signed-off-by: David S. Miller <davem@davemloft.net>

This also includes a patch that fixes an issue with LRO that crept in
during the backport.

This should resolve everything in RHBZ 441959.

diff --git a/drivers/net/cxgb3/common.h b/drivers/net/cxgb3/common.h
index 9ecf8a6..7c329f8 100644
--- a/drivers/net/cxgb3/common.h
+++ b/drivers/net/cxgb3/common.h
@@ -358,6 +358,7 @@ struct qset_params {		/* SGE queue set parameters */
 	unsigned int jumbo_size;	/* # of entries in jumbo free list */
 	unsigned int txq_size[SGE_TXQ_PER_SET];	/* Tx queue sizes */
 	unsigned int cong_thres;	/* FL congestion threshold */
+	unsigned int vector;		/* Interrupt (line or vector) number */
 };
 
 struct sge_params {
diff --git a/drivers/net/cxgb3/cxgb3_ioctl.h b/drivers/net/cxgb3/cxgb3_ioctl.h
index 68200a1..3e8d5fa 100644
--- a/drivers/net/cxgb3/cxgb3_ioctl.h
+++ b/drivers/net/cxgb3/cxgb3_ioctl.h
@@ -92,6 +92,8 @@ struct ch_qset_params {
 	int32_t polling;
 	int32_t lro;
 	int32_t cong_thres;
+	int32_t  vector;
+	int32_t  qnum;
 };
 
 struct ch_pktsched_params {
diff --git a/drivers/net/cxgb3/cxgb3_main.c b/drivers/net/cxgb3/cxgb3_main.c
index e4b0bda..83726ee 100644
--- a/drivers/net/cxgb3/cxgb3_main.c
+++ b/drivers/net/cxgb3/cxgb3_main.c
@@ -276,10 +276,10 @@ static void name_msix_vecs(struct adapter *adap)
 
 		for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 			snprintf(adap->msix_info[msi_idx].desc, n,
-				 "%s (queue %d)", d->name, i);
+				 "%s-%d", d->name, pi->first_qset + i);
 			adap->msix_info[msi_idx].desc[n] = 0;
 		}
- 	}
+	}
 }
 
 static int request_msix_data_irqs(struct adapter *adap)
@@ -308,6 +308,22 @@ static int request_msix_data_irqs(struct adapter *adap)
 	return 0;
 }
 
+static void free_irq_resources(struct adapter *adapter)
+{
+	if (adapter->flags & USING_MSIX) {
+		int i, n = 0;
+
+		free_irq(adapter->msix_info[0].vec, adapter);
+		for_each_port(adapter, i)
+		    n += adap2pinfo(adapter, i)->nqsets;
+
+		for (i = 0; i < n; ++i)
+			free_irq(adapter->msix_info[i + 1].vec,
+				 &adapter->sge.qs[i]);
+	} else
+		free_irq(adapter->pdev->irq, adapter);
+}
+
 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 			      unsigned long n)
 {
@@ -439,6 +455,7 @@ static int init_dummy_netdevs(struct adapter *adap)
 
 				p = netdev_priv(nd);
 				p->adapter = adap;
+				p->port_id = pi->port_id;
 				nd->weight = 64;
 				set_bit(__LINK_STATE_START, &nd->state);
 				adap->dummy_netdev[dummy_idx] = nd;
@@ -507,7 +524,10 @@ static int setup_sge_qsets(struct adapter *adap)
 		struct net_device *dev = adap->port[i];
 		const struct port_info *pi = netdev_priv(dev);
 
-		for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
+		for (j = pi->first_qset; j < pi->first_qset + pi->nqsets;
+		     ++j, ++qset_idx) {
+			if (!pi->rx_csum_offload)
+				adap->params.sge.qset[qset_idx].lro = 0;
 			err = t3_sge_alloc_qset(adap, qset_idx, 1,
 				(adap->flags & USING_MSIX) ? qset_idx + 1 :
 							     irq_idx,
@@ -773,11 +793,12 @@ static void init_port_mtus(struct adapter *adapter)
 	t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 }
 
-static void send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
+static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 			      int hi, int port)
 {
 	struct sk_buff *skb;
 	struct mngt_pktsched_wr *req;
+	int ret;
 
 	skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 	req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
@@ -788,20 +809,28 @@ static void send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 	req->min = lo;
 	req->max = hi;
 	req->binding = port;
-	t3_mgmt_tx(adap, skb);
+	ret = t3_mgmt_tx(adap, skb);
+
+	return ret;
 }
 
-static void bind_qsets(struct adapter *adap)
+static int bind_qsets(struct adapter *adap)
 {
-	int i, j;
+	int i, j, err = 0;
 
 	for_each_port(adap, i) {
 		const struct port_info *pi = adap2pinfo(adap, i);
 
-		for (j = 0; j < pi->nqsets; ++j)
-			send_pktsched_cmd(adap, 1, pi->first_qset + j, -1,
-					  -1, i);
+		for (j = 0; j < pi->nqsets; ++j) {
+			int ret = send_pktsched_cmd(adap, 1,
+						    pi->first_qset + j, -1,
+						    -1, i);
+			if (ret)
+				err = ret;
+		}
 	}
+
+	return err;
 }
 
 #define FW_FNAME "t3fw-%d.%d.%d.bin"
@@ -984,9 +1013,16 @@ static int cxgb_up(struct adapter *adap)
 		t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
 	}
 
-	if ((adap->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
-		bind_qsets(adap);
-	adap->flags |= QUEUES_BOUND;
+	if (!(adap->flags & QUEUES_BOUND)) {
+		err = bind_qsets(adap);
+		if (err) {
+			CH_ERR(adap, "failed to bind qsets, err %d\n", err);
+			t3_intr_disable(adap);
+			free_irq_resources(adap);
+			goto out;
+		}
+		adap->flags |= QUEUES_BOUND;
+	}
 
 out:
 	return err;
@@ -1005,19 +1041,7 @@ static void cxgb_down(struct adapter *adapter)
 	t3_intr_disable(adapter);
 	spin_unlock_irq(&adapter->work_lock);
 
-	if (adapter->flags & USING_MSIX) {
-		int i, n = 0;
-
-		free_irq(adapter->msix_info[0].vec, adapter);
-		for_each_port(adapter, i)
-		    n += adap2pinfo(adapter, i)->nqsets;
-
-		for (i = 0; i < n; ++i)
-			free_irq(adapter->msix_info[i + 1].vec,
-				 &adapter->sge.qs[i]);
-	} else
-		free_irq(adapter->pdev->irq, adapter);
-
+	free_irq_resources(adapter);
 	flush_workqueue(cxgb3_wq);	/* wait for external IRQ handler */
 	quiesce_rx(adapter);
 }
@@ -1318,8 +1342,8 @@ static unsigned long collect_sge_port_stats(struct adapter *adapter,
 	int i;
 	unsigned long tot = 0;
 
-	for (i = 0; i < p->nqsets; ++i)
-		tot += adapter->sge.qs[i + p->first_qset].port_stats[idx];
+	for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
+		tot += adapter->sge.qs[i].port_stats[idx];
 	return tot;
 }
 
@@ -1523,7 +1547,7 @@ static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 	struct link_config *lc = &p->link_config;
 
 	if (!(lc->supported & SUPPORTED_Autoneg))
-		return -EOPNOTSUPP;	/* can't change speed/duplex */
+		return -EOPNOTSUPP;     /* can't change speed/duplex */
 
 	if (cmd->autoneg == AUTONEG_DISABLE) {
 		int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
@@ -1602,8 +1626,10 @@ static int set_rx_csum(struct net_device *dev, u32 data)
 		struct adapter *adap = p->adapter;
 		int i;
 
-		for (i = p->first_qset; i < p->first_qset + p->nqsets; i++)
+		for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
+			adap->params.sge.qset[i].lro = 0;
 			adap->sge.qs[i].lro_enabled = 0;
+		}
 	}
 	return 0;
 }
@@ -1813,6 +1839,8 @@ static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
 		int i;
 		struct qset_params *q;
 		struct ch_qset_params t;
+		int q1 = pi->first_qset;
+		int nqsets = pi->nqsets;
 
 		if (!capable(CAP_NET_ADMIN))
 			return -EPERM;
@@ -1835,6 +1863,16 @@ static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
 			|| !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
 					MAX_RSPQ_ENTRIES))
 			return -EINVAL;
+
+		if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
+			for_each_port(adapter, i) {
+				pi = adap2pinfo(adapter, i);
+				if (t.qset_idx >= pi->first_qset &&
+				    t.qset_idx < pi->first_qset + pi->nqsets &&
+				    !pi->rx_csum_offload)
+					return -EINVAL;
+			}
+
 		if ((adapter->flags & FULL_INIT_DONE) &&
 			(t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
 			t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
@@ -1842,6 +1880,20 @@ static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
 			t.polling >= 0 || t.cong_thres >= 0))
 			return -EBUSY;
 
+		/* Allow setting of any available qset when offload enabled */
+		if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
+			q1 = 0;
+			for_each_port(adapter, i) {
+				pi = adap2pinfo(adapter, i);
+				nqsets += pi->first_qset + pi->nqsets;
+			}
+		}
+
+		if (t.qset_idx < q1)
+			return -EINVAL;
+		if (t.qset_idx > q1 + nqsets - 1)
+			return -EINVAL;
+
 		q = &adapter->params.sge.qset[t.qset_idx];
 
 		if (t.rspq_size >= 0)
@@ -1891,13 +1943,26 @@ static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
 	case CHELSIO_GET_QSET_PARAMS:{
 		struct qset_params *q;
 		struct ch_qset_params t;
+		int q1 = pi->first_qset;
+		int nqsets = pi->nqsets;
+		int i;
 
 		if (copy_from_user(&t, useraddr, sizeof(t)))
 			return -EFAULT;
-		if (t.qset_idx >= SGE_QSETS)
+
+		/* Display qsets for all ports when offload enabled */
+		if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
+			q1 = 0;
+			for_each_port(adapter, i) {
+				pi = adap2pinfo(adapter, i);
+				nqsets = pi->first_qset + pi->nqsets;
+			}
+		}
+
+		if (t.qset_idx >= nqsets)
 			return -EINVAL;
 
-		q = &adapter->params.sge.qset[t.qset_idx];
+		q = &adapter->params.sge.qset[q1 + t.qset_idx];
 		t.rspq_size = q->rspq_size;
 		t.txq_size[0] = q->txq_size[0];
 		t.txq_size[1] = q->txq_size[1];
@@ -1908,6 +1973,12 @@ static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
 		t.lro = q->lro;
 		t.intr_lat = q->coalesce_usecs;
 		t.cong_thres = q->cong_thres;
+		t.qnum = q1;
+
+		if (adapter->flags & USING_MSIX)
+			t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
+		else
+			t.vector = adapter->pdev->irq;
 
 		if (copy_to_user(useraddr, &t, sizeof(t)))
 			return -EFAULT;
@@ -2253,8 +2324,8 @@ static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
 {
 	int i;
 
-	for (i = 0; i < p->nqsets; i++) {
-		struct sge_rspq *q = &adap->sge.qs[i + p->first_qset].rspq;
+	for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
+		struct sge_rspq *q = &adap->sge.qs[i].rspq;
 
 		spin_lock_irq(&q->lock);
 		spin_unlock_irq(&q->lock);
@@ -2560,6 +2631,42 @@ static struct pci_error_handlers t3_err_handler = {
 	.resume = t3_io_resume,
 };
 
+/*
+ * Set the number of qsets based on the number of CPUs and the number of ports,
+ * not to exceed the number of available qsets, assuming there are enough qsets
+ * per port in HW.
+ */
+static void set_nqsets(struct adapter *adap)
+{
+	int i, j = 0;
+	int num_cpus = num_online_cpus();
+	int hwports = adap->params.nports;
+	int nqsets = SGE_QSETS;
+
+	if (adap->params.rev > 0) {
+		if (hwports == 2 &&
+		    (hwports * nqsets > SGE_QSETS ||
+		     num_cpus >= nqsets / hwports))
+			nqsets /= hwports;
+		if (nqsets > num_cpus)
+			nqsets = num_cpus;
+		if (nqsets < 1 || hwports == 4)
+			nqsets = 1;
+	} else
+		nqsets = 1;
+
+	for_each_port(adap, i) {
+		struct port_info *pi = adap2pinfo(adap, i);
+
+		pi->first_qset = j;
+		pi->nqsets = nqsets;
+		j = pi->first_qset + nqsets;
+
+		dev_info(&adap->pdev->dev,
+			 "Port %d using %d queue sets.\n", i, nqsets);
+	}
+}
+
 static int __devinit cxgb_enable_msix(struct adapter *adap)
 {
 	struct msix_entry entries[SGE_QSETS + 1];
@@ -2718,9 +2825,6 @@ static int __devinit init_one(struct pci_dev *pdev,
 		pi = netdev_priv(netdev);
 		pi->adapter = adapter;
 		pi->rx_csum_offload = 1;
-		pi->nqsets = 1;
-		pi->first_qset = i;
-		pi->activity = 0;
 		pi->port_id = i;
 		netif_carrier_off(netdev);
 		netdev->irq = pdev->irq;
@@ -2799,6 +2903,8 @@ static int __devinit init_one(struct pci_dev *pdev,
 	else if (msi > 0 && pci_enable_msi(pdev) == 0)
 		adapter->flags |= USING_MSI;
 
+	set_nqsets(adapter);
+
 	err = sysfs_create_group(&adapter->port[0]->class_dev.kobj,
 				 &cxgb3_attr_group);
 
diff --git a/drivers/net/cxgb3/sge.c b/drivers/net/cxgb3/sge.c
index d69e7ab..f958232 100644
--- a/drivers/net/cxgb3/sge.c
+++ b/drivers/net/cxgb3/sge.c
@@ -2045,7 +2045,11 @@ static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
  */
 static void init_lro_mgr(struct sge_qset *qs, struct net_lro_mgr *lro_mgr)
 {
-	lro_mgr->dev = qs->netdev;
+	struct adapter *adapter = qs->adap;
+	struct net_device *dev = qs->netdev;
+	struct port_info *pi = netdev_priv(dev);
+
+	lro_mgr->dev = adapter->port[pi->port_id];
 	lro_mgr->features = LRO_F_NAPI;
 	lro_mgr->ip_summed = CHECKSUM_UNNECESSARY;
 	lro_mgr->ip_summed_aggr = CHECKSUM_UNNECESSARY;