From: AMEET M. PARANJAPE <aparanja@redhat.com>
Date: Fri, 5 Jun 2009 10:35:00 -0400
Subject: [ppc64] resolves issues with pcie-save-restore-state
Message-id: 20090605143149.5066.91565.sendpatchset@squad5-lp1.lab.bos.redhat.com
O-Subject: [PATCH RHEL5.4 BZ504198] Resolves issues with pcie-save-restore-state
Bugzilla: 504198
RH-Acked-by: David Howells <dhowells@redhat.com>

RHBZ#:
======
https://bugzilla.redhat.com/show_bug.cgi?id=504198

Description:
===========
The patch fixes issues introduced by this patch in RH BZ 493152:

[RHEL5.4 PATCH 15/17] BZ493152: Backport: PCI: Restore PCI Express capability
registers after PM event

1)  Patch added calls to new pci_save_pcie_state()) & pci_restore_pcie_state()
to pci_save_state()/pci_restore_state() but did not remove the calls added in
RHEL5.3 to support EEH, save_pcie_reg() and restore_pcie_reg();

2)  In function pci_save_pcie_state(), the value for
save_state->cap_nr = PCI_CAP_ID_EXP;  must be set prior to call to
pci_add_saved_cap(dev, save_state);

3) In function pci_save_pcie_state(), save_state buffer should be allocated
once and saved in saved_cap for subsequent uses.
For example:
  save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
  if (!save_state) {
       save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeo(u16),
                                     GFP_KERNEL);
        if (!save_state) {
             printk(KERN_ERR "Out of memory in pci_save_pcie_state\n");
             return -ENOMEM;
         }
         save_state->cap_nr = PCI_CAP_ID_EXP;
         pci_add_saved_cap(dev, save_state);

4) In function pci_restore_pcie_state(), saved_cap should not be removed, this
is what breaks EEH recovery on second EEH error.
EEH recovery depends upon state saved during adapter initialization to be
restored after PCI slot reset.

RHEL Version Found:
================
RHEL 5.4

kABI Status:
============
No symbols were harmed.

Brew:
=====
Built on all platforms.
http://brewweb.devel.redhat.com/brew/taskinfo?taskID=1827916

Upstream Status:
================
The recent 5.4 patch:

[RHEL5.4 PATCH 15/17] BZ493152: Backport: PCI: Restore PCI Express capability
registers after PM event

looks similar to commit fb56a5a23bfecd9cac9187164a9d5f22d287c48b9.  In the
upstream code save_pcie_reg() and restore_pcie_reg() are no longer needed.

The save_state->cap_nr line comes from commit ID
ec0a3a27fbb5792980b8c3ce4a93bc2ee93d0b35.  Without it, the saved state cannot
be found in pci_restore_pcie_state().

Removal of the lines that delete the saved state after a restore comes from
commit ID 9f35575dfc172f0a93fb464761883c8f49599b7a.

Test Status:
============
Result of second eeh error injection on Emulex PCIe adapter before patch:
lpfc 0003:01:00.1: 3:0438 Adapter failed to init, chipset, status reg
xffffffff, FW Data: A8 xffffffff AC xffffffff
lpfc 0003:01:00.1: 3:0438 Adapter failed to init, chipset, status reg
xffffffff, FW Data: A8 xffffffff AC xffffffff

With patch applied EEH recovers as expected from 5 EEH error injections.
===============================================================
Ameet Paranjape 978-392-3903 ext 23903
IBM on-site partner

Proposed Patch:
===============

diff --git a/arch/powerpc/platforms/pseries/eeh.c b/arch/powerpc/platforms/pseries/eeh.c
index 778d2b5..34b87a6 100644
--- a/arch/powerpc/platforms/pseries/eeh.c
+++ b/arch/powerpc/platforms/pseries/eeh.c
@@ -128,59 +128,6 @@ static unsigned long slot_resets;
 /* --------------------------------------------------------------- */
 /* Below lies the EEH event infrastructure */
 
-int save_pcie_reg(struct pci_dev *dev)
-{
-	int pos, i = 0;
-	struct pci_cap_saved_state *save_state;
-	u16 *cap;
-
-	pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
-	if (pos <= 0)
-		return 0;
-
-	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
-	if (save_state)
-		return 0;
-
-	save_state = kzalloc(sizeof(*save_state) + sizeof(u16) * 4, GFP_KERNEL);
-	if (!save_state) {
-		dev_err(&dev->dev, "Out of memory in save_pcie_reg\n");
-		return -ENOMEM;
-	}
-
-	cap = (u16 *)&save_state->data[0];
-
-	pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &cap[i++]);
-	pci_read_config_word(dev, pos + PCI_EXP_LNKCTL, &cap[i++]);
-	pci_read_config_word(dev, pos + PCI_EXP_SLTCTL, &cap[i++]);
-	pci_read_config_word(dev, pos + PCI_EXP_RTCTL, &cap[i++]);
-	save_state->cap_nr = PCI_CAP_ID_EXP;
-	pci_add_saved_cap(dev, save_state);
-
-	return 0;
-}
-
-void restore_pcie_reg(struct pci_dev *dev)
-{
-	int i = 0, pos;
-	struct pci_cap_saved_state *save_state;
-	u16 *cap;
-
-	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
-	pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
-	if (!save_state || pos <= 0)
-		return;
-	cap = (u16 *)&save_state->data[0];
-
-	pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, cap[i++]);
-	pci_write_config_word(dev, pos + PCI_EXP_LNKCTL, cap[i++]);
-	pci_write_config_word(dev, pos + PCI_EXP_SLTCTL, cap[i++]);
-	pci_write_config_word(dev, pos + PCI_EXP_RTCTL, cap[i++]);
-}
-
-EXPORT_SYMBOL_GPL(save_pcie_reg);
-EXPORT_SYMBOL_GPL(restore_pcie_reg);
-
 static void rtas_slot_error_detail(struct pci_dn *pdn, int severity,
                                    char *driver_log, size_t loglen)
 {
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index a0e35fa..9bc4ffb 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -567,10 +567,15 @@ static int pci_save_pcie_state(struct pci_dev *dev)
 	if (pos <= 0)
 		return 0;
 
-	save_state = kzalloc(sizeof(*save_state) + sizeof(u16) * 7, GFP_KERNEL);
+	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
 	if (!save_state) {
-		dev_err(&dev->dev, "Out of memory in pci_save_pcie_state\n");
-		return -ENOMEM;
+		save_state = kzalloc(sizeof(*save_state) + sizeof(u16) * 7, GFP_KERNEL);
+		if (!save_state) {
+			printk(KERN_ERR "Out of memory in pci_save_pcie_state\n");
+			return -ENOMEM;
+		}
+		save_state->cap_nr = PCI_CAP_ID_EXP;
+		pci_add_saved_cap(dev, save_state);
 	}
 	cap = (u16 *)&save_state->data[0];
 
@@ -604,8 +609,6 @@ static void pci_restore_pcie_state(struct pci_dev *dev)
 	pci_write_config_word(dev, pos + PCI_EXP_DEVCTL2, cap[i++]);
 	pci_write_config_word(dev, pos + PCI_EXP_LNKCTL2, cap[i++]);
 	pci_write_config_word(dev, pos + PCI_EXP_SLTCTL2, cap[i++]);
-	pci_remove_saved_cap(save_state);
-	kfree(save_state);
 }
 
 /**
@@ -617,7 +620,6 @@ pci_save_state(struct pci_dev *dev)
 {
 	int i;
 
-	save_pcie_reg(dev);
 	/* XXX: 100% dword access ok here? */
 	for (i = 0; i < 16; i++)
 		pci_read_config_dword(dev, i * 4,&dev->saved_config_space[i]);
@@ -658,7 +660,6 @@ pci_restore_state(struct pci_dev *dev)
 				dev->saved_config_space[i]);
 		}
 	}
-	restore_pcie_reg(dev);
 	pci_restore_msi_state(dev);
 	pci_restore_msix_state(dev);
 	pci_restore_iov_state(dev);