From: David Milburn <dmilburn@redhat.com>
Date: Mon, 29 Mar 2010 17:44:21 -0400
Subject: [ata] ahci: support FIS-based switching
Message-id: <20100329174421.GA19927@localhost.localdomain>
Patchwork-id: 23806
O-Subject: [RHEL5.6 PATCH] ahci: support FIS-based switching
Bugzilla: 474294
RH-Acked-by: Jeff Garzik <jgarzik@redhat.com>
Some AHCI controllers support FIS-based switching which allows the
host to issue commands to a device while there are still outstanding
commands to other devices connected to the same host port through
a port multiplier. This patch is provided by Shane Huang and has
been tested by AMD, and has shown to improve performance over Command
based switching which only sends commands to one device at a time.
Upstream commits:
commit 5db5b0215af94a36d4bf10900ff9707b6d5c1610
commit d6ef31539d715db7a0eb2d67a1a008c9d8ccf059
commit 78d5ae39af19a02a3d16213c0f62930d9a5d25f7
This resolves BZ 474294, please review and ACK.
Thanks,
David
drivers/ata/ahci.c | 270 ++++++++++++++++++++++++++++++++++++++++++++-------
1 files changed, 233 insertions(+), 37 deletions(-)
Signed-off-by: Jarod Wilson <jarod@redhat.com>
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index 81a0c69..dcb0ea9 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -90,6 +90,9 @@ enum {
AHCI_CMD_TBL_AR_SZ = AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
AHCI_PORT_PRIV_DMA_SZ = AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
AHCI_RX_FIS_SZ,
+ AHCI_PORT_PRIV_FBS_DMA_SZ = AHCI_CMD_SLOT_SZ +
+ AHCI_CMD_TBL_AR_SZ +
+ (AHCI_RX_FIS_SZ * 16),
AHCI_IRQ_ON_SG = (1 << 31),
AHCI_CMD_ATAPI = (1 << 5),
AHCI_CMD_WRITE = (1 << 6),
@@ -126,6 +129,7 @@ enum {
HOST_CAP_EMS = (1 << 6), /* Enclosure Management support */
HOST_CAP_SSC = (1 << 14), /* Slumber capable */
HOST_CAP_PMP = (1 << 17), /* Port Multiplier support */
+ HOST_CAP_FBS = (1 << 16), /* FIS-based Switching support */
HOST_CAP_CLO = (1 << 24), /* Command List Override support */
HOST_CAP_ALPM = (1 << 26), /* Aggressive Link PM support */
HOST_CAP_SSS = (1 << 27), /* Staggered Spin-up */
@@ -149,6 +153,7 @@ enum {
PORT_SCR_ERR = 0x30, /* SATA phy register: SError */
PORT_SCR_ACT = 0x34, /* SATA phy register: SActive */
PORT_SCR_NTF = 0x3c, /* SATA phy register: SNotification */
+ PORT_FBS = 0x40, /* FIS-based Switching */
/* PORT_IRQ_{STAT,MASK} bits */
PORT_IRQ_COLD_PRES = (1 << 31), /* cold presence detect */
@@ -187,6 +192,7 @@ enum {
PORT_CMD_ASP = (1 << 27), /* Aggressive Slumber/Partial */
PORT_CMD_ALPE = (1 << 26), /* Aggressive Link PM enable */
PORT_CMD_ATAPI = (1 << 24), /* Device is ATAPI */
+ PORT_CMD_FBSCP = (1 << 22), /* FBS Capable Port */
PORT_CMD_PMP = (1 << 17), /* PMP attached */
PORT_CMD_LIST_ON = (1 << 15), /* cmd list DMA engine running */
PORT_CMD_FIS_ON = (1 << 14), /* FIS DMA engine running */
@@ -201,6 +207,14 @@ enum {
PORT_CMD_ICC_PARTIAL = (0x2 << 28), /* Put i/f in partial state */
PORT_CMD_ICC_SLUMBER = (0x6 << 28), /* Put i/f in slumber state */
+ PORT_FBS_DWE_OFFSET = 16, /* FBS device with error offset */
+ PORT_FBS_ADO_OFFSET = 12, /* FBS active dev optimization offset */
+ PORT_FBS_DEV_OFFSET = 8, /* FBS device to issue offset */
+ PORT_FBS_DEV_MASK = (0xf << PORT_FBS_DEV_OFFSET), /* FBS.DEV */
+ PORT_FBS_SDE = (1 << 2), /* FBS single device error */
+ PORT_FBS_DEC = (1 << 1), /* FBS device error clear */
+ PORT_FBS_EN = (1 << 0), /* Enable FBS */
+
/* hpriv->flags bits */
AHCI_HFLAG_NO_NCQ = (1 << 0),
AHCI_HFLAG_IGN_IRQ_IF_ERR = (1 << 1), /* ignore IRQ_IF_ERR */
@@ -272,6 +286,9 @@ struct ahci_port_priv {
unsigned int ncq_saw_dmas:1;
unsigned int ncq_saw_sdb:1;
u32 intr_mask; /* interrupts to enable */
+ bool fbs_supported; /* set iff FBS is supported */
+ bool fbs_enabled; /* set iff FBS is enabled */
+ int fbs_last_dev; /* save FBS.DEV of last FIS */
struct ahci_em_priv em_priv[MAX_SLOTS];/* enclosure management info
* per PM slot */
};
@@ -283,9 +300,12 @@ static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
static int ahci_port_start(struct ata_port *ap);
static void ahci_port_stop(struct ata_port *ap);
+static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
static void ahci_qc_prep(struct ata_queued_cmd *qc);
static void ahci_freeze(struct ata_port *ap);
static void ahci_thaw(struct ata_port *ap);
+static void ahci_enable_fbs(struct ata_port *ap);
+static void ahci_disable_fbs(struct ata_port *ap);
static void ahci_pmp_attach(struct ata_port *ap);
static void ahci_pmp_detach(struct ata_port *ap);
static int ahci_softreset(struct ata_link *link, unsigned int *class,
@@ -340,7 +360,7 @@ static struct scsi_host_template ahci_sht = {
static struct ata_port_operations ahci_ops = {
.inherits = &sata_pmp_port_ops,
- .qc_defer = sata_pmp_qc_defer_cmd_switch,
+ .qc_defer = ahci_pmp_qc_defer,
.qc_prep = ahci_qc_prep,
.qc_issue = ahci_qc_issue,
.qc_fill_rtf = ahci_qc_fill_rtf,
@@ -1533,7 +1553,7 @@ static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
}
-static int ahci_kick_engine(struct ata_port *ap, int force_restart)
+static int ahci_kick_engine(struct ata_port *ap)
{
void __iomem *port_mmio = ahci_port_base(ap);
struct ahci_host_priv *hpriv = ap->host->private_data;
@@ -1541,18 +1561,16 @@ static int ahci_kick_engine(struct ata_port *ap, int force_restart)
u32 tmp;
int busy, rc;
- /* do we need to kick the port? */
- busy = status & (ATA_BUSY | ATA_DRQ);
- if (!busy && !force_restart)
- return 0;
-
/* stop engine */
rc = ahci_stop_engine(ap);
if (rc)
goto out_restart;
- /* need to do CLO? */
- if (!busy) {
+ /* need to do CLO?
+ * always do CLO if PMP is attached (AHCI-1.3 9.2)
+ */
+ busy = status & (ATA_BUSY | ATA_DRQ);
+ if (!busy && !sata_pmp_attached(ap)) {
rc = 0;
goto out_restart;
}
@@ -1600,7 +1618,7 @@ static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
1, timeout_msec);
if (tmp & 0x1) {
- ahci_kick_engine(ap, 1);
+ ahci_kick_engine(ap);
return -EBUSY;
}
} else
@@ -1621,7 +1639,7 @@ static int ahci_do_softreset(struct ata_link *link, unsigned int *class,
DPRINTK("ENTER\n");
/* prepare for SRST (AHCI-1.1 10.4.1) */
- rc = ahci_kick_engine(ap, 1);
+ rc = ahci_kick_engine(ap);
if (rc && rc != -EOPNOTSUPP)
ata_link_printk(link, KERN_WARNING,
"failed to reset engine (errno=%d)\n", rc);
@@ -1827,7 +1845,7 @@ static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
rc = ata_wait_after_reset(link, jiffies + 2 * HZ,
ahci_check_ready);
if (rc)
- ahci_kick_engine(ap, 0);
+ ahci_kick_engine(ap);
}
return rc;
}
@@ -1879,6 +1897,17 @@ static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
return n_sg;
}
+static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
+{
+ struct ata_port *ap = qc->ap;
+ struct ahci_port_priv *pp = ap->private_data;
+
+ if (!sata_pmp_attached(ap) || pp->fbs_enabled)
+ return ata_std_qc_defer(qc);
+ else
+ return sata_pmp_qc_defer_cmd_switch(qc);
+}
+
static void ahci_qc_prep(struct ata_queued_cmd *qc)
{
struct ata_port *ap = qc->ap;
@@ -1917,6 +1946,31 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc)
ahci_fill_cmd_slot(pp, qc->tag, opts);
}
+static void ahci_fbs_dec_intr(struct ata_port *ap)
+{
+ struct ahci_port_priv *pp = ap->private_data;
+ void __iomem *port_mmio = ahci_port_base(ap);
+ u32 fbs = readl(port_mmio + PORT_FBS);
+ int retries = 3;
+
+ DPRINTK("ENTER\n");
+ BUG_ON(!pp->fbs_enabled);
+
+ /* time to wait for DEC is not specified by AHCI spec,
+ * add a retry loop for safety.
+ */
+ writel(fbs | PORT_FBS_DEC, port_mmio + PORT_FBS);
+ fbs = readl(port_mmio + PORT_FBS);
+ while ((fbs & PORT_FBS_DEC) && retries--) {
+ udelay(1);
+ fbs = readl(port_mmio + PORT_FBS);
+ }
+
+ if (fbs & PORT_FBS_DEC)
+ dev_printk(KERN_ERR, ap->host->dev,
+ "failed to clear device error\n");
+}
+
static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
{
struct ahci_host_priv *hpriv = ap->host->private_data;
@@ -1925,12 +1979,26 @@ static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
struct ata_link *link = NULL;
struct ata_queued_cmd *active_qc;
struct ata_eh_info *active_ehi;
+ bool fbs_need_dec = false;
u32 serror;
- /* determine active link */
- ata_port_for_each_link(link, ap)
- if (ata_link_active(link))
- break;
+ /* determine active link with error */
+ if (pp->fbs_enabled) {
+ void __iomem *port_mmio = ahci_port_base(ap);
+ u32 fbs = readl(port_mmio + PORT_FBS);
+ int pmp = fbs >> PORT_FBS_DWE_OFFSET;
+
+ if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links) &&
+ ata_link_online(&ap->pmp_link[pmp])) {
+ link = &ap->pmp_link[pmp];
+ fbs_need_dec = true;
+ }
+
+ } else
+ ata_port_for_each_link(link, ap)
+ if (ata_link_active(link))
+ break;
+
if (!link)
link = &ap->link;
@@ -1987,8 +2055,13 @@ static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
}
if (irq_stat & PORT_IRQ_IF_ERR) {
- host_ehi->err_mask |= AC_ERR_ATA_BUS;
- host_ehi->action |= ATA_EH_RESET;
+ if (fbs_need_dec)
+ active_ehi->err_mask |= AC_ERR_DEV;
+ else {
+ host_ehi->err_mask |= AC_ERR_ATA_BUS;
+ host_ehi->action |= ATA_EH_RESET;
+ }
+
ata_ehi_push_desc(host_ehi, "interface fatal error");
}
@@ -2003,7 +2076,10 @@ static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
if (irq_stat & PORT_IRQ_FREEZE)
ata_port_freeze(ap);
- else
+ else if (fbs_need_dec) {
+ ata_link_abort(link);
+ ahci_fbs_dec_intr(ap);
+ } else
ata_port_abort(ap);
}
@@ -2014,7 +2090,7 @@ static void ahci_port_intr(struct ata_port *ap)
struct ahci_port_priv *pp = ap->private_data;
struct ahci_host_priv *hpriv = ap->host->private_data;
int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
- u32 status, qc_active;
+ u32 status, qc_active = 0;
int rc;
status = readl(port_mmio + PORT_IRQ_STAT);
@@ -2056,20 +2132,38 @@ static void ahci_port_intr(struct ata_port *ap)
/* If the 'N' bit in word 0 of the FIS is set,
* we just received asynchronous notification.
* Tell libata about it.
+ *
+ * Lack of SNotification should not appear in
+ * ahci 1.2, so the workaround is unnecessary
+ * when FBS is enabled.
*/
- const __le32 *f = pp->rx_fis + RX_FIS_SDB;
- u32 f0 = le32_to_cpu(f[0]);
-
- if (f0 & (1 << 15))
- sata_async_notification(ap);
+ if (pp->fbs_enabled)
+ WARN_ON_ONCE(1);
+ else {
+ const __le32 *f = pp->rx_fis + RX_FIS_SDB;
+ u32 f0 = le32_to_cpu(f[0]);
+ if (f0 & (1 << 15))
+ sata_async_notification(ap);
+ }
}
}
- /* pp->active_link is valid iff any command is in flight */
- if (ap->qc_active && pp->active_link->sactive)
- qc_active = readl(port_mmio + PORT_SCR_ACT);
- else
- qc_active = readl(port_mmio + PORT_CMD_ISSUE);
+ /* pp->active_link is not reliable once FBS is enabled, both
+ * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
+ * NCQ and non-NCQ commands may be in flight at the same time.
+ */
+ if (pp->fbs_enabled) {
+ if (ap->qc_active) {
+ qc_active = readl(port_mmio + PORT_SCR_ACT);
+ qc_active |= readl(port_mmio + PORT_CMD_ISSUE);
+ }
+ } else {
+ /* pp->active_link is valid iff any command is in flight */
+ if (ap->qc_active && pp->active_link->sactive)
+ qc_active = readl(port_mmio + PORT_SCR_ACT);
+ else
+ qc_active = readl(port_mmio + PORT_CMD_ISSUE);
+ }
rc = ata_qc_complete_multiple(ap, qc_active);
@@ -2148,6 +2242,15 @@ static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
if (qc->tf.protocol == ATA_PROT_NCQ)
writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
+
+ if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
+ u32 fbs = readl(port_mmio + PORT_FBS);
+ fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
+ fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
+ writel(fbs, port_mmio + PORT_FBS);
+ pp->fbs_last_dev = qc->dev->link->pmp;
+ }
+
writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
readl(port_mmio + PORT_CMD_ISSUE); /* flush */
@@ -2161,6 +2264,9 @@ static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
struct ahci_port_priv *pp = qc->ap->private_data;
u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
+ if (pp->fbs_enabled)
+ d2h_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
+
ata_tf_from_fis(d2h_fis, &qc->result_tf);
return true;
}
@@ -2206,7 +2312,72 @@ static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
/* make DMA engine forget about the failed command */
if (qc->flags & ATA_QCFLAG_FAILED)
- ahci_kick_engine(ap, 1);
+ ahci_kick_engine(ap);
+}
+
+static void ahci_enable_fbs(struct ata_port *ap)
+{
+ struct ahci_port_priv *pp = ap->private_data;
+ void __iomem *port_mmio = ahci_port_base(ap);
+ u32 fbs;
+ int rc;
+
+ if (!pp->fbs_supported)
+ return;
+
+ fbs = readl(port_mmio + PORT_FBS);
+ if (fbs & PORT_FBS_EN) {
+ pp->fbs_enabled = true;
+ pp->fbs_last_dev = -1; /* initialization */
+ return;
+ }
+
+ rc = ahci_stop_engine(ap);
+ if (rc)
+ return;
+
+ writel(fbs | PORT_FBS_EN, port_mmio + PORT_FBS);
+ fbs = readl(port_mmio + PORT_FBS);
+ if (fbs & PORT_FBS_EN) {
+ dev_printk(KERN_INFO, ap->host->dev, "FBS is enabled.\n");
+ pp->fbs_enabled = true;
+ pp->fbs_last_dev = -1; /* initialization */
+ } else
+ dev_printk(KERN_ERR, ap->host->dev, "Failed to enable FBS\n");
+
+ ahci_start_engine(ap);
+}
+
+static void ahci_disable_fbs(struct ata_port *ap)
+{
+ struct ahci_port_priv *pp = ap->private_data;
+ void __iomem *port_mmio = ahci_port_base(ap);
+ u32 fbs;
+ int rc;
+
+ if (!pp->fbs_supported)
+ return;
+
+ fbs = readl(port_mmio + PORT_FBS);
+ if ((fbs & PORT_FBS_EN) == 0) {
+ pp->fbs_enabled = false;
+ return;
+ }
+
+ rc = ahci_stop_engine(ap);
+ if (rc)
+ return;
+
+ writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
+ fbs = readl(port_mmio + PORT_FBS);
+ if (fbs & PORT_FBS_EN)
+ dev_printk(KERN_ERR, ap->host->dev, "Failed to disable FBS\n");
+ else {
+ dev_printk(KERN_INFO, ap->host->dev, "FBS is disabled.\n");
+ pp->fbs_enabled = false;
+ }
+
+ ahci_start_engine(ap);
}
static void ahci_pmp_attach(struct ata_port *ap)
@@ -2219,6 +2390,8 @@ static void ahci_pmp_attach(struct ata_port *ap)
cmd |= PORT_CMD_PMP;
writel(cmd, port_mmio + PORT_CMD);
+ ahci_enable_fbs(ap);
+
pp->intr_mask |= PORT_IRQ_BAD_PMP;
writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
}
@@ -2229,6 +2402,8 @@ static void ahci_pmp_detach(struct ata_port *ap)
struct ahci_port_priv *pp = ap->private_data;
u32 cmd;
+ ahci_disable_fbs(ap);
+
cmd = readl(port_mmio + PORT_CMD);
cmd &= ~PORT_CMD_PMP;
writel(cmd, port_mmio + PORT_CMD);
@@ -2329,11 +2504,13 @@ static int ahci_pci_device_resume(struct pci_dev *pdev)
static int ahci_port_start(struct ata_port *ap)
{
+ struct ahci_host_priv *hpriv = ap->host->private_data;
struct device *dev = ap->host->dev;
struct ahci_port_priv *pp;
void *mem;
dma_addr_t mem_dma;
int rc;
+ size_t dma_sz, rx_fis_sz;
pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
if (!pp)
@@ -2343,11 +2520,29 @@ static int ahci_port_start(struct ata_port *ap)
if (rc)
return rc;
- mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
- GFP_KERNEL);
+ /* check FBS capability */
+ if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
+ void __iomem *port_mmio = ahci_port_base(ap);
+ u32 cmd = readl(port_mmio + PORT_CMD);
+ if (cmd & PORT_CMD_FBSCP)
+ pp->fbs_supported = true;
+ else
+ dev_printk(KERN_WARNING, dev,
+ "The port is not capable of FBS\n");
+ }
+
+ if (pp->fbs_supported) {
+ dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
+ rx_fis_sz = AHCI_RX_FIS_SZ * 16;
+ } else {
+ dma_sz = AHCI_PORT_PRIV_DMA_SZ;
+ rx_fis_sz = AHCI_RX_FIS_SZ;
+ }
+
+ mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
if (!mem)
return -ENOMEM;
- memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);
+ memset(mem, 0, dma_sz);
/*
* First item in chunk of DMA memory: 32-slot command table,
@@ -2365,8 +2560,8 @@ static int ahci_port_start(struct ata_port *ap)
pp->rx_fis = mem;
pp->rx_fis_dma = mem_dma;
- mem += AHCI_RX_FIS_SZ;
- mem_dma += AHCI_RX_FIS_SZ;
+ mem += rx_fis_sz;
+ mem_dma += rx_fis_sz;
/*
* Third item: data area for storing a single command
@@ -2484,7 +2679,7 @@ static void ahci_print_info(struct ata_host *host)
"flags: "
"%s%s%s%s%s%s%s"
"%s%s%s%s%s%s%s"
- "%s\n"
+ "%s%s\n"
,
cap & (1 << 31) ? "64bit " : "",
@@ -2499,6 +2694,7 @@ static void ahci_print_info(struct ata_host *host)
cap & (1 << 19) ? "nz " : "",
cap & (1 << 18) ? "only " : "",
cap & (1 << 17) ? "pmp " : "",
+ cap & (1 << 16) ? "fbs " : "",
cap & (1 << 15) ? "pio " : "",
cap & (1 << 14) ? "slum " : "",
cap & (1 << 13) ? "part " : "",
distrib
>
Scientific%20Linux
>
5x
>
x86_64
>
by-pkgid
>
27922b4260f65d317aabda37e42bbbff
>
files
>
160
