From: Bhavna Sarathy <bnagendr@redhat.com>
Date: Tue, 26 Aug 2008 16:51:19 -0400
Subject: [x86] amd oprofile: support instruction based sampling
Message-id: 20080826205257.4286.3472.sendpatchset@localhost.localdomain
O-Subject: [RHEL5.3 PATCH] Support Instruction based sampling (IBS) for AMD family 0x10 processors
Bugzilla: 438385
RH-Acked-by: Brian Maly <bmaly@redhat.com>
Resolves BZ 438385
IBS is a new profiling technique that provides rich, precise program performance
information, introduced by Family 0x10 Barcelona processors. IBS overcomes the
limitations of conventional performance counter sampling wherein the data can be
imprecise. IBS collects a wide range of performance information in a single
program run, making it easier to conduct performance testing.
This patch contains both Barcelona and new Shanghai support for IBS.
Notes:
1) AMD provides IBS and oprofile drivers on their website for developers to
download and use on all RHEL4 and RHEL5 updates. So IBS and RHEL get a lot of QA
action.
2) Not all upstream patches are needed in RHEL, and subset of the upstream
submission is needed in RHEL5.3. We added PMC support for oprofile in RHEL5.3.
This patch is adding IBS support.
Upstream status:
The IBS patches have been submitted upstream for inclusion in 2.6.27. Ingo has
applied the patches to the TIP. The core of the IBS support is in patches #10
and #11 (links below) and the RHEL5.3 patch contains PCI read and write that
is already upstream.
http://lkml.org/lkml/2008/7/22/417
Patches #10 and #11 has the core IBS implementation.
http://lkml.org/lkml/2008/7/22/428
http://lkml.org/lkml/2008/7/24/226
Testing: The driver has been tested on Shanghai system on both 32-bit and 64-bit
RHEL setup. The following samples were tested with AMD Code Analyst GUI running.
- IBS Fetch
- IBS Op (both modes)
- IBS Fetch + IBS-Op
- IBS fetch + IBS-Op + PMC
Please review and ACK.
diff --git a/arch/i386/oprofile/nmi_int.c b/arch/i386/oprofile/nmi_int.c
index 287278d..93ef62b 100644
--- a/arch/i386/oprofile/nmi_int.c
+++ b/arch/i386/oprofile/nmi_int.c
@@ -30,6 +30,8 @@ static void nmi_stop(void);
/* 0 == registered but off, 1 == registered and on */
static int nmi_enabled = 0;
+int ibs_allowed = 0; /* AMD Family 10h+ */
+extern unsigned long driver_version; /* driver version in oprof.c */
#ifdef CONFIG_PM
@@ -185,6 +187,11 @@ static int nmi_setup(void)
free_msrs();
return -EBUSY;
}
+
+ /*setup AMD Family10h IBS irq if needed */
+ if (ibs_allowed)
+ setup_ibs_nmi();
+
/* We need to serialize save and setup for HT because the subset
* of msrs are distinct for save and setup operations
*/
@@ -244,6 +251,10 @@ static void nmi_shutdown(void)
unset_nmi_callback();
release_lapic_nmi();
free_msrs();
+
+ /*clear AMD Family 10h IBS irq if needed */
+ if (ibs_allowed)
+ clear_ibs_nmi();
}
@@ -275,13 +286,14 @@ static void nmi_stop(void)
struct op_counter_config counter_config[OP_MAX_COUNTER];
+struct op_ibs_config ibs_config;
static int nmi_create_files(struct super_block * sb, struct dentry * root)
{
unsigned int i;
-
+ struct dentry *dir;
+
for (i = 0; i < model->num_counters; ++i) {
- struct dentry * dir;
char buf[4];
snprintf(buf, sizeof(buf), "%d", i);
@@ -289,11 +301,43 @@ static int nmi_create_files(struct super_block * sb, struct dentry * root)
oprofilefs_create_ulong(sb, dir, "enabled", &counter_config[i].enabled);
oprofilefs_create_ulong(sb, dir, "event", &counter_config[i].event);
oprofilefs_create_ulong(sb, dir, "count", &counter_config[i].count);
- oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask);
+ oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config [i].unit_mask);
oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel);
oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user);
}
+ /* Setup AMD Family 10h IBS control if needed */
+ if (ibs_allowed) {
+ char buf[12];
+
+ /* setup some reasonable defaults */
+ ibs_config.max_cnt_fetch = 250000;
+ ibs_config.FETCH_enabled = 0;
+ ibs_config.max_cnt_op = 250000;
+ ibs_config.OP_enabled = 0;
+ ibs_config.dispatched_ops = 1;
+ ibs_config.rand_en = 1;
+
+ oprofilefs_create_ulong(sb,root, "version",
+ &driver_version);
+
+ snprintf(buf, sizeof(buf), "ibs_fetch");
+ dir = oprofilefs_mkdir(sb, root, buf);
+ oprofilefs_create_ulong(sb, dir, "rand_enable",
+ &ibs_config.rand_en);
+ oprofilefs_create_ulong(sb, dir, "enable",
+ &ibs_config.FETCH_enabled);
+ oprofilefs_create_ulong(sb, dir, "max_count",
+ &ibs_config.max_cnt_fetch);
+ snprintf(buf, sizeof(buf), "ibs_op");
+ dir = oprofilefs_mkdir(sb, root, buf);
+ oprofilefs_create_ulong(sb, dir, "enable",
+ &ibs_config.OP_enabled);
+ oprofilefs_create_ulong(sb, dir, "max_count",
+ &ibs_config.max_cnt_op);
+ oprofilefs_create_ulong(sb, dir, "dispatched_ops",
+ &ibs_config.dispatched_ops);
+ }
return 0;
}
@@ -377,6 +421,7 @@ int __init op_nmi_init(struct oprofile_operations *ops)
__u8 vendor = boot_cpu_data.x86_vendor;
__u8 family = boot_cpu_data.x86;
char *cpu_type;
+ uint32_t eax, ebx, ecx, edx;
if (!cpu_has_apic)
return -ENODEV;
@@ -402,9 +447,20 @@ int __init op_nmi_init(struct oprofile_operations *ops)
model = &op_athlon_spec;
cpu_type = "x86-64/family10";
break;
+ case 0x11:
+ model = &op_athlon_spec;
+ cpu_type = "x86-64/family11h";
+ break;
+ }
+ /* see if IBS is available */
+ if (family >= 0x10) {
+ cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
+ if (ecx & 0x40)
+ /* This CPU has IBS capability */
+ ibs_allowed = 1;
}
break;
-
+
case X86_VENDOR_INTEL:
switch (family) {
/* Pentium IV */
diff --git a/arch/i386/oprofile/op_counter.h b/arch/i386/oprofile/op_counter.h
index 2880b15..ce60f59 100644
--- a/arch/i386/oprofile/op_counter.h
+++ b/arch/i386/oprofile/op_counter.h
@@ -26,4 +26,14 @@ struct op_counter_config {
extern struct op_counter_config counter_config[];
+struct op_ibs_config {
+ unsigned long OP_enabled;
+ unsigned long FETCH_enabled;
+ unsigned long max_cnt_fetch;
+ unsigned long max_cnt_op;
+ unsigned long rand_en;
+ unsigned long dispatched_ops;
+};
+
+extern struct op_ibs_config ibs_config;
#endif /* OP_COUNTER_H */
diff --git a/arch/i386/oprofile/op_model_athlon.c b/arch/i386/oprofile/op_model_athlon.c
index ddf41f4..d4f8db0 100644
--- a/arch/i386/oprofile/op_model_athlon.c
+++ b/arch/i386/oprofile/op_model_athlon.c
@@ -8,9 +8,13 @@
* @author John Levon
* @author Philippe Elie
* @author Graydon Hoare
+ * @author Barry Kasindorf
*/
#include <linux/oprofile.h>
+#include <linux/device.h>
+#include <linux/pci.h>
+
#include <asm/ptrace.h>
#include <asm/msr.h>
#include <asm/nmi.h>
@@ -40,7 +44,83 @@
#define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 1) << 9))
#define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 1) << 8))
+#ifndef MSR_AMD64_NB_CFG
+/* this is here in case the Perfmon2 patch was not applied yet */
+#define MSR_AMD64_NB_CFG 0xc001001f
+
+/* Definition of Family10h IBS register Addresses */
+#define MSR_AMD64_IBSFETCHCTL 0xc0011030
+#define MSR_AMD64_IBSFETCHLINAD 0xc0011031
+#define MSR_AMD64_IBSFETCHPHYSAD 0xc0011032
+#define MSR_AMD64_IBSOPCTL 0xc0011033
+#define MSR_AMD64_IBSOPRIP 0xc0011034
+#define MSR_AMD64_IBSOPDATA 0xc0011035
+#define MSR_AMD64_IBSOPDATA2 0xc0011036
+#define MSR_AMD64_IBSOPDATA3 0xc0011037
+#define MSR_AMD64_IBSDCLINAD 0xc0011038
+#define MSR_AMD64_IBSDCPHYSAD 0xc0011039
+#define MSR_AMD64_IBSCTL 0xc001103a
+#endif
+
+/* high dword IbsFetchCtl[bit 49] */
+#define IBS_FETCH_VALID_BIT 0x00020000
+/* high dword IbsFetchCtl[bit 52] */
+#define IBS_FETCH_PHY_ADDR_VALID_BIT 0x00100000
+#define IBS_FETCH_CTL_HIGH_MASK 0xFFFFFFFF
+/* high dword IbsFetchCtl[bit 48] */
+#define IBS_FETCH_ENABLE 0x00010000
+#define IBS_FETCH_CTL_CNT_MASK 0x00000000FFFF0000
+#define IBS_FETCH_CTL_MAX_CNT_MASK 0x000000000000FFFF
+
+/*IbsOpCtl masks/bits */
+#define IBS_OP_VALID_BIT 0x0000000000040000 /* IbsOpCtl[bit18] */
+#define IBS_OP_ENABLE 0x0000000000020000 /* IBS_OP_ENABLE[bit17]*/
+
+/*IbsOpData masks */
+#define IBS_OP_DATA_BRANCH_MASK 0x3F00000000 /* IbsOpData[32:37] */
+#define IBS_OP_DATA_HIGH_MASK 0x0000FFFF00000000 /* IbsOpData[32:47] */
+#define IBS_OP_DATA_LOW_MASK 0x00000000FFFFFFFF /*IbsOpData[0:31] */
+
+/*IbsOpData2 masks */
+#define IBS_OP_DATA2_MASK 0x000000000000002F
+
+/*IbsOpData3 masks */
+#define IBS_OP_DATA3_LS_MASK 0x0000000003
+
+#define IBS_OP_DATA3_PHY_ADDR_VALID_BIT 0x0000000000040000
+#define IBS_OP_DATA3_LIN_ADDR_VALID_BIT 0x0000000000020000
+#define IBS_CTL_LVT_OFFSET_VALID_BIT 0x100
+/* AMD ext internal APIC Local Vectors */
+#define APIC_IELVT 0x500
+/* number of APIC Entries for ieLVT */
+#define NUM_APIC_IELVT 4
+
+/*PCI Extended Configuration Constants */
+/* Northbridge Configuration Register */
+#define NB_CFG_MSR 0xC001001F
+/* Bit 46, EnableCf8ExtCfg: enable CF8 extended configuration cycles */
+#define ENABLE_CF8_EXT_CFG_MASK 0x4000
+/* MSR to set the IBS control register APIC LVT offset */
+#define IBS_LVT_OFFSET_PCI 0x1CC
+
+/* IBS rev [bit 10] 1 = IBS Rev B */
+#define IBS_REV_MASK 0x400
+
+/* When pci_ids.h gets caught up remove this */
+#ifndef PCI_DEVICE_ID_AMD_FAMILY10H_NB
+#define PCI_DEVICE_ID_AMD_FAMILY10H_NB 0x1200
+#endif
+
+/**
+ * Add an AMD IBS sample. This may be called from any context. Pass
+ * smp_processor_id() as cpu. Passes IBS registers as a unsigned int[8]
+ */
+void oprofile_add_ibs_op_sample(struct pt_regs * const regs, unsigned int * const ibs_op);
+
+void oprofile_add_ibs_fetch_sample(struct pt_regs * const regs, unsigned int * const ibs_fetch);
static unsigned long reset_value[NUM_COUNTERS];
+extern int ibs_allowed; /* AMD Family 10h+ */
+static int Extended_PCI_Enabled = 0;
static void athlon_fill_in_addresses(struct op_msrs * const msrs)
{
@@ -105,6 +185,8 @@ static int athlon_check_ctrs(struct pt_regs * const regs,
{
unsigned int low, high;
int i;
+ struct ibs_fetch_sample ibs_fetch;
+ struct ibs_op_sample ibs_op;
for (i = 0 ; i < NUM_COUNTERS; ++i) {
CTR_READ(low, high, msrs, i);
@@ -114,6 +196,63 @@ static int athlon_check_ctrs(struct pt_regs * const regs,
}
}
+ /*If AMD and IBS is available */
+ if (ibs_allowed && ibs_config.FETCH_enabled ) {
+ rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+ if ( high & IBS_FETCH_VALID_BIT) {
+ ibs_fetch.ibs_fetch_ctl_high = high;
+ ibs_fetch.ibs_fetch_ctl_low = low;
+ rdmsr(MSR_AMD64_IBSFETCHLINAD, low, high);
+ ibs_fetch.ibs_fetch_lin_addr_high = high;
+ ibs_fetch.ibs_fetch_lin_addr_low = low;
+ rdmsr(MSR_AMD64_IBSFETCHPHYSAD, low, high);
+ ibs_fetch.ibs_fetch_phys_addr_high = high;
+ ibs_fetch.ibs_fetch_phys_addr_low = low;
+
+ oprofile_add_ibs_fetch_sample(regs,
+ (unsigned int *)&ibs_fetch);
+
+ /*reenable the IRQ */
+ rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+ high &= ~(IBS_FETCH_VALID_BIT);
+ high |= IBS_FETCH_ENABLE;
+ low &= IBS_FETCH_CTL_MAX_CNT_MASK;
+ wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+ }
+ }
+
+ if (ibs_allowed && ibs_config.OP_enabled ) {
+ rdmsr(MSR_AMD64_IBSOPCTL, low, high);
+ if (low & IBS_OP_VALID_BIT) {
+ rdmsr(MSR_AMD64_IBSOPRIP, low, high);
+ ibs_op.ibs_op_rip_low = low;
+ ibs_op.ibs_op_rip_high = high;
+ rdmsr(MSR_AMD64_IBSOPDATA, low, high);
+ ibs_op.ibs_op_data1_low = low;
+ ibs_op.ibs_op_data1_high = high;
+ rdmsr(MSR_AMD64_IBSOPDATA2, low, high);
+ ibs_op.ibs_op_data2_low = low;
+ ibs_op.ibs_op_data2_high = high;
+ rdmsr(MSR_AMD64_IBSOPDATA3, low, high);
+ ibs_op.ibs_op_data3_low = low;
+ ibs_op.ibs_op_data3_high = high;
+ rdmsr(MSR_AMD64_IBSDCLINAD, low, high);
+ ibs_op.ibs_dc_linear_low = low;
+ ibs_op.ibs_dc_linear_high = high;
+ rdmsr(MSR_AMD64_IBSDCPHYSAD, low, high);
+ ibs_op.ibs_dc_phys_low = low;
+ ibs_op.ibs_dc_phys_high = high;
+
+ /* reenable the IRQ */
+ oprofile_add_ibs_op_sample(regs,
+ (unsigned int *)&ibs_op);
+ rdmsr(MSR_AMD64_IBSOPCTL, low, high);
+ low &= ~(IBS_OP_VALID_BIT);
+ low |= IBS_OP_ENABLE;
+ wrmsr(MSR_AMD64_IBSOPCTL, low, high);
+ }
+ }
+
/* See op_model_ppro.c */
return 1;
}
@@ -130,8 +269,19 @@ static void athlon_start(struct op_msrs const * const msrs)
CTRL_WRITE(low, high, msrs, i);
}
}
-}
+ if (ibs_allowed && ibs_config.FETCH_enabled ) {
+ low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
+ high = ((ibs_config.rand_en & 0x1) << 25) + IBS_FETCH_ENABLE;
+ wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+ }
+ if (ibs_allowed && ibs_config.OP_enabled ) {
+ low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF) +
+ ((ibs_config.dispatched_ops & 0x1) << 19) + IBS_OP_ENABLE;
+ high = 0;
+ wrmsr(MSR_AMD64_IBSOPCTL, low, high);
+ }
+}
static void athlon_stop(struct op_msrs const * const msrs)
{
@@ -145,8 +295,148 @@ static void athlon_stop(struct op_msrs const * const msrs)
CTRL_SET_INACTIVE(low);
CTRL_WRITE(low, high, msrs, i);
}
+ if (ibs_allowed && ibs_config.FETCH_enabled ) {
+ low = 0; /* clear max count and enable */
+ high = 0;
+ wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
+ }
+
+ if (ibs_allowed && ibs_config.OP_enabled ) {
+ low = 0; /* clear max count and enable */
+ high = 0;
+ wrmsr(MSR_AMD64_IBSOPCTL, low, high);
+ }
}
+static void
+ Enable_Extended_PCI_Config(void)
+{
+ unsigned int low, high;
+ rdmsr(MSR_AMD64_NB_CFG, low, high);
+ Extended_PCI_Enabled = high & ENABLE_CF8_EXT_CFG_MASK;
+ high |= ENABLE_CF8_EXT_CFG_MASK;
+ wrmsr(MSR_AMD64_NB_CFG, low, high);
+}
+
+/*
+ * Disable AMD extended PCI config space thru IO
+ * restore to previous state
+ */
+static void
+ Disable_Extended_PCI_Config(void)
+{
+ unsigned int low, high;
+ rdmsr(MSR_AMD64_NB_CFG, low, high);
+ high &= ~ENABLE_CF8_EXT_CFG_MASK;
+ high |= Extended_PCI_Enabled;
+ wrmsr(MSR_AMD64_NB_CFG, low, high);
+}
+
+/*
+ * Modified to use AMD extended PCI config space thru IO
+ * these 2 I/Os should be atomic but there is no easy way to do that.
+ * Should use the MMio version, will when it is fixed
+ */
+
+static void
+ PCI_Extended_Write(struct pci_dev *dev, unsigned int offset,
+ unsigned long val)
+{
+ outl(0x80000000 | (((offset >> 8) & 0x0f) << 24) |
+ ((dev->bus->number & 0xff) << 16) | ((dev->devfn | 3) << 8)
+ | (offset & 0x0fc), 0x0cf8);
+
+ outl(val, 0xcfc);
+}
+
+static inline void APIC_init_per_cpu(void *arg)
+{
+ unsigned long i = *(unsigned long *)arg;
+
+ apic_write(APIC_IELVT + (i << 4), APIC_DM_NMI);
+}
+
+static inline void APIC_clear_per_cpu(void *arg)
+{
+ unsigned long i = *(unsigned long *)arg;
+
+ apic_write(APIC_IELVT + (i << 4), APIC_LVT_MASKED);
+}
+
+/*
+ * initialize the APIC for the IBS interrupts
+ * if needed on AMD Family10h rev B0 and later
+ */
+void setup_ibs_nmi(void)
+{
+ struct pci_dev *gh_device = NULL;
+ //u32 low, high;
+
+ unsigned long i;
+ unsigned long apicLVT;
+
+#if 0 //Recent BIOS broke this
+ /*see if the IBS control register is already set */
+ rdmsr(MSR_AMD64_IBSCTL, low, high);
+ if (low & IBS_CTL_LVT_OFFSET_VALID_BIT)
+ /*nothing to do it is already setup correctly*/
+ return;
+#endif
+ for (i = 0; i < NUM_APIC_IELVT; i++) {
+ /* get ieLVT contents */
+ apicLVT = apic_read(APIC_IELVT + (i << 4));
+ if ( (apicLVT & APIC_LVT_MASKED) != 0 )
+ /* This slot is disabled, so we can use it */
+ break;
+ }
+
+ Enable_Extended_PCI_Config();
+
+ /**** Be sure to run loop until NULL is returned to
+ decrement reference count on any pci_dev structures returned ****/
+ while ( (gh_device = pci_get_device(PCI_VENDOR_ID_AMD,
+ PCI_DEVICE_ID_AMD_FAMILY10H_NB, gh_device)) != NULL ) {
+ /* This code may change if we can find a proper
+ * way to get at the PCI extended config space */
+ PCI_Extended_Write(
+ gh_device, IBS_LVT_OFFSET_PCI,
+ (i | IBS_CTL_LVT_OFFSET_VALID_BIT) );
+
+ }
+ Disable_Extended_PCI_Config();
+ on_each_cpu(APIC_init_per_cpu, (void *)&i, 1, 1);
+}
+
+/*
+ * unitialize the APIC for the IBS interrupts if needed on AMD Family10h
+ * rev B0 and later */
+void clear_ibs_nmi(void)
+{
+ unsigned long low, high;
+ struct pci_dev *gh_device = NULL;
+ unsigned long i;
+
+ /*see if the IBS control register is already set */
+ rdmsr(MSR_AMD64_IBSCTL, low, high);
+ if ( (low & IBS_CTL_LVT_OFFSET_VALID_BIT) == 0)
+ /*nothing to do it is already cleared
+ *(assume on all CPUS if any is done)
+ */
+ return;
+
+ i = low & 0x3; //get LVT vector number
+
+ on_each_cpu(APIC_clear_per_cpu, (void *)&i, 1, 1);
+ /**** Be sure to run loop until NULL is returned
+ * to decrement reference count on any pci_dev structures returned */
+ Enable_Extended_PCI_Config();
+ while ( (gh_device = pci_get_device(PCI_VENDOR_ID_AMD,
+ PCI_DEVICE_ID_AMD_FAMILY10H_NB, gh_device)) != NULL ) {
+ /* free the LVT entry */
+ PCI_Extended_Write(gh_device, IBS_LVT_OFFSET_PCI, ( 0 ));
+ }
+ Disable_Extended_PCI_Config();
+}
struct op_x86_model_spec const op_athlon_spec = {
.num_counters = NUM_COUNTERS,
diff --git a/arch/i386/oprofile/op_x86_model.h b/arch/i386/oprofile/op_x86_model.h
index 123b7e9..4b70531 100644
--- a/arch/i386/oprofile/op_x86_model.h
+++ b/arch/i386/oprofile/op_x86_model.h
@@ -26,6 +26,38 @@ struct op_msrs {
struct op_msr * controls;
};
+struct ibs_fetch_sample {
+ /* MSRC001_1031 IBS Fetch Linear Address Register */
+ unsigned int ibs_fetch_lin_addr_low;
+ unsigned int ibs_fetch_lin_addr_high;
+ /* MSRC001_1030 IBS Fetch Control Register */
+ unsigned int ibs_fetch_ctl_low;
+ unsigned int ibs_fetch_ctl_high;
+ /* MSRC001_1032 IBS Fetch Physical Address Register */
+ unsigned int ibs_fetch_phys_addr_low;
+ unsigned int ibs_fetch_phys_addr_high;
+};
+
+struct ibs_op_sample {
+ /* MSRC001_1034 IBS Op Logical Address Register (IbsRIP) */
+ unsigned int ibs_op_rip_low;
+ unsigned int ibs_op_rip_high;
+ /* MSRC001_1035 IBS Op Data Register */
+ unsigned int ibs_op_data1_low;
+ unsigned int ibs_op_data1_high;
+ /* MSRC001_1036 IBS Op Data 2 Register */
+ unsigned int ibs_op_data2_low;
+ unsigned int ibs_op_data2_high;
+ /* MSRC001_1037 IBS Op Data 3 Register */
+ unsigned int ibs_op_data3_low;
+ unsigned int ibs_op_data3_high;
+ /* MSRC001_1038 IBS DC Linear Address Register (IbsDcLinAd) */
+ unsigned int ibs_dc_linear_low;
+ unsigned int ibs_dc_linear_high;
+ /* MSRC001_1039 IBS DC Physical Address Register (IbsDcPhysAd) */
+ unsigned int ibs_dc_phys_low;
+ unsigned int ibs_dc_phys_high;
+};
struct pt_regs;
/* The model vtable abstracts the differences between
@@ -47,4 +79,8 @@ extern struct op_x86_model_spec const op_p4_spec;
extern struct op_x86_model_spec const op_p4_ht2_spec;
extern struct op_x86_model_spec const op_athlon_spec;
+/* setup AMD Family 10H IBS IRQ if needed */
+extern void setup_ibs_nmi(void);
+/* clearp AMD Family 10H IBS IRQ if needed */
+extern void clear_ibs_nmi(void);
#endif /* OP_X86_MODEL_H */
diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c
index 9f2a6de..5a6b9d1 100644
--- a/drivers/oprofile/buffer_sync.c
+++ b/drivers/oprofile/buffer_sync.c
@@ -5,6 +5,7 @@
* @remark Read the file COPYING
*
* @author John Levon <levon@movementarian.org>
+ * @author Barry Kasindorf
*
* Modified by Aravind Menon for Xen
* These modifications are:
@@ -37,6 +38,12 @@
#include "cpu_buffer.h"
#include "buffer_sync.h"
+#ifndef IBS_FETCH_CODE
+/* in case the kernel headers were not patched yet */
+#define IBS_FETCH_CODE 13
+#define IBS_OP_CODE 14
+#endif
+
static LIST_HEAD(dying_tasks);
static LIST_HEAD(dead_tasks);
static cpumask_t marked_cpus = CPU_MASK_NONE;
@@ -275,6 +282,17 @@ static unsigned long lookup_dcookie(struct mm_struct * mm, unsigned long addr, o
return cookie;
}
+static void increment_tail(struct oprofile_cpu_buffer *b)
+{
+ unsigned long new_tail = b->tail_pos + 1;
+
+ rmb(); /* be sure fifo pointers are synchromized */
+
+ if (new_tail < b->buffer_size)
+ b->tail_pos = new_tail;
+ else
+ b->tail_pos = 0;
+}
static unsigned long last_cookie = INVALID_COOKIE;
@@ -339,6 +357,68 @@ static void add_trace_begin(void)
add_event_entry(TRACE_BEGIN_CODE);
}
+/*
+ * Add IBS fetch and op entries to event buffer
+ */
+static void add_ibs_begin(struct oprofile_cpu_buffer *cpu_buf, int code,
+ int in_kernel, struct mm_struct *mm)
+{
+ unsigned long rip;
+ int i, count;
+ unsigned long ibs_cookie = 0;
+ off_t offset;
+
+ increment_tail(cpu_buf); /* move to RIP entry */
+
+ rip = ((struct op_sample *)&cpu_buf->buffer[cpu_buf->tail_pos])->eip;
+
+#ifdef __LP64__
+ rip += ((struct op_sample *)&cpu_buf->buffer[cpu_buf->tail_pos])->event << 32;
+#endif
+
+ if (mm) {
+ ibs_cookie = lookup_dcookie(mm, rip, &offset);
+
+ if(ibs_cookie == NO_COOKIE)
+ {
+ offset = rip;
+ }
+ if (ibs_cookie == INVALID_COOKIE) {
+ atomic_inc(&oprofile_stats.sample_lost_no_mapping);
+ offset = rip;
+ }
+ if (ibs_cookie != last_cookie) {
+ add_cookie_switch(ibs_cookie);
+ last_cookie = ibs_cookie;
+ }
+ }
+
+ else
+ offset = rip;
+
+ add_event_entry(ESCAPE_CODE);
+ add_event_entry(code);
+ add_event_entry(offset); /* Offset from Dcookie */
+
+ /* even though we send the Dcookie offset, send the raw Linear Address as well */
+ add_event_entry(
+ ((struct op_sample *)&cpu_buf->buffer[cpu_buf->tail_pos])->eip);
+ add_event_entry(
+ ((struct op_sample *)&cpu_buf->buffer[cpu_buf->tail_pos])->event);
+
+ if (code == IBS_FETCH_CODE)
+ count = 2; /*IBS FETCH is 2 int64s long */
+ else
+ count = 5; /*IBS OP is 5 int64s long */
+
+ for (i = 0; i < count; i++) {
+ increment_tail(cpu_buf);
+ add_event_entry(
+ ((struct op_sample *)&cpu_buf->buffer[cpu_buf->tail_pos])->eip);
+ add_event_entry(
+ ((struct op_sample *)&cpu_buf->buffer[cpu_buf->tail_pos])->event);
+ }
+}
static void add_sample_entry(unsigned long offset, unsigned long event)
{
@@ -440,19 +520,6 @@ static unsigned long get_slots(struct oprofile_cpu_buffer * b)
}
-static void increment_tail(struct oprofile_cpu_buffer * b)
-{
- unsigned long new_tail = b->tail_pos + 1;
-
- rmb();
-
- if (new_tail < b->buffer_size)
- b->tail_pos = new_tail;
- else
- b->tail_pos = 0;
-}
-
-
/* Move tasks along towards death. Any tasks on dead_tasks
* will definitely have no remaining references in any
* CPU buffers at this point, because we use two lists,
@@ -542,7 +609,7 @@ void sync_buffer(int cpu)
available = get_slots(cpu_buf);
- for (i = 0; i < available; ++i) {
+ while (get_slots(cpu_buf)) {
struct op_sample * s = &cpu_buf->buffer[cpu_buf->tail_pos];
if (is_code(s->eip) && !domain_switch) {
@@ -556,7 +623,15 @@ void sync_buffer(int cpu)
state = sb_bt_start;
add_trace_begin();
} else if (s->event == CPU_DOMAIN_SWITCH) {
- domain_switch = 1;
+ domain_switch = 1;
+ } else if (s->event == IBS_FETCH_BEGIN) {
+ state = sb_bt_start;
+ add_ibs_begin(cpu_buf,
+ IBS_FETCH_CODE, cpu_mode, mm);
+ } else if (s->event == IBS_OP_BEGIN) {
+ state = sb_bt_start;
+ add_ibs_begin(cpu_buf,
+ IBS_OP_CODE, cpu_mode, mm);
} else {
struct mm_struct * oldmm = mm;
diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c
index a59878e..2b08d71 100644
--- a/drivers/oprofile/cpu_buffer.c
+++ b/drivers/oprofile/cpu_buffer.c
@@ -208,6 +208,48 @@ static int log_sample(struct oprofile_cpu_buffer * cpu_buf, unsigned long pc,
return 1;
}
+static int log_ibs_sample(struct oprofile_cpu_buffer *cpu_buf,
+ unsigned long pc, int cpu_mode, unsigned int *ibs, int ibs_code)
+{
+ struct task_struct *task;
+
+ cpu_buf->sample_received++;
+
+ if (nr_available_slots(cpu_buf) < 14) {
+ cpu_buf->sample_lost_overflow++;
+ return 0;
+ }
+
+ task = current;
+
+ /* notice a switch from user->kernel or vice versa */
+ if (cpu_buf->last_cpu_mode != cpu_mode) {
+ cpu_buf->last_cpu_mode = cpu_mode;
+ add_code(cpu_buf, cpu_mode);
+ }
+
+ /* notice a task switch */
+ /* if not processing other domain samples */
+ if ((cpu_buf->last_task != task) &&
+ (current_domain == COORDINATOR_DOMAIN)) {
+ cpu_buf->last_task = task;
+ add_code(cpu_buf, (unsigned long)task);
+ }
+
+ add_code(cpu_buf, ibs_code);
+ add_sample(cpu_buf, ibs[0], ibs[1]);
+ add_sample(cpu_buf, ibs[2], ibs[3]);
+ add_sample(cpu_buf, ibs[4], ibs[5]);
+
+ if (ibs_code == IBS_OP_BEGIN) {
+ add_sample(cpu_buf, ibs[6], ibs[7]);
+ add_sample(cpu_buf, ibs[8], ibs[9]);
+ add_sample(cpu_buf, ibs[10], ibs[11]);
+ }
+
+ return 1;
+}
+
static int oprofile_begin_trace(struct oprofile_cpu_buffer * cpu_buf)
{
if (nr_available_slots(cpu_buf) < 4) {
@@ -253,6 +295,45 @@ void oprofile_add_sample(struct pt_regs * const regs, unsigned long event)
oprofile_add_ext_sample(pc, regs, event, is_kernel);
}
+void oprofile_add_ibs_fetch_sample(struct pt_regs *const regs,
+ unsigned int * const ibs_fetch)
+{
+ int is_kernel = !user_mode(regs);
+ unsigned long pc = profile_pc(regs);
+
+ struct oprofile_cpu_buffer *cpu_buf = &cpu_buffer[smp_processor_id()];
+
+ if (!backtrace_depth) {
+ log_ibs_sample(cpu_buf, pc, is_kernel,
+ ibs_fetch, IBS_FETCH_BEGIN);
+ return;
+ }
+
+ /* if log_sample() fails we can't backtrace since we lost the source
+ * of this event */
+ if (log_ibs_sample(cpu_buf, pc, is_kernel, ibs_fetch, IBS_FETCH_BEGIN))
+ oprofile_ops.backtrace(regs, backtrace_depth);
+}
+
+void oprofile_add_ibs_op_sample(struct pt_regs *const regs,
+ unsigned int * const ibs_op)
+{
+ int is_kernel = !user_mode(regs);
+ unsigned long pc = profile_pc(regs);
+
+ struct oprofile_cpu_buffer *cpu_buf = &cpu_buffer[smp_processor_id()];
+
+ if (!backtrace_depth) {
+ log_ibs_sample(cpu_buf, pc, is_kernel, ibs_op, IBS_OP_BEGIN);
+ return;
+ }
+
+ /* if log_sample() fails we can't backtrace since we lost the source
+ * of this event */
+ if (log_ibs_sample(cpu_buf, pc, is_kernel, ibs_op, IBS_OP_BEGIN))
+ oprofile_ops.backtrace(regs, backtrace_depth);
+}
+
void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event)
{
struct oprofile_cpu_buffer * cpu_buf = &cpu_buffer[smp_processor_id()];
diff --git a/drivers/oprofile/cpu_buffer.h b/drivers/oprofile/cpu_buffer.h
index cd94735..4af515a 100644
--- a/drivers/oprofile/cpu_buffer.h
+++ b/drivers/oprofile/cpu_buffer.h
@@ -56,5 +56,7 @@ void cpu_buffer_reset(struct oprofile_cpu_buffer * cpu_buf);
#define CPU_MODE_XEN 2
#define CPU_TRACE_BEGIN 3
#define CPU_DOMAIN_SWITCH 4
+#define IBS_FETCH_BEGIN 5
+#define IBS_OP_BEGIN 6
#endif /* OPROFILE_CPU_BUFFER_H */
diff --git a/drivers/oprofile/oprof.c b/drivers/oprofile/oprof.c
index cc11e87..0bfb11f 100644
--- a/drivers/oprofile/oprof.c
+++ b/drivers/oprofile/oprof.c
@@ -30,6 +30,7 @@ unsigned long oprofile_started;
unsigned long backtrace_depth;
static unsigned long is_setup;
static DEFINE_MUTEX(start_mutex);
+unsigned long driver_version = 0x1000; /*Update version here */
/* timer
0 - use performance monitoring hardware if available
distrib
>
Scientific%20Linux
>
5x
>
x86_64
>
by-pkgid
>
89877e42827f16fa5f86b1df0c2860b1
>
files
>
2419
