@@ -18,6 +18,7 @@
#include <linux/seq_file.h>
#include <linux/sizes.h>
#include <linux/spinlock.h>
+#include <linux/units.h>
#include <linux/interrupt.h>
#include <linux/of.h>
#include <linux/of_device.h>
@@ -34,6 +35,7 @@
/* DDR capabilities */
#define SNPS_CAP_ECC_SCRUB BIT(0)
+#define SNPS_CAP_ECC_SCRUBBER BIT(1)
#define SNPS_CAP_ZYNQMP BIT(31)
/* Synopsys uMCTL2 DDR controller registers that are relevant to ECC */
@@ -102,6 +104,12 @@
#define DDR_SARBASE0_OFST 0xF04
#define DDR_SARSIZE0_OFST 0xF08
+/* ECC Scrubber registers */
+#define ECC_SBRCTL_OFST 0xF24
+#define ECC_SBRSTAT_OFST 0xF28
+#define ECC_SBRWDATA0_OFST 0xF2C
+#define ECC_SBRWDATA1_OFST 0xF30
+
/* DDR Master Register 0 definitions */
#define DDR_MSTR_DEV_CFG_MASK GENMASK(31, 30)
#define DDR_MSTR_DEV_X4 0x0
@@ -244,6 +252,18 @@
#define DDR_MAX_NSAR 4
#define DDR_MIN_SARSIZE SZ_256M
+/* ECC Scrubber registers definitions */
+#define ECC_SBRCTL_SCRUB_INTERVAL GENMASK(20, 8)
+#define ECC_SBRCTL_INTERVAL_STEP 512
+#define ECC_SBRCTL_INTERVAL_MIN 0
+#define ECC_SBRCTL_INTERVAL_SAFE 1
+#define ECC_SBRCTL_INTERVAL_MAX (ECC_SBRCTL_SCRUB_INTERVAL >> 8)
+#define ECC_SBRCTL_SCRUB_BURST GENMASK(6, 4)
+#define ECC_SBRCTL_SCRUB_MODE_WR BIT(2)
+#define ECC_SBRCTL_SCRUB_EN BIT(0)
+#define ECC_SBRSTAT_SCRUB_DONE BIT(1)
+#define ECC_SBRSTAT_SCRUB_BUSY BIT(0)
+
/**
* enum snps_dq_width - SDRAM DQ bus width (ECC capable).
* SNPS_DQ_32: 32-bit memory data width.
@@ -906,6 +926,47 @@ static irqreturn_t snps_dfi_irq_handler(int irq, void *dev_id)
return IRQ_HANDLED;
}
+/**
+ * snps_sbr_irq_handler - Scrubber Done interrupt handler.
+ * @irq: IRQ number.
+ * @dev_id: Device ID.
+ *
+ * It just checks whether the IRQ has been caused by the Scrubber Done event
+ * and disables the back-to-back scrubbing by falling back to the smallest
+ * delay between the Scrubber read commands.
+ *
+ * Return: IRQ_NONE, if interrupt not set or IRQ_HANDLED otherwise.
+ */
+static irqreturn_t snps_sbr_irq_handler(int irq, void *dev_id)
+{
+ struct mem_ctl_info *mci = dev_id;
+ struct snps_edac_priv *priv = mci->pvt_info;
+ unsigned long flags;
+ u32 regval, en;
+
+ /* Make sure IRQ is caused by the Scrubber Done event */
+ regval = readl(priv->baseaddr + ECC_SBRSTAT_OFST);
+ if (!(regval & ECC_SBRSTAT_SCRUB_DONE))
+ return IRQ_NONE;
+
+ spin_lock_irqsave(&priv->lock, flags);
+
+ regval = readl(priv->baseaddr + ECC_SBRCTL_OFST);
+ en = regval & ECC_SBRCTL_SCRUB_EN;
+ writel(regval & ~en, priv->baseaddr + ECC_SBRCTL_OFST);
+
+ regval = FIELD_PREP(ECC_SBRCTL_SCRUB_INTERVAL, ECC_SBRCTL_INTERVAL_SAFE);
+ writel(regval, priv->baseaddr + ECC_SBRCTL_OFST);
+
+ writel(regval | en, priv->baseaddr + ECC_SBRCTL_OFST);
+
+ spin_unlock_irqrestore(&priv->lock, flags);
+
+ edac_mc_printk(mci, KERN_WARNING, "Back-to-back scrubbing disabled\n");
+
+ return IRQ_HANDLED;
+}
+
/**
* snps_com_irq_handler - Interrupt IRQ signal handler.
* @irq: IRQ number.
@@ -915,6 +976,8 @@ static irqreturn_t snps_dfi_irq_handler(int irq, void *dev_id)
*/
static irqreturn_t snps_com_irq_handler(int irq, void *dev_id)
{
+ struct mem_ctl_info *mci = dev_id;
+ struct snps_edac_priv *priv = mci->pvt_info;
irqreturn_t rc = IRQ_NONE;
rc |= snps_ce_irq_handler(irq, dev_id);
@@ -923,6 +986,9 @@ static irqreturn_t snps_com_irq_handler(int irq, void *dev_id)
rc |= snps_dfi_irq_handler(irq, dev_id);
+ if (priv->info.caps & SNPS_CAP_ECC_SCRUBBER)
+ rc |= snps_sbr_irq_handler(irq, dev_id);
+
return rc;
}
@@ -972,6 +1038,205 @@ static void snps_disable_irq(struct snps_edac_priv *priv)
spin_unlock_irqrestore(&priv->lock, flags);
}
+/**
+ * snps_get_sdram_bw - Get SDRAM bandwidth.
+ * @priv: DDR memory controller private instance data.
+ *
+ * The SDRAM interface bandwidth is calculated based on the DDRC Core clock rate
+ * and the DW uMCTL2 IP-core parameters like DQ-bus width and mode and
+ * Core/SDRAM clocks frequency ratio. Note it returns the theoretical bandwidth
+ * which in reality is hardly possible to reach.
+ *
+ * Return: SDRAM bandwidth or zero if no Core clock specified.
+ */
+static u64 snps_get_sdram_bw(struct snps_edac_priv *priv)
+{
+ unsigned long rate;
+
+ /*
+ * Depending on the ratio mode the SDRAM clock either matches the Core
+ * clock or runs with the twice its frequency.
+ */
+ rate = clk_get_rate(priv->clks[SNPS_CORE_CLK].clk);
+ rate *= priv->info.freq_ratio;
+
+ /*
+ * Scale up by 2 since it's DDR (Double Data Rate) and subtract the
+ * DQ-mode since in non-Full mode only a part of the DQ-bus is utilised
+ * on each SDRAM clock edge.
+ */
+ return (2U << (priv->info.dq_width - priv->info.dq_mode)) * (u64)rate;
+}
+
+/**
+ * snps_get_scrub_bw - Get Scrubber bandwidth.
+ * @priv: DDR memory controller private instance data.
+ * @interval: Scrub interval.
+ *
+ * DW uMCTL2 DDRC Scrubber performs periodical progressive burst reads (RMW if
+ * ECC CE is detected) commands from the whole memory space. The read commands
+ * can be delayed by means of the SBRCTL.scrub_interval field. The Scrubber
+ * cycles look as follows:
+ *
+ * |---HIF-burst-read---|-------delay-------|-HIF-burst-read-| etc
+ *
+ * Tb = Bl*[DQ]/Bw[RAM] Td = 512*interval/Fc - periods of the stages, where
+ * Bl - HIF burst length, [DQ] - Full DQ-bus width, Bw[RAM] - SDRAM bandwidth,
+ * Fc - Core clock frequency (Scrubber and Core clocks are synchronous).
+ *
+ * After some simple calculations the expressions above can be used to get the
+ * next Scrubber bandwidth formulae:
+ *
+ * Bw[Sbr] = Bw[RAM] / (1 + F * interval), where
+ * F = 2 * 512 * Fr * Fc * [DQ]e - interval scale factor with
+ * Fr - HIF/SDRAM clock frequency ratio (1 or 2), [DQ]e - DQ-bus width mode.
+ *
+ * Return: Scrubber bandwidth or zero if no Core clock specified.
+ */
+static u64 snps_get_scrub_bw(struct snps_edac_priv *priv, u32 interval)
+{
+ unsigned long fac;
+ u64 bw_ram;
+
+ fac = (2 * ECC_SBRCTL_INTERVAL_STEP * priv->info.freq_ratio) /
+ (priv->info.hif_burst_len * (1UL << priv->info.dq_mode));
+
+ bw_ram = snps_get_sdram_bw(priv);
+
+ do_div(bw_ram, 1 + fac * interval);
+
+ return bw_ram;
+}
+
+/**
+ * snps_get_scrub_interval - Get Scrubber delay interval.
+ * @priv: DDR memory controller private instance data.
+ * @bw: Scrubber bandwidth.
+ *
+ * Similarly to the Scrubber bandwidth the interval formulae can be inferred
+ * from the same expressions:
+ *
+ * interval = (Bw[RAM] - Bw[Sbr]) / (F * Bw[Sbr])
+ *
+ * Return: Scrubber delay interval or zero if no Core clock specified.
+ */
+static u32 snps_get_scrub_interval(struct snps_edac_priv *priv, u32 bw)
+{
+ unsigned long fac;
+ u64 bw_ram;
+
+ fac = (2 * priv->info.freq_ratio * ECC_SBRCTL_INTERVAL_STEP) /
+ (priv->info.hif_burst_len * (1UL << priv->info.dq_mode));
+
+ bw_ram = snps_get_sdram_bw(priv);
+
+ /* Divide twice so not to cause the integer overflow in (fac * bw) */
+ bw_ram -= bw;
+ do_div(bw_ram, bw);
+ do_div(bw_ram, fac);
+
+ return bw_ram;
+}
+
+/**
+ * snps_set_sdram_scrub_rate - Set the Scrubber bandwidth.
+ * @mci: EDAC memory controller instance.
+ * @bw: Bandwidth.
+ *
+ * It calculates the delay between the Scrubber read commands based on the
+ * specified bandwidth and the Core clock rate. If the Core clock is unavailable
+ * the passed bandwidth will be directly used as the interval value.
+ *
+ * Note the method warns about the back-to-back scrubbing since it may
+ * significantly degrade the system performance. This mode is supposed to be
+ * used for a single SDRAM scrubbing pass only. So it will be turned off in the
+ * Scrubber Done IRQ handler.
+ *
+ * Return: Actually set bandwidth (interval-based approximated bandwidth if the
+ * Core clock is unavailable) or zero if the Scrubber was disabled.
+ */
+static int snps_set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 bw)
+{
+ struct snps_edac_priv *priv = mci->pvt_info;
+ u32 regval, interval;
+ unsigned long flags;
+ u64 bw_min, bw_max;
+
+ /* Don't bother with the calculations just disable and return. */
+ if (!bw) {
+ spin_lock_irqsave(&priv->lock, flags);
+
+ regval = readl(priv->baseaddr + ECC_SBRCTL_OFST);
+ regval &= ~ECC_SBRCTL_SCRUB_EN;
+ writel(regval, priv->baseaddr + ECC_SBRCTL_OFST);
+
+ spin_unlock_irqrestore(&priv->lock, flags);
+
+ return 0;
+ }
+
+ /* If no Core clock specified fallback to the direct interval setup. */
+ bw_max = snps_get_scrub_bw(priv, ECC_SBRCTL_INTERVAL_MIN);
+ if (bw_max) {
+ bw_min = snps_get_scrub_bw(priv, ECC_SBRCTL_INTERVAL_MAX);
+ bw = clamp_t(u64, bw, bw_min, bw_max);
+
+ interval = snps_get_scrub_interval(priv, bw);
+ } else {
+ bw = clamp_val(bw, ECC_SBRCTL_INTERVAL_MIN, ECC_SBRCTL_INTERVAL_MAX);
+
+ interval = ECC_SBRCTL_INTERVAL_MAX - bw;
+ }
+
+ /*
+ * SBRCTL.scrub_en bitfield must be accessed separately from the other
+ * CSR bitfields. It means the flag must be set/clear with no updates
+ * to the rest of the fields.
+ */
+ spin_lock_irqsave(&priv->lock, flags);
+
+ regval = FIELD_PREP(ECC_SBRCTL_SCRUB_INTERVAL, interval);
+ writel(regval, priv->baseaddr + ECC_SBRCTL_OFST);
+
+ writel(regval | ECC_SBRCTL_SCRUB_EN, priv->baseaddr + ECC_SBRCTL_OFST);
+
+ spin_unlock_irqrestore(&priv->lock, flags);
+
+ if (!interval)
+ edac_mc_printk(mci, KERN_WARNING, "Back-to-back scrubbing enabled\n");
+
+ if (!bw_max)
+ return interval ? bw : INT_MAX;
+
+ return snps_get_scrub_bw(priv, interval);
+}
+
+/**
+ * snps_get_sdram_scrub_rate - Get the Scrubber bandwidth.
+ * @mci: EDAC memory controller instance.
+ *
+ * Return: Scrubber bandwidth (interval-based approximated bandwidth if the
+ * Core clock is unavailable) or zero if the Scrubber was disabled.
+ */
+static int snps_get_sdram_scrub_rate(struct mem_ctl_info *mci)
+{
+ struct snps_edac_priv *priv = mci->pvt_info;
+ u32 regval;
+ u64 bw;
+
+ regval = readl(priv->baseaddr + ECC_SBRCTL_OFST);
+ if (!(regval & ECC_SBRCTL_SCRUB_EN))
+ return 0;
+
+ regval = FIELD_GET(ECC_SBRCTL_SCRUB_INTERVAL, regval);
+
+ bw = snps_get_scrub_bw(priv, regval);
+ if (!bw)
+ return regval ? ECC_SBRCTL_INTERVAL_MAX - regval : INT_MAX;
+
+ return bw;
+}
+
/**
* snps_create_data - Create private data.
* @pdev: platform device.
@@ -1038,7 +1303,18 @@ static int snps_get_res(struct snps_edac_priv *priv)
return rc;
}
+ rc = clk_prepare_enable(priv->clks[SNPS_SBR_CLK].clk);
+ if (rc) {
+ edac_printk(KERN_INFO, EDAC_MC, "Couldn't enable Scrubber clock\n");
+ goto err_disable_pclk;
+ }
+
return 0;
+
+err_disable_pclk:
+ clk_disable_unprepare(priv->clks[SNPS_CSR_CLK].clk);
+
+ return rc;
}
/**
@@ -1047,6 +1323,8 @@ static int snps_get_res(struct snps_edac_priv *priv)
*/
static void snps_put_res(struct snps_edac_priv *priv)
{
+ clk_disable_unprepare(priv->clks[SNPS_SBR_CLK].clk);
+
clk_disable_unprepare(priv->clks[SNPS_CSR_CLK].clk);
}
@@ -1147,6 +1425,14 @@ static int snps_get_ddrc_info(struct snps_edac_priv *priv)
if (!(regval & ECC_CFG0_DIS_SCRUB))
priv->info.caps |= SNPS_CAP_ECC_SCRUB;
+ /* Auto-detect the scrubber by writing to the SBRWDATA0 CSR */
+ regval = readl(priv->baseaddr + ECC_SBRWDATA0_OFST);
+ writel(~regval, priv->baseaddr + ECC_SBRWDATA0_OFST);
+ if (regval != readl(priv->baseaddr + ECC_SBRWDATA0_OFST)) {
+ priv->info.caps |= SNPS_CAP_ECC_SCRUBBER;
+ writel(regval, priv->baseaddr + ECC_SBRWDATA0_OFST);
+ }
+
/* Auto-detect the basic HIF/SDRAM bus parameters */
regval = readl(priv->baseaddr + DDR_MSTR_OFST);
@@ -1633,6 +1919,12 @@ static struct mem_ctl_info *snps_mc_create(struct snps_edac_priv *priv)
mci->scrub_cap = SCRUB_FLAG_SW_SRC;
}
+ if (priv->info.caps & SNPS_CAP_ECC_SCRUBBER) {
+ mci->scrub_cap |= SCRUB_FLAG_HW_PROG | SCRUB_FLAG_HW_TUN;
+ mci->set_sdram_scrub_rate = snps_set_sdram_scrub_rate;
+ mci->get_sdram_scrub_rate = snps_get_sdram_scrub_rate;
+ }
+
mci->ctl_name = "snps_umctl2_ddrc";
mci->dev_name = SNPS_EDAC_MOD_STRING;
mci->mod_name = SNPS_EDAC_MOD_VER;
@@ -1707,6 +1999,16 @@ static int snps_request_ind_irq(struct mem_ctl_info *mci)
}
}
+ irq = platform_get_irq_byname_optional(priv->pdev, "ecc_sbr");
+ if (irq > 0) {
+ rc = devm_request_irq(dev, irq, snps_sbr_irq_handler, 0, "ecc_sbr", mci);
+ if (rc) {
+ edac_printk(KERN_ERR, EDAC_MC, "Failed to request Sbr IRQ\n");
+ return rc;
+ }
+ }
+
+
return 0;
}
@@ -1813,6 +2115,8 @@ static int snps_ddrc_info_show(struct seq_file *s, void *data)
if (priv->info.caps) {
if (priv->info.caps & SNPS_CAP_ECC_SCRUB)
seq_puts(s, " +Scrub");
+ if (priv->info.caps & SNPS_CAP_ECC_SCRUBBER)
+ seq_puts(s, " +Scrubber");
if (priv->info.caps & SNPS_CAP_ZYNQMP)
seq_puts(s, " +ZynqMP");
} else {
DW uMCTL2 DDR controller IP-core can by synthesized with an embedded Scrubber engine. The ECC Scrubber (SBR) is a block which initiates periodic background burst read commands to the DDRC and further towards the DDR memory in an attempt to trigger Correctable or Uncorrectable errors. If a Correctable error is detected the ECC Scrub feature will execute the Read-Modify-Write (RMW) procedure in order to fix the ECC. In case of the Uncorrectable error it will be just reported as the corresponding IRQ event. So it's definitely very useful feature. Let's add it to the driver then especially seeing the MCI core already has some infrastructure for it. First of all even though the Core clock rate is only used for the Scrub rate calculations we need to have the Scrubber clock enabled if one is supplied otherwise the engine won't work. Secondly the Scrubber engine support needs to be detected. Alas there is no any special CSR indicating whether the DW uMCTL2 DDRC IP-core has been synthesized with one embedded. Instead we suggest to implement the detection procedure based on the Scrubber-specific CSRs writability. So if the SBRWDATA0 CSR is writable then the CSR exists, which means the Scrubber is available, otherwise the capability will be considered as absent. Thirdly the MCI core provides two callbacks utilized for the Scrubber tuning: set the Scrubber bandwidth in bytes, which can also be used to disable the periodic scrubbing, and get the Scrubber bandwidth (zero if disabled). We can implement both of them by using the Scrubber CSRs the controller provides. In particular aside with the back-to-back periodic reads the Scrubber provides a way to delay the next read command for the predefined set of 512's Core/Scrubber clock cycles. It can be used to change the Scrubber bandwidth from the DDR maximal bandwidth (no delay) to up to (0x1FFF * 512) Core/Scrubber clock cycles (see the inline comments for details and utilized formulae). Note the Scrubber clock must be synchronous to the Core clock by the controller design so here we get to use the Core clock rate for the calculations. Pleas also note if no Core clock specified the Scrubber will still be supported, but the bandwidth will be used directly to calculate the Scrubber reads interval. The back-to-back reads mode in this case will be indicated by the INT_MAX bandwidth. Fourthly the back-to-back scrubbing most likely will cause the significant system performance drop. The manual says that it has been added to the controller for the initial SDRAM initialization and the fast SDRAM scrubbing after getting out of the low-power state. In anyway it is supposed to be enabled only for a single SDRAM pass. We get to preserve that semantic here so the back-to-back scrubbing will be disabled in the Scrubber Done IRQ handler. Finally the denoted scrub-rate callbacks and the SCRUB_FLAG_HW_PROG and SCRUB_FLAG_HW_TUN flags will set to the MCI descriptor based on the detected Scrubber capability. So no capability - no flags and no callbacks. Signed-off-by: Serge Semin <Sergey.Semin@baikalelectronics.ru> --- drivers/edac/synopsys_edac.c | 304 +++++++++++++++++++++++++++++++++++ 1 file changed, 304 insertions(+)