2 * Copyright (C) 2015 Masahiro Yamada <yamada.masahiro@socionext.com>
4 * based on commit 21b6e480f92ccc38fe0502e3116411d6509d3bf2 of Diag by:
5 * Copyright (C) 2015 Socionext Inc.
7 * SPDX-License-Identifier: GPL-2.0+
11 #include <linux/err.h>
13 #include <linux/sizes.h>
14 #include <asm/processor.h>
17 #include "../soc-info.h"
18 #include "ddrmphy-regs.h"
36 static u32 ddrphy_pgcr2[FREQ_NR] = {0x00FC7E5D, 0x00FC90AB};
37 static u32 ddrphy_ptr0[FREQ_NR] = {0x0EA09205, 0x10C0A6C6};
38 static u32 ddrphy_ptr1[FREQ_NR] = {0x0DAC041B, 0x0FA104B1};
39 static u32 ddrphy_ptr3[FREQ_NR] = {0x15171e45, 0x18182357};
40 static u32 ddrphy_ptr4[FREQ_NR] = {0x0e9ad8e9, 0x10b34157};
41 static u32 ddrphy_dtpr0[FREQ_NR] = {0x35a00d88, 0x39e40e88};
42 static u32 ddrphy_dtpr1[FREQ_NR] = {0x2288cc2c, 0x228a04d0};
43 static u32 ddrphy_dtpr2[FREQ_NR] = {0x50005e00, 0x50006a00};
44 static u32 ddrphy_dtpr3[FREQ_NR] = {0x0010cb49, 0x0010ec89};
45 static u32 ddrphy_mr0[FREQ_NR] = {0x00000115, 0x00000125};
46 static u32 ddrphy_mr2[FREQ_NR] = {0x000002a0, 0x000002a8};
48 /* dependent on package and board design */
49 static u32 ddrphy_acbdlr0[CH_NR] = {0x0000000c, 0x0000000c, 0x00000009};
51 static u32 umc_cmdctla[FREQ_NR] = {0x66DD131D, 0x77EE1722};
53 * The ch2 is a different generation UMC core.
54 * The register spec is different, unfortunately.
56 static u32 umc_cmdctlb_ch01[FREQ_NR] = {0x13E87C44, 0x18F88C44};
57 static u32 umc_cmdctlb_ch2[FREQ_NR] = {0x19E8DC44, 0x1EF8EC44};
58 static u32 umc_spcctla[FREQ_NR][SIZE_NR] = {
59 {0x00000000, 0x004A071D, 0x0078071D},
60 {0x00000000, 0x0055081E, 0x0089081E},
63 static u32 umc_spcctlb[] = {0x00FF000A, 0x00FF000B};
64 /* The ch2 is different for some reason only hardware guys know... */
65 static u32 umc_flowctla_ch01[] = {0x0800001E, 0x08000022};
66 static u32 umc_flowctla_ch2[] = {0x0800001E, 0x0800001E};
69 static inline int ddrphy_get_rank(int dx)
74 static void ddrphy_fifo_reset(void __iomem *phy_base)
78 tmp = readl(phy_base + DMPHY_PGCR0);
79 tmp &= ~DMPHY_PGCR0_PHYFRST;
80 writel(tmp, phy_base + DMPHY_PGCR0);
84 tmp |= DMPHY_PGCR0_PHYFRST;
85 writel(tmp, phy_base + DMPHY_PGCR0);
90 static void ddrphy_vt_ctrl(void __iomem *phy_base, int enable)
94 tmp = readl(phy_base + DMPHY_PGCR1);
97 tmp &= ~DMPHY_PGCR1_INHVT;
99 tmp |= DMPHY_PGCR1_INHVT;
101 writel(tmp, phy_base + DMPHY_PGCR1);
104 while (!(readl(phy_base + DMPHY_PGSR1) & DMPHY_PGSR1_VTSTOP))
109 static void ddrphy_dqs_delay_fixup(void __iomem *phy_base, int nr_dx, int step)
113 void __iomem *dx_base = phy_base + DMPHY_DX_BASE;
115 ddrphy_vt_ctrl(phy_base, 0);
117 for (dx = 0; dx < nr_dx; dx++) {
118 lcdlr1 = readl(dx_base + DMPHY_DX_LCDLR1);
119 rdqsd = (lcdlr1 >> 8) & 0xff;
120 rdqsd = clamp(rdqsd + step, 0U, 0xffU);
121 lcdlr1 = (lcdlr1 & ~(0xff << 8)) | (rdqsd << 8);
122 writel(lcdlr1, dx_base + DMPHY_DX_LCDLR1);
123 readl(dx_base + DMPHY_DX_LCDLR1); /* relax */
124 dx_base += DMPHY_DX_STRIDE;
127 ddrphy_vt_ctrl(phy_base, 1);
130 static int ddrphy_get_system_latency(void __iomem *phy_base, int width)
132 void __iomem *dx_base = phy_base + DMPHY_DX_BASE;
133 const int nr_dx = width / 8;
136 int dgsl, dgsl_min = INT_MAX, dgsl_max = 0;
138 for (dx = 0; dx < nr_dx; dx++) {
139 gtr = readl(dx_base + DMPHY_DX_GTR);
140 for (rank = 0; rank < 4; rank++) {
142 /* if dgsl is zero, this rank was not trained. skip. */
144 dgsl_min = min(dgsl_min, dgsl);
145 dgsl_max = max(dgsl_max, dgsl);
149 dx_base += DMPHY_DX_STRIDE;
152 if (dgsl_min != dgsl_max)
153 printf("DQS Gateing System Latencies are not all leveled.\n");
158 static void ddrphy_init(void __iomem *phy_base, enum dram_freq freq, int width,
162 void __iomem *zq_base, *dx_base;
168 writel(DMPHY_PIR_ZCALBYP, phy_base + DMPHY_PIR);
170 * Disable RGLVT bit (Read DQS Gating LCDL Delay VT Compensation)
171 * to avoid read error issue.
173 writel(0x07d81e37, phy_base + DMPHY_PGCR0);
174 writel(0x0200c4e0, phy_base + DMPHY_PGCR1);
176 tmp = ddrphy_pgcr2[freq];
178 tmp |= DMPHY_PGCR2_DUALCHN | DMPHY_PGCR2_ACPDDC;
179 writel(tmp, phy_base + DMPHY_PGCR2);
181 writel(ddrphy_ptr0[freq], phy_base + DMPHY_PTR0);
182 writel(ddrphy_ptr1[freq], phy_base + DMPHY_PTR1);
183 writel(0x00083def, phy_base + DMPHY_PTR2);
184 writel(ddrphy_ptr3[freq], phy_base + DMPHY_PTR3);
185 writel(ddrphy_ptr4[freq], phy_base + DMPHY_PTR4);
187 writel(ddrphy_acbdlr0[ch], phy_base + DMPHY_ACBDLR0);
189 writel(0x55555555, phy_base + DMPHY_ACIOCR1);
190 writel(0x00000000, phy_base + DMPHY_ACIOCR2);
191 writel(0x55555555, phy_base + DMPHY_ACIOCR3);
192 writel(0x00000000, phy_base + DMPHY_ACIOCR4);
193 writel(0x00000055, phy_base + DMPHY_ACIOCR5);
194 writel(0x00181aa4, phy_base + DMPHY_DXCCR);
196 writel(0x0024641e, phy_base + DMPHY_DSGCR);
197 writel(0x0000040b, phy_base + DMPHY_DCR);
198 writel(ddrphy_dtpr0[freq], phy_base + DMPHY_DTPR0);
199 writel(ddrphy_dtpr1[freq], phy_base + DMPHY_DTPR1);
200 writel(ddrphy_dtpr2[freq], phy_base + DMPHY_DTPR2);
201 writel(ddrphy_dtpr3[freq], phy_base + DMPHY_DTPR3);
202 writel(ddrphy_mr0[freq], phy_base + DMPHY_MR0);
203 writel(0x00000006, phy_base + DMPHY_MR1);
204 writel(ddrphy_mr2[freq], phy_base + DMPHY_MR2);
205 writel(0x00000000, phy_base + DMPHY_MR3);
208 for (dx = 0; dx < nr_dx; dx++)
209 tmp |= BIT(DMPHY_DTCR_RANKEN_SHIFT + ddrphy_get_rank(dx));
210 writel(0x90003087 | tmp, phy_base + DMPHY_DTCR);
212 writel(0x00000000, phy_base + DMPHY_DTAR0);
213 writel(0x00000008, phy_base + DMPHY_DTAR1);
214 writel(0x00000010, phy_base + DMPHY_DTAR2);
215 writel(0x00000018, phy_base + DMPHY_DTAR3);
216 writel(0xdd22ee11, phy_base + DMPHY_DTDR0);
217 writel(0x7788bb44, phy_base + DMPHY_DTDR1);
219 /* impedance control settings */
220 writel(0x04048900, phy_base + DMPHY_ZQCR);
222 zq_base = phy_base + DMPHY_ZQ_BASE;
223 for (zq = 0; zq < 4; zq++) {
226 * PXS2: CH0ZQ0=0x5B, CH1ZQ0=0x5B, CH2ZQ0=0x59, others=0x5D
228 writel(0x0007BB5D, zq_base + DMPHY_ZQ_PR);
229 zq_base += DMPHY_ZQ_STRIDE;
233 dx_base = phy_base + DMPHY_DX_BASE;
234 for (dx = 0; dx < 4; dx++) {
235 tmp = readl(dx_base + DMPHY_DX_GCR0);
236 tmp &= ~DMPHY_DX_GCR0_WLRKEN_MASK;
237 tmp |= BIT(DMPHY_DX_GCR0_WLRKEN_SHIFT + ddrphy_get_rank(dx)) &
238 DMPHY_DX_GCR0_WLRKEN_MASK;
239 writel(tmp, dx_base + DMPHY_DX_GCR0);
241 writel(0x00000000, dx_base + DMPHY_DX_GCR1);
242 writel(0x00000000, dx_base + DMPHY_DX_GCR2);
243 writel(0x00000000, dx_base + DMPHY_DX_GCR3);
244 dx_base += DMPHY_DX_STRIDE;
247 while (!(readl(phy_base + DMPHY_PGSR0) & DMPHY_PGSR0_IDONE))
250 ddrphy_dqs_delay_fixup(phy_base, nr_dx, -4);
253 struct ddrphy_init_sequence {
260 static const struct ddrphy_init_sequence impedance_calibration_sequence[] = {
262 "Impedance Calibration",
270 static const struct ddrphy_init_sequence dram_init_sequence[] = {
272 "DRAM Initialization",
273 DMPHY_PIR_DRAMRST | DMPHY_PIR_DRAMINIT,
280 static const struct ddrphy_init_sequence training_sequence[] = {
288 "Read DQS Gate Training",
294 "Write Leveling Adjustment",
318 "Write Eye Training",
326 static int __ddrphy_training(void __iomem *phy_base,
327 const struct ddrphy_init_sequence *seq)
329 const struct ddrphy_init_sequence *s;
331 u32 init_flag = DMPHY_PIR_INIT;
332 u32 done_flag = DMPHY_PGSR0_IDONE;
333 int timeout = 50000; /* 50 msec is long enough */
334 #ifdef DISPLAY_ELAPSED_TIME
335 ulong start = get_timer(0);
338 for (s = seq; s->description; s++) {
339 init_flag |= s->init_flag;
340 done_flag |= s->done_flag;
343 writel(init_flag, phy_base + DMPHY_PIR);
347 printf("%s: error: timeout during DDR training\n",
352 pgsr0 = readl(phy_base + DMPHY_PGSR0);
353 } while ((pgsr0 & done_flag) != done_flag);
355 for (s = seq; s->description; s++) {
356 if (pgsr0 & s->err_flag) {
357 printf("%s: error: %s failed\n", __func__,
363 #ifdef DISPLAY_ELAPSED_TIME
364 printf("%s: info: elapsed time %ld msec\n", get_timer(start));
370 static int ddrphy_impedance_calibration(void __iomem *phy_base)
375 ret = __ddrphy_training(phy_base, impedance_calibration_sequence);
380 * Because of a hardware bug, IDONE flag is set when the first ZQ block
381 * is calibrated. The flag does not guarantee the completion for all
382 * the ZQ blocks. Wait a little more just in case.
386 /* reflect ZQ settings and enable average algorithm*/
387 tmp = readl(phy_base + DMPHY_ZQCR);
388 tmp |= DMPHY_ZQCR_FORCE_ZCAL_VT_UPDATE;
389 writel(tmp, phy_base + DMPHY_ZQCR);
390 tmp &= ~DMPHY_ZQCR_FORCE_ZCAL_VT_UPDATE;
391 tmp |= DMPHY_ZQCR_AVGEN;
392 writel(tmp, phy_base + DMPHY_ZQCR);
397 static int ddrphy_dram_init(void __iomem *phy_base)
399 return __ddrphy_training(phy_base, dram_init_sequence);
402 static int ddrphy_training(void __iomem *phy_base)
404 return __ddrphy_training(phy_base, training_sequence);
408 static void umc_set_system_latency(void __iomem *umc_dc_base, int phy_latency)
413 val = readl(umc_dc_base + UMC_RDATACTL_D0);
414 latency = (val & UMC_RDATACTL_RADLTY_MASK) >> UMC_RDATACTL_RADLTY_SHIFT;
415 latency += (val & UMC_RDATACTL_RAD2LTY_MASK) >>
416 UMC_RDATACTL_RAD2LTY_SHIFT;
418 * UMC works at the half clock rate of the PHY.
419 * The LSB of latency is ignored
421 latency += phy_latency & ~1;
423 val &= ~(UMC_RDATACTL_RADLTY_MASK | UMC_RDATACTL_RAD2LTY_MASK);
425 val |= 0xf << UMC_RDATACTL_RADLTY_SHIFT;
426 val |= (latency - 0xf) << UMC_RDATACTL_RAD2LTY_SHIFT;
428 val |= latency << UMC_RDATACTL_RADLTY_SHIFT;
431 writel(val, umc_dc_base + UMC_RDATACTL_D0);
432 writel(val, umc_dc_base + UMC_RDATACTL_D1);
434 readl(umc_dc_base + UMC_RDATACTL_D1); /* relax */
437 /* enable/disable auto refresh */
438 void umc_refresh_ctrl(void __iomem *umc_dc_base, int enable)
442 tmp = readl(umc_dc_base + UMC_SPCSETB);
443 tmp &= ~UMC_SPCSETB_AREFMD_MASK;
446 tmp |= UMC_SPCSETB_AREFMD_ARB;
448 tmp |= UMC_SPCSETB_AREFMD_REG;
450 writel(tmp, umc_dc_base + UMC_SPCSETB);
454 static void umc_ud_init(void __iomem *umc_base, int ch)
456 writel(0x00000003, umc_base + UMC_BITPERPIXELMODE_D0);
459 writel(0x00000033, umc_base + UMC_PAIR1DOFF_D0);
462 static void umc_dc_init(void __iomem *umc_dc_base, enum dram_freq freq,
463 enum dram_size size, int ch, int width)
468 writel(umc_cmdctla[freq], umc_dc_base + UMC_CMDCTLA);
470 writel(ch == 2 ? umc_cmdctlb_ch2[freq] : umc_cmdctlb_ch01[freq],
471 umc_dc_base + UMC_CMDCTLB);
473 writel(umc_spcctla[freq][size / (width / 16)],
474 umc_dc_base + UMC_SPCCTLA);
475 writel(umc_spcctlb[freq], umc_dc_base + UMC_SPCCTLB);
479 /* ES2 inserted one more FF to the logic. */
480 if (uniphier_get_soc_model() >= 2)
484 val |= 0xf << UMC_RDATACTL_RADLTY_SHIFT;
485 val |= (latency - 0xf) << UMC_RDATACTL_RAD2LTY_SHIFT;
487 val |= latency << UMC_RDATACTL_RADLTY_SHIFT;
490 writel(val, umc_dc_base + UMC_RDATACTL_D0);
492 writel(val, umc_dc_base + UMC_RDATACTL_D1);
494 writel(0x04060A02, umc_dc_base + UMC_WDATACTL_D0);
496 writel(0x04060A02, umc_dc_base + UMC_WDATACTL_D1);
497 writel(0x04000000, umc_dc_base + UMC_DATASET);
498 writel(0x00400020, umc_dc_base + UMC_DCCGCTL);
499 writel(0x00000084, umc_dc_base + UMC_FLOWCTLG);
500 writel(0x00000000, umc_dc_base + UMC_ACSSETA);
502 writel(ch == 2 ? umc_flowctla_ch2[freq] : umc_flowctla_ch01[freq],
503 umc_dc_base + UMC_FLOWCTLA);
505 writel(0x00004400, umc_dc_base + UMC_FLOWCTLC);
506 writel(0x200A0A00, umc_dc_base + UMC_SPCSETB);
507 writel(0x00000520, umc_dc_base + UMC_DFICUPDCTLA);
508 writel(0x0000000D, umc_dc_base + UMC_RESPCTL);
511 writel(0x00202000, umc_dc_base + UMC_FLOWCTLB);
512 writel(0xFDBFFFFF, umc_dc_base + UMC_FLOWCTLOB0);
513 writel(0xFFFFFFFF, umc_dc_base + UMC_FLOWCTLOB1);
514 writel(0x00080700, umc_dc_base + UMC_BSICMAPSET);
516 writel(0x00200000, umc_dc_base + UMC_FLOWCTLB);
517 writel(0x00000000, umc_dc_base + UMC_BSICMAPSET);
520 writel(0x00000000, umc_dc_base + UMC_ERRMASKA);
521 writel(0x00000000, umc_dc_base + UMC_ERRMASKB);
524 static int umc_init(void __iomem *umc_base, enum dram_freq freq, int ch,
525 enum dram_size size, int width)
527 void __iomem *umc_dc_base = umc_base + 0x00011000;
528 void __iomem *phy_base = umc_base + 0x00030000;
531 writel(0x00000002, umc_dc_base + UMC_INITSET);
532 while (readl(umc_dc_base + UMC_INITSTAT) & BIT(2))
535 /* deassert PHY reset signals */
536 writel(UMC_DIOCTLA_CTL_NRST | UMC_DIOCTLA_CFG_NRST,
537 umc_dc_base + UMC_DIOCTLA);
539 ddrphy_init(phy_base, freq, width, ch);
541 ret = ddrphy_impedance_calibration(phy_base);
545 ddrphy_dram_init(phy_base);
549 umc_dc_init(umc_dc_base, freq, size, ch, width);
551 umc_ud_init(umc_base, ch);
554 ret = ddrphy_training(phy_base);
561 /* match the system latency between UMC and PHY */
562 umc_set_system_latency(umc_dc_base,
563 ddrphy_get_system_latency(phy_base, width));
567 /* stop auto refresh before clearing FIFO in PHY */
568 umc_refresh_ctrl(umc_dc_base, 0);
569 ddrphy_fifo_reset(phy_base);
570 umc_refresh_ctrl(umc_dc_base, 1);
577 static void um_init(void __iomem *um_base)
579 writel(0x000000ff, um_base + UMC_MBUS0);
580 writel(0x000000ff, um_base + UMC_MBUS1);
581 writel(0x000000ff, um_base + UMC_MBUS2);
582 writel(0x000000ff, um_base + UMC_MBUS3);
585 int proxstream2_umc_init(const struct uniphier_board_data *bd)
587 void __iomem *um_base = (void __iomem *)0x5b600000;
588 void __iomem *umc_ch0_base = (void __iomem *)0x5b800000;
589 void __iomem *umc_ch1_base = (void __iomem *)0x5ba00000;
590 void __iomem *umc_ch2_base = (void __iomem *)0x5bc00000;
594 switch (bd->dram_freq) {
602 printf("unsupported DRAM frequency %d MHz\n", bd->dram_freq);
606 ret = umc_init(umc_ch0_base, freq, 0, bd->dram_ch0_size / SZ_256M,
609 printf("failed to initialize UMC ch0\n");
613 ret = umc_init(umc_ch1_base, freq, 1, bd->dram_ch1_size / SZ_256M,
616 printf("failed to initialize UMC ch1\n");
620 ret = umc_init(umc_ch2_base, freq, 2, bd->dram_ch2_size / SZ_256M,
623 printf("failed to initialize UMC ch2\n");