2 * Copyright (C) 2013, Intel Corporation
3 * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com>
5 * Ported from Intel released Quark UEFI BIOS
6 * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei
8 * SPDX-License-Identifier: Intel
13 #include <asm/arch/device.h>
14 #include <asm/arch/mrc.h>
15 #include <asm/arch/msg_port.h>
20 /* t_rfc values (in picoseconds) per density */
21 static const uint32_t t_rfc[5] = {
29 /* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */
30 static const uint32_t t_ck[3] = {
36 /* Global variables */
37 static const uint16_t ddr_wclk[] = {193, 158};
38 static const uint16_t ddr_wctl[] = {1, 217};
39 static const uint16_t ddr_wcmd[] = {1, 220};
42 static const uint16_t ddr_rcvn[] = {129, 498};
46 static const uint16_t ddr_wdqs[] = {65, 289};
50 static const uint8_t ddr_rdqs[] = {32, 24};
54 static const uint16_t ddr_wdq[] = {32, 257};
57 /* Stop self refresh driven by MCU */
58 void clear_self_refresh(struct mrc_params *mrc_params)
62 /* clear the PMSTS Channel Self Refresh bits */
63 mrc_write_mask(MEM_CTLR, PMSTS, PMSTS_DISR, PMSTS_DISR);
68 /* It will initialize timing registers in the MCU (DTR0..DTR4) */
69 void prog_ddr_timing_control(struct mrc_params *mrc_params)
72 uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw;
74 u32 dtr0, dtr1, dtr2, dtr3, dtr4;
80 mrc_post_code(0x02, 0x00);
82 dtr0 = msg_port_read(MEM_CTLR, DTR0);
83 dtr1 = msg_port_read(MEM_CTLR, DTR1);
84 dtr2 = msg_port_read(MEM_CTLR, DTR2);
85 dtr3 = msg_port_read(MEM_CTLR, DTR3);
86 dtr4 = msg_port_read(MEM_CTLR, DTR4);
88 tck = t_ck[mrc_params->ddr_speed]; /* Clock in picoseconds */
89 tcl = mrc_params->params.cl; /* CAS latency in clocks */
90 trp = tcl; /* Per CAT MRC */
91 trcd = tcl; /* Per CAT MRC */
92 tras = MCEIL(mrc_params->params.ras, tck);
94 /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
95 twr = MCEIL(15000, tck);
97 twtr = MCEIL(mrc_params->params.wtr, tck);
98 trrd = MCEIL(mrc_params->params.rrd, tck);
99 trtp = 4; /* Valid for 800 and 1066, use 5 for 1333 */
100 tfaw = MCEIL(mrc_params->params.faw, tck);
102 wl = 5 + mrc_params->ddr_speed;
104 dtr0 &= ~DTR0_DFREQ_MASK;
105 dtr0 |= mrc_params->ddr_speed;
106 dtr0 &= ~DTR0_TCL_MASK;
108 dtr0 |= ((tcl - 5) << 12);
109 dtr0 &= ~DTR0_TRP_MASK;
110 dtr0 |= ((trp - 5) << 4); /* 5 bit DRAM Clock */
111 dtr0 &= ~DTR0_TRCD_MASK;
112 dtr0 |= ((trcd - 5) << 8); /* 5 bit DRAM Clock */
114 dtr1 &= ~DTR1_TWCL_MASK;
117 dtr1 &= ~DTR1_TWTP_MASK;
118 dtr1 |= ((wl + 4 + twr - 14) << 8); /* Change to tWTP */
119 dtr1 &= ~DTR1_TRTP_MASK;
120 dtr1 |= ((MMAX(trtp, 4) - 3) << 28); /* 4 bit DRAM Clock */
121 dtr1 &= ~DTR1_TRRD_MASK;
122 dtr1 |= ((trrd - 4) << 24); /* 4 bit DRAM Clock */
123 dtr1 &= ~DTR1_TCMD_MASK;
125 dtr1 &= ~DTR1_TRAS_MASK;
126 dtr1 |= ((tras - 14) << 20); /* 6 bit DRAM Clock */
127 dtr1 &= ~DTR1_TFAW_MASK;
128 dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */
129 /* Set 4 Clock CAS to CAS delay (multi-burst) */
130 dtr1 &= ~DTR1_TCCD_MASK;
132 dtr2 &= ~DTR2_TRRDR_MASK;
134 dtr2 &= ~DTR2_TWWDR_MASK;
136 dtr2 &= ~DTR2_TRWDR_MASK;
139 dtr3 &= ~DTR3_TWRDR_MASK;
141 dtr3 &= ~DTR3_TXXXX_MASK;
144 dtr3 &= ~DTR3_TRWSR_MASK;
145 if (mrc_params->ddr_speed == DDRFREQ_800) {
146 /* Extended RW delay (+1) */
147 dtr3 |= ((tcl - 5 + 1) << 8);
148 } else if (mrc_params->ddr_speed == DDRFREQ_1066) {
149 /* Extended RW delay (+1) */
150 dtr3 |= ((tcl - 5 + 1) << 8);
153 dtr3 &= ~DTR3_TWRSR_MASK;
154 dtr3 |= ((4 + wl + twtr - 11) << 13);
156 dtr3 &= ~DTR3_TXP_MASK;
157 if (mrc_params->ddr_speed == DDRFREQ_800)
158 dtr3 |= ((MMAX(0, 1 - 1)) << 22);
160 dtr3 |= ((MMAX(0, 2 - 1)) << 22);
162 dtr4 &= ~DTR4_WRODTSTRT_MASK;
164 dtr4 &= ~DTR4_WRODTSTOP_MASK;
166 dtr4 &= ~DTR4_XXXX1_MASK;
167 dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8);
168 dtr4 &= ~DTR4_XXXX2_MASK;
169 dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12);
170 dtr4 &= ~(DTR4_ODTDIS | DTR4_TRGSTRDIS);
172 msg_port_write(MEM_CTLR, DTR0, dtr0);
173 msg_port_write(MEM_CTLR, DTR1, dtr1);
174 msg_port_write(MEM_CTLR, DTR2, dtr2);
175 msg_port_write(MEM_CTLR, DTR3, dtr3);
176 msg_port_write(MEM_CTLR, DTR4, dtr4);
181 /* Configure MCU before jedec init sequence */
182 void prog_decode_before_jedec(struct mrc_params *mrc_params)
192 /* Disable power saving features */
193 dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
194 dpmc0 |= (DPMC0_CLKGTDIS | DPMC0_DISPWRDN);
195 dpmc0 &= ~DPMC0_PCLSTO_MASK;
196 dpmc0 &= ~DPMC0_DYNSREN;
197 msg_port_write(MEM_CTLR, DPMC0, dpmc0);
199 /* Disable out of order transactions */
200 dsch = msg_port_read(MEM_CTLR, DSCH);
201 dsch |= (DSCH_OOODIS | DSCH_NEWBYPDIS);
202 msg_port_write(MEM_CTLR, DSCH, dsch);
204 /* Disable issuing the REF command */
205 drfc = msg_port_read(MEM_CTLR, DRFC);
206 drfc &= ~DRFC_TREFI_MASK;
207 msg_port_write(MEM_CTLR, DRFC, drfc);
209 /* Disable ZQ calibration short */
210 dcal = msg_port_read(MEM_CTLR, DCAL);
211 dcal &= ~DCAL_ZQCINT_MASK;
212 dcal &= ~DCAL_SRXZQCL_MASK;
213 msg_port_write(MEM_CTLR, DCAL, dcal);
216 * Training performed in address mode 0, rank population has limited
217 * impact, however simulator complains if enabled non-existing rank.
220 if (mrc_params->rank_enables & 1)
222 if (mrc_params->rank_enables & 2)
224 msg_port_write(MEM_CTLR, DRP, drp);
230 * After Cold Reset, BIOS should set COLDWAKE bit to 1 before
231 * sending the WAKE message to the Dunit.
233 * For Standby Exit, or any other mode in which the DRAM is in
234 * SR, this bit must be set to 0.
236 void perform_ddr_reset(struct mrc_params *mrc_params)
240 /* Set COLDWAKE bit before sending the WAKE message */
241 mrc_write_mask(MEM_CTLR, DRMC, DRMC_COLDWAKE, DRMC_COLDWAKE);
243 /* Send wake command to DUNIT (MUST be done before JEDEC) */
246 /* Set default value */
247 msg_port_write(MEM_CTLR, DRMC,
248 mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0);
255 * This function performs some initialization on the DDRIO unit.
256 * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES.
258 void ddrphy_init(struct mrc_params *mrc_params)
261 uint8_t ch; /* channel counter */
262 uint8_t rk; /* rank counter */
263 uint8_t bl_grp; /* byte lane group counter (2 BLs per module) */
264 uint8_t bl_divisor = 1; /* byte lane divisor */
265 /* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */
266 uint8_t speed = mrc_params->ddr_speed & 3;
272 cas = mrc_params->params.cl;
273 cwl = 5 + mrc_params->ddr_speed;
275 /* ddrphy_init starts */
276 mrc_post_code(0x03, 0x00);
280 * Make sure IOBUFACT is deasserted before initializing the DDR PHY
283 * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY
285 for (ch = 0; ch < NUM_CHANNELS; ch++) {
286 if (mrc_params->channel_enables & (1 << ch)) {
287 /* Deassert DDRPHY Initialization Complete */
288 mrc_alt_write_mask(DDRPHY,
289 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
290 ~(1 << 20), 1 << 20); /* SPID_INIT_COMPLETE=0 */
291 /* Deassert IOBUFACT */
292 mrc_alt_write_mask(DDRPHY,
293 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
294 ~(1 << 2), 1 << 2); /* IOBUFACTRST_N=0 */
296 mrc_alt_write_mask(DDRPHY,
297 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
298 ~(1 << 0), 1 << 0); /* WRPTRENABLE=0 */
302 /* Put PHY in reset */
303 mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, 1);
305 /* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */
308 mrc_post_code(0x03, 0x10);
309 for (ch = 0; ch < NUM_CHANNELS; ch++) {
310 if (mrc_params->channel_enables & (1 << ch)) {
313 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
315 /* Analog MUX select - IO2xCLKSEL */
316 mrc_alt_write_mask(DDRPHY,
318 bl_grp * DDRIODQ_BL_OFFSET +
319 ch * DDRIODQ_CH_OFFSET,
320 bl_grp ? 0 : (1 << 22), 1 << 22);
323 switch (mrc_params->rd_odt_value) {
339 mrc_alt_write_mask(DDRPHY,
341 bl_grp * DDRIODQ_BL_OFFSET +
342 ch * DDRIODQ_CH_OFFSET,
345 mrc_alt_write_mask(DDRPHY,
347 bl_grp * DDRIODQ_BL_OFFSET +
348 ch * DDRIODQ_CH_OFFSET,
351 /* Dynamic ODT/DIFFAMP */
352 temp = (cas << 24) | (cas << 16) |
353 (cas << 8) | (cas << 0);
369 /* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */
370 mrc_alt_write_mask(DDRPHY,
372 bl_grp * DDRIODQ_BL_OFFSET +
373 ch * DDRIODQ_CH_OFFSET,
378 temp = (0x06 << 16) | (0x07 << 8);
381 temp = (0x07 << 16) | (0x08 << 8);
384 temp = (0x09 << 16) | (0x0a << 8);
387 temp = (0x0a << 16) | (0x0b << 8);
391 /* On Duration: ODT, DIFFAMP */
392 mrc_alt_write_mask(DDRPHY,
394 bl_grp * DDRIODQ_BL_OFFSET +
395 ch * DDRIODQ_CH_OFFSET,
397 /* On Duration: ODT, DIFFAMP */
398 mrc_alt_write_mask(DDRPHY,
400 bl_grp * DDRIODQ_BL_OFFSET +
401 ch * DDRIODQ_CH_OFFSET,
404 switch (mrc_params->rd_odt_value) {
406 /* override DIFFAMP=on, ODT=off */
407 temp = (0x3f << 16) | (0x3f << 10);
410 /* override DIFFAMP=on, ODT=on */
411 temp = (0x3f << 16) | (0x2a << 10);
415 /* Override: DIFFAMP, ODT */
416 mrc_alt_write_mask(DDRPHY,
418 bl_grp * DDRIODQ_BL_OFFSET +
419 ch * DDRIODQ_CH_OFFSET,
421 /* Override: DIFFAMP, ODT */
422 mrc_alt_write_mask(DDRPHY,
424 bl_grp * DDRIODQ_BL_OFFSET +
425 ch * DDRIODQ_CH_OFFSET,
430 /* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */
431 mrc_alt_write_mask(DDRPHY,
433 bl_grp * DDRIODQ_BL_OFFSET +
434 ch * DDRIODQ_CH_OFFSET,
435 ((cas + 7) << 16) | ((cas - 4) << 8) |
436 ((cwl - 2) << 0), 0x003f1f1f);
437 mrc_alt_write_mask(DDRPHY,
439 bl_grp * DDRIODQ_BL_OFFSET +
440 ch * DDRIODQ_CH_OFFSET,
441 ((cas + 7) << 16) | ((cas - 4) << 8) |
442 ((cwl - 2) << 0), 0x003f1f1f);
444 /* RCVEN Bypass (PO) */
445 mrc_alt_write_mask(DDRPHY,
447 bl_grp * DDRIODQ_BL_OFFSET +
448 ch * DDRIODQ_CH_OFFSET,
450 mrc_alt_write_mask(DDRPHY,
452 bl_grp * DDRIODQ_BL_OFFSET +
453 ch * DDRIODQ_CH_OFFSET,
457 mrc_alt_write_mask(DDRPHY,
459 bl_grp * DDRIODQ_BL_OFFSET +
460 ch * DDRIODQ_CH_OFFSET,
462 mrc_alt_write_mask(DDRPHY,
464 bl_grp * DDRIODQ_BL_OFFSET +
465 ch * DDRIODQ_CH_OFFSET,
469 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
470 mrc_alt_write_mask(DDRPHY,
472 bl_grp * DDRIODQ_BL_OFFSET +
473 ch * DDRIODQ_CH_OFFSET,
474 (0x03 << 2) | (0x0 << 1) | (0x0 << 0),
476 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
477 mrc_alt_write_mask(DDRPHY,
479 bl_grp * DDRIODQ_BL_OFFSET +
480 ch * DDRIODQ_CH_OFFSET,
481 (0x03 << 2) | (0x0 << 1) | (0x0 << 0),
483 /* Per-Bit De-Skew Enable */
484 mrc_alt_write_mask(DDRPHY,
486 bl_grp * DDRIODQ_BL_OFFSET +
487 ch * DDRIODQ_CH_OFFSET,
489 /* Per-Bit De-Skew Enable */
490 mrc_alt_write_mask(DDRPHY,
492 bl_grp * DDRIODQ_BL_OFFSET +
493 ch * DDRIODQ_CH_OFFSET,
498 mrc_alt_write_mask(DDRPHY,
499 CMDOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
502 /* Enable tristate control of cmd/address bus */
503 mrc_alt_write_mask(DDRPHY,
504 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
508 mrc_alt_write_mask(DDRPHY,
509 CMDRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
510 (0x03 << 5) | (0x03 << 0), 0x3ff);
512 /* CMDPM* registers must be programmed in this order */
514 /* Turn On Delays: SFR (regulator), MPLL */
515 mrc_alt_write_mask(DDRPHY,
516 CMDPMDLYREG4 + ch * DDRIOCCC_CH_OFFSET,
517 0xffffffff, 0xffffffff);
519 * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3,
520 * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT
521 * for_PM_MSG_gt0, MDLL Turn On
523 mrc_alt_write_mask(DDRPHY,
524 CMDPMDLYREG3 + ch * DDRIOCCC_CH_OFFSET,
525 0xfffff616, 0xffffffff);
526 /* MPLL Divider Reset Delays */
527 mrc_alt_write_mask(DDRPHY,
528 CMDPMDLYREG2 + ch * DDRIOCCC_CH_OFFSET,
529 0xffffffff, 0xffffffff);
530 /* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */
531 mrc_alt_write_mask(DDRPHY,
532 CMDPMDLYREG1 + ch * DDRIOCCC_CH_OFFSET,
533 0xffffffff, 0xffffffff);
534 /* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */
535 mrc_alt_write_mask(DDRPHY,
536 CMDPMDLYREG0 + ch * DDRIOCCC_CH_OFFSET,
537 0xffffffff, 0xffffffff);
538 /* Allow PUnit signals */
539 mrc_alt_write_mask(DDRPHY,
540 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
541 (0x6 << 8) | (0x1 << 6) | (0x4 << 0),
543 /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
544 mrc_alt_write_mask(DDRPHY,
545 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
546 (0x3 << 4) | (0x7 << 0), 0x7f);
549 mrc_alt_write_mask(DDRPHY,
550 CCOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
551 0, 1 << 24); /* CLKEBB */
552 /* Buffer Enable: CS,CKE,ODT,CLK */
553 mrc_alt_write_mask(DDRPHY,
554 CCCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
557 mrc_alt_write_mask(DDRPHY,
558 CCRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
559 (0x03 << 8) | (0x03 << 0), 0x00001f1f);
560 /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
561 mrc_alt_write_mask(DDRPHY,
562 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
563 (0x3 << 4) | (0x7 << 0), 0x7f);
566 * COMP (RON channel specific)
567 * - DQ/DQS/DM RON: 32 Ohm
568 * - CTRL/CMD RON: 27 Ohm
571 /* RCOMP Vref PU/PD */
572 mrc_alt_write_mask(DDRPHY,
573 DQVREFCH0 + ch * DDRCOMP_CH_OFFSET,
574 (0x08 << 24) | (0x03 << 16), 0x3f3f0000);
575 /* RCOMP Vref PU/PD */
576 mrc_alt_write_mask(DDRPHY,
577 CMDVREFCH0 + ch * DDRCOMP_CH_OFFSET,
578 (0x0C << 24) | (0x03 << 16), 0x3f3f0000);
579 /* RCOMP Vref PU/PD */
580 mrc_alt_write_mask(DDRPHY,
581 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
582 (0x0F << 24) | (0x03 << 16), 0x3f3f0000);
583 /* RCOMP Vref PU/PD */
584 mrc_alt_write_mask(DDRPHY,
585 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
586 (0x08 << 24) | (0x03 << 16), 0x3f3f0000);
587 /* RCOMP Vref PU/PD */
588 mrc_alt_write_mask(DDRPHY,
589 CTLVREFCH0 + ch * DDRCOMP_CH_OFFSET,
590 (0x0C << 24) | (0x03 << 16), 0x3f3f0000);
592 /* DQS Swapped Input Enable */
593 mrc_alt_write_mask(DDRPHY,
594 COMPEN1CH0 + ch * DDRCOMP_CH_OFFSET,
595 (1 << 19) | (1 << 17), 0xc00ac000);
597 /* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */
599 mrc_alt_write_mask(DDRPHY,
600 DQVREFCH0 + ch * DDRCOMP_CH_OFFSET,
601 (0x32 << 8) | (0x03 << 0), 0x00003f3f);
603 mrc_alt_write_mask(DDRPHY,
604 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
605 (0x32 << 8) | (0x03 << 0), 0x00003f3f);
607 mrc_alt_write_mask(DDRPHY,
608 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
609 (0x0E << 8) | (0x05 << 0), 0x00003f3f);
612 * Slew rate settings are frequency specific,
613 * numbers below are for 800Mhz (speed == 0)
614 * - DQ/DQS/DM/CLK SR: 4V/ns,
615 * - CTRL/CMD SR: 1.5V/ns
617 temp = (0x0e << 16) | (0x0e << 12) | (0x08 << 8) |
618 (0x0b << 4) | (0x0b << 0);
619 /* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */
620 mrc_alt_write_mask(DDRPHY,
621 DLYSELCH0 + ch * DDRCOMP_CH_OFFSET,
623 /* TCO Vref CLK,DQS,DQ */
624 mrc_alt_write_mask(DDRPHY,
625 TCOVREFCH0 + ch * DDRCOMP_CH_OFFSET,
626 (0x05 << 16) | (0x05 << 8) | (0x05 << 0),
628 /* ODTCOMP CMD/CTL PU/PD */
629 mrc_alt_write_mask(DDRPHY,
630 CCBUFODTCH0 + ch * DDRCOMP_CH_OFFSET,
631 (0x03 << 8) | (0x03 << 0),
634 mrc_alt_write_mask(DDRPHY,
635 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
639 /* DQ COMP Overrides */
641 mrc_alt_write_mask(DDRPHY,
642 DQDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
643 (1 << 31) | (0x0a << 16),
646 mrc_alt_write_mask(DDRPHY,
647 DQDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
648 (1 << 31) | (0x0a << 16),
651 mrc_alt_write_mask(DDRPHY,
652 DQDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
653 (1 << 31) | (0x10 << 16),
656 mrc_alt_write_mask(DDRPHY,
657 DQDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
658 (1 << 31) | (0x10 << 16),
661 mrc_alt_write_mask(DDRPHY,
662 DQODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
663 (1 << 31) | (0x0b << 16),
666 mrc_alt_write_mask(DDRPHY,
667 DQODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
668 (1 << 31) | (0x0b << 16),
671 mrc_alt_write_mask(DDRPHY,
672 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
675 mrc_alt_write_mask(DDRPHY,
676 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
679 /* DQS COMP Overrides */
681 mrc_alt_write_mask(DDRPHY,
682 DQSDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
683 (1 << 31) | (0x0a << 16),
686 mrc_alt_write_mask(DDRPHY,
687 DQSDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
688 (1 << 31) | (0x0a << 16),
691 mrc_alt_write_mask(DDRPHY,
692 DQSDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
693 (1 << 31) | (0x10 << 16),
696 mrc_alt_write_mask(DDRPHY,
697 DQSDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
698 (1 << 31) | (0x10 << 16),
701 mrc_alt_write_mask(DDRPHY,
702 DQSODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
703 (1 << 31) | (0x0b << 16),
706 mrc_alt_write_mask(DDRPHY,
707 DQSODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
708 (1 << 31) | (0x0b << 16),
711 mrc_alt_write_mask(DDRPHY,
712 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
715 mrc_alt_write_mask(DDRPHY,
716 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
719 /* CLK COMP Overrides */
721 mrc_alt_write_mask(DDRPHY,
722 CLKDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
723 (1 << 31) | (0x0c << 16),
726 mrc_alt_write_mask(DDRPHY,
727 CLKDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
728 (1 << 31) | (0x0c << 16),
731 mrc_alt_write_mask(DDRPHY,
732 CLKDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
733 (1 << 31) | (0x07 << 16),
736 mrc_alt_write_mask(DDRPHY,
737 CLKDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
738 (1 << 31) | (0x07 << 16),
741 mrc_alt_write_mask(DDRPHY,
742 CLKODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
743 (1 << 31) | (0x0b << 16),
746 mrc_alt_write_mask(DDRPHY,
747 CLKODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
748 (1 << 31) | (0x0b << 16),
751 mrc_alt_write_mask(DDRPHY,
752 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
755 mrc_alt_write_mask(DDRPHY,
756 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
759 /* CMD COMP Overrides */
761 mrc_alt_write_mask(DDRPHY,
762 CMDDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
763 (1 << 31) | (0x0d << 16),
766 mrc_alt_write_mask(DDRPHY,
767 CMDDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
768 (1 << 31) | (0x0d << 16),
771 mrc_alt_write_mask(DDRPHY,
772 CMDDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
773 (1 << 31) | (0x0a << 16),
776 mrc_alt_write_mask(DDRPHY,
777 CMDDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
778 (1 << 31) | (0x0a << 16),
781 /* CTL COMP Overrides */
783 mrc_alt_write_mask(DDRPHY,
784 CTLDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
785 (1 << 31) | (0x0d << 16),
788 mrc_alt_write_mask(DDRPHY,
789 CTLDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
790 (1 << 31) | (0x0d << 16),
793 mrc_alt_write_mask(DDRPHY,
794 CTLDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
795 (1 << 31) | (0x0a << 16),
798 mrc_alt_write_mask(DDRPHY,
799 CTLDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
800 (1 << 31) | (0x0a << 16),
803 /* DQ TCOCOMP Overrides */
805 mrc_alt_write_mask(DDRPHY,
806 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
807 (1 << 31) | (0x1f << 16),
810 mrc_alt_write_mask(DDRPHY,
811 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
812 (1 << 31) | (0x1f << 16),
815 /* DQS TCOCOMP Overrides */
817 mrc_alt_write_mask(DDRPHY,
818 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
819 (1 << 31) | (0x1f << 16),
822 mrc_alt_write_mask(DDRPHY,
823 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
824 (1 << 31) | (0x1f << 16),
827 /* CLK TCOCOMP Overrides */
829 mrc_alt_write_mask(DDRPHY,
830 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
831 (1 << 31) | (0x1f << 16),
834 mrc_alt_write_mask(DDRPHY,
835 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
836 (1 << 31) | (0x1f << 16),
840 /* program STATIC delays */
842 set_wcmd(ch, ddr_wcmd[PLATFORM_ID]);
844 set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK);
847 for (rk = 0; rk < NUM_RANKS; rk++) {
848 if (mrc_params->rank_enables & (1 << rk)) {
849 set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]);
851 set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]);
853 set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK);
860 /* COMP (non channel specific) */
861 /* RCOMP: Dither PU Enable */
862 mrc_alt_write_mask(DDRPHY, DQANADRVPUCTL, 1 << 30, 1 << 30);
863 /* RCOMP: Dither PD Enable */
864 mrc_alt_write_mask(DDRPHY, DQANADRVPDCTL, 1 << 30, 1 << 30);
865 /* RCOMP: Dither PU Enable */
866 mrc_alt_write_mask(DDRPHY, CMDANADRVPUCTL, 1 << 30, 1 << 30);
867 /* RCOMP: Dither PD Enable */
868 mrc_alt_write_mask(DDRPHY, CMDANADRVPDCTL, 1 << 30, 1 << 30);
869 /* RCOMP: Dither PU Enable */
870 mrc_alt_write_mask(DDRPHY, CLKANADRVPUCTL, 1 << 30, 1 << 30);
871 /* RCOMP: Dither PD Enable */
872 mrc_alt_write_mask(DDRPHY, CLKANADRVPDCTL, 1 << 30, 1 << 30);
873 /* RCOMP: Dither PU Enable */
874 mrc_alt_write_mask(DDRPHY, DQSANADRVPUCTL, 1 << 30, 1 << 30);
875 /* RCOMP: Dither PD Enable */
876 mrc_alt_write_mask(DDRPHY, DQSANADRVPDCTL, 1 << 30, 1 << 30);
877 /* RCOMP: Dither PU Enable */
878 mrc_alt_write_mask(DDRPHY, CTLANADRVPUCTL, 1 << 30, 1 << 30);
879 /* RCOMP: Dither PD Enable */
880 mrc_alt_write_mask(DDRPHY, CTLANADRVPDCTL, 1 << 30, 1 << 30);
881 /* ODT: Dither PU Enable */
882 mrc_alt_write_mask(DDRPHY, DQANAODTPUCTL, 1 << 30, 1 << 30);
883 /* ODT: Dither PD Enable */
884 mrc_alt_write_mask(DDRPHY, DQANAODTPDCTL, 1 << 30, 1 << 30);
885 /* ODT: Dither PU Enable */
886 mrc_alt_write_mask(DDRPHY, CLKANAODTPUCTL, 1 << 30, 1 << 30);
887 /* ODT: Dither PD Enable */
888 mrc_alt_write_mask(DDRPHY, CLKANAODTPDCTL, 1 << 30, 1 << 30);
889 /* ODT: Dither PU Enable */
890 mrc_alt_write_mask(DDRPHY, DQSANAODTPUCTL, 1 << 30, 1 << 30);
891 /* ODT: Dither PD Enable */
892 mrc_alt_write_mask(DDRPHY, DQSANAODTPDCTL, 1 << 30, 1 << 30);
893 /* DCOMP: Dither PU Enable */
894 mrc_alt_write_mask(DDRPHY, DQANADLYPUCTL, 1 << 30, 1 << 30);
895 /* DCOMP: Dither PD Enable */
896 mrc_alt_write_mask(DDRPHY, DQANADLYPDCTL, 1 << 30, 1 << 30);
897 /* DCOMP: Dither PU Enable */
898 mrc_alt_write_mask(DDRPHY, CMDANADLYPUCTL, 1 << 30, 1 << 30);
899 /* DCOMP: Dither PD Enable */
900 mrc_alt_write_mask(DDRPHY, CMDANADLYPDCTL, 1 << 30, 1 << 30);
901 /* DCOMP: Dither PU Enable */
902 mrc_alt_write_mask(DDRPHY, CLKANADLYPUCTL, 1 << 30, 1 << 30);
903 /* DCOMP: Dither PD Enable */
904 mrc_alt_write_mask(DDRPHY, CLKANADLYPDCTL, 1 << 30, 1 << 30);
905 /* DCOMP: Dither PU Enable */
906 mrc_alt_write_mask(DDRPHY, DQSANADLYPUCTL, 1 << 30, 1 << 30);
907 /* DCOMP: Dither PD Enable */
908 mrc_alt_write_mask(DDRPHY, DQSANADLYPDCTL, 1 << 30, 1 << 30);
909 /* DCOMP: Dither PU Enable */
910 mrc_alt_write_mask(DDRPHY, CTLANADLYPUCTL, 1 << 30, 1 << 30);
911 /* DCOMP: Dither PD Enable */
912 mrc_alt_write_mask(DDRPHY, CTLANADLYPDCTL, 1 << 30, 1 << 30);
913 /* TCO: Dither PU Enable */
914 mrc_alt_write_mask(DDRPHY, DQANATCOPUCTL, 1 << 30, 1 << 30);
915 /* TCO: Dither PD Enable */
916 mrc_alt_write_mask(DDRPHY, DQANATCOPDCTL, 1 << 30, 1 << 30);
917 /* TCO: Dither PU Enable */
918 mrc_alt_write_mask(DDRPHY, CLKANATCOPUCTL, 1 << 30, 1 << 30);
919 /* TCO: Dither PD Enable */
920 mrc_alt_write_mask(DDRPHY, CLKANATCOPDCTL, 1 << 30, 1 << 30);
921 /* TCO: Dither PU Enable */
922 mrc_alt_write_mask(DDRPHY, DQSANATCOPUCTL, 1 << 30, 1 << 30);
923 /* TCO: Dither PD Enable */
924 mrc_alt_write_mask(DDRPHY, DQSANATCOPDCTL, 1 << 30, 1 << 30);
925 /* TCOCOMP: Pulse Count */
926 mrc_alt_write_mask(DDRPHY, TCOCNTCTRL, 1, 3);
927 /* ODT: CMD/CTL PD/PU */
928 mrc_alt_write_mask(DDRPHY, CHNLBUFSTATIC,
929 (0x03 << 24) | (0x03 << 16), 0x1f1f0000);
930 /* Set 1us counter */
931 mrc_alt_write_mask(DDRPHY, MSCNTR, 0x64, 0xff);
932 mrc_alt_write_mask(DDRPHY, LATCH1CTL, 0x1 << 28, 0x70000000);
934 /* Release PHY from reset */
935 mrc_alt_write_mask(DDRPHY, MASTERRSTN, 1, 1);
938 mrc_post_code(0x03, 0x11);
940 for (ch = 0; ch < NUM_CHANNELS; ch++) {
941 if (mrc_params->channel_enables & (1 << ch)) {
944 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
946 mrc_alt_write_mask(DDRPHY,
948 bl_grp * DDRIODQ_BL_OFFSET +
949 ch * DDRIODQ_CH_OFFSET,
951 1 << 13); /* Enable VREG */
956 mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
957 1 << 13, 1 << 13); /* Enable VREG */
960 mrc_alt_write_mask(DDRPHY,
961 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
962 1 << 13, 1 << 13); /* Enable VREG */
965 mrc_alt_write_mask(DDRPHY,
966 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
967 1 << 13, 1 << 13); /* Enable VREG */
973 mrc_post_code(0x03, 0x12);
976 for (ch = 0; ch < NUM_CHANNELS; ch++) {
977 if (mrc_params->channel_enables & (1 << ch)) {
980 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
982 mrc_alt_write_mask(DDRPHY,
984 bl_grp * DDRIODQ_BL_OFFSET +
985 ch * DDRIODQ_CH_OFFSET,
987 1 << 17); /* Enable MCDLL */
992 mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
993 1 << 17, 1 << 17); /* Enable MCDLL */
996 mrc_alt_write_mask(DDRPHY,
997 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
998 1 << 18, 1 << 18); /* Enable MCDLL */
1001 mrc_alt_write_mask(DDRPHY,
1002 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
1003 1 << 18, 1 << 18); /* Enable MCDLL */
1009 mrc_post_code(0x03, 0x13);
1012 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1013 if (mrc_params->channel_enables & (1 << ch)) {
1016 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1018 #ifdef FORCE_16BIT_DDRIO
1020 (mrc_params->channel_width == X16)) ?
1026 mrc_alt_write_mask(DDRPHY,
1028 bl_grp * DDRIODQ_BL_OFFSET +
1029 ch * DDRIODQ_CH_OFFSET,
1033 mrc_alt_write_mask(DDRPHY,
1035 bl_grp * DDRIODQ_BL_OFFSET +
1036 ch * DDRIODQ_CH_OFFSET,
1039 /* Enable RXDLL Overrides BL0 */
1040 mrc_alt_write_mask(DDRPHY,
1042 bl_grp * DDRIODQ_BL_OFFSET +
1043 ch * DDRIODQ_CH_OFFSET,
1049 mrc_alt_write_mask(DDRPHY, ECCDLLTXCTL,
1054 mrc_alt_write_mask(DDRPHY,
1055 CMDDLLTXCTL + ch * DDRIOCCC_CH_OFFSET,
1062 mrc_post_code(0x03, 0x14);
1064 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1065 if (mrc_params->channel_enables & (1 << ch)) {
1066 /* Host To Memory Clock Alignment (HMC) for 800/1066 */
1068 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1070 /* CLK_ALIGN_MOD_ID */
1071 mrc_alt_write_mask(DDRPHY,
1073 bl_grp * DDRIODQ_BL_OFFSET +
1074 ch * DDRIODQ_CH_OFFSET,
1079 mrc_alt_write_mask(DDRPHY,
1080 ECCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1082 mrc_alt_write_mask(DDRPHY,
1083 CMDCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1085 mrc_alt_write_mask(DDRPHY,
1086 CCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1088 mrc_alt_write_mask(DDRPHY,
1089 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1092 * NUM_SAMPLES, MAX_SAMPLES,
1093 * MACRO_PI_STEP, MICRO_PI_STEP
1095 mrc_alt_write_mask(DDRPHY,
1096 CMDCLKALIGNREG1 + ch * DDRIOCCC_CH_OFFSET,
1097 (0x18 << 16) | (0x10 << 8) |
1098 (0x8 << 2) | (0x1 << 0),
1100 /* TOTAL_NUM_MODULES, FIRST_U_PARTITION */
1101 mrc_alt_write_mask(DDRPHY,
1102 CMDCLKALIGNREG2 + ch * DDRIOCCC_CH_OFFSET,
1103 (0x10 << 16) | (0x4 << 8) | (0x2 << 4),
1106 /* START_CLK_ALIGN=1 */
1107 mrc_alt_write_mask(DDRPHY,
1108 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1110 while (msg_port_alt_read(DDRPHY,
1111 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET) &
1113 ; /* wait for START_CLK_ALIGN=0 */
1116 /* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */
1117 mrc_alt_write_mask(DDRPHY,
1118 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
1119 1, 1); /* WRPTRENABLE=1 */
1122 /* enable bypass for CLK buffer (PO) */
1123 mrc_alt_write_mask(DDRPHY,
1124 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1126 /* Initial COMP Enable */
1127 mrc_alt_write_mask(DDRPHY, CMPCTRL, 1, 1);
1128 /* wait for Initial COMP Enable = 0 */
1129 while (msg_port_alt_read(DDRPHY, CMPCTRL) & 1)
1131 /* disable bypass for CLK buffer (PO) */
1132 mrc_alt_write_mask(DDRPHY,
1133 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1139 mrc_alt_write_mask(DDRPHY,
1140 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
1141 1 << 2, 1 << 2); /* IOBUFACTRST_N=1 */
1143 /* DDRPHY initialization complete */
1144 mrc_alt_write_mask(DDRPHY,
1145 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
1146 1 << 20, 1 << 20); /* SPID_INIT_COMPLETE=1 */
1153 /* This function performs JEDEC initialization on all enabled channels */
1154 void perform_jedec_init(struct mrc_params *mrc_params)
1156 uint8_t twr, wl, rank;
1168 /* jedec_init starts */
1169 mrc_post_code(0x04, 0x00);
1171 /* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */
1172 mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 2, 0x102);
1174 /* Assert RESET# for 200us */
1177 /* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */
1178 mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 0x100, 0x102);
1180 dtr0 = msg_port_read(MEM_CTLR, DTR0);
1183 * Set CKEVAL for populated ranks
1184 * then send NOP to each rank (#4550197)
1187 drp = msg_port_read(MEM_CTLR, DRP);
1190 drmc = msg_port_read(MEM_CTLR, DRMC);
1192 drmc |= (DRMC_CKEMODE | drp);
1194 msg_port_write(MEM_CTLR, DRMC, drmc);
1196 for (rank = 0; rank < NUM_RANKS; rank++) {
1197 /* Skip to next populated rank */
1198 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1201 dram_init_command(DCMD_NOP(rank));
1204 msg_port_write(MEM_CTLR, DRMC,
1205 (mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0));
1209 * BIT[15:11] --> Always "0"
1210 * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0)
1211 * BIT[08] --> Always "0"
1212 * BIT[07] --> SRT: use sr_temp_range
1213 * BIT[06] --> ASR: want "Manual SR Reference" (0)
1214 * BIT[05:03] --> CWL: use oem_tCWL
1215 * BIT[02:00] --> PASR: want "Full Array" (0)
1217 emrs2_cmd |= (2 << 3);
1218 wl = 5 + mrc_params->ddr_speed;
1219 emrs2_cmd |= ((wl - 5) << 9);
1220 emrs2_cmd |= (mrc_params->sr_temp_range << 13);
1224 * BIT[15:03] --> Always "0"
1225 * BIT[02] --> MPR: want "Normal Operation" (0)
1226 * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0)
1228 emrs3_cmd |= (3 << 3);
1232 * BIT[15:13] --> Always "0"
1233 * BIT[12:12] --> Qoff: want "Output Buffer Enabled" (0)
1234 * BIT[11:11] --> TDQS: want "Disabled" (0)
1235 * BIT[10:10] --> Always "0"
1236 * BIT[09,06,02] --> Rtt_nom: use rtt_nom_value
1237 * BIT[08] --> Always "0"
1238 * BIT[07] --> WR_LVL: want "Disabled" (0)
1239 * BIT[05,01] --> DIC: use ron_value
1240 * BIT[04:03] --> AL: additive latency want "0" (0)
1241 * BIT[00] --> DLL: want "Enable" (0)
1243 * (BIT5|BIT1) set Ron value
1244 * 00 --> RZQ/6 (40ohm)
1245 * 01 --> RZQ/7 (34ohm)
1248 * (BIT9|BIT6|BIT2) set Rtt_nom value
1250 * 001 --> RZQ/4 ( 60ohm)
1251 * 010 --> RZQ/2 (120ohm)
1252 * 011 --> RZQ/6 ( 40ohm)
1255 emrs1_cmd |= (1 << 3);
1256 emrs1_cmd &= ~(1 << 6);
1258 if (mrc_params->ron_value == 0)
1259 emrs1_cmd |= (1 << 7);
1261 emrs1_cmd &= ~(1 << 7);
1263 if (mrc_params->rtt_nom_value == 0)
1264 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6);
1265 else if (mrc_params->rtt_nom_value == 1)
1266 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6);
1267 else if (mrc_params->rtt_nom_value == 2)
1268 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6);
1270 /* save MRS1 value (excluding control fields) */
1271 mrc_params->mrs1 = emrs1_cmd >> 6;
1275 * BIT[15:13] --> Always "0"
1276 * BIT[12] --> PPD: for Quark (1)
1277 * BIT[11:09] --> WR: use oem_tWR
1278 * BIT[08] --> DLL: want "Reset" (1, self clearing)
1279 * BIT[07] --> MODE: want "Normal" (0)
1280 * BIT[06:04,02] --> CL: use oem_tCAS
1281 * BIT[03] --> RD_BURST_TYPE: want "Interleave" (1)
1282 * BIT[01:00] --> BL: want "8 Fixed" (0)
1293 * BIT[02:02] "0" if oem_tCAS <= 11 (1866?)
1294 * BIT[06:04] use oem_tCAS-4
1296 mrs0_cmd |= (1 << 14);
1297 mrs0_cmd |= (1 << 18);
1298 mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10);
1300 tck = t_ck[mrc_params->ddr_speed];
1301 /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
1302 twr = MCEIL(15000, tck);
1303 mrs0_cmd |= ((twr - 4) << 15);
1305 for (rank = 0; rank < NUM_RANKS; rank++) {
1306 /* Skip to next populated rank */
1307 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1310 emrs2_cmd |= (rank << 22);
1311 dram_init_command(emrs2_cmd);
1313 emrs3_cmd |= (rank << 22);
1314 dram_init_command(emrs3_cmd);
1316 emrs1_cmd |= (rank << 22);
1317 dram_init_command(emrs1_cmd);
1319 mrs0_cmd |= (rank << 22);
1320 dram_init_command(mrs0_cmd);
1322 dram_init_command(DCMD_ZQCL(rank));
1329 * Dunit Initialization Complete
1331 * Indicates that initialization of the Dunit has completed.
1333 * Memory accesses are permitted and maintenance operation begins.
1334 * Until this bit is set to a 1, the memory controller will not accept
1335 * DRAM requests from the MEMORY_MANAGER or HTE.
1337 void set_ddr_init_complete(struct mrc_params *mrc_params)
1343 dco = msg_port_read(MEM_CTLR, DCO);
1346 msg_port_write(MEM_CTLR, DCO, dco);
1352 * This function will retrieve relevant timing data
1354 * This data will be used on subsequent boots to speed up boot times
1355 * and is required for Suspend To RAM capabilities.
1357 void restore_timings(struct mrc_params *mrc_params)
1360 const struct mrc_timings *mt = &mrc_params->timings;
1362 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1363 for (rk = 0; rk < NUM_RANKS; rk++) {
1364 for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1365 set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]);
1366 set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]);
1367 set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]);
1368 set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]);
1370 /* VREF (RANK0 only) */
1371 set_vref(ch, bl, mt->vref[ch][bl]);
1374 set_wctl(ch, rk, mt->wctl[ch][rk]);
1376 set_wcmd(ch, mt->wcmd[ch]);
1381 * Configure default settings normally set as part of read training
1383 * Some defaults have to be set earlier as they may affect earlier
1386 void default_timings(struct mrc_params *mrc_params)
1390 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1391 for (rk = 0; rk < NUM_RANKS; rk++) {
1392 for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1393 set_rdqs(ch, rk, bl, 24);
1395 /* VREF (RANK0 only) */
1396 set_vref(ch, bl, 32);
1404 * This function will perform our RCVEN Calibration Algorithm.
1405 * We will only use the 2xCLK domain timings to perform RCVEN Calibration.
1406 * All byte lanes will be calibrated "simultaneously" per channel per rank.
1408 void rcvn_cal(struct mrc_params *mrc_params)
1410 uint8_t ch; /* channel counter */
1411 uint8_t rk; /* rank counter */
1412 uint8_t bl; /* byte lane counter */
1413 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1416 /* used to find placement for rank2rank sharing configs */
1417 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1419 /* used to find placement for rank2rank sharing configs */
1420 uint32_t num_ranks_enabled = 0;
1427 /* absolute PI value to be programmed on the byte lane */
1428 uint32_t delay[NUM_BYTE_LANES];
1429 u32 dtr1, dtr1_save;
1434 /* rcvn_cal starts */
1435 mrc_post_code(0x05, 0x00);
1438 /* need separate burst to sample DQS preamble */
1439 dtr1 = msg_port_read(MEM_CTLR, DTR1);
1441 dtr1 |= DTR1_TCCD_12CLK;
1442 msg_port_write(MEM_CTLR, DTR1, dtr1);
1446 /* need to set "final_delay[][]" elements to "0" */
1447 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1450 /* loop through each enabled channel */
1451 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1452 if (mrc_params->channel_enables & (1 << ch)) {
1453 /* perform RCVEN Calibration on a per rank basis */
1454 for (rk = 0; rk < NUM_RANKS; rk++) {
1455 if (mrc_params->rank_enables & (1 << rk)) {
1457 * POST_CODE here indicates the current
1458 * channel and rank being calibrated
1460 mrc_post_code(0x05, 0x10 + ((ch << 4) | rk));
1463 /* et hard-coded timing values */
1464 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++)
1465 set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]);
1467 /* enable FIFORST */
1468 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1469 mrc_alt_write_mask(DDRPHY,
1471 (bl >> 1) * DDRIODQ_BL_OFFSET +
1472 ch * DDRIODQ_CH_OFFSET,
1475 /* initialize the starting delay to 128 PI (cas +1 CLK) */
1476 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1477 /* 1x CLK domain timing is cas-4 */
1478 delay[bl] = (4 + 1) * FULL_CLK;
1480 set_rcvn(ch, rk, bl, delay[bl]);
1483 /* now find the rising edge */
1484 find_rising_edge(mrc_params, delay, ch, rk, true);
1486 /* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */
1487 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1488 delay[bl] += QRTR_CLK;
1489 set_rcvn(ch, rk, bl, delay[bl]);
1491 /* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */
1493 temp = sample_dqs(mrc_params, ch, rk, true);
1494 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1495 if (temp & (1 << bl)) {
1496 if (delay[bl] >= FULL_CLK) {
1497 delay[bl] -= FULL_CLK;
1498 set_rcvn(ch, rk, bl, delay[bl]);
1500 /* not enough delay */
1501 training_message(ch, rk, bl);
1502 mrc_post_code(0xee, 0x50);
1506 } while (temp & 0xff);
1509 /* increment "num_ranks_enabled" */
1510 num_ranks_enabled++;
1511 /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1512 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1513 delay[bl] += QRTR_CLK;
1514 /* add "delay[]" values to "final_delay[][]" for rolling average */
1515 final_delay[ch][bl] += delay[bl];
1516 /* set timing based on rolling average values */
1517 set_rcvn(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1520 /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1521 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1522 delay[bl] += QRTR_CLK;
1523 set_rcvn(ch, rk, bl, delay[bl]);
1527 /* disable FIFORST */
1528 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1529 mrc_alt_write_mask(DDRPHY,
1531 (bl >> 1) * DDRIODQ_BL_OFFSET +
1532 ch * DDRIODQ_CH_OFFSET,
1542 /* restore original */
1543 msg_port_write(MEM_CTLR, DTR1, dtr1_save);
1550 * This function will perform the Write Levelling algorithm
1551 * (align WCLK and WDQS).
1553 * This algorithm will act on each rank in each channel separately.
1555 void wr_level(struct mrc_params *mrc_params)
1557 uint8_t ch; /* channel counter */
1558 uint8_t rk; /* rank counter */
1559 uint8_t bl; /* byte lane counter */
1560 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1563 /* used to find placement for rank2rank sharing configs */
1564 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1566 /* used to find placement for rank2rank sharing configs */
1567 uint32_t num_ranks_enabled = 0;
1573 /* determines stop condition for CRS_WR_LVL */
1574 bool all_edges_found;
1575 /* absolute PI value to be programmed on the byte lane */
1576 uint32_t delay[NUM_BYTE_LANES];
1578 * static makes it so the data is loaded in the heap once by shadow(),
1579 * where non-static copies the data onto the stack every time this
1580 * function is called
1582 uint32_t address; /* address to be checked during COARSE_WR_LVL */
1583 u32 dtr4, dtr4_save;
1588 /* wr_level starts */
1589 mrc_post_code(0x06, 0x00);
1592 /* need to set "final_delay[][]" elements to "0" */
1593 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1596 /* loop through each enabled channel */
1597 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1598 if (mrc_params->channel_enables & (1 << ch)) {
1599 /* perform WRITE LEVELING algorithm on a per rank basis */
1600 for (rk = 0; rk < NUM_RANKS; rk++) {
1601 if (mrc_params->rank_enables & (1 << rk)) {
1603 * POST_CODE here indicates the current
1604 * rank and channel being calibrated
1606 mrc_post_code(0x06, 0x10 + ((ch << 4) | rk));
1609 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1610 set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]);
1611 set_wdq(ch, rk, bl, ddr_wdqs[PLATFORM_ID] - QRTR_CLK);
1615 * perform a single PRECHARGE_ALL command to
1616 * make DRAM state machine go to IDLE state
1618 dram_init_command(DCMD_PREA(rk));
1621 * enable Write Levelling Mode
1622 * (EMRS1 w/ Write Levelling Mode Enable)
1624 dram_init_command(DCMD_MRS1(rk, 0x82));
1627 * set ODT DRAM Full Time Termination
1631 dtr4 = msg_port_read(MEM_CTLR, DTR4);
1633 dtr4 |= DTR4_ODTDIS;
1634 msg_port_write(MEM_CTLR, DTR4, dtr4);
1636 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1638 * Enable Sandy Bridge Mode (WDQ Tri-State) &
1639 * Ensure 5 WDQS pulses during Write Leveling
1641 mrc_alt_write_mask(DDRPHY,
1642 DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1647 /* Write Leveling Mode enabled in IO */
1648 mrc_alt_write_mask(DDRPHY,
1649 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1652 /* Initialize the starting delay to WCLK */
1653 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1658 delay[bl] = get_wclk(ch, rk);
1660 set_wdqs(ch, rk, bl, delay[bl]);
1663 /* now find the rising edge */
1664 find_rising_edge(mrc_params, delay, ch, rk, false);
1666 /* disable Write Levelling Mode */
1667 mrc_alt_write_mask(DDRPHY,
1668 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1671 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1672 /* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */
1673 mrc_alt_write_mask(DDRPHY,
1674 DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1679 /* restore original DTR4 */
1680 msg_port_write(MEM_CTLR, DTR4, dtr4_save);
1683 * restore original value
1684 * (Write Levelling Mode Disable)
1686 dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1));
1689 * perform a single PRECHARGE_ALL command to
1690 * make DRAM state machine go to IDLE state
1692 dram_init_command(DCMD_PREA(rk));
1694 mrc_post_code(0x06, 0x30 + ((ch << 4) | rk));
1697 * COARSE WRITE LEVEL:
1698 * check that we're on the correct clock edge
1701 /* hte reconfiguration request */
1702 mrc_params->hte_setup = 1;
1704 /* start CRS_WR_LVL with WDQS = WDQS + 128 PI */
1705 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1706 delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK;
1707 set_wdqs(ch, rk, bl, delay[bl]);
1709 * program WDQ timings based on WDQS
1710 * (WDQ = WDQS - 32 PI)
1712 set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
1715 /* get an address in the targeted channel/rank */
1716 address = get_addr(ch, rk);
1718 uint32_t coarse_result = 0x00;
1719 uint32_t coarse_result_mask = byte_lane_mask(mrc_params);
1721 all_edges_found = true;
1723 mrc_params->hte_setup = 1;
1724 coarse_result = check_rw_coarse(mrc_params, address);
1726 /* check for failures and margin the byte lane back 128 PI (1 CLK) */
1727 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1728 if (coarse_result & (coarse_result_mask << bl)) {
1729 all_edges_found = false;
1730 delay[bl] -= FULL_CLK;
1731 set_wdqs(ch, rk, bl, delay[bl]);
1732 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1733 set_wdq(ch, rk, bl, delay[bl] - QRTR_CLK);
1736 } while (!all_edges_found);
1739 /* increment "num_ranks_enabled" */
1740 num_ranks_enabled++;
1741 /* accumulate "final_delay[][]" values from "delay[]" values for rolling average */
1742 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1743 final_delay[ch][bl] += delay[bl];
1744 set_wdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1745 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1746 set_wdq(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled - QRTR_CLK);
1758 void prog_page_ctrl(struct mrc_params *mrc_params)
1764 dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
1765 dpmc0 &= ~DPMC0_PCLSTO_MASK;
1767 dpmc0 |= DPMC0_PREAPWDEN;
1768 msg_port_write(MEM_CTLR, DPMC0, dpmc0);
1772 * This function will perform the READ TRAINING Algorithm on all
1773 * channels/ranks/byte_lanes simultaneously to minimize execution time.
1775 * The idea here is to train the VREF and RDQS (and eventually RDQ) values
1776 * to achieve maximum READ margins. The algorithm will first determine the
1777 * X coordinate (RDQS setting). This is done by collapsing the VREF eye
1778 * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX.
1779 * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX,
1780 * then average those; this will be the final X coordinate. The algorithm
1781 * will then determine the Y coordinate (VREF setting). This is done by
1782 * collapsing the RDQS eye until we find a minimum required VREF eye for
1783 * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at
1784 * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y
1787 * NOTE: this algorithm assumes the eye curves have a one-to-one relationship,
1788 * meaning for each X the curve has only one Y and vice-a-versa.
1790 void rd_train(struct mrc_params *mrc_params)
1792 uint8_t ch; /* channel counter */
1793 uint8_t rk; /* rank counter */
1794 uint8_t bl; /* byte lane counter */
1795 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1798 uint8_t side_x; /* tracks LEFT/RIGHT approach vectors */
1799 uint8_t side_y; /* tracks BOTTOM/TOP approach vectors */
1800 /* X coordinate data (passing RDQS values) for approach vectors */
1801 uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1802 /* Y coordinate data (passing VREF values) for approach vectors */
1803 uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES];
1804 /* centered X (RDQS) */
1805 uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1806 /* centered Y (VREF) */
1807 uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES];
1808 uint32_t address; /* target address for check_bls_ex() */
1809 uint32_t result; /* result of check_bls_ex() */
1810 uint32_t bl_mask; /* byte lane mask for result checking */
1812 /* used to find placement for rank2rank sharing configs */
1813 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1814 /* used to find placement for rank2rank sharing configs */
1815 uint32_t num_ranks_enabled = 0;
1819 /* rd_train starts */
1820 mrc_post_code(0x07, 0x00);
1825 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1826 if (mrc_params->channel_enables & (1 << ch)) {
1827 for (rk = 0; rk < NUM_RANKS; rk++) {
1828 if (mrc_params->rank_enables & (1 << rk)) {
1830 bl < NUM_BYTE_LANES / bl_divisor;
1832 set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]);
1839 /* initialize x/y_coordinate arrays */
1840 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1841 if (mrc_params->channel_enables & (1 << ch)) {
1842 for (rk = 0; rk < NUM_RANKS; rk++) {
1843 if (mrc_params->rank_enables & (1 << rk)) {
1845 bl < NUM_BYTE_LANES / bl_divisor;
1848 x_coordinate[L][B][ch][rk][bl] = RDQS_MIN;
1849 x_coordinate[R][B][ch][rk][bl] = RDQS_MAX;
1850 x_coordinate[L][T][ch][rk][bl] = RDQS_MIN;
1851 x_coordinate[R][T][ch][rk][bl] = RDQS_MAX;
1853 y_coordinate[L][B][ch][bl] = VREF_MIN;
1854 y_coordinate[R][B][ch][bl] = VREF_MIN;
1855 y_coordinate[L][T][ch][bl] = VREF_MAX;
1856 y_coordinate[R][T][ch][bl] = VREF_MAX;
1863 /* initialize other variables */
1864 bl_mask = byte_lane_mask(mrc_params);
1865 address = get_addr(0, 0);
1868 /* need to set "final_delay[][]" elements to "0" */
1869 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1872 /* look for passing coordinates */
1873 for (side_y = B; side_y <= T; side_y++) {
1874 for (side_x = L; side_x <= R; side_x++) {
1875 mrc_post_code(0x07, 0x10 + side_y * 2 + side_x);
1877 /* find passing values */
1878 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1879 if (mrc_params->channel_enables & (0x1 << ch)) {
1880 for (rk = 0; rk < NUM_RANKS; rk++) {
1881 if (mrc_params->rank_enables &
1883 /* set x/y_coordinate search starting settings */
1885 bl < NUM_BYTE_LANES / bl_divisor;
1887 set_rdqs(ch, rk, bl,
1888 x_coordinate[side_x][side_y][ch][rk][bl]);
1890 y_coordinate[side_x][side_y][ch][bl]);
1893 /* get an address in the target channel/rank */
1894 address = get_addr(ch, rk);
1896 /* request HTE reconfiguration */
1897 mrc_params->hte_setup = 1;
1899 /* test the settings */
1901 /* result[07:00] == failing byte lane (MAX 8) */
1902 result = check_bls_ex(mrc_params, address);
1904 /* check for failures */
1905 if (result & 0xff) {
1906 /* at least 1 byte lane failed */
1907 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1910 /* adjust the RDQS values accordingly */
1912 x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP;
1914 x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP;
1916 /* check that we haven't closed the RDQS_EYE too much */
1917 if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) ||
1918 (x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) ||
1919 (x_coordinate[L][side_y][ch][rk][bl] ==
1920 x_coordinate[R][side_y][ch][rk][bl])) {
1922 * not enough RDQS margin available at this VREF
1923 * update VREF values accordingly
1926 y_coordinate[side_x][B][ch][bl] += VREF_STEP;
1928 y_coordinate[side_x][T][ch][bl] -= VREF_STEP;
1930 /* check that we haven't closed the VREF_EYE too much */
1931 if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) ||
1932 (y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) ||
1933 (y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) {
1934 /* VREF_EYE collapsed below MIN_VREF_EYE */
1935 training_message(ch, rk, bl);
1936 mrc_post_code(0xEE, 0x70 + side_y * 2 + side_x);
1938 /* update the VREF setting */
1939 set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]);
1940 /* reset the X coordinate to begin the search at the new VREF */
1941 x_coordinate[side_x][side_y][ch][rk][bl] =
1942 (side_x == L) ? RDQS_MIN : RDQS_MAX;
1946 /* update the RDQS setting */
1947 set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]);
1951 } while (result & 0xff);
1959 mrc_post_code(0x07, 0x20);
1961 /* find final RDQS (X coordinate) & final VREF (Y coordinate) */
1962 for (ch = 0; ch < NUM_CHANNELS; ch++) {
1963 if (mrc_params->channel_enables & (1 << ch)) {
1964 for (rk = 0; rk < NUM_RANKS; rk++) {
1965 if (mrc_params->rank_enables & (1 << rk)) {
1966 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1972 "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n",
1974 x_coordinate[L][T][ch][rk][bl],
1975 x_coordinate[R][T][ch][rk][bl],
1976 x_coordinate[L][B][ch][rk][bl],
1977 x_coordinate[R][B][ch][rk][bl]);
1979 /* average the TOP side LEFT & RIGHT values */
1980 temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2;
1981 /* average the BOTTOM side LEFT & RIGHT values */
1982 temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2;
1983 /* average the above averages */
1984 x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2);
1988 "VREF R/L eye lane%d : %d-%d %d-%d\n",
1990 y_coordinate[R][B][ch][bl],
1991 y_coordinate[R][T][ch][bl],
1992 y_coordinate[L][B][ch][bl],
1993 y_coordinate[L][T][ch][bl]);
1995 /* average the RIGHT side TOP & BOTTOM values */
1996 temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2;
1997 /* average the LEFT side TOP & BOTTOM values */
1998 temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2;
1999 /* average the above averages */
2000 y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2);
2008 /* perform an eye check */
2009 for (side_y = B; side_y <= T; side_y++) {
2010 for (side_x = L; side_x <= R; side_x++) {
2011 mrc_post_code(0x07, 0x30 + side_y * 2 + side_x);
2013 /* update the settings for the eye check */
2014 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2015 if (mrc_params->channel_enables & (1 << ch)) {
2016 for (rk = 0; rk < NUM_RANKS; rk++) {
2017 if (mrc_params->rank_enables & (1 << rk)) {
2018 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2020 set_rdqs(ch, rk, bl, x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2));
2022 set_rdqs(ch, rk, bl, x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2));
2025 set_vref(ch, bl, y_center[ch][bl] - (MIN_VREF_EYE / 2));
2027 set_vref(ch, bl, y_center[ch][bl] + (MIN_VREF_EYE / 2));
2034 /* request HTE reconfiguration */
2035 mrc_params->hte_setup = 1;
2038 if (check_bls_ex(mrc_params, address) & 0xff) {
2039 /* one or more byte lanes failed */
2040 mrc_post_code(0xee, 0x74 + side_x * 2 + side_y);
2046 mrc_post_code(0x07, 0x40);
2048 /* set final placements */
2049 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2050 if (mrc_params->channel_enables & (1 << ch)) {
2051 for (rk = 0; rk < NUM_RANKS; rk++) {
2052 if (mrc_params->rank_enables & (1 << rk)) {
2054 /* increment "num_ranks_enabled" */
2055 num_ranks_enabled++;
2057 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2060 final_delay[ch][bl] += x_center[ch][rk][bl];
2061 set_rdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
2063 set_rdqs(ch, rk, bl, x_center[ch][rk][bl]);
2066 set_vref(ch, bl, y_center[ch][bl]);
2078 * This function will perform the WRITE TRAINING Algorithm on all
2079 * channels/ranks/byte_lanes simultaneously to minimize execution time.
2081 * The idea here is to train the WDQ timings to achieve maximum WRITE margins.
2082 * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS
2083 * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data
2084 * patterns pass. This is because WDQS will be aligned to WCLK by the
2085 * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window
2088 void wr_train(struct mrc_params *mrc_params)
2090 uint8_t ch; /* channel counter */
2091 uint8_t rk; /* rank counter */
2092 uint8_t bl; /* byte lane counter */
2093 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
2096 uint8_t side; /* LEFT/RIGHT side indicator (0=L, 1=R) */
2097 uint32_t temp; /* temporary DWORD */
2098 /* 2 arrays, for L & R side passing delays */
2099 uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
2100 uint32_t address; /* target address for check_bls_ex() */
2101 uint32_t result; /* result of check_bls_ex() */
2102 uint32_t bl_mask; /* byte lane mask for result checking */
2104 /* used to find placement for rank2rank sharing configs */
2105 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
2106 /* used to find placement for rank2rank sharing configs */
2107 uint32_t num_ranks_enabled = 0;
2111 /* wr_train starts */
2112 mrc_post_code(0x08, 0x00);
2117 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2118 if (mrc_params->channel_enables & (1 << ch)) {
2119 for (rk = 0; rk < NUM_RANKS; rk++) {
2120 if (mrc_params->rank_enables & (1 << rk)) {
2122 bl < NUM_BYTE_LANES / bl_divisor;
2124 set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]);
2131 /* initialize "delay" */
2132 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2133 if (mrc_params->channel_enables & (1 << ch)) {
2134 for (rk = 0; rk < NUM_RANKS; rk++) {
2135 if (mrc_params->rank_enables & (1 << rk)) {
2137 bl < NUM_BYTE_LANES / bl_divisor;
2140 * want to start with
2141 * WDQ = (WDQS - QRTR_CLK)
2144 temp = get_wdqs(ch, rk, bl) - QRTR_CLK;
2145 delay[L][ch][rk][bl] = temp - QRTR_CLK;
2146 delay[R][ch][rk][bl] = temp + QRTR_CLK;
2153 /* initialize other variables */
2154 bl_mask = byte_lane_mask(mrc_params);
2155 address = get_addr(0, 0);
2158 /* need to set "final_delay[][]" elements to "0" */
2159 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
2163 * start algorithm on the LEFT side and train each channel/bl
2164 * until no failures are observed, then repeat for the RIGHT side.
2166 for (side = L; side <= R; side++) {
2167 mrc_post_code(0x08, 0x10 + side);
2169 /* set starting values */
2170 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2171 if (mrc_params->channel_enables & (1 << ch)) {
2172 for (rk = 0; rk < NUM_RANKS; rk++) {
2173 if (mrc_params->rank_enables &
2176 bl < NUM_BYTE_LANES / bl_divisor;
2178 set_wdq(ch, rk, bl, delay[side][ch][rk][bl]);
2185 /* find passing values */
2186 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2187 if (mrc_params->channel_enables & (1 << ch)) {
2188 for (rk = 0; rk < NUM_RANKS; rk++) {
2189 if (mrc_params->rank_enables &
2191 /* get an address in the target channel/rank */
2192 address = get_addr(ch, rk);
2194 /* request HTE reconfiguration */
2195 mrc_params->hte_setup = 1;
2197 /* check the settings */
2199 /* result[07:00] == failing byte lane (MAX 8) */
2200 result = check_bls_ex(mrc_params, address);
2201 /* check for failures */
2202 if (result & 0xff) {
2203 /* at least 1 byte lane failed */
2204 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2208 delay[L][ch][rk][bl] += WDQ_STEP;
2210 delay[R][ch][rk][bl] -= WDQ_STEP;
2212 /* check for algorithm failure */
2213 if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) {
2216 * update delay setting
2219 delay[side][ch][rk][bl]);
2222 * no margin available
2223 * notify the user and halt
2225 training_message(ch, rk, bl);
2226 mrc_post_code(0xee, 0x80 + side);
2231 /* stop when all byte lanes pass */
2232 } while (result & 0xff);
2239 /* program WDQ to the middle of passing window */
2240 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2241 if (mrc_params->channel_enables & (1 << ch)) {
2242 for (rk = 0; rk < NUM_RANKS; rk++) {
2243 if (mrc_params->rank_enables & (1 << rk)) {
2245 /* increment "num_ranks_enabled" */
2246 num_ranks_enabled++;
2248 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2250 "WDQ eye rank%d lane%d : %d-%d\n",
2252 delay[L][ch][rk][bl],
2253 delay[R][ch][rk][bl]);
2255 temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2;
2258 final_delay[ch][bl] += temp;
2260 final_delay[ch][bl] / num_ranks_enabled);
2262 set_wdq(ch, rk, bl, temp);
2275 * This function will store relevant timing data
2277 * This data will be used on subsequent boots to speed up boot times
2278 * and is required for Suspend To RAM capabilities.
2280 void store_timings(struct mrc_params *mrc_params)
2283 struct mrc_timings *mt = &mrc_params->timings;
2285 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2286 for (rk = 0; rk < NUM_RANKS; rk++) {
2287 for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
2288 mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl);
2289 mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl);
2290 mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl);
2291 mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl);
2294 mt->vref[ch][bl] = get_vref(ch, bl);
2297 mt->wctl[ch][rk] = get_wctl(ch, rk);
2300 mt->wcmd[ch] = get_wcmd(ch);
2303 /* need to save for a case of changing frequency after warm reset */
2304 mt->ddr_speed = mrc_params->ddr_speed;
2308 * The purpose of this function is to ensure the SEC comes out of reset
2309 * and IA initiates the SEC enabling Memory Scrambling.
2311 void enable_scrambling(struct mrc_params *mrc_params)
2316 if (mrc_params->scrambling_enables == 0)
2321 /* 32 bit seed is always stored in BIOS NVM */
2322 lfsr = mrc_params->timings.scrambler_seed;
2324 if (mrc_params->boot_mode == BM_COLD) {
2326 * factory value is 0 and in first boot,
2327 * a clock based seed is loaded.
2331 * get seed from system clock
2332 * and make sure it is not all 1's
2334 lfsr = rdtsc() & 0x0fffffff;
2337 * Need to replace scrambler
2339 * get next 32bit LFSR 16 times which is the last
2340 * part of the previous scrambler vector
2342 for (i = 0; i < 16; i++)
2347 mrc_params->timings.scrambler_seed = lfsr;
2351 * In warm boot or S3 exit, we have the previous seed.
2352 * In cold boot, we have the last 32bit LFSR which is the new seed.
2354 lfsr32(&lfsr); /* shift to next value */
2355 msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003ffff));
2357 for (i = 0; i < 2; i++)
2358 msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xaaaaaaaa));
2364 * Configure MCU Power Management Control Register
2365 * and Scheduler Control Register
2367 void prog_ddr_control(struct mrc_params *mrc_params)
2374 dsch = msg_port_read(MEM_CTLR, DSCH);
2375 dsch &= ~(DSCH_OOODIS | DSCH_OOOST3DIS | DSCH_NEWBYPDIS);
2376 msg_port_write(MEM_CTLR, DSCH, dsch);
2378 dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2379 dpmc0 &= ~DPMC0_DISPWRDN;
2380 dpmc0 |= (mrc_params->power_down_disable << 25);
2381 dpmc0 &= ~DPMC0_CLKGTDIS;
2382 dpmc0 &= ~DPMC0_PCLSTO_MASK;
2384 dpmc0 |= DPMC0_PREAPWDEN;
2385 msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2387 /* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */
2388 mrc_write_mask(MEM_CTLR, DPMC1, 0x20, 0x30);
2394 * After training complete configure MCU Rank Population Register
2395 * specifying: ranks enabled, device width, density, address mode
2397 void prog_dra_drb(struct mrc_params *mrc_params)
2401 u8 density = mrc_params->params.density;
2405 dco = msg_port_read(MEM_CTLR, DCO);
2407 msg_port_write(MEM_CTLR, DCO, dco);
2410 if (mrc_params->rank_enables & 1)
2412 if (mrc_params->rank_enables & 2)
2414 if (mrc_params->dram_width == X16) {
2420 * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb
2421 * has to be mapped RANKDENSx encoding (0=1Gb)
2426 drp |= ((density - 1) << 6);
2427 drp |= ((density - 1) << 11);
2429 /* Address mode can be overwritten if ECC enabled */
2430 drp |= (mrc_params->address_mode << 14);
2432 msg_port_write(MEM_CTLR, DRP, drp);
2436 msg_port_write(MEM_CTLR, DCO, dco);
2441 /* Send DRAM wake command */
2442 void perform_wake(struct mrc_params *mrc_params)
2446 dram_wake_command();
2452 * Configure refresh rate and short ZQ calibration interval
2453 * Activate dynamic self refresh
2455 void change_refresh_period(struct mrc_params *mrc_params)
2463 drfc = msg_port_read(MEM_CTLR, DRFC);
2464 drfc &= ~DRFC_TREFI_MASK;
2465 drfc |= (mrc_params->refresh_rate << 12);
2466 drfc |= DRFC_REFDBTCLR;
2467 msg_port_write(MEM_CTLR, DRFC, drfc);
2469 dcal = msg_port_read(MEM_CTLR, DCAL);
2470 dcal &= ~DCAL_ZQCINT_MASK;
2471 dcal |= (3 << 8); /* 63ms */
2472 msg_port_write(MEM_CTLR, DCAL, dcal);
2474 dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2475 dpmc0 |= (DPMC0_DYNSREN | DPMC0_ENPHYCLKGATE);
2476 msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2482 * Configure DDRPHY for Auto-Refresh, Periodic Compensations,
2483 * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down
2485 void set_auto_refresh(struct mrc_params *mrc_params)
2490 uint32_t bl_divisor = 1;
2496 * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp,
2497 * ZQSPERIOD, Auto-Precharge, CKE Power-Down
2499 for (channel = 0; channel < NUM_CHANNELS; channel++) {
2500 if (mrc_params->channel_enables & (1 << channel)) {
2501 /* Enable Periodic RCOMPS */
2502 mrc_alt_write_mask(DDRPHY, CMPCTRL, 2, 2);
2504 /* Enable Dynamic DiffAmp & Set Read ODT Value */
2505 switch (mrc_params->rd_odt_value) {
2507 temp = 0x3f; /* OFF */
2510 temp = 0x00; /* Auto */
2514 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
2515 /* Override: DIFFAMP, ODT */
2516 mrc_alt_write_mask(DDRPHY,
2517 B0OVRCTL + bl * DDRIODQ_BL_OFFSET +
2518 channel * DDRIODQ_CH_OFFSET,
2522 /* Override: DIFFAMP, ODT */
2523 mrc_alt_write_mask(DDRPHY,
2524 B1OVRCTL + bl * DDRIODQ_BL_OFFSET +
2525 channel * DDRIODQ_CH_OFFSET,
2530 /* Issue ZQCS command */
2531 for (rank = 0; rank < NUM_RANKS; rank++) {
2532 if (mrc_params->rank_enables & (1 << rank))
2533 dram_init_command(DCMD_ZQCS(rank));
2544 * Depending on configuration enables ECC support
2546 * Available memory size is decreased, and updated with 0s
2547 * in order to clear error status. Address mode 2 forced.
2549 void ecc_enable(struct mrc_params *mrc_params)
2555 if (mrc_params->ecc_enables == 0)
2560 /* Configuration required in ECC mode */
2561 drp = msg_port_read(MEM_CTLR, DRP);
2562 drp &= ~DRP_ADDRMAP_MASK;
2563 drp |= DRP_ADDRMAP_MAP1;
2564 drp |= DRP_PRI64BSPLITEN;
2565 msg_port_write(MEM_CTLR, DRP, drp);
2567 /* Disable new request bypass */
2568 dsch = msg_port_read(MEM_CTLR, DSCH);
2569 dsch |= DSCH_NEWBYPDIS;
2570 msg_port_write(MEM_CTLR, DSCH, dsch);
2573 ecc_ctrl = (DECCCTRL_SBEEN | DECCCTRL_DBEEN | DECCCTRL_ENCBGEN);
2574 msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl);
2576 /* Assume 8 bank memory, one bank is gone for ECC */
2577 mrc_params->mem_size -= mrc_params->mem_size / 8;
2579 /* For S3 resume memory content has to be preserved */
2580 if (mrc_params->boot_mode != BM_S3) {
2582 hte_mem_init(mrc_params, MRC_MEM_INIT);
2590 * Execute memory test
2591 * if error detected it is indicated in mrc_params->status
2593 void memory_test(struct mrc_params *mrc_params)
2595 uint32_t result = 0;
2600 result = hte_mem_init(mrc_params, MRC_MEM_TEST);
2603 DPF(D_INFO, "Memory test result %x\n", result);
2604 mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST);
2608 /* Lock MCU registers at the end of initialization sequence */
2609 void lock_registers(struct mrc_params *mrc_params)
2615 dco = msg_port_read(MEM_CTLR, DCO);
2616 dco &= ~(DCO_PMICTL | DCO_PMIDIS);
2617 dco |= (DCO_DRPLOCK | DCO_CPGCLOCK);
2618 msg_port_write(MEM_CTLR, DCO, dco);