arch/x86/cpu/quark/smc.c

   1 /*
   2  * Copyright (C) 2013, Intel Corporation
   3  * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com>
   4  *
   5  * Ported from Intel released Quark UEFI BIOS
   6  * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei
   7  *
   8  * SPDX-License-Identifier:     Intel
   9  */
  10
  11 #include <common.h>
  12 #include <pci.h>
  13 #include <asm/arch/device.h>
  14 #include <asm/arch/mrc.h>
  15 #include <asm/arch/msg_port.h>
  16 #include "mrc_util.h"
  17 #include "hte.h"
  18 #include "smc.h"
  19
  20 /* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */
  21 static const uint32_t t_ck[3] = {
  22         2500,
  23         1875,
  24         1500
  25 };
  26
  27 /* Global variables */
  28 static const uint16_t ddr_wclk[] = {193, 158};
  29 #ifdef BACKUP_WCTL
  30 static const uint16_t ddr_wctl[] = {1, 217};
  31 #endif
  32 #ifdef BACKUP_WCMD
  33 static const uint16_t ddr_wcmd[] = {1, 220};
  34 #endif
  35
  36 #ifdef BACKUP_RCVN
  37 static const uint16_t ddr_rcvn[] = {129, 498};
  38 #endif
  39
  40 #ifdef BACKUP_WDQS
  41 static const uint16_t ddr_wdqs[] = {65, 289};
  42 #endif
  43
  44 #ifdef BACKUP_RDQS
  45 static const uint8_t ddr_rdqs[] = {32, 24};
  46 #endif
  47
  48 #ifdef BACKUP_WDQ
  49 static const uint16_t ddr_wdq[] = {32, 257};
  50 #endif
  51
  52 /* Stop self refresh driven by MCU */
  53 void clear_self_refresh(struct mrc_params *mrc_params)
  54 {
  55         ENTERFN();
  56
  57         /* clear the PMSTS Channel Self Refresh bits */
  58         mrc_write_mask(MEM_CTLR, PMSTS, PMSTS_DISR, PMSTS_DISR);
  59
  60         LEAVEFN();
  61 }
  62
  63 /* It will initialize timing registers in the MCU (DTR0..DTR4) */
  64 void prog_ddr_timing_control(struct mrc_params *mrc_params)
  65 {
  66         uint8_t tcl, wl;
  67         uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw;
  68         uint32_t tck;
  69         u32 dtr0, dtr1, dtr2, dtr3, dtr4;
  70         u32 tmp1, tmp2;
  71
  72         ENTERFN();
  73
  74         /* mcu_init starts */
  75         mrc_post_code(0x02, 0x00);
  76
  77         dtr0 = msg_port_read(MEM_CTLR, DTR0);
  78         dtr1 = msg_port_read(MEM_CTLR, DTR1);
  79         dtr2 = msg_port_read(MEM_CTLR, DTR2);
  80         dtr3 = msg_port_read(MEM_CTLR, DTR3);
  81         dtr4 = msg_port_read(MEM_CTLR, DTR4);
  82
  83         tck = t_ck[mrc_params->ddr_speed];      /* Clock in picoseconds */
  84         tcl = mrc_params->params.cl;            /* CAS latency in clocks */
  85         trp = tcl;      /* Per CAT MRC */
  86         trcd = tcl;     /* Per CAT MRC */
  87         tras = MCEIL(mrc_params->params.ras, tck);
  88
  89         /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
  90         twr = MCEIL(15000, tck);
  91
  92         twtr = MCEIL(mrc_params->params.wtr, tck);
  93         trrd = MCEIL(mrc_params->params.rrd, tck);
  94         trtp = 4;       /* Valid for 800 and 1066, use 5 for 1333 */
  95         tfaw = MCEIL(mrc_params->params.faw, tck);
  96
  97         wl = 5 + mrc_params->ddr_speed;
  98
  99         dtr0 &= ~DTR0_DFREQ_MASK;
 100         dtr0 |= mrc_params->ddr_speed;
 101         dtr0 &= ~DTR0_TCL_MASK;
 102         tmp1 = tcl - 5;
 103         dtr0 |= ((tcl - 5) << 12);
 104         dtr0 &= ~DTR0_TRP_MASK;
 105         dtr0 |= ((trp - 5) << 4);       /* 5 bit DRAM Clock */
 106         dtr0 &= ~DTR0_TRCD_MASK;
 107         dtr0 |= ((trcd - 5) << 8);      /* 5 bit DRAM Clock */
 108
 109         dtr1 &= ~DTR1_TWCL_MASK;
 110         tmp2 = wl - 3;
 111         dtr1 |= (wl - 3);
 112         dtr1 &= ~DTR1_TWTP_MASK;
 113         dtr1 |= ((wl + 4 + twr - 14) << 8);     /* Change to tWTP */
 114         dtr1 &= ~DTR1_TRTP_MASK;
 115         dtr1 |= ((MMAX(trtp, 4) - 3) << 28);    /* 4 bit DRAM Clock */
 116         dtr1 &= ~DTR1_TRRD_MASK;
 117         dtr1 |= ((trrd - 4) << 24);             /* 4 bit DRAM Clock */
 118         dtr1 &= ~DTR1_TCMD_MASK;
 119         dtr1 |= (1 << 4);
 120         dtr1 &= ~DTR1_TRAS_MASK;
 121         dtr1 |= ((tras - 14) << 20);            /* 6 bit DRAM Clock */
 122         dtr1 &= ~DTR1_TFAW_MASK;
 123         dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */
 124         /* Set 4 Clock CAS to CAS delay (multi-burst) */
 125         dtr1 &= ~DTR1_TCCD_MASK;
 126
 127         dtr2 &= ~DTR2_TRRDR_MASK;
 128         dtr2 |= 1;
 129         dtr2 &= ~DTR2_TWWDR_MASK;
 130         dtr2 |= (2 << 8);
 131         dtr2 &= ~DTR2_TRWDR_MASK;
 132         dtr2 |= (2 << 16);
 133
 134         dtr3 &= ~DTR3_TWRDR_MASK;
 135         dtr3 |= 2;
 136         dtr3 &= ~DTR3_TXXXX_MASK;
 137         dtr3 |= (2 << 4);
 138
 139         dtr3 &= ~DTR3_TRWSR_MASK;
 140         if (mrc_params->ddr_speed == DDRFREQ_800) {
 141                 /* Extended RW delay (+1) */
 142                 dtr3 |= ((tcl - 5 + 1) << 8);
 143         } else if (mrc_params->ddr_speed == DDRFREQ_1066) {
 144                 /* Extended RW delay (+1) */
 145                 dtr3 |= ((tcl - 5 + 1) << 8);
 146         }
 147
 148         dtr3 &= ~DTR3_TWRSR_MASK;
 149         dtr3 |= ((4 + wl + twtr - 11) << 13);
 150
 151         dtr3 &= ~DTR3_TXP_MASK;
 152         if (mrc_params->ddr_speed == DDRFREQ_800)
 153                 dtr3 |= ((MMAX(0, 1 - 1)) << 22);
 154         else
 155                 dtr3 |= ((MMAX(0, 2 - 1)) << 22);
 156
 157         dtr4 &= ~DTR4_WRODTSTRT_MASK;
 158         dtr4 |= 1;
 159         dtr4 &= ~DTR4_WRODTSTOP_MASK;
 160         dtr4 |= (1 << 4);
 161         dtr4 &= ~DTR4_XXXX1_MASK;
 162         dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8);
 163         dtr4 &= ~DTR4_XXXX2_MASK;
 164         dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12);
 165         dtr4 &= ~(DTR4_ODTDIS | DTR4_TRGSTRDIS);
 166
 167         msg_port_write(MEM_CTLR, DTR0, dtr0);
 168         msg_port_write(MEM_CTLR, DTR1, dtr1);
 169         msg_port_write(MEM_CTLR, DTR2, dtr2);
 170         msg_port_write(MEM_CTLR, DTR3, dtr3);
 171         msg_port_write(MEM_CTLR, DTR4, dtr4);
 172
 173         LEAVEFN();
 174 }
 175
 176 /* Configure MCU before jedec init sequence */
 177 void prog_decode_before_jedec(struct mrc_params *mrc_params)
 178 {
 179         u32 drp;
 180         u32 drfc;
 181         u32 dcal;
 182         u32 dsch;
 183         u32 dpmc0;
 184
 185         ENTERFN();
 186
 187         /* Disable power saving features */
 188         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
 189         dpmc0 |= (DPMC0_CLKGTDIS | DPMC0_DISPWRDN);
 190         dpmc0 &= ~DPMC0_PCLSTO_MASK;
 191         dpmc0 &= ~DPMC0_DYNSREN;
 192         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
 193
 194         /* Disable out of order transactions */
 195         dsch = msg_port_read(MEM_CTLR, DSCH);
 196         dsch |= (DSCH_OOODIS | DSCH_NEWBYPDIS);
 197         msg_port_write(MEM_CTLR, DSCH, dsch);
 198
 199         /* Disable issuing the REF command */
 200         drfc = msg_port_read(MEM_CTLR, DRFC);
 201         drfc &= ~DRFC_TREFI_MASK;
 202         msg_port_write(MEM_CTLR, DRFC, drfc);
 203
 204         /* Disable ZQ calibration short */
 205         dcal = msg_port_read(MEM_CTLR, DCAL);
 206         dcal &= ~DCAL_ZQCINT_MASK;
 207         dcal &= ~DCAL_SRXZQCL_MASK;
 208         msg_port_write(MEM_CTLR, DCAL, dcal);
 209
 210         /*
 211          * Training performed in address mode 0, rank population has limited
 212          * impact, however simulator complains if enabled non-existing rank.
 213          */
 214         drp = 0;
 215         if (mrc_params->rank_enables & 1)
 216                 drp |= DRP_RKEN0;
 217         if (mrc_params->rank_enables & 2)
 218                 drp |= DRP_RKEN1;
 219         msg_port_write(MEM_CTLR, DRP, drp);
 220
 221         LEAVEFN();
 222 }
 223
 224 /*
 225  * After Cold Reset, BIOS should set COLDWAKE bit to 1 before
 226  * sending the WAKE message to the Dunit.
 227  *
 228  * For Standby Exit, or any other mode in which the DRAM is in
 229  * SR, this bit must be set to 0.
 230  */
 231 void perform_ddr_reset(struct mrc_params *mrc_params)
 232 {
 233         ENTERFN();
 234
 235         /* Set COLDWAKE bit before sending the WAKE message */
 236         mrc_write_mask(MEM_CTLR, DRMC, DRMC_COLDWAKE, DRMC_COLDWAKE);
 237
 238         /* Send wake command to DUNIT (MUST be done before JEDEC) */
 239         dram_wake_command();
 240
 241         /* Set default value */
 242         msg_port_write(MEM_CTLR, DRMC,
 243                        mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0);
 244
 245         LEAVEFN();
 246 }
 247
 248
 249 /*
 250  * This function performs some initialization on the DDRIO unit.
 251  * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES.
 252  */
 253 void ddrphy_init(struct mrc_params *mrc_params)
 254 {
 255         uint32_t temp;
 256         uint8_t ch;     /* channel counter */
 257         uint8_t rk;     /* rank counter */
 258         uint8_t bl_grp; /*  byte lane group counter (2 BLs per module) */
 259         uint8_t bl_divisor = 1; /* byte lane divisor */
 260         /* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */
 261         uint8_t speed = mrc_params->ddr_speed & 3;
 262         uint8_t cas;
 263         uint8_t cwl;
 264
 265         ENTERFN();
 266
 267         cas = mrc_params->params.cl;
 268         cwl = 5 + mrc_params->ddr_speed;
 269
 270         /* ddrphy_init starts */
 271         mrc_post_code(0x03, 0x00);
 272
 273         /*
 274          * HSD#231531
 275          * Make sure IOBUFACT is deasserted before initializing the DDR PHY
 276          *
 277          * HSD#234845
 278          * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY
 279          */
 280         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 281                 if (mrc_params->channel_enables & (1 << ch)) {
 282                         /* Deassert DDRPHY Initialization Complete */
 283                         mrc_alt_write_mask(DDRPHY,
 284                                 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
 285                                 ~(1 << 20), 1 << 20);   /* SPID_INIT_COMPLETE=0 */
 286                         /* Deassert IOBUFACT */
 287                         mrc_alt_write_mask(DDRPHY,
 288                                 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
 289                                 ~(1 << 2), 1 << 2);     /* IOBUFACTRST_N=0 */
 290                         /* Disable WRPTR */
 291                         mrc_alt_write_mask(DDRPHY,
 292                                 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
 293                                 ~(1 << 0), 1 << 0);     /* WRPTRENABLE=0 */
 294                 }
 295         }
 296
 297         /* Put PHY in reset */
 298         mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, 1);
 299
 300         /* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */
 301
 302         /* STEP0 */
 303         mrc_post_code(0x03, 0x10);
 304         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 305                 if (mrc_params->channel_enables & (1 << ch)) {
 306                         /* DQ01-DQ23 */
 307                         for (bl_grp = 0;
 308                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
 309                              bl_grp++) {
 310                                 /* Analog MUX select - IO2xCLKSEL */
 311                                 mrc_alt_write_mask(DDRPHY,
 312                                         DQOBSCKEBBCTL +
 313                                         bl_grp * DDRIODQ_BL_OFFSET +
 314                                         ch * DDRIODQ_CH_OFFSET,
 315                                         bl_grp ? 0 : (1 << 22), 1 << 22);
 316
 317                                 /* ODT Strength */
 318                                 switch (mrc_params->rd_odt_value) {
 319                                 case 1:
 320                                         temp = 0x3;
 321                                         break;  /* 60 ohm */
 322                                 case 2:
 323                                         temp = 0x3;
 324                                         break;  /* 120 ohm */
 325                                 case 3:
 326                                         temp = 0x3;
 327                                         break;  /* 180 ohm */
 328                                 default:
 329                                         temp = 0x3;
 330                                         break;  /* 120 ohm */
 331                                 }
 332
 333                                 /* ODT strength */
 334                                 mrc_alt_write_mask(DDRPHY,
 335                                         B0RXIOBUFCTL +
 336                                         bl_grp * DDRIODQ_BL_OFFSET +
 337                                         ch * DDRIODQ_CH_OFFSET,
 338                                         temp << 5, 0x60);
 339                                 /* ODT strength */
 340                                 mrc_alt_write_mask(DDRPHY,
 341                                         B1RXIOBUFCTL +
 342                                         bl_grp * DDRIODQ_BL_OFFSET +
 343                                         ch * DDRIODQ_CH_OFFSET,
 344                                         temp << 5, 0x60);
 345
 346                                 /* Dynamic ODT/DIFFAMP */
 347                                 temp = (cas << 24) | (cas << 16) |
 348                                         (cas << 8) | (cas << 0);
 349                                 switch (speed) {
 350                                 case 0:
 351                                         temp -= 0x01010101;
 352                                         break;  /* 800 */
 353                                 case 1:
 354                                         temp -= 0x02020202;
 355                                         break;  /* 1066 */
 356                                 case 2:
 357                                         temp -= 0x03030303;
 358                                         break;  /* 1333 */
 359                                 case 3:
 360                                         temp -= 0x04040404;
 361                                         break;  /* 1600 */
 362                                 }
 363
 364                                 /* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */
 365                                 mrc_alt_write_mask(DDRPHY,
 366                                         B01LATCTL1 +
 367                                         bl_grp * DDRIODQ_BL_OFFSET +
 368                                         ch * DDRIODQ_CH_OFFSET,
 369                                         temp, 0x1f1f1f1f);
 370                                 switch (speed) {
 371                                 /* HSD#234715 */
 372                                 case 0:
 373                                         temp = (0x06 << 16) | (0x07 << 8);
 374                                         break;  /* 800 */
 375                                 case 1:
 376                                         temp = (0x07 << 16) | (0x08 << 8);
 377                                         break;  /* 1066 */
 378                                 case 2:
 379                                         temp = (0x09 << 16) | (0x0a << 8);
 380                                         break;  /* 1333 */
 381                                 case 3:
 382                                         temp = (0x0a << 16) | (0x0b << 8);
 383                                         break;  /* 1600 */
 384                                 }
 385
 386                                 /* On Duration: ODT, DIFFAMP */
 387                                 mrc_alt_write_mask(DDRPHY,
 388                                         B0ONDURCTL +
 389                                         bl_grp * DDRIODQ_BL_OFFSET +
 390                                         ch * DDRIODQ_CH_OFFSET,
 391                                         temp, 0x003f3f00);
 392                                 /* On Duration: ODT, DIFFAMP */
 393                                 mrc_alt_write_mask(DDRPHY,
 394                                         B1ONDURCTL +
 395                                         bl_grp * DDRIODQ_BL_OFFSET +
 396                                         ch * DDRIODQ_CH_OFFSET,
 397                                         temp, 0x003f3f00);
 398
 399                                 switch (mrc_params->rd_odt_value) {
 400                                 case 0:
 401                                         /* override DIFFAMP=on, ODT=off */
 402                                         temp = (0x3f << 16) | (0x3f << 10);
 403                                         break;
 404                                 default:
 405                                         /* override DIFFAMP=on, ODT=on */
 406                                         temp = (0x3f << 16) | (0x2a << 10);
 407                                         break;
 408                                 }
 409
 410                                 /* Override: DIFFAMP, ODT */
 411                                 mrc_alt_write_mask(DDRPHY,
 412                                         B0OVRCTL +
 413                                         bl_grp * DDRIODQ_BL_OFFSET +
 414                                         ch * DDRIODQ_CH_OFFSET,
 415                                         temp, 0x003ffc00);
 416                                 /* Override: DIFFAMP, ODT */
 417                                 mrc_alt_write_mask(DDRPHY,
 418                                         B1OVRCTL +
 419                                         bl_grp * DDRIODQ_BL_OFFSET +
 420                                         ch * DDRIODQ_CH_OFFSET,
 421                                         temp, 0x003ffc00);
 422
 423                                 /* DLL Setup */
 424
 425                                 /* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */
 426                                 mrc_alt_write_mask(DDRPHY,
 427                                         B0LATCTL0 +
 428                                         bl_grp * DDRIODQ_BL_OFFSET +
 429                                         ch * DDRIODQ_CH_OFFSET,
 430                                         ((cas + 7) << 16) | ((cas - 4) << 8) |
 431                                         ((cwl - 2) << 0), 0x003f1f1f);
 432                                 mrc_alt_write_mask(DDRPHY,
 433                                         B1LATCTL0 +
 434                                         bl_grp * DDRIODQ_BL_OFFSET +
 435                                         ch * DDRIODQ_CH_OFFSET,
 436                                         ((cas + 7) << 16) | ((cas - 4) << 8) |
 437                                         ((cwl - 2) << 0), 0x003f1f1f);
 438
 439                                 /* RCVEN Bypass (PO) */
 440                                 mrc_alt_write_mask(DDRPHY,
 441                                         B0RXIOBUFCTL +
 442                                         bl_grp * DDRIODQ_BL_OFFSET +
 443                                         ch * DDRIODQ_CH_OFFSET,
 444                                         0, 0x81);
 445                                 mrc_alt_write_mask(DDRPHY,
 446                                         B1RXIOBUFCTL +
 447                                         bl_grp * DDRIODQ_BL_OFFSET +
 448                                         ch * DDRIODQ_CH_OFFSET,
 449                                         0, 0x81);
 450
 451                                 /* TX */
 452                                 mrc_alt_write_mask(DDRPHY,
 453                                         DQCTL +
 454                                         bl_grp * DDRIODQ_BL_OFFSET +
 455                                         ch * DDRIODQ_CH_OFFSET,
 456                                         1 << 16, 1 << 16);
 457                                 mrc_alt_write_mask(DDRPHY,
 458                                         B01PTRCTL1 +
 459                                         bl_grp * DDRIODQ_BL_OFFSET +
 460                                         ch * DDRIODQ_CH_OFFSET,
 461                                         1 << 8, 1 << 8);
 462
 463                                 /* RX (PO) */
 464                                 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
 465                                 mrc_alt_write_mask(DDRPHY,
 466                                         B0VREFCTL +
 467                                         bl_grp * DDRIODQ_BL_OFFSET +
 468                                         ch * DDRIODQ_CH_OFFSET,
 469                                         (0x03 << 2) | (0x0 << 1) | (0x0 << 0),
 470                                         0xff);
 471                                 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
 472                                 mrc_alt_write_mask(DDRPHY,
 473                                         B1VREFCTL +
 474                                         bl_grp * DDRIODQ_BL_OFFSET +
 475                                         ch * DDRIODQ_CH_OFFSET,
 476                                         (0x03 << 2) | (0x0 << 1) | (0x0 << 0),
 477                                         0xff);
 478                                 /* Per-Bit De-Skew Enable */
 479                                 mrc_alt_write_mask(DDRPHY,
 480                                         B0RXIOBUFCTL +
 481                                         bl_grp * DDRIODQ_BL_OFFSET +
 482                                         ch * DDRIODQ_CH_OFFSET,
 483                                         0, 0x10);
 484                                 /* Per-Bit De-Skew Enable */
 485                                 mrc_alt_write_mask(DDRPHY,
 486                                         B1RXIOBUFCTL +
 487                                         bl_grp * DDRIODQ_BL_OFFSET +
 488                                         ch * DDRIODQ_CH_OFFSET,
 489                                         0, 0x10);
 490                         }
 491
 492                         /* CLKEBB */
 493                         mrc_alt_write_mask(DDRPHY,
 494                                 CMDOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
 495                                 0, 1 << 23);
 496
 497                         /* Enable tristate control of cmd/address bus */
 498                         mrc_alt_write_mask(DDRPHY,
 499                                 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
 500                                 0, 0x03);
 501
 502                         /* ODT RCOMP */
 503                         mrc_alt_write_mask(DDRPHY,
 504                                 CMDRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
 505                                 (0x03 << 5) | (0x03 << 0), 0x3ff);
 506
 507                         /* CMDPM* registers must be programmed in this order */
 508
 509                         /* Turn On Delays: SFR (regulator), MPLL */
 510                         mrc_alt_write_mask(DDRPHY,
 511                                 CMDPMDLYREG4 + ch * DDRIOCCC_CH_OFFSET,
 512                                 0xffffffff, 0xffffffff);
 513                         /*
 514                          * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3,
 515                          * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT
 516                          * for_PM_MSG_gt0, MDLL Turn On
 517                          */
 518                         mrc_alt_write_mask(DDRPHY,
 519                                 CMDPMDLYREG3 + ch * DDRIOCCC_CH_OFFSET,
 520                                 0xfffff616, 0xffffffff);
 521                         /* MPLL Divider Reset Delays */
 522                         mrc_alt_write_mask(DDRPHY,
 523                                 CMDPMDLYREG2 + ch * DDRIOCCC_CH_OFFSET,
 524                                 0xffffffff, 0xffffffff);
 525                         /* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */
 526                         mrc_alt_write_mask(DDRPHY,
 527                                 CMDPMDLYREG1 + ch * DDRIOCCC_CH_OFFSET,
 528                                 0xffffffff, 0xffffffff);
 529                         /* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */
 530                         mrc_alt_write_mask(DDRPHY,
 531                                 CMDPMDLYREG0 + ch * DDRIOCCC_CH_OFFSET,
 532                                 0xffffffff, 0xffffffff);
 533                         /* Allow PUnit signals */
 534                         mrc_alt_write_mask(DDRPHY,
 535                                 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
 536                                 (0x6 << 8) | (0x1 << 6) | (0x4 << 0),
 537                                 0xffe00f4f);
 538                         /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
 539                         mrc_alt_write_mask(DDRPHY,
 540                                 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 541                                 (0x3 << 4) | (0x7 << 0), 0x7f);
 542
 543                         /* CLK-CTL */
 544                         mrc_alt_write_mask(DDRPHY,
 545                                 CCOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET,
 546                                 0, 1 << 24);    /* CLKEBB */
 547                         /* Buffer Enable: CS,CKE,ODT,CLK */
 548                         mrc_alt_write_mask(DDRPHY,
 549                                 CCCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
 550                                 0x1f, 0x000ffff1);
 551                         /* ODT RCOMP */
 552                         mrc_alt_write_mask(DDRPHY,
 553                                 CCRCOMPODT + ch * DDRIOCCC_CH_OFFSET,
 554                                 (0x03 << 8) | (0x03 << 0), 0x00001f1f);
 555                         /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
 556                         mrc_alt_write_mask(DDRPHY,
 557                                 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 558                                 (0x3 << 4) | (0x7 << 0), 0x7f);
 559
 560                         /*
 561                          * COMP (RON channel specific)
 562                          * - DQ/DQS/DM RON: 32 Ohm
 563                          * - CTRL/CMD RON: 27 Ohm
 564                          * - CLK RON: 26 Ohm
 565                          */
 566                         /* RCOMP Vref PU/PD */
 567                         mrc_alt_write_mask(DDRPHY,
 568                                 DQVREFCH0 +  ch * DDRCOMP_CH_OFFSET,
 569                                 (0x08 << 24) | (0x03 << 16), 0x3f3f0000);
 570                         /* RCOMP Vref PU/PD */
 571                         mrc_alt_write_mask(DDRPHY,
 572                                 CMDVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 573                                 (0x0C << 24) | (0x03 << 16), 0x3f3f0000);
 574                         /* RCOMP Vref PU/PD */
 575                         mrc_alt_write_mask(DDRPHY,
 576                                 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 577                                 (0x0F << 24) | (0x03 << 16), 0x3f3f0000);
 578                         /* RCOMP Vref PU/PD */
 579                         mrc_alt_write_mask(DDRPHY,
 580                                 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 581                                 (0x08 << 24) | (0x03 << 16), 0x3f3f0000);
 582                         /* RCOMP Vref PU/PD */
 583                         mrc_alt_write_mask(DDRPHY,
 584                                 CTLVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 585                                 (0x0C << 24) | (0x03 << 16), 0x3f3f0000);
 586
 587                         /* DQS Swapped Input Enable */
 588                         mrc_alt_write_mask(DDRPHY,
 589                                 COMPEN1CH0 + ch * DDRCOMP_CH_OFFSET,
 590                                 (1 << 19) | (1 << 17), 0xc00ac000);
 591
 592                         /* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */
 593                         /* ODT Vref PU/PD */
 594                         mrc_alt_write_mask(DDRPHY,
 595                                 DQVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 596                                 (0x32 << 8) | (0x03 << 0), 0x00003f3f);
 597                         /* ODT Vref PU/PD */
 598                         mrc_alt_write_mask(DDRPHY,
 599                                 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 600                                 (0x32 << 8) | (0x03 << 0), 0x00003f3f);
 601                         /* ODT Vref PU/PD */
 602                         mrc_alt_write_mask(DDRPHY,
 603                                 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 604                                 (0x0E << 8) | (0x05 << 0), 0x00003f3f);
 605
 606                         /*
 607                          * Slew rate settings are frequency specific,
 608                          * numbers below are for 800Mhz (speed == 0)
 609                          * - DQ/DQS/DM/CLK SR: 4V/ns,
 610                          * - CTRL/CMD SR: 1.5V/ns
 611                          */
 612                         temp = (0x0e << 16) | (0x0e << 12) | (0x08 << 8) |
 613                                 (0x0b << 4) | (0x0b << 0);
 614                         /* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */
 615                         mrc_alt_write_mask(DDRPHY,
 616                                 DLYSELCH0 + ch * DDRCOMP_CH_OFFSET,
 617                                 temp, 0x000fffff);
 618                         /* TCO Vref CLK,DQS,DQ */
 619                         mrc_alt_write_mask(DDRPHY,
 620                                 TCOVREFCH0 + ch * DDRCOMP_CH_OFFSET,
 621                                 (0x05 << 16) | (0x05 << 8) | (0x05 << 0),
 622                                 0x003f3f3f);
 623                         /* ODTCOMP CMD/CTL PU/PD */
 624                         mrc_alt_write_mask(DDRPHY,
 625                                 CCBUFODTCH0 + ch * DDRCOMP_CH_OFFSET,
 626                                 (0x03 << 8) | (0x03 << 0),
 627                                 0x00001f1f);
 628                         /* COMP */
 629                         mrc_alt_write_mask(DDRPHY,
 630                                 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
 631                                 0, 0xc0000100);
 632
 633 #ifdef BACKUP_COMPS
 634                         /* DQ COMP Overrides */
 635                         /* RCOMP PU */
 636                         mrc_alt_write_mask(DDRPHY,
 637                                 DQDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 638                                 (1 << 31) | (0x0a << 16),
 639                                 0x801f0000);
 640                         /* RCOMP PD */
 641                         mrc_alt_write_mask(DDRPHY,
 642                                 DQDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 643                                 (1 << 31) | (0x0a << 16),
 644                                 0x801f0000);
 645                         /* DCOMP PU */
 646                         mrc_alt_write_mask(DDRPHY,
 647                                 DQDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 648                                 (1 << 31) | (0x10 << 16),
 649                                 0x801f0000);
 650                         /* DCOMP PD */
 651                         mrc_alt_write_mask(DDRPHY,
 652                                 DQDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 653                                 (1 << 31) | (0x10 << 16),
 654                                 0x801f0000);
 655                         /* ODTCOMP PU */
 656                         mrc_alt_write_mask(DDRPHY,
 657                                 DQODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 658                                 (1 << 31) | (0x0b << 16),
 659                                 0x801f0000);
 660                         /* ODTCOMP PD */
 661                         mrc_alt_write_mask(DDRPHY,
 662                                 DQODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 663                                 (1 << 31) | (0x0b << 16),
 664                                 0x801f0000);
 665                         /* TCOCOMP PU */
 666                         mrc_alt_write_mask(DDRPHY,
 667                                 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 668                                 1 << 31, 1 << 31);
 669                         /* TCOCOMP PD */
 670                         mrc_alt_write_mask(DDRPHY,
 671                                 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 672                                 1 << 31, 1 << 31);
 673
 674                         /* DQS COMP Overrides */
 675                         /* RCOMP PU */
 676                         mrc_alt_write_mask(DDRPHY,
 677                                 DQSDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 678                                 (1 << 31) | (0x0a << 16),
 679                                 0x801f0000);
 680                         /* RCOMP PD */
 681                         mrc_alt_write_mask(DDRPHY,
 682                                 DQSDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 683                                 (1 << 31) | (0x0a << 16),
 684                                 0x801f0000);
 685                         /* DCOMP PU */
 686                         mrc_alt_write_mask(DDRPHY,
 687                                 DQSDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 688                                 (1 << 31) | (0x10 << 16),
 689                                 0x801f0000);
 690                         /* DCOMP PD */
 691                         mrc_alt_write_mask(DDRPHY,
 692                                 DQSDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 693                                 (1 << 31) | (0x10 << 16),
 694                                 0x801f0000);
 695                         /* ODTCOMP PU */
 696                         mrc_alt_write_mask(DDRPHY,
 697                                 DQSODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 698                                 (1 << 31) | (0x0b << 16),
 699                                 0x801f0000);
 700                         /* ODTCOMP PD */
 701                         mrc_alt_write_mask(DDRPHY,
 702                                 DQSODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 703                                 (1 << 31) | (0x0b << 16),
 704                                 0x801f0000);
 705                         /* TCOCOMP PU */
 706                         mrc_alt_write_mask(DDRPHY,
 707                                 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 708                                 1 << 31, 1 << 31);
 709                         /* TCOCOMP PD */
 710                         mrc_alt_write_mask(DDRPHY,
 711                                 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 712                                 1 << 31, 1 << 31);
 713
 714                         /* CLK COMP Overrides */
 715                         /* RCOMP PU */
 716                         mrc_alt_write_mask(DDRPHY,
 717                                 CLKDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 718                                 (1 << 31) | (0x0c << 16),
 719                                 0x801f0000);
 720                         /* RCOMP PD */
 721                         mrc_alt_write_mask(DDRPHY,
 722                                 CLKDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 723                                 (1 << 31) | (0x0c << 16),
 724                                 0x801f0000);
 725                         /* DCOMP PU */
 726                         mrc_alt_write_mask(DDRPHY,
 727                                 CLKDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 728                                 (1 << 31) | (0x07 << 16),
 729                                 0x801f0000);
 730                         /* DCOMP PD */
 731                         mrc_alt_write_mask(DDRPHY,
 732                                 CLKDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 733                                 (1 << 31) | (0x07 << 16),
 734                                 0x801f0000);
 735                         /* ODTCOMP PU */
 736                         mrc_alt_write_mask(DDRPHY,
 737                                 CLKODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 738                                 (1 << 31) | (0x0b << 16),
 739                                 0x801f0000);
 740                         /* ODTCOMP PD */
 741                         mrc_alt_write_mask(DDRPHY,
 742                                 CLKODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 743                                 (1 << 31) | (0x0b << 16),
 744                                 0x801f0000);
 745                         /* TCOCOMP PU */
 746                         mrc_alt_write_mask(DDRPHY,
 747                                 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 748                                 1 << 31, 1 << 31);
 749                         /* TCOCOMP PD */
 750                         mrc_alt_write_mask(DDRPHY,
 751                                 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 752                                 1 << 31, 1 << 31);
 753
 754                         /* CMD COMP Overrides */
 755                         /* RCOMP PU */
 756                         mrc_alt_write_mask(DDRPHY,
 757                                 CMDDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 758                                 (1 << 31) | (0x0d << 16),
 759                                 0x803f0000);
 760                         /* RCOMP PD */
 761                         mrc_alt_write_mask(DDRPHY,
 762                                 CMDDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 763                                 (1 << 31) | (0x0d << 16),
 764                                 0x803f0000);
 765                         /* DCOMP PU */
 766                         mrc_alt_write_mask(DDRPHY,
 767                                 CMDDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 768                                 (1 << 31) | (0x0a << 16),
 769                                 0x801f0000);
 770                         /* DCOMP PD */
 771                         mrc_alt_write_mask(DDRPHY,
 772                                 CMDDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 773                                 (1 << 31) | (0x0a << 16),
 774                                 0x801f0000);
 775
 776                         /* CTL COMP Overrides */
 777                         /* RCOMP PU */
 778                         mrc_alt_write_mask(DDRPHY,
 779                                 CTLDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 780                                 (1 << 31) | (0x0d << 16),
 781                                 0x803f0000);
 782                         /* RCOMP PD */
 783                         mrc_alt_write_mask(DDRPHY,
 784                                 CTLDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 785                                 (1 << 31) | (0x0d << 16),
 786                                 0x803f0000);
 787                         /* DCOMP PU */
 788                         mrc_alt_write_mask(DDRPHY,
 789                                 CTLDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 790                                 (1 << 31) | (0x0a << 16),
 791                                 0x801f0000);
 792                         /* DCOMP PD */
 793                         mrc_alt_write_mask(DDRPHY,
 794                                 CTLDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 795                                 (1 << 31) | (0x0a << 16),
 796                                 0x801f0000);
 797 #else
 798                         /* DQ TCOCOMP Overrides */
 799                         /* TCOCOMP PU */
 800                         mrc_alt_write_mask(DDRPHY,
 801                                 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 802                                 (1 << 31) | (0x1f << 16),
 803                                 0x801f0000);
 804                         /* TCOCOMP PD */
 805                         mrc_alt_write_mask(DDRPHY,
 806                                 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 807                                 (1 << 31) | (0x1f << 16),
 808                                 0x801f0000);
 809
 810                         /* DQS TCOCOMP Overrides */
 811                         /* TCOCOMP PU */
 812                         mrc_alt_write_mask(DDRPHY,
 813                                 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 814                                 (1 << 31) | (0x1f << 16),
 815                                 0x801f0000);
 816                         /* TCOCOMP PD */
 817                         mrc_alt_write_mask(DDRPHY,
 818                                 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 819                                 (1 << 31) | (0x1f << 16),
 820                                 0x801f0000);
 821
 822                         /* CLK TCOCOMP Overrides */
 823                         /* TCOCOMP PU */
 824                         mrc_alt_write_mask(DDRPHY,
 825                                 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 826                                 (1 << 31) | (0x1f << 16),
 827                                 0x801f0000);
 828                         /* TCOCOMP PD */
 829                         mrc_alt_write_mask(DDRPHY,
 830                                 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET,
 831                                 (1 << 31) | (0x1f << 16),
 832                                 0x801f0000);
 833 #endif
 834
 835                         /* program STATIC delays */
 836 #ifdef BACKUP_WCMD
 837                         set_wcmd(ch, ddr_wcmd[PLATFORM_ID]);
 838 #else
 839                         set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK);
 840 #endif
 841
 842                         for (rk = 0; rk < NUM_RANKS; rk++) {
 843                                 if (mrc_params->rank_enables & (1 << rk)) {
 844                                         set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]);
 845 #ifdef BACKUP_WCTL
 846                                         set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]);
 847 #else
 848                                         set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK);
 849 #endif
 850                                 }
 851                         }
 852                 }
 853         }
 854
 855         /* COMP (non channel specific) */
 856         /* RCOMP: Dither PU Enable */
 857         mrc_alt_write_mask(DDRPHY, DQANADRVPUCTL, 1 << 30, 1 << 30);
 858         /* RCOMP: Dither PD Enable */
 859         mrc_alt_write_mask(DDRPHY, DQANADRVPDCTL, 1 << 30, 1 << 30);
 860         /* RCOMP: Dither PU Enable */
 861         mrc_alt_write_mask(DDRPHY, CMDANADRVPUCTL, 1 << 30, 1 << 30);
 862         /* RCOMP: Dither PD Enable */
 863         mrc_alt_write_mask(DDRPHY, CMDANADRVPDCTL, 1 << 30, 1 << 30);
 864         /* RCOMP: Dither PU Enable */
 865         mrc_alt_write_mask(DDRPHY, CLKANADRVPUCTL, 1 << 30, 1 << 30);
 866         /* RCOMP: Dither PD Enable */
 867         mrc_alt_write_mask(DDRPHY, CLKANADRVPDCTL, 1 << 30, 1 << 30);
 868         /* RCOMP: Dither PU Enable */
 869         mrc_alt_write_mask(DDRPHY, DQSANADRVPUCTL, 1 << 30, 1 << 30);
 870         /* RCOMP: Dither PD Enable */
 871         mrc_alt_write_mask(DDRPHY, DQSANADRVPDCTL, 1 << 30, 1 << 30);
 872         /* RCOMP: Dither PU Enable */
 873         mrc_alt_write_mask(DDRPHY, CTLANADRVPUCTL, 1 << 30, 1 << 30);
 874         /* RCOMP: Dither PD Enable */
 875         mrc_alt_write_mask(DDRPHY, CTLANADRVPDCTL, 1 << 30, 1 << 30);
 876         /* ODT: Dither PU Enable */
 877         mrc_alt_write_mask(DDRPHY, DQANAODTPUCTL, 1 << 30, 1 << 30);
 878         /* ODT: Dither PD Enable */
 879         mrc_alt_write_mask(DDRPHY, DQANAODTPDCTL, 1 << 30, 1 << 30);
 880         /* ODT: Dither PU Enable */
 881         mrc_alt_write_mask(DDRPHY, CLKANAODTPUCTL, 1 << 30, 1 << 30);
 882         /* ODT: Dither PD Enable */
 883         mrc_alt_write_mask(DDRPHY, CLKANAODTPDCTL, 1 << 30, 1 << 30);
 884         /* ODT: Dither PU Enable */
 885         mrc_alt_write_mask(DDRPHY, DQSANAODTPUCTL, 1 << 30, 1 << 30);
 886         /* ODT: Dither PD Enable */
 887         mrc_alt_write_mask(DDRPHY, DQSANAODTPDCTL, 1 << 30, 1 << 30);
 888         /* DCOMP: Dither PU Enable */
 889         mrc_alt_write_mask(DDRPHY, DQANADLYPUCTL, 1 << 30, 1 << 30);
 890         /* DCOMP: Dither PD Enable */
 891         mrc_alt_write_mask(DDRPHY, DQANADLYPDCTL, 1 << 30, 1 << 30);
 892         /* DCOMP: Dither PU Enable */
 893         mrc_alt_write_mask(DDRPHY, CMDANADLYPUCTL, 1 << 30, 1 << 30);
 894         /* DCOMP: Dither PD Enable */
 895         mrc_alt_write_mask(DDRPHY, CMDANADLYPDCTL, 1 << 30, 1 << 30);
 896         /* DCOMP: Dither PU Enable */
 897         mrc_alt_write_mask(DDRPHY, CLKANADLYPUCTL, 1 << 30, 1 << 30);
 898         /* DCOMP: Dither PD Enable */
 899         mrc_alt_write_mask(DDRPHY, CLKANADLYPDCTL, 1 << 30, 1 << 30);
 900         /* DCOMP: Dither PU Enable */
 901         mrc_alt_write_mask(DDRPHY, DQSANADLYPUCTL, 1 << 30, 1 << 30);
 902         /* DCOMP: Dither PD Enable */
 903         mrc_alt_write_mask(DDRPHY, DQSANADLYPDCTL, 1 << 30, 1 << 30);
 904         /* DCOMP: Dither PU Enable */
 905         mrc_alt_write_mask(DDRPHY, CTLANADLYPUCTL, 1 << 30, 1 << 30);
 906         /* DCOMP: Dither PD Enable */
 907         mrc_alt_write_mask(DDRPHY, CTLANADLYPDCTL, 1 << 30, 1 << 30);
 908         /* TCO: Dither PU Enable */
 909         mrc_alt_write_mask(DDRPHY, DQANATCOPUCTL, 1 << 30, 1 << 30);
 910         /* TCO: Dither PD Enable */
 911         mrc_alt_write_mask(DDRPHY, DQANATCOPDCTL, 1 << 30, 1 << 30);
 912         /* TCO: Dither PU Enable */
 913         mrc_alt_write_mask(DDRPHY, CLKANATCOPUCTL, 1 << 30, 1 << 30);
 914         /* TCO: Dither PD Enable */
 915         mrc_alt_write_mask(DDRPHY, CLKANATCOPDCTL, 1 << 30, 1 << 30);
 916         /* TCO: Dither PU Enable */
 917         mrc_alt_write_mask(DDRPHY, DQSANATCOPUCTL, 1 << 30, 1 << 30);
 918         /* TCO: Dither PD Enable */
 919         mrc_alt_write_mask(DDRPHY, DQSANATCOPDCTL, 1 << 30, 1 << 30);
 920         /* TCOCOMP: Pulse Count */
 921         mrc_alt_write_mask(DDRPHY, TCOCNTCTRL, 1, 3);
 922         /* ODT: CMD/CTL PD/PU */
 923         mrc_alt_write_mask(DDRPHY, CHNLBUFSTATIC,
 924                 (0x03 << 24) | (0x03 << 16), 0x1f1f0000);
 925         /* Set 1us counter */
 926         mrc_alt_write_mask(DDRPHY, MSCNTR, 0x64, 0xff);
 927         mrc_alt_write_mask(DDRPHY, LATCH1CTL, 0x1 << 28, 0x70000000);
 928
 929         /* Release PHY from reset */
 930         mrc_alt_write_mask(DDRPHY, MASTERRSTN, 1, 1);
 931
 932         /* STEP1 */
 933         mrc_post_code(0x03, 0x11);
 934
 935         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 936                 if (mrc_params->channel_enables & (1 << ch)) {
 937                         /* DQ01-DQ23 */
 938                         for (bl_grp = 0;
 939                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
 940                              bl_grp++) {
 941                                 mrc_alt_write_mask(DDRPHY,
 942                                         DQMDLLCTL +
 943                                         bl_grp * DDRIODQ_BL_OFFSET +
 944                                         ch * DDRIODQ_CH_OFFSET,
 945                                         1 << 13,
 946                                         1 << 13);       /* Enable VREG */
 947                                 delay_n(3);
 948                         }
 949
 950                         /* ECC */
 951                         mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
 952                                 1 << 13, 1 << 13);      /* Enable VREG */
 953                         delay_n(3);
 954                         /* CMD */
 955                         mrc_alt_write_mask(DDRPHY,
 956                                 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 957                                 1 << 13, 1 << 13);      /* Enable VREG */
 958                         delay_n(3);
 959                         /* CLK-CTL */
 960                         mrc_alt_write_mask(DDRPHY,
 961                                 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 962                                 1 << 13, 1 << 13);      /* Enable VREG */
 963                         delay_n(3);
 964                 }
 965         }
 966
 967         /* STEP2 */
 968         mrc_post_code(0x03, 0x12);
 969         delay_n(200);
 970
 971         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 972                 if (mrc_params->channel_enables & (1 << ch)) {
 973                         /* DQ01-DQ23 */
 974                         for (bl_grp = 0;
 975                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
 976                              bl_grp++) {
 977                                 mrc_alt_write_mask(DDRPHY,
 978                                         DQMDLLCTL +
 979                                         bl_grp * DDRIODQ_BL_OFFSET +
 980                                         ch * DDRIODQ_CH_OFFSET,
 981                                         1 << 17,
 982                                         1 << 17);       /* Enable MCDLL */
 983                                 delay_n(50);
 984                         }
 985
 986                 /* ECC */
 987                 mrc_alt_write_mask(DDRPHY, ECCMDLLCTL,
 988                         1 << 17, 1 << 17);      /* Enable MCDLL */
 989                 delay_n(50);
 990                 /* CMD */
 991                 mrc_alt_write_mask(DDRPHY,
 992                         CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 993                         1 << 18, 1 << 18);      /* Enable MCDLL */
 994                 delay_n(50);
 995                 /* CLK-CTL */
 996                 mrc_alt_write_mask(DDRPHY,
 997                         CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET,
 998                         1 << 18, 1 << 18);      /* Enable MCDLL */
 999                 delay_n(50);
1000                 }
1001         }
1002
1003         /* STEP3: */
1004         mrc_post_code(0x03, 0x13);
1005         delay_n(100);
1006
1007         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1008                 if (mrc_params->channel_enables & (1 << ch)) {
1009                         /* DQ01-DQ23 */
1010                         for (bl_grp = 0;
1011                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1012                              bl_grp++) {
1013 #ifdef FORCE_16BIT_DDRIO
1014                                 temp = (bl_grp &&
1015                                         (mrc_params->channel_width == X16)) ?
1016                                         0x11ff : 0xffff;
1017 #else
1018                                 temp = 0xffff;
1019 #endif
1020                                 /* Enable TXDLL */
1021                                 mrc_alt_write_mask(DDRPHY,
1022                                         DQDLLTXCTL +
1023                                         bl_grp * DDRIODQ_BL_OFFSET +
1024                                         ch * DDRIODQ_CH_OFFSET,
1025                                         temp, 0xffff);
1026                                 delay_n(3);
1027                                 /* Enable RXDLL */
1028                                 mrc_alt_write_mask(DDRPHY,
1029                                         DQDLLRXCTL +
1030                                         bl_grp * DDRIODQ_BL_OFFSET +
1031                                         ch * DDRIODQ_CH_OFFSET,
1032                                         0xf, 0xf);
1033                                 delay_n(3);
1034                                 /* Enable RXDLL Overrides BL0 */
1035                                 mrc_alt_write_mask(DDRPHY,
1036                                         B0OVRCTL +
1037                                         bl_grp * DDRIODQ_BL_OFFSET +
1038                                         ch * DDRIODQ_CH_OFFSET,
1039                                         0xf, 0xf);
1040                         }
1041
1042                         /* ECC */
1043                         temp = 0xffff;
1044                         mrc_alt_write_mask(DDRPHY, ECCDLLTXCTL,
1045                                 temp, 0xffff);
1046                         delay_n(3);
1047
1048                         /* CMD (PO) */
1049                         mrc_alt_write_mask(DDRPHY,
1050                                 CMDDLLTXCTL + ch * DDRIOCCC_CH_OFFSET,
1051                                 temp, 0xffff);
1052                         delay_n(3);
1053                 }
1054         }
1055
1056         /* STEP4 */
1057         mrc_post_code(0x03, 0x14);
1058
1059         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1060                 if (mrc_params->channel_enables & (1 << ch)) {
1061                         /* Host To Memory Clock Alignment (HMC) for 800/1066 */
1062                         for (bl_grp = 0;
1063                              bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2;
1064                              bl_grp++) {
1065                                 /* CLK_ALIGN_MOD_ID */
1066                                 mrc_alt_write_mask(DDRPHY,
1067                                         DQCLKALIGNREG2 +
1068                                         bl_grp * DDRIODQ_BL_OFFSET +
1069                                         ch * DDRIODQ_CH_OFFSET,
1070                                         bl_grp ? 3 : 1,
1071                                         0xf);
1072                         }
1073
1074                         mrc_alt_write_mask(DDRPHY,
1075                                 ECCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1076                                 0x2, 0xf);
1077                         mrc_alt_write_mask(DDRPHY,
1078                                 CMDCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1079                                 0x0, 0xf);
1080                         mrc_alt_write_mask(DDRPHY,
1081                                 CCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET,
1082                                 0x2, 0xf);
1083                         mrc_alt_write_mask(DDRPHY,
1084                                 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1085                                 0x20, 0x30);
1086                         /*
1087                          * NUM_SAMPLES, MAX_SAMPLES,
1088                          * MACRO_PI_STEP, MICRO_PI_STEP
1089                          */
1090                         mrc_alt_write_mask(DDRPHY,
1091                                 CMDCLKALIGNREG1 + ch * DDRIOCCC_CH_OFFSET,
1092                                 (0x18 << 16) | (0x10 << 8) |
1093                                 (0x8 << 2) | (0x1 << 0),
1094                                 0x007f7fff);
1095                         /* TOTAL_NUM_MODULES, FIRST_U_PARTITION */
1096                         mrc_alt_write_mask(DDRPHY,
1097                                 CMDCLKALIGNREG2 + ch * DDRIOCCC_CH_OFFSET,
1098                                 (0x10 << 16) | (0x4 << 8) | (0x2 << 4),
1099                                 0x001f0ff0);
1100 #ifdef HMC_TEST
1101                         /* START_CLK_ALIGN=1 */
1102                         mrc_alt_write_mask(DDRPHY,
1103                                 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET,
1104                                 1 << 24, 1 << 24);
1105                         while (msg_port_alt_read(DDRPHY,
1106                                 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET) &
1107                                 (1 << 24))
1108                                 ;       /* wait for START_CLK_ALIGN=0 */
1109 #endif
1110
1111                         /* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */
1112                         mrc_alt_write_mask(DDRPHY,
1113                                 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET,
1114                                 1, 1);  /* WRPTRENABLE=1 */
1115
1116                         /* COMP initial */
1117                         /* enable bypass for CLK buffer (PO) */
1118                         mrc_alt_write_mask(DDRPHY,
1119                                 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1120                                 1 << 5, 1 << 5);
1121                         /* Initial COMP Enable */
1122                         mrc_alt_write_mask(DDRPHY, CMPCTRL, 1, 1);
1123                         /* wait for Initial COMP Enable = 0 */
1124                         while (msg_port_alt_read(DDRPHY, CMPCTRL) & 1)
1125                                 ;
1126                         /* disable bypass for CLK buffer (PO) */
1127                         mrc_alt_write_mask(DDRPHY,
1128                                 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET,
1129                                 ~(1 << 5), 1 << 5);
1130
1131                         /* IOBUFACT */
1132
1133                         /* STEP4a */
1134                         mrc_alt_write_mask(DDRPHY,
1135                                 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET,
1136                                 1 << 2, 1 << 2);        /* IOBUFACTRST_N=1 */
1137
1138                         /* DDRPHY initialization complete */
1139                         mrc_alt_write_mask(DDRPHY,
1140                                 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET,
1141                                 1 << 20, 1 << 20);      /* SPID_INIT_COMPLETE=1 */
1142                 }
1143         }
1144
1145         LEAVEFN();
1146 }
1147
1148 /* This function performs JEDEC initialization on all enabled channels */
1149 void perform_jedec_init(struct mrc_params *mrc_params)
1150 {
1151         uint8_t twr, wl, rank;
1152         uint32_t tck;
1153         u32 dtr0;
1154         u32 drp;
1155         u32 drmc;
1156         u32 mrs0_cmd = 0;
1157         u32 emrs1_cmd = 0;
1158         u32 emrs2_cmd = 0;
1159         u32 emrs3_cmd = 0;
1160
1161         ENTERFN();
1162
1163         /* jedec_init starts */
1164         mrc_post_code(0x04, 0x00);
1165
1166         /* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */
1167         mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 2, 0x102);
1168
1169         /* Assert RESET# for 200us */
1170         delay_u(200);
1171
1172         /* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */
1173         mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 0x100, 0x102);
1174
1175         dtr0 = msg_port_read(MEM_CTLR, DTR0);
1176
1177         /*
1178          * Set CKEVAL for populated ranks
1179          * then send NOP to each rank (#4550197)
1180          */
1181
1182         drp = msg_port_read(MEM_CTLR, DRP);
1183         drp &= 0x3;
1184
1185         drmc = msg_port_read(MEM_CTLR, DRMC);
1186         drmc &= 0xfffffffc;
1187         drmc |= (DRMC_CKEMODE | drp);
1188
1189         msg_port_write(MEM_CTLR, DRMC, drmc);
1190
1191         for (rank = 0; rank < NUM_RANKS; rank++) {
1192                 /* Skip to next populated rank */
1193                 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1194                         continue;
1195
1196                 dram_init_command(DCMD_NOP(rank));
1197         }
1198
1199         msg_port_write(MEM_CTLR, DRMC,
1200                 (mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0));
1201
1202         /*
1203          * setup for emrs 2
1204          * BIT[15:11] --> Always "0"
1205          * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0)
1206          * BIT[08]    --> Always "0"
1207          * BIT[07]    --> SRT: use sr_temp_range
1208          * BIT[06]    --> ASR: want "Manual SR Reference" (0)
1209          * BIT[05:03] --> CWL: use oem_tCWL
1210          * BIT[02:00] --> PASR: want "Full Array" (0)
1211          */
1212         emrs2_cmd |= (2 << 3);
1213         wl = 5 + mrc_params->ddr_speed;
1214         emrs2_cmd |= ((wl - 5) << 9);
1215         emrs2_cmd |= (mrc_params->sr_temp_range << 13);
1216
1217         /*
1218          * setup for emrs 3
1219          * BIT[15:03] --> Always "0"
1220          * BIT[02]    --> MPR: want "Normal Operation" (0)
1221          * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0)
1222          */
1223         emrs3_cmd |= (3 << 3);
1224
1225         /*
1226          * setup for emrs 1
1227          * BIT[15:13]     --> Always "0"
1228          * BIT[12:12]     --> Qoff: want "Output Buffer Enabled" (0)
1229          * BIT[11:11]     --> TDQS: want "Disabled" (0)
1230          * BIT[10:10]     --> Always "0"
1231          * BIT[09,06,02]  --> Rtt_nom: use rtt_nom_value
1232          * BIT[08]        --> Always "0"
1233          * BIT[07]        --> WR_LVL: want "Disabled" (0)
1234          * BIT[05,01]     --> DIC: use ron_value
1235          * BIT[04:03]     --> AL: additive latency want "0" (0)
1236          * BIT[00]        --> DLL: want "Enable" (0)
1237          *
1238          * (BIT5|BIT1) set Ron value
1239          * 00 --> RZQ/6 (40ohm)
1240          * 01 --> RZQ/7 (34ohm)
1241          * 1* --> RESERVED
1242          *
1243          * (BIT9|BIT6|BIT2) set Rtt_nom value
1244          * 000 --> Disabled
1245          * 001 --> RZQ/4 ( 60ohm)
1246          * 010 --> RZQ/2 (120ohm)
1247          * 011 --> RZQ/6 ( 40ohm)
1248          * 1** --> RESERVED
1249          */
1250         emrs1_cmd |= (1 << 3);
1251         emrs1_cmd &= ~(1 << 6);
1252
1253         if (mrc_params->ron_value == 0)
1254                 emrs1_cmd |= (1 << 7);
1255         else
1256                 emrs1_cmd &= ~(1 << 7);
1257
1258         if (mrc_params->rtt_nom_value == 0)
1259                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6);
1260         else if (mrc_params->rtt_nom_value == 1)
1261                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6);
1262         else if (mrc_params->rtt_nom_value == 2)
1263                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6);
1264
1265         /* save MRS1 value (excluding control fields) */
1266         mrc_params->mrs1 = emrs1_cmd >> 6;
1267
1268         /*
1269          * setup for mrs 0
1270          * BIT[15:13]     --> Always "0"
1271          * BIT[12]        --> PPD: for Quark (1)
1272          * BIT[11:09]     --> WR: use oem_tWR
1273          * BIT[08]        --> DLL: want "Reset" (1, self clearing)
1274          * BIT[07]        --> MODE: want "Normal" (0)
1275          * BIT[06:04,02]  --> CL: use oem_tCAS
1276          * BIT[03]        --> RD_BURST_TYPE: want "Interleave" (1)
1277          * BIT[01:00]     --> BL: want "8 Fixed" (0)
1278          * WR:
1279          * 0 --> 16
1280          * 1 --> 5
1281          * 2 --> 6
1282          * 3 --> 7
1283          * 4 --> 8
1284          * 5 --> 10
1285          * 6 --> 12
1286          * 7 --> 14
1287          * CL:
1288          * BIT[02:02] "0" if oem_tCAS <= 11 (1866?)
1289          * BIT[06:04] use oem_tCAS-4
1290          */
1291         mrs0_cmd |= (1 << 14);
1292         mrs0_cmd |= (1 << 18);
1293         mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10);
1294
1295         tck = t_ck[mrc_params->ddr_speed];
1296         /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
1297         twr = MCEIL(15000, tck);
1298         mrs0_cmd |= ((twr - 4) << 15);
1299
1300         for (rank = 0; rank < NUM_RANKS; rank++) {
1301                 /* Skip to next populated rank */
1302                 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1303                         continue;
1304
1305                 emrs2_cmd |= (rank << 22);
1306                 dram_init_command(emrs2_cmd);
1307
1308                 emrs3_cmd |= (rank << 22);
1309                 dram_init_command(emrs3_cmd);
1310
1311                 emrs1_cmd |= (rank << 22);
1312                 dram_init_command(emrs1_cmd);
1313
1314                 mrs0_cmd |= (rank << 22);
1315                 dram_init_command(mrs0_cmd);
1316
1317                 dram_init_command(DCMD_ZQCL(rank));
1318         }
1319
1320         LEAVEFN();
1321 }
1322
1323 /*
1324  * Dunit Initialization Complete
1325  *
1326  * Indicates that initialization of the Dunit has completed.
1327  *
1328  * Memory accesses are permitted and maintenance operation begins.
1329  * Until this bit is set to a 1, the memory controller will not accept
1330  * DRAM requests from the MEMORY_MANAGER or HTE.
1331  */
1332 void set_ddr_init_complete(struct mrc_params *mrc_params)
1333 {
1334         u32 dco;
1335
1336         ENTERFN();
1337
1338         dco = msg_port_read(MEM_CTLR, DCO);
1339         dco &= ~DCO_PMICTL;
1340         dco |= DCO_IC;
1341         msg_port_write(MEM_CTLR, DCO, dco);
1342
1343         LEAVEFN();
1344 }
1345
1346 /*
1347  * This function will retrieve relevant timing data
1348  *
1349  * This data will be used on subsequent boots to speed up boot times
1350  * and is required for Suspend To RAM capabilities.
1351  */
1352 void restore_timings(struct mrc_params *mrc_params)
1353 {
1354         uint8_t ch, rk, bl;
1355         const struct mrc_timings *mt = &mrc_params->timings;
1356
1357         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1358                 for (rk = 0; rk < NUM_RANKS; rk++) {
1359                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1360                                 set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]);
1361                                 set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]);
1362                                 set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]);
1363                                 set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]);
1364                                 if (rk == 0) {
1365                                         /* VREF (RANK0 only) */
1366                                         set_vref(ch, bl, mt->vref[ch][bl]);
1367                                 }
1368                         }
1369                         set_wctl(ch, rk, mt->wctl[ch][rk]);
1370                 }
1371                 set_wcmd(ch, mt->wcmd[ch]);
1372         }
1373 }
1374
1375 /*
1376  * Configure default settings normally set as part of read training
1377  *
1378  * Some defaults have to be set earlier as they may affect earlier
1379  * training steps.
1380  */
1381 void default_timings(struct mrc_params *mrc_params)
1382 {
1383         uint8_t ch, rk, bl;
1384
1385         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1386                 for (rk = 0; rk < NUM_RANKS; rk++) {
1387                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1388                                 set_rdqs(ch, rk, bl, 24);
1389                                 if (rk == 0) {
1390                                         /* VREF (RANK0 only) */
1391                                         set_vref(ch, bl, 32);
1392                                 }
1393                         }
1394                 }
1395         }
1396 }
1397
1398 /*
1399  * This function will perform our RCVEN Calibration Algorithm.
1400  * We will only use the 2xCLK domain timings to perform RCVEN Calibration.
1401  * All byte lanes will be calibrated "simultaneously" per channel per rank.
1402  */
1403 void rcvn_cal(struct mrc_params *mrc_params)
1404 {
1405         uint8_t ch;     /* channel counter */
1406         uint8_t rk;     /* rank counter */
1407         uint8_t bl;     /* byte lane counter */
1408         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1409
1410 #ifdef R2R_SHARING
1411         /* used to find placement for rank2rank sharing configs */
1412         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1413 #ifndef BACKUP_RCVN
1414         /* used to find placement for rank2rank sharing configs */
1415         uint32_t num_ranks_enabled = 0;
1416 #endif
1417 #endif
1418
1419 #ifdef BACKUP_RCVN
1420 #else
1421         uint32_t temp;
1422         /* absolute PI value to be programmed on the byte lane */
1423         uint32_t delay[NUM_BYTE_LANES];
1424         u32 dtr1, dtr1_save;
1425 #endif
1426
1427         ENTERFN();
1428
1429         /* rcvn_cal starts */
1430         mrc_post_code(0x05, 0x00);
1431
1432 #ifndef BACKUP_RCVN
1433         /* need separate burst to sample DQS preamble */
1434         dtr1 = msg_port_read(MEM_CTLR, DTR1);
1435         dtr1_save = dtr1;
1436         dtr1 |= DTR1_TCCD_12CLK;
1437         msg_port_write(MEM_CTLR, DTR1, dtr1);
1438 #endif
1439
1440 #ifdef R2R_SHARING
1441         /* need to set "final_delay[][]" elements to "0" */
1442         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1443 #endif
1444
1445         /* loop through each enabled channel */
1446         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1447                 if (mrc_params->channel_enables & (1 << ch)) {
1448                         /* perform RCVEN Calibration on a per rank basis */
1449                         for (rk = 0; rk < NUM_RANKS; rk++) {
1450                                 if (mrc_params->rank_enables & (1 << rk)) {
1451                                         /*
1452                                          * POST_CODE here indicates the current
1453                                          * channel and rank being calibrated
1454                                          */
1455                                         mrc_post_code(0x05, 0x10 + ((ch << 4) | rk));
1456
1457 #ifdef BACKUP_RCVN
1458                                         /* et hard-coded timing values */
1459                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++)
1460                                                 set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]);
1461 #else
1462                                         /* enable FIFORST */
1463                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1464                                                 mrc_alt_write_mask(DDRPHY,
1465                                                         B01PTRCTL1 +
1466                                                         (bl >> 1) * DDRIODQ_BL_OFFSET +
1467                                                         ch * DDRIODQ_CH_OFFSET,
1468                                                         0, 1 << 8);
1469                                         }
1470                                         /* initialize the starting delay to 128 PI (cas +1 CLK) */
1471                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1472                                                 /* 1x CLK domain timing is cas-4 */
1473                                                 delay[bl] = (4 + 1) * FULL_CLK;
1474
1475                                                 set_rcvn(ch, rk, bl, delay[bl]);
1476                                         }
1477
1478                                         /* now find the rising edge */
1479                                         find_rising_edge(mrc_params, delay, ch, rk, true);
1480
1481                                         /* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */
1482                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1483                                                 delay[bl] += QRTR_CLK;
1484                                                 set_rcvn(ch, rk, bl, delay[bl]);
1485                                         }
1486                                         /* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */
1487                                         do {
1488                                                 temp = sample_dqs(mrc_params, ch, rk, true);
1489                                                 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1490                                                         if (temp & (1 << bl)) {
1491                                                                 if (delay[bl] >= FULL_CLK) {
1492                                                                         delay[bl] -= FULL_CLK;
1493                                                                         set_rcvn(ch, rk, bl, delay[bl]);
1494                                                                 } else {
1495                                                                         /* not enough delay */
1496                                                                         training_message(ch, rk, bl);
1497                                                                         mrc_post_code(0xee, 0x50);
1498                                                                 }
1499                                                         }
1500                                                 }
1501                                         } while (temp & 0xff);
1502
1503 #ifdef R2R_SHARING
1504                                         /* increment "num_ranks_enabled" */
1505                                         num_ranks_enabled++;
1506                                         /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1507                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1508                                                 delay[bl] += QRTR_CLK;
1509                                                 /* add "delay[]" values to "final_delay[][]" for rolling average */
1510                                                 final_delay[ch][bl] += delay[bl];
1511                                                 /* set timing based on rolling average values */
1512                                                 set_rcvn(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1513                                         }
1514 #else
1515                                         /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1516                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1517                                                 delay[bl] += QRTR_CLK;
1518                                                 set_rcvn(ch, rk, bl, delay[bl]);
1519                                         }
1520 #endif
1521
1522                                         /* disable FIFORST */
1523                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1524                                                 mrc_alt_write_mask(DDRPHY,
1525                                                         B01PTRCTL1 +
1526                                                         (bl >> 1) * DDRIODQ_BL_OFFSET +
1527                                                         ch * DDRIODQ_CH_OFFSET,
1528                                                         1 << 8, 1 << 8);
1529                                         }
1530 #endif
1531                                 }
1532                         }
1533                 }
1534         }
1535
1536 #ifndef BACKUP_RCVN
1537         /* restore original */
1538         msg_port_write(MEM_CTLR, DTR1, dtr1_save);
1539 #endif
1540
1541         LEAVEFN();
1542 }
1543
1544 /*
1545  * This function will perform the Write Levelling algorithm
1546  * (align WCLK and WDQS).
1547  *
1548  * This algorithm will act on each rank in each channel separately.
1549  */
1550 void wr_level(struct mrc_params *mrc_params)
1551 {
1552         uint8_t ch;     /* channel counter */
1553         uint8_t rk;     /* rank counter */
1554         uint8_t bl;     /* byte lane counter */
1555         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1556
1557 #ifdef R2R_SHARING
1558         /* used to find placement for rank2rank sharing configs */
1559         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1560 #ifndef BACKUP_WDQS
1561         /* used to find placement for rank2rank sharing configs */
1562         uint32_t num_ranks_enabled = 0;
1563 #endif
1564 #endif
1565
1566 #ifdef BACKUP_WDQS
1567 #else
1568         /* determines stop condition for CRS_WR_LVL */
1569         bool all_edges_found;
1570         /* absolute PI value to be programmed on the byte lane */
1571         uint32_t delay[NUM_BYTE_LANES];
1572         /*
1573          * static makes it so the data is loaded in the heap once by shadow(),
1574          * where non-static copies the data onto the stack every time this
1575          * function is called
1576          */
1577         uint32_t address;       /* address to be checked during COARSE_WR_LVL */
1578         u32 dtr4, dtr4_save;
1579 #endif
1580
1581         ENTERFN();
1582
1583         /* wr_level starts */
1584         mrc_post_code(0x06, 0x00);
1585
1586 #ifdef R2R_SHARING
1587         /* need to set "final_delay[][]" elements to "0" */
1588         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1589 #endif
1590
1591         /* loop through each enabled channel */
1592         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1593                 if (mrc_params->channel_enables & (1 << ch)) {
1594                         /* perform WRITE LEVELING algorithm on a per rank basis */
1595                         for (rk = 0; rk < NUM_RANKS; rk++) {
1596                                 if (mrc_params->rank_enables & (1 << rk)) {
1597                                         /*
1598                                          * POST_CODE here indicates the current
1599                                          * rank and channel being calibrated
1600                                          */
1601                                         mrc_post_code(0x06, 0x10 + ((ch << 4) | rk));
1602
1603 #ifdef BACKUP_WDQS
1604                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1605                                                 set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]);
1606                                                 set_wdq(ch, rk, bl, ddr_wdqs[PLATFORM_ID] - QRTR_CLK);
1607                                         }
1608 #else
1609                                         /*
1610                                          * perform a single PRECHARGE_ALL command to
1611                                          * make DRAM state machine go to IDLE state
1612                                          */
1613                                         dram_init_command(DCMD_PREA(rk));
1614
1615                                         /*
1616                                          * enable Write Levelling Mode
1617                                          * (EMRS1 w/ Write Levelling Mode Enable)
1618                                          */
1619                                         dram_init_command(DCMD_MRS1(rk, 0x82));
1620
1621                                         /*
1622                                          * set ODT DRAM Full Time Termination
1623                                          * disable in MCU
1624                                          */
1625
1626                                         dtr4 = msg_port_read(MEM_CTLR, DTR4);
1627                                         dtr4_save = dtr4;
1628                                         dtr4 |= DTR4_ODTDIS;
1629                                         msg_port_write(MEM_CTLR, DTR4, dtr4);
1630
1631                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1632                                                 /*
1633                                                  * Enable Sandy Bridge Mode (WDQ Tri-State) &
1634                                                  * Ensure 5 WDQS pulses during Write Leveling
1635                                                  */
1636                                                 mrc_alt_write_mask(DDRPHY,
1637                                                         DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1638                                                         0x10000154,
1639                                                         0x100003fc);
1640                                         }
1641
1642                                         /* Write Leveling Mode enabled in IO */
1643                                         mrc_alt_write_mask(DDRPHY,
1644                                                 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1645                                                 1 << 16, 1 << 16);
1646
1647                                         /* Initialize the starting delay to WCLK */
1648                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1649                                                 /*
1650                                                  * CLK0 --> RK0
1651                                                  * CLK1 --> RK1
1652                                                  */
1653                                                 delay[bl] = get_wclk(ch, rk);
1654
1655                                                 set_wdqs(ch, rk, bl, delay[bl]);
1656                                         }
1657
1658                                         /* now find the rising edge */
1659                                         find_rising_edge(mrc_params, delay, ch, rk, false);
1660
1661                                         /* disable Write Levelling Mode */
1662                                         mrc_alt_write_mask(DDRPHY,
1663                                                 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch,
1664                                                 0, 1 << 16);
1665
1666                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
1667                                                 /* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */
1668                                                 mrc_alt_write_mask(DDRPHY,
1669                                                         DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch,
1670                                                         0x00000154,
1671                                                         0x100003fc);
1672                                         }
1673
1674                                         /* restore original DTR4 */
1675                                         msg_port_write(MEM_CTLR, DTR4, dtr4_save);
1676
1677                                         /*
1678                                          * restore original value
1679                                          * (Write Levelling Mode Disable)
1680                                          */
1681                                         dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1));
1682
1683                                         /*
1684                                          * perform a single PRECHARGE_ALL command to
1685                                          * make DRAM state machine go to IDLE state
1686                                          */
1687                                         dram_init_command(DCMD_PREA(rk));
1688
1689                                         mrc_post_code(0x06, 0x30 + ((ch << 4) | rk));
1690
1691                                         /*
1692                                          * COARSE WRITE LEVEL:
1693                                          * check that we're on the correct clock edge
1694                                          */
1695
1696                                         /* hte reconfiguration request */
1697                                         mrc_params->hte_setup = 1;
1698
1699                                         /* start CRS_WR_LVL with WDQS = WDQS + 128 PI */
1700                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1701                                                 delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK;
1702                                                 set_wdqs(ch, rk, bl, delay[bl]);
1703                                                 /*
1704                                                  * program WDQ timings based on WDQS
1705                                                  * (WDQ = WDQS - 32 PI)
1706                                                  */
1707                                                 set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
1708                                         }
1709
1710                                         /* get an address in the targeted channel/rank */
1711                                         address = get_addr(ch, rk);
1712                                         do {
1713                                                 uint32_t coarse_result = 0x00;
1714                                                 uint32_t coarse_result_mask = byte_lane_mask(mrc_params);
1715                                                 /* assume pass */
1716                                                 all_edges_found = true;
1717
1718                                                 mrc_params->hte_setup = 1;
1719                                                 coarse_result = check_rw_coarse(mrc_params, address);
1720
1721                                                 /* check for failures and margin the byte lane back 128 PI (1 CLK) */
1722                                                 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1723                                                         if (coarse_result & (coarse_result_mask << bl)) {
1724                                                                 all_edges_found = false;
1725                                                                 delay[bl] -= FULL_CLK;
1726                                                                 set_wdqs(ch, rk, bl, delay[bl]);
1727                                                                 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1728                                                                 set_wdq(ch, rk, bl, delay[bl] - QRTR_CLK);
1729                                                         }
1730                                                 }
1731                                         } while (!all_edges_found);
1732
1733 #ifdef R2R_SHARING
1734                                         /* increment "num_ranks_enabled" */
1735                                          num_ranks_enabled++;
1736                                         /* accumulate "final_delay[][]" values from "delay[]" values for rolling average */
1737                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1738                                                 final_delay[ch][bl] += delay[bl];
1739                                                 set_wdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
1740                                                 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1741                                                 set_wdq(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled - QRTR_CLK);
1742                                         }
1743 #endif
1744 #endif
1745                                 }
1746                         }
1747                 }
1748         }
1749
1750         LEAVEFN();
1751 }
1752
1753 void prog_page_ctrl(struct mrc_params *mrc_params)
1754 {
1755         u32 dpmc0;
1756
1757         ENTERFN();
1758
1759         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
1760         dpmc0 &= ~DPMC0_PCLSTO_MASK;
1761         dpmc0 |= (4 << 16);
1762         dpmc0 |= DPMC0_PREAPWDEN;
1763         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
1764 }
1765
1766 /*
1767  * This function will perform the READ TRAINING Algorithm on all
1768  * channels/ranks/byte_lanes simultaneously to minimize execution time.
1769  *
1770  * The idea here is to train the VREF and RDQS (and eventually RDQ) values
1771  * to achieve maximum READ margins. The algorithm will first determine the
1772  * X coordinate (RDQS setting). This is done by collapsing the VREF eye
1773  * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX.
1774  * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX,
1775  * then average those; this will be the final X coordinate. The algorithm
1776  * will then determine the Y coordinate (VREF setting). This is done by
1777  * collapsing the RDQS eye until we find a minimum required VREF eye for
1778  * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at
1779  * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y
1780  * coordinate.
1781  *
1782  * NOTE: this algorithm assumes the eye curves have a one-to-one relationship,
1783  * meaning for each X the curve has only one Y and vice-a-versa.
1784  */
1785 void rd_train(struct mrc_params *mrc_params)
1786 {
1787         uint8_t ch;     /* channel counter */
1788         uint8_t rk;     /* rank counter */
1789         uint8_t bl;     /* byte lane counter */
1790         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1791 #ifdef BACKUP_RDQS
1792 #else
1793         uint8_t side_x; /* tracks LEFT/RIGHT approach vectors */
1794         uint8_t side_y; /* tracks BOTTOM/TOP approach vectors */
1795         /* X coordinate data (passing RDQS values) for approach vectors */
1796         uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1797         /* Y coordinate data (passing VREF values) for approach vectors */
1798         uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES];
1799         /* centered X (RDQS) */
1800         uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1801         /* centered Y (VREF) */
1802         uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES];
1803         uint32_t address;       /* target address for check_bls_ex() */
1804         uint32_t result;        /* result of check_bls_ex() */
1805         uint32_t bl_mask;       /* byte lane mask for result checking */
1806 #ifdef R2R_SHARING
1807         /* used to find placement for rank2rank sharing configs */
1808         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1809         /* used to find placement for rank2rank sharing configs */
1810         uint32_t num_ranks_enabled = 0;
1811 #endif
1812 #endif
1813
1814         /* rd_train starts */
1815         mrc_post_code(0x07, 0x00);
1816
1817         ENTERFN();
1818
1819 #ifdef BACKUP_RDQS
1820         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1821                 if (mrc_params->channel_enables & (1 << ch)) {
1822                         for (rk = 0; rk < NUM_RANKS; rk++) {
1823                                 if (mrc_params->rank_enables & (1 << rk)) {
1824                                         for (bl = 0;
1825                                              bl < NUM_BYTE_LANES / bl_divisor;
1826                                              bl++) {
1827                                                 set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]);
1828                                         }
1829                                 }
1830                         }
1831                 }
1832         }
1833 #else
1834         /* initialize x/y_coordinate arrays */
1835         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1836                 if (mrc_params->channel_enables & (1 << ch)) {
1837                         for (rk = 0; rk < NUM_RANKS; rk++) {
1838                                 if (mrc_params->rank_enables & (1 << rk)) {
1839                                         for (bl = 0;
1840                                              bl < NUM_BYTE_LANES / bl_divisor;
1841                                              bl++) {
1842                                                 /* x_coordinate */
1843                                                 x_coordinate[L][B][ch][rk][bl] = RDQS_MIN;
1844                                                 x_coordinate[R][B][ch][rk][bl] = RDQS_MAX;
1845                                                 x_coordinate[L][T][ch][rk][bl] = RDQS_MIN;
1846                                                 x_coordinate[R][T][ch][rk][bl] = RDQS_MAX;
1847                                                 /* y_coordinate */
1848                                                 y_coordinate[L][B][ch][bl] = VREF_MIN;
1849                                                 y_coordinate[R][B][ch][bl] = VREF_MIN;
1850                                                 y_coordinate[L][T][ch][bl] = VREF_MAX;
1851                                                 y_coordinate[R][T][ch][bl] = VREF_MAX;
1852                                         }
1853                                 }
1854                         }
1855                 }
1856         }
1857
1858         /* initialize other variables */
1859         bl_mask = byte_lane_mask(mrc_params);
1860         address = get_addr(0, 0);
1861
1862 #ifdef R2R_SHARING
1863         /* need to set "final_delay[][]" elements to "0" */
1864         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1865 #endif
1866
1867         /* look for passing coordinates */
1868         for (side_y = B; side_y <= T; side_y++) {
1869                 for (side_x = L; side_x <= R; side_x++) {
1870                         mrc_post_code(0x07, 0x10 + side_y * 2 + side_x);
1871
1872                         /* find passing values */
1873                         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1874                                 if (mrc_params->channel_enables & (0x1 << ch)) {
1875                                         for (rk = 0; rk < NUM_RANKS; rk++) {
1876                                                 if (mrc_params->rank_enables &
1877                                                         (0x1 << rk)) {
1878                                                         /* set x/y_coordinate search starting settings */
1879                                                         for (bl = 0;
1880                                                              bl < NUM_BYTE_LANES / bl_divisor;
1881                                                              bl++) {
1882                                                                 set_rdqs(ch, rk, bl,
1883                                                                          x_coordinate[side_x][side_y][ch][rk][bl]);
1884                                                                 set_vref(ch, bl,
1885                                                                          y_coordinate[side_x][side_y][ch][bl]);
1886                                                         }
1887
1888                                                         /* get an address in the target channel/rank */
1889                                                         address = get_addr(ch, rk);
1890
1891                                                         /* request HTE reconfiguration */
1892                                                         mrc_params->hte_setup = 1;
1893
1894                                                         /* test the settings */
1895                                                         do {
1896                                                                 /* result[07:00] == failing byte lane (MAX 8) */
1897                                                                 result = check_bls_ex(mrc_params, address);
1898
1899                                                                 /* check for failures */
1900                                                                 if (result & 0xff) {
1901                                                                         /* at least 1 byte lane failed */
1902                                                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
1903                                                                                 if (result &
1904                                                                                         (bl_mask << bl)) {
1905                                                                                         /* adjust the RDQS values accordingly */
1906                                                                                         if (side_x == L)
1907                                                                                                 x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP;
1908                                                                                         else
1909                                                                                                 x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP;
1910
1911                                                                                         /* check that we haven't closed the RDQS_EYE too much */
1912                                                                                         if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) ||
1913                                                                                                 (x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) ||
1914                                                                                                 (x_coordinate[L][side_y][ch][rk][bl] ==
1915                                                                                                 x_coordinate[R][side_y][ch][rk][bl])) {
1916                                                                                                 /*
1917                                                                                                  * not enough RDQS margin available at this VREF
1918                                                                                                  * update VREF values accordingly
1919                                                                                                  */
1920                                                                                                 if (side_y == B)
1921                                                                                                         y_coordinate[side_x][B][ch][bl] += VREF_STEP;
1922                                                                                                 else
1923                                                                                                         y_coordinate[side_x][T][ch][bl] -= VREF_STEP;
1924
1925                                                                                                 /* check that we haven't closed the VREF_EYE too much */
1926                                                                                                 if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) ||
1927                                                                                                         (y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) ||
1928                                                                                                         (y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) {
1929                                                                                                         /* VREF_EYE collapsed below MIN_VREF_EYE */
1930                                                                                                         training_message(ch, rk, bl);
1931                                                                                                         mrc_post_code(0xEE, 0x70 + side_y * 2 + side_x);
1932                                                                                                 } else {
1933                                                                                                         /* update the VREF setting */
1934                                                                                                         set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]);
1935                                                                                                         /* reset the X coordinate to begin the search at the new VREF */
1936                                                                                                         x_coordinate[side_x][side_y][ch][rk][bl] =
1937                                                                                                                 (side_x == L) ? RDQS_MIN : RDQS_MAX;
1938                                                                                                 }
1939                                                                                         }
1940
1941                                                                                         /* update the RDQS setting */
1942                                                                                         set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]);
1943                                                                                 }
1944                                                                         }
1945                                                                 }
1946                                                         } while (result & 0xff);
1947                                                 }
1948                                         }
1949                                 }
1950                         }
1951                 }
1952         }
1953
1954         mrc_post_code(0x07, 0x20);
1955
1956         /* find final RDQS (X coordinate) & final VREF (Y coordinate) */
1957         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1958                 if (mrc_params->channel_enables & (1 << ch)) {
1959                         for (rk = 0; rk < NUM_RANKS; rk++) {
1960                                 if (mrc_params->rank_enables & (1 << rk)) {
1961                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1962                                                 uint32_t temp1;
1963                                                 uint32_t temp2;
1964
1965                                                 /* x_coordinate */
1966                                                 DPF(D_INFO,
1967                                                     "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n",
1968                                                     rk, bl,
1969                                                     x_coordinate[L][T][ch][rk][bl],
1970                                                     x_coordinate[R][T][ch][rk][bl],
1971                                                     x_coordinate[L][B][ch][rk][bl],
1972                                                     x_coordinate[R][B][ch][rk][bl]);
1973
1974                                                 /* average the TOP side LEFT & RIGHT values */
1975                                                 temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2;
1976                                                 /* average the BOTTOM side LEFT & RIGHT values */
1977                                                 temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2;
1978                                                 /* average the above averages */
1979                                                 x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2);
1980
1981                                                 /* y_coordinate */
1982                                                 DPF(D_INFO,
1983                                                     "VREF R/L eye lane%d : %d-%d %d-%d\n",
1984                                                     bl,
1985                                                     y_coordinate[R][B][ch][bl],
1986                                                     y_coordinate[R][T][ch][bl],
1987                                                     y_coordinate[L][B][ch][bl],
1988                                                     y_coordinate[L][T][ch][bl]);
1989
1990                                                 /* average the RIGHT side TOP & BOTTOM values */
1991                                                 temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2;
1992                                                 /* average the LEFT side TOP & BOTTOM values */
1993                                                 temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2;
1994                                                 /* average the above averages */
1995                                                 y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2);
1996                                         }
1997                                 }
1998                         }
1999                 }
2000         }
2001
2002 #ifdef RX_EYE_CHECK
2003         /* perform an eye check */
2004         for (side_y = B; side_y <= T; side_y++) {
2005                 for (side_x = L; side_x <= R; side_x++) {
2006                         mrc_post_code(0x07, 0x30 + side_y * 2 + side_x);
2007
2008                         /* update the settings for the eye check */
2009                         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2010                                 if (mrc_params->channel_enables & (1 << ch)) {
2011                                         for (rk = 0; rk < NUM_RANKS; rk++) {
2012                                                 if (mrc_params->rank_enables & (1 << rk)) {
2013                                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2014                                                                 if (side_x == L)
2015                                                                         set_rdqs(ch, rk, bl, x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2));
2016                                                                 else
2017                                                                         set_rdqs(ch, rk, bl, x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2));
2018
2019                                                                 if (side_y == B)
2020                                                                         set_vref(ch, bl, y_center[ch][bl] - (MIN_VREF_EYE / 2));
2021                                                                 else
2022                                                                         set_vref(ch, bl, y_center[ch][bl] + (MIN_VREF_EYE / 2));
2023                                                         }
2024                                                 }
2025                                         }
2026                                 }
2027                         }
2028
2029                         /* request HTE reconfiguration */
2030                         mrc_params->hte_setup = 1;
2031
2032                         /* check the eye */
2033                         if (check_bls_ex(mrc_params, address) & 0xff) {
2034                                 /* one or more byte lanes failed */
2035                                 mrc_post_code(0xee, 0x74 + side_x * 2 + side_y);
2036                         }
2037                 }
2038         }
2039 #endif
2040
2041         mrc_post_code(0x07, 0x40);
2042
2043         /* set final placements */
2044         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2045                 if (mrc_params->channel_enables & (1 << ch)) {
2046                         for (rk = 0; rk < NUM_RANKS; rk++) {
2047                                 if (mrc_params->rank_enables & (1 << rk)) {
2048 #ifdef R2R_SHARING
2049                                         /* increment "num_ranks_enabled" */
2050                                         num_ranks_enabled++;
2051 #endif
2052                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2053                                                 /* x_coordinate */
2054 #ifdef R2R_SHARING
2055                                                 final_delay[ch][bl] += x_center[ch][rk][bl];
2056                                                 set_rdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled);
2057 #else
2058                                                 set_rdqs(ch, rk, bl, x_center[ch][rk][bl]);
2059 #endif
2060                                                 /* y_coordinate */
2061                                                 set_vref(ch, bl, y_center[ch][bl]);
2062                                         }
2063                                 }
2064                         }
2065                 }
2066         }
2067 #endif
2068
2069         LEAVEFN();
2070 }
2071
2072 /*
2073  * This function will perform the WRITE TRAINING Algorithm on all
2074  * channels/ranks/byte_lanes simultaneously to minimize execution time.
2075  *
2076  * The idea here is to train the WDQ timings to achieve maximum WRITE margins.
2077  * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS
2078  * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data
2079  * patterns pass. This is because WDQS will be aligned to WCLK by the
2080  * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window
2081  * of validity.
2082  */
2083 void wr_train(struct mrc_params *mrc_params)
2084 {
2085         uint8_t ch;     /* channel counter */
2086         uint8_t rk;     /* rank counter */
2087         uint8_t bl;     /* byte lane counter */
2088         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
2089 #ifdef BACKUP_WDQ
2090 #else
2091         uint8_t side;           /* LEFT/RIGHT side indicator (0=L, 1=R) */
2092         uint32_t temp;          /* temporary DWORD */
2093         /* 2 arrays, for L & R side passing delays */
2094         uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
2095         uint32_t address;       /* target address for check_bls_ex() */
2096         uint32_t result;        /* result of check_bls_ex() */
2097         uint32_t bl_mask;       /* byte lane mask for result checking */
2098 #ifdef R2R_SHARING
2099         /* used to find placement for rank2rank sharing configs */
2100         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
2101         /* used to find placement for rank2rank sharing configs */
2102         uint32_t num_ranks_enabled = 0;
2103 #endif
2104 #endif
2105
2106         /* wr_train starts */
2107         mrc_post_code(0x08, 0x00);
2108
2109         ENTERFN();
2110
2111 #ifdef BACKUP_WDQ
2112         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2113                 if (mrc_params->channel_enables & (1 << ch)) {
2114                         for (rk = 0; rk < NUM_RANKS; rk++) {
2115                                 if (mrc_params->rank_enables & (1 << rk)) {
2116                                         for (bl = 0;
2117                                              bl < NUM_BYTE_LANES / bl_divisor;
2118                                              bl++) {
2119                                                 set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]);
2120                                         }
2121                                 }
2122                         }
2123                 }
2124         }
2125 #else
2126         /* initialize "delay" */
2127         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2128                 if (mrc_params->channel_enables & (1 << ch)) {
2129                         for (rk = 0; rk < NUM_RANKS; rk++) {
2130                                 if (mrc_params->rank_enables & (1 << rk)) {
2131                                         for (bl = 0;
2132                                              bl < NUM_BYTE_LANES / bl_divisor;
2133                                              bl++) {
2134                                                 /*
2135                                                  * want to start with
2136                                                  * WDQ = (WDQS - QRTR_CLK)
2137                                                  * +/- QRTR_CLK
2138                                                  */
2139                                                 temp = get_wdqs(ch, rk, bl) - QRTR_CLK;
2140                                                 delay[L][ch][rk][bl] = temp - QRTR_CLK;
2141                                                 delay[R][ch][rk][bl] = temp + QRTR_CLK;
2142                                         }
2143                                 }
2144                         }
2145                 }
2146         }
2147
2148         /* initialize other variables */
2149         bl_mask = byte_lane_mask(mrc_params);
2150         address = get_addr(0, 0);
2151
2152 #ifdef R2R_SHARING
2153         /* need to set "final_delay[][]" elements to "0" */
2154         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
2155 #endif
2156
2157         /*
2158          * start algorithm on the LEFT side and train each channel/bl
2159          * until no failures are observed, then repeat for the RIGHT side.
2160          */
2161         for (side = L; side <= R; side++) {
2162                 mrc_post_code(0x08, 0x10 + side);
2163
2164                 /* set starting values */
2165                 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2166                         if (mrc_params->channel_enables & (1 << ch)) {
2167                                 for (rk = 0; rk < NUM_RANKS; rk++) {
2168                                         if (mrc_params->rank_enables &
2169                                                 (1 << rk)) {
2170                                                 for (bl = 0;
2171                                                      bl < NUM_BYTE_LANES / bl_divisor;
2172                                                      bl++) {
2173                                                         set_wdq(ch, rk, bl, delay[side][ch][rk][bl]);
2174                                                 }
2175                                         }
2176                                 }
2177                         }
2178                 }
2179
2180                 /* find passing values */
2181                 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2182                         if (mrc_params->channel_enables & (1 << ch)) {
2183                                 for (rk = 0; rk < NUM_RANKS; rk++) {
2184                                         if (mrc_params->rank_enables &
2185                                                 (1 << rk)) {
2186                                                 /* get an address in the target channel/rank */
2187                                                 address = get_addr(ch, rk);
2188
2189                                                 /* request HTE reconfiguration */
2190                                                 mrc_params->hte_setup = 1;
2191
2192                                                 /* check the settings */
2193                                                 do {
2194                                                         /* result[07:00] == failing byte lane (MAX 8) */
2195                                                         result = check_bls_ex(mrc_params, address);
2196                                                         /* check for failures */
2197                                                         if (result & 0xff) {
2198                                                                 /* at least 1 byte lane failed */
2199                                                                 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2200                                                                         if (result &
2201                                                                                 (bl_mask << bl)) {
2202                                                                                 if (side == L)
2203                                                                                         delay[L][ch][rk][bl] += WDQ_STEP;
2204                                                                                 else
2205                                                                                         delay[R][ch][rk][bl] -= WDQ_STEP;
2206
2207                                                                                 /* check for algorithm failure */
2208                                                                                 if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) {
2209                                                                                         /*
2210                                                                                          * margin available
2211                                                                                          * update delay setting
2212                                                                                          */
2213                                                                                         set_wdq(ch, rk, bl,
2214                                                                                                 delay[side][ch][rk][bl]);
2215                                                                                 } else {
2216                                                                                         /*
2217                                                                                          * no margin available
2218                                                                                          * notify the user and halt
2219                                                                                          */
2220                                                                                         training_message(ch, rk, bl);
2221                                                                                         mrc_post_code(0xee, 0x80 + side);
2222                                                                                 }
2223                                                                         }
2224                                                                 }
2225                                                         }
2226                                                 /* stop when all byte lanes pass */
2227                                                 } while (result & 0xff);
2228                                         }
2229                                 }
2230                         }
2231                 }
2232         }
2233
2234         /* program WDQ to the middle of passing window */
2235         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2236                 if (mrc_params->channel_enables & (1 << ch)) {
2237                         for (rk = 0; rk < NUM_RANKS; rk++) {
2238                                 if (mrc_params->rank_enables & (1 << rk)) {
2239 #ifdef R2R_SHARING
2240                                         /* increment "num_ranks_enabled" */
2241                                         num_ranks_enabled++;
2242 #endif
2243                                         for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) {
2244                                                 DPF(D_INFO,
2245                                                     "WDQ eye rank%d lane%d : %d-%d\n",
2246                                                     rk, bl,
2247                                                     delay[L][ch][rk][bl],
2248                                                     delay[R][ch][rk][bl]);
2249
2250                                                 temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2;
2251
2252 #ifdef R2R_SHARING
2253                                                 final_delay[ch][bl] += temp;
2254                                                 set_wdq(ch, rk, bl,
2255                                                         final_delay[ch][bl] / num_ranks_enabled);
2256 #else
2257                                                 set_wdq(ch, rk, bl, temp);
2258 #endif
2259                                         }
2260                                 }
2261                         }
2262                 }
2263         }
2264 #endif
2265
2266         LEAVEFN();
2267 }
2268
2269 /*
2270  * This function will store relevant timing data
2271  *
2272  * This data will be used on subsequent boots to speed up boot times
2273  * and is required for Suspend To RAM capabilities.
2274  */
2275 void store_timings(struct mrc_params *mrc_params)
2276 {
2277         uint8_t ch, rk, bl;
2278         struct mrc_timings *mt = &mrc_params->timings;
2279
2280         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2281                 for (rk = 0; rk < NUM_RANKS; rk++) {
2282                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
2283                                 mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl);
2284                                 mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl);
2285                                 mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl);
2286                                 mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl);
2287
2288                                 if (rk == 0)
2289                                         mt->vref[ch][bl] = get_vref(ch, bl);
2290                         }
2291
2292                         mt->wctl[ch][rk] = get_wctl(ch, rk);
2293                 }
2294
2295                 mt->wcmd[ch] = get_wcmd(ch);
2296         }
2297
2298         /* need to save for a case of changing frequency after warm reset */
2299         mt->ddr_speed = mrc_params->ddr_speed;
2300 }
2301
2302 /*
2303  * The purpose of this function is to ensure the SEC comes out of reset
2304  * and IA initiates the SEC enabling Memory Scrambling.
2305  */
2306 void enable_scrambling(struct mrc_params *mrc_params)
2307 {
2308         uint32_t lfsr = 0;
2309         uint8_t i;
2310
2311         if (mrc_params->scrambling_enables == 0)
2312                 return;
2313
2314         ENTERFN();
2315
2316         /* 32 bit seed is always stored in BIOS NVM */
2317         lfsr = mrc_params->timings.scrambler_seed;
2318
2319         if (mrc_params->boot_mode == BM_COLD) {
2320                 /*
2321                  * factory value is 0 and in first boot,
2322                  * a clock based seed is loaded.
2323                  */
2324                 if (lfsr == 0) {
2325                         /*
2326                          * get seed from system clock
2327                          * and make sure it is not all 1's
2328                          */
2329                         lfsr = rdtsc() & 0x0fffffff;
2330                 } else {
2331                         /*
2332                          * Need to replace scrambler
2333                          *
2334                          * get next 32bit LFSR 16 times which is the last
2335                          * part of the previous scrambler vector
2336                          */
2337                         for (i = 0; i < 16; i++)
2338                                 lfsr32(&lfsr);
2339                 }
2340
2341                 /* save new seed */
2342                 mrc_params->timings.scrambler_seed = lfsr;
2343         }
2344
2345         /*
2346          * In warm boot or S3 exit, we have the previous seed.
2347          * In cold boot, we have the last 32bit LFSR which is the new seed.
2348          */
2349         lfsr32(&lfsr);  /* shift to next value */
2350         msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003ffff));
2351
2352         for (i = 0; i < 2; i++)
2353                 msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xaaaaaaaa));
2354
2355         LEAVEFN();
2356 }
2357
2358 /*
2359  * Configure MCU Power Management Control Register
2360  * and Scheduler Control Register
2361  */
2362 void prog_ddr_control(struct mrc_params *mrc_params)
2363 {
2364         u32 dsch;
2365         u32 dpmc0;
2366
2367         ENTERFN();
2368
2369         dsch = msg_port_read(MEM_CTLR, DSCH);
2370         dsch &= ~(DSCH_OOODIS | DSCH_OOOST3DIS | DSCH_NEWBYPDIS);
2371         msg_port_write(MEM_CTLR, DSCH, dsch);
2372
2373         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2374         dpmc0 &= ~DPMC0_DISPWRDN;
2375         dpmc0 |= (mrc_params->power_down_disable << 25);
2376         dpmc0 &= ~DPMC0_CLKGTDIS;
2377         dpmc0 &= ~DPMC0_PCLSTO_MASK;
2378         dpmc0 |= (4 << 16);
2379         dpmc0 |= DPMC0_PREAPWDEN;
2380         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2381
2382         /* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */
2383         mrc_write_mask(MEM_CTLR, DPMC1, 0x20, 0x30);
2384
2385         LEAVEFN();
2386 }
2387
2388 /*
2389  * After training complete configure MCU Rank Population Register
2390  * specifying: ranks enabled, device width, density, address mode
2391  */
2392 void prog_dra_drb(struct mrc_params *mrc_params)
2393 {
2394         u32 drp;
2395         u32 dco;
2396         u8 density = mrc_params->params.density;
2397
2398         ENTERFN();
2399
2400         dco = msg_port_read(MEM_CTLR, DCO);
2401         dco &= ~DCO_IC;
2402         msg_port_write(MEM_CTLR, DCO, dco);
2403
2404         drp = 0;
2405         if (mrc_params->rank_enables & 1)
2406                 drp |= DRP_RKEN0;
2407         if (mrc_params->rank_enables & 2)
2408                 drp |= DRP_RKEN1;
2409         if (mrc_params->dram_width == X16) {
2410                 drp |= (1 << 4);
2411                 drp |= (1 << 9);
2412         }
2413
2414         /*
2415          * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb
2416          * has to be mapped RANKDENSx encoding (0=1Gb)
2417          */
2418         if (density == 0)
2419                 density = 4;
2420
2421         drp |= ((density - 1) << 6);
2422         drp |= ((density - 1) << 11);
2423
2424         /* Address mode can be overwritten if ECC enabled */
2425         drp |= (mrc_params->address_mode << 14);
2426
2427         msg_port_write(MEM_CTLR, DRP, drp);
2428
2429         dco &= ~DCO_PMICTL;
2430         dco |= DCO_IC;
2431         msg_port_write(MEM_CTLR, DCO, dco);
2432
2433         LEAVEFN();
2434 }
2435
2436 /* Send DRAM wake command */
2437 void perform_wake(struct mrc_params *mrc_params)
2438 {
2439         ENTERFN();
2440
2441         dram_wake_command();
2442
2443         LEAVEFN();
2444 }
2445
2446 /*
2447  * Configure refresh rate and short ZQ calibration interval
2448  * Activate dynamic self refresh
2449  */
2450 void change_refresh_period(struct mrc_params *mrc_params)
2451 {
2452         u32 drfc;
2453         u32 dcal;
2454         u32 dpmc0;
2455
2456         ENTERFN();
2457
2458         drfc = msg_port_read(MEM_CTLR, DRFC);
2459         drfc &= ~DRFC_TREFI_MASK;
2460         drfc |= (mrc_params->refresh_rate << 12);
2461         drfc |= DRFC_REFDBTCLR;
2462         msg_port_write(MEM_CTLR, DRFC, drfc);
2463
2464         dcal = msg_port_read(MEM_CTLR, DCAL);
2465         dcal &= ~DCAL_ZQCINT_MASK;
2466         dcal |= (3 << 8);       /* 63ms */
2467         msg_port_write(MEM_CTLR, DCAL, dcal);
2468
2469         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2470         dpmc0 |= (DPMC0_DYNSREN | DPMC0_ENPHYCLKGATE);
2471         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2472
2473         LEAVEFN();
2474 }
2475
2476 /*
2477  * Configure DDRPHY for Auto-Refresh, Periodic Compensations,
2478  * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down
2479  */
2480 void set_auto_refresh(struct mrc_params *mrc_params)
2481 {
2482         uint32_t channel;
2483         uint32_t rank;
2484         uint32_t bl;
2485         uint32_t bl_divisor = 1;
2486         uint32_t temp;
2487
2488         ENTERFN();
2489
2490         /*
2491          * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp,
2492          * ZQSPERIOD, Auto-Precharge, CKE Power-Down
2493          */
2494         for (channel = 0; channel < NUM_CHANNELS; channel++) {
2495                 if (mrc_params->channel_enables & (1 << channel)) {
2496                         /* Enable Periodic RCOMPS */
2497                         mrc_alt_write_mask(DDRPHY, CMPCTRL, 2, 2);
2498
2499                         /* Enable Dynamic DiffAmp & Set Read ODT Value */
2500                         switch (mrc_params->rd_odt_value) {
2501                         case 0:
2502                                 temp = 0x3f;    /* OFF */
2503                                 break;
2504                         default:
2505                                 temp = 0x00;    /* Auto */
2506                                 break;
2507                         }
2508
2509                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) {
2510                                 /* Override: DIFFAMP, ODT */
2511                                 mrc_alt_write_mask(DDRPHY,
2512                                         B0OVRCTL + bl * DDRIODQ_BL_OFFSET +
2513                                         channel * DDRIODQ_CH_OFFSET,
2514                                         temp << 10,
2515                                         0x003ffc00);
2516
2517                                 /* Override: DIFFAMP, ODT */
2518                                 mrc_alt_write_mask(DDRPHY,
2519                                         B1OVRCTL + bl * DDRIODQ_BL_OFFSET +
2520                                         channel * DDRIODQ_CH_OFFSET,
2521                                         temp << 10,
2522                                         0x003ffc00);
2523                         }
2524
2525                         /* Issue ZQCS command */
2526                         for (rank = 0; rank < NUM_RANKS; rank++) {
2527                                 if (mrc_params->rank_enables & (1 << rank))
2528                                         dram_init_command(DCMD_ZQCS(rank));
2529                         }
2530                 }
2531         }
2532
2533         clear_pointers();
2534
2535         LEAVEFN();
2536 }
2537
2538 /*
2539  * Depending on configuration enables ECC support
2540  *
2541  * Available memory size is decreased, and updated with 0s
2542  * in order to clear error status. Address mode 2 forced.
2543  */
2544 void ecc_enable(struct mrc_params *mrc_params)
2545 {
2546         u32 drp;
2547         u32 dsch;
2548         u32 ecc_ctrl;
2549
2550         if (mrc_params->ecc_enables == 0)
2551                 return;
2552
2553         ENTERFN();
2554
2555         /* Configuration required in ECC mode */
2556         drp = msg_port_read(MEM_CTLR, DRP);
2557         drp &= ~DRP_ADDRMAP_MASK;
2558         drp |= DRP_ADDRMAP_MAP1;
2559         drp |= DRP_PRI64BSPLITEN;
2560         msg_port_write(MEM_CTLR, DRP, drp);
2561
2562         /* Disable new request bypass */
2563         dsch = msg_port_read(MEM_CTLR, DSCH);
2564         dsch |= DSCH_NEWBYPDIS;
2565         msg_port_write(MEM_CTLR, DSCH, dsch);
2566
2567         /* Enable ECC */
2568         ecc_ctrl = (DECCCTRL_SBEEN | DECCCTRL_DBEEN | DECCCTRL_ENCBGEN);
2569         msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl);
2570
2571         /* Assume 8 bank memory, one bank is gone for ECC */
2572         mrc_params->mem_size -= mrc_params->mem_size / 8;
2573
2574         /* For S3 resume memory content has to be preserved */
2575         if (mrc_params->boot_mode != BM_S3) {
2576                 select_hte();
2577                 hte_mem_init(mrc_params, MRC_MEM_INIT);
2578                 select_mem_mgr();
2579         }
2580
2581         LEAVEFN();
2582 }
2583
2584 /*
2585  * Execute memory test
2586  * if error detected it is indicated in mrc_params->status
2587  */
2588 void memory_test(struct mrc_params *mrc_params)
2589 {
2590         uint32_t result = 0;
2591
2592         ENTERFN();
2593
2594         select_hte();
2595         result = hte_mem_init(mrc_params, MRC_MEM_TEST);
2596         select_mem_mgr();
2597
2598         DPF(D_INFO, "Memory test result %x\n", result);
2599         mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST);
2600         LEAVEFN();
2601 }
2602
2603 /* Lock MCU registers at the end of initialization sequence */
2604 void lock_registers(struct mrc_params *mrc_params)
2605 {
2606         u32 dco;
2607
2608         ENTERFN();
2609
2610         dco = msg_port_read(MEM_CTLR, DCO);
2611         dco &= ~(DCO_PMICTL | DCO_PMIDIS);
2612         dco |= (DCO_DRPLOCK | DCO_CPGCLOCK);
2613         msg_port_write(MEM_CTLR, DCO, dco);
2614
2615         LEAVEFN();
2616 }