drivers/ddr/marvell/axp/ddr3_write_leveling.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Marvell International Ltd. and its affiliates
   4  */
   5
   6 #include <common.h>
   7 #include <i2c.h>
   8 #include <log.h>
   9 #include <spl.h>
  10 #include <asm/io.h>
  11 #include <asm/arch/cpu.h>
  12 #include <asm/arch/soc.h>
  13 #include <linux/delay.h>
  14
  15 #include "ddr3_hw_training.h"
  16
  17 /*
  18  * Debug
  19  */
  20 #define DEBUG_WL_C(s, d, l) \
  21         DEBUG_WL_S(s); DEBUG_WL_D(d, l); DEBUG_WL_S("\n")
  22 #define DEBUG_WL_FULL_C(s, d, l) \
  23         DEBUG_WL_FULL_S(s); DEBUG_WL_FULL_D(d, l); DEBUG_WL_FULL_S("\n")
  24
  25 #ifdef MV_DEBUG_WL
  26 #define DEBUG_WL_S(s)                   puts(s)
  27 #define DEBUG_WL_D(d, l)                printf("%x", d)
  28 #define DEBUG_RL_S(s) \
  29         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  30 #define DEBUG_RL_D(d, l) \
  31         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  32 #else
  33 #define DEBUG_WL_S(s)
  34 #define DEBUG_WL_D(d, l)
  35 #endif
  36
  37 #ifdef MV_DEBUG_WL_FULL
  38 #define DEBUG_WL_FULL_S(s)              puts(s)
  39 #define DEBUG_WL_FULL_D(d, l)           printf("%x", d)
  40 #else
  41 #define DEBUG_WL_FULL_S(s)
  42 #define DEBUG_WL_FULL_D(d, l)
  43 #endif
  44
  45 #define WL_SUP_EXPECTED_DATA            0x21
  46 #define WL_SUP_READ_DRAM_ENTRY          0x8
  47
  48 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
  49                                          u32 *result,
  50                                          MV_DRAM_INFO *dram_info);
  51 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr,
  52                                     u32 data);
  53
  54 extern u16 odt_static[ODT_OPT][MAX_CS];
  55 extern u16 odt_dynamic[ODT_OPT][MAX_CS];
  56 extern u32 wl_sup_pattern[LEN_WL_SUP_PATTERN];
  57
  58 /*
  59  * Name:     ddr3_write_leveling_hw
  60  * Desc:     Execute Write leveling phase by HW
  61  * Args:     freq      - current sequence frequency
  62  *           dram_info   - main struct
  63  * Notes:
  64  * Returns:  MV_OK if success, MV_FAIL if fail.
  65  */
  66 int ddr3_write_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  67 {
  68         u32 reg, phase, delay, cs, pup;
  69 #ifdef MV88F67XX
  70         int dpde_flag = 0;
  71 #endif
  72         /* Debug message - Start Read leveling procedure */
  73         DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
  74
  75 #ifdef MV88F67XX
  76         /* Dynamic pad issue (BTS669) during WL */
  77         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  78         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  79                 dpde_flag = 1;
  80                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  81                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  82         }
  83 #endif
  84
  85         reg = 1 << REG_DRAM_TRAINING_WL_OFFS;
  86         /* Config the retest number */
  87         reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
  88         reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
  89         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  90
  91         reg =  reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  92                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  93         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  94
  95         /* Wait */
  96         do {
  97                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  98                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  99         } while (reg);          /* Wait for '0' */
 100
 101         reg = reg_read(REG_DRAM_TRAINING_ADDR);
 102         /* Check if Successful */
 103         if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 104                 /*
 105                  * Read results to arrays - Results are required for WL
 106                  * High freq Supplement and DQS Centralization
 107                  */
 108                 for (cs = 0; cs < MAX_CS; cs++) {
 109                         if (dram_info->cs_ena & (1 << cs)) {
 110                                 for (pup = 0;
 111                                      pup < dram_info->num_of_total_pups;
 112                                      pup++) {
 113                                         if (pup == dram_info->num_of_std_pups
 114                                             && dram_info->ecc_ena)
 115                                                 pup = ECC_PUP;
 116                                         reg =
 117                                             ddr3_read_pup_reg(PUP_WL_MODE, cs,
 118                                                               pup);
 119                                         phase =
 120                                             (reg >> REG_PHY_PHASE_OFFS) &
 121                                             PUP_PHASE_MASK;
 122                                         delay = reg & PUP_DELAY_MASK;
 123                                         dram_info->wl_val[cs][pup][P] = phase;
 124                                         dram_info->wl_val[cs][pup][D] = delay;
 125                                         dram_info->wl_val[cs][pup][S] =
 126                                             WL_HI_FREQ_STATE - 1;
 127                                         reg =
 128                                             ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 129                                                               cs, pup);
 130                                         dram_info->wl_val[cs][pup][DQS] =
 131                                             (reg & 0x3F);
 132                                 }
 133
 134 #ifdef MV_DEBUG_WL
 135                                 /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
 136                                 DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 137                                 DEBUG_WL_D((u32) cs, 1);
 138                                 DEBUG_WL_S(" Results:\n");
 139                                 for (pup = 0;
 140                                      pup < dram_info->num_of_total_pups;
 141                                      pup++) {
 142                                         if (pup == dram_info->num_of_std_pups
 143                                             && dram_info->ecc_ena)
 144                                                 pup = ECC_PUP;
 145                                         DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
 146                                         DEBUG_WL_D((u32) pup, 1);
 147                                         DEBUG_WL_S(", Phase: ");
 148                                         DEBUG_WL_D((u32)
 149                                                    dram_info->wl_val[cs][pup]
 150                                                    [P], 1);
 151                                         DEBUG_WL_S(", Delay: ");
 152                                         DEBUG_WL_D((u32)
 153                                                    dram_info->wl_val[cs][pup]
 154                                                    [D], 2);
 155                                         DEBUG_WL_S("\n");
 156                                 }
 157 #endif
 158                         }
 159                 }
 160
 161                 /* Dynamic pad issue (BTS669) during WL */
 162 #ifdef MV88F67XX
 163                 if (dpde_flag) {
 164                         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 165                                 (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 166                         reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 167                 }
 168 #endif
 169
 170                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 171
 172                 return MV_OK;
 173         } else {
 174                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Error\n");
 175                 return MV_FAIL;
 176         }
 177 }
 178
 179 /*
 180  * Name:     ddr3_wl_supplement
 181  * Desc:     Write Leveling Supplement
 182  * Args:     dram_info   - main struct
 183  * Notes:
 184  * Returns:  MV_OK if success, MV_FAIL if fail.
 185  */
 186 int ddr3_wl_supplement(MV_DRAM_INFO *dram_info)
 187 {
 188         u32 cs, cnt, pup_num, sum, phase, delay, max_pup_num, pup, sdram_offset;
 189         u32 tmp_count, ecc, reg;
 190         u32 ddr_width, tmp_pup, idx;
 191         u32 sdram_pup_val, uj;
 192         u32 one_clk_err = 0, align_err = 0, no_err = 0, err = 0, err_n = 0;
 193         u32 sdram_data[LEN_WL_SUP_PATTERN] __aligned(32) = { 0 };
 194
 195         ddr_width = dram_info->ddr_width;
 196         no_err = 0;
 197
 198         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Starting\n");
 199
 200         switch (ddr_width) {
 201                 /* Data error from pos-adge to pos-adge */
 202         case 16:
 203                 one_clk_err = 4;
 204                 align_err = 4;
 205                 break;
 206         case 32:
 207                 one_clk_err = 8;
 208                 align_err = 8;
 209                 break;
 210         case 64:
 211                 one_clk_err = 0x10;
 212                 align_err = 0x10;
 213                 break;
 214         default:
 215                 DEBUG_WL_S("Error - bus width!!!\n");
 216                 return MV_FAIL;
 217         }
 218
 219         /* Enable SW override */
 220         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 221                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 222
 223         /* [0] = 1 - Enable SW override  */
 224         /* 0x15B8 - Training SW 2 Register */
 225         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 226         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - SW Override Enabled\n");
 227         reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 228         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 229         tmp_count = 0;
 230         for (cs = 0; cs < MAX_CS; cs++) {
 231                 if (dram_info->cs_ena & (1 << cs)) {
 232                         sum = 0;
 233                         /*
 234                          * 2 iterations loop: 1)actual WL results 2) fix WL
 235                          * if needed
 236                          */
 237                         for (cnt = 0; cnt < COUNT_WL_HI_FREQ; cnt++) {
 238                                 DEBUG_WL_C("COUNT = ", cnt, 1);
 239                                 for (ecc = 0; ecc < (dram_info->ecc_ena + 1);
 240                                      ecc++) {
 241                                         if (ecc) {
 242                                                 DEBUG_WL_S("ECC PUP:\n");
 243                                         } else {
 244                                                 DEBUG_WL_S("DATA PUP:\n");
 245                                         }
 246
 247                                         max_pup_num =
 248                                             dram_info->num_of_std_pups * (1 -
 249                                                                           ecc) +
 250                                             ecc;
 251                                         /* ECC Support - Switch ECC Mux on ecc=1 */
 252                                         reg =
 253                                             (reg_read(REG_DRAM_TRAINING_2_ADDR)
 254                                              & ~(1 <<
 255                                                  REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 256                                         reg |=
 257                                             (dram_info->ecc_ena *
 258                                              ecc <<
 259                                              REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 260                                         reg_write(REG_DRAM_TRAINING_2_ADDR,
 261                                                   reg);
 262                                         ddr3_reset_phy_read_fifo();
 263
 264                                         /* Write to memory */
 265                                         sdram_offset =
 266                                             tmp_count * (SDRAM_CS_SIZE + 1) +
 267                                             0x200;
 268                                         if (MV_OK != ddr3_dram_sram_burst((u32)
 269                                                                           wl_sup_pattern,
 270                                                                           sdram_offset,
 271                                                                           LEN_WL_SUP_PATTERN))
 272                                                 return MV_FAIL;
 273
 274                                         /* Read from memory */
 275                                         if (MV_OK !=
 276                                             ddr3_dram_sram_burst(sdram_offset,
 277                                                                  (u32)
 278                                                                  sdram_data,
 279                                                                  LEN_WL_SUP_PATTERN))
 280                                                 return MV_FAIL;
 281
 282                                         /* Print the buffer */
 283                                         for (uj = 0; uj < LEN_WL_SUP_PATTERN;
 284                                              uj++) {
 285                                                 if ((uj % 4 == 0) && (uj != 0)) {
 286                                                         DEBUG_WL_S("\n");
 287                                                 }
 288                                                 DEBUG_WL_D(sdram_data[uj],
 289                                                            8);
 290                                                 DEBUG_WL_S(" ");
 291                                         }
 292
 293                                         /* Check pup which DQS/DATA is error */
 294                                         for (pup = 0; pup < max_pup_num; pup++) {
 295                                                 /* ECC support - bit 8 */
 296                                                 pup_num = (ecc) ? ECC_PUP : pup;
 297                                                 if (pup < 4) {  /* lower 32 bit */
 298                                                         tmp_pup = pup;
 299                                                         idx =
 300                                                             WL_SUP_READ_DRAM_ENTRY;
 301                                                 } else {        /* higher 32 bit */
 302                                                         tmp_pup = pup - 4;
 303                                                         idx =
 304                                                             WL_SUP_READ_DRAM_ENTRY
 305                                                             + 1;
 306                                                 }
 307                                                 DEBUG_WL_S("\nCS: ");
 308                                                 DEBUG_WL_D((u32) cs, 1);
 309                                                 DEBUG_WL_S(" PUP: ");
 310                                                 DEBUG_WL_D((u32) pup_num, 1);
 311                                                 DEBUG_WL_S("\n");
 312                                                 sdram_pup_val =
 313                                                     ((sdram_data[idx] >>
 314                                                       ((tmp_pup) * 8)) & 0xFF);
 315                                                 DEBUG_WL_C("Actual Data = ",
 316                                                            sdram_pup_val, 2);
 317                                                 DEBUG_WL_C("Expected Data = ",
 318                                                            (WL_SUP_EXPECTED_DATA
 319                                                             + pup), 2);
 320                                                 /*
 321                                                  * ALINGHMENT: calculate
 322                                                  * expected data vs actual data
 323                                                  */
 324                                                 err =
 325                                                     (WL_SUP_EXPECTED_DATA +
 326                                                      pup) - sdram_pup_val;
 327                                                 /*
 328                                                  * CLOCK LONG: calculate
 329                                                  * expected data vs actual data
 330                                                  */
 331                                                 err_n =
 332                                                     sdram_pup_val -
 333                                                     (WL_SUP_EXPECTED_DATA +
 334                                                      pup);
 335                                                 DEBUG_WL_C("err = ", err, 2);
 336                                                 DEBUG_WL_C("err_n = ", err_n,
 337                                                            2);
 338                                                 if (err == no_err) {
 339                                                         /* PUP is correct - increment State */
 340                                                         dram_info->wl_val[cs]
 341                                                             [pup_num]
 342                                                             [S] = 1;
 343                                                 } else if (err_n == one_clk_err) {
 344                                                         /* clock is longer than DQS */
 345                                                         phase =
 346                                                             ((dram_info->wl_val
 347                                                               [cs]
 348                                                               [pup_num][P] +
 349                                                               WL_HI_FREQ_SHIFT)
 350                                                              % MAX_PHASE_2TO1);
 351                                                         dram_info->wl_val[cs]
 352                                                             [pup_num]
 353                                                             [P] = phase;
 354                                                         delay =
 355                                                             dram_info->wl_val
 356                                                             [cs][pup_num]
 357                                                             [D];
 358                                                         DEBUG_WL_S("#### Clock is longer than DQS more than one clk cycle ####\n");
 359                                                         ddr3_write_pup_reg
 360                                                             (PUP_WL_MODE, cs,
 361                                                              pup * (1 - ecc) +
 362                                                              ECC_PUP * ecc,
 363                                                              phase, delay);
 364                                                 } else if (err == align_err) {
 365                                                         /* clock is align to DQS */
 366                                                         phase =
 367                                                             dram_info->wl_val
 368                                                             [cs][pup_num]
 369                                                             [P];
 370                                                         delay =
 371                                                             dram_info->wl_val
 372                                                             [cs][pup_num]
 373                                                             [D];
 374                                                         DEBUG_WL_S("#### Alignment PUPS problem ####\n");
 375                                                         if ((phase == 0)
 376                                                             || ((phase == 1)
 377                                                                 && (delay <=
 378                                                                     0x10))) {
 379                                                                 DEBUG_WL_S("#### Warning - Possible Layout Violation (DQS is longer than CLK)####\n");
 380                                                         }
 381
 382                                                         phase = 0x0;
 383                                                         delay = 0x0;
 384                                                         dram_info->wl_val[cs]
 385                                                             [pup_num]
 386                                                             [P] = phase;
 387                                                         dram_info->wl_val[cs]
 388                                                             [pup_num]
 389                                                             [D] = delay;
 390                                                         ddr3_write_pup_reg
 391                                                             (PUP_WL_MODE, cs,
 392                                                              pup * (1 - ecc) +
 393                                                              ECC_PUP * ecc,
 394                                                              phase, delay);
 395                                                 }
 396                                                 /* Stop condition for ECC phase */
 397                                                 pup = (ecc) ? max_pup_num : pup;
 398                                         }
 399
 400                                         /* ECC Support - Disable ECC MUX */
 401                                         reg =
 402                                             (reg_read(REG_DRAM_TRAINING_2_ADDR)
 403                                              & ~(1 <<
 404                                                  REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 405                                         reg_write(REG_DRAM_TRAINING_2_ADDR,
 406                                                   reg);
 407                                 }
 408                         }
 409
 410                         for (pup = 0; pup < dram_info->num_of_std_pups; pup++)
 411                                 sum += dram_info->wl_val[cs][pup][S];
 412
 413                         if (dram_info->ecc_ena)
 414                                 sum += dram_info->wl_val[cs][ECC_PUP][S];
 415
 416                         /* Checks if any pup is not locked after the change */
 417                         if (sum < (WL_HI_FREQ_STATE * (dram_info->num_of_total_pups))) {
 418                                 DEBUG_WL_C("DDR3 - Write Leveling Hi-Freq Supplement - didn't work for Cs - ",
 419                                            (u32) cs, 1);
 420                                 return MV_FAIL;
 421                         }
 422                         tmp_count++;
 423                 }
 424         }
 425
 426         dram_info->wl_max_phase = 0;
 427         dram_info->wl_min_phase = 10;
 428
 429         /*
 430          * Read results to arrays - Results are required for DQS Centralization
 431          */
 432         for (cs = 0; cs < MAX_CS; cs++) {
 433                 if (dram_info->cs_ena & (1 << cs)) {
 434                         for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 435                                 if (pup == dram_info->num_of_std_pups
 436                                     && dram_info->ecc_ena)
 437                                         pup = ECC_PUP;
 438                                 reg = ddr3_read_pup_reg(PUP_WL_MODE, cs, pup);
 439                                 phase =
 440                                     (reg >> REG_PHY_PHASE_OFFS) &
 441                                     PUP_PHASE_MASK;
 442                                 if (phase > dram_info->wl_max_phase)
 443                                         dram_info->wl_max_phase = phase;
 444                                 if (phase < dram_info->wl_min_phase)
 445                                         dram_info->wl_min_phase = phase;
 446                         }
 447                 }
 448         }
 449
 450         /* Disable SW override - Must be in a different stage */
 451         /* [0]=0 - Enable SW override  */
 452         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 453         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 454         /* 0x15B8 - Training SW 2 Register */
 455         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 456
 457         reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 458                 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 459         reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 460
 461         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Ended Successfully\n");
 462
 463         return MV_OK;
 464 }
 465
 466 /*
 467  * Name:     ddr3_write_leveling_hw_reg_dimm
 468  * Desc:     Execute Write leveling phase by HW
 469  * Args:     freq      - current sequence frequency
 470  *           dram_info   - main struct
 471  * Notes:
 472  * Returns:  MV_OK if success, MV_FAIL if fail.
 473  */
 474 int ddr3_write_leveling_hw_reg_dimm(u32 freq, MV_DRAM_INFO *dram_info)
 475 {
 476         u32 reg, phase, delay, cs, pup, pup_num;
 477         __maybe_unused int dpde_flag = 0;
 478
 479         /* Debug message - Start Read leveling procedure */
 480         DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
 481
 482         if (dram_info->num_cs > 2) {
 483                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 484                 return MV_NO_CHANGE;
 485         }
 486
 487         /* If target freq = 400 move clock start point */
 488         /* Write to control PUP to Control Deskew Regs */
 489         if (freq <= DDR_400) {
 490                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 491                         /* PUP_DELAY_MASK 0x1F */
 492                         /* reg = 0x0C10001F + (uj << 16); */
 493                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 494                                                 0x1F);
 495                 }
 496         }
 497
 498 #ifdef MV88F67XX
 499         /* Dynamic pad issue (BTS669) during WL */
 500         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 501         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 502                 dpde_flag = 1;
 503                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 504                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 505         }
 506 #endif
 507
 508         reg = (1 << REG_DRAM_TRAINING_WL_OFFS);
 509         /* Config the retest number */
 510         reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
 511         reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
 512         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 513
 514         reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
 515                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 516         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
 517
 518         /* Wait */
 519         do {
 520                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
 521                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 522         } while (reg);          /* Wait for '0' */
 523
 524         reg = reg_read(REG_DRAM_TRAINING_ADDR);
 525         /* Check if Successful */
 526         if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 527                 /*
 528                  * Read results to arrays - Results are required for WL High
 529                  * freq Supplement and DQS Centralization
 530                  */
 531                 for (cs = 0; cs < MAX_CS; cs++) {
 532                         if (dram_info->cs_ena & (1 << cs)) {
 533                                 for (pup = 0;
 534                                      pup < dram_info->num_of_total_pups;
 535                                      pup++) {
 536                                         if (pup == dram_info->num_of_std_pups
 537                                             && dram_info->ecc_ena)
 538                                                 pup = ECC_BIT;
 539                                         reg =
 540                                             ddr3_read_pup_reg(PUP_WL_MODE, cs,
 541                                                               pup);
 542                                         phase =
 543                                             (reg >> REG_PHY_PHASE_OFFS) &
 544                                             PUP_PHASE_MASK;
 545                                         delay = reg & PUP_DELAY_MASK;
 546                                         dram_info->wl_val[cs][pup][P] = phase;
 547                                         dram_info->wl_val[cs][pup][D] = delay;
 548                                         if ((phase == 1) && (delay >= 0x1D)) {
 549                                                 /*
 550                                                  * Need to do it here for
 551                                                  * uncorrect WL values
 552                                                  */
 553                                                 ddr3_write_pup_reg(PUP_WL_MODE,
 554                                                                    cs, pup, 0,
 555                                                                    0);
 556                                                 dram_info->wl_val[cs][pup][P] =
 557                                                     0;
 558                                                 dram_info->wl_val[cs][pup][D] =
 559                                                     0;
 560                                         }
 561                                         dram_info->wl_val[cs][pup][S] =
 562                                             WL_HI_FREQ_STATE - 1;
 563                                         reg =
 564                                             ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 565                                                               cs, pup);
 566                                         dram_info->wl_val[cs][pup][DQS] =
 567                                             (reg & 0x3F);
 568                                 }
 569 #ifdef MV_DEBUG_WL
 570                                 /*
 571                                  * Debug message - Print res for cs[i]:
 572                                  * cs,PUP,Phase,Delay
 573                                  */
 574                                 DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 575                                 DEBUG_WL_D((u32) cs, 1);
 576                                 DEBUG_WL_S(" Results:\n");
 577                                 for (pup = 0;
 578                                      pup < dram_info->num_of_total_pups;
 579                                      pup++) {
 580                                         DEBUG_WL_S
 581                                             ("DDR3 - Write Leveling - PUP: ");
 582                                         DEBUG_WL_D((u32) pup, 1);
 583                                         DEBUG_WL_S(", Phase: ");
 584                                         DEBUG_WL_D((u32)
 585                                                    dram_info->wl_val[cs][pup]
 586                                                    [P], 1);
 587                                         DEBUG_WL_S(", Delay: ");
 588                                         DEBUG_WL_D((u32)
 589                                                    dram_info->wl_val[cs][pup]
 590                                                    [D], 2);
 591                                         DEBUG_WL_S("\n");
 592                                 }
 593 #endif
 594                         }
 595                 }
 596
 597 #ifdef MV88F67XX
 598                 /* Dynamic pad issue (BTS669) during WL */
 599                 if (dpde_flag) {
 600                         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 601                                 (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 602                         reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 603                 }
 604 #endif
 605                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 606
 607                 /* If target freq = 400 move clock back */
 608                 /* Write to control PUP to Control Deskew Regs */
 609                 if (freq <= DDR_400) {
 610                         for (pup = 0; pup <= dram_info->num_of_total_pups;
 611                              pup++) {
 612                                 ddr3_write_ctrl_pup_reg(1, pup,
 613                                                         CNTRL_PUP_DESKEW + pup, 0);
 614                         }
 615                 }
 616
 617                 return MV_OK;
 618         } else {
 619                 /* Configure Each PUP with locked leveling settings */
 620                 for (cs = 0; cs < MAX_CS; cs++) {
 621                         if (dram_info->cs_ena & (1 << cs)) {
 622                                 for (pup = 0;
 623                                      pup < dram_info->num_of_total_pups;
 624                                      pup++) {
 625                                         /* ECC support - bit 8 */
 626                                         pup_num = (pup == dram_info->num_of_std_pups) ?
 627                                                 ECC_BIT : pup;
 628                                         ddr3_write_pup_reg(PUP_WL_MODE, cs,
 629                                                            pup_num, 0, 0);
 630                                 }
 631                         }
 632                 }
 633
 634                 reg_write(REG_DRAM_TRAINING_ADDR, 0);
 635
 636                 /* If target freq = 400 move clock back */
 637                 /* Write to control PUP to Control Deskew Regs */
 638                 if (freq <= DDR_400) {
 639                         for (pup = 0; pup <= dram_info->num_of_total_pups;
 640                              pup++) {
 641                                 ddr3_write_ctrl_pup_reg(1, pup,
 642                                                         CNTRL_PUP_DESKEW + pup, 0);
 643                         }
 644                 }
 645
 646                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 647                 return MV_NO_CHANGE;
 648         }
 649 }
 650
 651 /*
 652  * Name:     ddr3_write_leveling_sw
 653  * Desc:     Execute Write leveling phase by SW
 654  * Args:     freq      - current sequence frequency
 655  *           dram_info   - main struct
 656  * Notes:
 657  * Returns:  MV_OK if success, MV_FAIL if fail.
 658  */
 659 int ddr3_write_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 660 {
 661         u32 reg, cs, cnt, pup, max_pup_num;
 662         u32 res[MAX_CS];
 663         max_pup_num = dram_info->num_of_total_pups;
 664         __maybe_unused int dpde_flag = 0;
 665
 666         /* Debug message - Start Write leveling procedure */
 667         DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 668
 669 #ifdef MV88F67XX
 670         /* Dynamic pad issue (BTS669) during WL */
 671         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 672         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 673                 dpde_flag = 1;
 674                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 675                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 676         }
 677 #endif
 678
 679         /* Set Output buffer-off to all CS and correct ODT values */
 680         for (cs = 0; cs < MAX_CS; cs++) {
 681                 if (dram_info->cs_ena & (1 << cs)) {
 682                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 683                                 REG_DDR3_MR1_ODT_MASK;
 684                         reg |= odt_static[dram_info->cs_ena][cs];
 685                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 686
 687                         /* 0x15D0 - DDR3 MR0 Register */
 688                         reg_write(REG_DDR3_MR1_ADDR, reg);
 689                         /* Issue MRS Command to current cs */
 690                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 691                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 692                         /*
 693                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 694                          * enable current cs
 695                          */
 696                         /* 0x1418 - SDRAM Operation Register */
 697                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 698
 699                         udelay(MRS_DELAY);
 700                 }
 701         }
 702
 703         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 704
 705         /* Enable SW override */
 706         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 707                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 708         /* [0] = 1 - Enable SW override  */
 709         /* 0x15B8 - Training SW 2 Register */
 710         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 711         DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 712
 713         /* Enable PHY write leveling mode */
 714         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 715                 ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 716         /* [2] = 0 - TrnWLMode - Enable */
 717         /* 0x15B8 - Training SW 2 Register */
 718         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 719         /* Reset WL results arry */
 720         memset(dram_info->wl_val, 0, sizeof(u32) * MAX_CS * MAX_PUP_NUM * 7);
 721
 722         /* Loop for each cs */
 723         for (cs = 0; cs < MAX_CS; cs++) {
 724                 if (dram_info->cs_ena & (1 << cs)) {
 725                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 726                                         (u32) cs, 1);
 727                         /* Refresh X9 current cs */
 728                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 729                         for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 730                                 reg =
 731                                     REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 732                                                                      (REG_SDRAM_OPERATION_CS_OFFS
 733                                                                       + cs));
 734                                 /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 735                                 reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 736
 737                                 do {
 738                                         reg =
 739                                             ((reg_read
 740                                               (REG_SDRAM_OPERATION_ADDR)) &
 741                                              REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 742                                 } while (reg);  /* Wait for '0' */
 743                         }
 744
 745                         /* Configure MR1 in Cs[CsNum] - write leveling on, output buffer on */
 746                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 747                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 748                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 749                         /* Set ODT Values */
 750                         reg &= REG_DDR3_MR1_ODT_MASK;
 751                         reg |= odt_static[dram_info->cs_ena][cs];
 752                         /* Enable WL MODE */
 753                         reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 754                         /* [7]=1, [12]=0 - Output Buffer and write leveling enabled */
 755                         reg_write(REG_DDR3_MR1_ADDR, reg);      /* 0x15D4 - DDR3 MR1 Register */
 756                         /* Issue MRS Command to current cs */
 757                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 758                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 759                         /*
 760                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 761                          * enable current cs
 762                          */
 763                         /* 0x1418 - SDRAM Operation Register */
 764                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 765
 766                         udelay(MRS_DELAY);
 767
 768                         /* Write leveling  cs[cs] */
 769                         if (MV_OK !=
 770                             ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
 771                                                           (u32 *)(res + cs),
 772                                                           dram_info)) {
 773                                 DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
 774                                                 (u32) cs, 1);
 775                                 for (pup = 0; pup < max_pup_num; pup++) {
 776                                         if (((res[cs] >> pup) & 0x1) == 0) {
 777                                                 DEBUG_WL_C("Failed Byte : ",
 778                                                            pup, 1);
 779                                         }
 780                                 }
 781                                 return MV_FAIL;
 782                         }
 783
 784                         /* Set TrnWLDeUpd - After each CS is done */
 785                         reg = reg_read(REG_TRAINING_WL_ADDR) |
 786                                 (1 << REG_TRAINING_WL_CS_DONE_OFFS);
 787                         /* 0x16AC - Training Write leveling register */
 788                         reg_write(REG_TRAINING_WL_ADDR, reg);
 789
 790                         /*
 791                          * Debug message - Finished Write leveling cs[cs] -
 792                          * each PUP Fail/Success
 793                          */
 794                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
 795                                         1);
 796                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
 797                                         (u32) res[cs], 3);
 798
 799                         /*
 800                          * Configure MR1 in cs[cs] - write leveling off (0),
 801                          * output buffer off (1)
 802                          */
 803                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 804                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 805                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 806                         /* No need to sort ODT since it is same CS */
 807                         /* 0x15D4 - DDR3 MR1 Register */
 808                         reg_write(REG_DDR3_MR1_ADDR, reg);
 809                         /* Issue MRS Command to current cs */
 810                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 811                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 812                         /*
 813                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 814                          * enable current cs
 815                          */
 816                         /* 0x1418 - SDRAM Operation Register */
 817                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 818
 819                         udelay(MRS_DELAY);
 820                 }
 821         }
 822
 823         /* Disable WL Mode */
 824         /* [2]=1 - TrnWLMode - Disable */
 825         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 826         reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 827         /* 0x15B8 - Training SW 2 Register */
 828         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 829
 830         /* Disable SW override - Must be in a different stage */
 831         /* [0]=0 - Enable SW override  */
 832         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 833         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 834         /* 0x15B8 - Training SW 2 Register */
 835         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 836
 837         /* Set Output buffer-on to all CS and correct ODT values */
 838         for (cs = 0; cs < MAX_CS; cs++) {
 839                 if (dram_info->cs_ena & (1 << cs)) {
 840                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 841                                 REG_DDR3_MR1_ODT_MASK;
 842                         reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
 843                         reg |= odt_static[dram_info->cs_ena][cs];
 844
 845                         /* 0x15D0 - DDR3 MR1 Register */
 846                         reg_write(REG_DDR3_MR1_ADDR, reg);
 847                         /* Issue MRS Command to current cs */
 848                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 849                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 850                         /*
 851                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 852                          * enable current cs
 853                          */
 854                         /* 0x1418 - SDRAM Operation Register */
 855                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 856
 857                         udelay(MRS_DELAY);
 858                 }
 859         }
 860
 861 #ifdef MV88F67XX
 862         /* Dynamic pad issue (BTS669) during WL */
 863         if (dpde_flag) {
 864                 reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 865                         (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 866                 reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 867         }
 868 #endif
 869         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
 870
 871         return MV_OK;
 872 }
 873
 874 #if !defined(MV88F672X)
 875 /*
 876  * Name:     ddr3_write_leveling_sw
 877  * Desc:     Execute Write leveling phase by SW
 878  * Args:     freq        - current sequence frequency
 879  *           dram_info   - main struct
 880  * Notes:
 881  * Returns:  MV_OK if success, MV_FAIL if fail.
 882  */
 883 int ddr3_write_leveling_sw_reg_dimm(u32 freq, int ratio_2to1,
 884                                     MV_DRAM_INFO *dram_info)
 885 {
 886         u32 reg, cs, cnt, pup;
 887         u32 res[MAX_CS];
 888         __maybe_unused int dpde_flag = 0;
 889
 890         /* Debug message - Start Write leveling procedure */
 891         DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 892
 893 #ifdef MV88F67XX
 894         /* Dynamic pad issue (BTS669) during WL */
 895         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 896         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 897                 dpde_flag = 1;
 898                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 899                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 900         }
 901 #endif
 902
 903         /* If target freq = 400 move clock start point */
 904         /* Write to control PUP to Control Deskew Regs */
 905         if (freq <= DDR_400) {
 906                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 907                         /* PUP_DELAY_MASK 0x1F */
 908                         /* reg = 0x0C10001F + (uj << 16); */
 909                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 910                                                 0x1F);
 911                 }
 912         }
 913
 914         /* Set Output buffer-off to all CS and correct ODT values */
 915         for (cs = 0; cs < MAX_CS; cs++) {
 916                 if (dram_info->cs_ena & (1 << cs)) {
 917                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 918                                 REG_DDR3_MR1_ODT_MASK;
 919                         reg |= odt_static[dram_info->cs_ena][cs];
 920                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 921
 922                         /* 0x15D0 - DDR3 MR0 Register */
 923                         reg_write(REG_DDR3_MR1_ADDR, reg);
 924                         /* Issue MRS Command to current cs */
 925                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 926                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 927                         /*
 928                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 929                          * enable current cs
 930                          */
 931                         /* 0x1418 - SDRAM Operation Register */
 932                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 933
 934                         udelay(MRS_DELAY);
 935                 }
 936         }
 937
 938         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 939
 940         /* Enable SW override */
 941         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 942                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 943         /* [0] = 1 - Enable SW override  */
 944         /* 0x15B8 - Training SW 2 Register */
 945         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 946         DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 947
 948         /* Enable PHY write leveling mode */
 949         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 950                 ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 951         /* [2] = 0 - TrnWLMode - Enable */
 952         /* 0x15B8 - Training SW 2 Register */
 953         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 954
 955         /* Loop for each cs */
 956         for (cs = 0; cs < MAX_CS; cs++) {
 957                 if (dram_info->cs_ena & (1 << cs)) {
 958                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 959                                         (u32) cs, 1);
 960
 961                         /* Refresh X9 current cs */
 962                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 963                         for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 964                                 reg =
 965                                     REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 966                                                                      (REG_SDRAM_OPERATION_CS_OFFS
 967                                                                       + cs));
 968                                 /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 969                                 reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 970
 971                                 do {
 972                                         reg =
 973                                             ((reg_read
 974                                               (REG_SDRAM_OPERATION_ADDR)) &
 975                                              REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 976                                 } while (reg);  /* Wait for '0' */
 977                         }
 978
 979                         /*
 980                          * Configure MR1 in Cs[CsNum] - write leveling on,
 981                          * output buffer on
 982                          */
 983                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 984                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 985                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 986                         /* Set ODT Values */
 987                         reg &= REG_DDR3_MR1_ODT_MASK;
 988                         reg |= odt_static[dram_info->cs_ena][cs];
 989                         /* Enable WL MODE */
 990                         reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 991                         /*
 992                          * [7]=1, [12]=0 - Output Buffer and write leveling
 993                          * enabled
 994                          */
 995                         /* 0x15D4 - DDR3 MR1 Register */
 996                         reg_write(REG_DDR3_MR1_ADDR, reg);
 997                         /* Issue MRS Command to current cs */
 998                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 999                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1000                         /*
1001                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1002                          * enable current cs
1003                          */
1004                         /* 0x1418 - SDRAM Operation Register */
1005                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1006
1007                         udelay(MRS_DELAY);
1008
1009                         /* Write leveling  cs[cs] */
1010                         if (MV_OK !=
1011                             ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
1012                                                           (u32 *)(res + cs),
1013                                                           dram_info)) {
1014                                 DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
1015                                                 (u32) cs, 1);
1016                                 return MV_FAIL;
1017                         }
1018
1019                         /* Set TrnWLDeUpd - After each CS is done */
1020                         reg = reg_read(REG_TRAINING_WL_ADDR) |
1021                                 (1 << REG_TRAINING_WL_CS_DONE_OFFS);
1022                         /* 0x16AC - Training Write leveling register */
1023                         reg_write(REG_TRAINING_WL_ADDR, reg);
1024
1025                         /*
1026                          * Debug message - Finished Write leveling cs[cs] -
1027                          * each PUP Fail/Success
1028                          */
1029                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
1030                                         1);
1031                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
1032                                         (u32) res[cs], 3);
1033
1034                         /* Configure MR1 in cs[cs] - write leveling off (0), output buffer off (1) */
1035                         reg = reg_read(REG_DDR3_MR1_ADDR) &
1036                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
1037                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
1038                         /* No need to sort ODT since it is same CS */
1039                         /* 0x15D4 - DDR3 MR1 Register */
1040                         reg_write(REG_DDR3_MR1_ADDR, reg);
1041                         /* Issue MRS Command to current cs */
1042                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
1043                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1044                         /*
1045                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1046                          * enable current cs
1047                          */
1048                         /* 0x1418 - SDRAM Operation Register */
1049                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1050
1051                         udelay(MRS_DELAY);
1052                 }
1053         }
1054
1055         /* Disable WL Mode */
1056         /* [2]=1 - TrnWLMode - Disable */
1057         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1058         reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
1059         /* 0x15B8 - Training SW 2 Register */
1060         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1061
1062         /* Disable SW override - Must be in a different stage */
1063         /* [0]=0 - Enable SW override  */
1064         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1065         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
1066         /* 0x15B8 - Training SW 2 Register */
1067         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1068
1069         /* Set Output buffer-on to all CS and correct ODT values */
1070         for (cs = 0; cs < MAX_CS; cs++) {
1071                 if (dram_info->cs_ena & (1 << cs)) {
1072                         reg = reg_read(REG_DDR3_MR1_ADDR) &
1073                                 REG_DDR3_MR1_ODT_MASK;
1074                         reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
1075                         reg |= odt_static[dram_info->cs_ena][cs];
1076
1077                         /* 0x15D0 - DDR3 MR1 Register */
1078                         reg_write(REG_DDR3_MR1_ADDR, reg);
1079                         /* Issue MRS Command to current cs */
1080                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
1081                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1082                         /*
1083                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1084                          * enable current cs
1085                          */
1086                         /* 0x1418 - SDRAM Operation Register */
1087                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1088
1089                         udelay(MRS_DELAY);
1090                 }
1091         }
1092
1093 #ifdef MV88F67XX
1094         /* Dynamic pad issue (BTS669) during WL */
1095         if (dpde_flag) {
1096                 reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
1097                         (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
1098                 reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
1099         }
1100 #endif
1101
1102         /* If target freq = 400 move clock back */
1103         /* Write to control PUP to Control Deskew Regs */
1104         if (freq <= DDR_400) {
1105                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
1106                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
1107                                                 0);
1108                 }
1109         }
1110
1111         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
1112         return MV_OK;
1113 }
1114 #endif
1115
1116 /*
1117  * Name:     ddr3_write_leveling_single_cs
1118  * Desc:     Execute Write leveling for single Chip select
1119  * Args:     cs          - current chip select
1120  *           freq        - current sequence frequency
1121  *           result      - res array
1122  *           dram_info   - main struct
1123  * Notes:
1124  * Returns:  MV_OK if success, MV_FAIL if fail.
1125  */
1126 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
1127                                          u32 *result, MV_DRAM_INFO *dram_info)
1128 {
1129         u32 reg, pup_num, delay, phase, phaseMax, max_pup_num, pup,
1130                 max_pup_mask;
1131
1132         max_pup_num = dram_info->num_of_total_pups;
1133         *result = 0;
1134         u32 flag[MAX_PUP_NUM] = { 0 };
1135
1136         DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - WL for Cs - ",
1137                         (u32) cs, 1);
1138
1139         switch (max_pup_num) {
1140         case 2:
1141                 max_pup_mask = 0x3;
1142                 break;
1143         case 4:
1144                 max_pup_mask = 0xf;
1145                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1146                 break;
1147         case 5:
1148                 max_pup_mask = 0x1f;
1149                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1150                 break;
1151         case 8:
1152                 max_pup_mask = 0xff;
1153                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1154                 break;
1155         case 9:
1156                 max_pup_mask = 0x1ff;
1157                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1158                 break;
1159         default:
1160                 DEBUG_WL_C("ddr3_write_leveling_single_cs wrong max_pup_num =  ",
1161                            max_pup_num, 3);
1162                 return MV_FAIL;
1163         }
1164
1165         /* CS ODT Override */
1166         reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1167                 REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1168         reg |= (REG_SDRAM_ODT_CTRL_HIGH_OVRD_ENA << (2 * cs));
1169         /* Set 0x3 - Enable ODT on the curent cs and disable on other cs */
1170         /* 0x1498 - SDRAM ODT Control high */
1171         reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1172
1173         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - ODT Asserted for current Cs\n");
1174
1175         /* tWLMRD Delay */
1176         /* Delay of minimum 40 Dram clock cycles - 20 Tclk cycles */
1177         udelay(1);
1178
1179         /* [1:0] - current cs number */
1180         reg = (reg_read(REG_TRAINING_WL_ADDR) & REG_TRAINING_WL_CS_MASK) | cs;
1181         reg |= (1 << REG_TRAINING_WL_UPD_OFFS); /* [2] - trnWLCsUpd */
1182         /* 0x16AC - Training Write leveling register */
1183         reg_write(REG_TRAINING_WL_ADDR, reg);
1184
1185         /* Broadcast to all PUPs: Reset DQS phase, reset leveling delay */
1186         ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, 0, 0);
1187
1188         /* Seek Edge */
1189         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Current Cs\n");
1190
1191         /* Drive DQS high for one cycle - All data PUPs */
1192         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Driving DQS high for one cycle\n");
1193         if (!ratio_2to1) {
1194                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1195                        REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_1TO1;
1196         } else {
1197                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1198                        REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_2TO1;
1199         }
1200         /* 0x16AC - Training Write leveling register */
1201         reg_write(REG_TRAINING_WL_ADDR, reg);
1202
1203         /* Wait tWLdelay */
1204         do {
1205                 /* [29] - trnWLDelayExp */
1206                 reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1207                         REG_TRAINING_WL_DELAYEXP_MASK;
1208         } while (reg == 0x0);   /* Wait for '1' */
1209
1210         /* Read WL res */
1211         reg = (reg_read(REG_TRAINING_WL_ADDR) >> REG_TRAINING_WL_RESULTS_OFFS) &
1212                 REG_TRAINING_WL_RESULTS_MASK;
1213         /* [28:20] - TrnWLResult */
1214
1215         if (!ratio_2to1) /* Different phase options for 2:1 or 1:1 modes */
1216                 phaseMax = MAX_PHASE_1TO1;
1217         else
1218                 phaseMax = MAX_PHASE_2TO1;
1219
1220         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Shift DQS + Octet Leveling\n");
1221
1222         /* Shift DQS + Octet leveling */
1223         for (phase = 0; phase < phaseMax; phase++) {
1224                 for (delay = 0; delay < MAX_DELAY; delay++) {
1225                         /*  Broadcast to all PUPs: DQS phase,leveling delay */
1226                         ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, phase,
1227                                            delay);
1228
1229                         udelay(1);      /* Delay of  3 Tclk cycles */
1230
1231                         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge: Phase = ");
1232                         DEBUG_WL_FULL_D((u32) phase, 1);
1233                         DEBUG_WL_FULL_S(", Delay = ");
1234                         DEBUG_WL_FULL_D((u32) delay, 1);
1235                         DEBUG_WL_FULL_S("\n");
1236
1237                         /* Drive DQS high for one cycle - All data PUPs */
1238                         if (!ratio_2to1) {
1239                                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1240                                        REG_TRAINING_WL_RATIO_MASK) |
1241                                         REG_TRAINING_WL_1TO1;
1242                         } else {
1243                                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1244                                        REG_TRAINING_WL_RATIO_MASK) |
1245                                         REG_TRAINING_WL_2TO1;
1246                         }
1247                         reg_write(REG_TRAINING_WL_ADDR, reg);   /* 0x16AC  */
1248
1249                         /* Wait tWLdelay */
1250                         do {
1251                                 reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1252                                         REG_TRAINING_WL_DELAYEXP_MASK;
1253                         } while (reg == 0x0);   /* [29] Wait for '1' */
1254
1255                         /* Read WL res */
1256                         reg = reg_read(REG_TRAINING_WL_ADDR);
1257                         reg = (reg >> REG_TRAINING_WL_RESULTS_OFFS) &
1258                                 REG_TRAINING_WL_RESULTS_MASK;   /* [28:20] */
1259
1260                         DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - Seek Edge: Results =  ",
1261                                         (u32) reg, 3);
1262
1263                         /* Update State machine */
1264                         for (pup = 0; pup < (max_pup_num); pup++) {
1265                                 /* ECC support - bit 8 */
1266                                 pup_num = (pup == dram_info->num_of_std_pups) ?
1267                                         ECC_BIT : pup;
1268                                 if (dram_info->wl_val[cs][pup][S] == 0) {
1269                                         /* Update phase to PUP */
1270                                         dram_info->wl_val[cs][pup][P] = phase;
1271                                         /* Update delay to PUP */
1272                                         dram_info->wl_val[cs][pup][D] = delay;
1273                                 }
1274
1275                                 if (((reg >> pup_num) & 0x1) == 0)
1276                                         flag[pup_num] = 1;
1277
1278                                 if (((reg >> pup_num) & 0x1)
1279                                     && (flag[pup_num] == 1)
1280                                     && (dram_info->wl_val[cs][pup][S] == 0)) {
1281                                         /*
1282                                          * If the PUP is locked now and in last
1283                                          * counter states
1284                                          */
1285                                         /* Go to next state */
1286                                         dram_info->wl_val[cs][pup][S] = 1;
1287                                         /* Set res */
1288                                         *result = *result | (1 << pup_num);
1289                                 }
1290                         }
1291
1292                         /* If all locked - Break the loops - Finished */
1293                         if (*result == max_pup_mask) {
1294                                 phase = phaseMax;
1295                                 delay = MAX_DELAY;
1296                                 DEBUG_WL_S("DDR3 - Write Leveling Single Cs - Seek Edge: All Locked\n");
1297                         }
1298                 }
1299         }
1300
1301         /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
1302         DEBUG_WL_C("DDR3 - Write Leveling - Results for CS - ", (u32) cs, 1);
1303         for (pup = 0; pup < (max_pup_num); pup++) {
1304                 DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
1305                 DEBUG_WL_D((u32) pup, 1);
1306                 DEBUG_WL_S(", Phase: ");
1307                 DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][P], 1);
1308                 DEBUG_WL_S(", Delay: ");
1309                 DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][D], 2);
1310                 DEBUG_WL_S("\n");
1311         }
1312
1313         /* Check if some not locked and return error */
1314         if (*result != max_pup_mask) {
1315                 DEBUG_WL_S("DDR3 - Write Leveling - ERROR - not all PUPS were locked\n");
1316                 return MV_FAIL;
1317         }
1318
1319         /* Configure Each PUP with locked leveling settings */
1320         for (pup = 0; pup < (max_pup_num); pup++) {
1321                 /* ECC support - bit 8 */
1322                 pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
1323                 phase = dram_info->wl_val[cs][pup][P];
1324                 delay = dram_info->wl_val[cs][pup][D];
1325                 ddr3_write_pup_reg(PUP_WL_MODE, cs, pup_num, phase, delay);
1326         }
1327
1328         /* CS ODT Override */
1329         reg =  reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1330                 REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1331         /* 0x1498 - SDRAM ODT Control high */
1332         reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1333
1334         return MV_OK;
1335 }
1336
1337 /*
1338  * Perform DDR3 Control PUP Indirect Write
1339  */
1340 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr, u32 data)
1341 {
1342         u32 reg = 0;
1343
1344         /* Store value for write */
1345         reg = (data & 0xFFFF);
1346
1347         /* Set bit 26 for control PHY access */
1348         reg |= (1 << REG_PHY_CNTRL_OFFS);
1349
1350         /* Configure BC or UC access to PHYs */
1351         if (bc_acc == 1)
1352                 reg |= (1 << REG_PHY_BC_OFFS);
1353         else
1354                 reg |= (pup << REG_PHY_PUP_OFFS);
1355
1356         /* Set PHY register address to write to */
1357         reg |= (reg_addr << REG_PHY_CS_OFFS);
1358
1359         reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1360         reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1361         reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1362
1363         do {
1364                 reg = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &
1365                         REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1366         } while (reg);          /* Wait for '0' to mark the end of the transaction */
1367 }