drivers/ddr/marvell/axp/ddr3_write_leveling.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Marvell International Ltd. and its affiliates
   4  */
   5
   6 #include <common.h>
   7 #include <i2c.h>
   8 #include <log.h>
   9 #include <spl.h>
  10 #include <asm/io.h>
  11 #include <asm/arch/cpu.h>
  12 #include <asm/arch/soc.h>
  13
  14 #include "ddr3_hw_training.h"
  15
  16 /*
  17  * Debug
  18  */
  19 #define DEBUG_WL_C(s, d, l) \
  20         DEBUG_WL_S(s); DEBUG_WL_D(d, l); DEBUG_WL_S("\n")
  21 #define DEBUG_WL_FULL_C(s, d, l) \
  22         DEBUG_WL_FULL_S(s); DEBUG_WL_FULL_D(d, l); DEBUG_WL_FULL_S("\n")
  23
  24 #ifdef MV_DEBUG_WL
  25 #define DEBUG_WL_S(s)                   puts(s)
  26 #define DEBUG_WL_D(d, l)                printf("%x", d)
  27 #define DEBUG_RL_S(s) \
  28         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  29 #define DEBUG_RL_D(d, l) \
  30         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  31 #else
  32 #define DEBUG_WL_S(s)
  33 #define DEBUG_WL_D(d, l)
  34 #endif
  35
  36 #ifdef MV_DEBUG_WL_FULL
  37 #define DEBUG_WL_FULL_S(s)              puts(s)
  38 #define DEBUG_WL_FULL_D(d, l)           printf("%x", d)
  39 #else
  40 #define DEBUG_WL_FULL_S(s)
  41 #define DEBUG_WL_FULL_D(d, l)
  42 #endif
  43
  44 #define WL_SUP_EXPECTED_DATA            0x21
  45 #define WL_SUP_READ_DRAM_ENTRY          0x8
  46
  47 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
  48                                          u32 *result,
  49                                          MV_DRAM_INFO *dram_info);
  50 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr,
  51                                     u32 data);
  52
  53 extern u16 odt_static[ODT_OPT][MAX_CS];
  54 extern u16 odt_dynamic[ODT_OPT][MAX_CS];
  55 extern u32 wl_sup_pattern[LEN_WL_SUP_PATTERN];
  56
  57 /*
  58  * Name:     ddr3_write_leveling_hw
  59  * Desc:     Execute Write leveling phase by HW
  60  * Args:     freq      - current sequence frequency
  61  *           dram_info   - main struct
  62  * Notes:
  63  * Returns:  MV_OK if success, MV_FAIL if fail.
  64  */
  65 int ddr3_write_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  66 {
  67         u32 reg, phase, delay, cs, pup;
  68 #ifdef MV88F67XX
  69         int dpde_flag = 0;
  70 #endif
  71         /* Debug message - Start Read leveling procedure */
  72         DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
  73
  74 #ifdef MV88F67XX
  75         /* Dynamic pad issue (BTS669) during WL */
  76         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  77         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  78                 dpde_flag = 1;
  79                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  80                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  81         }
  82 #endif
  83
  84         reg = 1 << REG_DRAM_TRAINING_WL_OFFS;
  85         /* Config the retest number */
  86         reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
  87         reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
  88         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  89
  90         reg =  reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  91                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  92         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  93
  94         /* Wait */
  95         do {
  96                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  97                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  98         } while (reg);          /* Wait for '0' */
  99
 100         reg = reg_read(REG_DRAM_TRAINING_ADDR);
 101         /* Check if Successful */
 102         if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 103                 /*
 104                  * Read results to arrays - Results are required for WL
 105                  * High freq Supplement and DQS Centralization
 106                  */
 107                 for (cs = 0; cs < MAX_CS; cs++) {
 108                         if (dram_info->cs_ena & (1 << cs)) {
 109                                 for (pup = 0;
 110                                      pup < dram_info->num_of_total_pups;
 111                                      pup++) {
 112                                         if (pup == dram_info->num_of_std_pups
 113                                             && dram_info->ecc_ena)
 114                                                 pup = ECC_PUP;
 115                                         reg =
 116                                             ddr3_read_pup_reg(PUP_WL_MODE, cs,
 117                                                               pup);
 118                                         phase =
 119                                             (reg >> REG_PHY_PHASE_OFFS) &
 120                                             PUP_PHASE_MASK;
 121                                         delay = reg & PUP_DELAY_MASK;
 122                                         dram_info->wl_val[cs][pup][P] = phase;
 123                                         dram_info->wl_val[cs][pup][D] = delay;
 124                                         dram_info->wl_val[cs][pup][S] =
 125                                             WL_HI_FREQ_STATE - 1;
 126                                         reg =
 127                                             ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 128                                                               cs, pup);
 129                                         dram_info->wl_val[cs][pup][DQS] =
 130                                             (reg & 0x3F);
 131                                 }
 132
 133 #ifdef MV_DEBUG_WL
 134                                 /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
 135                                 DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 136                                 DEBUG_WL_D((u32) cs, 1);
 137                                 DEBUG_WL_S(" Results:\n");
 138                                 for (pup = 0;
 139                                      pup < dram_info->num_of_total_pups;
 140                                      pup++) {
 141                                         if (pup == dram_info->num_of_std_pups
 142                                             && dram_info->ecc_ena)
 143                                                 pup = ECC_PUP;
 144                                         DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
 145                                         DEBUG_WL_D((u32) pup, 1);
 146                                         DEBUG_WL_S(", Phase: ");
 147                                         DEBUG_WL_D((u32)
 148                                                    dram_info->wl_val[cs][pup]
 149                                                    [P], 1);
 150                                         DEBUG_WL_S(", Delay: ");
 151                                         DEBUG_WL_D((u32)
 152                                                    dram_info->wl_val[cs][pup]
 153                                                    [D], 2);
 154                                         DEBUG_WL_S("\n");
 155                                 }
 156 #endif
 157                         }
 158                 }
 159
 160                 /* Dynamic pad issue (BTS669) during WL */
 161 #ifdef MV88F67XX
 162                 if (dpde_flag) {
 163                         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 164                                 (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 165                         reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 166                 }
 167 #endif
 168
 169                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 170
 171                 return MV_OK;
 172         } else {
 173                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Error\n");
 174                 return MV_FAIL;
 175         }
 176 }
 177
 178 /*
 179  * Name:     ddr3_wl_supplement
 180  * Desc:     Write Leveling Supplement
 181  * Args:     dram_info   - main struct
 182  * Notes:
 183  * Returns:  MV_OK if success, MV_FAIL if fail.
 184  */
 185 int ddr3_wl_supplement(MV_DRAM_INFO *dram_info)
 186 {
 187         u32 cs, cnt, pup_num, sum, phase, delay, max_pup_num, pup, sdram_offset;
 188         u32 tmp_count, ecc, reg;
 189         u32 ddr_width, tmp_pup, idx;
 190         u32 sdram_pup_val, uj;
 191         u32 one_clk_err = 0, align_err = 0, no_err = 0, err = 0, err_n = 0;
 192         u32 sdram_data[LEN_WL_SUP_PATTERN] __aligned(32) = { 0 };
 193
 194         ddr_width = dram_info->ddr_width;
 195         no_err = 0;
 196
 197         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Starting\n");
 198
 199         switch (ddr_width) {
 200                 /* Data error from pos-adge to pos-adge */
 201         case 16:
 202                 one_clk_err = 4;
 203                 align_err = 4;
 204                 break;
 205         case 32:
 206                 one_clk_err = 8;
 207                 align_err = 8;
 208                 break;
 209         case 64:
 210                 one_clk_err = 0x10;
 211                 align_err = 0x10;
 212                 break;
 213         default:
 214                 DEBUG_WL_S("Error - bus width!!!\n");
 215                 return MV_FAIL;
 216         }
 217
 218         /* Enable SW override */
 219         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 220                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 221
 222         /* [0] = 1 - Enable SW override  */
 223         /* 0x15B8 - Training SW 2 Register */
 224         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 225         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - SW Override Enabled\n");
 226         reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 227         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 228         tmp_count = 0;
 229         for (cs = 0; cs < MAX_CS; cs++) {
 230                 if (dram_info->cs_ena & (1 << cs)) {
 231                         sum = 0;
 232                         /*
 233                          * 2 iterations loop: 1)actual WL results 2) fix WL
 234                          * if needed
 235                          */
 236                         for (cnt = 0; cnt < COUNT_WL_HI_FREQ; cnt++) {
 237                                 DEBUG_WL_C("COUNT = ", cnt, 1);
 238                                 for (ecc = 0; ecc < (dram_info->ecc_ena + 1);
 239                                      ecc++) {
 240                                         if (ecc) {
 241                                                 DEBUG_WL_S("ECC PUP:\n");
 242                                         } else {
 243                                                 DEBUG_WL_S("DATA PUP:\n");
 244                                         }
 245
 246                                         max_pup_num =
 247                                             dram_info->num_of_std_pups * (1 -
 248                                                                           ecc) +
 249                                             ecc;
 250                                         /* ECC Support - Switch ECC Mux on ecc=1 */
 251                                         reg =
 252                                             (reg_read(REG_DRAM_TRAINING_2_ADDR)
 253                                              & ~(1 <<
 254                                                  REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 255                                         reg |=
 256                                             (dram_info->ecc_ena *
 257                                              ecc <<
 258                                              REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 259                                         reg_write(REG_DRAM_TRAINING_2_ADDR,
 260                                                   reg);
 261                                         ddr3_reset_phy_read_fifo();
 262
 263                                         /* Write to memory */
 264                                         sdram_offset =
 265                                             tmp_count * (SDRAM_CS_SIZE + 1) +
 266                                             0x200;
 267                                         if (MV_OK != ddr3_dram_sram_burst((u32)
 268                                                                           wl_sup_pattern,
 269                                                                           sdram_offset,
 270                                                                           LEN_WL_SUP_PATTERN))
 271                                                 return MV_FAIL;
 272
 273                                         /* Read from memory */
 274                                         if (MV_OK !=
 275                                             ddr3_dram_sram_burst(sdram_offset,
 276                                                                  (u32)
 277                                                                  sdram_data,
 278                                                                  LEN_WL_SUP_PATTERN))
 279                                                 return MV_FAIL;
 280
 281                                         /* Print the buffer */
 282                                         for (uj = 0; uj < LEN_WL_SUP_PATTERN;
 283                                              uj++) {
 284                                                 if ((uj % 4 == 0) && (uj != 0)) {
 285                                                         DEBUG_WL_S("\n");
 286                                                 }
 287                                                 DEBUG_WL_D(sdram_data[uj],
 288                                                            8);
 289                                                 DEBUG_WL_S(" ");
 290                                         }
 291
 292                                         /* Check pup which DQS/DATA is error */
 293                                         for (pup = 0; pup < max_pup_num; pup++) {
 294                                                 /* ECC support - bit 8 */
 295                                                 pup_num = (ecc) ? ECC_PUP : pup;
 296                                                 if (pup < 4) {  /* lower 32 bit */
 297                                                         tmp_pup = pup;
 298                                                         idx =
 299                                                             WL_SUP_READ_DRAM_ENTRY;
 300                                                 } else {        /* higher 32 bit */
 301                                                         tmp_pup = pup - 4;
 302                                                         idx =
 303                                                             WL_SUP_READ_DRAM_ENTRY
 304                                                             + 1;
 305                                                 }
 306                                                 DEBUG_WL_S("\nCS: ");
 307                                                 DEBUG_WL_D((u32) cs, 1);
 308                                                 DEBUG_WL_S(" PUP: ");
 309                                                 DEBUG_WL_D((u32) pup_num, 1);
 310                                                 DEBUG_WL_S("\n");
 311                                                 sdram_pup_val =
 312                                                     ((sdram_data[idx] >>
 313                                                       ((tmp_pup) * 8)) & 0xFF);
 314                                                 DEBUG_WL_C("Actual Data = ",
 315                                                            sdram_pup_val, 2);
 316                                                 DEBUG_WL_C("Expected Data = ",
 317                                                            (WL_SUP_EXPECTED_DATA
 318                                                             + pup), 2);
 319                                                 /*
 320                                                  * ALINGHMENT: calculate
 321                                                  * expected data vs actual data
 322                                                  */
 323                                                 err =
 324                                                     (WL_SUP_EXPECTED_DATA +
 325                                                      pup) - sdram_pup_val;
 326                                                 /*
 327                                                  * CLOCK LONG: calculate
 328                                                  * expected data vs actual data
 329                                                  */
 330                                                 err_n =
 331                                                     sdram_pup_val -
 332                                                     (WL_SUP_EXPECTED_DATA +
 333                                                      pup);
 334                                                 DEBUG_WL_C("err = ", err, 2);
 335                                                 DEBUG_WL_C("err_n = ", err_n,
 336                                                            2);
 337                                                 if (err == no_err) {
 338                                                         /* PUP is correct - increment State */
 339                                                         dram_info->wl_val[cs]
 340                                                             [pup_num]
 341                                                             [S] = 1;
 342                                                 } else if (err_n == one_clk_err) {
 343                                                         /* clock is longer than DQS */
 344                                                         phase =
 345                                                             ((dram_info->wl_val
 346                                                               [cs]
 347                                                               [pup_num][P] +
 348                                                               WL_HI_FREQ_SHIFT)
 349                                                              % MAX_PHASE_2TO1);
 350                                                         dram_info->wl_val[cs]
 351                                                             [pup_num]
 352                                                             [P] = phase;
 353                                                         delay =
 354                                                             dram_info->wl_val
 355                                                             [cs][pup_num]
 356                                                             [D];
 357                                                         DEBUG_WL_S("#### Clock is longer than DQS more than one clk cycle ####\n");
 358                                                         ddr3_write_pup_reg
 359                                                             (PUP_WL_MODE, cs,
 360                                                              pup * (1 - ecc) +
 361                                                              ECC_PUP * ecc,
 362                                                              phase, delay);
 363                                                 } else if (err == align_err) {
 364                                                         /* clock is align to DQS */
 365                                                         phase =
 366                                                             dram_info->wl_val
 367                                                             [cs][pup_num]
 368                                                             [P];
 369                                                         delay =
 370                                                             dram_info->wl_val
 371                                                             [cs][pup_num]
 372                                                             [D];
 373                                                         DEBUG_WL_S("#### Alignment PUPS problem ####\n");
 374                                                         if ((phase == 0)
 375                                                             || ((phase == 1)
 376                                                                 && (delay <=
 377                                                                     0x10))) {
 378                                                                 DEBUG_WL_S("#### Warning - Possible Layout Violation (DQS is longer than CLK)####\n");
 379                                                         }
 380
 381                                                         phase = 0x0;
 382                                                         delay = 0x0;
 383                                                         dram_info->wl_val[cs]
 384                                                             [pup_num]
 385                                                             [P] = phase;
 386                                                         dram_info->wl_val[cs]
 387                                                             [pup_num]
 388                                                             [D] = delay;
 389                                                         ddr3_write_pup_reg
 390                                                             (PUP_WL_MODE, cs,
 391                                                              pup * (1 - ecc) +
 392                                                              ECC_PUP * ecc,
 393                                                              phase, delay);
 394                                                 }
 395                                                 /* Stop condition for ECC phase */
 396                                                 pup = (ecc) ? max_pup_num : pup;
 397                                         }
 398
 399                                         /* ECC Support - Disable ECC MUX */
 400                                         reg =
 401                                             (reg_read(REG_DRAM_TRAINING_2_ADDR)
 402                                              & ~(1 <<
 403                                                  REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 404                                         reg_write(REG_DRAM_TRAINING_2_ADDR,
 405                                                   reg);
 406                                 }
 407                         }
 408
 409                         for (pup = 0; pup < dram_info->num_of_std_pups; pup++)
 410                                 sum += dram_info->wl_val[cs][pup][S];
 411
 412                         if (dram_info->ecc_ena)
 413                                 sum += dram_info->wl_val[cs][ECC_PUP][S];
 414
 415                         /* Checks if any pup is not locked after the change */
 416                         if (sum < (WL_HI_FREQ_STATE * (dram_info->num_of_total_pups))) {
 417                                 DEBUG_WL_C("DDR3 - Write Leveling Hi-Freq Supplement - didn't work for Cs - ",
 418                                            (u32) cs, 1);
 419                                 return MV_FAIL;
 420                         }
 421                         tmp_count++;
 422                 }
 423         }
 424
 425         dram_info->wl_max_phase = 0;
 426         dram_info->wl_min_phase = 10;
 427
 428         /*
 429          * Read results to arrays - Results are required for DQS Centralization
 430          */
 431         for (cs = 0; cs < MAX_CS; cs++) {
 432                 if (dram_info->cs_ena & (1 << cs)) {
 433                         for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 434                                 if (pup == dram_info->num_of_std_pups
 435                                     && dram_info->ecc_ena)
 436                                         pup = ECC_PUP;
 437                                 reg = ddr3_read_pup_reg(PUP_WL_MODE, cs, pup);
 438                                 phase =
 439                                     (reg >> REG_PHY_PHASE_OFFS) &
 440                                     PUP_PHASE_MASK;
 441                                 if (phase > dram_info->wl_max_phase)
 442                                         dram_info->wl_max_phase = phase;
 443                                 if (phase < dram_info->wl_min_phase)
 444                                         dram_info->wl_min_phase = phase;
 445                         }
 446                 }
 447         }
 448
 449         /* Disable SW override - Must be in a different stage */
 450         /* [0]=0 - Enable SW override  */
 451         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 452         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 453         /* 0x15B8 - Training SW 2 Register */
 454         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 455
 456         reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 457                 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 458         reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 459
 460         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Ended Successfully\n");
 461
 462         return MV_OK;
 463 }
 464
 465 /*
 466  * Name:     ddr3_write_leveling_hw_reg_dimm
 467  * Desc:     Execute Write leveling phase by HW
 468  * Args:     freq      - current sequence frequency
 469  *           dram_info   - main struct
 470  * Notes:
 471  * Returns:  MV_OK if success, MV_FAIL if fail.
 472  */
 473 int ddr3_write_leveling_hw_reg_dimm(u32 freq, MV_DRAM_INFO *dram_info)
 474 {
 475         u32 reg, phase, delay, cs, pup, pup_num;
 476         __maybe_unused int dpde_flag = 0;
 477
 478         /* Debug message - Start Read leveling procedure */
 479         DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
 480
 481         if (dram_info->num_cs > 2) {
 482                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 483                 return MV_NO_CHANGE;
 484         }
 485
 486         /* If target freq = 400 move clock start point */
 487         /* Write to control PUP to Control Deskew Regs */
 488         if (freq <= DDR_400) {
 489                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 490                         /* PUP_DELAY_MASK 0x1F */
 491                         /* reg = 0x0C10001F + (uj << 16); */
 492                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 493                                                 0x1F);
 494                 }
 495         }
 496
 497 #ifdef MV88F67XX
 498         /* Dynamic pad issue (BTS669) during WL */
 499         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 500         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 501                 dpde_flag = 1;
 502                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 503                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 504         }
 505 #endif
 506
 507         reg = (1 << REG_DRAM_TRAINING_WL_OFFS);
 508         /* Config the retest number */
 509         reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
 510         reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
 511         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 512
 513         reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
 514                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 515         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
 516
 517         /* Wait */
 518         do {
 519                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
 520                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 521         } while (reg);          /* Wait for '0' */
 522
 523         reg = reg_read(REG_DRAM_TRAINING_ADDR);
 524         /* Check if Successful */
 525         if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 526                 /*
 527                  * Read results to arrays - Results are required for WL High
 528                  * freq Supplement and DQS Centralization
 529                  */
 530                 for (cs = 0; cs < MAX_CS; cs++) {
 531                         if (dram_info->cs_ena & (1 << cs)) {
 532                                 for (pup = 0;
 533                                      pup < dram_info->num_of_total_pups;
 534                                      pup++) {
 535                                         if (pup == dram_info->num_of_std_pups
 536                                             && dram_info->ecc_ena)
 537                                                 pup = ECC_BIT;
 538                                         reg =
 539                                             ddr3_read_pup_reg(PUP_WL_MODE, cs,
 540                                                               pup);
 541                                         phase =
 542                                             (reg >> REG_PHY_PHASE_OFFS) &
 543                                             PUP_PHASE_MASK;
 544                                         delay = reg & PUP_DELAY_MASK;
 545                                         dram_info->wl_val[cs][pup][P] = phase;
 546                                         dram_info->wl_val[cs][pup][D] = delay;
 547                                         if ((phase == 1) && (delay >= 0x1D)) {
 548                                                 /*
 549                                                  * Need to do it here for
 550                                                  * uncorrect WL values
 551                                                  */
 552                                                 ddr3_write_pup_reg(PUP_WL_MODE,
 553                                                                    cs, pup, 0,
 554                                                                    0);
 555                                                 dram_info->wl_val[cs][pup][P] =
 556                                                     0;
 557                                                 dram_info->wl_val[cs][pup][D] =
 558                                                     0;
 559                                         }
 560                                         dram_info->wl_val[cs][pup][S] =
 561                                             WL_HI_FREQ_STATE - 1;
 562                                         reg =
 563                                             ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 564                                                               cs, pup);
 565                                         dram_info->wl_val[cs][pup][DQS] =
 566                                             (reg & 0x3F);
 567                                 }
 568 #ifdef MV_DEBUG_WL
 569                                 /*
 570                                  * Debug message - Print res for cs[i]:
 571                                  * cs,PUP,Phase,Delay
 572                                  */
 573                                 DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 574                                 DEBUG_WL_D((u32) cs, 1);
 575                                 DEBUG_WL_S(" Results:\n");
 576                                 for (pup = 0;
 577                                      pup < dram_info->num_of_total_pups;
 578                                      pup++) {
 579                                         DEBUG_WL_S
 580                                             ("DDR3 - Write Leveling - PUP: ");
 581                                         DEBUG_WL_D((u32) pup, 1);
 582                                         DEBUG_WL_S(", Phase: ");
 583                                         DEBUG_WL_D((u32)
 584                                                    dram_info->wl_val[cs][pup]
 585                                                    [P], 1);
 586                                         DEBUG_WL_S(", Delay: ");
 587                                         DEBUG_WL_D((u32)
 588                                                    dram_info->wl_val[cs][pup]
 589                                                    [D], 2);
 590                                         DEBUG_WL_S("\n");
 591                                 }
 592 #endif
 593                         }
 594                 }
 595
 596 #ifdef MV88F67XX
 597                 /* Dynamic pad issue (BTS669) during WL */
 598                 if (dpde_flag) {
 599                         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 600                                 (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 601                         reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 602                 }
 603 #endif
 604                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 605
 606                 /* If target freq = 400 move clock back */
 607                 /* Write to control PUP to Control Deskew Regs */
 608                 if (freq <= DDR_400) {
 609                         for (pup = 0; pup <= dram_info->num_of_total_pups;
 610                              pup++) {
 611                                 ddr3_write_ctrl_pup_reg(1, pup,
 612                                                         CNTRL_PUP_DESKEW + pup, 0);
 613                         }
 614                 }
 615
 616                 return MV_OK;
 617         } else {
 618                 /* Configure Each PUP with locked leveling settings */
 619                 for (cs = 0; cs < MAX_CS; cs++) {
 620                         if (dram_info->cs_ena & (1 << cs)) {
 621                                 for (pup = 0;
 622                                      pup < dram_info->num_of_total_pups;
 623                                      pup++) {
 624                                         /* ECC support - bit 8 */
 625                                         pup_num = (pup == dram_info->num_of_std_pups) ?
 626                                                 ECC_BIT : pup;
 627                                         ddr3_write_pup_reg(PUP_WL_MODE, cs,
 628                                                            pup_num, 0, 0);
 629                                 }
 630                         }
 631                 }
 632
 633                 reg_write(REG_DRAM_TRAINING_ADDR, 0);
 634
 635                 /* If target freq = 400 move clock back */
 636                 /* Write to control PUP to Control Deskew Regs */
 637                 if (freq <= DDR_400) {
 638                         for (pup = 0; pup <= dram_info->num_of_total_pups;
 639                              pup++) {
 640                                 ddr3_write_ctrl_pup_reg(1, pup,
 641                                                         CNTRL_PUP_DESKEW + pup, 0);
 642                         }
 643                 }
 644
 645                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 646                 return MV_NO_CHANGE;
 647         }
 648 }
 649
 650 /*
 651  * Name:     ddr3_write_leveling_sw
 652  * Desc:     Execute Write leveling phase by SW
 653  * Args:     freq      - current sequence frequency
 654  *           dram_info   - main struct
 655  * Notes:
 656  * Returns:  MV_OK if success, MV_FAIL if fail.
 657  */
 658 int ddr3_write_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 659 {
 660         u32 reg, cs, cnt, pup, max_pup_num;
 661         u32 res[MAX_CS];
 662         max_pup_num = dram_info->num_of_total_pups;
 663         __maybe_unused int dpde_flag = 0;
 664
 665         /* Debug message - Start Write leveling procedure */
 666         DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 667
 668 #ifdef MV88F67XX
 669         /* Dynamic pad issue (BTS669) during WL */
 670         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 671         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 672                 dpde_flag = 1;
 673                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 674                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 675         }
 676 #endif
 677
 678         /* Set Output buffer-off to all CS and correct ODT values */
 679         for (cs = 0; cs < MAX_CS; cs++) {
 680                 if (dram_info->cs_ena & (1 << cs)) {
 681                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 682                                 REG_DDR3_MR1_ODT_MASK;
 683                         reg |= odt_static[dram_info->cs_ena][cs];
 684                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 685
 686                         /* 0x15D0 - DDR3 MR0 Register */
 687                         reg_write(REG_DDR3_MR1_ADDR, reg);
 688                         /* Issue MRS Command to current cs */
 689                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 690                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 691                         /*
 692                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 693                          * enable current cs
 694                          */
 695                         /* 0x1418 - SDRAM Operation Register */
 696                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 697
 698                         udelay(MRS_DELAY);
 699                 }
 700         }
 701
 702         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 703
 704         /* Enable SW override */
 705         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 706                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 707         /* [0] = 1 - Enable SW override  */
 708         /* 0x15B8 - Training SW 2 Register */
 709         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 710         DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 711
 712         /* Enable PHY write leveling mode */
 713         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 714                 ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 715         /* [2] = 0 - TrnWLMode - Enable */
 716         /* 0x15B8 - Training SW 2 Register */
 717         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 718         /* Reset WL results arry */
 719         memset(dram_info->wl_val, 0, sizeof(u32) * MAX_CS * MAX_PUP_NUM * 7);
 720
 721         /* Loop for each cs */
 722         for (cs = 0; cs < MAX_CS; cs++) {
 723                 if (dram_info->cs_ena & (1 << cs)) {
 724                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 725                                         (u32) cs, 1);
 726                         /* Refresh X9 current cs */
 727                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 728                         for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 729                                 reg =
 730                                     REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 731                                                                      (REG_SDRAM_OPERATION_CS_OFFS
 732                                                                       + cs));
 733                                 /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 734                                 reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 735
 736                                 do {
 737                                         reg =
 738                                             ((reg_read
 739                                               (REG_SDRAM_OPERATION_ADDR)) &
 740                                              REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 741                                 } while (reg);  /* Wait for '0' */
 742                         }
 743
 744                         /* Configure MR1 in Cs[CsNum] - write leveling on, output buffer on */
 745                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 746                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 747                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 748                         /* Set ODT Values */
 749                         reg &= REG_DDR3_MR1_ODT_MASK;
 750                         reg |= odt_static[dram_info->cs_ena][cs];
 751                         /* Enable WL MODE */
 752                         reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 753                         /* [7]=1, [12]=0 - Output Buffer and write leveling enabled */
 754                         reg_write(REG_DDR3_MR1_ADDR, reg);      /* 0x15D4 - DDR3 MR1 Register */
 755                         /* Issue MRS Command to current cs */
 756                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 757                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 758                         /*
 759                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 760                          * enable current cs
 761                          */
 762                         /* 0x1418 - SDRAM Operation Register */
 763                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 764
 765                         udelay(MRS_DELAY);
 766
 767                         /* Write leveling  cs[cs] */
 768                         if (MV_OK !=
 769                             ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
 770                                                           (u32 *)(res + cs),
 771                                                           dram_info)) {
 772                                 DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
 773                                                 (u32) cs, 1);
 774                                 for (pup = 0; pup < max_pup_num; pup++) {
 775                                         if (((res[cs] >> pup) & 0x1) == 0) {
 776                                                 DEBUG_WL_C("Failed Byte : ",
 777                                                            pup, 1);
 778                                         }
 779                                 }
 780                                 return MV_FAIL;
 781                         }
 782
 783                         /* Set TrnWLDeUpd - After each CS is done */
 784                         reg = reg_read(REG_TRAINING_WL_ADDR) |
 785                                 (1 << REG_TRAINING_WL_CS_DONE_OFFS);
 786                         /* 0x16AC - Training Write leveling register */
 787                         reg_write(REG_TRAINING_WL_ADDR, reg);
 788
 789                         /*
 790                          * Debug message - Finished Write leveling cs[cs] -
 791                          * each PUP Fail/Success
 792                          */
 793                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
 794                                         1);
 795                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
 796                                         (u32) res[cs], 3);
 797
 798                         /*
 799                          * Configure MR1 in cs[cs] - write leveling off (0),
 800                          * output buffer off (1)
 801                          */
 802                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 803                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 804                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 805                         /* No need to sort ODT since it is same CS */
 806                         /* 0x15D4 - DDR3 MR1 Register */
 807                         reg_write(REG_DDR3_MR1_ADDR, reg);
 808                         /* Issue MRS Command to current cs */
 809                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 810                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 811                         /*
 812                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 813                          * enable current cs
 814                          */
 815                         /* 0x1418 - SDRAM Operation Register */
 816                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 817
 818                         udelay(MRS_DELAY);
 819                 }
 820         }
 821
 822         /* Disable WL Mode */
 823         /* [2]=1 - TrnWLMode - Disable */
 824         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 825         reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 826         /* 0x15B8 - Training SW 2 Register */
 827         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 828
 829         /* Disable SW override - Must be in a different stage */
 830         /* [0]=0 - Enable SW override  */
 831         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 832         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 833         /* 0x15B8 - Training SW 2 Register */
 834         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 835
 836         /* Set Output buffer-on to all CS and correct ODT values */
 837         for (cs = 0; cs < MAX_CS; cs++) {
 838                 if (dram_info->cs_ena & (1 << cs)) {
 839                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 840                                 REG_DDR3_MR1_ODT_MASK;
 841                         reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
 842                         reg |= odt_static[dram_info->cs_ena][cs];
 843
 844                         /* 0x15D0 - DDR3 MR1 Register */
 845                         reg_write(REG_DDR3_MR1_ADDR, reg);
 846                         /* Issue MRS Command to current cs */
 847                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 848                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 849                         /*
 850                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 851                          * enable current cs
 852                          */
 853                         /* 0x1418 - SDRAM Operation Register */
 854                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 855
 856                         udelay(MRS_DELAY);
 857                 }
 858         }
 859
 860 #ifdef MV88F67XX
 861         /* Dynamic pad issue (BTS669) during WL */
 862         if (dpde_flag) {
 863                 reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 864                         (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 865                 reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 866         }
 867 #endif
 868         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
 869
 870         return MV_OK;
 871 }
 872
 873 #if !defined(MV88F672X)
 874 /*
 875  * Name:     ddr3_write_leveling_sw
 876  * Desc:     Execute Write leveling phase by SW
 877  * Args:     freq        - current sequence frequency
 878  *           dram_info   - main struct
 879  * Notes:
 880  * Returns:  MV_OK if success, MV_FAIL if fail.
 881  */
 882 int ddr3_write_leveling_sw_reg_dimm(u32 freq, int ratio_2to1,
 883                                     MV_DRAM_INFO *dram_info)
 884 {
 885         u32 reg, cs, cnt, pup;
 886         u32 res[MAX_CS];
 887         __maybe_unused int dpde_flag = 0;
 888
 889         /* Debug message - Start Write leveling procedure */
 890         DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 891
 892 #ifdef MV88F67XX
 893         /* Dynamic pad issue (BTS669) during WL */
 894         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 895         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 896                 dpde_flag = 1;
 897                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 898                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 899         }
 900 #endif
 901
 902         /* If target freq = 400 move clock start point */
 903         /* Write to control PUP to Control Deskew Regs */
 904         if (freq <= DDR_400) {
 905                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 906                         /* PUP_DELAY_MASK 0x1F */
 907                         /* reg = 0x0C10001F + (uj << 16); */
 908                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 909                                                 0x1F);
 910                 }
 911         }
 912
 913         /* Set Output buffer-off to all CS and correct ODT values */
 914         for (cs = 0; cs < MAX_CS; cs++) {
 915                 if (dram_info->cs_ena & (1 << cs)) {
 916                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 917                                 REG_DDR3_MR1_ODT_MASK;
 918                         reg |= odt_static[dram_info->cs_ena][cs];
 919                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 920
 921                         /* 0x15D0 - DDR3 MR0 Register */
 922                         reg_write(REG_DDR3_MR1_ADDR, reg);
 923                         /* Issue MRS Command to current cs */
 924                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 925                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 926                         /*
 927                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 928                          * enable current cs
 929                          */
 930                         /* 0x1418 - SDRAM Operation Register */
 931                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 932
 933                         udelay(MRS_DELAY);
 934                 }
 935         }
 936
 937         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 938
 939         /* Enable SW override */
 940         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 941                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 942         /* [0] = 1 - Enable SW override  */
 943         /* 0x15B8 - Training SW 2 Register */
 944         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 945         DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 946
 947         /* Enable PHY write leveling mode */
 948         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 949                 ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 950         /* [2] = 0 - TrnWLMode - Enable */
 951         /* 0x15B8 - Training SW 2 Register */
 952         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 953
 954         /* Loop for each cs */
 955         for (cs = 0; cs < MAX_CS; cs++) {
 956                 if (dram_info->cs_ena & (1 << cs)) {
 957                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 958                                         (u32) cs, 1);
 959
 960                         /* Refresh X9 current cs */
 961                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 962                         for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 963                                 reg =
 964                                     REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 965                                                                      (REG_SDRAM_OPERATION_CS_OFFS
 966                                                                       + cs));
 967                                 /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 968                                 reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 969
 970                                 do {
 971                                         reg =
 972                                             ((reg_read
 973                                               (REG_SDRAM_OPERATION_ADDR)) &
 974                                              REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 975                                 } while (reg);  /* Wait for '0' */
 976                         }
 977
 978                         /*
 979                          * Configure MR1 in Cs[CsNum] - write leveling on,
 980                          * output buffer on
 981                          */
 982                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 983                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 984                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 985                         /* Set ODT Values */
 986                         reg &= REG_DDR3_MR1_ODT_MASK;
 987                         reg |= odt_static[dram_info->cs_ena][cs];
 988                         /* Enable WL MODE */
 989                         reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 990                         /*
 991                          * [7]=1, [12]=0 - Output Buffer and write leveling
 992                          * enabled
 993                          */
 994                         /* 0x15D4 - DDR3 MR1 Register */
 995                         reg_write(REG_DDR3_MR1_ADDR, reg);
 996                         /* Issue MRS Command to current cs */
 997                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 998                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 999                         /*
1000                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1001                          * enable current cs
1002                          */
1003                         /* 0x1418 - SDRAM Operation Register */
1004                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1005
1006                         udelay(MRS_DELAY);
1007
1008                         /* Write leveling  cs[cs] */
1009                         if (MV_OK !=
1010                             ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
1011                                                           (u32 *)(res + cs),
1012                                                           dram_info)) {
1013                                 DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
1014                                                 (u32) cs, 1);
1015                                 return MV_FAIL;
1016                         }
1017
1018                         /* Set TrnWLDeUpd - After each CS is done */
1019                         reg = reg_read(REG_TRAINING_WL_ADDR) |
1020                                 (1 << REG_TRAINING_WL_CS_DONE_OFFS);
1021                         /* 0x16AC - Training Write leveling register */
1022                         reg_write(REG_TRAINING_WL_ADDR, reg);
1023
1024                         /*
1025                          * Debug message - Finished Write leveling cs[cs] -
1026                          * each PUP Fail/Success
1027                          */
1028                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
1029                                         1);
1030                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
1031                                         (u32) res[cs], 3);
1032
1033                         /* Configure MR1 in cs[cs] - write leveling off (0), output buffer off (1) */
1034                         reg = reg_read(REG_DDR3_MR1_ADDR) &
1035                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
1036                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
1037                         /* No need to sort ODT since it is same CS */
1038                         /* 0x15D4 - DDR3 MR1 Register */
1039                         reg_write(REG_DDR3_MR1_ADDR, reg);
1040                         /* Issue MRS Command to current cs */
1041                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
1042                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1043                         /*
1044                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1045                          * enable current cs
1046                          */
1047                         /* 0x1418 - SDRAM Operation Register */
1048                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1049
1050                         udelay(MRS_DELAY);
1051                 }
1052         }
1053
1054         /* Disable WL Mode */
1055         /* [2]=1 - TrnWLMode - Disable */
1056         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1057         reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
1058         /* 0x15B8 - Training SW 2 Register */
1059         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1060
1061         /* Disable SW override - Must be in a different stage */
1062         /* [0]=0 - Enable SW override  */
1063         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1064         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
1065         /* 0x15B8 - Training SW 2 Register */
1066         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1067
1068         /* Set Output buffer-on to all CS and correct ODT values */
1069         for (cs = 0; cs < MAX_CS; cs++) {
1070                 if (dram_info->cs_ena & (1 << cs)) {
1071                         reg = reg_read(REG_DDR3_MR1_ADDR) &
1072                                 REG_DDR3_MR1_ODT_MASK;
1073                         reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
1074                         reg |= odt_static[dram_info->cs_ena][cs];
1075
1076                         /* 0x15D0 - DDR3 MR1 Register */
1077                         reg_write(REG_DDR3_MR1_ADDR, reg);
1078                         /* Issue MRS Command to current cs */
1079                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
1080                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1081                         /*
1082                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1083                          * enable current cs
1084                          */
1085                         /* 0x1418 - SDRAM Operation Register */
1086                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1087
1088                         udelay(MRS_DELAY);
1089                 }
1090         }
1091
1092 #ifdef MV88F67XX
1093         /* Dynamic pad issue (BTS669) during WL */
1094         if (dpde_flag) {
1095                 reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
1096                         (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
1097                 reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
1098         }
1099 #endif
1100
1101         /* If target freq = 400 move clock back */
1102         /* Write to control PUP to Control Deskew Regs */
1103         if (freq <= DDR_400) {
1104                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
1105                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
1106                                                 0);
1107                 }
1108         }
1109
1110         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
1111         return MV_OK;
1112 }
1113 #endif
1114
1115 /*
1116  * Name:     ddr3_write_leveling_single_cs
1117  * Desc:     Execute Write leveling for single Chip select
1118  * Args:     cs          - current chip select
1119  *           freq        - current sequence frequency
1120  *           result      - res array
1121  *           dram_info   - main struct
1122  * Notes:
1123  * Returns:  MV_OK if success, MV_FAIL if fail.
1124  */
1125 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
1126                                          u32 *result, MV_DRAM_INFO *dram_info)
1127 {
1128         u32 reg, pup_num, delay, phase, phaseMax, max_pup_num, pup,
1129                 max_pup_mask;
1130
1131         max_pup_num = dram_info->num_of_total_pups;
1132         *result = 0;
1133         u32 flag[MAX_PUP_NUM] = { 0 };
1134
1135         DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - WL for Cs - ",
1136                         (u32) cs, 1);
1137
1138         switch (max_pup_num) {
1139         case 2:
1140                 max_pup_mask = 0x3;
1141                 break;
1142         case 4:
1143                 max_pup_mask = 0xf;
1144                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1145                 break;
1146         case 5:
1147                 max_pup_mask = 0x1f;
1148                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1149                 break;
1150         case 8:
1151                 max_pup_mask = 0xff;
1152                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1153                 break;
1154         case 9:
1155                 max_pup_mask = 0x1ff;
1156                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1157                 break;
1158         default:
1159                 DEBUG_WL_C("ddr3_write_leveling_single_cs wrong max_pup_num =  ",
1160                            max_pup_num, 3);
1161                 return MV_FAIL;
1162         }
1163
1164         /* CS ODT Override */
1165         reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1166                 REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1167         reg |= (REG_SDRAM_ODT_CTRL_HIGH_OVRD_ENA << (2 * cs));
1168         /* Set 0x3 - Enable ODT on the curent cs and disable on other cs */
1169         /* 0x1498 - SDRAM ODT Control high */
1170         reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1171
1172         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - ODT Asserted for current Cs\n");
1173
1174         /* tWLMRD Delay */
1175         /* Delay of minimum 40 Dram clock cycles - 20 Tclk cycles */
1176         udelay(1);
1177
1178         /* [1:0] - current cs number */
1179         reg = (reg_read(REG_TRAINING_WL_ADDR) & REG_TRAINING_WL_CS_MASK) | cs;
1180         reg |= (1 << REG_TRAINING_WL_UPD_OFFS); /* [2] - trnWLCsUpd */
1181         /* 0x16AC - Training Write leveling register */
1182         reg_write(REG_TRAINING_WL_ADDR, reg);
1183
1184         /* Broadcast to all PUPs: Reset DQS phase, reset leveling delay */
1185         ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, 0, 0);
1186
1187         /* Seek Edge */
1188         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Current Cs\n");
1189
1190         /* Drive DQS high for one cycle - All data PUPs */
1191         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Driving DQS high for one cycle\n");
1192         if (!ratio_2to1) {
1193                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1194                        REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_1TO1;
1195         } else {
1196                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1197                        REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_2TO1;
1198         }
1199         /* 0x16AC - Training Write leveling register */
1200         reg_write(REG_TRAINING_WL_ADDR, reg);
1201
1202         /* Wait tWLdelay */
1203         do {
1204                 /* [29] - trnWLDelayExp */
1205                 reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1206                         REG_TRAINING_WL_DELAYEXP_MASK;
1207         } while (reg == 0x0);   /* Wait for '1' */
1208
1209         /* Read WL res */
1210         reg = (reg_read(REG_TRAINING_WL_ADDR) >> REG_TRAINING_WL_RESULTS_OFFS) &
1211                 REG_TRAINING_WL_RESULTS_MASK;
1212         /* [28:20] - TrnWLResult */
1213
1214         if (!ratio_2to1) /* Different phase options for 2:1 or 1:1 modes */
1215                 phaseMax = MAX_PHASE_1TO1;
1216         else
1217                 phaseMax = MAX_PHASE_2TO1;
1218
1219         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Shift DQS + Octet Leveling\n");
1220
1221         /* Shift DQS + Octet leveling */
1222         for (phase = 0; phase < phaseMax; phase++) {
1223                 for (delay = 0; delay < MAX_DELAY; delay++) {
1224                         /*  Broadcast to all PUPs: DQS phase,leveling delay */
1225                         ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, phase,
1226                                            delay);
1227
1228                         udelay(1);      /* Delay of  3 Tclk cycles */
1229
1230                         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge: Phase = ");
1231                         DEBUG_WL_FULL_D((u32) phase, 1);
1232                         DEBUG_WL_FULL_S(", Delay = ");
1233                         DEBUG_WL_FULL_D((u32) delay, 1);
1234                         DEBUG_WL_FULL_S("\n");
1235
1236                         /* Drive DQS high for one cycle - All data PUPs */
1237                         if (!ratio_2to1) {
1238                                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1239                                        REG_TRAINING_WL_RATIO_MASK) |
1240                                         REG_TRAINING_WL_1TO1;
1241                         } else {
1242                                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1243                                        REG_TRAINING_WL_RATIO_MASK) |
1244                                         REG_TRAINING_WL_2TO1;
1245                         }
1246                         reg_write(REG_TRAINING_WL_ADDR, reg);   /* 0x16AC  */
1247
1248                         /* Wait tWLdelay */
1249                         do {
1250                                 reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1251                                         REG_TRAINING_WL_DELAYEXP_MASK;
1252                         } while (reg == 0x0);   /* [29] Wait for '1' */
1253
1254                         /* Read WL res */
1255                         reg = reg_read(REG_TRAINING_WL_ADDR);
1256                         reg = (reg >> REG_TRAINING_WL_RESULTS_OFFS) &
1257                                 REG_TRAINING_WL_RESULTS_MASK;   /* [28:20] */
1258
1259                         DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - Seek Edge: Results =  ",
1260                                         (u32) reg, 3);
1261
1262                         /* Update State machine */
1263                         for (pup = 0; pup < (max_pup_num); pup++) {
1264                                 /* ECC support - bit 8 */
1265                                 pup_num = (pup == dram_info->num_of_std_pups) ?
1266                                         ECC_BIT : pup;
1267                                 if (dram_info->wl_val[cs][pup][S] == 0) {
1268                                         /* Update phase to PUP */
1269                                         dram_info->wl_val[cs][pup][P] = phase;
1270                                         /* Update delay to PUP */
1271                                         dram_info->wl_val[cs][pup][D] = delay;
1272                                 }
1273
1274                                 if (((reg >> pup_num) & 0x1) == 0)
1275                                         flag[pup_num] = 1;
1276
1277                                 if (((reg >> pup_num) & 0x1)
1278                                     && (flag[pup_num] == 1)
1279                                     && (dram_info->wl_val[cs][pup][S] == 0)) {
1280                                         /*
1281                                          * If the PUP is locked now and in last
1282                                          * counter states
1283                                          */
1284                                         /* Go to next state */
1285                                         dram_info->wl_val[cs][pup][S] = 1;
1286                                         /* Set res */
1287                                         *result = *result | (1 << pup_num);
1288                                 }
1289                         }
1290
1291                         /* If all locked - Break the loops - Finished */
1292                         if (*result == max_pup_mask) {
1293                                 phase = phaseMax;
1294                                 delay = MAX_DELAY;
1295                                 DEBUG_WL_S("DDR3 - Write Leveling Single Cs - Seek Edge: All Locked\n");
1296                         }
1297                 }
1298         }
1299
1300         /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
1301         DEBUG_WL_C("DDR3 - Write Leveling - Results for CS - ", (u32) cs, 1);
1302         for (pup = 0; pup < (max_pup_num); pup++) {
1303                 DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
1304                 DEBUG_WL_D((u32) pup, 1);
1305                 DEBUG_WL_S(", Phase: ");
1306                 DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][P], 1);
1307                 DEBUG_WL_S(", Delay: ");
1308                 DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][D], 2);
1309                 DEBUG_WL_S("\n");
1310         }
1311
1312         /* Check if some not locked and return error */
1313         if (*result != max_pup_mask) {
1314                 DEBUG_WL_S("DDR3 - Write Leveling - ERROR - not all PUPS were locked\n");
1315                 return MV_FAIL;
1316         }
1317
1318         /* Configure Each PUP with locked leveling settings */
1319         for (pup = 0; pup < (max_pup_num); pup++) {
1320                 /* ECC support - bit 8 */
1321                 pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
1322                 phase = dram_info->wl_val[cs][pup][P];
1323                 delay = dram_info->wl_val[cs][pup][D];
1324                 ddr3_write_pup_reg(PUP_WL_MODE, cs, pup_num, phase, delay);
1325         }
1326
1327         /* CS ODT Override */
1328         reg =  reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1329                 REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1330         /* 0x1498 - SDRAM ODT Control high */
1331         reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1332
1333         return MV_OK;
1334 }
1335
1336 /*
1337  * Perform DDR3 Control PUP Indirect Write
1338  */
1339 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr, u32 data)
1340 {
1341         u32 reg = 0;
1342
1343         /* Store value for write */
1344         reg = (data & 0xFFFF);
1345
1346         /* Set bit 26 for control PHY access */
1347         reg |= (1 << REG_PHY_CNTRL_OFFS);
1348
1349         /* Configure BC or UC access to PHYs */
1350         if (bc_acc == 1)
1351                 reg |= (1 << REG_PHY_BC_OFFS);
1352         else
1353                 reg |= (pup << REG_PHY_PUP_OFFS);
1354
1355         /* Set PHY register address to write to */
1356         reg |= (reg_addr << REG_PHY_CS_OFFS);
1357
1358         reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1359         reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1360         reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1361
1362         do {
1363                 reg = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &
1364                         REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1365         } while (reg);          /* Wait for '0' to mark the end of the transaction */
1366 }