drivers/ddr/marvell/axp/ddr3_read_leveling.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Marvell International Ltd. and its affiliates
   4  */
   5
   6 #include <common.h>
   7 #include <i2c.h>
   8 #include <spl.h>
   9 #include <asm/io.h>
  10 #include <asm/arch/cpu.h>
  11 #include <asm/arch/soc.h>
  12
  13 #include "ddr3_hw_training.h"
  14
  15 /*
  16  * Debug
  17  */
  18 #define DEBUG_RL_C(s, d, l) \
  19         DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
  20 #define DEBUG_RL_FULL_C(s, d, l) \
  21         DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")
  22
  23 #ifdef MV_DEBUG_RL
  24 #define DEBUG_RL_S(s) \
  25         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  26 #define DEBUG_RL_D(d, l) \
  27         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  28 #else
  29 #define DEBUG_RL_S(s)
  30 #define DEBUG_RL_D(d, l)
  31 #endif
  32
  33 #ifdef MV_DEBUG_RL_FULL
  34 #define DEBUG_RL_FULL_S(s)              puts(s)
  35 #define DEBUG_RL_FULL_D(d, l)           printf("%x", d)
  36 #else
  37 #define DEBUG_RL_FULL_S(s)
  38 #define DEBUG_RL_FULL_D(d, l)
  39 #endif
  40
  41 extern u32 rl_pattern[LEN_STD_PATTERN];
  42
  43 #ifdef RL_MODE
  44 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
  45                                                 int ratio_2to1, u32 ecc,
  46                                                 MV_DRAM_INFO *dram_info);
  47 #else
  48 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
  49                                                     int ratio_2to1, u32 ecc,
  50                                                     MV_DRAM_INFO *dram_info);
  51 #endif
  52
  53 /*
  54  * Name:     ddr3_read_leveling_hw
  55  * Desc:     Execute the Read leveling phase by HW
  56  * Args:     dram_info - main struct
  57  *           freq      - current sequence frequency
  58  * Notes:
  59  * Returns:  MV_OK if success, MV_FAIL if fail.
  60  */
  61 int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  62 {
  63         u32 reg;
  64
  65         /* Debug message - Start Read leveling procedure */
  66         DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");
  67
  68         /* Start Auto Read Leveling procedure */
  69         reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
  70         /* Config the retest number */
  71         reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);
  72
  73         /* Enable CS in the automatic process */
  74         reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);
  75
  76         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  77
  78         reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  79                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  80         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  81
  82         /* Wait */
  83         do {
  84                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  85                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  86         } while (reg);          /* Wait for '0' */
  87
  88         /* Check if Successful */
  89         if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  90             (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
  91                 u32 delay, phase, pup, cs;
  92
  93                 dram_info->rl_max_phase = 0;
  94                 dram_info->rl_min_phase = 10;
  95
  96                 /* Read results to arrays */
  97                 for (cs = 0; cs < MAX_CS; cs++) {
  98                         if (dram_info->cs_ena & (1 << cs)) {
  99                                 for (pup = 0;
 100                                      pup < dram_info->num_of_total_pups;
 101                                      pup++) {
 102                                         if (pup == dram_info->num_of_std_pups
 103                                             && dram_info->ecc_ena)
 104                                                 pup = ECC_PUP;
 105                                         reg =
 106                                             ddr3_read_pup_reg(PUP_RL_MODE, cs,
 107                                                               pup);
 108                                         phase = (reg >> REG_PHY_PHASE_OFFS) &
 109                                                 PUP_PHASE_MASK;
 110                                         delay = reg & PUP_DELAY_MASK;
 111                                         dram_info->rl_val[cs][pup][P] = phase;
 112                                         if (phase > dram_info->rl_max_phase)
 113                                                 dram_info->rl_max_phase = phase;
 114                                         if (phase < dram_info->rl_min_phase)
 115                                                 dram_info->rl_min_phase = phase;
 116                                         dram_info->rl_val[cs][pup][D] = delay;
 117                                         dram_info->rl_val[cs][pup][S] =
 118                                             RL_FINAL_STATE;
 119                                         reg =
 120                                             ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
 121                                                               cs, pup);
 122                                         dram_info->rl_val[cs][pup][DQS] =
 123                                             (reg & 0x3F);
 124                                 }
 125 #ifdef MV_DEBUG_RL
 126                                 /* Print results */
 127                                 DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
 128                                            (u32) cs, 1);
 129
 130                                 for (pup = 0;
 131                                      pup < (dram_info->num_of_total_pups);
 132                                      pup++) {
 133                                         if (pup == dram_info->num_of_std_pups
 134                                             && dram_info->ecc_ena)
 135                                                 pup = ECC_PUP;
 136                                         DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 137                                         DEBUG_RL_D((u32) pup, 1);
 138                                         DEBUG_RL_S(", Phase: ");
 139                                         DEBUG_RL_D((u32) dram_info->
 140                                                    rl_val[cs][pup][P], 1);
 141                                         DEBUG_RL_S(", Delay: ");
 142                                         DEBUG_RL_D((u32) dram_info->
 143                                                    rl_val[cs][pup][D], 2);
 144                                         DEBUG_RL_S("\n");
 145                                 }
 146 #endif
 147                         }
 148                 }
 149
 150                 dram_info->rd_rdy_dly =
 151                         reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
 152                         REG_READ_DATA_SAMPLE_DELAYS_MASK;
 153                 dram_info->rd_smpl_dly =
 154                         reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
 155                         REG_READ_DATA_READY_DELAYS_MASK;
 156 #ifdef MV_DEBUG_RL
 157                 DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 158                            dram_info->rd_smpl_dly, 2);
 159                 DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 160                            dram_info->rd_rdy_dly, 2);
 161                 DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
 162 #endif
 163                 return MV_OK;
 164
 165         } else {
 166                 DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
 167                 return MV_FAIL;
 168         }
 169 }
 170
 171 /*
 172  * Name:     ddr3_read_leveling_sw
 173  * Desc:     Execute the Read leveling phase by SW
 174  * Args:     dram_info - main struct
 175  *           freq      - current sequence frequency
 176  * Notes:
 177  * Returns:  MV_OK if success, MV_FAIL if fail.
 178  */
 179 int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 180 {
 181         u32 reg, cs, ecc, pup_num, phase, delay, pup;
 182         int status;
 183
 184         /* Debug message - Start Read leveling procedure */
 185         DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");
 186
 187         /* Enable SW Read Leveling */
 188         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 189                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 190         reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
 191         /* [0]=1 - Enable SW override  */
 192         /* 0x15B8 - Training SW 2 Register */
 193         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 194
 195 #ifdef RL_MODE
 196         reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
 197                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 198         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 199 #endif
 200
 201         /* Loop for each CS */
 202         for (cs = 0; cs < dram_info->num_cs; cs++) {
 203                 DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);
 204
 205                 for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
 206                         /* ECC Support - Switch ECC Mux on ecc=1 */
 207                         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 208                                 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 209                         reg |= (dram_info->ecc_ena *
 210                                 ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 211                         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 212
 213                         if (ecc)
 214                                 DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
 215                         else
 216                                 DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");
 217
 218                         /* Set current sample delays */
 219                         reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 220                         reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
 221                                  (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 222                         reg |= (dram_info->cl <<
 223                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 224                         reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);
 225
 226                         /* Set current Ready delay */
 227                         reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 228                         reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 229                                  (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 230                         if (!ratio_2to1) {
 231                                 /* 1:1 mode */
 232                                 reg |= ((dram_info->cl + 1) <<
 233                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 234                         } else {
 235                                 /* 2:1 mode */
 236                                 reg |= ((dram_info->cl + 2) <<
 237                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 238                         }
 239                         reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 240
 241                         /* Read leveling Single CS[cs] */
 242 #ifdef RL_MODE
 243                         status =
 244                             ddr3_read_leveling_single_cs_rl_mode(cs, freq,
 245                                                                  ratio_2to1,
 246                                                                  ecc,
 247                                                                  dram_info);
 248                         if (MV_OK != status)
 249                                 return status;
 250 #else
 251                         status =
 252                             ddr3_read_leveling_single_cs_window_mode(cs, freq,
 253                                                                      ratio_2to1,
 254                                                                      ecc,
 255                                                                      dram_info)
 256                             if (MV_OK != status)
 257                                 return status;
 258 #endif
 259                 }
 260
 261                 /* Print results */
 262                 DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
 263                            1);
 264
 265                 for (pup = 0;
 266                      pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 267                      pup++) {
 268                         DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 269                         DEBUG_RL_D((u32) pup, 1);
 270                         DEBUG_RL_S(", Phase: ");
 271                         DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
 272                         DEBUG_RL_S(", Delay: ");
 273                         DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
 274                         DEBUG_RL_S("\n");
 275                 }
 276
 277                 DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 278                            dram_info->rd_smpl_dly, 2);
 279                 DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 280                            dram_info->rd_rdy_dly, 2);
 281
 282                 /* Configure PHY with average of 3 locked leveling settings */
 283                 for (pup = 0;
 284                      pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 285                      pup++) {
 286                         /* ECC support - bit 8 */
 287                         pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
 288
 289                         /* For now, set last cnt result */
 290                         phase = dram_info->rl_val[cs][pup][P];
 291                         delay = dram_info->rl_val[cs][pup][D];
 292                         ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
 293                                            delay);
 294                 }
 295         }
 296
 297         /* Reset PHY read FIFO */
 298         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 299                 (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 300         /* 0x15B8 - Training SW 2 Register */
 301         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 302
 303         do {
 304                 reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 305                         (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 306         } while (reg);          /* Wait for '0' */
 307
 308         /* ECC Support - Switch ECC Mux off ecc=0 */
 309         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 310                 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 311         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 312
 313 #ifdef RL_MODE
 314         reg_write(REG_DRAM_TRAINING_ADDR, 0);   /* 0x15B0 - Training Register */
 315 #endif
 316
 317         /* Disable SW Read Leveling */
 318         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 319                 ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 320         /* [0] = 0 - Disable SW override  */
 321         reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
 322         /* [3] = 1 - Disable RL MODE */
 323         /* 0x15B8 - Training SW 2 Register */
 324         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 325
 326         DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
 327         return MV_OK;
 328 }
 329
 330 #ifdef RL_MODE
 331 /*
 332  * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
 333  * This just got too much indented making it hard to read / edit.
 334  */
 335 static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
 336                     u32 *locked_sum, u32 ecc, int *first_octet_locked,
 337                     int *counter_in_progress, int final_delay, u32 delay,
 338                     u32 phase)
 339 {
 340         /* If no OverRun */
 341         if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
 342                 int idx;
 343
 344                 idx = pup + ecc * ECC_BIT;
 345
 346                 /* PUP passed, start examining */
 347                 if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
 348                         /* Must be RL_UNLOCK_STATE */
 349                         /* Match expected value ? - Update State Machine */
 350                         if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
 351                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 352                                                 (u32)pup, 1);
 353                                 info->rl_val[cs][idx][C]++;
 354
 355                                 /* If pup got to last state - lock the delays */
 356                                 if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
 357                                         info->rl_val[cs][idx][C] = 0;
 358                                         info->rl_val[cs][idx][DS] = delay;
 359                                         info->rl_val[cs][idx][PS] = phase;
 360
 361                                         /* Go to Final State */
 362                                         info->rl_val[cs][idx][S] = RL_FINAL_STATE;
 363                                         *locked_sum = *locked_sum + 1;
 364                                         DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
 365                                                         (u32)pup, 1);
 366
 367                                         /*
 368                                          * If first lock - need to lock delays
 369                                          */
 370                                         if (*first_octet_locked == 0) {
 371                                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 372                                                                 (u32)pup, 1);
 373                                                 *first_octet_locked = 1;
 374                                         }
 375
 376                                         /*
 377                                          * If pup is in not in final state but
 378                                          * there was match - dont increment
 379                                          * counter
 380                                          */
 381                                 } else {
 382                                         *counter_in_progress = 1;
 383                                 }
 384                         }
 385                 }
 386         }
 387 }
 388
 389 /*
 390  * Name:     ddr3_read_leveling_single_cs_rl_mode
 391  * Desc:     Execute Read leveling for single Chip select
 392  * Args:     cs        - current chip select
 393  *           freq      - current sequence frequency
 394  *           ecc       - ecc iteration indication
 395  *           dram_info - main struct
 396  * Notes:
 397  * Returns:  MV_OK if success, MV_FAIL if fail.
 398  */
 399 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
 400                                                 int ratio_2to1, u32 ecc,
 401                                                 MV_DRAM_INFO *dram_info)
 402 {
 403         u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
 404                 repeat_max_cnt, sdram_offset, locked_sum;
 405         u32 phase_min, ui_max_delay;
 406         int all_locked, first_octet_locked, counter_in_progress;
 407         int final_delay = 0;
 408
 409         DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 410
 411         /* Init values */
 412         phase = 0;
 413         delay = 0;
 414         rd_sample_delay = dram_info->cl;
 415         all_locked = 0;
 416         first_octet_locked = 0;
 417         repeat_max_cnt = 0;
 418         locked_sum = 0;
 419
 420         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 421              pup++)
 422                 dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 423
 424         /* Main loop */
 425         while (!all_locked) {
 426                 counter_in_progress = 0;
 427
 428                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 429                 DEBUG_RL_FULL_D(rd_sample_delay, 2);
 430                 DEBUG_RL_FULL_S(", RdRdyDly = ");
 431                 DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 432                 DEBUG_RL_FULL_S(", Phase = ");
 433                 DEBUG_RL_FULL_D(phase, 1);
 434                 DEBUG_RL_FULL_S(", Delay = ");
 435                 DEBUG_RL_FULL_D(delay, 2);
 436                 DEBUG_RL_FULL_S("\n");
 437
 438                 /*
 439                  * Broadcast to all PUPs current RL delays: DQS phase,
 440                  * leveling delay
 441                  */
 442                 ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 443
 444                 /* Reset PHY read FIFO */
 445                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 446                         (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 447                 /* 0x15B8 - Training SW 2 Register */
 448                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 449
 450                 do {
 451                         reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 452                                 (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 453                 } while (reg);  /* Wait for '0' */
 454
 455                 /* Read pattern from SDRAM */
 456                 sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 457                 locked_pups = 0;
 458                 if (MV_OK !=
 459                     ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 460                                        rl_pattern, LEN_STD_PATTERN,
 461                                        sdram_offset, 0, 0, NULL, 0))
 462                         return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;
 463
 464                 /* Octet evaluation */
 465                 /* pup_num = Q or 1 for ECC */
 466                 for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 467                         /* Check Overrun */
 468                         if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 469                                (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
 470                                 overrun(cs, dram_info, pup, locked_pups,
 471                                         &locked_sum, ecc, &first_octet_locked,
 472                                         &counter_in_progress, final_delay,
 473                                         delay, phase);
 474                         } else {
 475                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 476                                                 (u32)pup, 1);
 477                         }
 478                 }
 479
 480                 if (locked_sum == (dram_info->num_of_std_pups *
 481                                    (1 - ecc) + ecc)) {
 482                         all_locked = 1;
 483                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 484                 }
 485
 486                 /*
 487                  * This is a fix for unstable condition where pups are
 488                  * toggling between match and no match
 489                  */
 490                 /*
 491                  * If some of the pups is >1 <3, check if we did it too
 492                  * many times
 493                  */
 494                 if (counter_in_progress == 1) {
 495                         /* Notify at least one Counter is >=1 and < 3 */
 496                         if (repeat_max_cnt < RL_RETRY_COUNT) {
 497                                 repeat_max_cnt++;
 498                                 counter_in_progress = 1;
 499                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 500                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 501                         } else {
 502                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 503                                 counter_in_progress = 0;
 504                         }
 505                 }
 506
 507                 /*
 508                  * Check some of the pups are in the middle of state machine
 509                  * and don't increment the delays
 510                  */
 511                 if (!counter_in_progress && !all_locked) {
 512                         int idx;
 513
 514                         idx = pup + ecc * ECC_BIT;
 515
 516                         repeat_max_cnt = 0;
 517                         /* if 1:1 mode */
 518                         if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
 519                                 ui_max_delay = MAX_DELAY_INV;
 520                         else
 521                                 ui_max_delay = MAX_DELAY;
 522
 523                         /* Increment Delay */
 524                         if (delay < ui_max_delay) {
 525                                 delay++;
 526                                 /*
 527                                  * Mark the last delay/pahse place for
 528                                  * window final place
 529                                  */
 530                                 if (delay == ui_max_delay) {
 531                                         if ((!ratio_2to1 && phase ==
 532                                              MAX_PHASE_RL_L_1TO1)
 533                                             || (ratio_2to1 && phase ==
 534                                                 MAX_PHASE_RL_L_2TO1))
 535                                                 final_delay = 1;
 536                                 }
 537                         } else {
 538                                 /* Phase+CL Incrementation */
 539                                 delay = 0;
 540
 541                                 if (!ratio_2to1) {
 542                                         /* 1:1 mode */
 543                                         if (first_octet_locked) {
 544                                                 /* some Pup was Locked */
 545                                                 if (phase < MAX_PHASE_RL_L_1TO1) {
 546                                                         if (phase == 1) {
 547                                                                 phase = 4;
 548                                                         } else {
 549                                                                 phase++;
 550                                                                 delay = MIN_DELAY_PHASE_1_LIMIT;
 551                                                         }
 552                                                 } else {
 553                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 554                                                         DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
 555                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 556                                                 }
 557                                         } else {
 558                                                 /* NO Pup was Locked */
 559                                                 if (phase < MAX_PHASE_RL_UL_1TO1) {
 560                                                         phase++;
 561                                                         delay =
 562                                                             MIN_DELAY_PHASE_1_LIMIT;
 563                                                 } else {
 564                                                         phase = 0;
 565                                                 }
 566                                         }
 567                                 } else {
 568                                         /* 2:1 mode */
 569                                         if (first_octet_locked) {
 570                                                 /* some Pup was Locked */
 571                                                 if (phase < MAX_PHASE_RL_L_2TO1) {
 572                                                         phase++;
 573                                                 } else {
 574                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 575                                                         DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 576                                                         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 577                                                                 /* pup_num = Q or 1 for ECC */
 578                                                                 if (dram_info->rl_val[cs][idx][S]
 579                                                                     == 0) {
 580                                                                         DEBUG_RL_C("Failed byte is = ",
 581                                                                                    pup, 1);
 582                                                                 }
 583                                                         }
 584                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 585                                                 }
 586                                         } else {
 587                                                 /* No Pup was Locked */
 588                                                 if (phase < MAX_PHASE_RL_UL_2TO1)
 589                                                         phase++;
 590                                                 else
 591                                                         phase = 0;
 592                                         }
 593                                 }
 594
 595                                 /*
 596                                  * If we finished a full Phases cycle (so now
 597                                  * phase = 0, need to increment rd_sample_dly
 598                                  */
 599                                 if (phase == 0 && first_octet_locked == 0) {
 600                                         rd_sample_delay++;
 601                                         if (rd_sample_delay == 0x10) {
 602                                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 603                                                 DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 604                                                 for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 605                                                         /* pup_num = Q or 1 for ECC */
 606                                                         if (dram_info->
 607                                                             rl_val[cs][idx][S] == 0) {
 608                                                                 DEBUG_RL_C("Failed byte is = ",
 609                                                                            pup, 1);
 610                                                         }
 611                                                 }
 612                                                 return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
 613                                         }
 614
 615                                         /* Set current rd_sample_delay  */
 616                                         reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 617                                         reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
 618                                               << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
 619                                                   * cs));
 620                                         reg |= (rd_sample_delay <<
 621                                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
 622                                                  cs));
 623                                         reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
 624                                                   reg);
 625                                 }
 626
 627                                 /*
 628                                  * Set current rdReadyDelay according to the
 629                                  * hash table (Need to do this in every phase
 630                                  * change)
 631                                  */
 632                                 if (!ratio_2to1) {
 633                                         /* 1:1 mode */
 634                                         add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 635                                         switch (phase) {
 636                                         case 0:
 637                                                 add = (add >>
 638                                                        REG_TRAINING_DEBUG_2_OFFS);
 639                                                 break;
 640                                         case 1:
 641                                                 add = (add >>
 642                                                        (REG_TRAINING_DEBUG_2_OFFS
 643                                                         + 3));
 644                                                 break;
 645                                         case 4:
 646                                                 add = (add >>
 647                                                        (REG_TRAINING_DEBUG_2_OFFS
 648                                                         + 6));
 649                                                 break;
 650                                         case 5:
 651                                                 add = (add >>
 652                                                        (REG_TRAINING_DEBUG_2_OFFS
 653                                                         + 9));
 654                                                 break;
 655                                         }
 656                                         add &= REG_TRAINING_DEBUG_2_MASK;
 657                                 } else {
 658                                         /* 2:1 mode */
 659                                         add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 660                                         add = (add >>
 661                                                (phase *
 662                                                 REG_TRAINING_DEBUG_3_OFFS));
 663                                         add &= REG_TRAINING_DEBUG_3_MASK;
 664                                 }
 665
 666                                 reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 667                                 reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 668                                          (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 669                                 reg |= ((rd_sample_delay + add) <<
 670                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 671                                 reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 672                                 dram_info->rd_smpl_dly = rd_sample_delay;
 673                                 dram_info->rd_rdy_dly = rd_sample_delay + add;
 674                         }
 675
 676                         /* Reset counters for pups with states<RD_STATE_COUNT */
 677                         for (pup = 0; pup <
 678                                      (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 679                              pup++) {
 680                                 if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
 681                                         dram_info->rl_val[cs][idx][C] = 0;
 682                         }
 683                 }
 684         }
 685
 686         phase_min = 10;
 687
 688         for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
 689                 if (dram_info->rl_val[cs][pup][PS] < phase_min)
 690                         phase_min = dram_info->rl_val[cs][pup][PS];
 691         }
 692
 693         /*
 694          * Set current rdReadyDelay according to the hash table (Need to
 695          * do this in every phase change)
 696          */
 697         if (!ratio_2to1) {
 698                 /* 1:1 mode */
 699                 add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 700                 switch (phase_min) {
 701                 case 0:
 702                         add = (add >> REG_TRAINING_DEBUG_2_OFFS);
 703                         break;
 704                 case 1:
 705                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
 706                         break;
 707                 case 4:
 708                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
 709                         break;
 710                 case 5:
 711                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
 712                         break;
 713                 }
 714                 add &= REG_TRAINING_DEBUG_2_MASK;
 715         } else {
 716                 /* 2:1 mode */
 717                 add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 718                 add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
 719                 add &= REG_TRAINING_DEBUG_3_MASK;
 720         }
 721
 722         reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 723         reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 724                  (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 725         reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 726         reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 727         dram_info->rd_rdy_dly = rd_sample_delay + add;
 728
 729         for (cs = 0; cs < dram_info->num_cs; cs++) {
 730                 for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 731                         reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
 732                         dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
 733                 }
 734         }
 735
 736         return MV_OK;
 737 }
 738
 739 #else
 740
 741 /*
 742  * Name:     ddr3_read_leveling_single_cs_window_mode
 743  * Desc:     Execute Read leveling for single Chip select
 744  * Args:     cs        - current chip select
 745  *           freq      - current sequence frequency
 746  *           ecc       - ecc iteration indication
 747  *           dram_info - main struct
 748  * Notes:
 749  * Returns:  MV_OK if success, MV_FAIL if fail.
 750  */
 751 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
 752                                                     int ratio_2to1, u32 ecc,
 753                                                     MV_DRAM_INFO *dram_info)
 754 {
 755         u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
 756             repeat_max_cnt, sdram_offset, final_sum, locked_sum;
 757         u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
 758         int all_locked, first_octet_locked, counter_in_progress;
 759         int final_delay = 0;
 760
 761         DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 762
 763         /* Init values */
 764         phase = 0;
 765         delay = 0;
 766         rd_sample_delay = dram_info->cl;
 767         all_locked = 0;
 768         first_octet_locked = 0;
 769         repeat_max_cnt = 0;
 770         sum = 0;
 771         final_sum = 0;
 772         locked_sum = 0;
 773
 774         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 775              pup++)
 776                 dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 777
 778         /* Main loop */
 779         while (!all_locked) {
 780                 counter_in_progress = 0;
 781
 782                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 783                 DEBUG_RL_FULL_D(rd_sample_delay, 2);
 784                 DEBUG_RL_FULL_S(", RdRdyDly = ");
 785                 DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 786                 DEBUG_RL_FULL_S(", Phase = ");
 787                 DEBUG_RL_FULL_D(phase, 1);
 788                 DEBUG_RL_FULL_S(", Delay = ");
 789                 DEBUG_RL_FULL_D(delay, 2);
 790                 DEBUG_RL_FULL_S("\n");
 791
 792                 /*
 793                  * Broadcast to all PUPs current RL delays: DQS phase,leveling
 794                  * delay
 795                  */
 796                 ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 797
 798                 /* Reset PHY read FIFO */
 799                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 800                         (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 801                 /* 0x15B8 - Training SW 2 Register */
 802                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 803
 804                 do {
 805                         reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 806                                 (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 807                 } while (reg);  /* Wait for '0' */
 808
 809                 /* Read pattern from SDRAM */
 810                 sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 811                 locked_pups = 0;
 812                 if (MV_OK !=
 813                     ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 814                                        rl_pattern, LEN_STD_PATTERN,
 815                                        sdram_offset, 0, 0, NULL, 0))
 816                         return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;
 817
 818                 /* Octet evaluation */
 819                 for (pup = 0; pup < (dram_info->num_of_std_pups *
 820                                      (1 - ecc) + ecc); pup++) {
 821                         /* pup_num = Q or 1 for ECC */
 822                         int idx;
 823
 824                         idx = pup + ecc * ECC_BIT;
 825
 826                         /* Check Overrun */
 827                         if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 828                               (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
 829                                pup)) & 0x1)) {
 830                                 /* If no OverRun */
 831
 832                                 /* Inside the window */
 833                                 if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
 834                                         /*
 835                                          * Match expected value ? - Update
 836                                          * State Machine
 837                                          */
 838                                         if (((~locked_pups >> pup) & 0x1)
 839                                             && (final_delay == 0)) {
 840                                                 /* Match - Still inside the Window */
 841                                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
 842                                                                 (u32)pup, 1);
 843
 844                                         } else {
 845                                                 /* We got fail -> this is the end of the window */
 846                                                 dram_info->rl_val[cs][idx][DE] = delay;
 847                                                 dram_info->rl_val[cs][idx][PE] = phase;
 848                                                 /* Go to Final State */
 849                                                 dram_info->rl_val[cs][idx][S]++;
 850                                                 final_sum++;
 851                                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
 852                                                                 (u32)pup, 1);
 853                                         }
 854
 855                                         /* Before the start of the window */
 856                                 } else if (dram_info->rl_val[cs][idx][S] ==
 857                                            RL_UNLOCK_STATE) {
 858                                         /* Must be RL_UNLOCK_STATE */
 859                                         /*
 860                                          * Match expected value ? - Update
 861                                          * State Machine
 862                                          */
 863                                         if (dram_info->rl_val[cs][idx][C] <
 864                                             RL_RETRY_COUNT) {
 865                                                 if (((~locked_pups >> pup) & 0x1)) {
 866                                                         /* Match */
 867                                                         DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 868                                                                         (u32)pup, 1);
 869                                                         dram_info->rl_val[cs][idx][C]++;
 870
 871                                                         /* If pup got to last state - lock the delays */
 872                                                         if (dram_info->rl_val[cs][idx][C] ==
 873                                                             RL_RETRY_COUNT) {
 874                                                                 dram_info->rl_val[cs][idx][C] = 0;
 875                                                                 dram_info->rl_val[cs][idx][DS] =
 876                                                                         delay;
 877                                                                 dram_info->rl_val[cs][idx][PS] =
 878                                                                         phase;
 879                                                                 dram_info->rl_val[cs][idx][S]++;        /* Go to Window State */
 880                                                                 locked_sum++;
 881                                                                 /* Will count the pups that got locked */
 882
 883                                                                 /* IF First lock - need to lock delays */
 884                                                                 if (first_octet_locked == 0) {
 885                                                                         DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 886                                                                                         (u32)pup, 1);
 887                                                                         first_octet_locked
 888                                                                             =
 889                                                                             1;
 890                                                                 }
 891                                                         }
 892
 893                                                         /* if pup is in not in final state but there was match - dont increment counter */
 894                                                         else {
 895                                                                 counter_in_progress
 896                                                                     = 1;
 897                                                         }
 898                                                 }
 899                                         }
 900                                 }
 901                         } else {
 902                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 903                                                 (u32)pup, 1);
 904                                 counter_in_progress = 1;
 905                         }
 906                 }
 907
 908                 if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
 909                         all_locked = 1;
 910                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 911                 }
 912
 913                 /*
 914                  * This is a fix for unstable condition where pups are
 915                  * toggling between match and no match
 916                  */
 917                 /*
 918                  * If some of the pups is >1 <3, check if we did it too many
 919                  * times
 920                  */
 921                 if (counter_in_progress == 1) {
 922                         if (repeat_max_cnt < RL_RETRY_COUNT) {
 923                                 /* Notify at least one Counter is >=1 and < 3 */
 924                                 repeat_max_cnt++;
 925                                 counter_in_progress = 1;
 926                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 927                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 928                         } else {
 929                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 930                                 counter_in_progress = 0;
 931                         }
 932                 }
 933
 934                 /*
 935                  * Check some of the pups are in the middle of state machine
 936                  * and don't increment the delays
 937                  */
 938                 if (!counter_in_progress && !all_locked) {
 939                         repeat_max_cnt = 0;
 940                         if (!ratio_2to1)
 941                                 ui_max_delay = MAX_DELAY_INV;
 942                         else
 943                                 ui_max_delay = MAX_DELAY;
 944
 945                         /* Increment Delay */
 946                         if (delay < ui_max_delay) {
 947                                 /* Delay Incrementation */
 948                                 delay++;
 949                                 if (delay == ui_max_delay) {
 950                                         /*
 951                                          * Mark the last delay/pahse place
 952                                          * for window final place
 953                                          */
 954                                         if ((!ratio_2to1
 955                                              && phase == MAX_PHASE_RL_L_1TO1)
 956                                             || (ratio_2to1
 957                                                 && phase ==
 958                                                 MAX_PHASE_RL_L_2TO1))
 959                                                 final_delay = 1;
 960                                 }
 961                         } else {
 962                                 /* Phase+CL Incrementation */
 963                                 delay = 0;
 964                                 if (!ratio_2to1) {
 965                                         /* 1:1 mode */
 966                                         if (first_octet_locked) {
 967                                                 /* some pupet was Locked */
 968                                                 if (phase < MAX_PHASE_RL_L_1TO1) {
 969 #ifdef RL_WINDOW_WA
 970                                                         if (phase == 0)
 971 #else
 972                                                         if (phase == 1)
 973 #endif
 974                                                                 phase = 4;
 975                                                         else
 976                                                                 phase++;
 977                                                 } else {
 978                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 979                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
 980                                                 }
 981                                         } else {
 982                                                 /* No Pup was Locked */
 983                                                 if (phase < MAX_PHASE_RL_UL_1TO1) {
 984 #ifdef RL_WINDOW_WA
 985                                                         if (phase == 0)
 986                                                                 phase = 4;
 987 #else
 988                                                         phase++;
 989 #endif
 990                                                 } else
 991                                                         phase = 0;
 992                                         }
 993                                 } else {
 994                                         /* 2:1 mode */
 995                                         if (first_octet_locked) {
 996                                                 /* Some Pup was Locked */
 997                                                 if (phase < MAX_PHASE_RL_L_2TO1) {
 998                                                         phase++;
 999                                                 } else {
1000                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
1001                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
1002                                                 }
1003                                         } else {
1004                                                 /* No Pup was Locked */
1005                                                 if (phase < MAX_PHASE_RL_UL_2TO1)
1006                                                         phase++;
1007                                                 else
1008                                                         phase = 0;
1009                                         }
1010                                 }
1011
1012                                 /*
1013                                  * If we finished a full Phases cycle (so
1014                                  * now phase = 0, need to increment
1015                                  * rd_sample_dly
1016                                  */
1017                                 if (phase == 0 && first_octet_locked == 0) {
1018                                         rd_sample_delay++;
1019
1020                                         /* Set current rd_sample_delay  */
1021                                         reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
1022                                         reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
1023                                                  (REG_READ_DATA_SAMPLE_DELAYS_OFFS
1024                                                   * cs));
1025                                         reg |= (rd_sample_delay <<
1026                                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
1027                                                  cs));
1028                                         reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
1029                                                   reg);
1030                                 }
1031
1032                                 /*
1033                                  * Set current rdReadyDelay according to the
1034                                  * hash table (Need to do this in every phase
1035                                  * change)
1036                                  */
1037                                 if (!ratio_2to1) {
1038                                         /* 1:1 mode */
1039                                         add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1040                                         switch (phase) {
1041                                         case 0:
1042                                                 add = add >>
1043                                                         REG_TRAINING_DEBUG_2_OFFS;
1044                                                 break;
1045                                         case 1:
1046                                                 add = add >>
1047                                                         (REG_TRAINING_DEBUG_2_OFFS
1048                                                          + 3);
1049                                                 break;
1050                                         case 4:
1051                                                 add = add >>
1052                                                         (REG_TRAINING_DEBUG_2_OFFS
1053                                                          + 6);
1054                                                 break;
1055                                         case 5:
1056                                                 add = add >>
1057                                                         (REG_TRAINING_DEBUG_2_OFFS
1058                                                          + 9);
1059                                                 break;
1060                                         }
1061                                 } else {
1062                                         /* 2:1 mode */
1063                                         add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1064                                         add = (add >> phase *
1065                                                REG_TRAINING_DEBUG_3_OFFS);
1066                                 }
1067                                 add &= REG_TRAINING_DEBUG_2_MASK;
1068                                 reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1069                                 reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
1070                                          (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1071                                 reg |= ((rd_sample_delay + add) <<
1072                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1073                                 reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1074                                 dram_info->rd_smpl_dly = rd_sample_delay;
1075                                 dram_info->rd_rdy_dly = rd_sample_delay + add;
1076                         }
1077
1078                         /* Reset counters for pups with states<RD_STATE_COUNT */
1079                         for (pup = 0;
1080                              pup <
1081                              (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1082                              pup++) {
1083                                 if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
1084                                         dram_info->rl_val[cs][idx][C] = 0;
1085                         }
1086                 }
1087         }
1088
1089         phase_min = 10;
1090
1091         for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
1092                 DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
1093                 DEBUG_RL_D((u32) pup, 1);
1094                 DEBUG_RL_S(", PS: ");
1095                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
1096                 DEBUG_RL_S(", DS: ");
1097                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
1098                 DEBUG_RL_S(", PE: ");
1099                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
1100                 DEBUG_RL_S(", DE: ");
1101                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
1102                 DEBUG_RL_S("\n");
1103         }
1104
1105         /* Find center of the window procedure */
1106         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1107              pup++) {
1108 #ifdef RL_WINDOW_WA
1109                 if (!ratio_2to1) {      /* 1:1 mode */
1110                         if (dram_info->rl_val[cs][idx][PS] == 4)
1111                                 dram_info->rl_val[cs][idx][PS] = 1;
1112                         if (dram_info->rl_val[cs][idx][PE] == 4)
1113                                 dram_info->rl_val[cs][idx][PE] = 1;
1114
1115                         delay_s = dram_info->rl_val[cs][idx][PS] *
1116                                 MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
1117                         delay_e = dram_info->rl_val[cs][idx][PE] *
1118                                 MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];
1119
1120                         tmp = (delay_e - delay_s) / 2 + delay_s;
1121                         phase = tmp / MAX_DELAY_INV;
1122                         if (phase == 1) /* 1:1 mode */
1123                                 phase = 4;
1124
1125                         if (phase < phase_min)  /* for the read ready delay */
1126                                 phase_min = phase;
1127
1128                         dram_info->rl_val[cs][idx][P] = phase;
1129                         dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;
1130
1131                 } else {
1132                         delay_s = dram_info->rl_val[cs][idx][PS] *
1133                                 MAX_DELAY + dram_info->rl_val[cs][idx][DS];
1134                         delay_e = dram_info->rl_val[cs][idx][PE] *
1135                                 MAX_DELAY + dram_info->rl_val[cs][idx][DE];
1136
1137                         tmp = (delay_e - delay_s) / 2 + delay_s;
1138                         phase = tmp / MAX_DELAY;
1139
1140                         if (phase < phase_min)  /* for the read ready delay */
1141                                 phase_min = phase;
1142
1143                         dram_info->rl_val[cs][idx][P] = phase;
1144                         dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1145                 }
1146 #else
1147                 if (!ratio_2to1) {      /* 1:1 mode */
1148                         if (dram_info->rl_val[cs][idx][PS] > 1)
1149                                 dram_info->rl_val[cs][idx][PS] -= 2;
1150                         if (dram_info->rl_val[cs][idx][PE] > 1)
1151                                 dram_info->rl_val[cs][idx][PE] -= 2;
1152                 }
1153
1154                 delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
1155                         dram_info->rl_val[cs][idx][DS];
1156                 delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
1157                         dram_info->rl_val[cs][idx][DE];
1158
1159                 tmp = (delay_e - delay_s) / 2 + delay_s;
1160                 phase = tmp / MAX_DELAY;
1161                 if (!ratio_2to1 && phase > 1)   /* 1:1 mode */
1162                         phase += 2;
1163
1164                 if (phase < phase_min)  /* for the read ready delay */
1165                         phase_min = phase;
1166
1167                 dram_info->rl_val[cs][idx][P] = phase;
1168                 dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1169 #endif
1170         }
1171
1172         /* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
1173         if (!ratio_2to1) {      /* 1:1 mode */
1174                 add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1175                 switch (phase_min) {
1176                 case 0:
1177                         add = (add >> REG_TRAINING_DEBUG_2_OFFS);
1178                         break;
1179                 case 1:
1180                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
1181                         break;
1182                 case 4:
1183                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
1184                         break;
1185                 case 5:
1186                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
1187                         break;
1188                 }
1189         } else {                /* 2:1 mode */
1190                 add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1191                 add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
1192         }
1193
1194         add &= REG_TRAINING_DEBUG_2_MASK;
1195         reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1196         reg &=
1197             ~(REG_READ_DATA_READY_DELAYS_MASK <<
1198               (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1199         reg |=
1200             ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1201         reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1202         dram_info->rd_rdy_dly = rd_sample_delay + add;
1203
1204         for (cs = 0; cs < dram_info->num_cs; cs++) {
1205                 for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
1206                         reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
1207                         dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
1208                 }
1209         }
1210
1211         return MV_OK;
1212 }
1213 #endif