drivers/ddr/marvell/axp/ddr3_pbs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Marvell International Ltd. and its affiliates
   4  */
   5
   6 #include <common.h>
   7 #include <i2c.h>
   8 #include <spl.h>
   9 #include <asm/io.h>
  10 #include <asm/arch/cpu.h>
  11 #include <asm/arch/soc.h>
  12 #include <linux/delay.h>
  13
  14 #include "ddr3_hw_training.h"
  15
  16 /*
  17  * Debug
  18  */
  19 #define DEBUG_PBS_FULL_C(s, d, l) \
  20         DEBUG_PBS_FULL_S(s); DEBUG_PBS_FULL_D(d, l); DEBUG_PBS_FULL_S("\n")
  21 #define DEBUG_PBS_C(s, d, l) \
  22         DEBUG_PBS_S(s); DEBUG_PBS_D(d, l); DEBUG_PBS_S("\n")
  23
  24 #ifdef MV_DEBUG_PBS
  25 #define DEBUG_PBS_S(s)                  puts(s)
  26 #define DEBUG_PBS_D(d, l)               printf("%x", d)
  27 #else
  28 #define DEBUG_PBS_S(s)
  29 #define DEBUG_PBS_D(d, l)
  30 #endif
  31
  32 #ifdef MV_DEBUG_FULL_PBS
  33 #define DEBUG_PBS_FULL_S(s)             puts(s)
  34 #define DEBUG_PBS_FULL_D(d, l)          printf("%x", d)
  35 #else
  36 #define DEBUG_PBS_FULL_S(s)
  37 #define DEBUG_PBS_FULL_D(d, l)
  38 #endif
  39
  40 #if defined(MV88F78X60) || defined(MV88F672X)
  41
  42 /* Temp array for skew data storage */
  43 static u32 skew_array[(MAX_PUP_NUM) * DQ_NUM] = { 0 };
  44
  45 /* PBS locked dq (per pup) */
  46 extern u32 pbs_locked_dq[MAX_PUP_NUM][DQ_NUM];
  47 extern u32 pbs_locked_dm[MAX_PUP_NUM];
  48 extern u32 pbs_locked_value[MAX_PUP_NUM][DQ_NUM];
  49
  50 #if defined(MV88F672X)
  51 extern u32 pbs_pattern[2][LEN_16BIT_PBS_PATTERN];
  52 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
  53 #else
  54 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN];
  55 extern u32 pbs_pattern_64b[2][LEN_PBS_PATTERN];
  56 #endif
  57
  58 extern u32 pbs_dq_mapping[PUP_NUM_64BIT + 1][DQ_NUM];
  59
  60 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
  61                 u32 cur_pup, u32 pbs_pattern_idx, u32 ecc);
  62 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
  63                 u32 pbs_pattern_idx, u32 ecc);
  64 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
  65                 u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc);
  66 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx);
  67 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay);
  68
  69 /*
  70  * Name:     ddr3_pbs_tx
  71  * Desc:     Execute the PBS TX phase.
  72  * Args:     dram_info   ddr3 training information struct
  73  * Notes:
  74  * Returns:  MV_OK if success, other error code if fail.
  75  */
  76 int ddr3_pbs_tx(MV_DRAM_INFO *dram_info)
  77 {
  78         /* Array of Deskew results */
  79
  80         /*
  81          * Array to hold the total sum of skew from all iterations
  82          * (for average purpose)
  83          */
  84         u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
  85
  86         /*
  87          * Array to hold the total average skew from both patterns
  88          * (for average purpose)
  89          */
  90         u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
  91
  92         u32 pbs_rep_time = 0;   /* counts number of loop in case of fail */
  93         /* bit array for unlock pups - used to repeat on the RX operation */
  94         u32 cur_pup;
  95         u32 max_pup;
  96         u32 pbs_retry;
  97         u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
  98         u32 pattern_idx;
  99         u32 ecc;
 100         /* indicates whether we need to start the loop again */
 101         int start_over;
 102
 103         DEBUG_PBS_S("DDR3 - PBS TX - Starting PBS TX procedure\n");
 104
 105         pups = dram_info->num_of_total_pups;
 106         max_pup = dram_info->num_of_total_pups;
 107
 108         /* Enable SW override */
 109         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 110                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 111         /* [0] = 1 - Enable SW override  */
 112         /* 0x15B8 - Training SW 2 Register */
 113         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 114         DEBUG_PBS_S("DDR3 - PBS RX - SW Override Enabled\n");
 115
 116         reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
 117         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 118
 119         /* Running twice for 2 different patterns. each patterns - 3 times */
 120         for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
 121                 DEBUG_PBS_C("DDR3 - PBS TX - Working with pattern - ",
 122                             pattern_idx, 1);
 123
 124                 /* Reset sum array */
 125                 for (pup = 0; pup < pups; pup++) {
 126                         for (dq = 0; dq < DQ_NUM; dq++)
 127                                 skew_sum_array[pup][dq] = 0;
 128                 }
 129
 130                 /*
 131                  * Perform PBS several of times (3 for each pattern).
 132                  * At the end, we'll use the average
 133                  */
 134                 /* If there is ECC, do each PBS again with mux change */
 135                 for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
 136                         for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 137
 138                                 /*
 139                                  * This parameter stores the current PUP
 140                                  * num - ecc mode dependent - 4-8 / 1 pups
 141                                  */
 142                                 cur_max_pup = (1 - ecc) *
 143                                         dram_info->num_of_std_pups + ecc;
 144
 145                                 if (ecc) {
 146                                         /* Only 1 pup in this case */
 147                                         valid_pup = 0x1;
 148                                 } else if (cur_max_pup > 4) {
 149                                         /* 64 bit - 8 pups */
 150                                         valid_pup = 0xFF;
 151                                 } else if (cur_max_pup == 4) {
 152                                         /* 32 bit - 4 pups */
 153                                         valid_pup = 0xF;
 154                                 } else {
 155                                         /* 16 bit - 2 pups */
 156                                         valid_pup = 0x3;
 157                                 }
 158
 159                                 /* ECC Support - Switch ECC Mux on ecc=1 */
 160                                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 161                                         ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 162                                 reg |= (dram_info->ecc_ena * ecc <<
 163                                         REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 164                                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 165
 166                                 if (ecc)
 167                                         DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Enabled\n");
 168                                 else
 169                                         DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Disabled\n");
 170
 171                                 /* Init iteration values */
 172                                 /* Clear the locked DQs */
 173                                 for (pup = 0; pup < cur_max_pup; pup++) {
 174                                         for (dq = 0; dq < DQ_NUM; dq++) {
 175                                                 pbs_locked_dq[
 176                                                         pup + ecc *
 177                                                         (max_pup - 1)][dq] =
 178                                                         0;
 179                                         }
 180                                 }
 181
 182                                 pbs_rep_time = 0;
 183                                 cur_pup = valid_pup;
 184                                 start_over = 0;
 185
 186                                 /*
 187                                  * Run loop On current Pattern and current
 188                                  * pattern iteration (just to cover the false
 189                                  * fail problem)
 190                                  */
 191                                 do {
 192                                         DEBUG_PBS_S("DDR3 - PBS Tx - Pbs Rep Loop is ");
 193                                         DEBUG_PBS_D(pbs_rep_time, 1);
 194                                         DEBUG_PBS_S(", for Retry No.");
 195                                         DEBUG_PBS_D(pbs_retry, 1);
 196                                         DEBUG_PBS_S("\n");
 197
 198                                         /* Set all PBS values to MIN (0) */
 199                                         DEBUG_PBS_S("DDR3 - PBS Tx - Set all PBS values to MIN\n");
 200
 201                                         for (dq = 0; dq < DQ_NUM; dq++) {
 202                                                 ddr3_write_pup_reg(
 203                                                         PUP_PBS_TX +
 204                                                         pbs_dq_mapping[pup *
 205                                                                 (1 - ecc) +
 206                                                                 ecc * ECC_PUP]
 207                                                         [dq], CS0, (1 - ecc) *
 208                                                         PUP_BC + ecc * ECC_PUP, 0,
 209                                                         0);
 210                                         }
 211
 212                                         /*
 213                                          * Shift DQ ADLL right, One step before
 214                                          * fail
 215                                          */
 216                                         DEBUG_PBS_S("DDR3 - PBS Tx - ADLL shift right one phase before fail\n");
 217
 218                                         if (MV_OK != ddr3_tx_shift_dqs_adll_step_before_fail
 219                                             (dram_info, cur_pup, pattern_idx,
 220                                              ecc))
 221                                                 return MV_DDR3_TRAINING_ERR_PBS_ADLL_SHR_1PHASE;
 222
 223                                         /* PBS For each bit */
 224                                         DEBUG_PBS_S("DDR3 - PBS Tx - perform PBS for each bit\n");
 225
 226                                         /*
 227                                          * In this stage - start_over = 0
 228                                          */
 229                                         if (MV_OK != ddr3_pbs_per_bit(
 230                                                     dram_info, &start_over, 1,
 231                                                     &cur_pup, pattern_idx, ecc))
 232                                                 return MV_DDR3_TRAINING_ERR_PBS_TX_PER_BIT;
 233
 234                                 } while ((start_over == 1) &&
 235                                          (++pbs_rep_time < COUNT_PBS_STARTOVER));
 236
 237                                 if (pbs_rep_time == COUNT_PBS_STARTOVER &&
 238                                     start_over == 1) {
 239                                         DEBUG_PBS_S("DDR3 - PBS Tx - FAIL - Adll reach max value\n");
 240                                         return MV_DDR3_TRAINING_ERR_PBS_TX_MAX_VAL;
 241                                 }
 242
 243                                 DEBUG_PBS_FULL_C("DDR3 - PBS TX - values for iteration - ",
 244                                                  pbs_retry, 1);
 245                                 for (pup = 0; pup < cur_max_pup; pup++) {
 246                                         /*
 247                                          * To minimize delay elements, inc
 248                                          * from pbs value the min pbs val
 249                                          */
 250                                         DEBUG_PBS_S("DDR3 - PBS - PUP");
 251                                         DEBUG_PBS_D((pup + (ecc * ECC_PUP)), 1);
 252                                         DEBUG_PBS_S(": ");
 253
 254                                         for (dq = 0; dq < DQ_NUM; dq++) {
 255                                                 /* Set skew value for all dq */
 256                                                 /*
 257                                                  * Bit# Deskew <- Bit# Deskew -
 258                                                  * last / first  failing bit
 259                                                  * Deskew For all bits (per PUP)
 260                                                  * (minimize delay elements)
 261                                                  */
 262                                                 DEBUG_PBS_S("DQ");
 263                                                 DEBUG_PBS_D(dq, 1);
 264                                                 DEBUG_PBS_S("-");
 265                                                 DEBUG_PBS_D(skew_array
 266                                                             [((pup) * DQ_NUM) +
 267                                                              dq], 2);
 268                                                 DEBUG_PBS_S(", ");
 269                                         }
 270                                         DEBUG_PBS_S("\n");
 271                                 }
 272
 273                                 /*
 274                                  * Collect the results we got on this trial
 275                                  * of PBS
 276                                  */
 277                                 for (pup = 0; pup < cur_max_pup; pup++) {
 278                                         for (dq = 0; dq < DQ_NUM; dq++) {
 279                                                 skew_sum_array[pup + (ecc * (max_pup - 1))]
 280                                                         [dq] += skew_array
 281                                                         [((pup) * DQ_NUM) + dq];
 282                                         }
 283                                 }
 284
 285                                 /* ECC Support - Disable ECC MUX */
 286                                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 287                                         ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 288                                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 289                         }
 290                 }
 291
 292                 DEBUG_PBS_C("DDR3 - PBS TX - values for current pattern - ",
 293                             pattern_idx, 1);
 294                 for (pup = 0; pup < max_pup; pup++) {
 295                         /*
 296                          * To minimize delay elements, inc from pbs value the
 297                          * min pbs val
 298                          */
 299                         DEBUG_PBS_S("DDR3 - PBS - PUP");
 300                         DEBUG_PBS_D(pup, 1);
 301                         DEBUG_PBS_S(": ");
 302
 303                         for (dq = 0; dq < DQ_NUM; dq++) {
 304                                 /* set skew value for all dq */
 305                                 /* Bit# Deskew <- Bit# Deskew - last / first  failing bit Deskew For all bits (per PUP) (minimize delay elements) */
 306                                 DEBUG_PBS_S("DQ");
 307                                 DEBUG_PBS_D(dq, 1);
 308                                 DEBUG_PBS_S("-");
 309                                 DEBUG_PBS_D(skew_sum_array[pup][dq] /
 310                                             COUNT_PBS_REPEAT, 2);
 311                                 DEBUG_PBS_S(", ");
 312                         }
 313                         DEBUG_PBS_S("\n");
 314                 }
 315
 316                 /*
 317                  * Calculate the average skew for current pattern for each
 318                  * pup and each bit
 319                  */
 320                 DEBUG_PBS_C("DDR3 - PBS TX - Average for pattern - ",
 321                             pattern_idx, 1);
 322
 323                 for (pup = 0; pup < max_pup; pup++) {
 324                         /*
 325                          * FOR ECC only :: found min and max value for current
 326                          * pattern skew array
 327                          */
 328                         /* Loop for all dqs */
 329                         for (dq = 0; dq < DQ_NUM; dq++) {
 330                                 pattern_skew_array[pup][dq] +=
 331                                         (skew_sum_array[pup][dq] /
 332                                          COUNT_PBS_REPEAT);
 333                         }
 334                 }
 335         }
 336
 337         /* Calculate the average skew */
 338         for (pup = 0; pup < max_pup; pup++) {
 339                 for (dq = 0; dq < DQ_NUM; dq++)
 340                         skew_array[((pup) * DQ_NUM) + dq] =
 341                                 pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
 342         }
 343
 344         DEBUG_PBS_S("DDR3 - PBS TX - Average for all patterns:\n");
 345         for (pup = 0; pup < max_pup; pup++) {
 346                 /*
 347                  * To minimize delay elements, inc from pbs value the min
 348                  * pbs val
 349                  */
 350                 DEBUG_PBS_S("DDR3 - PBS - PUP");
 351                 DEBUG_PBS_D(pup, 1);
 352                 DEBUG_PBS_S(": ");
 353
 354                 for (dq = 0; dq < DQ_NUM; dq++) {
 355                         /* Set skew value for all dq */
 356                         /*
 357                          * Bit# Deskew <- Bit# Deskew - last / first
 358                          * failing bit Deskew For all bits (per PUP)
 359                          * (minimize delay elements)
 360                          */
 361                         DEBUG_PBS_S("DQ");
 362                         DEBUG_PBS_D(dq, 1);
 363                         DEBUG_PBS_S("-");
 364                         DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
 365                         DEBUG_PBS_S(", ");
 366                 }
 367                 DEBUG_PBS_S("\n");
 368         }
 369
 370         /* Return ADLL to default value */
 371         for (pup = 0; pup < max_pup; pup++) {
 372                 if (pup == (max_pup - 1) && dram_info->ecc_ena)
 373                         pup = ECC_PUP;
 374                 ddr3_pbs_write_pup_dqs_reg(CS0, pup, INIT_WL_DELAY);
 375         }
 376
 377         /* Set averaged PBS results */
 378         ddr3_set_pbs_results(dram_info, 1);
 379
 380         /* Disable SW override - Must be in a different stage */
 381         /* [0]=0 - Enable SW override  */
 382         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 383         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 384         /* 0x15B8 - Training SW 2 Register */
 385         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 386
 387         reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 388                 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 389         reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 390
 391         DEBUG_PBS_S("DDR3 - PBS Tx - PBS TX ended successfuly\n");
 392
 393         return MV_OK;
 394 }
 395
 396 /*
 397  * Name:     ddr3_tx_shift_dqs_adll_step_before_fail
 398  * Desc:     Execute the Tx shift DQ phase.
 399  * Args:     dram_info            ddr3 training information struct
 400  *           cur_pup              bit array of the function active pups.
 401  *           pbs_pattern_idx      Index of PBS pattern
 402  * Notes:
 403  * Returns:  MV_OK if success, other error code if fail.
 404  */
 405 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info,
 406                                                    u32 cur_pup,
 407                                                    u32 pbs_pattern_idx, u32 ecc)
 408 {
 409         u32 unlock_pup;         /* bit array of unlock pups  */
 410         u32 new_lockup_pup;     /* bit array of compare failed pups */
 411         u32 adll_val = 4;       /* INIT_WL_DELAY */
 412         u32 cur_max_pup, pup;
 413         u32 dqs_dly_set[MAX_PUP_NUM] = { 0 };
 414         u32 *pattern_ptr;
 415
 416         /* Choose pattern */
 417         switch (dram_info->ddr_width) {
 418 #if defined(MV88F672X)
 419         case 16:
 420                 pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
 421                 break;
 422 #endif
 423         case 32:
 424                 pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
 425                 break;
 426 #if defined(MV88F78X60)
 427         case 64:
 428                 pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
 429                 break;
 430 #endif
 431         default:
 432                 return MV_FAIL;
 433         }
 434
 435         /* Set current pup number */
 436         if (cur_pup == 0x1)     /* Ecc mode */
 437                 cur_max_pup = 1;
 438         else
 439                 cur_max_pup = dram_info->num_of_std_pups;
 440
 441         unlock_pup = cur_pup;   /* '1' for each unlocked pup */
 442
 443         /* Loop on all ADLL Vaules */
 444         do {
 445                 /* Loop until found first fail */
 446                 adll_val++;
 447
 448                 /*
 449                  * Increment (Move to right - ADLL) DQ TX delay
 450                  * (broadcast to all Data PUPs)
 451                  */
 452                 for (pup = 0; pup < cur_max_pup; pup++)
 453                         ddr3_pbs_write_pup_dqs_reg(CS0,
 454                                                    pup * (1 - ecc) +
 455                                                    ECC_PUP * ecc, adll_val);
 456
 457                 /*
 458                  * Write and Read, compare results (read was already verified)
 459                  */
 460                 /* 0 - all locked */
 461                 new_lockup_pup = 0;
 462
 463                 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
 464                                                 &new_lockup_pup,
 465                                                 pattern_ptr, LEN_PBS_PATTERN,
 466                                                 SDRAM_PBS_TX_OFFS, 1, 0,
 467                                                 NULL,
 468                                                 0))
 469                         return MV_FAIL;
 470
 471                 unlock_pup &= ~new_lockup_pup;
 472
 473                 DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
 474                 DEBUG_PBS_FULL_D(unlock_pup, 2);
 475                 DEBUG_PBS_FULL_C(", Set ADLL value = ", adll_val, 2);
 476
 477                 /* If any PUP failed there is '1' to mark the PUP */
 478                 if (new_lockup_pup != 0) {
 479                         /*
 480                          * Decrement (Move Back to Left two steps - ADLL)
 481                          * DQ TX delay for current failed pups and save
 482                          */
 483                         for (pup = 0; pup < cur_max_pup; pup++) {
 484                                 if (((new_lockup_pup >> pup) & 0x1) &&
 485                                     dqs_dly_set[pup] == 0)
 486                                         dqs_dly_set[pup] = adll_val - 1;
 487                         }
 488                 }
 489         } while ((unlock_pup != 0) && (adll_val != ADLL_MAX));
 490
 491         if (unlock_pup != 0) {
 492                 DEBUG_PBS_FULL_S("DDR3 - PBS Tx - Shift DQ - Adll value reached maximum\n");
 493
 494                 for (pup = 0; pup < cur_max_pup; pup++) {
 495                         if (((unlock_pup >> pup) & 0x1) &&
 496                             dqs_dly_set[pup] == 0)
 497                                 dqs_dly_set[pup] = adll_val - 1;
 498                 }
 499         }
 500
 501         DEBUG_PBS_FULL_C("PBS TX one step before fail last pups locked Adll ",
 502                          adll_val - 2, 2);
 503
 504         /* Set the PUP DQS DLY Values */
 505         for (pup = 0; pup < cur_max_pup; pup++)
 506                 ddr3_pbs_write_pup_dqs_reg(CS0, pup * (1 - ecc) + ECC_PUP * ecc,
 507                                            dqs_dly_set[pup]);
 508
 509         /* Found one phase before fail */
 510         return MV_OK;
 511 }
 512
 513 /*
 514  * Name:     ddr3_pbs_rx
 515  * Desc:     Execute the PBS RX phase.
 516  * Args:     dram_info   ddr3 training information struct
 517  * Notes:
 518  * Returns:  MV_OK if success, other error code if fail.
 519  */
 520 int ddr3_pbs_rx(MV_DRAM_INFO *dram_info)
 521 {
 522         /*
 523          * Array to hold the total sum of skew from all iterations
 524          * (for average purpose)
 525          */
 526         u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
 527
 528         /*
 529          * Array to hold the total average skew from both patterns
 530          * (for average purpose)
 531          */
 532         u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} };
 533
 534         u32 pbs_rep_time = 0;   /* counts number of loop in case of fail */
 535         /* bit array for unlock pups - used to repeat on the RX operation */
 536         u32 cur_pup;
 537         u32 max_pup;
 538         u32 pbs_retry;
 539         u32 pup, dq, pups, cur_max_pup, valid_pup, reg;
 540         u32 pattern_idx;
 541         u32 ecc;
 542         /* indicates whether we need to start the loop again */
 543         int start_over;
 544         int status;
 545
 546         DEBUG_PBS_S("DDR3 - PBS RX - Starting PBS RX procedure\n");
 547
 548         pups = dram_info->num_of_total_pups;
 549         max_pup = dram_info->num_of_total_pups;
 550
 551         /* Enable SW override */
 552         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 553                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 554         /* [0] = 1 - Enable SW override  */
 555         /* 0x15B8 - Training SW 2 Register */
 556         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 557         DEBUG_PBS_FULL_S("DDR3 - PBS RX - SW Override Enabled\n");
 558
 559         reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS;
 560         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 561
 562         /* Running twice for 2 different patterns. each patterns - 3 times */
 563         for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) {
 564                 DEBUG_PBS_FULL_C("DDR3 - PBS RX - Working with pattern - ",
 565                                  pattern_idx, 1);
 566
 567                 /* Reset sum array */
 568                 for (pup = 0; pup < pups; pup++) {
 569                         for (dq = 0; dq < DQ_NUM; dq++)
 570                                 skew_sum_array[pup][dq] = 0;
 571                 }
 572
 573                 /*
 574                  * Perform PBS several of times (3 for each pattern).
 575                  * At the end, we'll use the average
 576                  */
 577                 /* If there is ECC, do each PBS again with mux change */
 578                 for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) {
 579                         for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 580                                 /*
 581                                  * This parameter stores the current PUP
 582                                  * num - ecc mode dependent - 4-8 / 1 pups
 583                                  */
 584                                 cur_max_pup = (1 - ecc) *
 585                                         dram_info->num_of_std_pups + ecc;
 586
 587                                 if (ecc) {
 588                                         /* Only 1 pup in this case */
 589                                         valid_pup = 0x1;
 590                                 } else if (cur_max_pup > 4) {
 591                                         /* 64 bit - 8 pups */
 592                                         valid_pup = 0xFF;
 593                                 } else if (cur_max_pup == 4) {
 594                                         /* 32 bit - 4 pups */
 595                                         valid_pup = 0xF;
 596                                 } else {
 597                                         /* 16 bit - 2 pups */
 598                                         valid_pup = 0x3;
 599                                 }
 600
 601                                 /* ECC Support - Switch ECC Mux on ecc=1 */
 602                                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 603                                         ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 604                                 reg |= (dram_info->ecc_ena * ecc <<
 605                                         REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 606                                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 607
 608                                 if (ecc)
 609                                         DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Enabled\n");
 610                                 else
 611                                         DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Disabled\n");
 612
 613                                 /* Init iteration values */
 614                                 /* Clear the locked DQs */
 615                                 for (pup = 0; pup < cur_max_pup; pup++) {
 616                                         for (dq = 0; dq < DQ_NUM; dq++) {
 617                                                 pbs_locked_dq[
 618                                                         pup + ecc * (max_pup - 1)][dq] =
 619                                                         0;
 620                                         }
 621                                 }
 622
 623                                 pbs_rep_time = 0;
 624                                 cur_pup = valid_pup;
 625                                 start_over = 0;
 626
 627                                 /*
 628                                  * Run loop On current Pattern and current
 629                                  * pattern iteration (just to cover the false
 630                                  * fail problem
 631                                  */
 632                                 do {
 633                                         DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Pbs Rep Loop is ");
 634                                         DEBUG_PBS_FULL_D(pbs_rep_time, 1);
 635                                         DEBUG_PBS_FULL_S(", for Retry No.");
 636                                         DEBUG_PBS_FULL_D(pbs_retry, 1);
 637                                         DEBUG_PBS_FULL_S("\n");
 638
 639                                         /* Set all PBS values to MAX (31) */
 640                                         for (pup = 0; pup < cur_max_pup; pup++) {
 641                                                 for (dq = 0; dq < DQ_NUM; dq++)
 642                                                         ddr3_write_pup_reg(
 643                                                                 PUP_PBS_RX +
 644                                                                 pbs_dq_mapping[
 645                                                                 pup * (1 - ecc)
 646                                                                 + ecc * ECC_PUP]
 647                                                                 [dq], CS0,
 648                                                                 pup + ecc * ECC_PUP,
 649                                                                 0, MAX_PBS);
 650                                         }
 651
 652                                         /* Set all DQS PBS values to MIN (0) */
 653                                         for (pup = 0; pup < cur_max_pup; pup++) {
 654                                                 ddr3_write_pup_reg(PUP_PBS_RX +
 655                                                                    DQ_NUM, CS0,
 656                                                                    pup +
 657                                                                    ecc *
 658                                                                    ECC_PUP, 0,
 659                                                                    0);
 660                                         }
 661
 662                                         /* Shift DQS, To first Fail */
 663                                         DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift RX DQS to first fail\n");
 664
 665                                         status = ddr3_rx_shift_dqs_to_first_fail
 666                                                 (dram_info, cur_pup,
 667                                                  pattern_idx, ecc);
 668                                         if (MV_OK != status) {
 669                                                 DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_rx_shift_dqs_to_first_fail failed.\n");
 670                                                 DEBUG_PBS_D(status, 8);
 671                                                 DEBUG_PBS_S("\nDDR3 - PBS Rx - SKIP.\n");
 672
 673                                                 /* Reset read FIFO */
 674                                                 reg = reg_read(REG_DRAM_TRAINING_ADDR);
 675                                                 /* Start Auto Read Leveling procedure */
 676                                                 reg |= (1 << REG_DRAM_TRAINING_RL_OFFS);
 677                                                 /* 0x15B0 - Training Register */
 678                                                 reg_write(REG_DRAM_TRAINING_ADDR, reg);
 679
 680                                                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 681                                                 reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS)
 682                                                         + (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS));
 683                                                 /* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset  */
 684                                                 /* 0x15B8 - Training SW 2 Register */
 685                                                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 686
 687                                                 do {
 688                                                         reg = (reg_read(REG_DRAM_TRAINING_2_ADDR))
 689                                                                 & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 690                                                 } while (reg);  /* Wait for '0' */
 691
 692                                                 reg = reg_read(REG_DRAM_TRAINING_ADDR);
 693                                                 /* Clear Auto Read Leveling procedure */
 694                                                 reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS);
 695                                                 /* 0x15B0 - Training Register */
 696                                                 reg_write(REG_DRAM_TRAINING_ADDR, reg);
 697
 698                                                 /* Set ADLL to 15 */
 699                                                 for (pup = 0; pup < max_pup;
 700                                                      pup++) {
 701                                                         ddr3_write_pup_reg
 702                                                             (PUP_DQS_RD, CS0,
 703                                                              pup +
 704                                                              (ecc * ECC_PUP), 0,
 705                                                              15);
 706                                                 }
 707
 708                                                 /* Set all PBS values to MIN (0) */
 709                                                 for (pup = 0; pup < cur_max_pup;
 710                                                      pup++) {
 711                                                         for (dq = 0;
 712                                                              dq < DQ_NUM; dq++)
 713                                                                 ddr3_write_pup_reg
 714                                                                     (PUP_PBS_RX +
 715                                                                      pbs_dq_mapping
 716                                                                      [pup * (1 - ecc) +
 717                                                                       ecc * ECC_PUP]
 718                                                                      [dq], CS0,
 719                                                                      pup + ecc * ECC_PUP,
 720                                                                      0, MIN_PBS);
 721                                                 }
 722
 723                                                 return MV_OK;
 724                                         }
 725
 726                                         /* PBS For each bit */
 727                                         DEBUG_PBS_FULL_S("DDR3 - PBS Rx - perform PBS for each bit\n");
 728                                         /* in this stage - start_over = 0; */
 729                                         if (MV_OK != ddr3_pbs_per_bit(
 730                                                     dram_info, &start_over,
 731                                                     0, &cur_pup,
 732                                                     pattern_idx, ecc)) {
 733                                                 DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_pbs_per_bit failed.");
 734                                                 return MV_DDR3_TRAINING_ERR_PBS_RX_PER_BIT;
 735                                         }
 736
 737                                 } while ((start_over == 1) &&
 738                                          (++pbs_rep_time < COUNT_PBS_STARTOVER));
 739
 740                                 if (pbs_rep_time == COUNT_PBS_STARTOVER &&
 741                                     start_over == 1) {
 742                                         DEBUG_PBS_FULL_S("DDR3 - PBS Rx - FAIL - Algorithm failed doing RX PBS\n");
 743                                         return MV_DDR3_TRAINING_ERR_PBS_RX_MAX_VAL;
 744                                 }
 745
 746                                 /* Return DQS ADLL to default value - 15 */
 747                                 /* Set all DQS PBS values to MIN (0) */
 748                                 for (pup = 0; pup < cur_max_pup; pup++)
 749                                         ddr3_write_pup_reg(PUP_DQS_RD, CS0,
 750                                                            pup + ecc * ECC_PUP,
 751                                                            0, INIT_RL_DELAY);
 752
 753                                 DEBUG_PBS_FULL_C("DDR3 - PBS RX - values for iteration - ",
 754                                                  pbs_retry, 1);
 755                                 for (pup = 0; pup < cur_max_pup; pup++) {
 756                                         /*
 757                                          * To minimize delay elements, inc from
 758                                          * pbs value the min pbs val
 759                                          */
 760                                         DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
 761                                         DEBUG_PBS_FULL_D((pup +
 762                                                           (ecc * ECC_PUP)), 1);
 763                                         DEBUG_PBS_FULL_S(": ");
 764
 765                                         for (dq = 0; dq < DQ_NUM; dq++) {
 766                                                 /* Set skew value for all dq */
 767                                                 /*
 768                                                  * Bit# Deskew <- Bit# Deskew -
 769                                                  * last / first  failing bit
 770                                                  * Deskew For all bits (per PUP)
 771                                                  * (minimize delay elements)
 772                                                  */
 773                                                 DEBUG_PBS_FULL_S("DQ");
 774                                                 DEBUG_PBS_FULL_D(dq, 1);
 775                                                 DEBUG_PBS_FULL_S("-");
 776                                                 DEBUG_PBS_FULL_D(skew_array
 777                                                                  [((pup) *
 778                                                                    DQ_NUM) +
 779                                                                   dq], 2);
 780                                                 DEBUG_PBS_FULL_S(", ");
 781                                         }
 782                                         DEBUG_PBS_FULL_S("\n");
 783                                 }
 784
 785                                 /*
 786                                  * Collect the results we got on this trial
 787                                  * of PBS
 788                                  */
 789                                 for (pup = 0; pup < cur_max_pup; pup++) {
 790                                         for (dq = 0; dq < DQ_NUM; dq++) {
 791                                                 skew_sum_array
 792                                                         [pup + (ecc * (max_pup - 1))]
 793                                                         [dq] +=
 794                                                         skew_array[((pup) * DQ_NUM) + dq];
 795                                         }
 796                                 }
 797
 798                                 /* ECC Support - Disable ECC MUX */
 799                                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 800                                         ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 801                                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 802                         }
 803                 }
 804
 805                 /*
 806                  * Calculate the average skew for current pattern for each
 807                  * pup and each bit
 808                  */
 809                 DEBUG_PBS_FULL_C("DDR3 - PBS RX - Average for pattern - ",
 810                                  pattern_idx, 1);
 811                 for (pup = 0; pup < max_pup; pup++) {
 812                         /*
 813                          * FOR ECC only :: found min and max value for
 814                          * current pattern skew array
 815                          */
 816                         /* Loop for all dqs */
 817                         for (dq = 0; dq < DQ_NUM; dq++) {
 818                                 pattern_skew_array[pup][dq] +=
 819                                         (skew_sum_array[pup][dq] /
 820                                          COUNT_PBS_REPEAT);
 821                         }
 822                 }
 823
 824                 DEBUG_PBS_C("DDR3 - PBS RX - values for current pattern - ",
 825                             pattern_idx, 1);
 826                 for (pup = 0; pup < max_pup; pup++) {
 827                         /*
 828                          * To minimize delay elements, inc from pbs value the
 829                          * min pbs val
 830                          */
 831                         DEBUG_PBS_S("DDR3 - PBS RX - PUP");
 832                         DEBUG_PBS_D(pup, 1);
 833                         DEBUG_PBS_S(": ");
 834
 835                         for (dq = 0; dq < DQ_NUM; dq++) {
 836                                 /* Set skew value for all dq */
 837                                 /*
 838                                  * Bit# Deskew <- Bit# Deskew - last / first
 839                                  * failing bit Deskew For all bits (per PUP)
 840                                  * (minimize delay elements)
 841                                  */
 842                                 DEBUG_PBS_S("DQ");
 843                                 DEBUG_PBS_D(dq, 1);
 844                                 DEBUG_PBS_S("-");
 845                                 DEBUG_PBS_D(skew_sum_array[pup][dq] /
 846                                             COUNT_PBS_REPEAT, 2);
 847                                 DEBUG_PBS_S(", ");
 848                         }
 849                         DEBUG_PBS_S("\n");
 850                 }
 851         }
 852
 853         /* Calculate the average skew */
 854         for (pup = 0; pup < max_pup; pup++) {
 855                 for (dq = 0; dq < DQ_NUM; dq++)
 856                         skew_array[((pup) * DQ_NUM) + dq] =
 857                                 pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;
 858         }
 859
 860         DEBUG_PBS_S("DDR3 - PBS RX - Average for all patterns:\n");
 861         for (pup = 0; pup < max_pup; pup++) {
 862                 /*
 863                  * To minimize delay elements, inc from pbs value the
 864                  * min pbs val
 865                  */
 866                 DEBUG_PBS_S("DDR3 - PBS - PUP");
 867                 DEBUG_PBS_D(pup, 1);
 868                 DEBUG_PBS_S(": ");
 869
 870                 for (dq = 0; dq < DQ_NUM; dq++) {
 871                         /* Set skew value for all dq */
 872                         /*
 873                          * Bit# Deskew <- Bit# Deskew - last / first
 874                          * failing bit Deskew For all bits (per PUP)
 875                          * (minimize delay elements)
 876                          */
 877                         DEBUG_PBS_S("DQ");
 878                         DEBUG_PBS_D(dq, 1);
 879                         DEBUG_PBS_S("-");
 880                         DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);
 881                         DEBUG_PBS_S(", ");
 882                 }
 883                 DEBUG_PBS_S("\n");
 884         }
 885
 886         /* Return ADLL to default value */
 887         ddr3_write_pup_reg(PUP_DQS_RD, CS0, PUP_BC, 0, INIT_RL_DELAY);
 888
 889         /* Set averaged PBS results */
 890         ddr3_set_pbs_results(dram_info, 0);
 891
 892         /* Disable SW override - Must be in a different stage */
 893         /* [0]=0 - Enable SW override  */
 894         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 895         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 896         /* 0x15B8 - Training SW 2 Register */
 897         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 898
 899         reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 900                 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 901         reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 902
 903         DEBUG_PBS_FULL_S("DDR3 - PBS RX - ended successfuly\n");
 904
 905         return MV_OK;
 906 }
 907
 908 /*
 909  * Name:     ddr3_rx_shift_dqs_to_first_fail
 910  * Desc:     Execute the Rx shift DQ phase.
 911  * Args:     dram_info           ddr3 training information struct
 912  *           cur_pup             bit array of the function active pups.
 913  *           pbs_pattern_idx     Index of PBS pattern
 914  * Notes:
 915  * Returns:  MV_OK if success, other error code if fail.
 916  */
 917 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup,
 918                                            u32 pbs_pattern_idx, u32 ecc)
 919 {
 920         u32 unlock_pup;         /* bit array of unlock pups  */
 921         u32 new_lockup_pup;     /* bit array of compare failed pups */
 922         u32 adll_val = MAX_DELAY;
 923         u32 dqs_deskew_val = 0; /* current value of DQS PBS deskew */
 924         u32 cur_max_pup, pup, pass_pup;
 925         u32 *pattern_ptr;
 926
 927         /* Choose pattern */
 928         switch (dram_info->ddr_width) {
 929 #if defined(MV88F672X)
 930         case 16:
 931                 pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx];
 932                 break;
 933 #endif
 934         case 32:
 935                 pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx];
 936                 break;
 937 #if defined(MV88F78X60)
 938         case 64:
 939                 pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx];
 940                 break;
 941 #endif
 942         default:
 943                 return MV_FAIL;
 944         }
 945
 946         /* Set current pup number */
 947         if (cur_pup == 0x1)     /* Ecc mode */
 948                 cur_max_pup = 1;
 949         else
 950                 cur_max_pup = dram_info->num_of_std_pups;
 951
 952         unlock_pup = cur_pup;   /* '1' for each unlocked pup */
 953
 954         DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Starting...\n");
 955
 956         /* Set DQS ADLL to MAX */
 957         DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Set DQS ADLL to Max for all PUPs\n");
 958         for (pup = 0; pup < cur_max_pup; pup++)
 959                 ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 0,
 960                                    MAX_DELAY);
 961
 962         /* Loop on all ADLL Vaules */
 963         do {
 964                 /* Loop until found fail for all pups */
 965                 new_lockup_pup = 0;
 966                 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup,
 967                                                 &new_lockup_pup,
 968                                                 pattern_ptr, LEN_PBS_PATTERN,
 969                                                 SDRAM_PBS_I_OFFS +
 970                                                 pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
 971                                                 0, 0, NULL, 0)) {
 972                         DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
 973                         return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
 974                 }
 975
 976                 if ((new_lockup_pup != 0) && (dqs_deskew_val <= 1)) {
 977                         /* Fail on start with first deskew value */
 978                         /* Decrement DQS ADLL */
 979                         --adll_val;
 980                         if (adll_val == ADLL_MIN) {
 981                                 DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - fail on start with first deskew value\n");
 982                                 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
 983                         }
 984                         ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP,
 985                                            0, adll_val);
 986                         continue;
 987                 }
 988
 989                 /* Update all new locked pups */
 990                 unlock_pup &= ~new_lockup_pup;
 991
 992                 if ((unlock_pup == 0) || (dqs_deskew_val == MAX_PBS)) {
 993                         if (dqs_deskew_val == MAX_PBS) {
 994                                 /*
 995                                  * Reach max value of dqs deskew or get fail
 996                                  * for all pups
 997                                  */
 998                                 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - DQS deskew reached maximum value\n");
 999                         }
1000                         break;
1001                 }
1002
1003                 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Inc DQS deskew for PUPs: ");
1004                 DEBUG_PBS_FULL_D(unlock_pup, 2);
1005                 DEBUG_PBS_FULL_C(", deskew = ", dqs_deskew_val, 2);
1006
1007                 /* Increment DQS deskew elements - Only for unlocked pups */
1008                 dqs_deskew_val++;
1009                 for (pup = 0; pup < cur_max_pup; pup++) {
1010                         if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1011                                 ddr3_write_pup_reg(PUP_PBS_RX + DQS_DQ_NUM, CS0,
1012                                                    pup + ecc * ECC_PUP, 0,
1013                                                    dqs_deskew_val);
1014                         }
1015                 }
1016         } while (1);
1017
1018         DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - ADLL shift one step before fail\n");
1019         /* Continue to ADLL shift one step before fail */
1020         unlock_pup = cur_pup;
1021         do {
1022                 /* Loop until pass compare for all pups */
1023                 new_lockup_pup = 0;
1024                 /* Read and compare results  */
1025                 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, &new_lockup_pup,
1026                                                 pattern_ptr, LEN_PBS_PATTERN,
1027                                                 SDRAM_PBS_I_OFFS +
1028                                                 pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS,
1029                                                 1, 0, NULL, 0)) {
1030                         DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n");
1031                         return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP;
1032                 }
1033
1034                 /*
1035                  * Get mask for pup which passed so their adll will be
1036                  * changed to 2 steps before fails
1037                  */
1038                 pass_pup = unlock_pup & ~new_lockup_pup;
1039
1040                 DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: ");
1041                 DEBUG_PBS_FULL_D(pass_pup, 2);
1042                 DEBUG_PBS_FULL_C(", Set ADLL value = ", (adll_val - 2), 2);
1043
1044                 /* Only for pass pups   */
1045                 for (pup = 0; pup < cur_max_pup; pup++) {
1046                         if (IS_PUP_ACTIVE(pass_pup, pup) == 1) {
1047                                 ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1048                                                    pup + ecc * ECC_PUP, 0,
1049                                                    (adll_val - 2));
1050                         }
1051                 }
1052
1053                 /* Locked pups that compare success  */
1054                 unlock_pup &= new_lockup_pup;
1055
1056                 if (unlock_pup == 0) {
1057                         /* All pups locked */
1058                         break;
1059                 }
1060
1061                 /* Found error */
1062                 if (adll_val == 0) {
1063                         DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift DQS - Adll reach min value\n");
1064                         return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_MAX_VAL;
1065                 }
1066
1067                 /*
1068                  * Decrement (Move Back to Left one phase - ADLL) dqs RX delay
1069                  */
1070                 adll_val--;
1071                 for (pup = 0; pup < cur_max_pup; pup++) {
1072                         if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1073                                 ddr3_write_pup_reg(PUP_DQS_RD, CS0,
1074                                                    pup + ecc * ECC_PUP, 0,
1075                                                    adll_val);
1076                         }
1077                 }
1078         } while (1);
1079
1080         return MV_OK;
1081 }
1082
1083 /*
1084  * lock_pups() extracted from ddr3_pbs_per_bit(). This just got too
1085  * much indented making it hard to read / edit.
1086  */
1087 static void lock_pups(u32 pup, u32 *pup_locked, u8 *unlock_pup_dq_array,
1088                       u32 pbs_curr_val, u32 start_pbs, u32 ecc, int is_tx)
1089 {
1090         u32 dq;
1091         int idx;
1092
1093         /* Lock PBS value for all remaining PUPs bits */
1094         DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Lock PBS value for all remaining PUPs bits, pup ");
1095         DEBUG_PBS_FULL_D(pup, 1);
1096         DEBUG_PBS_FULL_C(" pbs value ", pbs_curr_val, 2);
1097
1098         idx = pup * (1 - ecc) + ecc * ECC_PUP;
1099         *pup_locked &= ~(1 << pup);
1100
1101         for (dq = 0; dq < DQ_NUM; dq++) {
1102                 if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) == 1) {
1103                         int offs;
1104
1105                         /* Lock current dq */
1106                         unlock_pup_dq_array[dq] &= ~(1 << pup);
1107                         skew_array[(pup * DQ_NUM) + dq] = pbs_curr_val;
1108
1109                         if (is_tx == 1)
1110                                 offs = PUP_PBS_TX;
1111                         else
1112                                 offs = PUP_PBS_RX;
1113
1114                         ddr3_write_pup_reg(offs +
1115                                            pbs_dq_mapping[idx][dq], CS0,
1116                                            idx, 0, start_pbs);
1117                 }
1118         }
1119 }
1120
1121 /*
1122  * Name:     ddr3_pbs_per_bit
1123  * Desc:     Execute the Per Bit Skew phase.
1124  * Args:     start_over      Return whether need to start over the algorithm
1125  *           is_tx           Indicate whether Rx or Tx
1126  *           pcur_pup        bit array of the function active pups. return the
1127  *                           pups that need to repeat on the PBS
1128  *           pbs_pattern_idx Index of PBS pattern
1129  *
1130  * Notes:    Current implementation supports double activation of this function.
1131  *           i.e. in order to activate this function (using start_over) more than
1132  *           twice, the implementation should change.
1133  *           imlementation limitation are marked using
1134  *           ' CHIP-ONLY! - Implementation Limitation '
1135  * Returns:  MV_OK if success, other error code if fail.
1136  */
1137 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx,
1138                             u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc)
1139 {
1140         /*
1141          * Bit array to indicate if we already get fail on bit per pup & dq bit
1142          */
1143         u8 unlock_pup_dq_array[DQ_NUM] = {
1144                 *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup,
1145                 *pcur_pup, *pcur_pup, *pcur_pup
1146         };
1147
1148         u8 cmp_unlock_pup_dq_array[COUNT_PBS_COMP_RETRY_NUM][DQ_NUM];
1149         u32 pup, dq;
1150         /* value of pbs is according to RX or TX */
1151         u32 start_pbs, last_pbs;
1152         u32 pbs_curr_val;
1153         /* bit array that indicates all dq of the pup locked */
1154         u32 pup_locked;
1155         u32 first_fail[MAX_PUP_NUM] = { 0 };    /* count first fail per pup */
1156         /* indicates whether we get first fail per pup */
1157         int first_failed[MAX_PUP_NUM] = { 0 };
1158         /* bit array that indicates pup already get fail */
1159         u32 sum_pup_fail;
1160         /* use to calculate diff between curr pbs to first fail pbs */
1161         u32 calc_pbs_diff;
1162         u32 pbs_cmp_retry;
1163         u32 max_pup;
1164
1165         /* Set init values for retry array - 8 retry */
1166         for (pbs_cmp_retry = 0; pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1167              pbs_cmp_retry++) {
1168                 for (dq = 0; dq < DQ_NUM; dq++)
1169                         cmp_unlock_pup_dq_array[pbs_cmp_retry][dq] = *pcur_pup;
1170         }
1171
1172         memset(&skew_array, 0, MAX_PUP_NUM * DQ_NUM * sizeof(u32));
1173
1174         DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Started\n");
1175
1176         /* The pbs value depends if rx or tx */
1177         if (is_tx == 1) {
1178                 start_pbs = MIN_PBS;
1179                 last_pbs = MAX_PBS;
1180         } else {
1181                 start_pbs = MAX_PBS;
1182                 last_pbs = MIN_PBS;
1183         }
1184
1185         pbs_curr_val = start_pbs;
1186         pup_locked = *pcur_pup;
1187
1188         /* Set current pup number */
1189         if (pup_locked == 0x1)  /* Ecc mode */
1190                 max_pup = 1;
1191         else
1192                 max_pup = dram_info->num_of_std_pups;
1193
1194         do {
1195                 /* Increment/ decrement PBS for un-lock bits only */
1196                 if (is_tx == 1)
1197                         pbs_curr_val++;
1198                 else
1199                         pbs_curr_val--;
1200
1201                 /* Set Current PBS delay  */
1202                 for (dq = 0; dq < DQ_NUM; dq++) {
1203                         /* Check DQ bits to see if locked in all pups */
1204                         if (unlock_pup_dq_array[dq] == 0) {
1205                                 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All pups are locked for DQ ");
1206                                 DEBUG_PBS_FULL_D(dq, 1);
1207                                 DEBUG_PBS_FULL_S("\n");
1208                                 continue;
1209                         }
1210
1211                         for (pup = 0; pup < max_pup; pup++) {
1212                                 int idx;
1213
1214                                 idx = pup * (1 - ecc) + ecc * ECC_PUP;
1215
1216                                 if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup)
1217                                     == 0)
1218                                         continue;
1219
1220                                 if (is_tx == 1)
1221                                         ddr3_write_pup_reg(
1222                                                 PUP_PBS_TX + pbs_dq_mapping[idx][dq],
1223                                                 CS0, idx, 0, pbs_curr_val);
1224                                 else
1225                                         ddr3_write_pup_reg(
1226                                                 PUP_PBS_RX + pbs_dq_mapping[idx][dq],
1227                                                 CS0, idx, 0, pbs_curr_val);
1228                         }
1229                 }
1230
1231                 /*
1232                  * Write Read and compare results - run the test
1233                  * DDR_PBS_COMP_RETRY_NUM times
1234                  */
1235                 /* Run number of read and write to verify */
1236                 for (pbs_cmp_retry = 0;
1237                      pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM;
1238                      pbs_cmp_retry++) {
1239
1240                         if (MV_OK !=
1241                             ddr3_sdram_pbs_compare(dram_info, pup_locked, is_tx,
1242                                                    pbs_pattern_idx,
1243                                                    pbs_curr_val, start_pbs,
1244                                                    skew_array,
1245                                                    cmp_unlock_pup_dq_array
1246                                                    [pbs_cmp_retry], ecc))
1247                                 return MV_FAIL;
1248
1249                         for (pup = 0; pup < max_pup; pup++) {
1250                                 for (dq = 0; dq < DQ_NUM; dq++) {
1251                                         if ((IS_PUP_ACTIVE(unlock_pup_dq_array[dq],
1252                                                            pup) == 1)
1253                                             && (IS_PUP_ACTIVE(cmp_unlock_pup_dq_array
1254                                               [pbs_cmp_retry][dq],
1255                                               pup) == 0)) {
1256                                                 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PbsCurrVal: ");
1257                                                 DEBUG_PBS_FULL_D(pbs_curr_val, 2);
1258                                                 DEBUG_PBS_FULL_S(" PUP: ");
1259                                                 DEBUG_PBS_FULL_D(pup, 1);
1260                                                 DEBUG_PBS_FULL_S(" DQ: ");
1261                                                 DEBUG_PBS_FULL_D(dq, 1);
1262                                                 DEBUG_PBS_FULL_S(" - failed\n");
1263                                         }
1264                                 }
1265                         }
1266
1267                         for (dq = 0; dq < DQ_NUM; dq++) {
1268                                 unlock_pup_dq_array[dq] &=
1269                                     cmp_unlock_pup_dq_array[pbs_cmp_retry][dq];
1270                         }
1271                 }
1272
1273                 pup_locked = 0;
1274                 sum_pup_fail = *pcur_pup;
1275
1276                 /* Check which DQ is failed */
1277                 for (dq = 0; dq < DQ_NUM; dq++) {
1278                         /* Summarize the locked pup */
1279                         pup_locked |= unlock_pup_dq_array[dq];
1280
1281                         /* Check if get fail */
1282                         sum_pup_fail &= unlock_pup_dq_array[dq];
1283                 }
1284
1285                 /* If all PUPS are locked in all DQ - Break */
1286                 if (pup_locked == 0) {
1287                         /* All pups are locked */
1288                         *start_over = 0;
1289                         DEBUG_PBS_FULL_S("DDR3 - PBS Per bit -  All bit in all pups are successfully locked\n");
1290                         break;
1291                 }
1292
1293                 /* PBS deskew elements reach max ? */
1294                 if (pbs_curr_val == last_pbs) {
1295                         DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PBS deskew elements reach max\n");
1296                         /* CHIP-ONLY! - Implementation Limitation */
1297                         *start_over = (sum_pup_fail != 0) && (!(*start_over));
1298                         *pcur_pup = pup_locked;
1299
1300                         DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - StartOver: ");
1301                         DEBUG_PBS_FULL_D(*start_over, 1);
1302                         DEBUG_PBS_FULL_S("  pup_locked: ");
1303                         DEBUG_PBS_FULL_D(pup_locked, 2);
1304                         DEBUG_PBS_FULL_S("  sum_pup_fail: ");
1305                         DEBUG_PBS_FULL_D(sum_pup_fail, 2);
1306                         DEBUG_PBS_FULL_S("\n");
1307
1308                         /* Lock PBS value for all remaining  bits */
1309                         for (pup = 0; pup < max_pup; pup++) {
1310                                 /* Check if current pup already received error */
1311                                 if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1312                                         /* Valid pup for current function */
1313                                         if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1314                                             1 && (*start_over == 1)) {
1315                                                 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - skipping lock of pup (first loop of pbs)",
1316                                                                  pup, 1);
1317                                                 continue;
1318                                         } else
1319                                             if (IS_PUP_ACTIVE(sum_pup_fail, pup)
1320                                                 == 1) {
1321                                                 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - Locking pup %d (even though it wasn't supposed to be locked)",
1322                                                                  pup, 1);
1323                                         }
1324
1325                                         /* Already got fail on the PUP */
1326                                         /* Lock PBS value for all remaining bits */
1327                                         DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Locking remaning DQs for pup - ");
1328                                         DEBUG_PBS_FULL_D(pup, 1);
1329                                         DEBUG_PBS_FULL_S(": ");
1330
1331                                         for (dq = 0; dq < DQ_NUM; dq++) {
1332                                                 if (IS_PUP_ACTIVE
1333                                                     (unlock_pup_dq_array[dq],
1334                                                      pup) == 1) {
1335                                                         DEBUG_PBS_FULL_D(dq, 1);
1336                                                         DEBUG_PBS_FULL_S(",");
1337                                                         /* set current PBS */
1338                                                         skew_array[((pup) *
1339                                                                     DQ_NUM) +
1340                                                                    dq] =
1341                                                             pbs_curr_val;
1342                                                 }
1343                                         }
1344
1345                                         if (*start_over == 1) {
1346                                                 /*
1347                                                  * Reset this pup bit - when
1348                                                  * restart the PBS, ignore this
1349                                                  * pup
1350                                                  */
1351                                                 *pcur_pup &= ~(1 << pup);
1352                                         }
1353                                         DEBUG_PBS_FULL_S("\n");
1354                                 } else {
1355                                         DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Pup ");
1356                                         DEBUG_PBS_FULL_D(pup, 1);
1357                                         DEBUG_PBS_FULL_C(" is not set in puplocked - ",
1358                                                          pup_locked, 1);
1359                                 }
1360                         }
1361
1362                         /* Need to start the PBS again */
1363                         if (*start_over == 1) {
1364                                 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - false fail - returning to start\n");
1365                                 return MV_OK;
1366                         }
1367                         break;
1368                 }
1369
1370                 /* Diff Check */
1371                 for (pup = 0; pup < max_pup; pup++) {
1372                         if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {
1373                                 /* pup is not locked */
1374                                 if (first_failed[pup] == 0) {
1375                                         /* No first fail until now */
1376                                         if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==
1377                                             0) {
1378                                                 /* Get first fail */
1379                                                 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - First fail in pup ",
1380                                                                  pup, 1);
1381                                                 first_failed[pup] = 1;
1382                                                 first_fail[pup] = pbs_curr_val;
1383                                         }
1384                                 } else {
1385                                         /* Already got first fail */
1386                                         if (is_tx == 1) {
1387                                                 /* TX - inc pbs */
1388                                                 calc_pbs_diff = pbs_curr_val -
1389                                                         first_fail[pup];
1390                                         } else {
1391                                                 /* RX - dec pbs */
1392                                                 calc_pbs_diff = first_fail[pup] -
1393                                                         pbs_curr_val;
1394                                         }
1395
1396                                         if (calc_pbs_diff >= PBS_DIFF_LIMIT) {
1397                                                 lock_pups(pup, &pup_locked,
1398                                                           unlock_pup_dq_array,
1399                                                           pbs_curr_val,
1400                                                           start_pbs, ecc, is_tx);
1401                                         }
1402                                 }
1403                         }
1404                 }
1405         } while (1);
1406
1407         return MV_OK;
1408 }
1409
1410 /*
1411  * Name:         ddr3_set_pbs_results
1412  * Desc:         Set to HW the PBS phase results.
1413  * Args:         is_tx       Indicates whether to set Tx or RX results
1414  * Notes:
1415  * Returns:      MV_OK if success, other error code if fail.
1416  */
1417 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx)
1418 {
1419         u32 pup, phys_pup, dq;
1420         u32 max_pup;            /* number of valid pups */
1421         u32 pbs_min;            /* minimal pbs val per pup */
1422         u32 pbs_max;            /* maximum pbs val per pup */
1423         u32 val[9];
1424
1425         max_pup = dram_info->num_of_total_pups;
1426         DEBUG_PBS_FULL_S("DDR3 - PBS - ddr3_set_pbs_results:\n");
1427
1428         /* Loop for all dqs & pups */
1429         for (pup = 0; pup < max_pup; pup++) {
1430                 if (pup == (max_pup - 1) && dram_info->ecc_ena)
1431                         phys_pup = ECC_PUP;
1432                 else
1433                         phys_pup = pup;
1434
1435                 /*
1436                  * To minimize delay elements, inc from pbs value the min
1437                  * pbs val
1438                  */
1439                 pbs_min = MAX_PBS;
1440                 pbs_max = 0;
1441                 for (dq = 0; dq < DQ_NUM; dq++) {
1442                         if (pbs_min > skew_array[(pup * DQ_NUM) + dq])
1443                                 pbs_min = skew_array[(pup * DQ_NUM) + dq];
1444
1445                         if (pbs_max < skew_array[(pup * DQ_NUM) + dq])
1446                                 pbs_max = skew_array[(pup * DQ_NUM) + dq];
1447                 }
1448
1449                 pbs_max -= pbs_min;
1450
1451                 DEBUG_PBS_FULL_S("DDR3 - PBS - PUP");
1452                 DEBUG_PBS_FULL_D(phys_pup, 1);
1453                 DEBUG_PBS_FULL_S(": Min Val = ");
1454                 DEBUG_PBS_FULL_D(pbs_min, 2);
1455                 DEBUG_PBS_FULL_C(", Max Val = ", pbs_max, 2);
1456
1457                 val[pup] = 0;
1458
1459                 for (dq = 0; dq < DQ_NUM; dq++) {
1460                         int idx;
1461                         int offs;
1462
1463                         /* Set skew value for all dq */
1464                         /*
1465                          * Bit# Deskew <- Bit# Deskew - last / first
1466                          * failing bit Deskew For all bits (per PUP)
1467                          * (minimize delay elements)
1468                          */
1469
1470                         DEBUG_PBS_FULL_S("DQ");
1471                         DEBUG_PBS_FULL_D(dq, 1);
1472                         DEBUG_PBS_FULL_S("-");
1473                         DEBUG_PBS_FULL_D((skew_array[(pup * DQ_NUM) + dq] -
1474                                           pbs_min), 2);
1475                         DEBUG_PBS_FULL_S(", ");
1476
1477                         idx = (pup * DQ_NUM) + dq;
1478
1479                         if (is_tx == 1)
1480                                 offs = PUP_PBS_TX;
1481                         else
1482                                 offs = PUP_PBS_RX;
1483
1484                         ddr3_write_pup_reg(offs + pbs_dq_mapping[phys_pup][dq],
1485                                            CS0, phys_pup, 0,
1486                                            skew_array[idx] - pbs_min);
1487
1488                         if (is_tx == 1)
1489                                 val[pup] += skew_array[idx] - pbs_min;
1490                 }
1491
1492                 DEBUG_PBS_FULL_S("\n");
1493
1494                 /* Set the DQS the half of the Max PBS of the DQs  */
1495                 if (is_tx == 1) {
1496                         ddr3_write_pup_reg(PUP_PBS_TX + 8, CS0, phys_pup, 0,
1497                                            pbs_max / 2);
1498                         ddr3_write_pup_reg(PUP_PBS_TX + 0xa, CS0, phys_pup, 0,
1499                                            val[pup] / 8);
1500                 } else
1501                         ddr3_write_pup_reg(PUP_PBS_RX + 8, CS0, phys_pup, 0,
1502                                            pbs_max / 2);
1503         }
1504
1505         return MV_OK;
1506 }
1507
1508 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay)
1509 {
1510         u32 reg, delay;
1511
1512         reg = (ddr3_read_pup_reg(PUP_WL_MODE, cs, pup) & 0x3FF);
1513         delay = reg & PUP_DELAY_MASK;
1514         reg |= ((dqs_delay + delay) << REG_PHY_DQS_REF_DLY_OFFS);
1515         reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1516         reg |= (pup << REG_PHY_PUP_OFFS);
1517         reg |= ((0x4 * cs + PUP_WL_MODE) << REG_PHY_CS_OFFS);
1518
1519         reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1520         do {
1521                 reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) &
1522                         REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1523         } while (reg);  /* Wait for '0' to mark the end of the transaction */
1524
1525         udelay(10);
1526 }
1527
1528 /*
1529  * Set training patterns
1530  */
1531 int ddr3_load_pbs_patterns(MV_DRAM_INFO *dram_info)
1532 {
1533         u32 cs, cs_count, cs_tmp;
1534         u32 sdram_addr;
1535         u32 *pattern_ptr0, *pattern_ptr1;
1536
1537         /* Choose pattern */
1538         switch (dram_info->ddr_width) {
1539 #if defined(MV88F672X)
1540         case 16:
1541                 pattern_ptr0 = (u32 *)&pbs_pattern[0];
1542                 pattern_ptr1 = (u32 *)&pbs_pattern[1];
1543                 break;
1544 #endif
1545         case 32:
1546                 pattern_ptr0 = (u32 *)&pbs_pattern_32b[0];
1547                 pattern_ptr1 = (u32 *)&pbs_pattern_32b[1];
1548                 break;
1549 #if defined(MV88F78X60)
1550         case 64:
1551                 pattern_ptr0 = (u32 *)&pbs_pattern_64b[0];
1552                 pattern_ptr1 = (u32 *)&pbs_pattern_64b[1];
1553                 break;
1554 #endif
1555         default:
1556                 return MV_FAIL;
1557         }
1558
1559         /* Loop for each CS */
1560         for (cs = 0; cs < MAX_CS; cs++) {
1561                 if (dram_info->cs_ena & (1 << cs)) {
1562                         cs_count = 0;
1563                         for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1564                                 if (dram_info->cs_ena & (1 << cs_tmp))
1565                                         cs_count++;
1566                         }
1567
1568                         /* Init PBS I pattern */
1569                         sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1570                                       SDRAM_PBS_I_OFFS);
1571                         if (MV_OK !=
1572                             ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1573                                                pattern_ptr0, LEN_STD_PATTERN,
1574                                                sdram_addr, 1, 0, NULL,
1575                                                0))
1576                                 return MV_FAIL;
1577
1578                         /* Init PBS II pattern */
1579                         sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1580                                       SDRAM_PBS_II_OFFS);
1581                         if (MV_OK !=
1582                             ddr3_sdram_compare(dram_info, (u32) NULL, NULL,
1583                                                pattern_ptr1, LEN_STD_PATTERN,
1584                                                sdram_addr, 1, 0, NULL,
1585                                                0))
1586                                 return MV_FAIL;
1587                 }
1588         }
1589
1590         return MV_OK;
1591 }
1592 #endif