drivers/ddr/marvell/a38x/ddr3_training_leveling.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Marvell International Ltd. and its affiliates
   4  */
   5
   6 #include "ddr3_init.h"
   7 #include "mv_ddr_training_db.h"
   8 #include "ddr_training_ip_db.h"
   9 #include "mv_ddr_regs.h"
  10 #include <linux/delay.h>
  11
  12 #define WL_ITERATION_NUM        10
  13
  14 static u32 pup_mask_table[] = {
  15         0x000000ff,
  16         0x0000ff00,
  17         0x00ff0000,
  18         0xff000000
  19 };
  20
  21 static struct write_supp_result wr_supp_res[MAX_INTERFACE_NUM][MAX_BUS_NUM];
  22
  23 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num);
  24 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num);
  25 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num);
  26 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
  27                                               u32 bus_id);
  28 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
  29                                      u32 edge_offset);
  30
  31 enum {
  32         PASS,
  33         FAIL
  34 };
  35 /*****************************************************************************
  36 Dynamic read leveling
  37 ******************************************************************************/
  38 int ddr3_tip_dynamic_read_leveling(u32 dev_num, u32 freq)
  39 {
  40         u32 data, mask;
  41         unsigned int max_cs = mv_ddr_cs_num_get();
  42         u32 bus_num, if_id, cl_val;
  43         enum mv_ddr_speed_bin speed_bin_index;
  44         /* save current CS value */
  45         u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
  46         int is_any_pup_fail = 0;
  47         u32 data_read[MAX_INTERFACE_NUM + 1] = { 0 };
  48         u8 rl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM];
  49         struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
  50         u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
  51         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
  52         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
  53
  54         for (effective_cs = 0; effective_cs < MAX_CS_NUM; effective_cs++)
  55                 for (bus_num = 0; bus_num < MAX_BUS_NUM; bus_num++)
  56                         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++)
  57                                 rl_values[effective_cs][bus_num][if_id] = 0;
  58
  59         for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
  60                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
  61                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
  62                         training_result[training_stage][if_id] = TEST_SUCCESS;
  63
  64                         /* save current cs enable reg val */
  65                         CHECK_STATUS(ddr3_tip_if_read
  66                                      (dev_num, ACCESS_TYPE_UNICAST, if_id,
  67                                       DUAL_DUNIT_CFG_REG, cs_enable_reg_val,
  68                                       MASK_ALL_BITS));
  69                         /* enable single cs */
  70                         CHECK_STATUS(ddr3_tip_if_write
  71                                      (dev_num, ACCESS_TYPE_UNICAST, if_id,
  72                                       DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
  73                 }
  74
  75                 ddr3_tip_reset_fifo_ptr(dev_num);
  76
  77                 /*
  78                  *     Phase 1: Load pattern (using ODPG)
  79                  *
  80                  * enter Read Leveling mode
  81                  * only 27 bits are masked
  82                  * assuming non multi-CS configuration
  83                  * write to CS = 0 for the non multi CS configuration, note
  84                  * that the results shall be read back to the required CS !!!
  85                  */
  86
  87                 /* BUS count is 0 shifted 26 */
  88                 CHECK_STATUS(ddr3_tip_if_write
  89                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
  90                               ODPG_DATA_CTRL_REG, 0x3, 0x3));
  91                 CHECK_STATUS(ddr3_tip_configure_odpg
  92                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
  93                               pattern_table[PATTERN_RL].num_of_phases_tx, 0,
  94                               pattern_table[PATTERN_RL].num_of_phases_rx, 0, 0,
  95                               effective_cs, STRESS_NONE, DURATION_SINGLE));
  96
  97                 /* load pattern to ODPG */
  98                 ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
  99                                               PARAM_NOT_CARE, PATTERN_RL,
 100                                               pattern_table[PATTERN_RL].
 101                                               start_addr);
 102
 103                 /*
 104                  *     Phase 2: ODPG to Read Leveling mode
 105                  */
 106
 107                 /* General Training Opcode register */
 108                 CHECK_STATUS(ddr3_tip_if_write
 109                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 110                               ODPG_WR_RD_MODE_ENA_REG, 0,
 111                               MASK_ALL_BITS));
 112
 113                 CHECK_STATUS(ddr3_tip_if_write
 114                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 115                               GENERAL_TRAINING_OPCODE_REG,
 116                               (0x301b01 | effective_cs << 2), 0x3c3fef));
 117
 118                 /* Object1 opcode register 0 & 1 */
 119                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 120                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 121                         speed_bin_index =
 122                                 tm->interface_params[if_id].speed_bin_index;
 123                         cl_val = mv_ddr_cl_val_get(speed_bin_index, freq);
 124                         data = (cl_val << 17) | (0x3 << 25);
 125                         mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
 126                         CHECK_STATUS(ddr3_tip_if_write
 127                                      (dev_num, ACCESS_TYPE_UNICAST, if_id,
 128                                       OPCODE_REG0_REG(1), data, mask));
 129                 }
 130
 131                 /* Set iteration count to max value */
 132                 CHECK_STATUS(ddr3_tip_if_write
 133                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 134                               OPCODE_REG1_REG(1), 0xd00, 0xd00));
 135
 136                 /*
 137                  *     Phase 2: Mask config
 138                  */
 139
 140                 ddr3_tip_dynamic_read_leveling_seq(dev_num);
 141
 142                 /*
 143                  *     Phase 3: Read Leveling execution
 144                  */
 145
 146                 /* temporary jira dunit=14751 */
 147                 CHECK_STATUS(ddr3_tip_if_write
 148                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 149                               TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
 150                 /* configure phy reset value */
 151                 CHECK_STATUS(ddr3_tip_if_write
 152                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 153                               TRAINING_DBG_3_REG, (0x7f << 24),
 154                               (u32)(0xff << 24)));
 155                 /* data pup rd reset enable  */
 156                 CHECK_STATUS(ddr3_tip_if_write
 157                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 158                               SDRAM_CFG_REG, 0, (1 << 30)));
 159                 /* data pup rd reset disable */
 160                 CHECK_STATUS(ddr3_tip_if_write
 161                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 162                               SDRAM_CFG_REG, (1 << 30), (1 << 30)));
 163                 /* training SW override & training RL mode */
 164                 CHECK_STATUS(ddr3_tip_if_write
 165                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 166                               TRAINING_SW_2_REG, 0x1, 0x9));
 167                 /* training enable */
 168                 CHECK_STATUS(ddr3_tip_if_write
 169                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 170                               TRAINING_REG, (1 << 24) | (1 << 20),
 171                               (1 << 24) | (1 << 20)));
 172                 CHECK_STATUS(ddr3_tip_if_write
 173                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 174                               TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
 175
 176                 /* trigger training */
 177                 mv_ddr_training_enable();
 178
 179                 /* check for training done */
 180                 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
 181                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
 182                         return MV_FAIL;
 183                 }
 184                 /* check for training pass */
 185                 if (data != PASS)
 186                         DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
 187
 188                 /* disable odpg; switch back to functional mode */
 189                 mv_ddr_odpg_disable();
 190
 191                 if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
 192                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
 193                         return MV_FAIL;
 194                 }
 195
 196                 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 197                                   ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
 198
 199                 /* double loop on bus, pup */
 200                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 201                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 202                         /* check training done */
 203                         is_any_pup_fail = 0;
 204                         for (bus_num = 0;
 205                              bus_num < octets_per_if_num;
 206                              bus_num++) {
 207                                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
 208                                 if (ddr3_tip_if_polling
 209                                     (dev_num, ACCESS_TYPE_UNICAST,
 210                                      if_id, (1 << 25), (1 << 25),
 211                                      mask_results_pup_reg_map[bus_num],
 212                                      MAX_POLLING_ITERATIONS) != MV_OK) {
 213                                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
 214                                                        ("\n_r_l: DDR3 poll failed(2) for IF %d CS %d bus %d",
 215                                                         if_id, effective_cs, bus_num));
 216                                         is_any_pup_fail = 1;
 217                                 } else {
 218                                         /* read result per pup */
 219                                         CHECK_STATUS(ddr3_tip_if_read
 220                                                      (dev_num,
 221                                                       ACCESS_TYPE_UNICAST,
 222                                                       if_id,
 223                                                       mask_results_pup_reg_map
 224                                                       [bus_num], data_read,
 225                                                       0xff));
 226                                         rl_values[effective_cs][bus_num]
 227                                                 [if_id] = (u8)data_read[if_id];
 228                                 }
 229                         }
 230
 231                         if (is_any_pup_fail == 1) {
 232                                 training_result[training_stage][if_id] =
 233                                         TEST_FAILED;
 234                                 if (debug_mode == 0)
 235                                         return MV_FAIL;
 236                         }
 237                 }
 238
 239                 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
 240
 241                 /*
 242                  *     Phase 3: Exit Read Leveling
 243                  */
 244
 245                 CHECK_STATUS(ddr3_tip_if_write
 246                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 247                               TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
 248                 CHECK_STATUS(ddr3_tip_if_write
 249                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 250                               TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
 251                 /* set ODPG to functional */
 252                 CHECK_STATUS(ddr3_tip_if_write
 253                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 254                               ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
 255
 256                 /*
 257                  * Copy the result from the effective CS search to the
 258                  * real Functional CS
 259                  */
 260                 /*ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0); */
 261                 CHECK_STATUS(ddr3_tip_if_write
 262                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 263                               ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
 264         }
 265
 266         for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
 267                 /* double loop on bus, pup */
 268                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 269                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 270                         for (bus_num = 0;
 271                              bus_num < octets_per_if_num;
 272                              bus_num++) {
 273                                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
 274                                 /* read result per pup from arry */
 275                                 data = rl_values[effective_cs][bus_num][if_id];
 276                                 data = (data & 0x1f) |
 277                                         (((data & 0xe0) >> 5) << 6);
 278                                 ddr3_tip_bus_write(dev_num,
 279                                                    ACCESS_TYPE_UNICAST,
 280                                                    if_id,
 281                                                    ACCESS_TYPE_UNICAST,
 282                                                    bus_num, DDR_PHY_DATA,
 283                                                    RL_PHY_REG(effective_cs),
 284                                                    data);
 285                         }
 286                 }
 287         }
 288         /* Set to 0 after each loop to avoid illegal value may be used */
 289         effective_cs = 0;
 290
 291         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 292                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 293                 /* restore cs enable value */
 294                 CHECK_STATUS(ddr3_tip_if_write
 295                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 296                               DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
 297                               MASK_ALL_BITS));
 298                 if (odt_config != 0) {
 299                         CHECK_STATUS(ddr3_tip_write_additional_odt_setting
 300                                      (dev_num, if_id));
 301                 }
 302         }
 303
 304         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 305                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 306                 if (training_result[training_stage][if_id] == TEST_FAILED)
 307                         return MV_FAIL;
 308         }
 309
 310         return MV_OK;
 311 }
 312
 313 /*
 314  * Legacy Dynamic write leveling
 315  */
 316 int ddr3_tip_legacy_dynamic_write_leveling(u32 dev_num)
 317 {
 318         u32 c_cs, if_id, cs_mask = 0;
 319         unsigned int max_cs = mv_ddr_cs_num_get();
 320         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
 321
 322         /*
 323          * In TRAINIUNG reg (0x15b0) write 0x80000008 | cs_mask:
 324          * Trn_start
 325          * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
 326          * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
 327          * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
 328          * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
 329          * Trn_auto_seq =  write leveling
 330          */
 331         for (c_cs = 0; c_cs < max_cs; c_cs++)
 332                 cs_mask = cs_mask | 1 << (20 + c_cs);
 333
 334         for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
 335                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 336                 CHECK_STATUS(ddr3_tip_if_write
 337                              (dev_num, ACCESS_TYPE_MULTICAST, 0,
 338                               TRAINING_REG, (0x80000008 | cs_mask),
 339                               0xffffffff));
 340                 mdelay(20);
 341                 if (ddr3_tip_if_polling
 342                     (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
 343                      (u32)0x80000000, TRAINING_REG,
 344                      MAX_POLLING_ITERATIONS) != MV_OK) {
 345                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
 346                                        ("polling failed for Old WL result\n"));
 347                         return MV_FAIL;
 348                 }
 349         }
 350
 351         return MV_OK;
 352 }
 353
 354 /*
 355  * Legacy Dynamic read leveling
 356  */
 357 int ddr3_tip_legacy_dynamic_read_leveling(u32 dev_num)
 358 {
 359         u32 c_cs, if_id, cs_mask = 0;
 360         unsigned int max_cs = mv_ddr_cs_num_get();
 361         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
 362
 363         /*
 364          * In TRAINIUNG reg (0x15b0) write 0x80000040 | cs_mask:
 365          * Trn_start
 366          * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
 367          * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
 368          * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
 369          * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
 370          * Trn_auto_seq =  Read Leveling using training pattern
 371          */
 372         for (c_cs = 0; c_cs < max_cs; c_cs++)
 373                 cs_mask = cs_mask | 1 << (20 + c_cs);
 374
 375         CHECK_STATUS(ddr3_tip_if_write
 376                      (dev_num, ACCESS_TYPE_MULTICAST, 0, TRAINING_REG,
 377                       (0x80000040 | cs_mask), 0xffffffff));
 378         mdelay(100);
 379
 380         for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
 381                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 382                 if (ddr3_tip_if_polling
 383                     (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
 384                      (u32)0x80000000, TRAINING_REG,
 385                      MAX_POLLING_ITERATIONS) != MV_OK) {
 386                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
 387                                        ("polling failed for Old RL result\n"));
 388                         return MV_FAIL;
 389                 }
 390         }
 391
 392         return MV_OK;
 393 }
 394
 395 /*
 396  * Dynamic per bit read leveling
 397  */
 398 int ddr3_tip_dynamic_per_bit_read_leveling(u32 dev_num, u32 freq)
 399 {
 400         u32 data, mask;
 401         u32 bus_num, if_id, cl_val, bit_num;
 402         u32 curr_numb, curr_min_delay;
 403         int adll_array[3] = { 0, -0xa, 0x14 };
 404         u32 phyreg3_arr[MAX_INTERFACE_NUM][MAX_BUS_NUM];
 405         enum mv_ddr_speed_bin speed_bin_index;
 406         int is_any_pup_fail = 0;
 407         int break_loop = 0;
 408         u32 cs_enable_reg_val[MAX_INTERFACE_NUM]; /* save current CS value */
 409         u32 data_read[MAX_INTERFACE_NUM];
 410         int per_bit_rl_pup_status[MAX_INTERFACE_NUM][MAX_BUS_NUM];
 411         u32 data2_write[MAX_INTERFACE_NUM][MAX_BUS_NUM];
 412         struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
 413         u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
 414         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
 415         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
 416
 417         for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
 418                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 419                 for (bus_num = 0;
 420                      bus_num <= octets_per_if_num; bus_num++) {
 421                         VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
 422                         per_bit_rl_pup_status[if_id][bus_num] = 0;
 423                         data2_write[if_id][bus_num] = 0;
 424                         /* read current value of phy register 0x3 */
 425                         CHECK_STATUS(ddr3_tip_bus_read
 426                                      (dev_num, if_id, ACCESS_TYPE_UNICAST,
 427                                       bus_num, DDR_PHY_DATA,
 428                                       CRX_PHY_REG(0),
 429                                       &phyreg3_arr[if_id][bus_num]));
 430                 }
 431         }
 432
 433         /* NEW RL machine */
 434         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 435                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 436                 training_result[training_stage][if_id] = TEST_SUCCESS;
 437
 438                 /* save current cs enable reg val */
 439                 CHECK_STATUS(ddr3_tip_if_read
 440                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 441                               DUAL_DUNIT_CFG_REG, &cs_enable_reg_val[if_id],
 442                               MASK_ALL_BITS));
 443                 /* enable single cs */
 444                 CHECK_STATUS(ddr3_tip_if_write
 445                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 446                               DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
 447         }
 448
 449         ddr3_tip_reset_fifo_ptr(dev_num);
 450         for (curr_numb = 0; curr_numb < 3; curr_numb++) {
 451                 /*
 452                  *     Phase 1: Load pattern (using ODPG)
 453                  *
 454                  * enter Read Leveling mode
 455                  * only 27 bits are masked
 456                  * assuming non multi-CS configuration
 457                  * write to CS = 0 for the non multi CS configuration, note that
 458                  * the results shall be read back to the required CS !!!
 459                  */
 460
 461                 /* BUS count is 0 shifted 26 */
 462                 CHECK_STATUS(ddr3_tip_if_write
 463                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 464                               ODPG_DATA_CTRL_REG, 0x3, 0x3));
 465                 CHECK_STATUS(ddr3_tip_configure_odpg
 466                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
 467                               pattern_table[PATTERN_TEST].num_of_phases_tx, 0,
 468                               pattern_table[PATTERN_TEST].num_of_phases_rx, 0,
 469                               0, 0, STRESS_NONE, DURATION_SINGLE));
 470
 471                 /* load pattern to ODPG */
 472                 ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
 473                                               PARAM_NOT_CARE, PATTERN_TEST,
 474                                               pattern_table[PATTERN_TEST].
 475                                               start_addr);
 476
 477                 /*
 478                  *     Phase 2: ODPG to Read Leveling mode
 479                  */
 480
 481                 /* General Training Opcode register */
 482                 CHECK_STATUS(ddr3_tip_if_write
 483                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 484                               ODPG_WR_RD_MODE_ENA_REG, 0,
 485                               MASK_ALL_BITS));
 486                 CHECK_STATUS(ddr3_tip_if_write
 487                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 488                               GENERAL_TRAINING_OPCODE_REG, 0x301b01, 0x3c3fef));
 489
 490                 /* Object1 opcode register 0 & 1 */
 491                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 492                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 493                         speed_bin_index =
 494                                 tm->interface_params[if_id].speed_bin_index;
 495                         cl_val = mv_ddr_cl_val_get(speed_bin_index, freq);
 496                         data = (cl_val << 17) | (0x3 << 25);
 497                         mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
 498                         CHECK_STATUS(ddr3_tip_if_write
 499                                      (dev_num, ACCESS_TYPE_UNICAST, if_id,
 500                                       OPCODE_REG0_REG(1), data, mask));
 501                 }
 502
 503                 /* Set iteration count to max value */
 504                 CHECK_STATUS(ddr3_tip_if_write
 505                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 506                               OPCODE_REG1_REG(1), 0xd00, 0xd00));
 507
 508                 /*
 509                  *     Phase 2: Mask config
 510                  */
 511
 512                 ddr3_tip_dynamic_per_bit_read_leveling_seq(dev_num);
 513
 514                 /*
 515                  *     Phase 3: Read Leveling execution
 516                  */
 517
 518                 /* temporary jira dunit=14751 */
 519                 CHECK_STATUS(ddr3_tip_if_write
 520                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 521                               TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
 522                 /* configure phy reset value */
 523                 CHECK_STATUS(ddr3_tip_if_write
 524                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 525                               TRAINING_DBG_3_REG, (0x7f << 24),
 526                               (u32)(0xff << 24)));
 527                 /* data pup rd reset enable  */
 528                 CHECK_STATUS(ddr3_tip_if_write
 529                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 530                               SDRAM_CFG_REG, 0, (1 << 30)));
 531                 /* data pup rd reset disable */
 532                 CHECK_STATUS(ddr3_tip_if_write
 533                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 534                               SDRAM_CFG_REG, (1 << 30), (1 << 30)));
 535                 /* training SW override & training RL mode */
 536                 CHECK_STATUS(ddr3_tip_if_write
 537                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 538                               TRAINING_SW_2_REG, 0x1, 0x9));
 539                 /* training enable */
 540                 CHECK_STATUS(ddr3_tip_if_write
 541                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 542                               TRAINING_REG, (1 << 24) | (1 << 20),
 543                               (1 << 24) | (1 << 20)));
 544                 CHECK_STATUS(ddr3_tip_if_write
 545                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 546                               TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
 547
 548                 /* trigger training */
 549                 mv_ddr_training_enable();
 550
 551                 /* check for training done */
 552                 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
 553                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
 554                         return MV_FAIL;
 555                 }
 556                 /* check for training pass */
 557                 if (data != PASS)
 558                         DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
 559
 560                 /* disable odpg; switch back to functional mode */
 561                 mv_ddr_odpg_disable();
 562
 563                 if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
 564                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
 565                         return MV_FAIL;
 566                 }
 567
 568                 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 569                                   ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
 570
 571                 /* double loop on bus, pup */
 572                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 573                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 574                         /* check training done */
 575                         for (bus_num = 0;
 576                              bus_num < octets_per_if_num;
 577                              bus_num++) {
 578                                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
 579
 580                                 if (per_bit_rl_pup_status[if_id][bus_num]
 581                                     == 0) {
 582                                         curr_min_delay = 0;
 583                                         for (bit_num = 0; bit_num < 8;
 584                                              bit_num++) {
 585                                                 if (ddr3_tip_if_polling
 586                                                     (dev_num,
 587                                                      ACCESS_TYPE_UNICAST,
 588                                                      if_id, (1 << 25),
 589                                                      (1 << 25),
 590                                                      mask_results_dq_reg_map
 591                                                      [bus_num * 8 + bit_num],
 592                                                      MAX_POLLING_ITERATIONS) !=
 593                                                     MV_OK) {
 594                                                         DEBUG_LEVELING
 595                                                                 (DEBUG_LEVEL_ERROR,
 596                                                                  ("\n_r_l: DDR3 poll failed(2) for bus %d bit %d\n",
 597                                                                   bus_num,
 598                                                                   bit_num));
 599                                                 } else {
 600                                                         /* read result per pup */
 601                                                         CHECK_STATUS
 602                                                                 (ddr3_tip_if_read
 603                                                                  (dev_num,
 604                                                                   ACCESS_TYPE_UNICAST,
 605                                                                   if_id,
 606                                                                   mask_results_dq_reg_map
 607                                                                   [bus_num * 8 +
 608                                                                    bit_num],
 609                                                                   data_read,
 610                                                                   MASK_ALL_BITS));
 611                                                         data =
 612                                                                 (data_read
 613                                                                  [if_id] &
 614                                                                  0x1f) |
 615                                                                 ((data_read
 616                                                                   [if_id] &
 617                                                                   0xe0) << 1);
 618                                                         if (curr_min_delay == 0)
 619                                                                 curr_min_delay =
 620                                                                         data;
 621                                                         else if (data <
 622                                                                  curr_min_delay)
 623                                                                 curr_min_delay =
 624                                                                         data;
 625                                                         if (data > data2_write[if_id][bus_num])
 626                                                                 data2_write
 627                                                                         [if_id]
 628                                                                         [bus_num] =
 629                                                                         data;
 630                                                 }
 631                                         }
 632
 633                                         if (data2_write[if_id][bus_num] <=
 634                                             (curr_min_delay +
 635                                              MAX_DQ_READ_LEVELING_DELAY)) {
 636                                                 per_bit_rl_pup_status[if_id]
 637                                                         [bus_num] = 1;
 638                                         }
 639                                 }
 640                         }
 641                 }
 642
 643                 /* check if there is need to search new phyreg3 value */
 644                 if (curr_numb < 2) {
 645                         /* if there is DLL that is not checked yet */
 646                         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
 647                              if_id++) {
 648                                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 649                                 for (bus_num = 0;
 650                                      bus_num < octets_per_if_num;
 651                                      bus_num++) {
 652                                         VALIDATE_BUS_ACTIVE(tm->bus_act_mask,
 653                                                         bus_num);
 654                                         if (per_bit_rl_pup_status[if_id]
 655                                             [bus_num] != 1) {
 656                                                 /* go to next ADLL value */
 657                                                 CHECK_STATUS
 658                                                         (ddr3_tip_bus_write
 659                                                          (dev_num,
 660                                                           ACCESS_TYPE_UNICAST,
 661                                                           if_id,
 662                                                           ACCESS_TYPE_UNICAST,
 663                                                           bus_num, DDR_PHY_DATA,
 664                                                           CRX_PHY_REG(0),
 665                                                           (phyreg3_arr[if_id]
 666                                                            [bus_num] +
 667                                                            adll_array[curr_numb])));
 668                                                 break_loop = 1;
 669                                                 break;
 670                                         }
 671                                 }
 672                                 if (break_loop)
 673                                         break;
 674                         }
 675                 }               /* if (curr_numb < 2) */
 676                 if (!break_loop)
 677                         break;
 678         }               /* for ( curr_numb = 0; curr_numb <3; curr_numb++) */
 679
 680         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 681                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 682                 for (bus_num = 0; bus_num < octets_per_if_num;
 683                      bus_num++) {
 684                         VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
 685                         if (per_bit_rl_pup_status[if_id][bus_num] == 1)
 686                                 ddr3_tip_bus_write(dev_num,
 687                                                    ACCESS_TYPE_UNICAST,
 688                                                    if_id,
 689                                                    ACCESS_TYPE_UNICAST,
 690                                                    bus_num, DDR_PHY_DATA,
 691                                                    RL_PHY_REG(effective_cs),
 692                                                    data2_write[if_id]
 693                                                    [bus_num]);
 694                         else
 695                                 is_any_pup_fail = 1;
 696                 }
 697
 698                 /* TBD flow does not support multi CS */
 699                 /*
 700                  * cs_bitmask = tm->interface_params[if_id].
 701                  * as_bus_params[bus_num].cs_bitmask;
 702                  */
 703                 /* divide by 4 is used for retrieving the CS number */
 704                 /*
 705                  * TBD BC2 - what is the PHY address for other
 706                  * CS ddr3_tip_write_cs_result() ???
 707                  */
 708                 /*
 709                  * find what should be written to PHY
 710                  * - max delay that is less than threshold
 711                  */
 712                 if (is_any_pup_fail == 1) {
 713                         training_result[training_stage][if_id] = TEST_FAILED;
 714                         if (debug_mode == 0)
 715                                 return MV_FAIL;
 716                 }
 717         }
 718         DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
 719
 720         /*
 721          *     Phase 3: Exit Read Leveling
 722          */
 723
 724         CHECK_STATUS(ddr3_tip_if_write
 725                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 726                       TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
 727         CHECK_STATUS(ddr3_tip_if_write
 728                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 729                       TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
 730         /* set ODPG to functional */
 731         CHECK_STATUS(ddr3_tip_if_write
 732                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 733                       ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
 734         /*
 735          * Copy the result from the effective CS search to the real
 736          * Functional CS
 737          */
 738         ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0));
 739         CHECK_STATUS(ddr3_tip_if_write
 740                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 741                       ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
 742
 743         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 744                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 745                 /* restore cs enable value */
 746                 CHECK_STATUS(ddr3_tip_if_write
 747                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 748                               DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
 749                               MASK_ALL_BITS));
 750                 if (odt_config != 0) {
 751                         CHECK_STATUS(ddr3_tip_write_additional_odt_setting
 752                                      (dev_num, if_id));
 753                 }
 754         }
 755
 756         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 757                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 758                 if (training_result[training_stage][if_id] == TEST_FAILED)
 759                         return MV_FAIL;
 760         }
 761
 762         return MV_OK;
 763 }
 764
 765 int ddr3_tip_calc_cs_mask(u32 dev_num, u32 if_id, u32 effective_cs,
 766                           u32 *cs_mask)
 767 {
 768         u32 all_bus_cs = 0, same_bus_cs;
 769         u32 bus_cnt;
 770         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
 771         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
 772
 773         *cs_mask = same_bus_cs = CS_BIT_MASK;
 774
 775         /*
 776          * In some of the devices (such as BC2), the CS is per pup and there
 777          * for mixed mode is valid on like other devices where CS configuration
 778          * is per interface.
 779          * In order to know that, we do 'Or' and 'And' operation between all
 780          * CS (of the pups).
 781          * If they are they are not the same then it's mixed mode so all CS
 782          * should be configured (when configuring the MRS)
 783          */
 784         for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
 785                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
 786
 787                 all_bus_cs |= tm->interface_params[if_id].
 788                         as_bus_params[bus_cnt].cs_bitmask;
 789                 same_bus_cs &= tm->interface_params[if_id].
 790                         as_bus_params[bus_cnt].cs_bitmask;
 791
 792                 /* cs enable is active low */
 793                 *cs_mask &= ~tm->interface_params[if_id].
 794                         as_bus_params[bus_cnt].cs_bitmask;
 795         }
 796
 797         if (all_bus_cs == same_bus_cs)
 798                 *cs_mask = (*cs_mask | (~(1 << effective_cs))) & CS_BIT_MASK;
 799
 800         return MV_OK;
 801 }
 802
 803 /*
 804  * Dynamic write leveling
 805  */
 806 int ddr3_tip_dynamic_write_leveling(u32 dev_num, int phase_remove)
 807 {
 808         u32 reg_data = 0, temp = 0, iter, if_id, bus_cnt;
 809         u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
 810         u32 cs_mask[MAX_INTERFACE_NUM];
 811         u32 read_data_sample_delay_vals[MAX_INTERFACE_NUM] = { 0 };
 812         u32 read_data_ready_delay_vals[MAX_INTERFACE_NUM] = { 0 };
 813         /* 0 for failure */
 814         u32 res_values[MAX_INTERFACE_NUM * MAX_BUS_NUM] = { 0 };
 815         u32 test_res = 0;       /* 0 - success for all pup */
 816         u32 data_read[MAX_INTERFACE_NUM];
 817         u8 wl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM];
 818         u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
 819         u32 cs_mask0[MAX_INTERFACE_NUM] = { 0 };
 820         unsigned int max_cs = mv_ddr_cs_num_get();
 821         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
 822         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
 823
 824         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 825                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 826
 827                 training_result[training_stage][if_id] = TEST_SUCCESS;
 828
 829                 /* save Read Data Sample Delay */
 830                 CHECK_STATUS(ddr3_tip_if_read
 831                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 832                               RD_DATA_SMPL_DLYS_REG,
 833                               read_data_sample_delay_vals, MASK_ALL_BITS));
 834                 /* save Read Data Ready Delay */
 835                 CHECK_STATUS(ddr3_tip_if_read
 836                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 837                               RD_DATA_RDY_DLYS_REG, read_data_ready_delay_vals,
 838                               MASK_ALL_BITS));
 839                 /* save current cs reg val */
 840                 CHECK_STATUS(ddr3_tip_if_read
 841                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 842                               DUAL_DUNIT_CFG_REG, cs_enable_reg_val, MASK_ALL_BITS));
 843         }
 844
 845         if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) < MV_TIP_REV_3) {
 846                 /* Enable multi-CS */
 847                 CHECK_STATUS(ddr3_tip_if_write
 848                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
 849                              DUAL_DUNIT_CFG_REG, 0, (1 << 3)));
 850         }
 851
 852         /*
 853          *     Phase 1: DRAM 2 Write Leveling mode
 854          */
 855
 856         /*Assert 10 refresh commands to DRAM to all CS */
 857         for (iter = 0; iter < WL_ITERATION_NUM; iter++) {
 858                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 859                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 860                         CHECK_STATUS(ddr3_tip_if_write
 861                                      (dev_num, ACCESS_TYPE_UNICAST,
 862                                       if_id, SDRAM_OP_REG,
 863                                       (u32)((~(0xf) << 8) | 0x2), 0xf1f));
 864                 }
 865         }
 866         /* check controller back to normal */
 867         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 868                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 869                 if (ddr3_tip_if_polling
 870                     (dev_num, ACCESS_TYPE_UNICAST, if_id, 0, 0x1f,
 871                      SDRAM_OP_REG, MAX_POLLING_ITERATIONS) != MV_OK) {
 872                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
 873                                        ("WL: DDR3 poll failed(3)"));
 874                 }
 875         }
 876
 877         for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
 878                 /*enable write leveling to all cs  - Q off , WL n */
 879                 /* calculate interface cs mask */
 880                 CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
 881                                                     0x1000, 0x1080));
 882
 883                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
 884                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 885                         /* cs enable is active low */
 886                         ddr3_tip_calc_cs_mask(dev_num, if_id, effective_cs,
 887                                               &cs_mask[if_id]);
 888                 }
 889
 890                 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
 891                         /* Enable Output buffer to relevant CS - Q on , WL on */
 892                         CHECK_STATUS(ddr3_tip_write_mrs_cmd
 893                                      (dev_num, cs_mask, MR_CMD1, 0x80, 0x1080));
 894
 895                         /*enable odt for relevant CS */
 896                         CHECK_STATUS(ddr3_tip_if_write
 897                                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
 898                                       0x1498, (0x3 << (effective_cs * 2)), 0xf));
 899                 } else {
 900                         /* FIXME: should be the same as _CPU case */
 901                         CHECK_STATUS(ddr3_tip_write_mrs_cmd
 902                                      (dev_num, cs_mask, MR_CMD1, 0xc0, 0x12c4));
 903                 }
 904
 905                 /*
 906                  *     Phase 2: Set training IP to write leveling mode
 907                  */
 908
 909                 CHECK_STATUS(ddr3_tip_dynamic_write_leveling_seq(dev_num));
 910
 911                 /* phase 3: trigger training */
 912                 mv_ddr_training_enable();
 913
 914                 /* check for training done */
 915                 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, data_read) != MV_OK) {
 916                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
 917                 } else { /* check for training pass */
 918                         reg_data = data_read[0];
 919 #if defined(CONFIG_ARMADA_38X) /* JIRA #1498 for 16 bit with ECC */
 920                         if (tm->bus_act_mask == 0xb) /* set to data to 0 to skip the check */
 921                                 reg_data = 0;
 922 #endif
 923                         if (reg_data != PASS)
 924                                 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
 925
 926                         /* check for training completion per bus */
 927                         for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
 928                                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
 929                                 /* training status */
 930                                 ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
 931                                               mask_results_pup_reg_map[bus_cnt],
 932                                               data_read, MASK_ALL_BITS);
 933                                 reg_data = data_read[0];
 934                                 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("WL: IF %d BUS %d reg 0x%x\n",
 935                                                                    0, bus_cnt, reg_data));
 936                                 if ((reg_data & (1 << 25)) == 0)
 937                                         res_values[bus_cnt] = 1;
 938                                 ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
 939                                               mask_results_pup_reg_map[bus_cnt],
 940                                               data_read, 0xff);
 941                                 /*
 942                                  * Save the read value that should be
 943                                  * write to PHY register
 944                                  */
 945                                 wl_values[effective_cs][bus_cnt][0] = (u8)data_read[0];
 946                         }
 947                 }
 948
 949                 /*
 950                  *     Phase 3.5: Validate result
 951                  */
 952                 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
 953                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
 954                         for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
 955                                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
 956                                 /*
 957                                  * Read result control register according to subphy
 958                                  * "16" below is for a half-phase
 959                                  */
 960                                 reg_data = wl_values[effective_cs][bus_cnt][if_id] + 16;
 961                                 /*
 962                                  * Write to WL register: ADLL [4:0], Phase [8:6],
 963                                  * Centralization ADLL [15:10] + 0x10
 964                                  */
 965                                 reg_data = (reg_data & 0x1f) |
 966                                            (((reg_data & 0xe0) >> 5) << 6) |
 967                                            (((reg_data & 0x1f) + phy_reg1_val) << 10);
 968                                 /* Search with WL CS0 subphy reg */
 969                                 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
 970                                                    ACCESS_TYPE_UNICAST, bus_cnt,
 971                                                    DDR_PHY_DATA, WL_PHY_REG(0), reg_data);
 972                                 /*
 973                                  * Check for change in data read from DRAM.
 974                                  * If changed, fix the result
 975                                  */
 976                                 CHECK_STATUS(ddr3_tip_if_read
 977                                              (dev_num,
 978                                               ACCESS_TYPE_UNICAST,
 979                                               if_id,
 980                                               TRAINING_WL_REG,
 981                                               data_read, MASK_ALL_BITS));
 982                                 if (((data_read[if_id] & (1 << (bus_cnt + 20))) >>
 983                                      (bus_cnt + 20)) == 0) {
 984                                         DEBUG_LEVELING(
 985                                                 DEBUG_LEVEL_ERROR,
 986                                                 ("WLValues was changed from 0x%X",
 987                                                  wl_values[effective_cs]
 988                                                  [bus_cnt][if_id]));
 989                                         wl_values[effective_cs]
 990                                         [bus_cnt][if_id] += 32;
 991                                         DEBUG_LEVELING(
 992                                                 DEBUG_LEVEL_ERROR,
 993                                                 ("to 0x%X",
 994                                                  wl_values[effective_cs]
 995                                                  [bus_cnt][if_id]));
 996                                 }
 997                         }
 998                 }
 999
1000                 /*
1001                  *     Phase 4: Exit write leveling mode
1002                  */
1003
1004                 /* disable DQs toggling */
1005                 CHECK_STATUS(ddr3_tip_if_write
1006                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1007                               WL_DQS_PATTERN_REG, 0x0, 0x1));
1008
1009                 /* Update MRS 1 (WL off) */
1010                 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1011                         CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1012                                                             0x1000, 0x1080));
1013                 } else {
1014                         /* FIXME: should be same as _CPU case */
1015                         CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1016                                                             0x1000, 0x12c4));
1017                 }
1018
1019                 /* Update MRS 1 (return to functional mode - Q on , WL off) */
1020                 CHECK_STATUS(ddr3_tip_write_mrs_cmd
1021                              (dev_num, cs_mask0, MR_CMD1, 0x0, 0x1080));
1022
1023                 /* set phy to normal mode */
1024                 CHECK_STATUS(ddr3_tip_if_write
1025                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1026                               TRAINING_SW_2_REG, 0x5, 0x7));
1027
1028                 /* exit sw override mode  */
1029                 CHECK_STATUS(ddr3_tip_if_write
1030                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1031                               TRAINING_SW_2_REG, 0x4, 0x7));
1032         }
1033
1034         /*
1035          *     Phase 5: Load WL values to each PHY
1036          */
1037
1038         for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1039                 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1040                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1041                         test_res = 0;
1042                         for (bus_cnt = 0;
1043                              bus_cnt < octets_per_if_num;
1044                              bus_cnt++) {
1045                                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
1046                                 /* check if result == pass */
1047                                 if (res_values
1048                                     [(if_id *
1049                                       octets_per_if_num) +
1050                                      bus_cnt] == 0) {
1051                                         /*
1052                                          * read result control register
1053                                          * according to pup
1054                                          */
1055                                         reg_data =
1056                                                 wl_values[effective_cs][bus_cnt]
1057                                                 [if_id];
1058                                         /*
1059                                          * Write into write leveling register
1060                                          * ([4:0] ADLL, [8:6] Phase, [15:10]
1061                                          * (centralization) ADLL + 0x10)
1062                                          */
1063                                         reg_data =
1064                                                 (reg_data & 0x1f) |
1065                                                 (((reg_data & 0xe0) >> 5) << 6) |
1066                                                 (((reg_data & 0x1f) +
1067                                                   phy_reg1_val) << 10);
1068                                         /*
1069                                          * in case phase remove should be executed
1070                                          * need to remove more than one phase.
1071                                          * this will take place only in low frequency,
1072                                          * where there could be more than one phase between sub-phys
1073                                          */
1074                                         if (phase_remove == 1) {
1075                                                 temp = (reg_data >> WR_LVL_PH_SEL_OFFS) & WR_LVL_PH_SEL_PHASE1;
1076                                                 reg_data &= ~(WR_LVL_PH_SEL_MASK << WR_LVL_PH_SEL_OFFS);
1077                                                 reg_data |= (temp << WR_LVL_PH_SEL_OFFS);
1078                                         }
1079
1080                                         ddr3_tip_bus_write(
1081                                                 dev_num,
1082                                                 ACCESS_TYPE_UNICAST,
1083                                                 if_id,
1084                                                 ACCESS_TYPE_UNICAST,
1085                                                 bus_cnt,
1086                                                 DDR_PHY_DATA,
1087                                                 WL_PHY_REG(effective_cs),
1088                                                 reg_data);
1089                                 } else {
1090                                         test_res = 1;
1091                                         /*
1092                                          * read result control register
1093                                          * according to pup
1094                                          */
1095                                         CHECK_STATUS(ddr3_tip_if_read
1096                                                      (dev_num,
1097                                                       ACCESS_TYPE_UNICAST,
1098                                                       if_id,
1099                                                       mask_results_pup_reg_map
1100                                                       [bus_cnt], data_read,
1101                                                       0xff));
1102                                         reg_data = data_read[if_id];
1103                                         DEBUG_LEVELING(
1104                                                 DEBUG_LEVEL_ERROR,
1105                                                 ("WL: IF %d BUS %d failed, reg 0x%x\n",
1106                                                  if_id, bus_cnt, reg_data));
1107                                 }
1108                         }
1109
1110                         if (test_res != 0) {
1111                                 training_result[training_stage][if_id] =
1112                                         TEST_FAILED;
1113                         }
1114                 }
1115         }
1116         /* Set to 0 after each loop to avoid illegal value may be used */
1117         effective_cs = 0;
1118
1119         /*
1120          * Copy the result from the effective CS search to the real
1121          * Functional CS
1122          */
1123         /* ddr3_tip_write_cs_result(dev_num, WL_PHY_REG(0); */
1124         /* restore saved values */
1125         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1126                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1127                 /* restore Read Data Sample Delay */
1128                 CHECK_STATUS(ddr3_tip_if_write
1129                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
1130                               RD_DATA_SMPL_DLYS_REG,
1131                               read_data_sample_delay_vals[if_id],
1132                               MASK_ALL_BITS));
1133
1134                 /* restore Read Data Ready Delay */
1135                 CHECK_STATUS(ddr3_tip_if_write
1136                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
1137                               RD_DATA_RDY_DLYS_REG,
1138                               read_data_ready_delay_vals[if_id],
1139                               MASK_ALL_BITS));
1140
1141                 /* enable multi cs */
1142                 CHECK_STATUS(ddr3_tip_if_write
1143                              (dev_num, ACCESS_TYPE_UNICAST, if_id,
1144                               DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
1145                               MASK_ALL_BITS));
1146         }
1147
1148         if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1149                 /* Disable modt0 for CS0 training - need to adjust for multi-CS
1150                  * in case of ddr4 set 0xf else 0
1151                  */
1152                 if (odt_config != 0) {
1153                         CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1154                                                        SDRAM_ODT_CTRL_HIGH_REG, 0x0, 0xf));
1155                 }
1156                 else {
1157                         CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1158                                                        SDRAM_ODT_CTRL_HIGH_REG, 0xf, 0xf));
1159                 }
1160
1161         }
1162
1163         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1164                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1165                 if (training_result[training_stage][if_id] == TEST_FAILED)
1166                         return MV_FAIL;
1167         }
1168
1169         return MV_OK;
1170 }
1171
1172 /*
1173  * Dynamic write leveling supplementary
1174  */
1175 int ddr3_tip_dynamic_write_leveling_supp(u32 dev_num)
1176 {
1177         int adll_offset;
1178         u32 if_id, bus_id, data, data_tmp;
1179         int is_if_fail = 0;
1180         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1181         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1182
1183         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1184                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1185                 is_if_fail = 0;
1186
1187                 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1188                         VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1189                         wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1190                         CHECK_STATUS(ddr3_tip_bus_read
1191                                      (dev_num, if_id, ACCESS_TYPE_UNICAST,
1192                                       bus_id, DDR_PHY_DATA,
1193                                       CTX_PHY_REG(effective_cs),
1194                                       &data));
1195                         DEBUG_LEVELING(
1196                                 DEBUG_LEVEL_TRACE,
1197                                 ("WL Supp: adll_offset=0 data delay = %d\n",
1198                                  data));
1199                         if (ddr3_tip_wl_supp_align_phase_shift
1200                             (dev_num, if_id, bus_id) == MV_OK) {
1201                                 DEBUG_LEVELING(
1202                                         DEBUG_LEVEL_TRACE,
1203                                         ("WL Supp: IF %d bus_id %d adll_offset=0 Success !\n",
1204                                          if_id, bus_id));
1205                                 continue;
1206                         }
1207
1208                         /* change adll */
1209                         adll_offset = 5;
1210                         CHECK_STATUS(ddr3_tip_bus_write
1211                                      (dev_num, ACCESS_TYPE_UNICAST, if_id,
1212                                       ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1213                                       CTX_PHY_REG(effective_cs),
1214                                       data + adll_offset));
1215                         CHECK_STATUS(ddr3_tip_bus_read
1216                                      (dev_num, if_id, ACCESS_TYPE_UNICAST,
1217                                       bus_id, DDR_PHY_DATA,
1218                                       CTX_PHY_REG(effective_cs),
1219                                       &data_tmp));
1220                         DEBUG_LEVELING(
1221                                 DEBUG_LEVEL_TRACE,
1222                                 ("WL Supp: adll_offset= %d data delay = %d\n",
1223                                  adll_offset, data_tmp));
1224
1225                         if (ddr3_tip_wl_supp_align_phase_shift
1226                             (dev_num, if_id, bus_id) == MV_OK) {
1227                                 DEBUG_LEVELING(
1228                                         DEBUG_LEVEL_TRACE,
1229                                         ("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1230                                          if_id, bus_id, adll_offset));
1231                                 continue;
1232                         }
1233
1234                         /* change adll */
1235                         adll_offset = -5;
1236                         CHECK_STATUS(ddr3_tip_bus_write
1237                                      (dev_num, ACCESS_TYPE_UNICAST, if_id,
1238                                       ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1239                                       CTX_PHY_REG(effective_cs),
1240                                       data + adll_offset));
1241                         CHECK_STATUS(ddr3_tip_bus_read
1242                                      (dev_num, if_id, ACCESS_TYPE_UNICAST,
1243                                       bus_id, DDR_PHY_DATA,
1244                                       CTX_PHY_REG(effective_cs),
1245                                       &data_tmp));
1246                         DEBUG_LEVELING(
1247                                 DEBUG_LEVEL_TRACE,
1248                                 ("WL Supp: adll_offset= %d data delay = %d\n",
1249                                  adll_offset, data_tmp));
1250                         if (ddr3_tip_wl_supp_align_phase_shift
1251                             (dev_num, if_id, bus_id) == MV_OK) {
1252                                 DEBUG_LEVELING(
1253                                         DEBUG_LEVEL_TRACE,
1254                                         ("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1255                                          if_id, bus_id, adll_offset));
1256                                 continue;
1257                         } else {
1258                                 DEBUG_LEVELING(
1259                                         DEBUG_LEVEL_ERROR,
1260                                         ("WL Supp: IF %d bus_id %d Failed !\n",
1261                                          if_id, bus_id));
1262                                 is_if_fail = 1;
1263                         }
1264                 }
1265
1266                 if (is_if_fail == 1) {
1267                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1268                                        ("WL Supp: CS# %d: IF %d failed\n",
1269                                         effective_cs, if_id));
1270                         training_result[training_stage][if_id] = TEST_FAILED;
1271                 } else {
1272                         training_result[training_stage][if_id] = TEST_SUCCESS;
1273                 }
1274         }
1275
1276         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1277                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1278                 if (training_result[training_stage][if_id] == TEST_FAILED)
1279                         return MV_FAIL;
1280         }
1281
1282         return MV_OK;
1283 }
1284
1285 /*
1286  * Phase Shift
1287  */
1288 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
1289                                               u32 bus_id)
1290 {
1291         u32 original_phase;
1292         u32 data, write_data;
1293
1294         wr_supp_res[if_id][bus_id].stage = PHASE_SHIFT;
1295         if (ddr3_tip_xsb_compare_test
1296             (dev_num, if_id, bus_id, 0) == MV_OK)
1297                 return MV_OK;
1298
1299         /* Read current phase */
1300         CHECK_STATUS(ddr3_tip_bus_read
1301                      (dev_num, if_id, ACCESS_TYPE_UNICAST, bus_id,
1302                       DDR_PHY_DATA, WL_PHY_REG(effective_cs), &data));
1303         original_phase = (data >> 6) & 0x7;
1304
1305         /* Set phase (0x0[6-8]) -2 */
1306         if (original_phase >= 1) {
1307                 if (original_phase == 1)
1308                         write_data = data & ~0x1df;
1309                 else
1310                         write_data = (data & ~0x1c0) |
1311                                      ((original_phase - 2) << 6);
1312                 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1313                                    ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1314                                    WL_PHY_REG(effective_cs), write_data);
1315                 if (ddr3_tip_xsb_compare_test
1316                     (dev_num, if_id, bus_id, -2) == MV_OK)
1317                         return MV_OK;
1318         }
1319
1320         /* Set phase (0x0[6-8]) +2 */
1321         if (original_phase <= 5) {
1322                 write_data = (data & ~0x1c0) |
1323                              ((original_phase + 2) << 6);
1324                 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1325                                    ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1326                                    WL_PHY_REG(effective_cs), write_data);
1327                 if (ddr3_tip_xsb_compare_test
1328                     (dev_num, if_id, bus_id, 2) == MV_OK)
1329                         return MV_OK;
1330         }
1331
1332         /* Set phase (0x0[6-8]) +4 */
1333         if (original_phase <= 3) {
1334                 write_data = (data & ~0x1c0) |
1335                              ((original_phase + 4) << 6);
1336                 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1337                                    ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1338                                    WL_PHY_REG(effective_cs), write_data);
1339                 if (ddr3_tip_xsb_compare_test
1340                     (dev_num, if_id, bus_id, 4) == MV_OK)
1341                         return MV_OK;
1342         }
1343
1344         /* Set phase (0x0[6-8]) +6 */
1345         if (original_phase <= 1) {
1346                 write_data = (data & ~0x1c0) |
1347                              ((original_phase + 6) << 6);
1348                 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1349                                    ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1350                                    WL_PHY_REG(effective_cs), write_data);
1351                 if (ddr3_tip_xsb_compare_test
1352                     (dev_num, if_id, bus_id, 6) == MV_OK)
1353                         return MV_OK;
1354         }
1355
1356         /* Write original WL result back */
1357         ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1358                            ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1359                            WL_PHY_REG(effective_cs), data);
1360         wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1361
1362         return MV_FAIL;
1363 }
1364
1365 /*
1366  * Compare Test
1367  */
1368 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
1369                                      u32 edge_offset)
1370 {
1371         u32 num_of_succ_byte_compare, word_in_pattern;
1372         u32 word_offset, i, num_of_word_mult;
1373         u32 read_pattern[TEST_PATTERN_LENGTH * 2];
1374         struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
1375         u32 pattern_test_pattern_table[8];
1376         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1377
1378         /* 3 below for INTERFACE_BUS_MASK_16BIT */
1379         num_of_word_mult = (tm->bus_act_mask == 3) ? 1 : 2;
1380
1381         for (i = 0; i < 8; i++) {
1382                 pattern_test_pattern_table[i] =
1383                         pattern_table_get_word(dev_num, PATTERN_TEST, (u8)i);
1384         }
1385
1386         /* External write, read and compare */
1387         CHECK_STATUS(ddr3_tip_load_pattern_to_mem(dev_num, PATTERN_TEST));
1388
1389         CHECK_STATUS(ddr3_tip_reset_fifo_ptr(dev_num));
1390
1391         CHECK_STATUS(ddr3_tip_ext_read
1392                      (dev_num, if_id,
1393                       ((pattern_table[PATTERN_TEST].start_addr << 3) +
1394                        ((SDRAM_CS_SIZE + 1) * effective_cs)), 1, read_pattern));
1395
1396         DEBUG_LEVELING(
1397                 DEBUG_LEVEL_TRACE,
1398                 ("XSB-compt CS#%d: IF %d bus_id %d 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1399                  effective_cs, if_id, bus_id,
1400                  read_pattern[0], read_pattern[1],
1401                  read_pattern[2], read_pattern[3],
1402                  read_pattern[4], read_pattern[5],
1403                  read_pattern[6], read_pattern[7]));
1404
1405         /* compare byte per pup */
1406         num_of_succ_byte_compare = 0;
1407         for (word_in_pattern = start_xsb_offset;
1408              word_in_pattern < (TEST_PATTERN_LENGTH * num_of_word_mult);
1409              word_in_pattern++) {
1410                 word_offset = word_in_pattern;
1411                 if ((word_offset > (TEST_PATTERN_LENGTH * 2 - 1)))
1412                         continue;
1413
1414                 if ((read_pattern[word_in_pattern] & pup_mask_table[bus_id]) ==
1415                     (pattern_test_pattern_table[word_offset] &
1416                      pup_mask_table[bus_id]))
1417                         num_of_succ_byte_compare++;
1418         }
1419
1420         if ((TEST_PATTERN_LENGTH * num_of_word_mult - start_xsb_offset) ==
1421             num_of_succ_byte_compare) {
1422                 wr_supp_res[if_id][bus_id].stage = edge_offset;
1423                 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1424                                ("supplementary: shift to %d for if %d pup %d success\n",
1425                                 edge_offset, if_id, bus_id));
1426                 wr_supp_res[if_id][bus_id].is_pup_fail = 0;
1427
1428                 return MV_OK;
1429         } else {
1430                 DEBUG_LEVELING(
1431                         DEBUG_LEVEL_TRACE,
1432                         ("XSB-compt CS#%d: IF %d bus_id %d num_of_succ_byte_compare %d - Fail!\n",
1433                          effective_cs, if_id, bus_id, num_of_succ_byte_compare));
1434
1435                 DEBUG_LEVELING(
1436                         DEBUG_LEVEL_TRACE,
1437                         ("XSB-compt: expected 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1438                          pattern_test_pattern_table[0],
1439                          pattern_test_pattern_table[1],
1440                          pattern_test_pattern_table[2],
1441                          pattern_test_pattern_table[3],
1442                          pattern_test_pattern_table[4],
1443                          pattern_test_pattern_table[5],
1444                          pattern_test_pattern_table[6],
1445                          pattern_test_pattern_table[7]));
1446                 DEBUG_LEVELING(
1447                         DEBUG_LEVEL_TRACE,
1448                         ("XSB-compt: recieved 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1449                          read_pattern[0], read_pattern[1],
1450                          read_pattern[2], read_pattern[3],
1451                          read_pattern[4], read_pattern[5],
1452                          read_pattern[6], read_pattern[7]));
1453
1454                 return MV_FAIL;
1455         }
1456 }
1457
1458 /*
1459  * Dynamic write leveling sequence
1460  */
1461 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num)
1462 {
1463         u32 bus_id, dq_id;
1464         u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1465         u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1466         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1467         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1468
1469         CHECK_STATUS(ddr3_tip_if_write
1470                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1471                       TRAINING_SW_2_REG, 0x1, 0x5));
1472         CHECK_STATUS(ddr3_tip_if_write
1473                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1474                       TRAINING_WL_REG, 0x50, 0xff));
1475         CHECK_STATUS(ddr3_tip_if_write
1476                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1477                       TRAINING_WL_REG, 0x5c, 0xff));
1478         CHECK_STATUS(ddr3_tip_if_write
1479                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1480                       GENERAL_TRAINING_OPCODE_REG, 0x381b82, 0x3c3faf));
1481         CHECK_STATUS(ddr3_tip_if_write
1482                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1483                       OPCODE_REG0_REG(1), (0x3 << 25), (0x3ffff << 9)));
1484         CHECK_STATUS(ddr3_tip_if_write
1485                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1486                       OPCODE_REG1_REG(1), 0x80, 0xffff));
1487         CHECK_STATUS(ddr3_tip_if_write
1488                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1489                       WL_DONE_CNTR_REF_REG, 0x14, 0xff));
1490         CHECK_STATUS(ddr3_tip_if_write
1491                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1492                       TRAINING_WL_REG, 0xff5c, 0xffff));
1493
1494         /* mask PBS */
1495         for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1496                 CHECK_STATUS(ddr3_tip_if_write
1497                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1498                               mask_results_dq_reg_map[dq_id], 0x1 << 24,
1499                               0x1 << 24));
1500         }
1501
1502         /* Mask all results */
1503         for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1504                 CHECK_STATUS(ddr3_tip_if_write
1505                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1506                               mask_results_pup_reg_map[bus_id], 0x1 << 24,
1507                               0x1 << 24));
1508         }
1509
1510         /* Unmask only wanted */
1511         for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1512                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1513                 CHECK_STATUS(ddr3_tip_if_write
1514                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1515                               mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1516         }
1517
1518         CHECK_STATUS(ddr3_tip_if_write
1519                      (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1520                       WL_DQS_PATTERN_REG, 0x1, 0x1));
1521
1522         return MV_OK;
1523 }
1524
1525 /*
1526  * Dynamic read leveling sequence
1527  */
1528 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num)
1529 {
1530         u32 bus_id, dq_id;
1531         u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1532         u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1533         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1534         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1535
1536         /* mask PBS */
1537         for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1538                 CHECK_STATUS(ddr3_tip_if_write
1539                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1540                               mask_results_dq_reg_map[dq_id], 0x1 << 24,
1541                               0x1 << 24));
1542         }
1543
1544         /* Mask all results */
1545         for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1546                 CHECK_STATUS(ddr3_tip_if_write
1547                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1548                               mask_results_pup_reg_map[bus_id], 0x1 << 24,
1549                               0x1 << 24));
1550         }
1551
1552         /* Unmask only wanted */
1553         for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1554                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1555                 CHECK_STATUS(ddr3_tip_if_write
1556                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1557                               mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1558         }
1559
1560         return MV_OK;
1561 }
1562
1563 /*
1564  * Dynamic read leveling sequence
1565  */
1566 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num)
1567 {
1568         u32 bus_id, dq_id;
1569         u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1570         u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1571         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1572         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1573
1574         /* mask PBS */
1575         for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1576                 CHECK_STATUS(ddr3_tip_if_write
1577                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1578                               mask_results_dq_reg_map[dq_id], 0x1 << 24,
1579                               0x1 << 24));
1580         }
1581
1582         /* Mask all results */
1583         for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1584                 CHECK_STATUS(ddr3_tip_if_write
1585                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1586                               mask_results_pup_reg_map[bus_id], 0x1 << 24,
1587                               0x1 << 24));
1588         }
1589
1590         /* Unmask only wanted */
1591         for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1592                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, dq_id / 8);
1593                 CHECK_STATUS(ddr3_tip_if_write
1594                              (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1595                               mask_results_dq_reg_map[dq_id], 0x0 << 24,
1596                               0x1 << 24));
1597         }
1598
1599         return MV_OK;
1600 }
1601
1602 /*
1603  * Print write leveling supplementary results
1604  */
1605 int ddr3_tip_print_wl_supp_result(u32 dev_num)
1606 {
1607         u32 bus_id = 0, if_id = 0;
1608         u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1609         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1610
1611         DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1612                        ("I/F0 PUP0 Result[0 - success, 1-fail] ...\n"));
1613
1614         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1615                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1616                 for (bus_id = 0; bus_id < octets_per_if_num;
1617                      bus_id++) {
1618                         VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1619                         DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1620                                        ("%d ,", wr_supp_res[if_id]
1621                                         [bus_id].is_pup_fail));
1622                 }
1623         }
1624         DEBUG_LEVELING(
1625                 DEBUG_LEVEL_INFO,
1626                 ("I/F0 PUP0 Stage[0-phase_shift, 1-clock_shift, 2-align_shift] ...\n"));
1627
1628         for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1629                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1630                 for (bus_id = 0; bus_id < octets_per_if_num;
1631                      bus_id++) {
1632                         VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1633                         DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1634                                        ("%d ,", wr_supp_res[if_id]
1635                                         [bus_id].stage));
1636                 }
1637         }
1638
1639         return MV_OK;
1640 }
1641
1642 #define RD_FIFO_PTR_LOW_STAT_INDIR_ADDR         0x9a
1643 #define RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR        0x9b
1644 /* position of falling dqs edge in fifo; walking 1 */
1645 #define RD_FIFO_DQS_FALL_EDGE_POS_0             0x1
1646 #define RD_FIFO_DQS_FALL_EDGE_POS_1             0x2
1647 #define RD_FIFO_DQS_FALL_EDGE_POS_2             0x4
1648 #define RD_FIFO_DQS_FALL_EDGE_POS_3             0x8
1649 #define RD_FIFO_DQS_FALL_EDGE_POS_4             0x10 /* lock */
1650 /* position of rising dqs edge in fifo; walking 0 */
1651 #define RD_FIFO_DQS_RISE_EDGE_POS_0             0x1fff
1652 #define RD_FIFO_DQS_RISE_EDGE_POS_1             0x3ffe
1653 #define RD_FIFO_DQS_RISE_EDGE_POS_2             0x3ffd
1654 #define RD_FIFO_DQS_RISE_EDGE_POS_3             0x3ffb
1655 #define RD_FIFO_DQS_RISE_EDGE_POS_4             0x3ff7 /* lock */
1656 #define TEST_ADDR               0x8
1657 #define TAPS_PER_UI             32
1658 #define UI_PER_RD_SAMPLE        4
1659 #define TAPS_PER_RD_SAMPLE      ((UI_PER_RD_SAMPLE) * (TAPS_PER_UI))
1660 #define MAX_RD_SAMPLES          32
1661 #define MAX_RL_VALUE            ((MAX_RD_SAMPLES) * (TAPS_PER_RD_SAMPLE))
1662 #define RD_FIFO_DLY             8
1663 #define STEP_SIZE               64
1664 #define RL_JITTER_WIDTH_LMT     20
1665 #define ADLL_TAPS_IN_CYCLE      64
1666
1667 enum rl_dqs_burst_state {
1668         RL_AHEAD = 0,
1669         RL_INSIDE,
1670         RL_BEHIND
1671 };
1672
1673
1674 int mv_ddr_rl_dqs_burst(u32 dev_num, u32 if_id, u32 freq)
1675 {
1676         enum rl_dqs_burst_state rl_state[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1677         enum hws_ddr_phy subphy_type = DDR_PHY_DATA;
1678         struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1679         int cl_val = tm->interface_params[0].cas_l;
1680         int rl_adll_val, rl_phase_val, sdr_cycle_incr, rd_sample, rd_ready;
1681         int final_rd_sample, final_rd_ready;
1682         int i, subphy_id, step;
1683         int pass_lock_num = 0;
1684         int init_pass_lock_num;
1685         int phase_delta;
1686         int min_phase, max_phase;
1687         unsigned int max_cs = mv_ddr_cs_num_get();
1688         u32 rl_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1689         u32 rl_min_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1690         u32 rl_max_values[MAX_CS_NUM][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1691         u32 rl_val, rl_min_val[MAX_CS_NUM], rl_max_val[MAX_CS_NUM];
1692         u32 reg_val_low, reg_val_high;
1693         u32 reg_val, reg_mask;
1694         uintptr_t test_addr = TEST_ADDR;
1695
1696
1697         /* initialization */
1698         if (mv_ddr_is_ecc_ena()) {
1699                 ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_SW_2_REG,
1700                                  &reg_val, MASK_ALL_BITS);
1701                 reg_mask = (TRAINING_ECC_MUX_MASK << TRAINING_ECC_MUX_OFFS) |
1702                            (TRAINING_SW_OVRD_MASK << TRAINING_SW_OVRD_OFFS);
1703                 reg_val &= ~reg_mask;
1704                 reg_val |= (TRAINING_ECC_MUX_DIS << TRAINING_ECC_MUX_OFFS) |
1705                            (TRAINING_SW_OVRD_ENA << TRAINING_SW_OVRD_OFFS);
1706                 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_SW_2_REG,
1707                                   reg_val, MASK_ALL_BITS);
1708                 ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_REG,
1709                                  &reg_val, MASK_ALL_BITS);
1710                 reg_mask = (TRN_START_MASK << TRN_START_OFFS);
1711                 reg_val &= ~reg_mask;
1712                 reg_val |= TRN_START_ENA << TRN_START_OFFS;
1713                 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_REG,
1714                                   reg_val, MASK_ALL_BITS);
1715         }
1716
1717         for (effective_cs = 0; effective_cs < max_cs; effective_cs++)
1718                 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++)
1719                         for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++)
1720                                 if (IS_BUS_ACTIVE(tm->bus_act_mask, subphy_id) == 0)
1721                                         pass_lock_num++; /* increment on inactive subphys */
1722
1723         init_pass_lock_num = pass_lock_num / max_cs;
1724         for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1725                 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1726                         VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1727                         training_result[training_stage][if_id] = TEST_SUCCESS;
1728                 }
1729         }
1730
1731         /* search for dqs edges per subphy */
1732         if_id = 0;
1733         for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1734
1735                 pass_lock_num = init_pass_lock_num;
1736                 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ODPG_DATA_CTRL_REG,
1737                                   effective_cs << ODPG_DATA_CS_OFFS,
1738                                   ODPG_DATA_CS_MASK << ODPG_DATA_CS_OFFS);
1739                 rl_min_val[effective_cs] = MAX_RL_VALUE;
1740                 rl_max_val[effective_cs] = 0;
1741                 step = STEP_SIZE;
1742                 for (i = 0; i < MAX_RL_VALUE; i += step) {
1743                         rl_val = 0;
1744                         sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1745                         rd_sample = cl_val + 2 * sdr_cycle_incr;
1746                         /* fifo out to in delay in search is constant */
1747                         rd_ready = rd_sample + RD_FIFO_DLY;
1748
1749                         ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1750                                           rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1751                                           RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1752                         ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1753                                           rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1754                                           RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1755
1756                         /* one sdr (single data rate) cycle incremented on every four phases of ddr clock */
1757                         sdr_cycle_incr = i % TAPS_PER_RD_SAMPLE;
1758                         rl_adll_val = sdr_cycle_incr % MAX_RD_SAMPLES;
1759                         rl_phase_val = sdr_cycle_incr / MAX_RD_SAMPLES;
1760                         rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1761                                  ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1762
1763                         /* write to all subphys (even to not connected or locked) */
1764                         ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_MULTICAST,
1765                                            0, DDR_PHY_DATA, RL_PHY_REG(effective_cs), rl_val);
1766
1767                         /* reset read fifo assertion */
1768                         ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1769                                           DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1770                                           DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1771
1772                         /* reset read fifo deassertion */
1773                         ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1774                                           DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1775                                           DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1776
1777                         /* perform one read burst */
1778                         if (MV_DDR_IS_64BIT_DRAM_MODE(tm->bus_act_mask))
1779                                 readq(test_addr);
1780                         else
1781                                 readl(test_addr);
1782
1783                         /* progress read ptr; decide on rl state per byte */
1784                         for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1785                                 if (rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND)
1786                                         continue; /* skip locked subphys */
1787                                 ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1788                                                   RD_FIFO_PTR_LOW_STAT_INDIR_ADDR, &reg_val_low);
1789                                 ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1790                                                   RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR, &reg_val_high);
1791                                 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1792                                                ("%s: cs %d, step %d, subphy %d, state %d, low 0x%04x, high 0x%04x; move to ",
1793                                                 __func__, effective_cs, i, subphy_id,
1794                                                 rl_state[effective_cs][subphy_id][if_id],
1795                                                 reg_val_low, reg_val_high));
1796
1797                                 switch (rl_state[effective_cs][subphy_id][if_id]) {
1798                                 case RL_AHEAD:
1799                                         /* improve search resolution getting closer to the window */
1800                                         if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1801                                             reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1802                                                 rl_state[effective_cs][subphy_id][if_id] = RL_INSIDE;
1803                                                 rl_values[effective_cs][subphy_id][if_id] = i;
1804                                                 rl_min_values[effective_cs][subphy_id][if_id] = i;
1805                                                 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1806                                                                ("new state %d\n",
1807                                                                 rl_state[effective_cs][subphy_id][if_id]));
1808                                         } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_3 &&
1809                                                    reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_3) {
1810                                                 step = (step < 2) ? step : 2;
1811                                         } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_2 &&
1812                                                    reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_2) {
1813                                                 step = (step < 16) ? step : 16;
1814                                         } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_1 &&
1815                                                    reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_1) {
1816                                                 step = (step < 32) ? step : 32;
1817                                         } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_0 &&
1818                                                    reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_0) {
1819                                                 step = (step < 64) ? step : 64;
1820                                         } else {
1821                                                 /* otherwise, step is unchanged */
1822                                         }
1823                                         break;
1824                                 case RL_INSIDE:
1825                                         if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1826                                             reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1827                                                 rl_max_values[effective_cs][subphy_id][if_id] = i;
1828                                                 if ((rl_max_values[effective_cs][subphy_id][if_id] -
1829                                                      rl_min_values[effective_cs][subphy_id][if_id]) >
1830                                                     ADLL_TAPS_IN_CYCLE) {
1831                                                         rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1832                                                         rl_values[effective_cs][subphy_id][if_id] =
1833                                                                 (i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1834                                                         pass_lock_num++;
1835                                                         DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1836                                                                        ("new lock %d\n", pass_lock_num));
1837                                                         if (rl_min_val[effective_cs] >
1838                                                             rl_values[effective_cs][subphy_id][if_id])
1839                                                                 rl_min_val[effective_cs] =
1840                                                                         rl_values[effective_cs][subphy_id][if_id];
1841                                                         if (rl_max_val[effective_cs] <
1842                                                             rl_values[effective_cs][subphy_id][if_id])
1843                                                                 rl_max_val[effective_cs] =
1844                                                                         rl_values[effective_cs][subphy_id][if_id];
1845                                                         step = 2;
1846                                                 }
1847                                         }
1848                                         if (reg_val_low != RD_FIFO_DQS_FALL_EDGE_POS_4 ||
1849                                             reg_val_high != RD_FIFO_DQS_RISE_EDGE_POS_4) {
1850                                                 if ((i - rl_values[effective_cs][subphy_id][if_id]) <
1851                                                     RL_JITTER_WIDTH_LMT) {
1852                                                         /* inside the jitter; not valid segment */
1853                                                         rl_state[effective_cs][subphy_id][if_id] = RL_AHEAD;
1854                                                         DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1855                                                                        ("new state %d; jitter on mask\n",
1856                                                                         rl_state[effective_cs][subphy_id][if_id]));
1857                                                 } else { /* finished valid segment */
1858                                                         rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1859                                                         rl_values[effective_cs][subphy_id][if_id] =
1860                                                                 (i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1861                                                         DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1862                                                                        ("new state %d, solution %d\n",
1863                                                                         rl_state[effective_cs][subphy_id][if_id],
1864                                                                         rl_values[effective_cs][subphy_id][if_id]));
1865                                                         pass_lock_num++;
1866                                                         DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1867                                                                        ("new lock %d\n", pass_lock_num));
1868                                                         if (rl_min_val[effective_cs] >
1869                                                             rl_values[effective_cs][subphy_id][if_id])
1870                                                                 rl_min_val[effective_cs] =
1871                                                                         rl_values[effective_cs][subphy_id][if_id];
1872                                                         if (rl_max_val[effective_cs] <
1873                                                             rl_values[effective_cs][subphy_id][if_id])
1874                                                                 rl_max_val[effective_cs] =
1875                                                                         rl_values[effective_cs][subphy_id][if_id];
1876                                                         step = 2;
1877                                                 }
1878                                         }
1879                                         break;
1880                                 case RL_BEHIND: /* do nothing */
1881                                         break;
1882                                 }
1883                                 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("\n"));
1884                         }
1885                         DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("pass_lock_num %d\n", pass_lock_num));
1886                         /* exit condition */
1887                         if (pass_lock_num == MAX_BUS_NUM)
1888                                 break;
1889                 } /* for-loop on i */
1890
1891                 if (pass_lock_num != MAX_BUS_NUM) {
1892                         DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1893                                        ("%s: cs %d, pass_lock_num %d, max_bus_num %d, init_pass_lock_num %d\n",
1894                                        __func__, effective_cs, pass_lock_num, MAX_BUS_NUM, init_pass_lock_num));
1895                         for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1896                                 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1897                                 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1898                                                ("%s: subphy %d %s\n",
1899                                                 __func__, subphy_id,
1900                                                 (rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND) ?
1901                                                 "locked" : "not locked"));
1902                         }
1903                 }
1904         } /* for-loop on effective_cs */
1905
1906         /* post-processing read leveling results */
1907         if_id = 0;
1908         for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1909                 phase_delta = 0;
1910                 i = rl_min_val[effective_cs];
1911                 sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1912                 rd_sample = cl_val + 2 * sdr_cycle_incr;
1913                 rd_ready = rd_sample + RD_FIFO_DLY;
1914                 min_phase = (rl_min_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1915                 max_phase = (rl_max_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1916                 final_rd_sample = rd_sample;
1917                 final_rd_ready = rd_ready;
1918
1919                 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1920                                   rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1921                                   RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1922                 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1923                                   rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1924                                   RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1925                 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1926                                ("%s: cs %d, min phase %d, max phase %d, read sample %d\n",
1927                                 __func__, effective_cs, min_phase, max_phase, rd_sample));
1928
1929                 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1930                         VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1931                         /* reduce sdr cycle per cs; extract rl adll and phase values */
1932                         i = rl_values[effective_cs][subphy_id][if_id] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE);
1933                         rl_adll_val = i % MAX_RD_SAMPLES;
1934                         rl_phase_val = i / MAX_RD_SAMPLES;
1935                         rl_phase_val -= phase_delta;
1936                         DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1937                                        ("%s: final results: cs %d, subphy %d, read sample %d read ready %d, rl_phase_val %d, rl_adll_val %d\n",
1938                                         __func__, effective_cs, subphy_id, final_rd_sample,
1939                                         final_rd_ready, rl_phase_val, rl_adll_val));
1940
1941                         rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1942                                  ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1943                         ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_UNICAST,
1944                                            subphy_id, subphy_type, RL_PHY_REG(effective_cs), rl_val);
1945                 }
1946         } /* for-loop on effective cs */
1947
1948         for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1949                 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1950                 if (odt_config != 0)
1951                         CHECK_STATUS(ddr3_tip_write_additional_odt_setting(dev_num, if_id));
1952         }
1953
1954
1955         /* reset read fifo assertion */
1956         ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1957                           DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1958                           DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1959
1960         /* reset read fifo deassertion */
1961         ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1962                           DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1963                           DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1964
1965         return MV_OK;
1966 }