drivers/ddr/altera/sequencer.c

   1 /*
   2  * Copyright Altera Corporation (C) 2012-2015
   3  *
   4  * SPDX-License-Identifier:    BSD-3-Clause
   5  */
   6
   7 #include <common.h>
   8 #include <asm/io.h>
   9 #include <asm/arch/sdram.h>
  10 #include "sequencer.h"
  11 #include "sequencer_auto.h"
  12 #include "sequencer_auto_ac_init.h"
  13 #include "sequencer_auto_inst_init.h"
  14 #include "sequencer_defines.h"
  15
  16 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group);
  17
  18 static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  19         (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
  20
  21 static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
  22         (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
  23
  24 static struct socfpga_sdr_reg_file *sdr_reg_file =
  25         (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
  26
  27 static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  28         (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
  29
  30 static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  31         (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  32
  33 static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  34         (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
  35
  36 static struct socfpga_data_mgr *data_mgr =
  37         (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
  38
  39 static struct socfpga_sdr_ctrl *sdr_ctrl =
  40         (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
  41
  42 #define DELTA_D         1
  43
  44 /*
  45  * In order to reduce ROM size, most of the selectable calibration steps are
  46  * decided at compile time based on the user's calibration mode selection,
  47  * as captured by the STATIC_CALIB_STEPS selection below.
  48  *
  49  * However, to support simulation-time selection of fast simulation mode, where
  50  * we skip everything except the bare minimum, we need a few of the steps to
  51  * be dynamic.  In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  52  * check, which is based on the rtl-supplied value, or we dynamically compute
  53  * the value to use based on the dynamically-chosen calibration mode
  54  */
  55
  56 #define DLEVEL 0
  57 #define STATIC_IN_RTL_SIM 0
  58 #define STATIC_SKIP_DELAY_LOOPS 0
  59
  60 #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  61         STATIC_SKIP_DELAY_LOOPS)
  62
  63 /* calibration steps requested by the rtl */
  64 uint16_t dyn_calib_steps;
  65
  66 /*
  67  * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
  68  * instead of static, we use boolean logic to select between
  69  * non-skip and skip values
  70  *
  71  * The mask is set to include all bits when not-skipping, but is
  72  * zero when skipping
  73  */
  74
  75 uint16_t skip_delay_mask;       /* mask off bits when skipping/not-skipping */
  76
  77 #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  78         ((non_skip_value) & skip_delay_mask)
  79
  80 struct gbl_type *gbl;
  81 struct param_type *param;
  82 uint32_t curr_shadow_reg;
  83
  84 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  85         uint32_t write_group, uint32_t use_dm,
  86         uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
  87
  88 static void set_failing_group_stage(uint32_t group, uint32_t stage,
  89         uint32_t substage)
  90 {
  91         /*
  92          * Only set the global stage if there was not been any other
  93          * failing group
  94          */
  95         if (gbl->error_stage == CAL_STAGE_NIL)  {
  96                 gbl->error_substage = substage;
  97                 gbl->error_stage = stage;
  98                 gbl->error_group = group;
  99         }
 100 }
 101
 102 static void reg_file_set_group(uint32_t set_group)
 103 {
 104         u32 addr = (u32)&sdr_reg_file->cur_stage;
 105
 106         /* Read the current group and stage */
 107         uint32_t cur_stage_group = readl(addr);
 108
 109         /* Clear the group */
 110         cur_stage_group &= 0x0000FFFF;
 111
 112         /* Set the group */
 113         cur_stage_group |= (set_group << 16);
 114
 115         /* Write the data back */
 116         writel(cur_stage_group, addr);
 117 }
 118
 119 static void reg_file_set_stage(uint32_t set_stage)
 120 {
 121         u32 addr = (u32)&sdr_reg_file->cur_stage;
 122
 123         /* Read the current group and stage */
 124         uint32_t cur_stage_group = readl(addr);
 125
 126         /* Clear the stage and substage */
 127         cur_stage_group &= 0xFFFF0000;
 128
 129         /* Set the stage */
 130         cur_stage_group |= (set_stage & 0x000000FF);
 131
 132         /* Write the data back */
 133         writel(cur_stage_group, addr);
 134 }
 135
 136 static void reg_file_set_sub_stage(uint32_t set_sub_stage)
 137 {
 138         u32 addr = (u32)&sdr_reg_file->cur_stage;
 139
 140         /* Read the current group and stage */
 141         uint32_t cur_stage_group = readl(addr);
 142
 143         /* Clear the substage */
 144         cur_stage_group &= 0xFFFF00FF;
 145
 146         /* Set the sub stage */
 147         cur_stage_group |= ((set_sub_stage << 8) & 0x0000FF00);
 148
 149         /* Write the data back */
 150         writel(cur_stage_group, addr);
 151 }
 152
 153 static void initialize(void)
 154 {
 155         u32 addr = (u32)&phy_mgr_cfg->mux_sel;
 156
 157         debug("%s:%d\n", __func__, __LINE__);
 158         /* USER calibration has control over path to memory */
 159         /*
 160          * In Hard PHY this is a 2-bit control:
 161          * 0: AFI Mux Select
 162          * 1: DDIO Mux Select
 163          */
 164         writel(0x3, addr);
 165
 166         /* USER memory clock is not stable we begin initialization  */
 167         addr = (u32)&phy_mgr_cfg->reset_mem_stbl;
 168         writel(0, addr);
 169
 170         /* USER calibration status all set to zero */
 171         addr = (u32)&phy_mgr_cfg->cal_status;
 172         writel(0, addr);
 173
 174         addr = (u32)&phy_mgr_cfg->cal_debug_info;
 175         writel(0, addr);
 176
 177         if ((dyn_calib_steps & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) {
 178                 param->read_correct_mask_vg  = ((uint32_t)1 <<
 179                         (RW_MGR_MEM_DQ_PER_READ_DQS /
 180                         RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
 181                 param->write_correct_mask_vg = ((uint32_t)1 <<
 182                         (RW_MGR_MEM_DQ_PER_READ_DQS /
 183                         RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
 184                 param->read_correct_mask     = ((uint32_t)1 <<
 185                         RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
 186                 param->write_correct_mask    = ((uint32_t)1 <<
 187                         RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
 188                 param->dm_correct_mask       = ((uint32_t)1 <<
 189                         (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH))
 190                         - 1;
 191         }
 192 }
 193
 194 static void set_rank_and_odt_mask(uint32_t rank, uint32_t odt_mode)
 195 {
 196         uint32_t odt_mask_0 = 0;
 197         uint32_t odt_mask_1 = 0;
 198         uint32_t cs_and_odt_mask;
 199         uint32_t addr;
 200
 201         if (odt_mode == RW_MGR_ODT_MODE_READ_WRITE) {
 202                 if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) {
 203                         /*
 204                          * 1 Rank
 205                          * Read: ODT = 0
 206                          * Write: ODT = 1
 207                          */
 208                         odt_mask_0 = 0x0;
 209                         odt_mask_1 = 0x1;
 210                 } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) {
 211                         /* 2 Ranks */
 212                         if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
 213                                 /* - Dual-Slot , Single-Rank
 214                                  * (1 chip-select per DIMM)
 215                                  * OR
 216                                  * - RDIMM, 4 total CS (2 CS per DIMM)
 217                                  * means 2 DIMM
 218                                  * Since MEM_NUMBER_OF_RANKS is 2 they are
 219                                  * both single rank
 220                                  * with 2 CS each (special for RDIMM)
 221                                  * Read: Turn on ODT on the opposite rank
 222                                  * Write: Turn on ODT on all ranks
 223                                  */
 224                                 odt_mask_0 = 0x3 & ~(1 << rank);
 225                                 odt_mask_1 = 0x3;
 226                         } else {
 227                                 /*
 228                                  * USER - Single-Slot , Dual-rank DIMMs
 229                                  * (2 chip-selects per DIMM)
 230                                  * USER Read: Turn on ODT off on all ranks
 231                                  * USER Write: Turn on ODT on active rank
 232                                  */
 233                                 odt_mask_0 = 0x0;
 234                                 odt_mask_1 = 0x3 & (1 << rank);
 235                         }
 236                 } else {
 237                         /* 4 Ranks
 238                          * Read:
 239                          * ----------+-----------------------+
 240                          *           |                       |
 241                          *           |         ODT           |
 242                          * Read From +-----------------------+
 243                          *   Rank    |  3  |  2  |  1  |  0  |
 244                          * ----------+-----+-----+-----+-----+
 245                          *     0     |  0  |  1  |  0  |  0  |
 246                          *     1     |  1  |  0  |  0  |  0  |
 247                          *     2     |  0  |  0  |  0  |  1  |
 248                          *     3     |  0  |  0  |  1  |  0  |
 249                          * ----------+-----+-----+-----+-----+
 250                          *
 251                          * Write:
 252                          * ----------+-----------------------+
 253                          *           |                       |
 254                          *           |         ODT           |
 255                          * Write To  +-----------------------+
 256                          *   Rank    |  3  |  2  |  1  |  0  |
 257                          * ----------+-----+-----+-----+-----+
 258                          *     0     |  0  |  1  |  0  |  1  |
 259                          *     1     |  1  |  0  |  1  |  0  |
 260                          *     2     |  0  |  1  |  0  |  1  |
 261                          *     3     |  1  |  0  |  1  |  0  |
 262                          * ----------+-----+-----+-----+-----+
 263                          */
 264                         switch (rank) {
 265                         case 0:
 266                                 odt_mask_0 = 0x4;
 267                                 odt_mask_1 = 0x5;
 268                                 break;
 269                         case 1:
 270                                 odt_mask_0 = 0x8;
 271                                 odt_mask_1 = 0xA;
 272                                 break;
 273                         case 2:
 274                                 odt_mask_0 = 0x1;
 275                                 odt_mask_1 = 0x5;
 276                                 break;
 277                         case 3:
 278                                 odt_mask_0 = 0x2;
 279                                 odt_mask_1 = 0xA;
 280                                 break;
 281                         }
 282                 }
 283         } else {
 284                 odt_mask_0 = 0x0;
 285                 odt_mask_1 = 0x0;
 286         }
 287
 288         cs_and_odt_mask =
 289                 (0xFF & ~(1 << rank)) |
 290                 ((0xFF & odt_mask_0) << 8) |
 291                 ((0xFF & odt_mask_1) << 16);
 292         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_SET_CS_AND_ODT_MASK_OFFSET;
 293         writel(cs_and_odt_mask, addr);
 294 }
 295
 296 static void scc_mgr_initialize(void)
 297 {
 298         u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_HHP_RFILE_OFFSET;
 299
 300         /*
 301          * Clear register file for HPS
 302          * 16 (2^4) is the size of the full register file in the scc mgr:
 303          *      RFILE_DEPTH = log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
 304          * MEM_IF_READ_DQS_WIDTH - 1) + 1;
 305          */
 306         uint32_t i;
 307         for (i = 0; i < 16; i++) {
 308                 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
 309                            __func__, __LINE__, i);
 310                 writel(0, addr + (i << 2));
 311         }
 312 }
 313
 314 static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group,
 315                                                 uint32_t delay)
 316 {
 317         u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
 318
 319         /* Load the setting in the SCC manager */
 320         writel(delay, addr + (read_group << 2));
 321 }
 322
 323 static void scc_mgr_set_dqs_io_in_delay(uint32_t write_group,
 324         uint32_t delay)
 325 {
 326         u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
 327
 328         writel(delay, addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
 329 }
 330
 331 static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
 332 {
 333         u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_EN_PHASE_OFFSET;
 334
 335         /* Load the setting in the SCC manager */
 336         writel(phase, addr + (read_group << 2));
 337 }
 338
 339 static void scc_mgr_set_dqs_en_phase_all_ranks(uint32_t read_group,
 340                                                uint32_t phase)
 341 {
 342         uint32_t r;
 343         uint32_t update_scan_chains;
 344         uint32_t addr;
 345
 346         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 347              r += NUM_RANKS_PER_SHADOW_REG) {
 348                 /*
 349                  * USER although the h/w doesn't support different phases per
 350                  * shadow register, for simplicity our scc manager modeling
 351                  * keeps different phase settings per shadow reg, and it's
 352                  * important for us to keep them in sync to match h/w.
 353                  * for efficiency, the scan chain update should occur only
 354                  * once to sr0.
 355                  */
 356                 update_scan_chains = (r == 0) ? 1 : 0;
 357
 358                 scc_mgr_set_dqs_en_phase(read_group, phase);
 359
 360                 if (update_scan_chains) {
 361                         addr = (u32)&sdr_scc_mgr->dqs_ena;
 362                         writel(read_group, addr);
 363
 364                         addr = (u32)&sdr_scc_mgr->update;
 365                         writel(0, addr);
 366                 }
 367         }
 368 }
 369
 370 static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group,
 371                                                   uint32_t phase)
 372 {
 373         u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQDQS_OUT_PHASE_OFFSET;
 374
 375         /* Load the setting in the SCC manager */
 376         writel(phase, addr + (write_group << 2));
 377 }
 378
 379 static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
 380                                                      uint32_t phase)
 381 {
 382         uint32_t r;
 383         uint32_t update_scan_chains;
 384         uint32_t addr;
 385
 386         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 387              r += NUM_RANKS_PER_SHADOW_REG) {
 388                 /*
 389                  * USER although the h/w doesn't support different phases per
 390                  * shadow register, for simplicity our scc manager modeling
 391                  * keeps different phase settings per shadow reg, and it's
 392                  * important for us to keep them in sync to match h/w.
 393                  * for efficiency, the scan chain update should occur only
 394                  * once to sr0.
 395                  */
 396                 update_scan_chains = (r == 0) ? 1 : 0;
 397
 398                 scc_mgr_set_dqdqs_output_phase(write_group, phase);
 399
 400                 if (update_scan_chains) {
 401                         addr = (u32)&sdr_scc_mgr->dqs_ena;
 402                         writel(write_group, addr);
 403
 404                         addr = (u32)&sdr_scc_mgr->update;
 405                         writel(0, addr);
 406                 }
 407         }
 408 }
 409
 410 static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
 411 {
 412         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_EN_DELAY_OFFSET;
 413
 414         /* Load the setting in the SCC manager */
 415         writel(delay + IO_DQS_EN_DELAY_OFFSET, addr +
 416                (read_group << 2));
 417 }
 418
 419 static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
 420                                                uint32_t delay)
 421 {
 422         uint32_t r;
 423         uint32_t addr;
 424
 425         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 426                 r += NUM_RANKS_PER_SHADOW_REG) {
 427                 scc_mgr_set_dqs_en_delay(read_group, delay);
 428
 429                 addr = (u32)&sdr_scc_mgr->dqs_ena;
 430                 writel(read_group, addr);
 431                 /*
 432                  * In shadow register mode, the T11 settings are stored in
 433                  * registers in the core, which are updated by the DQS_ENA
 434                  * signals. Not issuing the SCC_MGR_UPD command allows us to
 435                  * save lots of rank switching overhead, by calling
 436                  * select_shadow_regs_for_update with update_scan_chains
 437                  * set to 0.
 438                  */
 439                 addr = (u32)&sdr_scc_mgr->update;
 440                 writel(0, addr);
 441         }
 442         /*
 443          * In shadow register mode, the T11 settings are stored in
 444          * registers in the core, which are updated by the DQS_ENA
 445          * signals. Not issuing the SCC_MGR_UPD command allows us to
 446          * save lots of rank switching overhead, by calling
 447          * select_shadow_regs_for_update with update_scan_chains
 448          * set to 0.
 449          */
 450         addr = (u32)&sdr_scc_mgr->update;
 451         writel(0, addr);
 452 }
 453
 454 static void scc_mgr_set_oct_out1_delay(uint32_t write_group, uint32_t delay)
 455 {
 456         uint32_t read_group;
 457         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_OCT_OUT1_DELAY_OFFSET;
 458
 459         /*
 460          * Load the setting in the SCC manager
 461          * Although OCT affects only write data, the OCT delay is controlled
 462          * by the DQS logic block which is instantiated once per read group.
 463          * For protocols where a write group consists of multiple read groups,
 464          * the setting must be set multiple times.
 465          */
 466         for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 467              RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 468              read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 469              RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group)
 470                 writel(delay, addr + (read_group << 2));
 471 }
 472
 473 static void scc_mgr_set_dq_out1_delay(uint32_t write_group,
 474                                       uint32_t dq_in_group, uint32_t delay)
 475 {
 476         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
 477
 478         /* Load the setting in the SCC manager */
 479         writel(delay, addr + (dq_in_group << 2));
 480 }
 481
 482 static void scc_mgr_set_dq_in_delay(uint32_t write_group,
 483         uint32_t dq_in_group, uint32_t delay)
 484 {
 485         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
 486
 487         /* Load the setting in the SCC manager */
 488         writel(delay, addr + (dq_in_group << 2));
 489 }
 490
 491 static void scc_mgr_set_hhp_extras(void)
 492 {
 493         /*
 494          * Load the fixed setting in the SCC manager
 495          * bits: 0:0 = 1'b1   - dqs bypass
 496          * bits: 1:1 = 1'b1   - dq bypass
 497          * bits: 4:2 = 3'b001   - rfifo_mode
 498          * bits: 6:5 = 2'b01  - rfifo clock_select
 499          * bits: 7:7 = 1'b0  - separate gating from ungating setting
 500          * bits: 8:8 = 1'b0  - separate OE from Output delay setting
 501          */
 502         uint32_t value = (0<<8) | (0<<7) | (1<<5) | (1<<2) | (1<<1) | (1<<0);
 503         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_HHP_GLOBALS_OFFSET;
 504
 505         writel(value, addr + SCC_MGR_HHP_EXTRAS_OFFSET);
 506 }
 507
 508 static void scc_mgr_set_dqs_out1_delay(uint32_t write_group,
 509                                               uint32_t delay)
 510 {
 511         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
 512
 513         /* Load the setting in the SCC manager */
 514         writel(delay, addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
 515 }
 516
 517 static void scc_mgr_set_dm_out1_delay(uint32_t write_group,
 518                                              uint32_t dm, uint32_t delay)
 519 {
 520         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
 521
 522         /* Load the setting in the SCC manager */
 523         writel(delay, addr +
 524                 ((RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm) << 2));
 525 }
 526
 527 /*
 528  * USER Zero all DQS config
 529  * TODO: maybe rename to scc_mgr_zero_dqs_config (or something)
 530  */
 531 static void scc_mgr_zero_all(void)
 532 {
 533         uint32_t i, r;
 534         uint32_t addr;
 535
 536         /*
 537          * USER Zero all DQS config settings, across all groups and all
 538          * shadow registers
 539          */
 540         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
 541              NUM_RANKS_PER_SHADOW_REG) {
 542                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
 543                         /*
 544                          * The phases actually don't exist on a per-rank basis,
 545                          * but there's no harm updating them several times, so
 546                          * let's keep the code simple.
 547                          */
 548                         scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
 549                         scc_mgr_set_dqs_en_phase(i, 0);
 550                         scc_mgr_set_dqs_en_delay(i, 0);
 551                 }
 552
 553                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
 554                         scc_mgr_set_dqdqs_output_phase(i, 0);
 555                         /* av/cv don't have out2 */
 556                         scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
 557                 }
 558         }
 559
 560         /* multicast to all DQS group enables */
 561         addr = (u32)&sdr_scc_mgr->dqs_ena;
 562         writel(0xff, addr);
 563
 564         addr = (u32)&sdr_scc_mgr->update;
 565         writel(0, addr);
 566 }
 567
 568 static void scc_set_bypass_mode(uint32_t write_group, uint32_t mode)
 569 {
 570         uint32_t addr;
 571         /* mode = 0 : Do NOT bypass - Half Rate Mode */
 572         /* mode = 1 : Bypass - Full Rate Mode */
 573
 574         /* only need to set once for all groups, pins, dq, dqs, dm */
 575         if (write_group == 0) {
 576                 debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n", __func__,
 577                            __LINE__);
 578                 scc_mgr_set_hhp_extras();
 579                 debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
 580                           __func__, __LINE__);
 581         }
 582         /* multicast to all DQ enables */
 583         addr = (u32)&sdr_scc_mgr->dq_ena;
 584         writel(0xff, addr);
 585
 586         addr = (u32)&sdr_scc_mgr->dm_ena;
 587         writel(0xff, addr);
 588
 589         /* update current DQS IO enable */
 590         addr = (u32)&sdr_scc_mgr->dqs_io_ena;
 591         writel(0, addr);
 592
 593         /* update the DQS logic */
 594         addr = (u32)&sdr_scc_mgr->dqs_ena;
 595         writel(write_group, addr);
 596
 597         /* hit update */
 598         addr = (u32)&sdr_scc_mgr->update;
 599         writel(0, addr);
 600 }
 601
 602 static void scc_mgr_zero_group(uint32_t write_group, uint32_t test_begin,
 603                                int32_t out_only)
 604 {
 605         uint32_t i, r;
 606         uint32_t addr;
 607
 608         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
 609                 NUM_RANKS_PER_SHADOW_REG) {
 610                 /* Zero all DQ config settings */
 611                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
 612                         scc_mgr_set_dq_out1_delay(write_group, i, 0);
 613                         if (!out_only)
 614                                 scc_mgr_set_dq_in_delay(write_group, i, 0);
 615                 }
 616
 617                 /* multicast to all DQ enables */
 618                 addr = (u32)&sdr_scc_mgr->dq_ena;
 619                 writel(0xff, addr);
 620
 621                 /* Zero all DM config settings */
 622                 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 623                         scc_mgr_set_dm_out1_delay(write_group, i, 0);
 624                 }
 625
 626                 /* multicast to all DM enables */
 627                 addr = (u32)&sdr_scc_mgr->dm_ena;
 628                 writel(0xff, addr);
 629
 630                 /* zero all DQS io settings */
 631                 if (!out_only)
 632                         scc_mgr_set_dqs_io_in_delay(write_group, 0);
 633                 /* av/cv don't have out2 */
 634                 scc_mgr_set_dqs_out1_delay(write_group, IO_DQS_OUT_RESERVE);
 635                 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
 636                 scc_mgr_load_dqs_for_write_group(write_group);
 637
 638                 /* multicast to all DQS IO enables (only 1) */
 639                 addr = (u32)&sdr_scc_mgr->dqs_io_ena;
 640                 writel(0, addr);
 641
 642                 /* hit update to zero everything */
 643                 addr = (u32)&sdr_scc_mgr->update;
 644                 writel(0, addr);
 645         }
 646 }
 647
 648 /* load up dqs config settings */
 649 static void scc_mgr_load_dqs(uint32_t dqs)
 650 {
 651         uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena;
 652
 653         writel(dqs, addr);
 654 }
 655
 656 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group)
 657 {
 658         uint32_t read_group;
 659         uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena;
 660         /*
 661          * Although OCT affects only write data, the OCT delay is controlled
 662          * by the DQS logic block which is instantiated once per read group.
 663          * For protocols where a write group consists of multiple read groups,
 664          * the setting must be scanned multiple times.
 665          */
 666         for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 667              RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 668              read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 669              RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group)
 670                 writel(read_group, addr);
 671 }
 672
 673 /* load up dqs io config settings */
 674 static void scc_mgr_load_dqs_io(void)
 675 {
 676         uint32_t addr = (u32)&sdr_scc_mgr->dqs_io_ena;
 677
 678         writel(0, addr);
 679 }
 680
 681 /* load up dq config settings */
 682 static void scc_mgr_load_dq(uint32_t dq_in_group)
 683 {
 684         uint32_t addr = (u32)&sdr_scc_mgr->dq_ena;
 685
 686         writel(dq_in_group, addr);
 687 }
 688
 689 /* load up dm config settings */
 690 static void scc_mgr_load_dm(uint32_t dm)
 691 {
 692         uint32_t addr = (u32)&sdr_scc_mgr->dm_ena;
 693
 694         writel(dm, addr);
 695 }
 696
 697 /*
 698  * apply and load a particular input delay for the DQ pins in a group
 699  * group_bgn is the index of the first dq pin (in the write group)
 700  */
 701 static void scc_mgr_apply_group_dq_in_delay(uint32_t write_group,
 702                                             uint32_t group_bgn, uint32_t delay)
 703 {
 704         uint32_t i, p;
 705
 706         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
 707                 scc_mgr_set_dq_in_delay(write_group, p, delay);
 708                 scc_mgr_load_dq(p);
 709         }
 710 }
 711
 712 /* apply and load a particular output delay for the DQ pins in a group */
 713 static void scc_mgr_apply_group_dq_out1_delay(uint32_t write_group,
 714                                               uint32_t group_bgn,
 715                                               uint32_t delay1)
 716 {
 717         uint32_t i, p;
 718
 719         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
 720                 scc_mgr_set_dq_out1_delay(write_group, i, delay1);
 721                 scc_mgr_load_dq(i);
 722         }
 723 }
 724
 725 /* apply and load a particular output delay for the DM pins in a group */
 726 static void scc_mgr_apply_group_dm_out1_delay(uint32_t write_group,
 727                                               uint32_t delay1)
 728 {
 729         uint32_t i;
 730
 731         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 732                 scc_mgr_set_dm_out1_delay(write_group, i, delay1);
 733                 scc_mgr_load_dm(i);
 734         }
 735 }
 736
 737
 738 /* apply and load delay on both DQS and OCT out1 */
 739 static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
 740                                                     uint32_t delay)
 741 {
 742         scc_mgr_set_dqs_out1_delay(write_group, delay);
 743         scc_mgr_load_dqs_io();
 744
 745         scc_mgr_set_oct_out1_delay(write_group, delay);
 746         scc_mgr_load_dqs_for_write_group(write_group);
 747 }
 748
 749 /* apply a delay to the entire output side: DQ, DM, DQS, OCT */
 750 static void scc_mgr_apply_group_all_out_delay_add(uint32_t write_group,
 751                                                   uint32_t group_bgn,
 752                                                   uint32_t delay)
 753 {
 754         uint32_t i, p, new_delay;
 755
 756         /* dq shift */
 757         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
 758                 new_delay = READ_SCC_DQ_OUT2_DELAY;
 759                 new_delay += delay;
 760
 761                 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 762                         debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQ[%u,%u]:\
 763                                    %u > %lu => %lu", __func__, __LINE__,
 764                                    write_group, group_bgn, delay, i, p, new_delay,
 765                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX,
 766                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX);
 767                         new_delay = IO_IO_OUT2_DELAY_MAX;
 768                 }
 769
 770                 scc_mgr_load_dq(i);
 771         }
 772
 773         /* dm shift */
 774         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 775                 new_delay = READ_SCC_DM_IO_OUT2_DELAY;
 776                 new_delay += delay;
 777
 778                 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 779                         debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DM[%u]:\
 780                                    %u > %lu => %lu\n",  __func__, __LINE__,
 781                                    write_group, group_bgn, delay, i, new_delay,
 782                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX,
 783                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX);
 784                         new_delay = IO_IO_OUT2_DELAY_MAX;
 785                 }
 786
 787                 scc_mgr_load_dm(i);
 788         }
 789
 790         /* dqs shift */
 791         new_delay = READ_SCC_DQS_IO_OUT2_DELAY;
 792         new_delay += delay;
 793
 794         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 795                 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
 796                            " adding %u to OUT1\n", __func__, __LINE__,
 797                            write_group, group_bgn, delay, new_delay,
 798                            IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
 799                            new_delay - IO_IO_OUT2_DELAY_MAX);
 800                 scc_mgr_set_dqs_out1_delay(write_group, new_delay -
 801                                            IO_IO_OUT2_DELAY_MAX);
 802                 new_delay = IO_IO_OUT2_DELAY_MAX;
 803         }
 804
 805         scc_mgr_load_dqs_io();
 806
 807         /* oct shift */
 808         new_delay = READ_SCC_OCT_OUT2_DELAY;
 809         new_delay += delay;
 810
 811         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 812                 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
 813                            " adding %u to OUT1\n", __func__, __LINE__,
 814                            write_group, group_bgn, delay, new_delay,
 815                            IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
 816                            new_delay - IO_IO_OUT2_DELAY_MAX);
 817                 scc_mgr_set_oct_out1_delay(write_group, new_delay -
 818                                            IO_IO_OUT2_DELAY_MAX);
 819                 new_delay = IO_IO_OUT2_DELAY_MAX;
 820         }
 821
 822         scc_mgr_load_dqs_for_write_group(write_group);
 823 }
 824
 825 /*
 826  * USER apply a delay to the entire output side (DQ, DM, DQS, OCT)
 827  * and to all ranks
 828  */
 829 static void scc_mgr_apply_group_all_out_delay_add_all_ranks(
 830         uint32_t write_group, uint32_t group_bgn, uint32_t delay)
 831 {
 832         uint32_t r;
 833         uint32_t addr = (u32)&sdr_scc_mgr->update;
 834
 835         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 836                 r += NUM_RANKS_PER_SHADOW_REG) {
 837                 scc_mgr_apply_group_all_out_delay_add(write_group,
 838                                                       group_bgn, delay);
 839                 writel(0, addr);
 840         }
 841 }
 842
 843 /* optimization used to recover some slots in ddr3 inst_rom */
 844 /* could be applied to other protocols if we wanted to */
 845 static void set_jump_as_return(void)
 846 {
 847         uint32_t addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
 848
 849         /*
 850          * to save space, we replace return with jump to special shared
 851          * RETURN instruction so we set the counter to large value so that
 852          * we always jump
 853          */
 854         writel(0xff, addr);
 855         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
 856         writel(RW_MGR_RETURN, addr);
 857 }
 858
 859 /*
 860  * should always use constants as argument to ensure all computations are
 861  * performed at compile time
 862  */
 863 static void delay_for_n_mem_clocks(const uint32_t clocks)
 864 {
 865         uint32_t afi_clocks;
 866         uint8_t inner = 0;
 867         uint8_t outer = 0;
 868         uint16_t c_loop = 0;
 869         uint32_t addr;
 870
 871         debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
 872
 873
 874         afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
 875         /* scale (rounding up) to get afi clocks */
 876
 877         /*
 878          * Note, we don't bother accounting for being off a little bit
 879          * because of a few extra instructions in outer loops
 880          * Note, the loops have a test at the end, and do the test before
 881          * the decrement, and so always perform the loop
 882          * 1 time more than the counter value
 883          */
 884         if (afi_clocks == 0) {
 885                 ;
 886         } else if (afi_clocks <= 0x100) {
 887                 inner = afi_clocks-1;
 888                 outer = 0;
 889                 c_loop = 0;
 890         } else if (afi_clocks <= 0x10000) {
 891                 inner = 0xff;
 892                 outer = (afi_clocks-1) >> 8;
 893                 c_loop = 0;
 894         } else {
 895                 inner = 0xff;
 896                 outer = 0xff;
 897                 c_loop = (afi_clocks-1) >> 16;
 898         }
 899
 900         /*
 901          * rom instructions are structured as follows:
 902          *
 903          *    IDLE_LOOP2: jnz cntr0, TARGET_A
 904          *    IDLE_LOOP1: jnz cntr1, TARGET_B
 905          *                return
 906          *
 907          * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
 908          * TARGET_B is set to IDLE_LOOP2 as well
 909          *
 910          * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
 911          * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
 912          *
 913          * a little confusing, but it helps save precious space in the inst_rom
 914          * and sequencer rom and keeps the delays more accurate and reduces
 915          * overhead
 916          */
 917         if (afi_clocks <= 0x100) {
 918                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
 919                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), addr);
 920
 921                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
 922                 writel(RW_MGR_IDLE_LOOP1, addr);
 923
 924                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
 925                 writel(RW_MGR_IDLE_LOOP1, addr);
 926         } else {
 927                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
 928                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), addr);
 929
 930                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
 931                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer), addr);
 932
 933                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
 934                 writel(RW_MGR_IDLE_LOOP2, addr);
 935
 936                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
 937                 writel(RW_MGR_IDLE_LOOP2, addr);
 938
 939                 /* hack to get around compiler not being smart enough */
 940                 if (afi_clocks <= 0x10000) {
 941                         /* only need to run once */
 942                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
 943                         writel(RW_MGR_IDLE_LOOP2, addr);
 944                 } else {
 945                         do {
 946                                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
 947                                 writel(RW_MGR_IDLE_LOOP2, addr);
 948                         } while (c_loop-- != 0);
 949                 }
 950         }
 951         debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
 952 }
 953
 954 static void rw_mgr_mem_initialize(void)
 955 {
 956         uint32_t r;
 957         uint32_t addr;
 958
 959         debug("%s:%d\n", __func__, __LINE__);
 960
 961         /* The reset / cke part of initialization is broadcasted to all ranks */
 962         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_SET_CS_AND_ODT_MASK_OFFSET;
 963         writel(RW_MGR_RANK_ALL, addr);
 964
 965         /*
 966          * Here's how you load register for a loop
 967          * Counters are located @ 0x800
 968          * Jump address are located @ 0xC00
 969          * For both, registers 0 to 3 are selected using bits 3 and 2, like
 970          * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
 971          * I know this ain't pretty, but Avalon bus throws away the 2 least
 972          * significant bits
 973          */
 974
 975         /* start with memory RESET activated */
 976
 977         /* tINIT = 200us */
 978
 979         /*
 980          * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
 981          * If a and b are the number of iteration in 2 nested loops
 982          * it takes the following number of cycles to complete the operation:
 983          * number_of_cycles = ((2 + n) * a + 2) * b
 984          * where n is the number of instruction in the inner loop
 985          * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
 986          * b = 6A
 987          */
 988
 989         /* Load counters */
 990         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
 991         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL),
 992                addr);
 993         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
 994         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL),
 995                addr);
 996         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
 997         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL),
 998                addr);
 999
1000         /* Load jump address */
1001         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1002         writel(RW_MGR_INIT_RESET_0_CKE_0, addr);
1003
1004         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1005         writel(RW_MGR_INIT_RESET_0_CKE_0, addr);
1006
1007         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1008         writel(RW_MGR_INIT_RESET_0_CKE_0, addr);
1009
1010         /* Execute count instruction */
1011         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1012         writel(RW_MGR_INIT_RESET_0_CKE_0, addr);
1013
1014         /* indicate that memory is stable */
1015         addr = (u32)&phy_mgr_cfg->reset_mem_stbl;
1016         writel(1, addr);
1017
1018         /*
1019          * transition the RESET to high
1020          * Wait for 500us
1021          */
1022
1023         /*
1024          * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
1025          * If a and b are the number of iteration in 2 nested loops
1026          * it takes the following number of cycles to complete the operation
1027          * number_of_cycles = ((2 + n) * a + 2) * b
1028          * where n is the number of instruction in the inner loop
1029          * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
1030          * b = FF
1031          */
1032
1033         /* Load counters */
1034         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1035         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR0_VAL),
1036                addr);
1037         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1038         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR1_VAL),
1039                addr);
1040         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
1041         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR2_VAL),
1042                addr);
1043
1044         /* Load jump address */
1045         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1046         writel(RW_MGR_INIT_RESET_1_CKE_0, addr);
1047         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1048         writel(RW_MGR_INIT_RESET_1_CKE_0, addr);
1049         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1050         writel(RW_MGR_INIT_RESET_1_CKE_0, addr);
1051
1052         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1053         writel(RW_MGR_INIT_RESET_1_CKE_0, addr);
1054
1055         /* bring up clock enable */
1056
1057         /* tXRP < 250 ck cycles */
1058         delay_for_n_mem_clocks(250);
1059
1060         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
1061                 if (param->skip_ranks[r]) {
1062                         /* request to skip the rank */
1063                         continue;
1064                 }
1065
1066                 /* set rank */
1067                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
1068
1069                 /*
1070                  * USER Use Mirror-ed commands for odd ranks if address
1071                  * mirrorring is on
1072                  */
1073                 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
1074                         set_jump_as_return();
1075                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1076                         writel(RW_MGR_MRS2_MIRR, addr);
1077                         delay_for_n_mem_clocks(4);
1078                         set_jump_as_return();
1079                         writel(RW_MGR_MRS3_MIRR, addr);
1080                         delay_for_n_mem_clocks(4);
1081                         set_jump_as_return();
1082                         writel(RW_MGR_MRS1_MIRR, addr);
1083                         delay_for_n_mem_clocks(4);
1084                         set_jump_as_return();
1085                         writel(RW_MGR_MRS0_DLL_RESET_MIRR, addr);
1086                 } else {
1087                         set_jump_as_return();
1088                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1089                         writel(RW_MGR_MRS2, addr);
1090                         delay_for_n_mem_clocks(4);
1091                         set_jump_as_return();
1092                         writel(RW_MGR_MRS3, addr);
1093                         delay_for_n_mem_clocks(4);
1094                         set_jump_as_return();
1095                         writel(RW_MGR_MRS1, addr);
1096                         set_jump_as_return();
1097                         writel(RW_MGR_MRS0_DLL_RESET, addr);
1098                 }
1099                 set_jump_as_return();
1100                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1101                 writel(RW_MGR_ZQCL, addr);
1102
1103                 /* tZQinit = tDLLK = 512 ck cycles */
1104                 delay_for_n_mem_clocks(512);
1105         }
1106 }
1107
1108 /*
1109  * At the end of calibration we have to program the user settings in, and
1110  * USER  hand off the memory to the user.
1111  */
1112 static void rw_mgr_mem_handoff(void)
1113 {
1114         uint32_t r;
1115         uint32_t addr;
1116
1117         debug("%s:%d\n", __func__, __LINE__);
1118         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
1119                 if (param->skip_ranks[r])
1120                         /* request to skip the rank */
1121                         continue;
1122                 /* set rank */
1123                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
1124
1125                 /* precharge all banks ... */
1126                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1127                 writel(RW_MGR_PRECHARGE_ALL, addr);
1128
1129                 /* load up MR settings specified by user */
1130
1131                 /*
1132                  * Use Mirror-ed commands for odd ranks if address
1133                  * mirrorring is on
1134                  */
1135                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1136                 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
1137                         set_jump_as_return();
1138                         writel(RW_MGR_MRS2_MIRR, addr);
1139                         delay_for_n_mem_clocks(4);
1140                         set_jump_as_return();
1141                         writel(RW_MGR_MRS3_MIRR, addr);
1142                         delay_for_n_mem_clocks(4);
1143                         set_jump_as_return();
1144                         writel(RW_MGR_MRS1_MIRR, addr);
1145                         delay_for_n_mem_clocks(4);
1146                         set_jump_as_return();
1147                         writel(RW_MGR_MRS0_USER_MIRR, addr);
1148                 } else {
1149                         set_jump_as_return();
1150                         writel(RW_MGR_MRS2, addr);
1151                         delay_for_n_mem_clocks(4);
1152                         set_jump_as_return();
1153                         writel(RW_MGR_MRS3, addr);
1154                         delay_for_n_mem_clocks(4);
1155                         set_jump_as_return();
1156                         writel(RW_MGR_MRS1, addr);
1157                         delay_for_n_mem_clocks(4);
1158                         set_jump_as_return();
1159                         writel(RW_MGR_MRS0_USER, addr);
1160                 }
1161                 /*
1162                  * USER  need to wait tMOD (12CK or 15ns) time before issuing
1163                  * other commands, but we will have plenty of NIOS cycles before
1164                  * actual handoff so its okay.
1165                  */
1166         }
1167 }
1168
1169 /*
1170  * performs a guaranteed read on the patterns we are going to use during a
1171  * read test to ensure memory works
1172  */
1173 static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn,
1174         uint32_t group, uint32_t num_tries, uint32_t *bit_chk,
1175         uint32_t all_ranks)
1176 {
1177         uint32_t r, vg;
1178         uint32_t correct_mask_vg;
1179         uint32_t tmp_bit_chk;
1180         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1181                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1182         uint32_t addr;
1183         uint32_t base_rw_mgr;
1184
1185         *bit_chk = param->read_correct_mask;
1186         correct_mask_vg = param->read_correct_mask_vg;
1187
1188         for (r = rank_bgn; r < rank_end; r++) {
1189                 if (param->skip_ranks[r])
1190                         /* request to skip the rank */
1191                         continue;
1192
1193                 /* set rank */
1194                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1195
1196                 /* Load up a constant bursts of read commands */
1197                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1198                 writel(0x20, addr);
1199                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1200                 writel(RW_MGR_GUARANTEED_READ, addr);
1201
1202                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1203                 writel(0x20, addr);
1204                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1205                 writel(RW_MGR_GUARANTEED_READ_CONT, addr);
1206
1207                 tmp_bit_chk = 0;
1208                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1209                         /* reset the fifos to get pointers to known state */
1210
1211                         addr = (u32)&phy_mgr_cmd->fifo_reset;
1212                         writel(0, addr);
1213                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RESET_READ_DATAPATH_OFFSET;
1214                         writel(0, addr);
1215
1216                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1217                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1218
1219                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1220                         writel(RW_MGR_GUARANTEED_READ, addr +
1221                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1222                                 vg) << 2));
1223
1224                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
1225                         base_rw_mgr = readl(addr);
1226                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr));
1227
1228                         if (vg == 0)
1229                                 break;
1230                 }
1231                 *bit_chk &= tmp_bit_chk;
1232         }
1233
1234         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1235         writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
1236
1237         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1238         debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\
1239                    %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask,
1240                    (long unsigned int)(*bit_chk == param->read_correct_mask));
1241         return *bit_chk == param->read_correct_mask;
1242 }
1243
1244 static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks
1245         (uint32_t group, uint32_t num_tries, uint32_t *bit_chk)
1246 {
1247         return rw_mgr_mem_calibrate_read_test_patterns(0, group,
1248                 num_tries, bit_chk, 1);
1249 }
1250
1251 /* load up the patterns we are going to use during a read test */
1252 static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn,
1253         uint32_t all_ranks)
1254 {
1255         uint32_t r;
1256         uint32_t addr;
1257         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1258                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1259
1260         debug("%s:%d\n", __func__, __LINE__);
1261         for (r = rank_bgn; r < rank_end; r++) {
1262                 if (param->skip_ranks[r])
1263                         /* request to skip the rank */
1264                         continue;
1265
1266                 /* set rank */
1267                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1268
1269                 /* Load up a constant bursts */
1270                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1271                 writel(0x20, addr);
1272
1273                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1274                 writel(RW_MGR_GUARANTEED_WRITE_WAIT0, addr);
1275
1276                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1277                 writel(0x20, addr);
1278
1279                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1280                 writel(RW_MGR_GUARANTEED_WRITE_WAIT1, addr);
1281
1282                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
1283                 writel(0x04, addr);
1284
1285                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1286                 writel(RW_MGR_GUARANTEED_WRITE_WAIT2, addr);
1287
1288                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3;
1289                 writel(0x04, addr);
1290
1291                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
1292                 writel(RW_MGR_GUARANTEED_WRITE_WAIT3, addr);
1293
1294                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1295                 writel(RW_MGR_GUARANTEED_WRITE, addr);
1296         }
1297
1298         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1299 }
1300
1301 /*
1302  * try a read and see if it returns correct data back. has dummy reads
1303  * inserted into the mix used to align dqs enable. has more thorough checks
1304  * than the regular read test.
1305  */
1306 static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
1307         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1308         uint32_t all_groups, uint32_t all_ranks)
1309 {
1310         uint32_t r, vg;
1311         uint32_t correct_mask_vg;
1312         uint32_t tmp_bit_chk;
1313         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1314                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1315         uint32_t addr;
1316         uint32_t base_rw_mgr;
1317
1318         *bit_chk = param->read_correct_mask;
1319         correct_mask_vg = param->read_correct_mask_vg;
1320
1321         uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
1322                 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
1323
1324         for (r = rank_bgn; r < rank_end; r++) {
1325                 if (param->skip_ranks[r])
1326                         /* request to skip the rank */
1327                         continue;
1328
1329                 /* set rank */
1330                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1331
1332                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1333                 writel(0x10, addr);
1334
1335                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1336                 writel(RW_MGR_READ_B2B_WAIT1, addr);
1337
1338                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
1339                 writel(0x10, addr);
1340                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1341                 writel(RW_MGR_READ_B2B_WAIT2, addr);
1342
1343                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1344                 if (quick_read_mode)
1345                         writel(0x1, addr);
1346                         /* need at least two (1+1) reads to capture failures */
1347                 else if (all_groups)
1348                         writel(0x06, addr);
1349                 else
1350                         writel(0x32, addr);
1351
1352                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1353                 writel(RW_MGR_READ_B2B, addr);
1354                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3;
1355                 if (all_groups)
1356                         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
1357                                RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
1358                                addr);
1359                 else
1360                         writel(0x0, addr);
1361
1362                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
1363                 writel(RW_MGR_READ_B2B, addr);
1364
1365                 tmp_bit_chk = 0;
1366                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1367                         /* reset the fifos to get pointers to known state */
1368                         addr = (u32)&phy_mgr_cmd->fifo_reset;
1369                         writel(0, addr);
1370                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RESET_READ_DATAPATH_OFFSET;
1371                         writel(0, addr);
1372
1373                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1374                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1375
1376                         if (all_groups)
1377                                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET;
1378                         else
1379                                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1380
1381                         writel(RW_MGR_READ_B2B, addr +
1382                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1383                                vg) << 2));
1384
1385                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
1386                         base_rw_mgr = readl(addr);
1387                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
1388
1389                         if (vg == 0)
1390                                 break;
1391                 }
1392                 *bit_chk &= tmp_bit_chk;
1393         }
1394
1395         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1396         writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
1397
1398         if (all_correct) {
1399                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1400                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
1401                            (%u == %u) => %lu", __func__, __LINE__, group,
1402                            all_groups, *bit_chk, param->read_correct_mask,
1403                            (long unsigned int)(*bit_chk ==
1404                            param->read_correct_mask));
1405                 return *bit_chk == param->read_correct_mask;
1406         } else  {
1407                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1408                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
1409                            (%u != %lu) => %lu\n", __func__, __LINE__,
1410                            group, all_groups, *bit_chk, (long unsigned int)0,
1411                            (long unsigned int)(*bit_chk != 0x00));
1412                 return *bit_chk != 0x00;
1413         }
1414 }
1415
1416 static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
1417         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1418         uint32_t all_groups)
1419 {
1420         return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
1421                                               bit_chk, all_groups, 1);
1422 }
1423
1424 static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
1425 {
1426         uint32_t addr = (u32)&phy_mgr_cmd->inc_vfifo_hard_phy;
1427
1428         writel(grp, addr);
1429         (*v)++;
1430 }
1431
1432 static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
1433 {
1434         uint32_t i;
1435
1436         for (i = 0; i < VFIFO_SIZE-1; i++)
1437                 rw_mgr_incr_vfifo(grp, v);
1438 }
1439
1440 static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
1441 {
1442         uint32_t  v;
1443         uint32_t fail_cnt = 0;
1444         uint32_t test_status;
1445
1446         for (v = 0; v < VFIFO_SIZE; ) {
1447                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
1448                            __func__, __LINE__, v);
1449                 test_status = rw_mgr_mem_calibrate_read_test_all_ranks
1450                         (grp, 1, PASS_ONE_BIT, bit_chk, 0);
1451                 if (!test_status) {
1452                         fail_cnt++;
1453
1454                         if (fail_cnt == 2)
1455                                 break;
1456                 }
1457
1458                 /* fiddle with FIFO */
1459                 rw_mgr_incr_vfifo(grp, &v);
1460         }
1461
1462         if (v >= VFIFO_SIZE) {
1463                 /* no failing read found!! Something must have gone wrong */
1464                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
1465                            __func__, __LINE__);
1466                 return 0;
1467         } else {
1468                 return v;
1469         }
1470 }
1471
1472 static int find_working_phase(uint32_t *grp, uint32_t *bit_chk,
1473                               uint32_t dtaps_per_ptap, uint32_t *work_bgn,
1474                               uint32_t *v, uint32_t *d, uint32_t *p,
1475                               uint32_t *i, uint32_t *max_working_cnt)
1476 {
1477         uint32_t found_begin = 0;
1478         uint32_t tmp_delay = 0;
1479         uint32_t test_status;
1480
1481         for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
1482                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1483                 *work_bgn = tmp_delay;
1484                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1485
1486                 for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
1487                         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
1488                                 IO_DELAY_PER_OPA_TAP) {
1489                                 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1490
1491                                 test_status =
1492                                 rw_mgr_mem_calibrate_read_test_all_ranks
1493                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0);
1494
1495                                 if (test_status) {
1496                                         *max_working_cnt = 1;
1497                                         found_begin = 1;
1498                                         break;
1499                                 }
1500                         }
1501
1502                         if (found_begin)
1503                                 break;
1504
1505                         if (*p > IO_DQS_EN_PHASE_MAX)
1506                                 /* fiddle with FIFO */
1507                                 rw_mgr_incr_vfifo(*grp, v);
1508                 }
1509
1510                 if (found_begin)
1511                         break;
1512         }
1513
1514         if (*i >= VFIFO_SIZE) {
1515                 /* cannot find working solution */
1516                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
1517                            ptap/dtap\n", __func__, __LINE__);
1518                 return 0;
1519         } else {
1520                 return 1;
1521         }
1522 }
1523
1524 static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk,
1525                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1526                              uint32_t *p, uint32_t *max_working_cnt)
1527 {
1528         uint32_t found_begin = 0;
1529         uint32_t tmp_delay;
1530
1531         /* Special case code for backing up a phase */
1532         if (*p == 0) {
1533                 *p = IO_DQS_EN_PHASE_MAX;
1534                 rw_mgr_decr_vfifo(*grp, v);
1535         } else {
1536                 (*p)--;
1537         }
1538         tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
1539         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1540
1541         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
1542                 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1543                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1544
1545                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1546                                                              PASS_ONE_BIT,
1547                                                              bit_chk, 0)) {
1548                         found_begin = 1;
1549                         *work_bgn = tmp_delay;
1550                         break;
1551                 }
1552         }
1553
1554         /* We have found a working dtap before the ptap found above */
1555         if (found_begin == 1)
1556                 (*max_working_cnt)++;
1557
1558         /*
1559          * Restore VFIFO to old state before we decremented it
1560          * (if needed).
1561          */
1562         (*p)++;
1563         if (*p > IO_DQS_EN_PHASE_MAX) {
1564                 *p = 0;
1565                 rw_mgr_incr_vfifo(*grp, v);
1566         }
1567
1568         scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0);
1569 }
1570
1571 static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk,
1572                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1573                              uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
1574                              uint32_t *work_end)
1575 {
1576         uint32_t found_end = 0;
1577
1578         (*p)++;
1579         *work_end += IO_DELAY_PER_OPA_TAP;
1580         if (*p > IO_DQS_EN_PHASE_MAX) {
1581                 /* fiddle with FIFO */
1582                 *p = 0;
1583                 rw_mgr_incr_vfifo(*grp, v);
1584         }
1585
1586         for (; *i < VFIFO_SIZE + 1; (*i)++) {
1587                 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
1588                         += IO_DELAY_PER_OPA_TAP) {
1589                         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1590
1591                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1592                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
1593                                 found_end = 1;
1594                                 break;
1595                         } else {
1596                                 (*max_working_cnt)++;
1597                         }
1598                 }
1599
1600                 if (found_end)
1601                         break;
1602
1603                 if (*p > IO_DQS_EN_PHASE_MAX) {
1604                         /* fiddle with FIFO */
1605                         rw_mgr_incr_vfifo(*grp, v);
1606                         *p = 0;
1607                 }
1608         }
1609
1610         if (*i >= VFIFO_SIZE + 1) {
1611                 /* cannot see edge of failing read */
1612                 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
1613                            failed\n", __func__, __LINE__);
1614                 return 0;
1615         } else {
1616                 return 1;
1617         }
1618 }
1619
1620 static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk,
1621                                   uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1622                                   uint32_t *p, uint32_t *work_mid,
1623                                   uint32_t *work_end)
1624 {
1625         int i;
1626         int tmp_delay = 0;
1627
1628         *work_mid = (*work_bgn + *work_end) / 2;
1629
1630         debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
1631                    *work_bgn, *work_end, *work_mid);
1632         /* Get the middle delay to be less than a VFIFO delay */
1633         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX;
1634                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1635                 ;
1636         debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
1637         while (*work_mid > tmp_delay)
1638                 *work_mid -= tmp_delay;
1639         debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid);
1640
1641         tmp_delay = 0;
1642         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid;
1643                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1644                 ;
1645         tmp_delay -= IO_DELAY_PER_OPA_TAP;
1646         debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay);
1647         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++,
1648                 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP)
1649                 ;
1650         debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay);
1651
1652         scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1);
1653         scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1654
1655         /*
1656          * push vfifo until we can successfully calibrate. We can do this
1657          * because the largest possible margin in 1 VFIFO cycle.
1658          */
1659         for (i = 0; i < VFIFO_SIZE; i++) {
1660                 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
1661                            *v);
1662                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1663                                                              PASS_ONE_BIT,
1664                                                              bit_chk, 0)) {
1665                         break;
1666                 }
1667
1668                 /* fiddle with FIFO */
1669                 rw_mgr_incr_vfifo(*grp, v);
1670         }
1671
1672         if (i >= VFIFO_SIZE) {
1673                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \
1674                            failed\n", __func__, __LINE__);
1675                 return 0;
1676         } else {
1677                 return 1;
1678         }
1679 }
1680
1681 /* find a good dqs enable to use */
1682 static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
1683 {
1684         uint32_t v, d, p, i;
1685         uint32_t max_working_cnt;
1686         uint32_t bit_chk;
1687         uint32_t dtaps_per_ptap;
1688         uint32_t work_bgn, work_mid, work_end;
1689         uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
1690         uint32_t addr;
1691
1692         debug("%s:%d %u\n", __func__, __LINE__, grp);
1693
1694         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
1695
1696         scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
1697         scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
1698
1699         /* ************************************************************** */
1700         /* * Step 0 : Determine number of delay taps for each phase tap * */
1701         dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1702
1703         /* ********************************************************* */
1704         /* * Step 1 : First push vfifo until we get a failing read * */
1705         v = find_vfifo_read(grp, &bit_chk);
1706
1707         max_working_cnt = 0;
1708
1709         /* ******************************************************** */
1710         /* * step 2: find first working phase, increment in ptaps * */
1711         work_bgn = 0;
1712         if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
1713                                 &p, &i, &max_working_cnt) == 0)
1714                 return 0;
1715
1716         work_end = work_bgn;
1717
1718         /*
1719          * If d is 0 then the working window covers a phase tap and
1720          * we can follow the old procedure otherwise, we've found the beginning,
1721          * and we need to increment the dtaps until we find the end.
1722          */
1723         if (d == 0) {
1724                 /* ********************************************************* */
1725                 /* * step 3a: if we have room, back off by one and
1726                 increment in dtaps * */
1727
1728                 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1729                                  &max_working_cnt);
1730
1731                 /* ********************************************************* */
1732                 /* * step 4a: go forward from working phase to non working
1733                 phase, increment in ptaps * */
1734                 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1735                                          &i, &max_working_cnt, &work_end) == 0)
1736                         return 0;
1737
1738                 /* ********************************************************* */
1739                 /* * step 5a:  back off one from last, increment in dtaps  * */
1740
1741                 /* Special case code for backing up a phase */
1742                 if (p == 0) {
1743                         p = IO_DQS_EN_PHASE_MAX;
1744                         rw_mgr_decr_vfifo(grp, &v);
1745                 } else {
1746                         p = p - 1;
1747                 }
1748
1749                 work_end -= IO_DELAY_PER_OPA_TAP;
1750                 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1751
1752                 /* * The actual increment of dtaps is done outside of
1753                 the if/else loop to share code */
1754                 d = 0;
1755
1756                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
1757                            vfifo=%u ptap=%u\n", __func__, __LINE__,
1758                            v, p);
1759         } else {
1760                 /* ******************************************************* */
1761                 /* * step 3-5b:  Find the right edge of the window using
1762                 delay taps   * */
1763                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
1764                            ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
1765                            v, p, d, work_bgn);
1766
1767                 work_end = work_bgn;
1768
1769                 /* * The actual increment of dtaps is done outside of the
1770                 if/else loop to share code */
1771
1772                 /* Only here to counterbalance a subtract later on which is
1773                 not needed if this branch of the algorithm is taken */
1774                 max_working_cnt++;
1775         }
1776
1777         /* The dtap increment to find the failing edge is done here */
1778         for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
1779                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1780                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1781                                    end-2: dtap=%u\n", __func__, __LINE__, d);
1782                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1783
1784                         if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1785                                                                       PASS_ONE_BIT,
1786                                                                       &bit_chk, 0)) {
1787                                 break;
1788                         }
1789         }
1790
1791         /* Go back to working dtap */
1792         if (d != 0)
1793                 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1794
1795         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
1796                    ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
1797                    v, p, d-1, work_end);
1798
1799         if (work_end < work_bgn) {
1800                 /* nil range */
1801                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
1802                            failed\n", __func__, __LINE__);
1803                 return 0;
1804         }
1805
1806         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
1807                    __func__, __LINE__, work_bgn, work_end);
1808
1809         /* *************************************************************** */
1810         /*
1811          * * We need to calculate the number of dtaps that equal a ptap
1812          * * To do that we'll back up a ptap and re-find the edge of the
1813          * * window using dtaps
1814          */
1815
1816         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
1817                    for tracking\n", __func__, __LINE__);
1818
1819         /* Special case code for backing up a phase */
1820         if (p == 0) {
1821                 p = IO_DQS_EN_PHASE_MAX;
1822                 rw_mgr_decr_vfifo(grp, &v);
1823                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1824                            cycle/phase: v=%u p=%u\n", __func__, __LINE__,
1825                            v, p);
1826         } else {
1827                 p = p - 1;
1828                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1829                            phase only: v=%u p=%u", __func__, __LINE__,
1830                            v, p);
1831         }
1832
1833         scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1834
1835         /*
1836          * Increase dtap until we first see a passing read (in case the
1837          * window is smaller than a ptap),
1838          * and then a failing read to mark the edge of the window again
1839          */
1840
1841         /* Find a passing read */
1842         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
1843                    __func__, __LINE__);
1844         found_passing_read = 0;
1845         found_failing_read = 0;
1846         initial_failing_dtap = d;
1847         for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
1848                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
1849                            read d=%u\n", __func__, __LINE__, d);
1850                 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1851
1852                 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1853                                                              PASS_ONE_BIT,
1854                                                              &bit_chk, 0)) {
1855                         found_passing_read = 1;
1856                         break;
1857                 }
1858         }
1859
1860         if (found_passing_read) {
1861                 /* Find a failing read */
1862                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
1863                            read\n", __func__, __LINE__);
1864                 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
1865                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1866                                    testing read d=%u\n", __func__, __LINE__, d);
1867                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1868
1869                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1870                                 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
1871                                 found_failing_read = 1;
1872                                 break;
1873                         }
1874                 }
1875         } else {
1876                 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
1877                            calculate dtaps", __func__, __LINE__);
1878                 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
1879         }
1880
1881         /*
1882          * The dynamically calculated dtaps_per_ptap is only valid if we
1883          * found a passing/failing read. If we didn't, it means d hit the max
1884          * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
1885          * statically calculated value.
1886          */
1887         if (found_passing_read && found_failing_read)
1888                 dtaps_per_ptap = d - initial_failing_dtap;
1889
1890         addr = (u32)&sdr_reg_file->dtaps_per_ptap;
1891         writel(dtaps_per_ptap, addr);
1892         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
1893                    - %u = %u",  __func__, __LINE__, d,
1894                    initial_failing_dtap, dtaps_per_ptap);
1895
1896         /* ******************************************** */
1897         /* * step 6:  Find the centre of the window   * */
1898         if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1899                                    &work_mid, &work_end) == 0)
1900                 return 0;
1901
1902         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \
1903                    vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__,
1904                    v, p-1, d);
1905         return 1;
1906 }
1907
1908 /*
1909  * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
1910  * dq_in_delay values
1911  */
1912 static uint32_t
1913 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
1914 (uint32_t write_group, uint32_t read_group, uint32_t test_bgn)
1915 {
1916         uint32_t found;
1917         uint32_t i;
1918         uint32_t p;
1919         uint32_t d;
1920         uint32_t r;
1921         uint32_t addr;
1922
1923         const uint32_t delay_step = IO_IO_IN_DELAY_MAX /
1924                 (RW_MGR_MEM_DQ_PER_READ_DQS-1);
1925                 /* we start at zero, so have one less dq to devide among */
1926
1927         debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group,
1928               test_bgn);
1929
1930         /* try different dq_in_delays since the dq path is shorter than dqs */
1931
1932         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1933              r += NUM_RANKS_PER_SHADOW_REG) {
1934                 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1935                         i++, p++, d += delay_step) {
1936                         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\
1937                                    vfifo_find_dqs_", __func__, __LINE__);
1938                         debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ",
1939                                write_group, read_group);
1940                         debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d);
1941                         scc_mgr_set_dq_in_delay(write_group, p, d);
1942                         scc_mgr_load_dq(p);
1943                 }
1944                 addr = (u32)&sdr_scc_mgr->update;
1945                 writel(0, addr);
1946         }
1947
1948         found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group);
1949
1950         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\
1951                    en_phase_sweep_dq", __func__, __LINE__);
1952         debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \
1953                    chain to zero\n", write_group, read_group, found);
1954
1955         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1956              r += NUM_RANKS_PER_SHADOW_REG) {
1957                 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1958                         i++, p++) {
1959                         scc_mgr_set_dq_in_delay(write_group, p, 0);
1960                         scc_mgr_load_dq(p);
1961                 }
1962                 addr = (u32)&sdr_scc_mgr->update;
1963                 writel(0, addr);
1964         }
1965
1966         return found;
1967 }
1968
1969 /* per-bit deskew DQ and center */
1970 static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
1971         uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
1972         uint32_t use_read_test, uint32_t update_fom)
1973 {
1974         uint32_t i, p, d, min_index;
1975         /*
1976          * Store these as signed since there are comparisons with
1977          * signed numbers.
1978          */
1979         uint32_t bit_chk;
1980         uint32_t sticky_bit_chk;
1981         int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1982         int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1983         int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
1984         int32_t mid;
1985         int32_t orig_mid_min, mid_min;
1986         int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
1987                 final_dqs_en;
1988         int32_t dq_margin, dqs_margin;
1989         uint32_t stop;
1990         uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
1991         uint32_t addr;
1992
1993         debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
1994
1995         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
1996         start_dqs = readl(addr + (read_group << 2));
1997         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
1998                 start_dqs_en = readl(addr + ((read_group << 2)
1999                                      - IO_DQS_EN_DELAY_OFFSET));
2000
2001         /* set the left and right edge of each bit to an illegal value */
2002         /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
2003         sticky_bit_chk = 0;
2004         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2005                 left_edge[i]  = IO_IO_IN_DELAY_MAX + 1;
2006                 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
2007         }
2008
2009         addr = (u32)&sdr_scc_mgr->update;
2010         /* Search for the left edge of the window for each bit */
2011         for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
2012                 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
2013
2014                 writel(0, addr);
2015
2016                 /*
2017                  * Stop searching when the read test doesn't pass AND when
2018                  * we've seen a passing read on every bit.
2019                  */
2020                 if (use_read_test) {
2021                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
2022                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
2023                                 &bit_chk, 0, 0);
2024                 } else {
2025                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2026                                                         0, PASS_ONE_BIT,
2027                                                         &bit_chk, 0);
2028                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
2029                                 (read_group - (write_group *
2030                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
2031                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
2032                         stop = (bit_chk == 0);
2033                 }
2034                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2035                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
2036                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
2037                            && %u", __func__, __LINE__, d,
2038                            sticky_bit_chk,
2039                         param->read_correct_mask, stop);
2040
2041                 if (stop == 1) {
2042                         break;
2043                 } else {
2044                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2045                                 if (bit_chk & 1) {
2046                                         /* Remember a passing test as the
2047                                         left_edge */
2048                                         left_edge[i] = d;
2049                                 } else {
2050                                         /* If a left edge has not been seen yet,
2051                                         then a future passing test will mark
2052                                         this edge as the right edge */
2053                                         if (left_edge[i] ==
2054                                                 IO_IO_IN_DELAY_MAX + 1) {
2055                                                 right_edge[i] = -(d + 1);
2056                                         }
2057                                 }
2058                                 bit_chk = bit_chk >> 1;
2059                         }
2060                 }
2061         }
2062
2063         /* Reset DQ delay chains to 0 */
2064         scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, 0);
2065         sticky_bit_chk = 0;
2066         for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
2067                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2068                            %d right_edge[%u]: %d\n", __func__, __LINE__,
2069                            i, left_edge[i], i, right_edge[i]);
2070
2071                 /*
2072                  * Check for cases where we haven't found the left edge,
2073                  * which makes our assignment of the the right edge invalid.
2074                  * Reset it to the illegal value.
2075                  */
2076                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
2077                         right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
2078                         right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
2079                         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
2080                                    right_edge[%u]: %d\n", __func__, __LINE__,
2081                                    i, right_edge[i]);
2082                 }
2083
2084                 /*
2085                  * Reset sticky bit (except for bits where we have seen
2086                  * both the left and right edge).
2087                  */
2088                 sticky_bit_chk = sticky_bit_chk << 1;
2089                 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
2090                     (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
2091                         sticky_bit_chk = sticky_bit_chk | 1;
2092                 }
2093
2094                 if (i == 0)
2095                         break;
2096         }
2097
2098         addr = (u32)&sdr_scc_mgr->update;
2099         /* Search for the right edge of the window for each bit */
2100         for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
2101                 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
2102                 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2103                         uint32_t delay = d + start_dqs_en;
2104                         if (delay > IO_DQS_EN_DELAY_MAX)
2105                                 delay = IO_DQS_EN_DELAY_MAX;
2106                         scc_mgr_set_dqs_en_delay(read_group, delay);
2107                 }
2108                 scc_mgr_load_dqs(read_group);
2109
2110                 writel(0, addr);
2111
2112                 /*
2113                  * Stop searching when the read test doesn't pass AND when
2114                  * we've seen a passing read on every bit.
2115                  */
2116                 if (use_read_test) {
2117                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
2118                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
2119                                 &bit_chk, 0, 0);
2120                 } else {
2121                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2122                                                         0, PASS_ONE_BIT,
2123                                                         &bit_chk, 0);
2124                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
2125                                 (read_group - (write_group *
2126                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
2127                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
2128                         stop = (bit_chk == 0);
2129                 }
2130                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2131                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
2132
2133                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
2134                            %u && %u", __func__, __LINE__, d,
2135                            sticky_bit_chk, param->read_correct_mask, stop);
2136
2137                 if (stop == 1) {
2138                         break;
2139                 } else {
2140                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2141                                 if (bit_chk & 1) {
2142                                         /* Remember a passing test as
2143                                         the right_edge */
2144                                         right_edge[i] = d;
2145                                 } else {
2146                                         if (d != 0) {
2147                                                 /* If a right edge has not been
2148                                                 seen yet, then a future passing
2149                                                 test will mark this edge as the
2150                                                 left edge */
2151                                                 if (right_edge[i] ==
2152                                                 IO_IO_IN_DELAY_MAX + 1) {
2153                                                         left_edge[i] = -(d + 1);
2154                                                 }
2155                                         } else {
2156                                                 /* d = 0 failed, but it passed
2157                                                 when testing the left edge,
2158                                                 so it must be marginal,
2159                                                 set it to -1 */
2160                                                 if (right_edge[i] ==
2161                                                         IO_IO_IN_DELAY_MAX + 1 &&
2162                                                         left_edge[i] !=
2163                                                         IO_IO_IN_DELAY_MAX
2164                                                         + 1) {
2165                                                         right_edge[i] = -1;
2166                                                 }
2167                                                 /* If a right edge has not been
2168                                                 seen yet, then a future passing
2169                                                 test will mark this edge as the
2170                                                 left edge */
2171                                                 else if (right_edge[i] ==
2172                                                         IO_IO_IN_DELAY_MAX +
2173                                                         1) {
2174                                                         left_edge[i] = -(d + 1);
2175                                                 }
2176                                         }
2177                                 }
2178
2179                                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
2180                                            d=%u]: ", __func__, __LINE__, d);
2181                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
2182                                            (int)(bit_chk & 1), i, left_edge[i]);
2183                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2184                                            right_edge[i]);
2185                                 bit_chk = bit_chk >> 1;
2186                         }
2187                 }
2188         }
2189
2190         /* Check that all bits have a window */
2191         addr = (u32)&sdr_scc_mgr->update;
2192         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2193                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2194                            %d right_edge[%u]: %d", __func__, __LINE__,
2195                            i, left_edge[i], i, right_edge[i]);
2196                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
2197                         == IO_IO_IN_DELAY_MAX + 1)) {
2198                         /*
2199                          * Restore delay chain settings before letting the loop
2200                          * in rw_mgr_mem_calibrate_vfifo to retry different
2201                          * dqs/ck relationships.
2202                          */
2203                         scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
2204                         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2205                                 scc_mgr_set_dqs_en_delay(read_group,
2206                                                          start_dqs_en);
2207                         }
2208                         scc_mgr_load_dqs(read_group);
2209                         writel(0, addr);
2210
2211                         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
2212                                    find edge [%u]: %d %d", __func__, __LINE__,
2213                                    i, left_edge[i], right_edge[i]);
2214                         if (use_read_test) {
2215                                 set_failing_group_stage(read_group *
2216                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2217                                         CAL_STAGE_VFIFO,
2218                                         CAL_SUBSTAGE_VFIFO_CENTER);
2219                         } else {
2220                                 set_failing_group_stage(read_group *
2221                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2222                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2223                                         CAL_SUBSTAGE_VFIFO_CENTER);
2224                         }
2225                         return 0;
2226                 }
2227         }
2228
2229         /* Find middle of window for each DQ bit */
2230         mid_min = left_edge[0] - right_edge[0];
2231         min_index = 0;
2232         for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2233                 mid = left_edge[i] - right_edge[i];
2234                 if (mid < mid_min) {
2235                         mid_min = mid;
2236                         min_index = i;
2237                 }
2238         }
2239
2240         /*
2241          * -mid_min/2 represents the amount that we need to move DQS.
2242          * If mid_min is odd and positive we'll need to add one to
2243          * make sure the rounding in further calculations is correct
2244          * (always bias to the right), so just add 1 for all positive values.
2245          */
2246         if (mid_min > 0)
2247                 mid_min++;
2248
2249         mid_min = mid_min / 2;
2250
2251         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
2252                    __func__, __LINE__, mid_min, min_index);
2253
2254         /* Determine the amount we can change DQS (which is -mid_min) */
2255         orig_mid_min = mid_min;
2256         new_dqs = start_dqs - mid_min;
2257         if (new_dqs > IO_DQS_IN_DELAY_MAX)
2258                 new_dqs = IO_DQS_IN_DELAY_MAX;
2259         else if (new_dqs < 0)
2260                 new_dqs = 0;
2261
2262         mid_min = start_dqs - new_dqs;
2263         debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
2264                    mid_min, new_dqs);
2265
2266         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2267                 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
2268                         mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
2269                 else if (start_dqs_en - mid_min < 0)
2270                         mid_min += start_dqs_en - mid_min;
2271         }
2272         new_dqs = start_dqs - mid_min;
2273
2274         debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
2275                    new_dqs=%d mid_min=%d\n", start_dqs,
2276                    IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
2277                    new_dqs, mid_min);
2278
2279         /* Initialize data for export structures */
2280         dqs_margin = IO_IO_IN_DELAY_MAX + 1;
2281         dq_margin  = IO_IO_IN_DELAY_MAX + 1;
2282
2283         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
2284         /* add delay to bring centre of all DQ windows to the same "level" */
2285         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
2286                 /* Use values before divide by 2 to reduce round off error */
2287                 shift_dq = (left_edge[i] - right_edge[i] -
2288                         (left_edge[min_index] - right_edge[min_index]))/2  +
2289                         (orig_mid_min - mid_min);
2290
2291                 debug_cond(DLEVEL == 2, "vfifo_center: before: \
2292                            shift_dq[%u]=%d\n", i, shift_dq);
2293
2294                 temp_dq_in_delay1 = readl(addr + (p << 2));
2295                 temp_dq_in_delay2 = readl(addr + (i << 2));
2296
2297                 if (shift_dq + (int32_t)temp_dq_in_delay1 >
2298                         (int32_t)IO_IO_IN_DELAY_MAX) {
2299                         shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
2300                 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
2301                         shift_dq = -(int32_t)temp_dq_in_delay1;
2302                 }
2303                 debug_cond(DLEVEL == 2, "vfifo_center: after: \
2304                            shift_dq[%u]=%d\n", i, shift_dq);
2305                 final_dq[i] = temp_dq_in_delay1 + shift_dq;
2306                 scc_mgr_set_dq_in_delay(write_group, p, final_dq[i]);
2307                 scc_mgr_load_dq(p);
2308
2309                 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
2310                            left_edge[i] - shift_dq + (-mid_min),
2311                            right_edge[i] + shift_dq - (-mid_min));
2312                 /* To determine values for export structures */
2313                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2314                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
2315
2316                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2317                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2318         }
2319
2320         final_dqs = new_dqs;
2321         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2322                 final_dqs_en = start_dqs_en - mid_min;
2323
2324         /* Move DQS-en */
2325         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2326                 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
2327                 scc_mgr_load_dqs(read_group);
2328         }
2329
2330         /* Move DQS */
2331         scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
2332         scc_mgr_load_dqs(read_group);
2333         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
2334                    dqs_margin=%d", __func__, __LINE__,
2335                    dq_margin, dqs_margin);
2336
2337         /*
2338          * Do not remove this line as it makes sure all of our decisions
2339          * have been applied. Apply the update bit.
2340          */
2341         addr = (u32)&sdr_scc_mgr->update;
2342         writel(0, addr);
2343
2344         return (dq_margin >= 0) && (dqs_margin >= 0);
2345 }
2346
2347 /*
2348  * calibrate the read valid prediction FIFO.
2349  *
2350  *  - read valid prediction will consist of finding a good DQS enable phase,
2351  * DQS enable delay, DQS input phase, and DQS input delay.
2352  *  - we also do a per-bit deskew on the DQ lines.
2353  */
2354 static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group,
2355                                            uint32_t test_bgn)
2356 {
2357         uint32_t p, d, rank_bgn, sr;
2358         uint32_t dtaps_per_ptap;
2359         uint32_t tmp_delay;
2360         uint32_t bit_chk;
2361         uint32_t grp_calibrated;
2362         uint32_t write_group, write_test_bgn;
2363         uint32_t failed_substage;
2364
2365         debug("%s:%d: %u %u\n", __func__, __LINE__, read_group, test_bgn);
2366
2367         /* update info for sims */
2368         reg_file_set_stage(CAL_STAGE_VFIFO);
2369
2370         write_group = read_group;
2371         write_test_bgn = test_bgn;
2372
2373         /* USER Determine number of delay taps for each phase tap */
2374         dtaps_per_ptap = 0;
2375         tmp_delay = 0;
2376         while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
2377                 dtaps_per_ptap++;
2378                 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
2379         }
2380         dtaps_per_ptap--;
2381         tmp_delay = 0;
2382
2383         /* update info for sims */
2384         reg_file_set_group(read_group);
2385
2386         grp_calibrated = 0;
2387
2388         reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
2389         failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
2390
2391         for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) {
2392                 /*
2393                  * In RLDRAMX we may be messing the delay of pins in
2394                  * the same write group but outside of the current read
2395                  * the group, but that's ok because we haven't
2396                  * calibrated output side yet.
2397                  */
2398                 if (d > 0) {
2399                         scc_mgr_apply_group_all_out_delay_add_all_ranks
2400                         (write_group, write_test_bgn, d);
2401                 }
2402
2403                 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0;
2404                         p++) {
2405                         /* set a particular dqdqs phase */
2406                         scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p);
2407
2408                         debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \
2409                                    p=%u d=%u\n", __func__, __LINE__,
2410                                    read_group, p, d);
2411
2412                         /*
2413                          * Load up the patterns used by read calibration
2414                          * using current DQDQS phase.
2415                          */
2416                         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2417                         if (!(gbl->phy_debug_mode_flags &
2418                                 PHY_DEBUG_DISABLE_GUARANTEED_READ)) {
2419                                 if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks
2420                                     (read_group, 1, &bit_chk)) {
2421                                         debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:",
2422                                                    __func__, __LINE__);
2423                                         debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n",
2424                                                    read_group, p, d);
2425                                         break;
2426                                 }
2427                         }
2428
2429 /* case:56390 */
2430                         grp_calibrated = 1;
2431                 if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
2432                     (write_group, read_group, test_bgn)) {
2433                                 /*
2434                                  * USER Read per-bit deskew can be done on a
2435                                  * per shadow register basis.
2436                                  */
2437                                 for (rank_bgn = 0, sr = 0;
2438                                         rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2439                                         rank_bgn += NUM_RANKS_PER_SHADOW_REG,
2440                                         ++sr) {
2441                                         /*
2442                                          * Determine if this set of ranks
2443                                          * should be skipped entirely.
2444                                          */
2445                                         if (!param->skip_shadow_regs[sr]) {
2446                                                 /*
2447                                                  * If doing read after write
2448                                                  * calibration, do not update
2449                                                  * FOM, now - do it then.
2450                                                  */
2451                                         if (!rw_mgr_mem_calibrate_vfifo_center
2452                                                 (rank_bgn, write_group,
2453                                                 read_group, test_bgn, 1, 0)) {
2454                                                         grp_calibrated = 0;
2455                                                         failed_substage =
2456                                                 CAL_SUBSTAGE_VFIFO_CENTER;
2457                                                 }
2458                                         }
2459                                 }
2460                         } else {
2461                                 grp_calibrated = 0;
2462                                 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
2463                         }
2464                 }
2465         }
2466
2467         if (grp_calibrated == 0) {
2468                 set_failing_group_stage(write_group, CAL_STAGE_VFIFO,
2469                                         failed_substage);
2470                 return 0;
2471         }
2472
2473         /*
2474          * Reset the delay chains back to zero if they have moved > 1
2475          * (check for > 1 because loop will increase d even when pass in
2476          * first case).
2477          */
2478         if (d > 2)
2479                 scc_mgr_zero_group(write_group, write_test_bgn, 1);
2480
2481         return 1;
2482 }
2483
2484 /* VFIFO Calibration -- Read Deskew Calibration after write deskew */
2485 static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
2486                                                uint32_t test_bgn)
2487 {
2488         uint32_t rank_bgn, sr;
2489         uint32_t grp_calibrated;
2490         uint32_t write_group;
2491
2492         debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
2493
2494         /* update info for sims */
2495
2496         reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
2497         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
2498
2499         write_group = read_group;
2500
2501         /* update info for sims */
2502         reg_file_set_group(read_group);
2503
2504         grp_calibrated = 1;
2505         /* Read per-bit deskew can be done on a per shadow register basis */
2506         for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2507                 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
2508                 /* Determine if this set of ranks should be skipped entirely */
2509                 if (!param->skip_shadow_regs[sr]) {
2510                 /* This is the last calibration round, update FOM here */
2511                         if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
2512                                                                 write_group,
2513                                                                 read_group,
2514                                                                 test_bgn, 0,
2515                                                                 1)) {
2516                                 grp_calibrated = 0;
2517                         }
2518                 }
2519         }
2520
2521
2522         if (grp_calibrated == 0) {
2523                 set_failing_group_stage(write_group,
2524                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2525                                         CAL_SUBSTAGE_VFIFO_CENTER);
2526                 return 0;
2527         }
2528
2529         return 1;
2530 }
2531
2532 /* Calibrate LFIFO to find smallest read latency */
2533 static uint32_t rw_mgr_mem_calibrate_lfifo(void)
2534 {
2535         uint32_t found_one;
2536         uint32_t bit_chk;
2537         uint32_t addr;
2538
2539         debug("%s:%d\n", __func__, __LINE__);
2540
2541         /* update info for sims */
2542         reg_file_set_stage(CAL_STAGE_LFIFO);
2543         reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
2544
2545         /* Load up the patterns used by read calibration for all ranks */
2546         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2547         found_one = 0;
2548
2549         addr = (u32)&phy_mgr_cfg->phy_rlat;
2550         do {
2551                 writel(gbl->curr_read_lat, addr);
2552                 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
2553                            __func__, __LINE__, gbl->curr_read_lat);
2554
2555                 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
2556                                                               NUM_READ_TESTS,
2557                                                               PASS_ALL_BITS,
2558                                                               &bit_chk, 1)) {
2559                         break;
2560                 }
2561
2562                 found_one = 1;
2563                 /* reduce read latency and see if things are working */
2564                 /* correctly */
2565                 gbl->curr_read_lat--;
2566         } while (gbl->curr_read_lat > 0);
2567
2568         /* reset the fifos to get pointers to known state */
2569
2570         addr = (u32)&phy_mgr_cmd->fifo_reset;
2571         writel(0, addr);
2572
2573         if (found_one) {
2574                 /* add a fudge factor to the read latency that was determined */
2575                 gbl->curr_read_lat += 2;
2576                 addr = (u32)&phy_mgr_cfg->phy_rlat;
2577                 writel(gbl->curr_read_lat, addr);
2578                 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
2579                            read_lat=%u\n", __func__, __LINE__,
2580                            gbl->curr_read_lat);
2581                 return 1;
2582         } else {
2583                 set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
2584                                         CAL_SUBSTAGE_READ_LATENCY);
2585
2586                 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
2587                            read_lat=%u\n", __func__, __LINE__,
2588                            gbl->curr_read_lat);
2589                 return 0;
2590         }
2591 }
2592
2593 /*
2594  * issue write test command.
2595  * two variants are provided. one that just tests a write pattern and
2596  * another that tests datamask functionality.
2597  */
2598 static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
2599                                                   uint32_t test_dm)
2600 {
2601         uint32_t mcc_instruction;
2602         uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
2603                 ENABLE_SUPER_QUICK_CALIBRATION);
2604         uint32_t rw_wl_nop_cycles;
2605         uint32_t addr;
2606
2607         /*
2608          * Set counter and jump addresses for the right
2609          * number of NOP cycles.
2610          * The number of supported NOP cycles can range from -1 to infinity
2611          * Three different cases are handled:
2612          *
2613          * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
2614          *    mechanism will be used to insert the right number of NOPs
2615          *
2616          * 2. For a number of NOP cycles equals to 0, the micro-instruction
2617          *    issuing the write command will jump straight to the
2618          *    micro-instruction that turns on DQS (for DDRx), or outputs write
2619          *    data (for RLD), skipping
2620          *    the NOP micro-instruction all together
2621          *
2622          * 3. A number of NOP cycles equal to -1 indicates that DQS must be
2623          *    turned on in the same micro-instruction that issues the write
2624          *    command. Then we need
2625          *    to directly jump to the micro-instruction that sends out the data
2626          *
2627          * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
2628          *       (2 and 3). One jump-counter (0) is used to perform multiple
2629          *       write-read operations.
2630          *       one counter left to issue this command in "multiple-group" mode
2631          */
2632
2633         rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
2634
2635         if (rw_wl_nop_cycles == -1) {
2636                 /*
2637                  * CNTR 2 - We want to execute the special write operation that
2638                  * turns on DQS right away and then skip directly to the
2639                  * instruction that sends out the data. We set the counter to a
2640                  * large number so that the jump is always taken.
2641                  */
2642                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
2643                 writel(0xFF, addr);
2644
2645                 /* CNTR 3 - Not used */
2646                 if (test_dm) {
2647                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
2648                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2649                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
2650                                addr);
2651                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2652                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2653                                addr);
2654                 } else {
2655                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
2656                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2657                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA, addr);
2658                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2659                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, addr);
2660                 }
2661         } else if (rw_wl_nop_cycles == 0) {
2662                 /*
2663                  * CNTR 2 - We want to skip the NOP operation and go straight
2664                  * to the DQS enable instruction. We set the counter to a large
2665                  * number so that the jump is always taken.
2666                  */
2667                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
2668                 writel(0xFF, addr);
2669
2670                 /* CNTR 3 - Not used */
2671                 if (test_dm) {
2672                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2673                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2674                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
2675                                addr);
2676                 } else {
2677                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2678                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2679                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS, addr);
2680                 }
2681         } else {
2682                 /*
2683                  * CNTR 2 - In this case we want to execute the next instruction
2684                  * and NOT take the jump. So we set the counter to 0. The jump
2685                  * address doesn't count.
2686                  */
2687                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
2688                 writel(0x0, addr);
2689                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2690                 writel(0x0, addr);
2691
2692                 /*
2693                  * CNTR 3 - Set the nop counter to the number of cycles we
2694                  * need to loop for, minus 1.
2695                  */
2696                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3;
2697                 writel(rw_wl_nop_cycles - 1, addr);
2698                 if (test_dm) {
2699                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2700                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2701                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP, addr);
2702                 } else {
2703                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2704                         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2705                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, addr);
2706                 }
2707         }
2708
2709         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RESET_READ_DATAPATH_OFFSET;
2710         writel(0, addr);
2711
2712         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
2713         if (quick_write_mode)
2714                 writel(0x08, addr);
2715         else
2716                 writel(0x40, addr);
2717
2718         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
2719         writel(mcc_instruction, addr);
2720
2721         /*
2722          * CNTR 1 - This is used to ensure enough time elapses
2723          * for read data to come back.
2724          */
2725         addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
2726         writel(0x30, addr);
2727
2728         addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
2729         if (test_dm) {
2730                 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT, addr);
2731         } else {
2732                 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT, addr);
2733         }
2734
2735         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
2736         writel(mcc_instruction, addr + (group << 2));
2737 }
2738
2739 /* Test writes, can check for a single bit pass or multiple bit pass */
2740 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
2741         uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
2742         uint32_t *bit_chk, uint32_t all_ranks)
2743 {
2744         uint32_t addr;
2745         uint32_t r;
2746         uint32_t correct_mask_vg;
2747         uint32_t tmp_bit_chk;
2748         uint32_t vg;
2749         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
2750                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
2751         uint32_t addr_rw_mgr;
2752         uint32_t base_rw_mgr;
2753
2754         *bit_chk = param->write_correct_mask;
2755         correct_mask_vg = param->write_correct_mask_vg;
2756
2757         for (r = rank_bgn; r < rank_end; r++) {
2758                 if (param->skip_ranks[r]) {
2759                         /* request to skip the rank */
2760                         continue;
2761                 }
2762
2763                 /* set rank */
2764                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
2765
2766                 tmp_bit_chk = 0;
2767                 addr = (u32)&phy_mgr_cmd->fifo_reset;
2768                 addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
2769                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
2770                         /* reset the fifos to get pointers to known state */
2771                         writel(0, addr);
2772
2773                         tmp_bit_chk = tmp_bit_chk <<
2774                                 (RW_MGR_MEM_DQ_PER_WRITE_DQS /
2775                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
2776                         rw_mgr_mem_calibrate_write_test_issue(write_group *
2777                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
2778                                 use_dm);
2779
2780                         base_rw_mgr = readl(addr_rw_mgr);
2781                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
2782                         if (vg == 0)
2783                                 break;
2784                 }
2785                 *bit_chk &= tmp_bit_chk;
2786         }
2787
2788         if (all_correct) {
2789                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2790                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
2791                            %u => %lu", write_group, use_dm,
2792                            *bit_chk, param->write_correct_mask,
2793                            (long unsigned int)(*bit_chk ==
2794                            param->write_correct_mask));
2795                 return *bit_chk == param->write_correct_mask;
2796         } else {
2797                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2798                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
2799                        write_group, use_dm, *bit_chk);
2800                 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
2801                         (long unsigned int)(*bit_chk != 0));
2802                 return *bit_chk != 0x00;
2803         }
2804 }
2805
2806 /*
2807  * center all windows. do per-bit-deskew to possibly increase size of
2808  * certain windows.
2809  */
2810 static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
2811         uint32_t write_group, uint32_t test_bgn)
2812 {
2813         uint32_t i, p, min_index;
2814         int32_t d;
2815         /*
2816          * Store these as signed since there are comparisons with
2817          * signed numbers.
2818          */
2819         uint32_t bit_chk;
2820         uint32_t sticky_bit_chk;
2821         int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2822         int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2823         int32_t mid;
2824         int32_t mid_min, orig_mid_min;
2825         int32_t new_dqs, start_dqs, shift_dq;
2826         int32_t dq_margin, dqs_margin, dm_margin;
2827         uint32_t stop;
2828         uint32_t temp_dq_out1_delay;
2829         uint32_t addr;
2830
2831         debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
2832
2833         dm_margin = 0;
2834
2835         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
2836         start_dqs = readl(addr +
2837                           (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
2838
2839         /* per-bit deskew */
2840
2841         /*
2842          * set the left and right edge of each bit to an illegal value
2843          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
2844          */
2845         sticky_bit_chk = 0;
2846         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2847                 left_edge[i]  = IO_IO_OUT1_DELAY_MAX + 1;
2848                 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2849         }
2850
2851         /* Search for the left edge of the window for each bit */
2852         addr = (u32)&sdr_scc_mgr->update;
2853         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
2854                 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, d);
2855
2856                 writel(0, addr);
2857
2858                 /*
2859                  * Stop searching when the read test doesn't pass AND when
2860                  * we've seen a passing read on every bit.
2861                  */
2862                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2863                         0, PASS_ONE_BIT, &bit_chk, 0);
2864                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2865                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2866                 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
2867                            == %u && %u [bit_chk= %u ]\n",
2868                         d, sticky_bit_chk, param->write_correct_mask,
2869                         stop, bit_chk);
2870
2871                 if (stop == 1) {
2872                         break;
2873                 } else {
2874                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2875                                 if (bit_chk & 1) {
2876                                         /*
2877                                          * Remember a passing test as the
2878                                          * left_edge.
2879                                          */
2880                                         left_edge[i] = d;
2881                                 } else {
2882                                         /*
2883                                          * If a left edge has not been seen
2884                                          * yet, then a future passing test will
2885                                          * mark this edge as the right edge.
2886                                          */
2887                                         if (left_edge[i] ==
2888                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2889                                                 right_edge[i] = -(d + 1);
2890                                         }
2891                                 }
2892                                 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
2893                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2894                                            (int)(bit_chk & 1), i, left_edge[i]);
2895                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2896                                        right_edge[i]);
2897                                 bit_chk = bit_chk >> 1;
2898                         }
2899                 }
2900         }
2901
2902         /* Reset DQ delay chains to 0 */
2903         scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, 0);
2904         sticky_bit_chk = 0;
2905         for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
2906                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2907                            %d right_edge[%u]: %d\n", __func__, __LINE__,
2908                            i, left_edge[i], i, right_edge[i]);
2909
2910                 /*
2911                  * Check for cases where we haven't found the left edge,
2912                  * which makes our assignment of the the right edge invalid.
2913                  * Reset it to the illegal value.
2914                  */
2915                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
2916                     (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
2917                         right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2918                         debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
2919                                    right_edge[%u]: %d\n", __func__, __LINE__,
2920                                    i, right_edge[i]);
2921                 }
2922
2923                 /*
2924                  * Reset sticky bit (except for bits where we have
2925                  * seen the left edge).
2926                  */
2927                 sticky_bit_chk = sticky_bit_chk << 1;
2928                 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
2929                         sticky_bit_chk = sticky_bit_chk | 1;
2930
2931                 if (i == 0)
2932                         break;
2933         }
2934
2935         /* Search for the right edge of the window for each bit */
2936         addr = (u32)&sdr_scc_mgr->update;
2937         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
2938                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
2939                                                         d + start_dqs);
2940
2941                 writel(0, addr);
2942
2943                 /*
2944                  * Stop searching when the read test doesn't pass AND when
2945                  * we've seen a passing read on every bit.
2946                  */
2947                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2948                         0, PASS_ONE_BIT, &bit_chk, 0);
2949
2950                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2951                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2952
2953                 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
2954                            %u && %u\n", d, sticky_bit_chk,
2955                            param->write_correct_mask, stop);
2956
2957                 if (stop == 1) {
2958                         if (d == 0) {
2959                                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
2960                                         i++) {
2961                                         /* d = 0 failed, but it passed when
2962                                         testing the left edge, so it must be
2963                                         marginal, set it to -1 */
2964                                         if (right_edge[i] ==
2965                                                 IO_IO_OUT1_DELAY_MAX + 1 &&
2966                                                 left_edge[i] !=
2967                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2968                                                 right_edge[i] = -1;
2969                                         }
2970                                 }
2971                         }
2972                         break;
2973                 } else {
2974                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2975                                 if (bit_chk & 1) {
2976                                         /*
2977                                          * Remember a passing test as
2978                                          * the right_edge.
2979                                          */
2980                                         right_edge[i] = d;
2981                                 } else {
2982                                         if (d != 0) {
2983                                                 /*
2984                                                  * If a right edge has not
2985                                                  * been seen yet, then a future
2986                                                  * passing test will mark this
2987                                                  * edge as the left edge.
2988                                                  */
2989                                                 if (right_edge[i] ==
2990                                                     IO_IO_OUT1_DELAY_MAX + 1)
2991                                                         left_edge[i] = -(d + 1);
2992                                         } else {
2993                                                 /*
2994                                                  * d = 0 failed, but it passed
2995                                                  * when testing the left edge,
2996                                                  * so it must be marginal, set
2997                                                  * it to -1.
2998                                                  */
2999                                                 if (right_edge[i] ==
3000                                                     IO_IO_OUT1_DELAY_MAX + 1 &&
3001                                                     left_edge[i] !=
3002                                                     IO_IO_OUT1_DELAY_MAX + 1)
3003                                                         right_edge[i] = -1;
3004                                                 /*
3005                                                  * If a right edge has not been
3006                                                  * seen yet, then a future
3007                                                  * passing test will mark this
3008                                                  * edge as the left edge.
3009                                                  */
3010                                                 else if (right_edge[i] ==
3011                                                         IO_IO_OUT1_DELAY_MAX +
3012                                                         1)
3013                                                         left_edge[i] = -(d + 1);
3014                                         }
3015                                 }
3016                                 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
3017                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
3018                                            (int)(bit_chk & 1), i, left_edge[i]);
3019                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
3020                                            right_edge[i]);
3021                                 bit_chk = bit_chk >> 1;
3022                         }
3023                 }
3024         }
3025
3026         /* Check that all bits have a window */
3027         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
3028                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
3029                            %d right_edge[%u]: %d", __func__, __LINE__,
3030                            i, left_edge[i], i, right_edge[i]);
3031                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
3032                     (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
3033                         set_failing_group_stage(test_bgn + i,
3034                                                 CAL_STAGE_WRITES,
3035                                                 CAL_SUBSTAGE_WRITES_CENTER);
3036                         return 0;
3037                 }
3038         }
3039
3040         /* Find middle of window for each DQ bit */
3041         mid_min = left_edge[0] - right_edge[0];
3042         min_index = 0;
3043         for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
3044                 mid = left_edge[i] - right_edge[i];
3045                 if (mid < mid_min) {
3046                         mid_min = mid;
3047                         min_index = i;
3048                 }
3049         }
3050
3051         /*
3052          * -mid_min/2 represents the amount that we need to move DQS.
3053          * If mid_min is odd and positive we'll need to add one to
3054          * make sure the rounding in further calculations is correct
3055          * (always bias to the right), so just add 1 for all positive values.
3056          */
3057         if (mid_min > 0)
3058                 mid_min++;
3059         mid_min = mid_min / 2;
3060         debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
3061                    __LINE__, mid_min);
3062
3063         /* Determine the amount we can change DQS (which is -mid_min) */
3064         orig_mid_min = mid_min;
3065         new_dqs = start_dqs;
3066         mid_min = 0;
3067         debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
3068                    mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
3069         /* Initialize data for export structures */
3070         dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
3071         dq_margin  = IO_IO_OUT1_DELAY_MAX + 1;
3072
3073         /* add delay to bring centre of all DQ windows to the same "level" */
3074         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
3075         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
3076                 /* Use values before divide by 2 to reduce round off error */
3077                 shift_dq = (left_edge[i] - right_edge[i] -
3078                         (left_edge[min_index] - right_edge[min_index]))/2  +
3079                 (orig_mid_min - mid_min);
3080
3081                 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
3082                            [%u]=%d\n", __func__, __LINE__, i, shift_dq);
3083
3084                 temp_dq_out1_delay = readl(addr + (i << 2));
3085                 if (shift_dq + (int32_t)temp_dq_out1_delay >
3086                         (int32_t)IO_IO_OUT1_DELAY_MAX) {
3087                         shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
3088                 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
3089                         shift_dq = -(int32_t)temp_dq_out1_delay;
3090                 }
3091                 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
3092                            i, shift_dq);
3093                 scc_mgr_set_dq_out1_delay(write_group, i, temp_dq_out1_delay +
3094                                           shift_dq);
3095                 scc_mgr_load_dq(i);
3096
3097                 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
3098                            left_edge[i] - shift_dq + (-mid_min),
3099                            right_edge[i] + shift_dq - (-mid_min));
3100                 /* To determine values for export structures */
3101                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
3102                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
3103
3104                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
3105                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
3106         }
3107
3108         /* Move DQS */
3109         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3110         addr = (u32)&sdr_scc_mgr->update;
3111         writel(0, addr);
3112
3113         /* Centre DM */
3114         debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
3115
3116         /*
3117          * set the left and right edge of each bit to an illegal value,
3118          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
3119          */
3120         left_edge[0]  = IO_IO_OUT1_DELAY_MAX + 1;
3121         right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
3122         int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3123         int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3124         int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
3125         int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
3126         int32_t win_best = 0;
3127
3128         /* Search for the/part of the window with DM shift */
3129         addr = (u32)&sdr_scc_mgr->update;
3130         for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
3131                 scc_mgr_apply_group_dm_out1_delay(write_group, d);
3132                 writel(0, addr);
3133
3134                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3135                                                     PASS_ALL_BITS, &bit_chk,
3136                                                     0)) {
3137                         /* USE Set current end of the window */
3138                         end_curr = -d;
3139                         /*
3140                          * If a starting edge of our window has not been seen
3141                          * this is our current start of the DM window.
3142                          */
3143                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3144                                 bgn_curr = -d;
3145
3146                         /*
3147                          * If current window is bigger than best seen.
3148                          * Set best seen to be current window.
3149                          */
3150                         if ((end_curr-bgn_curr+1) > win_best) {
3151                                 win_best = end_curr-bgn_curr+1;
3152                                 bgn_best = bgn_curr;
3153                                 end_best = end_curr;
3154                         }
3155                 } else {
3156                         /* We just saw a failing test. Reset temp edge */
3157                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3158                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3159                         }
3160                 }
3161
3162
3163         /* Reset DM delay chains to 0 */
3164         scc_mgr_apply_group_dm_out1_delay(write_group, 0);
3165
3166         /*
3167          * Check to see if the current window nudges up aganist 0 delay.
3168          * If so we need to continue the search by shifting DQS otherwise DQS
3169          * search begins as a new search. */
3170         if (end_curr != 0) {
3171                 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3172                 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3173         }
3174
3175         /* Search for the/part of the window with DQS shifts */
3176         addr = (u32)&sdr_scc_mgr->update;
3177         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
3178                 /*
3179                  * Note: This only shifts DQS, so are we limiting ourselve to
3180                  * width of DQ unnecessarily.
3181                  */
3182                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
3183                                                         d + new_dqs);
3184
3185                 writel(0, addr);
3186                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3187                                                     PASS_ALL_BITS, &bit_chk,
3188                                                     0)) {
3189                         /* USE Set current end of the window */
3190                         end_curr = d;
3191                         /*
3192                          * If a beginning edge of our window has not been seen
3193                          * this is our current begin of the DM window.
3194                          */
3195                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3196                                 bgn_curr = d;
3197
3198                         /*
3199                          * If current window is bigger than best seen. Set best
3200                          * seen to be current window.
3201                          */
3202                         if ((end_curr-bgn_curr+1) > win_best) {
3203                                 win_best = end_curr-bgn_curr+1;
3204                                 bgn_best = bgn_curr;
3205                                 end_best = end_curr;
3206                         }
3207                 } else {
3208                         /* We just saw a failing test. Reset temp edge */
3209                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3210                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3211
3212                         /* Early exit optimization: if ther remaining delay
3213                         chain space is less than already seen largest window
3214                         we can exit */
3215                         if ((win_best-1) >
3216                                 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
3217                                         break;
3218                                 }
3219                         }
3220                 }
3221
3222         /* assign left and right edge for cal and reporting; */
3223         left_edge[0] = -1*bgn_best;
3224         right_edge[0] = end_best;
3225
3226         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
3227                    __LINE__, left_edge[0], right_edge[0]);
3228
3229         /* Move DQS (back to orig) */
3230         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3231
3232         /* Move DM */
3233
3234         /* Find middle of window for the DM bit */
3235         mid = (left_edge[0] - right_edge[0]) / 2;
3236
3237         /* only move right, since we are not moving DQS/DQ */
3238         if (mid < 0)
3239                 mid = 0;
3240
3241         /* dm_marign should fail if we never find a window */
3242         if (win_best == 0)
3243                 dm_margin = -1;
3244         else
3245                 dm_margin = left_edge[0] - mid;
3246
3247         scc_mgr_apply_group_dm_out1_delay(write_group, mid);
3248         addr = (u32)&sdr_scc_mgr->update;
3249         writel(0, addr);
3250
3251         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
3252                    dm_margin=%d\n", __func__, __LINE__, left_edge[0],
3253                    right_edge[0], mid, dm_margin);
3254         /* Export values */
3255         gbl->fom_out += dq_margin + dqs_margin;
3256
3257         debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
3258                    dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
3259                    dq_margin, dqs_margin, dm_margin);
3260
3261         /*
3262          * Do not remove this line as it makes sure all of our
3263          * decisions have been applied.
3264          */
3265         addr = (u32)&sdr_scc_mgr->update;
3266         writel(0, addr);
3267         return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
3268 }
3269
3270 /* calibrate the write operations */
3271 static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
3272         uint32_t test_bgn)
3273 {
3274         /* update info for sims */
3275         debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
3276
3277         reg_file_set_stage(CAL_STAGE_WRITES);
3278         reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
3279
3280         reg_file_set_group(g);
3281
3282         if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
3283                 set_failing_group_stage(g, CAL_STAGE_WRITES,
3284                                         CAL_SUBSTAGE_WRITES_CENTER);
3285                 return 0;
3286         }
3287
3288         return 1;
3289 }
3290
3291 /* precharge all banks and activate row 0 in bank "000..." and bank "111..." */
3292 static void mem_precharge_and_activate(void)
3293 {
3294         uint32_t r;
3295         uint32_t addr;
3296
3297         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
3298                 if (param->skip_ranks[r]) {
3299                         /* request to skip the rank */
3300                         continue;
3301                 }
3302
3303                 /* set rank */
3304                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
3305
3306                 /* precharge all banks ... */
3307                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
3308                 writel(RW_MGR_PRECHARGE_ALL, addr);
3309
3310                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
3311                 writel(0x0F, addr);
3312                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
3313                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1, addr);
3314
3315                 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
3316                 writel(0x0F, addr);
3317                 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
3318                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2, addr);
3319
3320                 /* activate rows */
3321                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
3322                 writel(RW_MGR_ACTIVATE_0_AND_1, addr);
3323         }
3324 }
3325
3326 /* Configure various memory related parameters. */
3327 static void mem_config(void)
3328 {
3329         uint32_t rlat, wlat;
3330         uint32_t rw_wl_nop_cycles;
3331         uint32_t max_latency;
3332         uint32_t addr;
3333
3334         debug("%s:%d\n", __func__, __LINE__);
3335         /* read in write and read latency */
3336         addr = (u32)&data_mgr->t_wl_add;
3337         wlat = readl(addr);
3338
3339         addr = (u32)&data_mgr->mem_t_add;
3340         wlat += readl(addr);
3341         /* WL for hard phy does not include additive latency */
3342
3343         /*
3344          * add addtional write latency to offset the address/command extra
3345          * clock cycle. We change the AC mux setting causing AC to be delayed
3346          * by one mem clock cycle. Only do this for DDR3
3347          */
3348         wlat = wlat + 1;
3349
3350         addr = (u32)&data_mgr->t_rl_add;
3351         rlat = readl(addr);
3352
3353         rw_wl_nop_cycles = wlat - 2;
3354         gbl->rw_wl_nop_cycles = rw_wl_nop_cycles;
3355
3356         /*
3357          * For AV/CV, lfifo is hardened and always runs at full rate so
3358          * max latency in AFI clocks, used here, is correspondingly smaller.
3359          */
3360         max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1;
3361         /* configure for a burst length of 8 */
3362
3363         /* write latency */
3364         /* Adjust Write Latency for Hard PHY */
3365         wlat = wlat + 1;
3366
3367         /* set a pretty high read latency initially */
3368         gbl->curr_read_lat = rlat + 16;
3369
3370         if (gbl->curr_read_lat > max_latency)
3371                 gbl->curr_read_lat = max_latency;
3372
3373         addr = (u32)&phy_mgr_cfg->phy_rlat;
3374         writel(gbl->curr_read_lat, addr);
3375
3376         /* advertise write latency */
3377         gbl->curr_write_lat = wlat;
3378         addr = (u32)&phy_mgr_cfg->afi_wlat;
3379         writel(wlat - 2, addr);
3380
3381         /* initialize bit slips */
3382         mem_precharge_and_activate();
3383 }
3384
3385 /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */
3386 static void mem_skip_calibrate(void)
3387 {
3388         uint32_t vfifo_offset;
3389         uint32_t i, j, r;
3390         uint32_t addr;
3391
3392         debug("%s:%d\n", __func__, __LINE__);
3393         /* Need to update every shadow register set used by the interface */
3394         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
3395                 r += NUM_RANKS_PER_SHADOW_REG) {
3396                 /*
3397                  * Set output phase alignment settings appropriate for
3398                  * skip calibration.
3399                  */
3400                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3401                         scc_mgr_set_dqs_en_phase(i, 0);
3402 #if IO_DLL_CHAIN_LENGTH == 6
3403                         scc_mgr_set_dqdqs_output_phase(i, 6);
3404 #else
3405                         scc_mgr_set_dqdqs_output_phase(i, 7);
3406 #endif
3407                         /*
3408                          * Case:33398
3409                          *
3410                          * Write data arrives to the I/O two cycles before write
3411                          * latency is reached (720 deg).
3412                          *   -> due to bit-slip in a/c bus
3413                          *   -> to allow board skew where dqs is longer than ck
3414                          *      -> how often can this happen!?
3415                          *      -> can claim back some ptaps for high freq
3416                          *       support if we can relax this, but i digress...
3417                          *
3418                          * The write_clk leads mem_ck by 90 deg
3419                          * The minimum ptap of the OPA is 180 deg
3420                          * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
3421                          * The write_clk is always delayed by 2 ptaps
3422                          *
3423                          * Hence, to make DQS aligned to CK, we need to delay
3424                          * DQS by:
3425                          *    (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
3426                          *
3427                          * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
3428                          * gives us the number of ptaps, which simplies to:
3429                          *
3430                          *    (1.25 * IO_DLL_CHAIN_LENGTH - 2)
3431                          */
3432                         scc_mgr_set_dqdqs_output_phase(i, (1.25 *
3433                                 IO_DLL_CHAIN_LENGTH - 2));
3434                 }
3435                 addr = (u32)&sdr_scc_mgr->dqs_ena;
3436                 writel(0xff, addr);
3437                 addr = (u32)&sdr_scc_mgr->dqs_io_ena;
3438                 writel(0xff, addr);
3439
3440                 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET;
3441                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
3442                         writel(i, addr);
3443                 }
3444                 addr = (u32)&sdr_scc_mgr->dq_ena;
3445                 writel(0xff, addr);
3446                 addr = (u32)&sdr_scc_mgr->dm_ena;
3447                 writel(0xff, addr);
3448                 addr = (u32)&sdr_scc_mgr->update;
3449                 writel(0, addr);
3450         }
3451
3452         /* Compensate for simulation model behaviour */
3453         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3454                 scc_mgr_set_dqs_bus_in_delay(i, 10);
3455                 scc_mgr_load_dqs(i);
3456         }
3457         addr = (u32)&sdr_scc_mgr->update;
3458         writel(0, addr);
3459
3460         /*
3461          * ArriaV has hard FIFOs that can only be initialized by incrementing
3462          * in sequencer.
3463          */
3464         vfifo_offset = CALIB_VFIFO_OFFSET;
3465         addr = (u32)&phy_mgr_cmd->inc_vfifo_hard_phy;
3466         for (j = 0; j < vfifo_offset; j++) {
3467                 writel(0xff, addr);
3468         }
3469         addr = (u32)&phy_mgr_cmd->fifo_reset;
3470         writel(0, addr);
3471
3472         /*
3473          * For ACV with hard lfifo, we get the skip-cal setting from
3474          * generation-time constant.
3475          */
3476         gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
3477         addr = (u32)&phy_mgr_cfg->phy_rlat;
3478         writel(gbl->curr_read_lat, addr);
3479 }
3480
3481 /* Memory calibration entry point */
3482 static uint32_t mem_calibrate(void)
3483 {
3484         uint32_t i;
3485         uint32_t rank_bgn, sr;
3486         uint32_t write_group, write_test_bgn;
3487         uint32_t read_group, read_test_bgn;
3488         uint32_t run_groups, current_run;
3489         uint32_t failing_groups = 0;
3490         uint32_t group_failed = 0;
3491         uint32_t sr_failed = 0;
3492         uint32_t addr;
3493
3494         debug("%s:%d\n", __func__, __LINE__);
3495         /* Initialize the data settings */
3496
3497         gbl->error_substage = CAL_SUBSTAGE_NIL;
3498         gbl->error_stage = CAL_STAGE_NIL;
3499         gbl->error_group = 0xff;
3500         gbl->fom_in = 0;
3501         gbl->fom_out = 0;
3502
3503         mem_config();
3504
3505         uint32_t bypass_mode = 0x1;
3506         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET;
3507         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3508                 writel(i, addr);
3509                 scc_set_bypass_mode(i, bypass_mode);
3510         }
3511
3512         if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
3513                 /*
3514                  * Set VFIFO and LFIFO to instant-on settings in skip
3515                  * calibration mode.
3516                  */
3517                 mem_skip_calibrate();
3518         } else {
3519                 for (i = 0; i < NUM_CALIB_REPEAT; i++) {
3520                         /*
3521                          * Zero all delay chain/phase settings for all
3522                          * groups and all shadow register sets.
3523                          */
3524                         scc_mgr_zero_all();
3525
3526                         run_groups = ~param->skip_groups;
3527
3528                         for (write_group = 0, write_test_bgn = 0; write_group
3529                                 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
3530                                 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
3531                                 /* Initialized the group failure */
3532                                 group_failed = 0;
3533
3534                                 current_run = run_groups & ((1 <<
3535                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
3536                                 run_groups = run_groups >>
3537                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP;
3538
3539                                 if (current_run == 0)
3540                                         continue;
3541
3542                                 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_GROUP_COUNTER_OFFSET;
3543                                 writel(write_group, addr);
3544                                 scc_mgr_zero_group(write_group, write_test_bgn,
3545                                                    0);
3546
3547                                 for (read_group = write_group *
3548                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3549                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3550                                         read_test_bgn = 0;
3551                                         read_group < (write_group + 1) *
3552                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3553                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3554                                         group_failed == 0;
3555                                         read_group++, read_test_bgn +=
3556                                         RW_MGR_MEM_DQ_PER_READ_DQS) {
3557                                         /* Calibrate the VFIFO */
3558                                         if (!((STATIC_CALIB_STEPS) &
3559                                                 CALIB_SKIP_VFIFO)) {
3560                                                 if (!rw_mgr_mem_calibrate_vfifo
3561                                                         (read_group,
3562                                                         read_test_bgn)) {
3563                                                         group_failed = 1;
3564
3565                                                         if (!(gbl->
3566                                                         phy_debug_mode_flags &
3567                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3568                                                                 return 0;
3569                                                         }
3570                                                 }
3571                                         }
3572                                 }
3573
3574                                 /* Calibrate the output side */
3575                                 if (group_failed == 0)  {
3576                                         for (rank_bgn = 0, sr = 0; rank_bgn
3577                                                 < RW_MGR_MEM_NUMBER_OF_RANKS;
3578                                                 rank_bgn +=
3579                                                 NUM_RANKS_PER_SHADOW_REG,
3580                                                 ++sr) {
3581                                                 sr_failed = 0;
3582                                                 if (!((STATIC_CALIB_STEPS) &
3583                                                 CALIB_SKIP_WRITES)) {
3584                                                         if ((STATIC_CALIB_STEPS)
3585                                                 & CALIB_SKIP_DELAY_SWEEPS) {
3586                                                 /* not needed in quick mode! */
3587                                                         } else {
3588                                                 /*
3589                                                  * Determine if this set of
3590                                                  * ranks should be skipped
3591                                                  * entirely.
3592                                                  */
3593                                         if (!param->skip_shadow_regs[sr]) {
3594                                                 if (!rw_mgr_mem_calibrate_writes
3595                                                 (rank_bgn, write_group,
3596                                                 write_test_bgn)) {
3597                                                         sr_failed = 1;
3598                                                         if (!(gbl->
3599                                                         phy_debug_mode_flags &
3600                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3601                                                                 return 0;
3602                                                                         }
3603                                                                         }
3604                                                                 }
3605                                                         }
3606                                                 }
3607                                                 if (sr_failed != 0)
3608                                                         group_failed = 1;
3609                                         }
3610                                 }
3611
3612                                 if (group_failed == 0) {
3613                                         for (read_group = write_group *
3614                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3615                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3616                                         read_test_bgn = 0;
3617                                                 read_group < (write_group + 1)
3618                                                 * RW_MGR_MEM_IF_READ_DQS_WIDTH
3619                                                 / RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3620                                                 group_failed == 0;
3621                                                 read_group++, read_test_bgn +=
3622                                                 RW_MGR_MEM_DQ_PER_READ_DQS) {
3623                                                 if (!((STATIC_CALIB_STEPS) &
3624                                                         CALIB_SKIP_WRITES)) {
3625                                         if (!rw_mgr_mem_calibrate_vfifo_end
3626                                                 (read_group, read_test_bgn)) {
3627                                                         group_failed = 1;
3628
3629                                                 if (!(gbl->phy_debug_mode_flags
3630                                                 & PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3631                                                                 return 0;
3632                                                                 }
3633                                                         }
3634                                                 }
3635                                         }
3636                                 }
3637
3638                                 if (group_failed != 0)
3639                                         failing_groups++;
3640                         }
3641
3642                         /*
3643                          * USER If there are any failing groups then report
3644                          * the failure.
3645                          */
3646                         if (failing_groups != 0)
3647                                 return 0;
3648
3649                         /* Calibrate the LFIFO */
3650                         if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) {
3651                                 /*
3652                                  * If we're skipping groups as part of debug,
3653                                  * don't calibrate LFIFO.
3654                                  */
3655                                 if (param->skip_groups == 0) {
3656                                         if (!rw_mgr_mem_calibrate_lfifo())
3657                                                 return 0;
3658                                 }
3659                         }
3660                 }
3661         }
3662
3663         /*
3664          * Do not remove this line as it makes sure all of our decisions
3665          * have been applied.
3666          */
3667         addr = (u32)&sdr_scc_mgr->update;
3668         writel(0, addr);
3669         return 1;
3670 }
3671
3672 static uint32_t run_mem_calibrate(void)
3673 {
3674         uint32_t pass;
3675         uint32_t debug_info;
3676         uint32_t addr;
3677
3678         debug("%s:%d\n", __func__, __LINE__);
3679
3680         /* Reset pass/fail status shown on afi_cal_success/fail */
3681         addr = (u32)&phy_mgr_cfg->cal_status;
3682         writel(PHY_MGR_CAL_RESET, addr);
3683
3684         /* stop tracking manger */
3685         uint32_t ctrlcfg = readl(&sdr_ctrl->ctrl_cfg);
3686
3687         writel(ctrlcfg & 0xFFBFFFFF, &sdr_ctrl->ctrl_cfg);
3688
3689         initialize();
3690         rw_mgr_mem_initialize();
3691
3692         pass = mem_calibrate();
3693
3694         mem_precharge_and_activate();
3695         addr = (u32)&phy_mgr_cmd->fifo_reset;
3696         writel(0, addr);
3697
3698         /*
3699          * Handoff:
3700          * Don't return control of the PHY back to AFI when in debug mode.
3701          */
3702         if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) {
3703                 rw_mgr_mem_handoff();
3704                 /*
3705                  * In Hard PHY this is a 2-bit control:
3706                  * 0: AFI Mux Select
3707                  * 1: DDIO Mux Select
3708                  */
3709                 addr = (u32)&phy_mgr_cfg->mux_sel;
3710                 writel(0x2, addr);
3711         }
3712
3713         writel(ctrlcfg, &sdr_ctrl->ctrl_cfg);
3714
3715         if (pass) {
3716                 printf("%s: CALIBRATION PASSED\n", __FILE__);
3717
3718                 gbl->fom_in /= 2;
3719                 gbl->fom_out /= 2;
3720
3721                 if (gbl->fom_in > 0xff)
3722                         gbl->fom_in = 0xff;
3723
3724                 if (gbl->fom_out > 0xff)
3725                         gbl->fom_out = 0xff;
3726
3727                 /* Update the FOM in the register file */
3728                 debug_info = gbl->fom_in;
3729                 debug_info |= gbl->fom_out << 8;
3730                 addr = (u32)&sdr_reg_file->fom;
3731                 writel(debug_info, addr);
3732
3733                 addr = (u32)&phy_mgr_cfg->cal_debug_info;
3734                 writel(debug_info, addr);
3735                 addr = (u32)&phy_mgr_cfg->cal_status;
3736                 writel(PHY_MGR_CAL_SUCCESS, addr);
3737         } else {
3738                 printf("%s: CALIBRATION FAILED\n", __FILE__);
3739
3740                 debug_info = gbl->error_stage;
3741                 debug_info |= gbl->error_substage << 8;
3742                 debug_info |= gbl->error_group << 16;
3743
3744                 addr = (u32)&sdr_reg_file->failing_stage;
3745                 writel(debug_info, addr);
3746                 addr = (u32)&phy_mgr_cfg->cal_debug_info;
3747                 writel(debug_info, addr);
3748                 addr = (u32)&phy_mgr_cfg->cal_status;
3749                 writel(PHY_MGR_CAL_FAIL, addr);
3750
3751                 /* Update the failing group/stage in the register file */
3752                 debug_info = gbl->error_stage;
3753                 debug_info |= gbl->error_substage << 8;
3754                 debug_info |= gbl->error_group << 16;
3755                 addr = (u32)&sdr_reg_file->failing_stage;
3756                 writel(debug_info, addr);
3757         }
3758
3759         return pass;
3760 }
3761
3762 static void hc_initialize_rom_data(void)
3763 {
3764         uint32_t i;
3765         uint32_t addr;
3766
3767         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
3768         for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++) {
3769                 uint32_t data = inst_rom_init[i];
3770                 writel(data, addr + (i << 2));
3771         }
3772
3773         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
3774         for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++) {
3775                 uint32_t data = ac_rom_init[i];
3776                 writel(data, addr + (i << 2));
3777         }
3778 }
3779
3780 static void initialize_reg_file(void)
3781 {
3782         uint32_t addr;
3783
3784         /* Initialize the register file with the correct data */
3785         addr = (u32)&sdr_reg_file->signature;
3786         writel(REG_FILE_INIT_SEQ_SIGNATURE, addr);
3787
3788         addr = (u32)&sdr_reg_file->debug_data_addr;
3789         writel(0, addr);
3790
3791         addr = (u32)&sdr_reg_file->cur_stage;
3792         writel(0, addr);
3793
3794         addr = (u32)&sdr_reg_file->fom;
3795         writel(0, addr);
3796
3797         addr = (u32)&sdr_reg_file->failing_stage;
3798         writel(0, addr);
3799
3800         addr = (u32)&sdr_reg_file->debug1;
3801         writel(0, addr);
3802
3803         addr = (u32)&sdr_reg_file->debug2;
3804         writel(0, addr);
3805 }
3806
3807 static void initialize_hps_phy(void)
3808 {
3809         uint32_t reg;
3810         /*
3811          * Tracking also gets configured here because it's in the
3812          * same register.
3813          */
3814         uint32_t trk_sample_count = 7500;
3815         uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
3816         /*
3817          * Format is number of outer loops in the 16 MSB, sample
3818          * count in 16 LSB.
3819          */
3820
3821         reg = 0;
3822         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
3823         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
3824         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
3825         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
3826         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
3827         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
3828         /*
3829          * This field selects the intrinsic latency to RDATA_EN/FULL path.
3830          * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
3831          */
3832         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
3833         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
3834                 trk_sample_count);
3835         writel(reg, &sdr_ctrl->phy_ctrl0);
3836
3837         reg = 0;
3838         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
3839                 trk_sample_count >>
3840                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
3841         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
3842                 trk_long_idle_sample_count);
3843         writel(reg, &sdr_ctrl->phy_ctrl1);
3844
3845         reg = 0;
3846         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
3847                 trk_long_idle_sample_count >>
3848                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
3849         writel(reg, &sdr_ctrl->phy_ctrl2);
3850 }
3851
3852 static void initialize_tracking(void)
3853 {
3854         uint32_t concatenated_longidle = 0x0;
3855         uint32_t concatenated_delays = 0x0;
3856         uint32_t concatenated_rw_addr = 0x0;
3857         uint32_t concatenated_refresh = 0x0;
3858         uint32_t trk_sample_count = 7500;
3859         uint32_t dtaps_per_ptap;
3860         uint32_t tmp_delay;
3861         uint32_t addr;
3862
3863         /*
3864          * compute usable version of value in case we skip full
3865          * computation later
3866          */
3867         dtaps_per_ptap = 0;
3868         tmp_delay = 0;
3869         while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
3870                 dtaps_per_ptap++;
3871                 tmp_delay += IO_DELAY_PER_DCHAIN_TAP;
3872         }
3873         dtaps_per_ptap--;
3874
3875         concatenated_longidle = concatenated_longidle ^ 10;
3876                 /*longidle outer loop */
3877         concatenated_longidle = concatenated_longidle << 16;
3878         concatenated_longidle = concatenated_longidle ^ 100;
3879                 /*longidle sample count */
3880         concatenated_delays = concatenated_delays ^ 243;
3881                 /* trfc, worst case of 933Mhz 4Gb */
3882         concatenated_delays = concatenated_delays << 8;
3883         concatenated_delays = concatenated_delays ^ 14;
3884                 /* trcd, worst case */
3885         concatenated_delays = concatenated_delays << 8;
3886         concatenated_delays = concatenated_delays ^ 10;
3887                 /* vfifo wait */
3888         concatenated_delays = concatenated_delays << 8;
3889         concatenated_delays = concatenated_delays ^ 4;
3890                 /* mux delay */
3891
3892         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE;
3893         concatenated_rw_addr = concatenated_rw_addr << 8;
3894         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1;
3895         concatenated_rw_addr = concatenated_rw_addr << 8;
3896         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ;
3897         concatenated_rw_addr = concatenated_rw_addr << 8;
3898         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL;
3899
3900         concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL;
3901         concatenated_refresh = concatenated_refresh << 24;
3902         concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */
3903
3904         /* Initialize the register file with the correct data */
3905         addr = (u32)&sdr_reg_file->dtaps_per_ptap;
3906         writel(dtaps_per_ptap, addr);
3907
3908         addr = (u32)&sdr_reg_file->trk_sample_count;
3909         writel(trk_sample_count, addr);
3910
3911         addr = (u32)&sdr_reg_file->trk_longidle;
3912         writel(concatenated_longidle, addr);
3913
3914         addr = (u32)&sdr_reg_file->delays;
3915         writel(concatenated_delays, addr);
3916
3917         addr = (u32)&sdr_reg_file->trk_rw_mgr_addr;
3918         writel(concatenated_rw_addr, addr);
3919
3920         addr = (u32)&sdr_reg_file->trk_read_dqs_width;
3921         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, addr);
3922
3923         addr = (u32)&sdr_reg_file->trk_rfsh;
3924         writel(concatenated_refresh, addr);
3925 }
3926
3927 int sdram_calibration_full(void)
3928 {
3929         struct param_type my_param;
3930         struct gbl_type my_gbl;
3931         uint32_t pass;
3932         uint32_t i;
3933
3934         param = &my_param;
3935         gbl = &my_gbl;
3936
3937         /* Initialize the debug mode flags */
3938         gbl->phy_debug_mode_flags = 0;
3939         /* Set the calibration enabled by default */
3940         gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
3941         /*
3942          * Only sweep all groups (regardless of fail state) by default
3943          * Set enabled read test by default.
3944          */
3945 #if DISABLE_GUARANTEED_READ
3946         gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
3947 #endif
3948         /* Initialize the register file */
3949         initialize_reg_file();
3950
3951         /* Initialize any PHY CSR */
3952         initialize_hps_phy();
3953
3954         scc_mgr_initialize();
3955
3956         initialize_tracking();
3957
3958         /* USER Enable all ranks, groups */
3959         for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++)
3960                 param->skip_ranks[i] = 0;
3961         for (i = 0; i < NUM_SHADOW_REGS; ++i)
3962                 param->skip_shadow_regs[i] = 0;
3963         param->skip_groups = 0;
3964
3965         printf("%s: Preparing to start memory calibration\n", __FILE__);
3966
3967         debug("%s:%d\n", __func__, __LINE__);
3968         debug_cond(DLEVEL == 1,
3969                    "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
3970                    RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
3971                    RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
3972                    RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
3973                    RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
3974         debug_cond(DLEVEL == 1,
3975                    "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
3976                    RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3977                    RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
3978                    IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
3979         debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
3980                    IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
3981         debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
3982                    IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
3983                    IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
3984         debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
3985                    IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
3986                    IO_IO_OUT2_DELAY_MAX);
3987         debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
3988                    IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);
3989
3990         hc_initialize_rom_data();
3991
3992         /* update info for sims */
3993         reg_file_set_stage(CAL_STAGE_NIL);
3994         reg_file_set_group(0);
3995
3996         /*
3997          * Load global needed for those actions that require
3998          * some dynamic calibration support.
3999          */
4000         dyn_calib_steps = STATIC_CALIB_STEPS;
4001         /*
4002          * Load global to allow dynamic selection of delay loop settings
4003          * based on calibration mode.
4004          */
4005         if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
4006                 skip_delay_mask = 0xff;
4007         else
4008                 skip_delay_mask = 0x0;
4009
4010         pass = run_mem_calibrate();
4011
4012         printf("%s: Calibration complete\n", __FILE__);
4013         return pass;
4014 }