drivers/ddr/altera/sequencer.c

   1 /*
   2  * Copyright Altera Corporation (C) 2012-2015
   3  *
   4  * SPDX-License-Identifier:    BSD-3-Clause
   5  */
   6
   7 #include <common.h>
   8 #include <asm/io.h>
   9 #include <asm/arch/sdram.h>
  10 #include "sequencer.h"
  11 #include "sequencer_auto.h"
  12 #include "sequencer_auto_ac_init.h"
  13 #include "sequencer_auto_inst_init.h"
  14 #include "sequencer_defines.h"
  15
  16 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group);
  17
  18 static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  19         (struct socfpga_sdr_rw_load_manager *)(BASE_RW_MGR + 0x800);
  20
  21 static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
  22         (struct socfpga_sdr_rw_load_jump_manager *)(BASE_RW_MGR + 0xC00);
  23
  24 static struct socfpga_sdr_reg_file *sdr_reg_file =
  25         (struct socfpga_sdr_reg_file *)(BASE_REG_FILE);
  26
  27 static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  28         (struct socfpga_sdr_scc_mgr *)(BASE_SCC_MGR + 0x0E00);
  29
  30 static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  31         (struct socfpga_phy_mgr_cmd *)(BASE_PHY_MGR);
  32
  33 static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  34         (struct socfpga_phy_mgr_cfg *)(BASE_PHY_MGR + 0x4000);
  35
  36 static struct socfpga_data_mgr *data_mgr =
  37         (struct socfpga_data_mgr *)(BASE_DATA_MGR);
  38
  39 #define DELTA_D         1
  40 #define MGR_SELECT_MASK         0xf8000
  41
  42 /*
  43  * In order to reduce ROM size, most of the selectable calibration steps are
  44  * decided at compile time based on the user's calibration mode selection,
  45  * as captured by the STATIC_CALIB_STEPS selection below.
  46  *
  47  * However, to support simulation-time selection of fast simulation mode, where
  48  * we skip everything except the bare minimum, we need a few of the steps to
  49  * be dynamic.  In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  50  * check, which is based on the rtl-supplied value, or we dynamically compute
  51  * the value to use based on the dynamically-chosen calibration mode
  52  */
  53
  54 #define DLEVEL 0
  55 #define STATIC_IN_RTL_SIM 0
  56 #define STATIC_SKIP_DELAY_LOOPS 0
  57
  58 #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  59         STATIC_SKIP_DELAY_LOOPS)
  60
  61 /* calibration steps requested by the rtl */
  62 uint16_t dyn_calib_steps;
  63
  64 /*
  65  * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
  66  * instead of static, we use boolean logic to select between
  67  * non-skip and skip values
  68  *
  69  * The mask is set to include all bits when not-skipping, but is
  70  * zero when skipping
  71  */
  72
  73 uint16_t skip_delay_mask;       /* mask off bits when skipping/not-skipping */
  74
  75 #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  76         ((non_skip_value) & skip_delay_mask)
  77
  78 struct gbl_type *gbl;
  79 struct param_type *param;
  80 uint32_t curr_shadow_reg;
  81
  82 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  83         uint32_t write_group, uint32_t use_dm,
  84         uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
  85
  86 static u32 sdr_get_addr(u32 *base)
  87 {
  88         u32 addr = (u32)base & MGR_SELECT_MASK;
  89
  90         switch (addr) {
  91         case BASE_PHY_MGR:
  92                 addr = (((u32)base >> 8) & (1 << 6)) | ((u32)base & 0x3f) |
  93                         SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  94                 break;
  95         case BASE_RW_MGR:
  96                 addr = ((u32)base & 0x1fff) | SDR_PHYGRP_RWMGRGRP_ADDRESS;
  97                 break;
  98         case BASE_DATA_MGR:
  99                 addr = ((u32)base & 0x7ff) | SDR_PHYGRP_DATAMGRGRP_ADDRESS;
 100                 break;
 101         case BASE_SCC_MGR:
 102                 addr = ((u32)base & 0xfff) | SDR_PHYGRP_SCCGRP_ADDRESS;
 103                 break;
 104         case BASE_REG_FILE:
 105                 addr = ((u32)base & 0x7ff) | SDR_PHYGRP_REGFILEGRP_ADDRESS;
 106                 break;
 107         case BASE_MMR:
 108                 addr = ((u32)base & 0xfff) | SDR_CTRLGRP_ADDRESS;
 109                 break;
 110         default:
 111                 return -1;
 112         }
 113
 114         return addr;
 115 }
 116
 117 static void set_failing_group_stage(uint32_t group, uint32_t stage,
 118         uint32_t substage)
 119 {
 120         /*
 121          * Only set the global stage if there was not been any other
 122          * failing group
 123          */
 124         if (gbl->error_stage == CAL_STAGE_NIL)  {
 125                 gbl->error_substage = substage;
 126                 gbl->error_stage = stage;
 127                 gbl->error_group = group;
 128         }
 129 }
 130
 131 static void reg_file_set_group(uint32_t set_group)
 132 {
 133         u32 addr = sdr_get_addr(&sdr_reg_file->cur_stage);
 134
 135         /* Read the current group and stage */
 136         uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr);
 137
 138         /* Clear the group */
 139         cur_stage_group &= 0x0000FFFF;
 140
 141         /* Set the group */
 142         cur_stage_group |= (set_group << 16);
 143
 144         /* Write the data back */
 145         writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr);
 146 }
 147
 148 static void reg_file_set_stage(uint32_t set_stage)
 149 {
 150         u32 addr = sdr_get_addr(&sdr_reg_file->cur_stage);
 151         /* Read the current group and stage */
 152         uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr);
 153
 154         /* Clear the stage and substage */
 155         cur_stage_group &= 0xFFFF0000;
 156
 157         /* Set the stage */
 158         cur_stage_group |= (set_stage & 0x000000FF);
 159
 160         /* Write the data back */
 161         writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr);
 162 }
 163
 164 static void reg_file_set_sub_stage(uint32_t set_sub_stage)
 165 {
 166         u32 addr = sdr_get_addr(&sdr_reg_file->cur_stage);
 167         /* Read the current group and stage */
 168         uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr);
 169
 170         /* Clear the substage */
 171         cur_stage_group &= 0xFFFF00FF;
 172
 173         /* Set the sub stage */
 174         cur_stage_group |= ((set_sub_stage << 8) & 0x0000FF00);
 175
 176         /* Write the data back */
 177         writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr);
 178 }
 179
 180 static void initialize(void)
 181 {
 182         u32 addr = sdr_get_addr(&phy_mgr_cfg->mux_sel);
 183
 184         debug("%s:%d\n", __func__, __LINE__);
 185         /* USER calibration has control over path to memory */
 186         /*
 187          * In Hard PHY this is a 2-bit control:
 188          * 0: AFI Mux Select
 189          * 1: DDIO Mux Select
 190          */
 191         writel(0x3, SOCFPGA_SDR_ADDRESS + addr);
 192
 193         /* USER memory clock is not stable we begin initialization  */
 194         addr = sdr_get_addr(&phy_mgr_cfg->reset_mem_stbl);
 195         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 196
 197         /* USER calibration status all set to zero */
 198         addr = sdr_get_addr(&phy_mgr_cfg->cal_status);
 199         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 200
 201         addr = sdr_get_addr(&phy_mgr_cfg->cal_debug_info);
 202         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 203
 204         if ((dyn_calib_steps & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) {
 205                 param->read_correct_mask_vg  = ((uint32_t)1 <<
 206                         (RW_MGR_MEM_DQ_PER_READ_DQS /
 207                         RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
 208                 param->write_correct_mask_vg = ((uint32_t)1 <<
 209                         (RW_MGR_MEM_DQ_PER_READ_DQS /
 210                         RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
 211                 param->read_correct_mask     = ((uint32_t)1 <<
 212                         RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
 213                 param->write_correct_mask    = ((uint32_t)1 <<
 214                         RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
 215                 param->dm_correct_mask       = ((uint32_t)1 <<
 216                         (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH))
 217                         - 1;
 218         }
 219 }
 220
 221 static void set_rank_and_odt_mask(uint32_t rank, uint32_t odt_mode)
 222 {
 223         uint32_t odt_mask_0 = 0;
 224         uint32_t odt_mask_1 = 0;
 225         uint32_t cs_and_odt_mask;
 226         uint32_t addr;
 227
 228         if (odt_mode == RW_MGR_ODT_MODE_READ_WRITE) {
 229                 if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) {
 230                         /*
 231                          * 1 Rank
 232                          * Read: ODT = 0
 233                          * Write: ODT = 1
 234                          */
 235                         odt_mask_0 = 0x0;
 236                         odt_mask_1 = 0x1;
 237                 } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) {
 238                         /* 2 Ranks */
 239                         if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
 240                                 /* - Dual-Slot , Single-Rank
 241                                  * (1 chip-select per DIMM)
 242                                  * OR
 243                                  * - RDIMM, 4 total CS (2 CS per DIMM)
 244                                  * means 2 DIMM
 245                                  * Since MEM_NUMBER_OF_RANKS is 2 they are
 246                                  * both single rank
 247                                  * with 2 CS each (special for RDIMM)
 248                                  * Read: Turn on ODT on the opposite rank
 249                                  * Write: Turn on ODT on all ranks
 250                                  */
 251                                 odt_mask_0 = 0x3 & ~(1 << rank);
 252                                 odt_mask_1 = 0x3;
 253                         } else {
 254                                 /*
 255                                  * USER - Single-Slot , Dual-rank DIMMs
 256                                  * (2 chip-selects per DIMM)
 257                                  * USER Read: Turn on ODT off on all ranks
 258                                  * USER Write: Turn on ODT on active rank
 259                                  */
 260                                 odt_mask_0 = 0x0;
 261                                 odt_mask_1 = 0x3 & (1 << rank);
 262                         }
 263                                 } else {
 264                         /* 4 Ranks
 265                          * Read:
 266                          * ----------+-----------------------+
 267                          *           |                       |
 268                          *           |         ODT           |
 269                          * Read From +-----------------------+
 270                          *   Rank    |  3  |  2  |  1  |  0  |
 271                          * ----------+-----+-----+-----+-----+
 272                          *     0     |  0  |  1  |  0  |  0  |
 273                          *     1     |  1  |  0  |  0  |  0  |
 274                          *     2     |  0  |  0  |  0  |  1  |
 275                          *     3     |  0  |  0  |  1  |  0  |
 276                          * ----------+-----+-----+-----+-----+
 277                          *
 278                          * Write:
 279                          * ----------+-----------------------+
 280                          *           |                       |
 281                          *           |         ODT           |
 282                          * Write To  +-----------------------+
 283                          *   Rank    |  3  |  2  |  1  |  0  |
 284                          * ----------+-----+-----+-----+-----+
 285                          *     0     |  0  |  1  |  0  |  1  |
 286                          *     1     |  1  |  0  |  1  |  0  |
 287                          *     2     |  0  |  1  |  0  |  1  |
 288                          *     3     |  1  |  0  |  1  |  0  |
 289                          * ----------+-----+-----+-----+-----+
 290                          */
 291                         switch (rank) {
 292                         case 0:
 293                                 odt_mask_0 = 0x4;
 294                                 odt_mask_1 = 0x5;
 295                                 break;
 296                         case 1:
 297                                 odt_mask_0 = 0x8;
 298                                 odt_mask_1 = 0xA;
 299                                 break;
 300                         case 2:
 301                                 odt_mask_0 = 0x1;
 302                                 odt_mask_1 = 0x5;
 303                                 break;
 304                         case 3:
 305                                 odt_mask_0 = 0x2;
 306                                 odt_mask_1 = 0xA;
 307                                 break;
 308                         }
 309                 }
 310         } else {
 311                 odt_mask_0 = 0x0;
 312                 odt_mask_1 = 0x0;
 313         }
 314
 315         cs_and_odt_mask =
 316                 (0xFF & ~(1 << rank)) |
 317                 ((0xFF & odt_mask_0) << 8) |
 318                 ((0xFF & odt_mask_1) << 16);
 319         addr = sdr_get_addr((u32 *)RW_MGR_SET_CS_AND_ODT_MASK);
 320         writel(cs_and_odt_mask, SOCFPGA_SDR_ADDRESS + addr);
 321 }
 322
 323 static void scc_mgr_initialize(void)
 324 {
 325         u32 addr = sdr_get_addr((u32 *)SCC_MGR_HHP_RFILE);
 326
 327         /*
 328          * Clear register file for HPS
 329          * 16 (2^4) is the size of the full register file in the scc mgr:
 330          *      RFILE_DEPTH = log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
 331          * MEM_IF_READ_DQS_WIDTH - 1) + 1;
 332          */
 333         uint32_t i;
 334         for (i = 0; i < 16; i++) {
 335                 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u",
 336                            __func__, __LINE__, i);
 337                 writel(0, SOCFPGA_SDR_ADDRESS + addr + (i << 2));
 338         }
 339 }
 340
 341 static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group,
 342                                                 uint32_t delay)
 343 {
 344         u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_IN_DELAY);
 345
 346         /* Load the setting in the SCC manager */
 347         writel(delay, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
 348 }
 349
 350 static void scc_mgr_set_dqs_io_in_delay(uint32_t write_group,
 351         uint32_t delay)
 352 {
 353         u32 addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY);
 354
 355         writel(delay, SOCFPGA_SDR_ADDRESS + addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
 356 }
 357
 358 static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
 359 {
 360         u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_EN_PHASE);
 361
 362         /* Load the setting in the SCC manager */
 363         writel(phase, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
 364 }
 365
 366 static void scc_mgr_set_dqs_en_phase_all_ranks(uint32_t read_group,
 367                                                uint32_t phase)
 368 {
 369         uint32_t r;
 370         uint32_t update_scan_chains;
 371         uint32_t addr;
 372
 373         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 374              r += NUM_RANKS_PER_SHADOW_REG) {
 375                 /*
 376                  * USER although the h/w doesn't support different phases per
 377                  * shadow register, for simplicity our scc manager modeling
 378                  * keeps different phase settings per shadow reg, and it's
 379                  * important for us to keep them in sync to match h/w.
 380                  * for efficiency, the scan chain update should occur only
 381                  * once to sr0.
 382                  */
 383                 update_scan_chains = (r == 0) ? 1 : 0;
 384
 385                 scc_mgr_set_dqs_en_phase(read_group, phase);
 386
 387                 if (update_scan_chains) {
 388                         addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
 389                         writel(read_group, SOCFPGA_SDR_ADDRESS + addr);
 390
 391                         addr = sdr_get_addr(&sdr_scc_mgr->update);
 392                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 393                 }
 394         }
 395 }
 396
 397 static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group,
 398                                                   uint32_t phase)
 399 {
 400         u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQDQS_OUT_PHASE);
 401
 402         /* Load the setting in the SCC manager */
 403         writel(phase, SOCFPGA_SDR_ADDRESS + addr + (write_group << 2));
 404 }
 405
 406 static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
 407                                                      uint32_t phase)
 408 {
 409         uint32_t r;
 410         uint32_t update_scan_chains;
 411         uint32_t addr;
 412
 413         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 414              r += NUM_RANKS_PER_SHADOW_REG) {
 415                 /*
 416                  * USER although the h/w doesn't support different phases per
 417                  * shadow register, for simplicity our scc manager modeling
 418                  * keeps different phase settings per shadow reg, and it's
 419                  * important for us to keep them in sync to match h/w.
 420                  * for efficiency, the scan chain update should occur only
 421                  * once to sr0.
 422                  */
 423                 update_scan_chains = (r == 0) ? 1 : 0;
 424
 425                 scc_mgr_set_dqdqs_output_phase(write_group, phase);
 426
 427                 if (update_scan_chains) {
 428                         addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
 429                         writel(write_group, SOCFPGA_SDR_ADDRESS + addr);
 430
 431                         addr = sdr_get_addr(&sdr_scc_mgr->update);
 432                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 433                 }
 434         }
 435 }
 436
 437 static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
 438 {
 439         uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_DQS_EN_DELAY);
 440
 441         /* Load the setting in the SCC manager */
 442         writel(delay + IO_DQS_EN_DELAY_OFFSET, SOCFPGA_SDR_ADDRESS + addr +
 443                (read_group << 2));
 444 }
 445
 446 static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
 447                                                uint32_t delay)
 448 {
 449         uint32_t r;
 450         uint32_t addr;
 451
 452         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 453                 r += NUM_RANKS_PER_SHADOW_REG) {
 454                 scc_mgr_set_dqs_en_delay(read_group, delay);
 455
 456                 addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
 457                 writel(read_group, SOCFPGA_SDR_ADDRESS + addr);
 458                 /*
 459                  * In shadow register mode, the T11 settings are stored in
 460                  * registers in the core, which are updated by the DQS_ENA
 461                  * signals. Not issuing the SCC_MGR_UPD command allows us to
 462                  * save lots of rank switching overhead, by calling
 463                  * select_shadow_regs_for_update with update_scan_chains
 464                  * set to 0.
 465                  */
 466                 addr = sdr_get_addr(&sdr_scc_mgr->update);
 467                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
 468         }
 469         /*
 470          * In shadow register mode, the T11 settings are stored in
 471          * registers in the core, which are updated by the DQS_ENA
 472          * signals. Not issuing the SCC_MGR_UPD command allows us to
 473          * save lots of rank switching overhead, by calling
 474          * select_shadow_regs_for_update with update_scan_chains
 475          * set to 0.
 476          */
 477         addr = sdr_get_addr(&sdr_scc_mgr->update);
 478         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 479 }
 480
 481 static void scc_mgr_set_oct_out1_delay(uint32_t write_group, uint32_t delay)
 482 {
 483         uint32_t read_group;
 484         uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_OCT_OUT1_DELAY);
 485
 486         /*
 487          * Load the setting in the SCC manager
 488          * Although OCT affects only write data, the OCT delay is controlled
 489          * by the DQS logic block which is instantiated once per read group.
 490          * For protocols where a write group consists of multiple read groups,
 491          * the setting must be set multiple times.
 492          */
 493         for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 494              RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 495              read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 496              RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group)
 497                 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
 498 }
 499
 500 static void scc_mgr_set_dq_out1_delay(uint32_t write_group,
 501                                       uint32_t dq_in_group, uint32_t delay)
 502 {
 503         uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
 504
 505         /* Load the setting in the SCC manager */
 506         writel(delay, SOCFPGA_SDR_ADDRESS + addr + (dq_in_group << 2));
 507 }
 508
 509 static void scc_mgr_set_dq_in_delay(uint32_t write_group,
 510         uint32_t dq_in_group, uint32_t delay)
 511 {
 512         uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY);
 513
 514         /* Load the setting in the SCC manager */
 515         writel(delay, SOCFPGA_SDR_ADDRESS + addr + (dq_in_group << 2));
 516 }
 517
 518 static void scc_mgr_set_hhp_extras(void)
 519 {
 520         /*
 521          * Load the fixed setting in the SCC manager
 522          * bits: 0:0 = 1'b1   - dqs bypass
 523          * bits: 1:1 = 1'b1   - dq bypass
 524          * bits: 4:2 = 3'b001   - rfifo_mode
 525          * bits: 6:5 = 2'b01  - rfifo clock_select
 526          * bits: 7:7 = 1'b0  - separate gating from ungating setting
 527          * bits: 8:8 = 1'b0  - separate OE from Output delay setting
 528          */
 529         uint32_t value = (0<<8) | (0<<7) | (1<<5) | (1<<2) | (1<<1) | (1<<0);
 530         uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_HHP_GLOBALS);
 531
 532         writel(value, SOCFPGA_SDR_ADDRESS + addr + SCC_MGR_HHP_EXTRAS_OFFSET);
 533 }
 534
 535 static void scc_mgr_set_dqs_out1_delay(uint32_t write_group,
 536                                               uint32_t delay)
 537 {
 538         uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
 539
 540         /* Load the setting in the SCC manager */
 541         writel(delay, SOCFPGA_SDR_ADDRESS + addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
 542 }
 543
 544 static void scc_mgr_set_dm_out1_delay(uint32_t write_group,
 545                                              uint32_t dm, uint32_t delay)
 546 {
 547         uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
 548
 549         /* Load the setting in the SCC manager */
 550         writel(delay, SOCFPGA_SDR_ADDRESS + addr +
 551                 ((RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm) << 2));
 552 }
 553
 554 /*
 555  * USER Zero all DQS config
 556  * TODO: maybe rename to scc_mgr_zero_dqs_config (or something)
 557  */
 558 static void scc_mgr_zero_all(void)
 559 {
 560         uint32_t i, r;
 561         uint32_t addr;
 562
 563         /*
 564          * USER Zero all DQS config settings, across all groups and all
 565          * shadow registers
 566          */
 567         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
 568              NUM_RANKS_PER_SHADOW_REG) {
 569                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
 570                         /*
 571                          * The phases actually don't exist on a per-rank basis,
 572                          * but there's no harm updating them several times, so
 573                          * let's keep the code simple.
 574                          */
 575                         scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
 576                         scc_mgr_set_dqs_en_phase(i, 0);
 577                         scc_mgr_set_dqs_en_delay(i, 0);
 578                 }
 579
 580                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
 581                         scc_mgr_set_dqdqs_output_phase(i, 0);
 582                         /* av/cv don't have out2 */
 583                         scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
 584                 }
 585         }
 586
 587         /* multicast to all DQS group enables */
 588         addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
 589         writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
 590
 591         addr = sdr_get_addr(&sdr_scc_mgr->update);
 592         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 593 }
 594
 595 static void scc_set_bypass_mode(uint32_t write_group, uint32_t mode)
 596 {
 597         uint32_t addr;
 598         /* mode = 0 : Do NOT bypass - Half Rate Mode */
 599         /* mode = 1 : Bypass - Full Rate Mode */
 600
 601         /* only need to set once for all groups, pins, dq, dqs, dm */
 602         if (write_group == 0) {
 603                 debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n", __func__,
 604                            __LINE__);
 605                 scc_mgr_set_hhp_extras();
 606                 debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
 607                           __func__, __LINE__);
 608         }
 609         /* multicast to all DQ enables */
 610         addr = sdr_get_addr(&sdr_scc_mgr->dq_ena);
 611         writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
 612
 613         addr = sdr_get_addr(&sdr_scc_mgr->dm_ena);
 614         writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
 615
 616         /* update current DQS IO enable */
 617         addr = sdr_get_addr(&sdr_scc_mgr->dqs_io_ena);
 618         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 619
 620         /* update the DQS logic */
 621         addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
 622         writel(write_group, SOCFPGA_SDR_ADDRESS + addr);
 623
 624         /* hit update */
 625         addr = sdr_get_addr(&sdr_scc_mgr->update);
 626         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 627 }
 628
 629 static void scc_mgr_zero_group(uint32_t write_group, uint32_t test_begin,
 630                                int32_t out_only)
 631 {
 632         uint32_t i, r;
 633         uint32_t addr;
 634
 635         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
 636                 NUM_RANKS_PER_SHADOW_REG) {
 637                 /* Zero all DQ config settings */
 638                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
 639                         scc_mgr_set_dq_out1_delay(write_group, i, 0);
 640                         if (!out_only)
 641                                 scc_mgr_set_dq_in_delay(write_group, i, 0);
 642                 }
 643
 644                 /* multicast to all DQ enables */
 645                 addr = sdr_get_addr(&sdr_scc_mgr->dq_ena);
 646                 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
 647
 648                 /* Zero all DM config settings */
 649                 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 650                         scc_mgr_set_dm_out1_delay(write_group, i, 0);
 651                 }
 652
 653                 /* multicast to all DM enables */
 654                 addr = sdr_get_addr(&sdr_scc_mgr->dm_ena);
 655                 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
 656
 657                 /* zero all DQS io settings */
 658                 if (!out_only)
 659                         scc_mgr_set_dqs_io_in_delay(write_group, 0);
 660                 /* av/cv don't have out2 */
 661                 scc_mgr_set_dqs_out1_delay(write_group, IO_DQS_OUT_RESERVE);
 662                 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
 663                 scc_mgr_load_dqs_for_write_group(write_group);
 664
 665                 /* multicast to all DQS IO enables (only 1) */
 666                 addr = sdr_get_addr(&sdr_scc_mgr->dqs_io_ena);
 667                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
 668
 669                 /* hit update to zero everything */
 670                 addr = sdr_get_addr(&sdr_scc_mgr->update);
 671                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
 672         }
 673 }
 674
 675 /* load up dqs config settings */
 676 static void scc_mgr_load_dqs(uint32_t dqs)
 677 {
 678         uint32_t addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
 679
 680         writel(dqs, SOCFPGA_SDR_ADDRESS + addr);
 681 }
 682
 683 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group)
 684 {
 685         uint32_t read_group;
 686         uint32_t addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
 687         /*
 688          * Although OCT affects only write data, the OCT delay is controlled
 689          * by the DQS logic block which is instantiated once per read group.
 690          * For protocols where a write group consists of multiple read groups,
 691          * the setting must be scanned multiple times.
 692          */
 693         for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 694              RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 695              read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH /
 696              RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group)
 697                 writel(read_group, SOCFPGA_SDR_ADDRESS + addr);
 698 }
 699
 700 /* load up dqs io config settings */
 701 static void scc_mgr_load_dqs_io(void)
 702 {
 703         uint32_t addr = sdr_get_addr(&sdr_scc_mgr->dqs_io_ena);
 704
 705         writel(0, SOCFPGA_SDR_ADDRESS + addr);
 706 }
 707
 708 /* load up dq config settings */
 709 static void scc_mgr_load_dq(uint32_t dq_in_group)
 710 {
 711         uint32_t addr = sdr_get_addr(&sdr_scc_mgr->dq_ena);
 712
 713         writel(dq_in_group, SOCFPGA_SDR_ADDRESS + addr);
 714 }
 715
 716 /* load up dm config settings */
 717 static void scc_mgr_load_dm(uint32_t dm)
 718 {
 719         uint32_t addr = sdr_get_addr(&sdr_scc_mgr->dm_ena);
 720
 721         writel(dm, SOCFPGA_SDR_ADDRESS + addr);
 722 }
 723
 724 /*
 725  * apply and load a particular input delay for the DQ pins in a group
 726  * group_bgn is the index of the first dq pin (in the write group)
 727  */
 728 static void scc_mgr_apply_group_dq_in_delay(uint32_t write_group,
 729                                             uint32_t group_bgn, uint32_t delay)
 730 {
 731         uint32_t i, p;
 732
 733         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
 734                 scc_mgr_set_dq_in_delay(write_group, p, delay);
 735                 scc_mgr_load_dq(p);
 736         }
 737 }
 738
 739 /* apply and load a particular output delay for the DQ pins in a group */
 740 static void scc_mgr_apply_group_dq_out1_delay(uint32_t write_group,
 741                                               uint32_t group_bgn,
 742                                               uint32_t delay1)
 743 {
 744         uint32_t i, p;
 745
 746         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
 747                 scc_mgr_set_dq_out1_delay(write_group, i, delay1);
 748                 scc_mgr_load_dq(i);
 749         }
 750 }
 751
 752 /* apply and load a particular output delay for the DM pins in a group */
 753 static void scc_mgr_apply_group_dm_out1_delay(uint32_t write_group,
 754                                               uint32_t delay1)
 755 {
 756         uint32_t i;
 757
 758         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 759                 scc_mgr_set_dm_out1_delay(write_group, i, delay1);
 760                 scc_mgr_load_dm(i);
 761         }
 762 }
 763
 764
 765 /* apply and load delay on both DQS and OCT out1 */
 766 static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
 767                                                     uint32_t delay)
 768 {
 769         scc_mgr_set_dqs_out1_delay(write_group, delay);
 770         scc_mgr_load_dqs_io();
 771
 772         scc_mgr_set_oct_out1_delay(write_group, delay);
 773         scc_mgr_load_dqs_for_write_group(write_group);
 774 }
 775
 776 /* apply a delay to the entire output side: DQ, DM, DQS, OCT */
 777 static void scc_mgr_apply_group_all_out_delay_add(uint32_t write_group,
 778                                                   uint32_t group_bgn,
 779                                                   uint32_t delay)
 780 {
 781         uint32_t i, p, new_delay;
 782
 783         /* dq shift */
 784         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
 785                 new_delay = READ_SCC_DQ_OUT2_DELAY;
 786                 new_delay += delay;
 787
 788                 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 789                         debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQ[%u,%u]:\
 790                                    %u > %lu => %lu", __func__, __LINE__,
 791                                    write_group, group_bgn, delay, i, p, new_delay,
 792                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX,
 793                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX);
 794                         new_delay = IO_IO_OUT2_DELAY_MAX;
 795                 }
 796
 797                 scc_mgr_load_dq(i);
 798         }
 799
 800         /* dm shift */
 801         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 802                 new_delay = READ_SCC_DM_IO_OUT2_DELAY;
 803                 new_delay += delay;
 804
 805                 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 806                         debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DM[%u]:\
 807                                    %u > %lu => %lu\n",  __func__, __LINE__,
 808                                    write_group, group_bgn, delay, i, new_delay,
 809                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX,
 810                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX);
 811                         new_delay = IO_IO_OUT2_DELAY_MAX;
 812                 }
 813
 814                 scc_mgr_load_dm(i);
 815         }
 816
 817         /* dqs shift */
 818         new_delay = READ_SCC_DQS_IO_OUT2_DELAY;
 819         new_delay += delay;
 820
 821         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 822                 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
 823                            " adding %u to OUT1\n", __func__, __LINE__,
 824                            write_group, group_bgn, delay, new_delay,
 825                            IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
 826                            new_delay - IO_IO_OUT2_DELAY_MAX);
 827                 scc_mgr_set_dqs_out1_delay(write_group, new_delay -
 828                                            IO_IO_OUT2_DELAY_MAX);
 829                 new_delay = IO_IO_OUT2_DELAY_MAX;
 830         }
 831
 832         scc_mgr_load_dqs_io();
 833
 834         /* oct shift */
 835         new_delay = READ_SCC_OCT_OUT2_DELAY;
 836         new_delay += delay;
 837
 838         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 839                 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
 840                            " adding %u to OUT1\n", __func__, __LINE__,
 841                            write_group, group_bgn, delay, new_delay,
 842                            IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
 843                            new_delay - IO_IO_OUT2_DELAY_MAX);
 844                 scc_mgr_set_oct_out1_delay(write_group, new_delay -
 845                                            IO_IO_OUT2_DELAY_MAX);
 846                 new_delay = IO_IO_OUT2_DELAY_MAX;
 847         }
 848
 849         scc_mgr_load_dqs_for_write_group(write_group);
 850 }
 851
 852 /*
 853  * USER apply a delay to the entire output side (DQ, DM, DQS, OCT)
 854  * and to all ranks
 855  */
 856 static void scc_mgr_apply_group_all_out_delay_add_all_ranks(
 857         uint32_t write_group, uint32_t group_bgn, uint32_t delay)
 858 {
 859         uint32_t r;
 860         uint32_t addr = sdr_get_addr(&sdr_scc_mgr->update);
 861
 862         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 863                 r += NUM_RANKS_PER_SHADOW_REG) {
 864                 scc_mgr_apply_group_all_out_delay_add(write_group,
 865                                                       group_bgn, delay);
 866                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
 867         }
 868 }
 869
 870 /* optimization used to recover some slots in ddr3 inst_rom */
 871 /* could be applied to other protocols if we wanted to */
 872 static void set_jump_as_return(void)
 873 {
 874         uint32_t addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
 875
 876         /*
 877          * to save space, we replace return with jump to special shared
 878          * RETURN instruction so we set the counter to large value so that
 879          * we always jump
 880          */
 881         writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
 882         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
 883         writel(RW_MGR_RETURN, SOCFPGA_SDR_ADDRESS + addr);
 884 }
 885
 886 /*
 887  * should always use constants as argument to ensure all computations are
 888  * performed at compile time
 889  */
 890 static void delay_for_n_mem_clocks(const uint32_t clocks)
 891 {
 892         uint32_t afi_clocks;
 893         uint8_t inner = 0;
 894         uint8_t outer = 0;
 895         uint16_t c_loop = 0;
 896         uint32_t addr;
 897
 898         debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
 899
 900
 901         afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
 902         /* scale (rounding up) to get afi clocks */
 903
 904         /*
 905          * Note, we don't bother accounting for being off a little bit
 906          * because of a few extra instructions in outer loops
 907          * Note, the loops have a test at the end, and do the test before
 908          * the decrement, and so always perform the loop
 909          * 1 time more than the counter value
 910          */
 911         if (afi_clocks == 0) {
 912                 ;
 913         } else if (afi_clocks <= 0x100) {
 914                 inner = afi_clocks-1;
 915                 outer = 0;
 916                 c_loop = 0;
 917         } else if (afi_clocks <= 0x10000) {
 918                 inner = 0xff;
 919                 outer = (afi_clocks-1) >> 8;
 920                 c_loop = 0;
 921         } else {
 922                 inner = 0xff;
 923                 outer = 0xff;
 924                 c_loop = (afi_clocks-1) >> 16;
 925         }
 926
 927         /*
 928          * rom instructions are structured as follows:
 929          *
 930          *    IDLE_LOOP2: jnz cntr0, TARGET_A
 931          *    IDLE_LOOP1: jnz cntr1, TARGET_B
 932          *                return
 933          *
 934          * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
 935          * TARGET_B is set to IDLE_LOOP2 as well
 936          *
 937          * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
 938          * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
 939          *
 940          * a little confusing, but it helps save precious space in the inst_rom
 941          * and sequencer rom and keeps the delays more accurate and reduces
 942          * overhead
 943          */
 944         if (afi_clocks <= 0x100) {
 945                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
 946                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), SOCFPGA_SDR_ADDRESS + addr);
 947
 948                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
 949                 writel(RW_MGR_IDLE_LOOP1, SOCFPGA_SDR_ADDRESS + addr);
 950
 951                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
 952                 writel(RW_MGR_IDLE_LOOP1, SOCFPGA_SDR_ADDRESS + addr);
 953         } else {
 954                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
 955                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), SOCFPGA_SDR_ADDRESS + addr);
 956
 957                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
 958                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer), SOCFPGA_SDR_ADDRESS + addr);
 959
 960                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
 961                 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
 962
 963                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
 964                 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
 965
 966                 /* hack to get around compiler not being smart enough */
 967                 if (afi_clocks <= 0x10000) {
 968                         /* only need to run once */
 969                         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
 970                         writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
 971                 } else {
 972                         do {
 973                                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
 974                                 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
 975                         } while (c_loop-- != 0);
 976                 }
 977         }
 978         debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
 979 }
 980
 981 static void rw_mgr_mem_initialize(void)
 982 {
 983         uint32_t r;
 984         uint32_t addr;
 985
 986         debug("%s:%d\n", __func__, __LINE__);
 987
 988         /* The reset / cke part of initialization is broadcasted to all ranks */
 989         addr = sdr_get_addr((u32 *)RW_MGR_SET_CS_AND_ODT_MASK);
 990         writel(RW_MGR_RANK_ALL, SOCFPGA_SDR_ADDRESS + addr);
 991
 992         /*
 993          * Here's how you load register for a loop
 994          * Counters are located @ 0x800
 995          * Jump address are located @ 0xC00
 996          * For both, registers 0 to 3 are selected using bits 3 and 2, like
 997          * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
 998          * I know this ain't pretty, but Avalon bus throws away the 2 least
 999          * significant bits
1000          */
1001
1002         /* start with memory RESET activated */
1003
1004         /* tINIT = 200us */
1005
1006         /*
1007          * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
1008          * If a and b are the number of iteration in 2 nested loops
1009          * it takes the following number of cycles to complete the operation:
1010          * number_of_cycles = ((2 + n) * a + 2) * b
1011          * where n is the number of instruction in the inner loop
1012          * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
1013          * b = 6A
1014          */
1015
1016         /* Load counters */
1017         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
1018         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL),
1019                SOCFPGA_SDR_ADDRESS + addr);
1020         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
1021         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL),
1022                SOCFPGA_SDR_ADDRESS + addr);
1023         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr2);
1024         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL),
1025                SOCFPGA_SDR_ADDRESS + addr);
1026
1027         /* Load jump address */
1028         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
1029         writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1030
1031         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
1032         writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1033
1034         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
1035         writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1036
1037         /* Execute count instruction */
1038         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1039         writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1040
1041         /* indicate that memory is stable */
1042         addr = sdr_get_addr(&phy_mgr_cfg->reset_mem_stbl);
1043         writel(1, SOCFPGA_SDR_ADDRESS + addr);
1044
1045         /*
1046          * transition the RESET to high
1047          * Wait for 500us
1048          */
1049
1050         /*
1051          * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
1052          * If a and b are the number of iteration in 2 nested loops
1053          * it takes the following number of cycles to complete the operation
1054          * number_of_cycles = ((2 + n) * a + 2) * b
1055          * where n is the number of instruction in the inner loop
1056          * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
1057          * b = FF
1058          */
1059
1060         /* Load counters */
1061         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
1062         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR0_VAL),
1063                SOCFPGA_SDR_ADDRESS + addr);
1064         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
1065         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR1_VAL),
1066                SOCFPGA_SDR_ADDRESS + addr);
1067         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr2);
1068         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR2_VAL),
1069                SOCFPGA_SDR_ADDRESS + addr);
1070
1071         /* Load jump address */
1072         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
1073         writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1074         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
1075         writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1076         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
1077         writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1078
1079         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1080         writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1081
1082         /* bring up clock enable */
1083
1084         /* tXRP < 250 ck cycles */
1085         delay_for_n_mem_clocks(250);
1086
1087         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
1088                 if (param->skip_ranks[r]) {
1089                         /* request to skip the rank */
1090                         continue;
1091                 }
1092
1093                 /* set rank */
1094                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
1095
1096                 /*
1097                  * USER Use Mirror-ed commands for odd ranks if address
1098                  * mirrorring is on
1099                  */
1100                 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
1101                         set_jump_as_return();
1102                         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1103                         writel(RW_MGR_MRS2_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1104                         delay_for_n_mem_clocks(4);
1105                         set_jump_as_return();
1106                         writel(RW_MGR_MRS3_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1107                         delay_for_n_mem_clocks(4);
1108                         set_jump_as_return();
1109                         writel(RW_MGR_MRS1_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1110                         delay_for_n_mem_clocks(4);
1111                         set_jump_as_return();
1112                         writel(RW_MGR_MRS0_DLL_RESET_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1113                 } else {
1114                         set_jump_as_return();
1115                         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1116                         writel(RW_MGR_MRS2, SOCFPGA_SDR_ADDRESS + addr);
1117                         delay_for_n_mem_clocks(4);
1118                         set_jump_as_return();
1119                         writel(RW_MGR_MRS3, SOCFPGA_SDR_ADDRESS + addr);
1120                         delay_for_n_mem_clocks(4);
1121                         set_jump_as_return();
1122                         writel(RW_MGR_MRS1, SOCFPGA_SDR_ADDRESS + addr);
1123                         set_jump_as_return();
1124                         writel(RW_MGR_MRS0_DLL_RESET, SOCFPGA_SDR_ADDRESS + addr);
1125                 }
1126                 set_jump_as_return();
1127                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1128                 writel(RW_MGR_ZQCL, SOCFPGA_SDR_ADDRESS + addr);
1129
1130                 /* tZQinit = tDLLK = 512 ck cycles */
1131                 delay_for_n_mem_clocks(512);
1132         }
1133 }
1134
1135 /*
1136  * At the end of calibration we have to program the user settings in, and
1137  * USER  hand off the memory to the user.
1138  */
1139 static void rw_mgr_mem_handoff(void)
1140 {
1141         uint32_t r;
1142         uint32_t addr;
1143
1144         debug("%s:%d\n", __func__, __LINE__);
1145         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
1146                 if (param->skip_ranks[r])
1147                         /* request to skip the rank */
1148                         continue;
1149                 /* set rank */
1150                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
1151
1152                 /* precharge all banks ... */
1153                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1154                 writel(RW_MGR_PRECHARGE_ALL, SOCFPGA_SDR_ADDRESS + addr);
1155
1156                 /* load up MR settings specified by user */
1157
1158                 /*
1159                  * Use Mirror-ed commands for odd ranks if address
1160                  * mirrorring is on
1161                  */
1162                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1163                 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
1164                         set_jump_as_return();
1165                         writel(RW_MGR_MRS2_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1166                         delay_for_n_mem_clocks(4);
1167                         set_jump_as_return();
1168                         writel(RW_MGR_MRS3_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1169                         delay_for_n_mem_clocks(4);
1170                         set_jump_as_return();
1171                         writel(RW_MGR_MRS1_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1172                         delay_for_n_mem_clocks(4);
1173                         set_jump_as_return();
1174                         writel(RW_MGR_MRS0_USER_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1175                 } else {
1176                         set_jump_as_return();
1177                         writel(RW_MGR_MRS2, SOCFPGA_SDR_ADDRESS + addr);
1178                         delay_for_n_mem_clocks(4);
1179                         set_jump_as_return();
1180                         writel(RW_MGR_MRS3, SOCFPGA_SDR_ADDRESS + addr);
1181                         delay_for_n_mem_clocks(4);
1182                         set_jump_as_return();
1183                         writel(RW_MGR_MRS1, SOCFPGA_SDR_ADDRESS + addr);
1184                         delay_for_n_mem_clocks(4);
1185                         set_jump_as_return();
1186                         writel(RW_MGR_MRS0_USER, SOCFPGA_SDR_ADDRESS + addr);
1187                 }
1188                 /*
1189                  * USER  need to wait tMOD (12CK or 15ns) time before issuing
1190                  * other commands, but we will have plenty of NIOS cycles before
1191                  * actual handoff so its okay.
1192                  */
1193         }
1194 }
1195
1196 /*
1197  * performs a guaranteed read on the patterns we are going to use during a
1198  * read test to ensure memory works
1199  */
1200 static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn,
1201         uint32_t group, uint32_t num_tries, uint32_t *bit_chk,
1202         uint32_t all_ranks)
1203 {
1204         uint32_t r, vg;
1205         uint32_t correct_mask_vg;
1206         uint32_t tmp_bit_chk;
1207         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1208                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1209         uint32_t addr;
1210         uint32_t base_rw_mgr;
1211
1212         *bit_chk = param->read_correct_mask;
1213         correct_mask_vg = param->read_correct_mask_vg;
1214
1215         for (r = rank_bgn; r < rank_end; r++) {
1216                 if (param->skip_ranks[r])
1217                         /* request to skip the rank */
1218                         continue;
1219
1220                 /* set rank */
1221                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1222
1223                 /* Load up a constant bursts of read commands */
1224                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
1225                 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1226                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
1227                 writel(RW_MGR_GUARANTEED_READ, SOCFPGA_SDR_ADDRESS + addr);
1228
1229                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
1230                 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1231                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
1232                 writel(RW_MGR_GUARANTEED_READ_CONT, SOCFPGA_SDR_ADDRESS + addr);
1233
1234                 tmp_bit_chk = 0;
1235                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1236                         /* reset the fifos to get pointers to known state */
1237
1238                         addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset);
1239                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
1240                         addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH);
1241                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
1242
1243                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1244                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1245
1246                         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1247                         writel(RW_MGR_GUARANTEED_READ, SOCFPGA_SDR_ADDRESS + addr +
1248                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1249                                 vg) << 2));
1250
1251                         addr = sdr_get_addr((u32 *)BASE_RW_MGR);
1252                         base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr);
1253                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr));
1254
1255                         if (vg == 0)
1256                                 break;
1257                 }
1258                 *bit_chk &= tmp_bit_chk;
1259         }
1260
1261         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1262         writel(RW_MGR_CLEAR_DQS_ENABLE, SOCFPGA_SDR_ADDRESS + addr + (group << 2));
1263
1264         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1265         debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\
1266                    %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask,
1267                    (long unsigned int)(*bit_chk == param->read_correct_mask));
1268         return *bit_chk == param->read_correct_mask;
1269 }
1270
1271 static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks
1272         (uint32_t group, uint32_t num_tries, uint32_t *bit_chk)
1273 {
1274         return rw_mgr_mem_calibrate_read_test_patterns(0, group,
1275                 num_tries, bit_chk, 1);
1276 }
1277
1278 /* load up the patterns we are going to use during a read test */
1279 static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn,
1280         uint32_t all_ranks)
1281 {
1282         uint32_t r;
1283         uint32_t addr;
1284         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1285                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1286
1287         debug("%s:%d\n", __func__, __LINE__);
1288         for (r = rank_bgn; r < rank_end; r++) {
1289                 if (param->skip_ranks[r])
1290                         /* request to skip the rank */
1291                         continue;
1292
1293                 /* set rank */
1294                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1295
1296                 /* Load up a constant bursts */
1297                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
1298                 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1299
1300                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
1301                 writel(RW_MGR_GUARANTEED_WRITE_WAIT0, SOCFPGA_SDR_ADDRESS + addr);
1302
1303                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
1304                 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1305
1306                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
1307                 writel(RW_MGR_GUARANTEED_WRITE_WAIT1, SOCFPGA_SDR_ADDRESS + addr);
1308
1309                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr2);
1310                 writel(0x04, SOCFPGA_SDR_ADDRESS + addr);
1311
1312                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
1313                 writel(RW_MGR_GUARANTEED_WRITE_WAIT2, SOCFPGA_SDR_ADDRESS + addr);
1314
1315                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr3);
1316                 writel(0x04, SOCFPGA_SDR_ADDRESS + addr);
1317
1318                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add3);
1319                 writel(RW_MGR_GUARANTEED_WRITE_WAIT3, SOCFPGA_SDR_ADDRESS + addr);
1320
1321                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1322                 writel(RW_MGR_GUARANTEED_WRITE, SOCFPGA_SDR_ADDRESS + addr);
1323         }
1324
1325         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1326 }
1327
1328 /*
1329  * try a read and see if it returns correct data back. has dummy reads
1330  * inserted into the mix used to align dqs enable. has more thorough checks
1331  * than the regular read test.
1332  */
1333 static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
1334         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1335         uint32_t all_groups, uint32_t all_ranks)
1336 {
1337         uint32_t r, vg;
1338         uint32_t correct_mask_vg;
1339         uint32_t tmp_bit_chk;
1340         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1341                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1342         uint32_t addr;
1343         uint32_t base_rw_mgr;
1344
1345         *bit_chk = param->read_correct_mask;
1346         correct_mask_vg = param->read_correct_mask_vg;
1347
1348         uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
1349                 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
1350
1351         for (r = rank_bgn; r < rank_end; r++) {
1352                 if (param->skip_ranks[r])
1353                         /* request to skip the rank */
1354                         continue;
1355
1356                 /* set rank */
1357                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1358
1359                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
1360                 writel(0x10, SOCFPGA_SDR_ADDRESS + addr);
1361
1362                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
1363                 writel(RW_MGR_READ_B2B_WAIT1, SOCFPGA_SDR_ADDRESS + addr);
1364
1365                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr2);
1366                 writel(0x10, SOCFPGA_SDR_ADDRESS + addr);
1367                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
1368                 writel(RW_MGR_READ_B2B_WAIT2, SOCFPGA_SDR_ADDRESS + addr);
1369
1370                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
1371                 if (quick_read_mode)
1372                         writel(0x1, SOCFPGA_SDR_ADDRESS + addr);
1373                         /* need at least two (1+1) reads to capture failures */
1374                 else if (all_groups)
1375                         writel(0x06, SOCFPGA_SDR_ADDRESS + addr);
1376                 else
1377                         writel(0x32, SOCFPGA_SDR_ADDRESS + addr);
1378
1379                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
1380                 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr);
1381                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr3);
1382                 if (all_groups)
1383                         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
1384                                RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
1385                                SOCFPGA_SDR_ADDRESS + addr);
1386                 else
1387                         writel(0x0, SOCFPGA_SDR_ADDRESS + addr);
1388
1389                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add3);
1390                 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr);
1391
1392                 tmp_bit_chk = 0;
1393                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1394                         /* reset the fifos to get pointers to known state */
1395                         addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset);
1396                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
1397                         addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH);
1398                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
1399
1400                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1401                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1402
1403                         addr = sdr_get_addr((u32 *)(all_groups ? RW_MGR_RUN_ALL_GROUPS :
1404                                             RW_MGR_RUN_SINGLE_GROUP));
1405                         writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr +
1406                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1407                                vg) << 2));
1408
1409                         addr = sdr_get_addr((u32 *)BASE_RW_MGR);
1410                         base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr);
1411                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
1412
1413                         if (vg == 0)
1414                                 break;
1415                 }
1416                 *bit_chk &= tmp_bit_chk;
1417         }
1418
1419         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1420         writel(RW_MGR_CLEAR_DQS_ENABLE, SOCFPGA_SDR_ADDRESS + addr + (group << 2));
1421
1422         if (all_correct) {
1423                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1424                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
1425                            (%u == %u) => %lu", __func__, __LINE__, group,
1426                            all_groups, *bit_chk, param->read_correct_mask,
1427                            (long unsigned int)(*bit_chk ==
1428                            param->read_correct_mask));
1429                 return *bit_chk == param->read_correct_mask;
1430         } else  {
1431                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1432                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
1433                            (%u != %lu) => %lu\n", __func__, __LINE__,
1434                            group, all_groups, *bit_chk, (long unsigned int)0,
1435                            (long unsigned int)(*bit_chk != 0x00));
1436                 return *bit_chk != 0x00;
1437         }
1438 }
1439
1440 static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
1441         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1442         uint32_t all_groups)
1443 {
1444         return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
1445                                               bit_chk, all_groups, 1);
1446 }
1447
1448 static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
1449 {
1450         uint32_t addr = sdr_get_addr(&phy_mgr_cmd->inc_vfifo_hard_phy);
1451
1452         writel(grp, SOCFPGA_SDR_ADDRESS + addr);
1453         (*v)++;
1454 }
1455
1456 static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
1457 {
1458         uint32_t i;
1459
1460         for (i = 0; i < VFIFO_SIZE-1; i++)
1461                 rw_mgr_incr_vfifo(grp, v);
1462 }
1463
1464 static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
1465 {
1466         uint32_t  v;
1467         uint32_t fail_cnt = 0;
1468         uint32_t test_status;
1469
1470         for (v = 0; v < VFIFO_SIZE; ) {
1471                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
1472                            __func__, __LINE__, v);
1473                 test_status = rw_mgr_mem_calibrate_read_test_all_ranks
1474                         (grp, 1, PASS_ONE_BIT, bit_chk, 0);
1475                 if (!test_status) {
1476                         fail_cnt++;
1477
1478                         if (fail_cnt == 2)
1479                                 break;
1480                 }
1481
1482                 /* fiddle with FIFO */
1483                 rw_mgr_incr_vfifo(grp, &v);
1484         }
1485
1486         if (v >= VFIFO_SIZE) {
1487                 /* no failing read found!! Something must have gone wrong */
1488                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
1489                            __func__, __LINE__);
1490                 return 0;
1491         } else {
1492                 return v;
1493         }
1494 }
1495
1496 static int find_working_phase(uint32_t *grp, uint32_t *bit_chk,
1497                               uint32_t dtaps_per_ptap, uint32_t *work_bgn,
1498                               uint32_t *v, uint32_t *d, uint32_t *p,
1499                               uint32_t *i, uint32_t *max_working_cnt)
1500 {
1501         uint32_t found_begin = 0;
1502         uint32_t tmp_delay = 0;
1503         uint32_t test_status;
1504
1505         for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
1506                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1507                 *work_bgn = tmp_delay;
1508                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1509
1510                 for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
1511                         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
1512                                 IO_DELAY_PER_OPA_TAP) {
1513                                 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1514
1515                                 test_status =
1516                                 rw_mgr_mem_calibrate_read_test_all_ranks
1517                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0);
1518
1519                                 if (test_status) {
1520                                         *max_working_cnt = 1;
1521                                         found_begin = 1;
1522                                         break;
1523                                 }
1524                         }
1525
1526                         if (found_begin)
1527                                 break;
1528
1529                         if (*p > IO_DQS_EN_PHASE_MAX)
1530                                 /* fiddle with FIFO */
1531                                 rw_mgr_incr_vfifo(*grp, v);
1532                 }
1533
1534                 if (found_begin)
1535                         break;
1536         }
1537
1538         if (*i >= VFIFO_SIZE) {
1539                 /* cannot find working solution */
1540                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
1541                            ptap/dtap\n", __func__, __LINE__);
1542                 return 0;
1543         } else {
1544                 return 1;
1545         }
1546 }
1547
1548 static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk,
1549                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1550                              uint32_t *p, uint32_t *max_working_cnt)
1551 {
1552         uint32_t found_begin = 0;
1553         uint32_t tmp_delay;
1554
1555         /* Special case code for backing up a phase */
1556         if (*p == 0) {
1557                 *p = IO_DQS_EN_PHASE_MAX;
1558                 rw_mgr_decr_vfifo(*grp, v);
1559         } else {
1560                 (*p)--;
1561         }
1562         tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
1563         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1564
1565         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
1566                 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1567                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1568
1569                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1570                                                              PASS_ONE_BIT,
1571                                                              bit_chk, 0)) {
1572                         found_begin = 1;
1573                         *work_bgn = tmp_delay;
1574                         break;
1575                 }
1576         }
1577
1578         /* We have found a working dtap before the ptap found above */
1579         if (found_begin == 1)
1580                 (*max_working_cnt)++;
1581
1582         /*
1583          * Restore VFIFO to old state before we decremented it
1584          * (if needed).
1585          */
1586         (*p)++;
1587         if (*p > IO_DQS_EN_PHASE_MAX) {
1588                 *p = 0;
1589                 rw_mgr_incr_vfifo(*grp, v);
1590         }
1591
1592         scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0);
1593 }
1594
1595 static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk,
1596                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1597                              uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
1598                              uint32_t *work_end)
1599 {
1600         uint32_t found_end = 0;
1601
1602         (*p)++;
1603         *work_end += IO_DELAY_PER_OPA_TAP;
1604         if (*p > IO_DQS_EN_PHASE_MAX) {
1605                 /* fiddle with FIFO */
1606                 *p = 0;
1607                 rw_mgr_incr_vfifo(*grp, v);
1608         }
1609
1610         for (; *i < VFIFO_SIZE + 1; (*i)++) {
1611                 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
1612                         += IO_DELAY_PER_OPA_TAP) {
1613                         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1614
1615                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1616                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
1617                                 found_end = 1;
1618                                 break;
1619                         } else {
1620                                 (*max_working_cnt)++;
1621                         }
1622                 }
1623
1624                 if (found_end)
1625                         break;
1626
1627                 if (*p > IO_DQS_EN_PHASE_MAX) {
1628                         /* fiddle with FIFO */
1629                         rw_mgr_incr_vfifo(*grp, v);
1630                         *p = 0;
1631                 }
1632         }
1633
1634         if (*i >= VFIFO_SIZE + 1) {
1635                 /* cannot see edge of failing read */
1636                 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
1637                            failed\n", __func__, __LINE__);
1638                 return 0;
1639         } else {
1640                 return 1;
1641         }
1642 }
1643
1644 static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk,
1645                                   uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1646                                   uint32_t *p, uint32_t *work_mid,
1647                                   uint32_t *work_end)
1648 {
1649         int i;
1650         int tmp_delay = 0;
1651
1652         *work_mid = (*work_bgn + *work_end) / 2;
1653
1654         debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
1655                    *work_bgn, *work_end, *work_mid);
1656         /* Get the middle delay to be less than a VFIFO delay */
1657         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX;
1658                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1659                 ;
1660         debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
1661         while (*work_mid > tmp_delay)
1662                 *work_mid -= tmp_delay;
1663         debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid);
1664
1665         tmp_delay = 0;
1666         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid;
1667                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1668                 ;
1669         tmp_delay -= IO_DELAY_PER_OPA_TAP;
1670         debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay);
1671         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++,
1672                 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP)
1673                 ;
1674         debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay);
1675
1676         scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1);
1677         scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1678
1679         /*
1680          * push vfifo until we can successfully calibrate. We can do this
1681          * because the largest possible margin in 1 VFIFO cycle.
1682          */
1683         for (i = 0; i < VFIFO_SIZE; i++) {
1684                 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
1685                            *v);
1686                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1687                                                              PASS_ONE_BIT,
1688                                                              bit_chk, 0)) {
1689                         break;
1690                 }
1691
1692                 /* fiddle with FIFO */
1693                 rw_mgr_incr_vfifo(*grp, v);
1694         }
1695
1696         if (i >= VFIFO_SIZE) {
1697                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \
1698                            failed\n", __func__, __LINE__);
1699                 return 0;
1700         } else {
1701                 return 1;
1702         }
1703 }
1704
1705 /* find a good dqs enable to use */
1706 static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
1707 {
1708         uint32_t v, d, p, i;
1709         uint32_t max_working_cnt;
1710         uint32_t bit_chk;
1711         uint32_t dtaps_per_ptap;
1712         uint32_t work_bgn, work_mid, work_end;
1713         uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
1714         uint32_t addr;
1715
1716         debug("%s:%d %u\n", __func__, __LINE__, grp);
1717
1718         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
1719
1720         scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
1721         scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
1722
1723         /* ************************************************************** */
1724         /* * Step 0 : Determine number of delay taps for each phase tap * */
1725         dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1726
1727         /* ********************************************************* */
1728         /* * Step 1 : First push vfifo until we get a failing read * */
1729         v = find_vfifo_read(grp, &bit_chk);
1730
1731         max_working_cnt = 0;
1732
1733         /* ******************************************************** */
1734         /* * step 2: find first working phase, increment in ptaps * */
1735         work_bgn = 0;
1736         if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
1737                                 &p, &i, &max_working_cnt) == 0)
1738                 return 0;
1739
1740         work_end = work_bgn;
1741
1742         /*
1743          * If d is 0 then the working window covers a phase tap and
1744          * we can follow the old procedure otherwise, we've found the beginning,
1745          * and we need to increment the dtaps until we find the end.
1746          */
1747         if (d == 0) {
1748                 /* ********************************************************* */
1749                 /* * step 3a: if we have room, back off by one and
1750                 increment in dtaps * */
1751
1752                 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1753                                  &max_working_cnt);
1754
1755                 /* ********************************************************* */
1756                 /* * step 4a: go forward from working phase to non working
1757                 phase, increment in ptaps * */
1758                 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1759                                          &i, &max_working_cnt, &work_end) == 0)
1760                         return 0;
1761
1762                 /* ********************************************************* */
1763                 /* * step 5a:  back off one from last, increment in dtaps  * */
1764
1765                 /* Special case code for backing up a phase */
1766                 if (p == 0) {
1767                         p = IO_DQS_EN_PHASE_MAX;
1768                         rw_mgr_decr_vfifo(grp, &v);
1769                 } else {
1770                         p = p - 1;
1771                 }
1772
1773                 work_end -= IO_DELAY_PER_OPA_TAP;
1774                 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1775
1776                 /* * The actual increment of dtaps is done outside of
1777                 the if/else loop to share code */
1778                 d = 0;
1779
1780                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
1781                            vfifo=%u ptap=%u\n", __func__, __LINE__,
1782                            v, p);
1783         } else {
1784                 /* ******************************************************* */
1785                 /* * step 3-5b:  Find the right edge of the window using
1786                 delay taps   * */
1787                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
1788                            ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
1789                            v, p, d, work_bgn);
1790
1791                 work_end = work_bgn;
1792
1793                 /* * The actual increment of dtaps is done outside of the
1794                 if/else loop to share code */
1795
1796                 /* Only here to counterbalance a subtract later on which is
1797                 not needed if this branch of the algorithm is taken */
1798                 max_working_cnt++;
1799         }
1800
1801         /* The dtap increment to find the failing edge is done here */
1802         for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
1803                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1804                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1805                                    end-2: dtap=%u\n", __func__, __LINE__, d);
1806                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1807
1808                         if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1809                                                                       PASS_ONE_BIT,
1810                                                                       &bit_chk, 0)) {
1811                                 break;
1812                         }
1813         }
1814
1815         /* Go back to working dtap */
1816         if (d != 0)
1817                 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1818
1819         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
1820                    ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
1821                    v, p, d-1, work_end);
1822
1823         if (work_end < work_bgn) {
1824                 /* nil range */
1825                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
1826                            failed\n", __func__, __LINE__);
1827                 return 0;
1828         }
1829
1830         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
1831                    __func__, __LINE__, work_bgn, work_end);
1832
1833         /* *************************************************************** */
1834         /*
1835          * * We need to calculate the number of dtaps that equal a ptap
1836          * * To do that we'll back up a ptap and re-find the edge of the
1837          * * window using dtaps
1838          */
1839
1840         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
1841                    for tracking\n", __func__, __LINE__);
1842
1843         /* Special case code for backing up a phase */
1844         if (p == 0) {
1845                 p = IO_DQS_EN_PHASE_MAX;
1846                 rw_mgr_decr_vfifo(grp, &v);
1847                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1848                            cycle/phase: v=%u p=%u\n", __func__, __LINE__,
1849                            v, p);
1850         } else {
1851                 p = p - 1;
1852                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1853                            phase only: v=%u p=%u", __func__, __LINE__,
1854                            v, p);
1855         }
1856
1857         scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1858
1859         /*
1860          * Increase dtap until we first see a passing read (in case the
1861          * window is smaller than a ptap),
1862          * and then a failing read to mark the edge of the window again
1863          */
1864
1865         /* Find a passing read */
1866         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
1867                    __func__, __LINE__);
1868         found_passing_read = 0;
1869         found_failing_read = 0;
1870         initial_failing_dtap = d;
1871         for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
1872                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
1873                            read d=%u\n", __func__, __LINE__, d);
1874                 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1875
1876                 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1877                                                              PASS_ONE_BIT,
1878                                                              &bit_chk, 0)) {
1879                         found_passing_read = 1;
1880                         break;
1881                 }
1882         }
1883
1884         if (found_passing_read) {
1885                 /* Find a failing read */
1886                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
1887                            read\n", __func__, __LINE__);
1888                 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
1889                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1890                                    testing read d=%u\n", __func__, __LINE__, d);
1891                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1892
1893                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1894                                 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
1895                                 found_failing_read = 1;
1896                                 break;
1897                         }
1898                 }
1899         } else {
1900                 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
1901                            calculate dtaps", __func__, __LINE__);
1902                 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
1903         }
1904
1905         /*
1906          * The dynamically calculated dtaps_per_ptap is only valid if we
1907          * found a passing/failing read. If we didn't, it means d hit the max
1908          * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
1909          * statically calculated value.
1910          */
1911         if (found_passing_read && found_failing_read)
1912                 dtaps_per_ptap = d - initial_failing_dtap;
1913
1914         addr = sdr_get_addr(&sdr_reg_file->dtaps_per_ptap);
1915         writel(dtaps_per_ptap, SOCFPGA_SDR_ADDRESS + addr);
1916         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
1917                    - %u = %u",  __func__, __LINE__, d,
1918                    initial_failing_dtap, dtaps_per_ptap);
1919
1920         /* ******************************************** */
1921         /* * step 6:  Find the centre of the window   * */
1922         if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1923                                    &work_mid, &work_end) == 0)
1924                 return 0;
1925
1926         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \
1927                    vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__,
1928                    v, p-1, d);
1929         return 1;
1930 }
1931
1932 /*
1933  * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
1934  * dq_in_delay values
1935  */
1936 static uint32_t
1937 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
1938 (uint32_t write_group, uint32_t read_group, uint32_t test_bgn)
1939 {
1940         uint32_t found;
1941         uint32_t i;
1942         uint32_t p;
1943         uint32_t d;
1944         uint32_t r;
1945         uint32_t addr;
1946
1947         const uint32_t delay_step = IO_IO_IN_DELAY_MAX /
1948                 (RW_MGR_MEM_DQ_PER_READ_DQS-1);
1949                 /* we start at zero, so have one less dq to devide among */
1950
1951         debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group,
1952               test_bgn);
1953
1954         /* try different dq_in_delays since the dq path is shorter than dqs */
1955
1956         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1957              r += NUM_RANKS_PER_SHADOW_REG) {
1958                 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1959                         i++, p++, d += delay_step) {
1960                         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\
1961                                    vfifo_find_dqs_", __func__, __LINE__);
1962                         debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ",
1963                                write_group, read_group);
1964                         debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d);
1965                         scc_mgr_set_dq_in_delay(write_group, p, d);
1966                         scc_mgr_load_dq(p);
1967                 }
1968                 addr = sdr_get_addr(&sdr_scc_mgr->update);
1969                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1970         }
1971
1972         found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group);
1973
1974         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\
1975                    en_phase_sweep_dq", __func__, __LINE__);
1976         debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \
1977                    chain to zero\n", write_group, read_group, found);
1978
1979         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1980              r += NUM_RANKS_PER_SHADOW_REG) {
1981                 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1982                         i++, p++) {
1983                         scc_mgr_set_dq_in_delay(write_group, p, 0);
1984                         scc_mgr_load_dq(p);
1985                 }
1986                 addr = sdr_get_addr(&sdr_scc_mgr->update);
1987                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1988         }
1989
1990         return found;
1991 }
1992
1993 /* per-bit deskew DQ and center */
1994 static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
1995         uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
1996         uint32_t use_read_test, uint32_t update_fom)
1997 {
1998         uint32_t i, p, d, min_index;
1999         /*
2000          * Store these as signed since there are comparisons with
2001          * signed numbers.
2002          */
2003         uint32_t bit_chk;
2004         uint32_t sticky_bit_chk;
2005         int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
2006         int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
2007         int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
2008         int32_t mid;
2009         int32_t orig_mid_min, mid_min;
2010         int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
2011                 final_dqs_en;
2012         int32_t dq_margin, dqs_margin;
2013         uint32_t stop;
2014         uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
2015         uint32_t addr;
2016
2017         debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
2018
2019         addr = sdr_get_addr((u32 *)SCC_MGR_DQS_IN_DELAY);
2020         start_dqs = readl(SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
2021         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2022                 start_dqs_en = readl(SOCFPGA_SDR_ADDRESS + addr + ((read_group << 2)
2023                                      - IO_DQS_EN_DELAY_OFFSET));
2024
2025         /* set the left and right edge of each bit to an illegal value */
2026         /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
2027         sticky_bit_chk = 0;
2028         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2029                 left_edge[i]  = IO_IO_IN_DELAY_MAX + 1;
2030                 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
2031         }
2032
2033         addr = sdr_get_addr(&sdr_scc_mgr->update);
2034         /* Search for the left edge of the window for each bit */
2035         for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
2036                 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
2037
2038                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2039
2040                 /*
2041                  * Stop searching when the read test doesn't pass AND when
2042                  * we've seen a passing read on every bit.
2043                  */
2044                 if (use_read_test) {
2045                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
2046                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
2047                                 &bit_chk, 0, 0);
2048                 } else {
2049                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2050                                                         0, PASS_ONE_BIT,
2051                                                         &bit_chk, 0);
2052                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
2053                                 (read_group - (write_group *
2054                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
2055                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
2056                         stop = (bit_chk == 0);
2057                 }
2058                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2059                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
2060                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
2061                            && %u", __func__, __LINE__, d,
2062                            sticky_bit_chk,
2063                         param->read_correct_mask, stop);
2064
2065                 if (stop == 1) {
2066                         break;
2067                 } else {
2068                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2069                                 if (bit_chk & 1) {
2070                                         /* Remember a passing test as the
2071                                         left_edge */
2072                                         left_edge[i] = d;
2073                                 } else {
2074                                         /* If a left edge has not been seen yet,
2075                                         then a future passing test will mark
2076                                         this edge as the right edge */
2077                                         if (left_edge[i] ==
2078                                                 IO_IO_IN_DELAY_MAX + 1) {
2079                                                 right_edge[i] = -(d + 1);
2080                                         }
2081                                 }
2082                                 bit_chk = bit_chk >> 1;
2083                         }
2084                 }
2085         }
2086
2087         /* Reset DQ delay chains to 0 */
2088         scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, 0);
2089         sticky_bit_chk = 0;
2090         for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
2091                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2092                            %d right_edge[%u]: %d\n", __func__, __LINE__,
2093                            i, left_edge[i], i, right_edge[i]);
2094
2095                 /*
2096                  * Check for cases where we haven't found the left edge,
2097                  * which makes our assignment of the the right edge invalid.
2098                  * Reset it to the illegal value.
2099                  */
2100                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
2101                         right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
2102                         right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
2103                         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
2104                                    right_edge[%u]: %d\n", __func__, __LINE__,
2105                                    i, right_edge[i]);
2106                 }
2107
2108                 /*
2109                  * Reset sticky bit (except for bits where we have seen
2110                  * both the left and right edge).
2111                  */
2112                 sticky_bit_chk = sticky_bit_chk << 1;
2113                 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
2114                     (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
2115                         sticky_bit_chk = sticky_bit_chk | 1;
2116                 }
2117
2118                 if (i == 0)
2119                         break;
2120         }
2121
2122         addr = sdr_get_addr(&sdr_scc_mgr->update);
2123         /* Search for the right edge of the window for each bit */
2124         for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
2125                 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
2126                 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2127                         uint32_t delay = d + start_dqs_en;
2128                         if (delay > IO_DQS_EN_DELAY_MAX)
2129                                 delay = IO_DQS_EN_DELAY_MAX;
2130                         scc_mgr_set_dqs_en_delay(read_group, delay);
2131                 }
2132                 scc_mgr_load_dqs(read_group);
2133
2134                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2135
2136                 /*
2137                  * Stop searching when the read test doesn't pass AND when
2138                  * we've seen a passing read on every bit.
2139                  */
2140                 if (use_read_test) {
2141                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
2142                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
2143                                 &bit_chk, 0, 0);
2144                 } else {
2145                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2146                                                         0, PASS_ONE_BIT,
2147                                                         &bit_chk, 0);
2148                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
2149                                 (read_group - (write_group *
2150                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
2151                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
2152                         stop = (bit_chk == 0);
2153                 }
2154                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2155                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
2156
2157                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
2158                            %u && %u", __func__, __LINE__, d,
2159                            sticky_bit_chk, param->read_correct_mask, stop);
2160
2161                 if (stop == 1) {
2162                         break;
2163                 } else {
2164                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2165                                 if (bit_chk & 1) {
2166                                         /* Remember a passing test as
2167                                         the right_edge */
2168                                         right_edge[i] = d;
2169                                 } else {
2170                                         if (d != 0) {
2171                                                 /* If a right edge has not been
2172                                                 seen yet, then a future passing
2173                                                 test will mark this edge as the
2174                                                 left edge */
2175                                                 if (right_edge[i] ==
2176                                                 IO_IO_IN_DELAY_MAX + 1) {
2177                                                         left_edge[i] = -(d + 1);
2178                                                 }
2179                                         } else {
2180                                                 /* d = 0 failed, but it passed
2181                                                 when testing the left edge,
2182                                                 so it must be marginal,
2183                                                 set it to -1 */
2184                                                 if (right_edge[i] ==
2185                                                         IO_IO_IN_DELAY_MAX + 1 &&
2186                                                         left_edge[i] !=
2187                                                         IO_IO_IN_DELAY_MAX
2188                                                         + 1) {
2189                                                         right_edge[i] = -1;
2190                                                 }
2191                                                 /* If a right edge has not been
2192                                                 seen yet, then a future passing
2193                                                 test will mark this edge as the
2194                                                 left edge */
2195                                                 else if (right_edge[i] ==
2196                                                         IO_IO_IN_DELAY_MAX +
2197                                                         1) {
2198                                                         left_edge[i] = -(d + 1);
2199                                                 }
2200                                         }
2201                                 }
2202
2203                                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
2204                                            d=%u]: ", __func__, __LINE__, d);
2205                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
2206                                            (int)(bit_chk & 1), i, left_edge[i]);
2207                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2208                                            right_edge[i]);
2209                                 bit_chk = bit_chk >> 1;
2210                         }
2211                 }
2212         }
2213
2214         /* Check that all bits have a window */
2215         addr = sdr_get_addr(&sdr_scc_mgr->update);
2216         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2217                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2218                            %d right_edge[%u]: %d", __func__, __LINE__,
2219                            i, left_edge[i], i, right_edge[i]);
2220                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
2221                         == IO_IO_IN_DELAY_MAX + 1)) {
2222                         /*
2223                          * Restore delay chain settings before letting the loop
2224                          * in rw_mgr_mem_calibrate_vfifo to retry different
2225                          * dqs/ck relationships.
2226                          */
2227                         scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
2228                         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2229                                 scc_mgr_set_dqs_en_delay(read_group,
2230                                                          start_dqs_en);
2231                         }
2232                         scc_mgr_load_dqs(read_group);
2233                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
2234
2235                         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
2236                                    find edge [%u]: %d %d", __func__, __LINE__,
2237                                    i, left_edge[i], right_edge[i]);
2238                         if (use_read_test) {
2239                                 set_failing_group_stage(read_group *
2240                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2241                                         CAL_STAGE_VFIFO,
2242                                         CAL_SUBSTAGE_VFIFO_CENTER);
2243                         } else {
2244                                 set_failing_group_stage(read_group *
2245                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2246                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2247                                         CAL_SUBSTAGE_VFIFO_CENTER);
2248                         }
2249                         return 0;
2250                 }
2251         }
2252
2253         /* Find middle of window for each DQ bit */
2254         mid_min = left_edge[0] - right_edge[0];
2255         min_index = 0;
2256         for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2257                 mid = left_edge[i] - right_edge[i];
2258                 if (mid < mid_min) {
2259                         mid_min = mid;
2260                         min_index = i;
2261                 }
2262         }
2263
2264         /*
2265          * -mid_min/2 represents the amount that we need to move DQS.
2266          * If mid_min is odd and positive we'll need to add one to
2267          * make sure the rounding in further calculations is correct
2268          * (always bias to the right), so just add 1 for all positive values.
2269          */
2270         if (mid_min > 0)
2271                 mid_min++;
2272
2273         mid_min = mid_min / 2;
2274
2275         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
2276                    __func__, __LINE__, mid_min, min_index);
2277
2278         /* Determine the amount we can change DQS (which is -mid_min) */
2279         orig_mid_min = mid_min;
2280         new_dqs = start_dqs - mid_min;
2281         if (new_dqs > IO_DQS_IN_DELAY_MAX)
2282                 new_dqs = IO_DQS_IN_DELAY_MAX;
2283         else if (new_dqs < 0)
2284                 new_dqs = 0;
2285
2286         mid_min = start_dqs - new_dqs;
2287         debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
2288                    mid_min, new_dqs);
2289
2290         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2291                 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
2292                         mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
2293                 else if (start_dqs_en - mid_min < 0)
2294                         mid_min += start_dqs_en - mid_min;
2295         }
2296         new_dqs = start_dqs - mid_min;
2297
2298         debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
2299                    new_dqs=%d mid_min=%d\n", start_dqs,
2300                    IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
2301                    new_dqs, mid_min);
2302
2303         /* Initialize data for export structures */
2304         dqs_margin = IO_IO_IN_DELAY_MAX + 1;
2305         dq_margin  = IO_IO_IN_DELAY_MAX + 1;
2306
2307         addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY);
2308         /* add delay to bring centre of all DQ windows to the same "level" */
2309         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
2310                 /* Use values before divide by 2 to reduce round off error */
2311                 shift_dq = (left_edge[i] - right_edge[i] -
2312                         (left_edge[min_index] - right_edge[min_index]))/2  +
2313                         (orig_mid_min - mid_min);
2314
2315                 debug_cond(DLEVEL == 2, "vfifo_center: before: \
2316                            shift_dq[%u]=%d\n", i, shift_dq);
2317
2318                 temp_dq_in_delay1 = readl(SOCFPGA_SDR_ADDRESS + addr + (p << 2));
2319                 temp_dq_in_delay2 = readl(SOCFPGA_SDR_ADDRESS + addr + (i << 2));
2320
2321                 if (shift_dq + (int32_t)temp_dq_in_delay1 >
2322                         (int32_t)IO_IO_IN_DELAY_MAX) {
2323                         shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
2324                 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
2325                         shift_dq = -(int32_t)temp_dq_in_delay1;
2326                 }
2327                 debug_cond(DLEVEL == 2, "vfifo_center: after: \
2328                            shift_dq[%u]=%d\n", i, shift_dq);
2329                 final_dq[i] = temp_dq_in_delay1 + shift_dq;
2330                 scc_mgr_set_dq_in_delay(write_group, p, final_dq[i]);
2331                 scc_mgr_load_dq(p);
2332
2333                 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
2334                            left_edge[i] - shift_dq + (-mid_min),
2335                            right_edge[i] + shift_dq - (-mid_min));
2336                 /* To determine values for export structures */
2337                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2338                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
2339
2340                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2341                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2342         }
2343
2344         final_dqs = new_dqs;
2345         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2346                 final_dqs_en = start_dqs_en - mid_min;
2347
2348         /* Move DQS-en */
2349         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2350                 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
2351                 scc_mgr_load_dqs(read_group);
2352         }
2353
2354         /* Move DQS */
2355         scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
2356         scc_mgr_load_dqs(read_group);
2357         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
2358                    dqs_margin=%d", __func__, __LINE__,
2359                    dq_margin, dqs_margin);
2360
2361         /*
2362          * Do not remove this line as it makes sure all of our decisions
2363          * have been applied. Apply the update bit.
2364          */
2365         addr = sdr_get_addr(&sdr_scc_mgr->update);
2366         writel(0, SOCFPGA_SDR_ADDRESS + addr);
2367
2368         return (dq_margin >= 0) && (dqs_margin >= 0);
2369 }
2370
2371 /*
2372  * calibrate the read valid prediction FIFO.
2373  *
2374  *  - read valid prediction will consist of finding a good DQS enable phase,
2375  * DQS enable delay, DQS input phase, and DQS input delay.
2376  *  - we also do a per-bit deskew on the DQ lines.
2377  */
2378 static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group,
2379                                            uint32_t test_bgn)
2380 {
2381         uint32_t p, d, rank_bgn, sr;
2382         uint32_t dtaps_per_ptap;
2383         uint32_t tmp_delay;
2384         uint32_t bit_chk;
2385         uint32_t grp_calibrated;
2386         uint32_t write_group, write_test_bgn;
2387         uint32_t failed_substage;
2388
2389         debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
2390
2391         /* update info for sims */
2392         reg_file_set_stage(CAL_STAGE_VFIFO);
2393
2394         write_group = read_group;
2395         write_test_bgn = test_bgn;
2396
2397         /* USER Determine number of delay taps for each phase tap */
2398         dtaps_per_ptap = 0;
2399         tmp_delay = 0;
2400         while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
2401                 dtaps_per_ptap++;
2402                 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
2403         }
2404         dtaps_per_ptap--;
2405         tmp_delay = 0;
2406
2407         /* update info for sims */
2408         reg_file_set_group(read_group);
2409
2410         grp_calibrated = 0;
2411
2412         reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
2413         failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
2414
2415         for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) {
2416                 /*
2417                  * In RLDRAMX we may be messing the delay of pins in
2418                  * the same write group but outside of the current read
2419                  * the group, but that's ok because we haven't
2420                  * calibrated output side yet.
2421                  */
2422                 if (d > 0) {
2423                         scc_mgr_apply_group_all_out_delay_add_all_ranks
2424                         (write_group, write_test_bgn, d);
2425                 }
2426
2427                 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0;
2428                         p++) {
2429                         /* set a particular dqdqs phase */
2430                         scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p);
2431
2432                         debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \
2433                                    p=%u d=%u\n", __func__, __LINE__,
2434                                    read_group, p, d);
2435
2436                         /*
2437                          * Load up the patterns used by read calibration
2438                          * using current DQDQS phase.
2439                          */
2440                         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2441                         if (!(gbl->phy_debug_mode_flags &
2442                                 PHY_DEBUG_DISABLE_GUARANTEED_READ)) {
2443                                 if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks
2444                                     (read_group, 1, &bit_chk)) {
2445                                         debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:",
2446                                                    __func__, __LINE__);
2447                                         debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n",
2448                                                    read_group, p, d);
2449                                         break;
2450                                 }
2451                         }
2452
2453 /* case:56390 */
2454                         grp_calibrated = 1;
2455                 if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
2456                     (write_group, read_group, test_bgn)) {
2457                                 /*
2458                                  * USER Read per-bit deskew can be done on a
2459                                  * per shadow register basis.
2460                                  */
2461                                 for (rank_bgn = 0, sr = 0;
2462                                         rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2463                                         rank_bgn += NUM_RANKS_PER_SHADOW_REG,
2464                                         ++sr) {
2465                                         /*
2466                                          * Determine if this set of ranks
2467                                          * should be skipped entirely.
2468                                          */
2469                                         if (!param->skip_shadow_regs[sr]) {
2470                                                 /*
2471                                                  * If doing read after write
2472                                                  * calibration, do not update
2473                                                  * FOM, now - do it then.
2474                                                  */
2475                                         if (!rw_mgr_mem_calibrate_vfifo_center
2476                                                 (rank_bgn, write_group,
2477                                                 read_group, test_bgn, 1, 0)) {
2478                                                         grp_calibrated = 0;
2479                                                         failed_substage =
2480                                                 CAL_SUBSTAGE_VFIFO_CENTER;
2481                                                 }
2482                                         }
2483                                 }
2484                         } else {
2485                                 grp_calibrated = 0;
2486                                 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
2487                         }
2488                 }
2489         }
2490
2491         if (grp_calibrated == 0) {
2492                 set_failing_group_stage(write_group, CAL_STAGE_VFIFO,
2493                                         failed_substage);
2494                 return 0;
2495         }
2496
2497         /*
2498          * Reset the delay chains back to zero if they have moved > 1
2499          * (check for > 1 because loop will increase d even when pass in
2500          * first case).
2501          */
2502         if (d > 2)
2503                 scc_mgr_zero_group(write_group, write_test_bgn, 1);
2504
2505         return 1;
2506 }
2507
2508 /* VFIFO Calibration -- Read Deskew Calibration after write deskew */
2509 static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
2510                                                uint32_t test_bgn)
2511 {
2512         uint32_t rank_bgn, sr;
2513         uint32_t grp_calibrated;
2514         uint32_t write_group;
2515
2516         debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
2517
2518         /* update info for sims */
2519
2520         reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
2521         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
2522
2523         write_group = read_group;
2524
2525         /* update info for sims */
2526         reg_file_set_group(read_group);
2527
2528         grp_calibrated = 1;
2529         /* Read per-bit deskew can be done on a per shadow register basis */
2530         for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2531                 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
2532                 /* Determine if this set of ranks should be skipped entirely */
2533                 if (!param->skip_shadow_regs[sr]) {
2534                 /* This is the last calibration round, update FOM here */
2535                         if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
2536                                                                 write_group,
2537                                                                 read_group,
2538                                                                 test_bgn, 0,
2539                                                                 1)) {
2540                                 grp_calibrated = 0;
2541                         }
2542                 }
2543         }
2544
2545
2546         if (grp_calibrated == 0) {
2547                 set_failing_group_stage(write_group,
2548                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2549                                         CAL_SUBSTAGE_VFIFO_CENTER);
2550                 return 0;
2551         }
2552
2553         return 1;
2554 }
2555
2556 /* Calibrate LFIFO to find smallest read latency */
2557 static uint32_t rw_mgr_mem_calibrate_lfifo(void)
2558 {
2559         uint32_t found_one;
2560         uint32_t bit_chk;
2561         uint32_t addr;
2562
2563         debug("%s:%d\n", __func__, __LINE__);
2564
2565         /* update info for sims */
2566         reg_file_set_stage(CAL_STAGE_LFIFO);
2567         reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
2568
2569         /* Load up the patterns used by read calibration for all ranks */
2570         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2571         found_one = 0;
2572
2573         addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat);
2574         do {
2575                 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
2576                 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
2577                            __func__, __LINE__, gbl->curr_read_lat);
2578
2579                 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
2580                                                               NUM_READ_TESTS,
2581                                                               PASS_ALL_BITS,
2582                                                               &bit_chk, 1)) {
2583                         break;
2584                 }
2585
2586                 found_one = 1;
2587                 /* reduce read latency and see if things are working */
2588                 /* correctly */
2589                 gbl->curr_read_lat--;
2590         } while (gbl->curr_read_lat > 0);
2591
2592         /* reset the fifos to get pointers to known state */
2593
2594         addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset);
2595         writel(0, SOCFPGA_SDR_ADDRESS + addr);
2596
2597         if (found_one) {
2598                 /* add a fudge factor to the read latency that was determined */
2599                 gbl->curr_read_lat += 2;
2600                 addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat);
2601                 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
2602                 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
2603                            read_lat=%u\n", __func__, __LINE__,
2604                            gbl->curr_read_lat);
2605                 return 1;
2606         } else {
2607                 set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
2608                                         CAL_SUBSTAGE_READ_LATENCY);
2609
2610                 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
2611                            read_lat=%u\n", __func__, __LINE__,
2612                            gbl->curr_read_lat);
2613                 return 0;
2614         }
2615 }
2616
2617 /*
2618  * issue write test command.
2619  * two variants are provided. one that just tests a write pattern and
2620  * another that tests datamask functionality.
2621  */
2622 static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
2623                                                   uint32_t test_dm)
2624 {
2625         uint32_t mcc_instruction;
2626         uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
2627                 ENABLE_SUPER_QUICK_CALIBRATION);
2628         uint32_t rw_wl_nop_cycles;
2629         uint32_t addr;
2630
2631         /*
2632          * Set counter and jump addresses for the right
2633          * number of NOP cycles.
2634          * The number of supported NOP cycles can range from -1 to infinity
2635          * Three different cases are handled:
2636          *
2637          * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
2638          *    mechanism will be used to insert the right number of NOPs
2639          *
2640          * 2. For a number of NOP cycles equals to 0, the micro-instruction
2641          *    issuing the write command will jump straight to the
2642          *    micro-instruction that turns on DQS (for DDRx), or outputs write
2643          *    data (for RLD), skipping
2644          *    the NOP micro-instruction all together
2645          *
2646          * 3. A number of NOP cycles equal to -1 indicates that DQS must be
2647          *    turned on in the same micro-instruction that issues the write
2648          *    command. Then we need
2649          *    to directly jump to the micro-instruction that sends out the data
2650          *
2651          * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
2652          *       (2 and 3). One jump-counter (0) is used to perform multiple
2653          *       write-read operations.
2654          *       one counter left to issue this command in "multiple-group" mode
2655          */
2656
2657         rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
2658
2659         if (rw_wl_nop_cycles == -1) {
2660                 /*
2661                  * CNTR 2 - We want to execute the special write operation that
2662                  * turns on DQS right away and then skip directly to the
2663                  * instruction that sends out the data. We set the counter to a
2664                  * large number so that the jump is always taken.
2665                  */
2666                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr2);
2667                 writel(0xFF, SOCFPGA_SDR_ADDRESS + addr);
2668
2669                 /* CNTR 3 - Not used */
2670                 if (test_dm) {
2671                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
2672                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
2673                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
2674                                SOCFPGA_SDR_ADDRESS + addr);
2675                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add3);
2676                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2677                                SOCFPGA_SDR_ADDRESS + addr);
2678                 } else {
2679                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
2680                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
2681                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA, SOCFPGA_SDR_ADDRESS + addr);
2682                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add3);
2683                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr);
2684                 }
2685         } else if (rw_wl_nop_cycles == 0) {
2686                 /*
2687                  * CNTR 2 - We want to skip the NOP operation and go straight
2688                  * to the DQS enable instruction. We set the counter to a large
2689                  * number so that the jump is always taken.
2690                  */
2691                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr2);
2692                 writel(0xFF, SOCFPGA_SDR_ADDRESS + addr);
2693
2694                 /* CNTR 3 - Not used */
2695                 if (test_dm) {
2696                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2697                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
2698                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
2699                                SOCFPGA_SDR_ADDRESS + addr);
2700                 } else {
2701                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2702                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
2703                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS, SOCFPGA_SDR_ADDRESS + addr);
2704                 }
2705         } else {
2706                 /*
2707                  * CNTR 2 - In this case we want to execute the next instruction
2708                  * and NOT take the jump. So we set the counter to 0. The jump
2709                  * address doesn't count.
2710                  */
2711                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr2);
2712                 writel(0x0, SOCFPGA_SDR_ADDRESS + addr);
2713                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add2);
2714                 writel(0x0, SOCFPGA_SDR_ADDRESS + addr);
2715
2716                 /*
2717                  * CNTR 3 - Set the nop counter to the number of cycles we
2718                  * need to loop for, minus 1.
2719                  */
2720                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr3);
2721                 writel(rw_wl_nop_cycles - 1, SOCFPGA_SDR_ADDRESS + addr);
2722                 if (test_dm) {
2723                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2724                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add3);
2725                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr);
2726                 } else {
2727                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2728                         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add3);
2729                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr);
2730                 }
2731         }
2732
2733         addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH);
2734         writel(0, SOCFPGA_SDR_ADDRESS + addr);
2735
2736         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
2737         if (quick_write_mode)
2738                 writel(0x08, SOCFPGA_SDR_ADDRESS + addr);
2739         else
2740                 writel(0x40, SOCFPGA_SDR_ADDRESS + addr);
2741
2742         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
2743         writel(mcc_instruction, SOCFPGA_SDR_ADDRESS + addr);
2744
2745         /*
2746          * CNTR 1 - This is used to ensure enough time elapses
2747          * for read data to come back.
2748          */
2749         addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
2750         writel(0x30, SOCFPGA_SDR_ADDRESS + addr);
2751
2752         addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
2753         if (test_dm) {
2754                 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT, SOCFPGA_SDR_ADDRESS + addr);
2755         } else {
2756                 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT, SOCFPGA_SDR_ADDRESS + addr);
2757         }
2758
2759         addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
2760         writel(mcc_instruction, SOCFPGA_SDR_ADDRESS + addr + (group << 2));
2761 }
2762
2763 /* Test writes, can check for a single bit pass or multiple bit pass */
2764 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
2765         uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
2766         uint32_t *bit_chk, uint32_t all_ranks)
2767 {
2768         uint32_t addr;
2769         uint32_t r;
2770         uint32_t correct_mask_vg;
2771         uint32_t tmp_bit_chk;
2772         uint32_t vg;
2773         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
2774                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
2775         uint32_t addr_rw_mgr;
2776         uint32_t base_rw_mgr;
2777
2778         *bit_chk = param->write_correct_mask;
2779         correct_mask_vg = param->write_correct_mask_vg;
2780
2781         for (r = rank_bgn; r < rank_end; r++) {
2782                 if (param->skip_ranks[r]) {
2783                         /* request to skip the rank */
2784                         continue;
2785                 }
2786
2787                 /* set rank */
2788                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
2789
2790                 tmp_bit_chk = 0;
2791                 addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset);
2792                 addr_rw_mgr = sdr_get_addr((u32 *)BASE_RW_MGR);
2793                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
2794                         /* reset the fifos to get pointers to known state */
2795                         writel(0, SOCFPGA_SDR_ADDRESS + addr);
2796
2797                         tmp_bit_chk = tmp_bit_chk <<
2798                                 (RW_MGR_MEM_DQ_PER_WRITE_DQS /
2799                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
2800                         rw_mgr_mem_calibrate_write_test_issue(write_group *
2801                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
2802                                 use_dm);
2803
2804                         base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr_rw_mgr);
2805                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
2806                         if (vg == 0)
2807                                 break;
2808                 }
2809                 *bit_chk &= tmp_bit_chk;
2810         }
2811
2812         if (all_correct) {
2813                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2814                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
2815                            %u => %lu", write_group, use_dm,
2816                            *bit_chk, param->write_correct_mask,
2817                            (long unsigned int)(*bit_chk ==
2818                            param->write_correct_mask));
2819                 return *bit_chk == param->write_correct_mask;
2820         } else {
2821                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2822                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
2823                        write_group, use_dm, *bit_chk);
2824                 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
2825                         (long unsigned int)(*bit_chk != 0));
2826                 return *bit_chk != 0x00;
2827         }
2828 }
2829
2830 /*
2831  * center all windows. do per-bit-deskew to possibly increase size of
2832  * certain windows.
2833  */
2834 static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
2835         uint32_t write_group, uint32_t test_bgn)
2836 {
2837         uint32_t i, p, min_index;
2838         int32_t d;
2839         /*
2840          * Store these as signed since there are comparisons with
2841          * signed numbers.
2842          */
2843         uint32_t bit_chk;
2844         uint32_t sticky_bit_chk;
2845         int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2846         int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2847         int32_t mid;
2848         int32_t mid_min, orig_mid_min;
2849         int32_t new_dqs, start_dqs, shift_dq;
2850         int32_t dq_margin, dqs_margin, dm_margin;
2851         uint32_t stop;
2852         uint32_t temp_dq_out1_delay;
2853         uint32_t addr;
2854
2855         debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
2856
2857         dm_margin = 0;
2858
2859         addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
2860         start_dqs = readl(SOCFPGA_SDR_ADDRESS + addr +
2861                           (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
2862
2863         /* per-bit deskew */
2864
2865         /*
2866          * set the left and right edge of each bit to an illegal value
2867          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
2868          */
2869         sticky_bit_chk = 0;
2870         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2871                 left_edge[i]  = IO_IO_OUT1_DELAY_MAX + 1;
2872                 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2873         }
2874
2875         /* Search for the left edge of the window for each bit */
2876         addr = sdr_get_addr(&sdr_scc_mgr->update);
2877         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
2878                 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, d);
2879
2880                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2881
2882                 /*
2883                  * Stop searching when the read test doesn't pass AND when
2884                  * we've seen a passing read on every bit.
2885                  */
2886                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2887                         0, PASS_ONE_BIT, &bit_chk, 0);
2888                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2889                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2890                 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
2891                            == %u && %u [bit_chk= %u ]\n",
2892                         d, sticky_bit_chk, param->write_correct_mask,
2893                         stop, bit_chk);
2894
2895                 if (stop == 1) {
2896                         break;
2897                 } else {
2898                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2899                                 if (bit_chk & 1) {
2900                                         /*
2901                                          * Remember a passing test as the
2902                                          * left_edge.
2903                                          */
2904                                         left_edge[i] = d;
2905                                 } else {
2906                                         /*
2907                                          * If a left edge has not been seen
2908                                          * yet, then a future passing test will
2909                                          * mark this edge as the right edge.
2910                                          */
2911                                         if (left_edge[i] ==
2912                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2913                                                 right_edge[i] = -(d + 1);
2914                                         }
2915                                 }
2916                                 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
2917                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2918                                            (int)(bit_chk & 1), i, left_edge[i]);
2919                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2920                                        right_edge[i]);
2921                                 bit_chk = bit_chk >> 1;
2922                         }
2923                 }
2924         }
2925
2926         /* Reset DQ delay chains to 0 */
2927         scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, 0);
2928         sticky_bit_chk = 0;
2929         for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
2930                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2931                            %d right_edge[%u]: %d\n", __func__, __LINE__,
2932                            i, left_edge[i], i, right_edge[i]);
2933
2934                 /*
2935                  * Check for cases where we haven't found the left edge,
2936                  * which makes our assignment of the the right edge invalid.
2937                  * Reset it to the illegal value.
2938                  */
2939                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
2940                     (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
2941                         right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2942                         debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
2943                                    right_edge[%u]: %d\n", __func__, __LINE__,
2944                                    i, right_edge[i]);
2945                 }
2946
2947                 /*
2948                  * Reset sticky bit (except for bits where we have
2949                  * seen the left edge).
2950                  */
2951                 sticky_bit_chk = sticky_bit_chk << 1;
2952                 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
2953                         sticky_bit_chk = sticky_bit_chk | 1;
2954
2955                 if (i == 0)
2956                         break;
2957         }
2958
2959         /* Search for the right edge of the window for each bit */
2960         addr = sdr_get_addr(&sdr_scc_mgr->update);
2961         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
2962                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
2963                                                         d + start_dqs);
2964
2965                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2966
2967                 /*
2968                  * Stop searching when the read test doesn't pass AND when
2969                  * we've seen a passing read on every bit.
2970                  */
2971                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2972                         0, PASS_ONE_BIT, &bit_chk, 0);
2973
2974                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2975                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2976
2977                 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
2978                            %u && %u\n", d, sticky_bit_chk,
2979                            param->write_correct_mask, stop);
2980
2981                 if (stop == 1) {
2982                         if (d == 0) {
2983                                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
2984                                         i++) {
2985                                         /* d = 0 failed, but it passed when
2986                                         testing the left edge, so it must be
2987                                         marginal, set it to -1 */
2988                                         if (right_edge[i] ==
2989                                                 IO_IO_OUT1_DELAY_MAX + 1 &&
2990                                                 left_edge[i] !=
2991                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2992                                                 right_edge[i] = -1;
2993                                         }
2994                                 }
2995                         }
2996                         break;
2997                 } else {
2998                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2999                                 if (bit_chk & 1) {
3000                                         /*
3001                                          * Remember a passing test as
3002                                          * the right_edge.
3003                                          */
3004                                         right_edge[i] = d;
3005                                 } else {
3006                                         if (d != 0) {
3007                                                 /*
3008                                                  * If a right edge has not
3009                                                  * been seen yet, then a future
3010                                                  * passing test will mark this
3011                                                  * edge as the left edge.
3012                                                  */
3013                                                 if (right_edge[i] ==
3014                                                     IO_IO_OUT1_DELAY_MAX + 1)
3015                                                         left_edge[i] = -(d + 1);
3016                                         } else {
3017                                                 /*
3018                                                  * d = 0 failed, but it passed
3019                                                  * when testing the left edge,
3020                                                  * so it must be marginal, set
3021                                                  * it to -1.
3022                                                  */
3023                                                 if (right_edge[i] ==
3024                                                     IO_IO_OUT1_DELAY_MAX + 1 &&
3025                                                     left_edge[i] !=
3026                                                     IO_IO_OUT1_DELAY_MAX + 1)
3027                                                         right_edge[i] = -1;
3028                                                 /*
3029                                                  * If a right edge has not been
3030                                                  * seen yet, then a future
3031                                                  * passing test will mark this
3032                                                  * edge as the left edge.
3033                                                  */
3034                                                 else if (right_edge[i] ==
3035                                                         IO_IO_OUT1_DELAY_MAX +
3036                                                         1)
3037                                                         left_edge[i] = -(d + 1);
3038                                         }
3039                                 }
3040                                 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
3041                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
3042                                            (int)(bit_chk & 1), i, left_edge[i]);
3043                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
3044                                            right_edge[i]);
3045                                 bit_chk = bit_chk >> 1;
3046                         }
3047                 }
3048         }
3049
3050         /* Check that all bits have a window */
3051         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
3052                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
3053                            %d right_edge[%u]: %d", __func__, __LINE__,
3054                            i, left_edge[i], i, right_edge[i]);
3055                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
3056                     (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
3057                         set_failing_group_stage(test_bgn + i,
3058                                                 CAL_STAGE_WRITES,
3059                                                 CAL_SUBSTAGE_WRITES_CENTER);
3060                         return 0;
3061                 }
3062         }
3063
3064         /* Find middle of window for each DQ bit */
3065         mid_min = left_edge[0] - right_edge[0];
3066         min_index = 0;
3067         for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
3068                 mid = left_edge[i] - right_edge[i];
3069                 if (mid < mid_min) {
3070                         mid_min = mid;
3071                         min_index = i;
3072                 }
3073         }
3074
3075         /*
3076          * -mid_min/2 represents the amount that we need to move DQS.
3077          * If mid_min is odd and positive we'll need to add one to
3078          * make sure the rounding in further calculations is correct
3079          * (always bias to the right), so just add 1 for all positive values.
3080          */
3081         if (mid_min > 0)
3082                 mid_min++;
3083         mid_min = mid_min / 2;
3084         debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
3085                    __LINE__, mid_min);
3086
3087         /* Determine the amount we can change DQS (which is -mid_min) */
3088         orig_mid_min = mid_min;
3089         new_dqs = start_dqs;
3090         mid_min = 0;
3091         debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
3092                    mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
3093         /* Initialize data for export structures */
3094         dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
3095         dq_margin  = IO_IO_OUT1_DELAY_MAX + 1;
3096
3097         /* add delay to bring centre of all DQ windows to the same "level" */
3098         addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
3099         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
3100                 /* Use values before divide by 2 to reduce round off error */
3101                 shift_dq = (left_edge[i] - right_edge[i] -
3102                         (left_edge[min_index] - right_edge[min_index]))/2  +
3103                 (orig_mid_min - mid_min);
3104
3105                 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
3106                            [%u]=%d\n", __func__, __LINE__, i, shift_dq);
3107
3108                 temp_dq_out1_delay = readl(SOCFPGA_SDR_ADDRESS + addr + (i << 2));
3109                 if (shift_dq + (int32_t)temp_dq_out1_delay >
3110                         (int32_t)IO_IO_OUT1_DELAY_MAX) {
3111                         shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
3112                 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
3113                         shift_dq = -(int32_t)temp_dq_out1_delay;
3114                 }
3115                 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
3116                            i, shift_dq);
3117                 scc_mgr_set_dq_out1_delay(write_group, i, temp_dq_out1_delay +
3118                                           shift_dq);
3119                 scc_mgr_load_dq(i);
3120
3121                 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
3122                            left_edge[i] - shift_dq + (-mid_min),
3123                            right_edge[i] + shift_dq - (-mid_min));
3124                 /* To determine values for export structures */
3125                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
3126                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
3127
3128                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
3129                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
3130         }
3131
3132         /* Move DQS */
3133         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3134         addr = sdr_get_addr(&sdr_scc_mgr->update);
3135         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3136
3137         /* Centre DM */
3138         debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
3139
3140         /*
3141          * set the left and right edge of each bit to an illegal value,
3142          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
3143          */
3144         left_edge[0]  = IO_IO_OUT1_DELAY_MAX + 1;
3145         right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
3146         int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3147         int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3148         int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
3149         int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
3150         int32_t win_best = 0;
3151
3152         /* Search for the/part of the window with DM shift */
3153         addr = sdr_get_addr(&sdr_scc_mgr->update);
3154         for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
3155                 scc_mgr_apply_group_dm_out1_delay(write_group, d);
3156                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3157
3158                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3159                                                     PASS_ALL_BITS, &bit_chk,
3160                                                     0)) {
3161                         /* USE Set current end of the window */
3162                         end_curr = -d;
3163                         /*
3164                          * If a starting edge of our window has not been seen
3165                          * this is our current start of the DM window.
3166                          */
3167                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3168                                 bgn_curr = -d;
3169
3170                         /*
3171                          * If current window is bigger than best seen.
3172                          * Set best seen to be current window.
3173                          */
3174                         if ((end_curr-bgn_curr+1) > win_best) {
3175                                 win_best = end_curr-bgn_curr+1;
3176                                 bgn_best = bgn_curr;
3177                                 end_best = end_curr;
3178                         }
3179                 } else {
3180                         /* We just saw a failing test. Reset temp edge */
3181                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3182                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3183                         }
3184                 }
3185
3186
3187         /* Reset DM delay chains to 0 */
3188         scc_mgr_apply_group_dm_out1_delay(write_group, 0);
3189
3190         /*
3191          * Check to see if the current window nudges up aganist 0 delay.
3192          * If so we need to continue the search by shifting DQS otherwise DQS
3193          * search begins as a new search. */
3194         if (end_curr != 0) {
3195                 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3196                 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3197         }
3198
3199         /* Search for the/part of the window with DQS shifts */
3200         addr = sdr_get_addr(&sdr_scc_mgr->update);
3201         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
3202                 /*
3203                  * Note: This only shifts DQS, so are we limiting ourselve to
3204                  * width of DQ unnecessarily.
3205                  */
3206                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
3207                                                         d + new_dqs);
3208
3209                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3210                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3211                                                     PASS_ALL_BITS, &bit_chk,
3212                                                     0)) {
3213                         /* USE Set current end of the window */
3214                         end_curr = d;
3215                         /*
3216                          * If a beginning edge of our window has not been seen
3217                          * this is our current begin of the DM window.
3218                          */
3219                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3220                                 bgn_curr = d;
3221
3222                         /*
3223                          * If current window is bigger than best seen. Set best
3224                          * seen to be current window.
3225                          */
3226                         if ((end_curr-bgn_curr+1) > win_best) {
3227                                 win_best = end_curr-bgn_curr+1;
3228                                 bgn_best = bgn_curr;
3229                                 end_best = end_curr;
3230                         }
3231                 } else {
3232                         /* We just saw a failing test. Reset temp edge */
3233                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3234                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3235
3236                         /* Early exit optimization: if ther remaining delay
3237                         chain space is less than already seen largest window
3238                         we can exit */
3239                         if ((win_best-1) >
3240                                 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
3241                                         break;
3242                                 }
3243                         }
3244                 }
3245
3246         /* assign left and right edge for cal and reporting; */
3247         left_edge[0] = -1*bgn_best;
3248         right_edge[0] = end_best;
3249
3250         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
3251                    __LINE__, left_edge[0], right_edge[0]);
3252
3253         /* Move DQS (back to orig) */
3254         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3255
3256         /* Move DM */
3257
3258         /* Find middle of window for the DM bit */
3259         mid = (left_edge[0] - right_edge[0]) / 2;
3260
3261         /* only move right, since we are not moving DQS/DQ */
3262         if (mid < 0)
3263                 mid = 0;
3264
3265         /* dm_marign should fail if we never find a window */
3266         if (win_best == 0)
3267                 dm_margin = -1;
3268         else
3269                 dm_margin = left_edge[0] - mid;
3270
3271         scc_mgr_apply_group_dm_out1_delay(write_group, mid);
3272         addr = sdr_get_addr(&sdr_scc_mgr->update);
3273         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3274
3275         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
3276                    dm_margin=%d\n", __func__, __LINE__, left_edge[0],
3277                    right_edge[0], mid, dm_margin);
3278         /* Export values */
3279         gbl->fom_out += dq_margin + dqs_margin;
3280
3281         debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
3282                    dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
3283                    dq_margin, dqs_margin, dm_margin);
3284
3285         /*
3286          * Do not remove this line as it makes sure all of our
3287          * decisions have been applied.
3288          */
3289         addr = sdr_get_addr(&sdr_scc_mgr->update);
3290         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3291         return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
3292 }
3293
3294 /* calibrate the write operations */
3295 static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
3296         uint32_t test_bgn)
3297 {
3298         /* update info for sims */
3299         debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
3300
3301         reg_file_set_stage(CAL_STAGE_WRITES);
3302         reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
3303
3304         reg_file_set_group(g);
3305
3306         if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
3307                 set_failing_group_stage(g, CAL_STAGE_WRITES,
3308                                         CAL_SUBSTAGE_WRITES_CENTER);
3309                 return 0;
3310         }
3311
3312         return 1;
3313 }
3314
3315 /* precharge all banks and activate row 0 in bank "000..." and bank "111..." */
3316 static void mem_precharge_and_activate(void)
3317 {
3318         uint32_t r;
3319         uint32_t addr;
3320
3321         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
3322                 if (param->skip_ranks[r]) {
3323                         /* request to skip the rank */
3324                         continue;
3325                 }
3326
3327                 /* set rank */
3328                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
3329
3330                 /* precharge all banks ... */
3331                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
3332                 writel(RW_MGR_PRECHARGE_ALL, SOCFPGA_SDR_ADDRESS + addr);
3333
3334                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr0);
3335                 writel(0x0F, SOCFPGA_SDR_ADDRESS + addr);
3336                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add0);
3337                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1, SOCFPGA_SDR_ADDRESS + addr);
3338
3339                 addr = sdr_get_addr(&sdr_rw_load_mgr_regs->load_cntr1);
3340                 writel(0x0F, SOCFPGA_SDR_ADDRESS + addr);
3341                 addr = sdr_get_addr(&sdr_rw_load_jump_mgr_regs->load_jump_add1);
3342                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2, SOCFPGA_SDR_ADDRESS + addr);
3343
3344                 /* activate rows */
3345                 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
3346                 writel(RW_MGR_ACTIVATE_0_AND_1, SOCFPGA_SDR_ADDRESS + addr);
3347         }
3348 }
3349
3350 /* Configure various memory related parameters. */
3351 static void mem_config(void)
3352 {
3353         uint32_t rlat, wlat;
3354         uint32_t rw_wl_nop_cycles;
3355         uint32_t max_latency;
3356         uint32_t addr;
3357
3358         debug("%s:%d\n", __func__, __LINE__);
3359         /* read in write and read latency */
3360         addr = sdr_get_addr(&data_mgr->t_wl_add);
3361         wlat = readl(SOCFPGA_SDR_ADDRESS + addr);
3362
3363         addr = sdr_get_addr(&data_mgr->mem_t_add);
3364         wlat += readl(SOCFPGA_SDR_ADDRESS + addr);
3365         /* WL for hard phy does not include additive latency */
3366
3367         /*
3368          * add addtional write latency to offset the address/command extra
3369          * clock cycle. We change the AC mux setting causing AC to be delayed
3370          * by one mem clock cycle. Only do this for DDR3
3371          */
3372         wlat = wlat + 1;
3373
3374         addr = sdr_get_addr(&data_mgr->t_rl_add);
3375         rlat = readl(SOCFPGA_SDR_ADDRESS + addr);
3376
3377         rw_wl_nop_cycles = wlat - 2;
3378         gbl->rw_wl_nop_cycles = rw_wl_nop_cycles;
3379
3380         /*
3381          * For AV/CV, lfifo is hardened and always runs at full rate so
3382          * max latency in AFI clocks, used here, is correspondingly smaller.
3383          */
3384         max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1;
3385         /* configure for a burst length of 8 */
3386
3387         /* write latency */
3388         /* Adjust Write Latency for Hard PHY */
3389         wlat = wlat + 1;
3390
3391         /* set a pretty high read latency initially */
3392         gbl->curr_read_lat = rlat + 16;
3393
3394         if (gbl->curr_read_lat > max_latency)
3395                 gbl->curr_read_lat = max_latency;
3396
3397         addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat);
3398         writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
3399
3400         /* advertise write latency */
3401         gbl->curr_write_lat = wlat;
3402         addr = sdr_get_addr(&phy_mgr_cfg->afi_wlat);
3403         writel(wlat - 2, SOCFPGA_SDR_ADDRESS + addr);
3404
3405         /* initialize bit slips */
3406         mem_precharge_and_activate();
3407 }
3408
3409 /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */
3410 static void mem_skip_calibrate(void)
3411 {
3412         uint32_t vfifo_offset;
3413         uint32_t i, j, r;
3414         uint32_t addr;
3415
3416         debug("%s:%d\n", __func__, __LINE__);
3417         /* Need to update every shadow register set used by the interface */
3418         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
3419                 r += NUM_RANKS_PER_SHADOW_REG) {
3420                 /*
3421                  * Set output phase alignment settings appropriate for
3422                  * skip calibration.
3423                  */
3424                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3425                         scc_mgr_set_dqs_en_phase(i, 0);
3426 #if IO_DLL_CHAIN_LENGTH == 6
3427                         scc_mgr_set_dqdqs_output_phase(i, 6);
3428 #else
3429                         scc_mgr_set_dqdqs_output_phase(i, 7);
3430 #endif
3431                         /*
3432                          * Case:33398
3433                          *
3434                          * Write data arrives to the I/O two cycles before write
3435                          * latency is reached (720 deg).
3436                          *   -> due to bit-slip in a/c bus
3437                          *   -> to allow board skew where dqs is longer than ck
3438                          *      -> how often can this happen!?
3439                          *      -> can claim back some ptaps for high freq
3440                          *       support if we can relax this, but i digress...
3441                          *
3442                          * The write_clk leads mem_ck by 90 deg
3443                          * The minimum ptap of the OPA is 180 deg
3444                          * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
3445                          * The write_clk is always delayed by 2 ptaps
3446                          *
3447                          * Hence, to make DQS aligned to CK, we need to delay
3448                          * DQS by:
3449                          *    (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
3450                          *
3451                          * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
3452                          * gives us the number of ptaps, which simplies to:
3453                          *
3454                          *    (1.25 * IO_DLL_CHAIN_LENGTH - 2)
3455                          */
3456                         scc_mgr_set_dqdqs_output_phase(i, (1.25 *
3457                                 IO_DLL_CHAIN_LENGTH - 2));
3458                 }
3459                 addr = sdr_get_addr(&sdr_scc_mgr->dqs_ena);
3460                 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3461                 addr = sdr_get_addr(&sdr_scc_mgr->dqs_io_ena);
3462                 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3463
3464                 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER);
3465                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
3466                         writel(i, SOCFPGA_SDR_ADDRESS + addr);
3467                 }
3468                 addr = sdr_get_addr(&sdr_scc_mgr->dq_ena);
3469                 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3470                 addr = sdr_get_addr(&sdr_scc_mgr->dm_ena);
3471                 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3472                 addr = sdr_get_addr(&sdr_scc_mgr->update);
3473                 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3474         }
3475
3476         /* Compensate for simulation model behaviour */
3477         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3478                 scc_mgr_set_dqs_bus_in_delay(i, 10);
3479                 scc_mgr_load_dqs(i);
3480         }
3481         addr = sdr_get_addr(&sdr_scc_mgr->update);
3482         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3483
3484         /*
3485          * ArriaV has hard FIFOs that can only be initialized by incrementing
3486          * in sequencer.
3487          */
3488         vfifo_offset = CALIB_VFIFO_OFFSET;
3489         addr = sdr_get_addr(&phy_mgr_cmd->inc_vfifo_hard_phy);
3490         for (j = 0; j < vfifo_offset; j++) {
3491                 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3492         }
3493         addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset);
3494         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3495
3496         /*
3497          * For ACV with hard lfifo, we get the skip-cal setting from
3498          * generation-time constant.
3499          */
3500         gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
3501         addr = sdr_get_addr(&phy_mgr_cfg->phy_rlat);
3502         writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
3503 }
3504
3505 /* Memory calibration entry point */
3506 static uint32_t mem_calibrate(void)
3507 {
3508         uint32_t i;
3509         uint32_t rank_bgn, sr;
3510         uint32_t write_group, write_test_bgn;
3511         uint32_t read_group, read_test_bgn;
3512         uint32_t run_groups, current_run;
3513         uint32_t failing_groups = 0;
3514         uint32_t group_failed = 0;
3515         uint32_t sr_failed = 0;
3516         uint32_t addr;
3517
3518         debug("%s:%d\n", __func__, __LINE__);
3519         /* Initialize the data settings */
3520
3521         gbl->error_substage = CAL_SUBSTAGE_NIL;
3522         gbl->error_stage = CAL_STAGE_NIL;
3523         gbl->error_group = 0xff;
3524         gbl->fom_in = 0;
3525         gbl->fom_out = 0;
3526
3527         mem_config();
3528
3529         uint32_t bypass_mode = 0x1;
3530         addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER);
3531         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3532                 writel(i, SOCFPGA_SDR_ADDRESS + addr);
3533                 scc_set_bypass_mode(i, bypass_mode);
3534         }
3535
3536         if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
3537                 /*
3538                  * Set VFIFO and LFIFO to instant-on settings in skip
3539                  * calibration mode.
3540                  */
3541                 mem_skip_calibrate();
3542         } else {
3543                 for (i = 0; i < NUM_CALIB_REPEAT; i++) {
3544                         /*
3545                          * Zero all delay chain/phase settings for all
3546                          * groups and all shadow register sets.
3547                          */
3548                         scc_mgr_zero_all();
3549
3550                         run_groups = ~param->skip_groups;
3551
3552                         for (write_group = 0, write_test_bgn = 0; write_group
3553                                 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
3554                                 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
3555                                 /* Initialized the group failure */
3556                                 group_failed = 0;
3557
3558                                 current_run = run_groups & ((1 <<
3559                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
3560                                 run_groups = run_groups >>
3561                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP;
3562
3563                                 if (current_run == 0)
3564                                         continue;
3565
3566                                 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER);
3567                                 writel(write_group, SOCFPGA_SDR_ADDRESS + addr);
3568                                 scc_mgr_zero_group(write_group, write_test_bgn,
3569                                                    0);
3570
3571                                 for (read_group = write_group *
3572                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3573                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3574                                         read_test_bgn = 0;
3575                                         read_group < (write_group + 1) *
3576                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3577                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3578                                         group_failed == 0;
3579                                         read_group++, read_test_bgn +=
3580                                         RW_MGR_MEM_DQ_PER_READ_DQS) {
3581                                         /* Calibrate the VFIFO */
3582                                         if (!((STATIC_CALIB_STEPS) &
3583                                                 CALIB_SKIP_VFIFO)) {
3584                                                 if (!rw_mgr_mem_calibrate_vfifo
3585                                                         (read_group,
3586                                                         read_test_bgn)) {
3587                                                         group_failed = 1;
3588
3589                                                         if (!(gbl->
3590                                                         phy_debug_mode_flags &
3591                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3592                                                                 return 0;
3593                                                         }
3594                                                 }
3595                                         }
3596                                 }
3597
3598                                 /* Calibrate the output side */
3599                                 if (group_failed == 0)  {
3600                                         for (rank_bgn = 0, sr = 0; rank_bgn
3601                                                 < RW_MGR_MEM_NUMBER_OF_RANKS;
3602                                                 rank_bgn +=
3603                                                 NUM_RANKS_PER_SHADOW_REG,
3604                                                 ++sr) {
3605                                                 sr_failed = 0;
3606                                                 if (!((STATIC_CALIB_STEPS) &
3607                                                 CALIB_SKIP_WRITES)) {
3608                                                         if ((STATIC_CALIB_STEPS)
3609                                                 & CALIB_SKIP_DELAY_SWEEPS) {
3610                                                 /* not needed in quick mode! */
3611                                                         } else {
3612                                                 /*
3613                                                  * Determine if this set of
3614                                                  * ranks should be skipped
3615                                                  * entirely.
3616                                                  */
3617                                         if (!param->skip_shadow_regs[sr]) {
3618                                                 if (!rw_mgr_mem_calibrate_writes
3619                                                 (rank_bgn, write_group,
3620                                                 write_test_bgn)) {
3621                                                         sr_failed = 1;
3622                                                         if (!(gbl->
3623                                                         phy_debug_mode_flags &
3624                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3625                                                                 return 0;
3626                                                                         }
3627                                                                         }
3628                                                                 }
3629                                                         }
3630                                                 }
3631                                                 if (sr_failed != 0)
3632                                                         group_failed = 1;
3633                                         }
3634                                 }
3635
3636                                 if (group_failed == 0) {
3637                                         for (read_group = write_group *
3638                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3639                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3640                                         read_test_bgn = 0;
3641                                                 read_group < (write_group + 1)
3642                                                 * RW_MGR_MEM_IF_READ_DQS_WIDTH
3643                                                 / RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3644                                                 group_failed == 0;
3645                                                 read_group++, read_test_bgn +=
3646                                                 RW_MGR_MEM_DQ_PER_READ_DQS) {
3647                                                 if (!((STATIC_CALIB_STEPS) &
3648                                                         CALIB_SKIP_WRITES)) {
3649                                         if (!rw_mgr_mem_calibrate_vfifo_end
3650                                                 (read_group, read_test_bgn)) {
3651                                                         group_failed = 1;
3652
3653                                                 if (!(gbl->phy_debug_mode_flags
3654                                                 & PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3655                                                                 return 0;
3656                                                                 }
3657                                                         }
3658                                                 }
3659                                         }
3660                                 }
3661
3662                                 if (group_failed != 0)
3663                                         failing_groups++;
3664                         }
3665
3666                         /*
3667                          * USER If there are any failing groups then report
3668                          * the failure.
3669                          */
3670                         if (failing_groups != 0)
3671                                 return 0;
3672
3673                         /* Calibrate the LFIFO */
3674                         if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) {
3675                                 /*
3676                                  * If we're skipping groups as part of debug,
3677                                  * don't calibrate LFIFO.
3678                                  */
3679                                 if (param->skip_groups == 0) {
3680                                         if (!rw_mgr_mem_calibrate_lfifo())
3681                                                 return 0;
3682                                 }
3683                         }
3684                 }
3685         }
3686
3687         /*
3688          * Do not remove this line as it makes sure all of our decisions
3689          * have been applied.
3690          */
3691         addr = sdr_get_addr(&sdr_scc_mgr->update);
3692         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3693         return 1;
3694 }
3695
3696 static uint32_t run_mem_calibrate(void)
3697 {
3698         uint32_t pass;
3699         uint32_t debug_info;
3700         uint32_t addr;
3701
3702         debug("%s:%d\n", __func__, __LINE__);
3703
3704         /* Reset pass/fail status shown on afi_cal_success/fail */
3705         addr = sdr_get_addr(&phy_mgr_cfg->cal_status);
3706         writel(PHY_MGR_CAL_RESET, SOCFPGA_SDR_ADDRESS + addr);
3707
3708         addr = sdr_get_addr((u32 *)BASE_MMR);
3709         /* stop tracking manger */
3710         uint32_t ctrlcfg = readl(SOCFPGA_SDR_ADDRESS + addr);
3711
3712         addr = sdr_get_addr((u32 *)BASE_MMR);
3713         writel(ctrlcfg & 0xFFBFFFFF, SOCFPGA_SDR_ADDRESS + addr);
3714
3715         initialize();
3716         rw_mgr_mem_initialize();
3717
3718         pass = mem_calibrate();
3719
3720         mem_precharge_and_activate();
3721         addr = sdr_get_addr(&phy_mgr_cmd->fifo_reset);
3722         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3723
3724         /*
3725          * Handoff:
3726          * Don't return control of the PHY back to AFI when in debug mode.
3727          */
3728         if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) {
3729                 rw_mgr_mem_handoff();
3730                 /*
3731                  * In Hard PHY this is a 2-bit control:
3732                  * 0: AFI Mux Select
3733                  * 1: DDIO Mux Select
3734                  */
3735                 addr = sdr_get_addr(&phy_mgr_cfg->mux_sel);
3736                 writel(0x2, SOCFPGA_SDR_ADDRESS + addr);
3737         }
3738
3739         addr = sdr_get_addr((u32 *)BASE_MMR);
3740         writel(ctrlcfg, SOCFPGA_SDR_ADDRESS + addr);
3741
3742         if (pass) {
3743                 printf("%s: CALIBRATION PASSED\n", __FILE__);
3744
3745                 gbl->fom_in /= 2;
3746                 gbl->fom_out /= 2;
3747
3748                 if (gbl->fom_in > 0xff)
3749                         gbl->fom_in = 0xff;
3750
3751                 if (gbl->fom_out > 0xff)
3752                         gbl->fom_out = 0xff;
3753
3754                 /* Update the FOM in the register file */
3755                 debug_info = gbl->fom_in;
3756                 debug_info |= gbl->fom_out << 8;
3757                 addr = sdr_get_addr(&sdr_reg_file->fom);
3758                 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3759
3760                 addr = sdr_get_addr(&phy_mgr_cfg->cal_debug_info);
3761                 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3762                 addr = sdr_get_addr(&phy_mgr_cfg->cal_status);
3763                 writel(PHY_MGR_CAL_SUCCESS, SOCFPGA_SDR_ADDRESS + addr);
3764         } else {
3765                 printf("%s: CALIBRATION FAILED\n", __FILE__);
3766
3767                 debug_info = gbl->error_stage;
3768                 debug_info |= gbl->error_substage << 8;
3769                 debug_info |= gbl->error_group << 16;
3770
3771                 addr = sdr_get_addr(&sdr_reg_file->failing_stage);
3772                 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3773                 addr = sdr_get_addr(&phy_mgr_cfg->cal_debug_info);
3774                 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3775                 addr = sdr_get_addr(&phy_mgr_cfg->cal_status);
3776                 writel(PHY_MGR_CAL_FAIL, SOCFPGA_SDR_ADDRESS + addr);
3777
3778                 /* Update the failing group/stage in the register file */
3779                 debug_info = gbl->error_stage;
3780                 debug_info |= gbl->error_substage << 8;
3781                 debug_info |= gbl->error_group << 16;
3782                 addr = sdr_get_addr(&sdr_reg_file->failing_stage);
3783                 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3784         }
3785
3786         return pass;
3787 }
3788
3789 static void hc_initialize_rom_data(void)
3790 {
3791         uint32_t i;
3792         uint32_t addr;
3793
3794         addr = sdr_get_addr((u32 *)(RW_MGR_INST_ROM_WRITE));
3795         for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++) {
3796                 uint32_t data = inst_rom_init[i];
3797                 writel(data, SOCFPGA_SDR_ADDRESS + addr + (i << 2));
3798         }
3799
3800         addr = sdr_get_addr((u32 *)(RW_MGR_AC_ROM_WRITE));
3801         for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++) {
3802                 uint32_t data = ac_rom_init[i];
3803                 writel(data, SOCFPGA_SDR_ADDRESS + addr + (i << 2));
3804         }
3805 }
3806
3807 static void initialize_reg_file(void)
3808 {
3809         uint32_t addr;
3810
3811         /* Initialize the register file with the correct data */
3812         addr = sdr_get_addr(&sdr_reg_file->signature);
3813         writel(REG_FILE_INIT_SEQ_SIGNATURE, SOCFPGA_SDR_ADDRESS + addr);
3814
3815         addr = sdr_get_addr(&sdr_reg_file->debug_data_addr);
3816         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3817
3818         addr = sdr_get_addr(&sdr_reg_file->cur_stage);
3819         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3820
3821         addr = sdr_get_addr(&sdr_reg_file->fom);
3822         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3823
3824         addr = sdr_get_addr(&sdr_reg_file->failing_stage);
3825         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3826
3827         addr = sdr_get_addr(&sdr_reg_file->debug1);
3828         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3829
3830         addr = sdr_get_addr(&sdr_reg_file->debug2);
3831         writel(0, SOCFPGA_SDR_ADDRESS + addr);
3832 }
3833
3834 static void initialize_hps_phy(void)
3835 {
3836         uint32_t reg;
3837         uint32_t addr;
3838         /*
3839          * Tracking also gets configured here because it's in the
3840          * same register.
3841          */
3842         uint32_t trk_sample_count = 7500;
3843         uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
3844         /*
3845          * Format is number of outer loops in the 16 MSB, sample
3846          * count in 16 LSB.
3847          */
3848
3849         reg = 0;
3850         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
3851         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
3852         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
3853         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
3854         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
3855         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
3856         /*
3857          * This field selects the intrinsic latency to RDATA_EN/FULL path.
3858          * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
3859          */
3860         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
3861         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
3862                 trk_sample_count);
3863         addr = sdr_get_addr((u32 *)BASE_MMR);
3864         writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_OFFSET);
3865
3866         reg = 0;
3867         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
3868                 trk_sample_count >>
3869                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
3870         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
3871                 trk_long_idle_sample_count);
3872         writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_OFFSET);
3873
3874         reg = 0;
3875         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
3876                 trk_long_idle_sample_count >>
3877                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
3878         writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_OFFSET);
3879 }
3880
3881 static void initialize_tracking(void)
3882 {
3883         uint32_t concatenated_longidle = 0x0;
3884         uint32_t concatenated_delays = 0x0;
3885         uint32_t concatenated_rw_addr = 0x0;
3886         uint32_t concatenated_refresh = 0x0;
3887         uint32_t trk_sample_count = 7500;
3888         uint32_t dtaps_per_ptap;
3889         uint32_t tmp_delay;
3890         uint32_t addr;
3891
3892         /*
3893          * compute usable version of value in case we skip full
3894          * computation later
3895          */
3896         dtaps_per_ptap = 0;
3897         tmp_delay = 0;
3898         while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
3899                 dtaps_per_ptap++;
3900                 tmp_delay += IO_DELAY_PER_DCHAIN_TAP;
3901         }
3902         dtaps_per_ptap--;
3903
3904         concatenated_longidle = concatenated_longidle ^ 10;
3905                 /*longidle outer loop */
3906         concatenated_longidle = concatenated_longidle << 16;
3907         concatenated_longidle = concatenated_longidle ^ 100;
3908                 /*longidle sample count */
3909         concatenated_delays = concatenated_delays ^ 243;
3910                 /* trfc, worst case of 933Mhz 4Gb */
3911         concatenated_delays = concatenated_delays << 8;
3912         concatenated_delays = concatenated_delays ^ 14;
3913                 /* trcd, worst case */
3914         concatenated_delays = concatenated_delays << 8;
3915         concatenated_delays = concatenated_delays ^ 10;
3916                 /* vfifo wait */
3917         concatenated_delays = concatenated_delays << 8;
3918         concatenated_delays = concatenated_delays ^ 4;
3919                 /* mux delay */
3920
3921         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE;
3922         concatenated_rw_addr = concatenated_rw_addr << 8;
3923         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1;
3924         concatenated_rw_addr = concatenated_rw_addr << 8;
3925         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ;
3926         concatenated_rw_addr = concatenated_rw_addr << 8;
3927         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL;
3928
3929         concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL;
3930         concatenated_refresh = concatenated_refresh << 24;
3931         concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */
3932
3933         /* Initialize the register file with the correct data */
3934         addr = sdr_get_addr(&sdr_reg_file->dtaps_per_ptap);
3935         writel(dtaps_per_ptap, SOCFPGA_SDR_ADDRESS + addr);
3936
3937         addr = sdr_get_addr(&sdr_reg_file->trk_sample_count);
3938         writel(trk_sample_count, SOCFPGA_SDR_ADDRESS + addr);
3939
3940         addr = sdr_get_addr(&sdr_reg_file->trk_longidle);
3941         writel(concatenated_longidle, SOCFPGA_SDR_ADDRESS + addr);
3942
3943         addr = sdr_get_addr(&sdr_reg_file->delays);
3944         writel(concatenated_delays, SOCFPGA_SDR_ADDRESS + addr);
3945
3946         addr = sdr_get_addr(&sdr_reg_file->trk_rw_mgr_addr);
3947         writel(concatenated_rw_addr, SOCFPGA_SDR_ADDRESS + addr);
3948
3949         addr = sdr_get_addr(&sdr_reg_file->trk_read_dqs_width);
3950         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, SOCFPGA_SDR_ADDRESS + addr);
3951
3952         addr = sdr_get_addr(&sdr_reg_file->trk_rfsh);
3953         writel(concatenated_refresh, SOCFPGA_SDR_ADDRESS + addr);
3954 }
3955
3956 int sdram_calibration_full(void)
3957 {
3958         struct param_type my_param;
3959         struct gbl_type my_gbl;
3960         uint32_t pass;
3961         uint32_t i;
3962
3963         param = &my_param;
3964         gbl = &my_gbl;
3965
3966         /* Initialize the debug mode flags */
3967         gbl->phy_debug_mode_flags = 0;
3968         /* Set the calibration enabled by default */
3969         gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
3970         /*
3971          * Only sweep all groups (regardless of fail state) by default
3972          * Set enabled read test by default.
3973          */
3974 #if DISABLE_GUARANTEED_READ
3975         gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
3976 #endif
3977         /* Initialize the register file */
3978         initialize_reg_file();
3979
3980         /* Initialize any PHY CSR */
3981         initialize_hps_phy();
3982
3983         scc_mgr_initialize();
3984
3985         initialize_tracking();
3986
3987         /* USER Enable all ranks, groups */
3988         for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++)
3989                 param->skip_ranks[i] = 0;
3990         for (i = 0; i < NUM_SHADOW_REGS; ++i)
3991                 param->skip_shadow_regs[i] = 0;
3992         param->skip_groups = 0;
3993
3994         printf("%s: Preparing to start memory calibration\n", __FILE__);
3995
3996         debug("%s:%d\n", __func__, __LINE__);
3997         debug_cond(DLEVEL == 1, "DDR3 FULL_RATE ranks=%lu cs/dimm=%lu dq/dqs=%lu,%lu vg/dqs=%lu,%lu",
3998                (long unsigned int)RW_MGR_MEM_NUMBER_OF_RANKS,
3999                (long unsigned int)RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
4000                (long unsigned int)RW_MGR_MEM_DQ_PER_READ_DQS,
4001                (long unsigned int)RW_MGR_MEM_DQ_PER_WRITE_DQS,
4002                (long unsigned int)RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
4003                (long unsigned int)RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
4004         debug_cond(DLEVEL == 1, "dqs=%lu,%lu dq=%lu dm=%lu ptap_delay=%lu dtap_delay=%lu",
4005                (long unsigned int)RW_MGR_MEM_IF_READ_DQS_WIDTH,
4006                (long unsigned int)RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
4007                (long unsigned int)RW_MGR_MEM_DATA_WIDTH,
4008                (long unsigned int)RW_MGR_MEM_DATA_MASK_WIDTH,
4009                (long unsigned int)IO_DELAY_PER_OPA_TAP,
4010                (long unsigned int)IO_DELAY_PER_DCHAIN_TAP);
4011         debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%lu, dll=%lu",
4012                (long unsigned int)IO_DELAY_PER_DQS_EN_DCHAIN_TAP,
4013                (long unsigned int)IO_DLL_CHAIN_LENGTH);
4014         debug_cond(DLEVEL == 1, "max values: en_p=%lu dqdqs_p=%lu en_d=%lu dqs_in_d=%lu",
4015                (long unsigned int)IO_DQS_EN_PHASE_MAX,
4016                (long unsigned int)IO_DQDQS_OUT_PHASE_MAX,
4017                (long unsigned int)IO_DQS_EN_DELAY_MAX,
4018                (long unsigned int)IO_DQS_IN_DELAY_MAX);
4019         debug_cond(DLEVEL == 1, "io_in_d=%lu io_out1_d=%lu io_out2_d=%lu",
4020                (long unsigned int)IO_IO_IN_DELAY_MAX,
4021                (long unsigned int)IO_IO_OUT1_DELAY_MAX,
4022                (long unsigned int)IO_IO_OUT2_DELAY_MAX);
4023         debug_cond(DLEVEL == 1, "dqs_in_reserve=%lu dqs_out_reserve=%lu",
4024                (long unsigned int)IO_DQS_IN_RESERVE,
4025                (long unsigned int)IO_DQS_OUT_RESERVE);
4026
4027         hc_initialize_rom_data();
4028
4029         /* update info for sims */
4030         reg_file_set_stage(CAL_STAGE_NIL);
4031         reg_file_set_group(0);
4032
4033         /*
4034          * Load global needed for those actions that require
4035          * some dynamic calibration support.
4036          */
4037         dyn_calib_steps = STATIC_CALIB_STEPS;
4038         /*
4039          * Load global to allow dynamic selection of delay loop settings
4040          * based on calibration mode.
4041          */
4042         if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
4043                 skip_delay_mask = 0xff;
4044         else
4045                 skip_delay_mask = 0x0;
4046
4047         pass = run_mem_calibrate();
4048
4049         printf("%s: Calibration complete\n", __FILE__);
4050         return pass;
4051 }