arch/mips/mach-mscc/include/mach/ddr.h

   1 /* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
   2 /*
   3  * Copyright (c) 2018 Microsemi Corporation
   4  */
   5
   6 #ifndef __ASM_MACH_DDR_H
   7 #define __ASM_MACH_DDR_H
   8
   9 #include <asm/cacheops.h>
  10 #include <asm/io.h>
  11 #include <asm/reboot.h>
  12 #include <mach/common.h>
  13
  14 #define MIPS_VCOREIII_MEMORY_DDR3
  15 #define MIPS_VCOREIII_DDR_SIZE CONFIG_SYS_SDRAM_SIZE
  16
  17 #if defined(CONFIG_DDRTYPE_H5TQ1G63BFA) /* Serval1 Refboard */
  18
  19 /* Hynix H5TQ1G63BFA (1Gbit DDR3, x16) @ 3.20ns */
  20 #define VC3_MPAR_bank_addr_cnt    3
  21 #define VC3_MPAR_row_addr_cnt     13
  22 #define VC3_MPAR_col_addr_cnt     10
  23 #define VC3_MPAR_tREFI            2437
  24 #define VC3_MPAR_tRAS_min         12
  25 #define VC3_MPAR_CL               6
  26 #define VC3_MPAR_tWTR             4
  27 #define VC3_MPAR_tRC              16
  28 #define VC3_MPR_tFAW             16
  29 #define VC3_MPAR_tRP              5
  30 #define VC3_MPAR_tRRD             4
  31 #define VC3_MPAR_tRCD             5
  32 #define VC3_MPAR_tMRD             4
  33 #define VC3_MPAR_tRFC             35
  34 #define VC3_MPAR_CWL              5
  35 #define VC3_MPAR_tXPR             38
  36 #define VC3_MPAR_tMOD             12
  37 #define VC3_MPAR_tDLLK            512
  38 #define VC3_MPAR_tWR              5
  39
  40 #elif defined(CONFIG_DDRTYPE_MT41J128M16HA)     /* Validation board */
  41
  42 /* Micron MT41J128M16HA-15E:D (2Gbit DDR3, x16) @ 3.20ns */
  43 #define VC3_MPAR_bank_addr_cnt    3
  44 #define VC3_MPAR_row_addr_cnt     14
  45 #define VC3_MPAR_col_addr_cnt     10
  46 #define VC3_MPAR_tREFI            2437
  47 #define VC3_MPAR_tRAS_min         12
  48 #define VC3_MPAR_CL               5
  49 #define VC3_MPAR_tWTR             4
  50 #define VC3_MPAR_tRC              16
  51 #define VC3_MPAR_tFAW             16
  52 #define VC3_MPAR_tRP              5
  53 #define VC3_MPAR_tRRD             4
  54 #define VC3_MPAR_tRCD             5
  55 #define VC3_MPAR_tMRD             4
  56 #define VC3_MPAR_tRFC             50
  57 #define VC3_MPAR_CWL              5
  58 #define VC3_MPAR_tXPR             54
  59 #define VC3_MPAR_tMOD             12
  60 #define VC3_MPAR_tDLLK            512
  61 #define VC3_MPAR_tWR              5
  62
  63 #elif defined(CONFIG_DDRTYPE_MT41K256M16)       /* JR2 Validation board */
  64
  65 /* Micron MT41K256M16 (4Gbit, DDR3L-800, 256Mbitx16) @ 3.20ns */
  66 #define VC3_MPAR_bank_addr_cnt    3
  67 #define VC3_MPAR_row_addr_cnt     15
  68 #define VC3_MPAR_col_addr_cnt     10
  69 #define VC3_MPAR_tREFI            2437
  70 #define VC3_MPAR_tRAS_min         12
  71 #define VC3_MPAR_CL               5
  72 #define VC3_MPAR_tWTR             4
  73 #define VC3_MPAR_tRC              16
  74 #define VC3_MPAR_tFAW             16
  75 #define VC3_MPAR_tRP              5
  76 #define VC3_MPAR_tRRD             4
  77 #define VC3_MPAR_tRCD             5
  78 #define VC3_MPAR_tMRD             4
  79 #define VC3_MPAR_tRFC             82
  80 #define VC3_MPAR_CWL              5
  81 #define VC3_MPAR_tXPR             85
  82 #define VC3_MPAR_tMOD             12
  83 #define VC3_MPAR_tDLLK            512
  84 #define VC3_MPAR_tWR              5
  85
  86 #elif defined(CONFIG_DDRTYPE_H5TQ4G63MFR)       /* JR2 Reference board */
  87
  88 /* Hynix H5TQ4G63MFR-PBC (4Gbit, DDR3-800, 256Mbitx16) - 2kb pages @ 3.20ns */
  89 #define VC3_MPAR_bank_addr_cnt    3
  90 #define VC3_MPAR_row_addr_cnt     15
  91 #define VC3_MPAR_col_addr_cnt     10
  92 #define VC3_MPAR_tREFI            2437
  93 #define VC3_MPAR_tRAS_min         12
  94 #define VC3_MPAR_CL               6
  95 #define VC3_MPAR_tWTR             4
  96 #define VC3_MPAR_tRC              17
  97 #define VC3_MPAR_tFAW             16
  98 #define VC3_MPAR_tRP              5
  99 #define VC3_MPAR_tRRD             4
 100 #define VC3_MPAR_tRCD             5
 101 #define VC3_MPAR_tMRD             4
 102 #define VC3_MPAR_tRFC             82
 103 #define VC3_MPAR_CWL              5
 104 #define VC3_MPAR_tXPR             85
 105 #define VC3_MPAR_tMOD             12
 106 #define VC3_MPAR_tDLLK            512
 107 #define VC3_MPAR_tWR              5
 108
 109 #elif defined(CONFIG_DDRTYPE_MT41K128M16JT)
 110
 111 /* Micron Micron MT41K128M16JT-125 (2Gbit DDR3L, 128Mbitx16) @ 3.20ns */
 112 #define VC3_MPAR_bank_addr_cnt    3
 113 #define VC3_MPAR_row_addr_cnt     14
 114 #define VC3_MPAR_col_addr_cnt     10
 115 #define VC3_MPAR_tREFI            2437
 116 #define VC3_MPAR_tRAS_min         12
 117 #define VC3_MPAR_CL               6
 118 #define VC3_MPAR_tWTR             4
 119 #define VC3_MPAR_tRC              16
 120 #define VC3_MPAR_tFAW             16
 121 #define VC3_MPAR_tRP              5
 122 #define VC3_MPAR_tRRD             4
 123 #define VC3_MPAR_tRCD             5
 124 #define VC3_MPAR_tMRD             4
 125 #define VC3_MPAR_tRFC             82
 126 #define VC3_MPAR_CWL              5
 127 #define VC3_MPAR_tXPR             85
 128 #define VC3_MPAR_tMOD             12
 129 #define VC3_MPAR_tDLLK            512
 130 #define VC3_MPAR_tWR              5
 131
 132 #elif defined(CONFIG_DDRTYPE_MT47H128M8HQ)      /* Luton10/26 Refboards */
 133
 134 /* Micron 1Gb MT47H128M8-3 16Meg x 8 x 8 banks, DDR-533@CL4 @ 4.80ns */
 135 #define VC3_MPAR_bank_addr_cnt    3
 136 #define VC3_MPAR_row_addr_cnt     14
 137 #define VC3_MPAR_col_addr_cnt     10
 138 #define VC3_MPAR_tREFI            1625
 139 #define VC3_MPAR_tRAS_min         9
 140 #define VC3_MPAR_CL               4
 141 #define VC3_MPAR_tWTR             2
 142 #define VC3_MPAR_tRC              12
 143 #define VC3_MPAR_tFAW             8
 144 #define VC3_MPAR_tRP              4
 145 #define VC3_MPAR_tRRD             2
 146 #define VC3_MPAR_tRCD             4
 147
 148 #define VC3_MPAR_tRPA             4
 149 #define VC3_MPAR_tRP              4
 150
 151 #define VC3_MPAR_tMRD             2
 152 #define VC3_MPAR_tRFC             27
 153
 154 #define VC3_MPAR__400_ns_dly      84
 155
 156 #define VC3_MPAR_tWR              4
 157 #undef MIPS_VCOREIII_MEMORY_DDR3
 158 #else
 159
 160 #error Unknown DDR system configuration - please add!
 161
 162 #endif
 163
 164 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
 165 #define MIPS_VCOREIII_MEMORY_16BIT 1
 166 #endif
 167
 168 #define MIPS_VCOREIII_MEMORY_SSTL_ODT 7
 169 #define MIPS_VCOREIII_MEMORY_SSTL_DRIVE 7
 170 #define VCOREIII_DDR_DQS_MODE_CALIBRATE
 171
 172 #ifdef MIPS_VCOREIII_MEMORY_16BIT
 173 #define VC3_MPAR_16BIT       1
 174 #else
 175 #define VC3_MPAR_16BIT       0
 176 #endif
 177
 178 #ifdef MIPS_VCOREIII_MEMORY_DDR3
 179 #define VC3_MPAR_DDR3_MODE    1 /* DDR3 */
 180 #define VC3_MPAR_BURST_LENGTH 8 /* Always 8 (1) for DDR3 */
 181 #ifdef MIPS_VCOREIII_MEMORY_16BIT
 182 #define VC3_MPAR_BURST_SIZE   1 /* Always 1 for DDR3/16bit */
 183 #else
 184 #define VC3_MPAR_BURST_SIZE   0
 185 #endif
 186 #else
 187 #define VC3_MPAR_DDR3_MODE    0 /* DDR2 */
 188 #ifdef MIPS_VCOREIII_MEMORY_16BIT
 189 #define VC3_MPAR_BURST_LENGTH 4 /* in DDR2 16-bit mode, use burstlen 4 */
 190 #else
 191 #define VC3_MPAR_BURST_LENGTH 8 /* For 8-bit IF we must run burst-8 */
 192 #endif
 193 #define VC3_MPAR_BURST_SIZE   0 /* Always 0 for DDR2 */
 194 #endif
 195
 196 #define VC3_MPAR_RL VC3_MPAR_CL
 197 #if !defined(MIPS_VCOREIII_MEMORY_DDR3)
 198 #define VC3_MPAR_WL (VC3_MPAR_RL - 1)
 199 #define VC3_MPAR_MD VC3_MPAR_tMRD
 200 #define VC3_MPAR_ID VC3_MPAR__400_ns_dly
 201 #define VC3_MPAR_SD VC3_MPAR_tXSRD
 202 #define VC3_MPAR_OW (VC3_MPAR_WL - 2)
 203 #define VC3_MPAR_OR (VC3_MPAR_WL - 3)
 204 #define VC3_MPAR_RP (VC3_MPAR_bank_addr_cnt < 3 ? VC3_MPAR_tRP : VC3_MPAR_tRPA)
 205 #define VC3_MPAR_FAW (VC3_MPAR_bank_addr_cnt < 3 ? 1 : VC3_MPAR_tFAW)
 206 #define VC3_MPAR_BL (VC3_MPAR_BURST_LENGTH == 4 ? 2 : 4)
 207 #define MSCC_MEMPARM_MR0 \
 208         (VC3_MPAR_BURST_LENGTH == 8 ? 3 : 2) | (VC3_MPAR_CL << 4) | \
 209         ((VC3_MPAR_tWR - 1) << 9)
 210 /* DLL-on, Full-OD, AL=0, RTT=off, nDQS-on, RDQS-off, out-en */
 211 #define MSCC_MEMPARM_MR1 0x382
 212 #define MSCC_MEMPARM_MR2 0
 213 #define MSCC_MEMPARM_MR3 0
 214 #else
 215 #define VC3_MPAR_WL VC3_MPAR_CWL
 216 #define VC3_MPAR_MD VC3_MPAR_tMOD
 217 #define VC3_MPAR_ID VC3_MPAR_tXPR
 218 #define VC3_MPAR_SD VC3_MPAR_tDLLK
 219 #define VC3_MPAR_OW 2
 220 #define VC3_MPAR_OR 2
 221 #define VC3_MPAR_RP VC3_MPAR_tRP
 222 #define VC3_MPAR_FAW VC3_MPAR_tFAW
 223 #define VC3_MPAR_BL 4
 224 #define MSCC_MEMPARM_MR0 ((VC3_MPAR_RL - 4) << 4) | ((VC3_MPAR_tWR - 4) << 9)
 225 /* ODT_RTT: “0x0040” for 120ohm, and “0x0004” for 60ohm. */
 226 #define MSCC_MEMPARM_MR1 0x0040
 227 #define MSCC_MEMPARM_MR2 ((VC3_MPAR_WL - 5) << 3)
 228 #define MSCC_MEMPARM_MR3 0
 229 #endif                          /* MIPS_VCOREIII_MEMORY_DDR3 */
 230
 231 #define MSCC_MEMPARM_MEMCFG                                             \
 232         ((MIPS_VCOREIII_DDR_SIZE > SZ_512M) ?                           \
 233          ICPU_MEMCTRL_CFG_DDR_512MBYTE_PLUS : 0) |                      \
 234         (VC3_MPAR_16BIT ? ICPU_MEMCTRL_CFG_DDR_WIDTH : 0) |             \
 235         (VC3_MPAR_DDR3_MODE ? ICPU_MEMCTRL_CFG_DDR_MODE : 0) |          \
 236         (VC3_MPAR_BURST_SIZE ? ICPU_MEMCTRL_CFG_BURST_SIZE : 0) |       \
 237         (VC3_MPAR_BURST_LENGTH == 8 ? ICPU_MEMCTRL_CFG_BURST_LEN : 0) | \
 238         (VC3_MPAR_bank_addr_cnt == 3 ? ICPU_MEMCTRL_CFG_BANK_CNT : 0) | \
 239         ICPU_MEMCTRL_CFG_MSB_ROW_ADDR(VC3_MPAR_row_addr_cnt - 1) |      \
 240         ICPU_MEMCTRL_CFG_MSB_COL_ADDR(VC3_MPAR_col_addr_cnt - 1)
 241
 242 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
 243 #define MSCC_MEMPARM_PERIOD                                     \
 244         ICPU_MEMCTRL_REF_PERIOD_MAX_PEND_REF(8) |               \
 245         ICPU_MEMCTRL_REF_PERIOD_REF_PERIOD(VC3_MPAR_tREFI)
 246
 247 #define MSCC_MEMPARM_TIMING0                                            \
 248         ICPU_MEMCTRL_TIMING0_RD_TO_WR_DLY(VC3_MPAR_RL + VC3_MPAR_BL + 1 - \
 249                                           VC3_MPAR_WL) |                \
 250         ICPU_MEMCTRL_TIMING0_WR_CS_CHANGE_DLY(VC3_MPAR_BL - 1) |        \
 251         ICPU_MEMCTRL_TIMING0_RD_CS_CHANGE_DLY(VC3_MPAR_BL) |            \
 252         ICPU_MEMCTRL_TIMING0_RAS_TO_PRECH_DLY(VC3_MPAR_tRAS_min - 1) |  \
 253         ICPU_MEMCTRL_TIMING0_WR_TO_PRECH_DLY(VC3_MPAR_WL +              \
 254                                              VC3_MPAR_BL +              \
 255                                              VC3_MPAR_tWR - 1) |        \
 256         ICPU_MEMCTRL_TIMING0_RD_TO_PRECH_DLY(VC3_MPAR_BL - 1) |         \
 257                 ICPU_MEMCTRL_TIMING0_WR_DATA_XFR_DLY(VC3_MPAR_WL - 1) | \
 258         ICPU_MEMCTRL_TIMING0_RD_DATA_XFR_DLY(VC3_MPAR_RL - 3)
 259
 260 #define MSCC_MEMPARM_TIMING1                                            \
 261         ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_SAME_BANK_DLY(VC3_MPAR_tRC - 1) | \
 262         ICPU_MEMCTRL_TIMING1_BANK8_FAW_DLY(VC3_MPAR_FAW - 1) |          \
 263         ICPU_MEMCTRL_TIMING1_PRECH_TO_RAS_DLY(VC3_MPAR_RP - 1) |        \
 264         ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_DLY(VC3_MPAR_tRRD - 1) |        \
 265         ICPU_MEMCTRL_TIMING1_RAS_TO_CAS_DLY(VC3_MPAR_tRCD - 1) |        \
 266         ICPU_MEMCTRL_TIMING1_WR_TO_RD_DLY(VC3_MPAR_WL +                 \
 267                                           VC3_MPAR_BL +                 \
 268                                           VC3_MPAR_tWTR - 1)
 269
 270 #define MSCC_MEMPARM_TIMING2                                    \
 271         ICPU_MEMCTRL_TIMING2_PRECH_ALL_DLY(VC3_MPAR_RP - 1) |   \
 272         ICPU_MEMCTRL_TIMING2_MDSET_DLY(VC3_MPAR_MD - 1) |               \
 273         ICPU_MEMCTRL_TIMING2_REF_DLY(VC3_MPAR_tRFC - 1) |               \
 274         ICPU_MEMCTRL_TIMING2_INIT_DLY(VC3_MPAR_ID - 1)
 275
 276 #define MSCC_MEMPARM_TIMING3                                            \
 277         ICPU_MEMCTRL_TIMING3_WR_TO_RD_CS_CHANGE_DLY(VC3_MPAR_WL +       \
 278                                                     VC3_MPAR_tWTR - 1) |\
 279         ICPU_MEMCTRL_TIMING3_ODT_RD_DLY(VC3_MPAR_OR - 1) |              \
 280         ICPU_MEMCTRL_TIMING3_ODT_WR_DLY(VC3_MPAR_OW - 1) |              \
 281         ICPU_MEMCTRL_TIMING3_LOCAL_ODT_RD_DLY(VC3_MPAR_RL - 3)
 282
 283 #else
 284 #define MSCC_MEMPARM_PERIOD                                     \
 285         ICPU_MEMCTRL_REF_PERIOD_MAX_PEND_REF(1) |               \
 286         ICPU_MEMCTRL_REF_PERIOD_REF_PERIOD(VC3_MPAR_tREFI)
 287
 288 #define MSCC_MEMPARM_TIMING0                                            \
 289         ICPU_MEMCTRL_TIMING0_RAS_TO_PRECH_DLY(VC3_MPAR_tRAS_min - 1) |  \
 290         ICPU_MEMCTRL_TIMING0_WR_TO_PRECH_DLY(VC3_MPAR_CL +              \
 291                                              (VC3_MPAR_BURST_LENGTH == 8 ? 2 : 0) + \
 292                                              VC3_MPAR_tWR) |            \
 293         ICPU_MEMCTRL_TIMING0_RD_TO_PRECH_DLY(VC3_MPAR_BURST_LENGTH == 8 ? 3 : 1) | \
 294         ICPU_MEMCTRL_TIMING0_WR_DATA_XFR_DLY(VC3_MPAR_CL - 3) |         \
 295         ICPU_MEMCTRL_TIMING0_RD_DATA_XFR_DLY(VC3_MPAR_CL - 3)
 296
 297 #define MSCC_MEMPARM_TIMING1                                            \
 298         ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_SAME_BANK_DLY(VC3_MPAR_tRC - 1) | \
 299         ICPU_MEMCTRL_TIMING1_BANK8_FAW_DLY(VC3_MPAR_tFAW - 1) |         \
 300         ICPU_MEMCTRL_TIMING1_PRECH_TO_RAS_DLY(VC3_MPAR_tRP - 1) |       \
 301         ICPU_MEMCTRL_TIMING1_RAS_TO_RAS_DLY(VC3_MPAR_tRRD - 1) |        \
 302         ICPU_MEMCTRL_TIMING1_RAS_TO_CAS_DLY(VC3_MPAR_tRCD - 1) |        \
 303         ICPU_MEMCTRL_TIMING1_WR_TO_RD_DLY(VC3_MPAR_CL +                 \
 304                                           (VC3_MPAR_BURST_LENGTH == 8 ? 2 : 0) + \
 305                                           VC3_MPAR_tWTR)
 306 #define MSCC_MEMPARM_TIMING2                                            \
 307         ICPU_MEMCTRL_TIMING2_PRECH_ALL_DLY(VC3_MPAR_tRPA - 1) |         \
 308         ICPU_MEMCTRL_TIMING2_MDSET_DLY(VC3_MPAR_tMRD - 1) |             \
 309         ICPU_MEMCTRL_TIMING2_REF_DLY(VC3_MPAR_tRFC - 1) |               \
 310         ICPU_MEMCTRL_TIMING2_FOUR_HUNDRED_NS_DLY(VC3_MPAR__400_ns_dly)
 311
 312 #define MSCC_MEMPARM_TIMING3                                            \
 313         ICPU_MEMCTRL_TIMING3_WR_TO_RD_CS_CHANGE_DLY(VC3_MPAR_CL - 1) |  \
 314         ICPU_MEMCTRL_TIMING3_ODT_WR_DLY(VC3_MPAR_CL - 1) |              \
 315         ICPU_MEMCTRL_TIMING3_LOCAL_ODT_RD_DLY(VC3_MPAR_CL - 1)
 316
 317 #endif
 318
 319 enum {
 320         DDR_TRAIN_OK,
 321         DDR_TRAIN_CONTINUE,
 322         DDR_TRAIN_ERROR,
 323 };
 324
 325 /*
 326  * We actually have very few 'pause' possibilities apart from
 327  * these assembly nops (at this very early stage).
 328  */
 329 #define PAUSE() asm volatile("nop; nop; nop; nop; nop; nop; nop; nop")
 330
 331 /* NB: Assumes inlining as no stack is available! */
 332 static inline void set_dly(u32 bytelane, u32 dly)
 333 {
 334         register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
 335
 336         r &= ~ICPU_MEMCTRL_DQS_DLY_DQS_DLY_M;
 337         r |= ICPU_MEMCTRL_DQS_DLY_DQS_DLY(dly);
 338         writel(r, BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
 339 }
 340
 341 static inline bool incr_dly(u32 bytelane)
 342 {
 343         register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
 344
 345         if (ICPU_MEMCTRL_DQS_DLY_DQS_DLY(r) < 31) {
 346                 writel(r + 1, BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
 347                 return true;
 348         }
 349
 350         return false;
 351 }
 352
 353 static inline bool adjust_dly(int adjust)
 354 {
 355         register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(0));
 356
 357         if (ICPU_MEMCTRL_DQS_DLY_DQS_DLY(r) < 31) {
 358                 writel(r + adjust, BASE_CFG + ICPU_MEMCTRL_DQS_DLY(0));
 359                 return true;
 360         }
 361
 362         return false;
 363 }
 364
 365 /* NB: Assumes inlining as no stack is available! */
 366 static inline void center_dly(u32 bytelane, u32 start)
 367 {
 368         register u32 r = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane)) - start;
 369
 370         writel(start + (r >> 1), BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
 371 }
 372
 373 static inline void memphy_soft_reset(void)
 374 {
 375         setbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_FIFO_RST);
 376         PAUSE();
 377         clrbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_FIFO_RST);
 378         PAUSE();
 379 }
 380
 381 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
 382 static u8 training_data[] = { 0xfe, 0x11, 0x33, 0x55, 0x77, 0x99, 0xbb, 0xdd };
 383
 384 static inline void sleep_100ns(u32 val)
 385 {
 386         /* Set the timer tick generator to 100 ns */
 387         writel(VCOREIII_TIMER_DIVIDER - 1, BASE_CFG + ICPU_TIMER_TICK_DIV);
 388
 389         /* Set the timer value */
 390         writel(val, BASE_CFG + ICPU_TIMER_VALUE(0));
 391
 392         /* Enable timer 0 for one-shot */
 393         writel(ICPU_TIMER_CTRL_ONE_SHOT_ENA | ICPU_TIMER_CTRL_TIMER_ENA,
 394                BASE_CFG + ICPU_TIMER_CTRL(0));
 395
 396         /* Wait for timer 0 to reach 0 */
 397         while (readl(BASE_CFG + ICPU_TIMER_VALUE(0)) != 0)
 398                 ;
 399 }
 400
 401 #if defined(CONFIG_SOC_OCELOT)
 402 static inline void hal_vcoreiii_ddr_reset_assert(void)
 403 {
 404         /* DDR has reset pin on GPIO 19 toggle Low-High to release */
 405         setbits_le32(BASE_DEVCPU_GCB + PERF_GPIO_OE, BIT(19));
 406         writel(BIT(19), BASE_DEVCPU_GCB + PERF_GPIO_OUT_CLR);
 407         sleep_100ns(10000);
 408 }
 409
 410 static inline void hal_vcoreiii_ddr_reset_release(void)
 411 {
 412         /* DDR has reset pin on GPIO 19 toggle Low-High to release */
 413         setbits_le32(BASE_DEVCPU_GCB + PERF_GPIO_OE, BIT(19));
 414         writel(BIT(19), BASE_DEVCPU_GCB + PERF_GPIO_OUT_SET);
 415         sleep_100ns(10000);
 416 }
 417
 418 /*
 419  * DDR memory sanity checking failed, tally and do hard reset
 420  *
 421  * NB: Assumes inlining as no stack is available!
 422  */
 423 static inline void hal_vcoreiii_ddr_failed(void)
 424 {
 425         register u32 reset;
 426
 427         writel(readl(BASE_CFG + ICPU_GPR(6)) + 1, BASE_CFG + ICPU_GPR(6));
 428
 429         clrbits_le32(BASE_DEVCPU_GCB + PERF_GPIO_OE, BIT(19));
 430
 431         /* We have to execute the reset function from cache. Indeed,
 432          * the reboot workaround in _machine_restart() will change the
 433          * SPI NOR into SW bitbang.
 434          *
 435          * This will render the CPU unable to execute directly from
 436          * the NOR, which is why the reset instructions are prefetched
 437          * into the I-cache.
 438          *
 439          * When failing the DDR initialization we are executing from
 440          * NOR.
 441          *
 442          * The last instruction in _machine_restart() will reset the
 443          * MIPS CPU (and the cache), and the CPU will start executing
 444          * from the reset vector.
 445          */
 446         reset = KSEG0ADDR(_machine_restart);
 447         icache_lock((void *)reset, 128);
 448         asm volatile ("jr %0"::"r" (reset));
 449
 450         panic("DDR init failed\n");
 451 }
 452 #else                           /* JR2 */
 453 static inline void hal_vcoreiii_ddr_reset_assert(void)
 454 {
 455         /* Ensure the memory controller physical iface is forced reset */
 456         writel(readl(BASE_CFG + ICPU_MEMPHY_CFG) |
 457                ICPU_MEMPHY_CFG_PHY_RST, BASE_CFG + ICPU_MEMPHY_CFG);
 458
 459         /* Ensure the memory controller is forced reset */
 460         writel(readl(BASE_CFG + ICPU_RESET) |
 461                ICPU_RESET_MEM_RST_FORCE, BASE_CFG + ICPU_RESET);
 462 }
 463
 464 static inline void hal_vcoreiii_ddr_failed(void)
 465 {
 466         writel(0, BASE_CFG + ICPU_RESET);
 467         writel(PERF_SOFT_RST_SOFT_CHIP_RST, BASE_CFG + PERF_SOFT_RST);
 468
 469         panic("DDR init failed\n");
 470 }
 471 #endif
 472
 473 /*
 474  * DDR memory sanity checking done, possibly enable ECC.
 475  *
 476  * NB: Assumes inlining as no stack is available!
 477  */
 478 static inline void hal_vcoreiii_ddr_verified(void)
 479 {
 480 #ifdef MIPS_VCOREIII_MEMORY_ECC
 481         /* Finally, enable ECC */
 482         register u32 val = readl(BASE_CFG + ICPU_MEMCTRL_CFG);
 483
 484         val |= ICPU_MEMCTRL_CFG_DDR_ECC_ERR_ENA;
 485         val &= ~ICPU_MEMCTRL_CFG_BURST_SIZE;
 486
 487         writel(val, BASE_CFG + ICPU_MEMCTRL_CFG);
 488 #endif
 489
 490         /* Reset Status register - sticky bits */
 491         writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT), BASE_CFG + ICPU_MEMCTRL_STAT);
 492 }
 493
 494 /* NB: Assumes inlining as no stack is available! */
 495 static inline int look_for(u32 bytelane)
 496 {
 497         register u32 i;
 498
 499         /* Reset FIFO in case any previous access failed */
 500         for (i = 0; i < sizeof(training_data); i++) {
 501                 register u32 byte;
 502
 503                 memphy_soft_reset();
 504                 /* Reset sticky bits */
 505                 writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT),
 506                        BASE_CFG + ICPU_MEMCTRL_STAT);
 507                 /* Read data */
 508                 byte = __raw_readb((void __iomem *)MSCC_DDR_TO + bytelane +
 509                                    (i * 4));
 510
 511                 /*
 512                  * Prevent the compiler reordering the instruction so
 513                  * the read of RAM happens after the check of the
 514                  * errors.
 515                  */
 516                 rmb();
 517                 if (readl(BASE_CFG + ICPU_MEMCTRL_STAT) &
 518                     (ICPU_MEMCTRL_STAT_RDATA_MASKED |
 519                      ICPU_MEMCTRL_STAT_RDATA_DUMMY)) {
 520                         /* Noise on the line */
 521                         goto read_error;
 522                 }
 523                 /* If mismatch, increment DQS - if possible */
 524                 if (byte != training_data[i]) {
 525  read_error:
 526                         if (!incr_dly(bytelane))
 527                                 return DDR_TRAIN_ERROR;
 528                         return DDR_TRAIN_CONTINUE;
 529                 }
 530         }
 531         return DDR_TRAIN_OK;
 532 }
 533
 534 /* NB: Assumes inlining as no stack is available! */
 535 static inline int look_past(u32 bytelane)
 536 {
 537         register u32 i;
 538
 539         /* Reset FIFO in case any previous access failed */
 540         for (i = 0; i < sizeof(training_data); i++) {
 541                 register u32 byte;
 542
 543                 memphy_soft_reset();
 544                 /* Ack sticky bits */
 545                 writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT),
 546                        BASE_CFG + ICPU_MEMCTRL_STAT);
 547                 byte = __raw_readb((void __iomem *)MSCC_DDR_TO + bytelane +
 548                                    (i * 4));
 549                 /*
 550                  * Prevent the compiler reordering the instruction so
 551                  * the read of RAM happens after the check of the
 552                  * errors.
 553                  */
 554                 rmb();
 555                 if (readl(BASE_CFG + ICPU_MEMCTRL_STAT) &
 556                     (ICPU_MEMCTRL_STAT_RDATA_MASKED |
 557                      ICPU_MEMCTRL_STAT_RDATA_DUMMY)) {
 558                         /* Noise on the line */
 559                         goto read_error;
 560                 }
 561                 /* Bail out when we see first mismatch */
 562                 if (byte != training_data[i]) {
 563  read_error:
 564                         return DDR_TRAIN_OK;
 565                 }
 566         }
 567         /* All data compares OK, increase DQS and retry */
 568         if (!incr_dly(bytelane))
 569                 return DDR_TRAIN_ERROR;
 570
 571         return DDR_TRAIN_CONTINUE;
 572 }
 573
 574 static inline int hal_vcoreiii_train_bytelane(u32 bytelane)
 575 {
 576         register int res;
 577         register u32 dqs_s;
 578
 579         set_dly(bytelane, 0);   /* Start training at DQS=0 */
 580         while ((res = look_for(bytelane)) == DDR_TRAIN_CONTINUE)
 581                 ;
 582         if (res != DDR_TRAIN_OK)
 583                 return res;
 584
 585         dqs_s = readl(BASE_CFG + ICPU_MEMCTRL_DQS_DLY(bytelane));
 586         while ((res = look_past(bytelane)) == DDR_TRAIN_CONTINUE)
 587                 ;
 588         if (res != DDR_TRAIN_OK)
 589                 return res;
 590         /* Reset FIFO - for good measure */
 591         memphy_soft_reset();
 592         /* Adjust to center [dqs_s;cur] */
 593         center_dly(bytelane, dqs_s);
 594         return DDR_TRAIN_OK;
 595 }
 596
 597 /* This algorithm is converted from the TCL training algorithm used
 598  * during silicon simulation.
 599  * NB: Assumes inlining as no stack is available!
 600  */
 601 static inline int hal_vcoreiii_init_dqs(void)
 602 {
 603 #define MAX_DQS 32
 604         register u32 i, j;
 605
 606         for (i = 0; i < MAX_DQS; i++) {
 607                 set_dly(0, i);  /* Byte-lane 0 */
 608                 for (j = 0; j < MAX_DQS; j++) {
 609                         __maybe_unused register u32  byte;
 610
 611                         set_dly(1, j);  /* Byte-lane 1 */
 612                         /* Reset FIFO in case any previous access failed */
 613                         memphy_soft_reset();
 614                         writel(readl(BASE_CFG + ICPU_MEMCTRL_STAT),
 615                                BASE_CFG + ICPU_MEMCTRL_STAT);
 616                         byte = __raw_readb((void __iomem *)MSCC_DDR_TO);
 617                         byte = __raw_readb((void __iomem *)(MSCC_DDR_TO + 1));
 618                         if (!(readl(BASE_CFG + ICPU_MEMCTRL_STAT) &
 619                             (ICPU_MEMCTRL_STAT_RDATA_MASKED |
 620                              ICPU_MEMCTRL_STAT_RDATA_DUMMY)))
 621                                 return 0;
 622                 }
 623         }
 624         return -1;
 625 }
 626
 627 static inline int dram_check(void)
 628 {
 629         register u32 i;
 630
 631         for (i = 0; i < 8; i++) {
 632                 __raw_writel(~i, (void __iomem *)(MSCC_DDR_TO + (i * 4)));
 633                 if (__raw_readl((void __iomem *)(MSCC_DDR_TO + (i * 4))) != ~i)
 634                         return 1;
 635         }
 636         return 0;
 637 }
 638 #else                           /* Luton */
 639
 640 static inline void sleep_100ns(u32 val)
 641 {
 642 }
 643
 644 static inline void hal_vcoreiii_ddr_reset_assert(void)
 645 {
 646         setbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_RST);
 647         setbits_le32(BASE_CFG + ICPU_RESET, ICPU_RESET_MEM_RST_FORCE);
 648 }
 649
 650 static inline void hal_vcoreiii_ddr_reset_release(void)
 651 {
 652 }
 653
 654 static inline void hal_vcoreiii_ddr_failed(void)
 655 {
 656         register u32 memphy_cfg = readl(BASE_CFG + ICPU_MEMPHY_CFG);
 657
 658         /* Do a fifo reset and start over */
 659         writel(memphy_cfg | ICPU_MEMPHY_CFG_PHY_FIFO_RST,
 660                BASE_CFG + ICPU_MEMPHY_CFG);
 661         writel(memphy_cfg & ~ICPU_MEMPHY_CFG_PHY_FIFO_RST,
 662                BASE_CFG + ICPU_MEMPHY_CFG);
 663         writel(memphy_cfg | ICPU_MEMPHY_CFG_PHY_FIFO_RST,
 664                BASE_CFG + ICPU_MEMPHY_CFG);
 665 }
 666
 667 static inline void hal_vcoreiii_ddr_verified(void)
 668 {
 669 }
 670
 671 static inline int look_for(u32 data)
 672 {
 673         register u32 byte = __raw_readb((void __iomem *)MSCC_DDR_TO);
 674
 675         if (data != byte) {
 676                 if (!incr_dly(0))
 677                         return DDR_TRAIN_ERROR;
 678                 return DDR_TRAIN_CONTINUE;
 679         }
 680
 681         return DDR_TRAIN_OK;
 682 }
 683
 684 /* This algorithm is converted from the TCL training algorithm used
 685  * during silicon simulation.
 686  * NB: Assumes inlining as no stack is available!
 687  */
 688 static inline int hal_vcoreiii_train_bytelane(u32 bytelane)
 689 {
 690         register int res;
 691
 692         set_dly(bytelane, 0);   /* Start training at DQS=0 */
 693         while ((res = look_for(0xff)) == DDR_TRAIN_CONTINUE)
 694                 ;
 695         if (res != DDR_TRAIN_OK)
 696                 return res;
 697
 698         set_dly(bytelane, 0);   /* Start training at DQS=0 */
 699         while ((res = look_for(0x00)) == DDR_TRAIN_CONTINUE)
 700
 701                 ;
 702
 703         if (res != DDR_TRAIN_OK)
 704                 return res;
 705
 706         adjust_dly(-3);
 707
 708         return DDR_TRAIN_OK;
 709 }
 710
 711 static inline int hal_vcoreiii_init_dqs(void)
 712 {
 713         return 0;
 714 }
 715
 716 static inline int dram_check(void)
 717 {
 718         register u32 i;
 719
 720         for (i = 0; i < 8; i++) {
 721                 __raw_writel(~i, (void __iomem *)(MSCC_DDR_TO + (i * 4)));
 722
 723                 if (__raw_readl((void __iomem *)(MSCC_DDR_TO + (i * 4))) != ~i)
 724                         return 1;
 725         }
 726
 727         return 0;
 728 }
 729 #endif
 730
 731 /*
 732  * NB: Called *early* to init memory controller - assumes inlining as
 733  * no stack is available!
 734  */
 735 static inline void hal_vcoreiii_init_memctl(void)
 736 {
 737         /* Ensure DDR is in reset */
 738         hal_vcoreiii_ddr_reset_assert();
 739
 740         /* Wait maybe not needed, but ... */
 741         PAUSE();
 742
 743         /* Drop sys ctl memory controller forced reset */
 744         clrbits_le32(BASE_CFG + ICPU_RESET, ICPU_RESET_MEM_RST_FORCE);
 745
 746         PAUSE();
 747
 748         /* Drop Reset, enable SSTL */
 749         writel(ICPU_MEMPHY_CFG_PHY_SSTL_ENA, BASE_CFG + ICPU_MEMPHY_CFG);
 750         PAUSE();
 751
 752         /* Start the automatic SSTL output and ODT drive-strength calibration */
 753         writel(ICPU_MEMPHY_ZCAL_ZCAL_PROG_ODT(MIPS_VCOREIII_MEMORY_SSTL_ODT) |
 754                /* drive strength */
 755                ICPU_MEMPHY_ZCAL_ZCAL_PROG(MIPS_VCOREIII_MEMORY_SSTL_DRIVE) |
 756                /* Start calibration process */
 757                ICPU_MEMPHY_ZCAL_ZCAL_ENA, BASE_CFG + ICPU_MEMPHY_ZCAL);
 758
 759         /* Wait for ZCAL to clear */
 760         while (readl(BASE_CFG + ICPU_MEMPHY_ZCAL) & ICPU_MEMPHY_ZCAL_ZCAL_ENA)
 761                 ;
 762 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
 763         /* Check no ZCAL_ERR */
 764         if (readl(BASE_CFG + ICPU_MEMPHY_ZCAL_STAT)
 765             & ICPU_MEMPHY_ZCAL_STAT_ZCAL_ERR)
 766                 hal_vcoreiii_ddr_failed();
 767 #endif
 768         /* Drive CL, CK, ODT */
 769         setbits_le32(BASE_CFG + ICPU_MEMPHY_CFG, ICPU_MEMPHY_CFG_PHY_ODT_OE |
 770                      ICPU_MEMPHY_CFG_PHY_CK_OE | ICPU_MEMPHY_CFG_PHY_CL_OE);
 771
 772         /* Initialize memory controller */
 773         writel(MSCC_MEMPARM_MEMCFG, BASE_CFG + ICPU_MEMCTRL_CFG);
 774         writel(MSCC_MEMPARM_PERIOD, BASE_CFG + ICPU_MEMCTRL_REF_PERIOD);
 775
 776 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
 777         writel(MSCC_MEMPARM_TIMING0, BASE_CFG + ICPU_MEMCTRL_TIMING0);
 778 #else /* Luton */
 779         clrbits_le32(BASE_CFG + ICPU_MEMCTRL_TIMING0, ((1 << 20) - 1));
 780         setbits_le32(BASE_CFG + ICPU_MEMCTRL_TIMING0, MSCC_MEMPARM_TIMING0);
 781 #endif
 782
 783         writel(MSCC_MEMPARM_TIMING1, BASE_CFG + ICPU_MEMCTRL_TIMING1);
 784         writel(MSCC_MEMPARM_TIMING2, BASE_CFG + ICPU_MEMCTRL_TIMING2);
 785         writel(MSCC_MEMPARM_TIMING3, BASE_CFG + ICPU_MEMCTRL_TIMING3);
 786         writel(MSCC_MEMPARM_MR0, BASE_CFG + ICPU_MEMCTRL_MR0_VAL);
 787         writel(MSCC_MEMPARM_MR1, BASE_CFG + ICPU_MEMCTRL_MR1_VAL);
 788         writel(MSCC_MEMPARM_MR2, BASE_CFG + ICPU_MEMCTRL_MR2_VAL);
 789         writel(MSCC_MEMPARM_MR3, BASE_CFG + ICPU_MEMCTRL_MR3_VAL);
 790
 791 #if defined(CONFIG_SOC_OCELOT)
 792         /* Termination setup - enable ODT */
 793         writel(ICPU_MEMCTRL_TERMRES_CTRL_LOCAL_ODT_RD_ENA |
 794                /* Assert ODT0 for any write */
 795                ICPU_MEMCTRL_TERMRES_CTRL_ODT_WR_ENA(3),
 796                BASE_CFG + ICPU_MEMCTRL_TERMRES_CTRL);
 797
 798         /* Release Reset from DDR */
 799         hal_vcoreiii_ddr_reset_release();
 800
 801         writel(readl(BASE_CFG + ICPU_GPR(7)) + 1, BASE_CFG + ICPU_GPR(7));
 802 #elif defined(CONFIG_SOC_JR2)
 803         writel(ICPU_MEMCTRL_TERMRES_CTRL_ODT_WR_ENA(3),
 804                BASE_CFG + ICPU_MEMCTRL_TERMRES_CTRL);
 805 #else                           /* Luton */
 806         /* Termination setup - disable ODT */
 807         writel(0, BASE_CFG + ICPU_MEMCTRL_TERMRES_CTRL);
 808
 809 #endif
 810 }
 811
 812 static inline void hal_vcoreiii_wait_memctl(void)
 813 {
 814         /* Now, rip it! */
 815         writel(ICPU_MEMCTRL_CTRL_INITIALIZE, BASE_CFG + ICPU_MEMCTRL_CTRL);
 816
 817         while (!(readl(BASE_CFG + ICPU_MEMCTRL_STAT)
 818                  & ICPU_MEMCTRL_STAT_INIT_DONE))
 819                 ;
 820
 821         /* Settle...? */
 822         sleep_100ns(10000);
 823 #if defined(CONFIG_SOC_OCELOT) || defined(CONFIG_SOC_JR2)
 824         /* Establish data contents in DDR RAM for training */
 825
 826         __raw_writel(0xcacafefe, ((void __iomem *)MSCC_DDR_TO));
 827         __raw_writel(0x22221111, ((void __iomem *)MSCC_DDR_TO + 0x4));
 828         __raw_writel(0x44443333, ((void __iomem *)MSCC_DDR_TO + 0x8));
 829         __raw_writel(0x66665555, ((void __iomem *)MSCC_DDR_TO + 0xC));
 830         __raw_writel(0x88887777, ((void __iomem *)MSCC_DDR_TO + 0x10));
 831         __raw_writel(0xaaaa9999, ((void __iomem *)MSCC_DDR_TO + 0x14));
 832         __raw_writel(0xccccbbbb, ((void __iomem *)MSCC_DDR_TO + 0x18));
 833         __raw_writel(0xeeeedddd, ((void __iomem *)MSCC_DDR_TO + 0x1C));
 834 #else
 835         __raw_writel(0xff, ((void __iomem *)MSCC_DDR_TO));
 836 #endif
 837 }
 838 #endif                          /* __ASM_MACH_DDR_H */