drivers/ram/rockchip/sdram_rk3399.c

   1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
   2 /*
   3  * (C) Copyright 2016-2017 Rockchip Inc.
   4  *
   5  * Adapted from coreboot.
   6  */
   7
   8 #include <common.h>
   9 #include <clk.h>
  10 #include <dm.h>
  11 #include <dt-structs.h>
  12 #include <ram.h>
  13 #include <regmap.h>
  14 #include <syscon.h>
  15 #include <asm/io.h>
  16 #include <asm/arch-rockchip/clock.h>
  17 #include <asm/arch-rockchip/cru_rk3399.h>
  18 #include <asm/arch-rockchip/grf_rk3399.h>
  19 #include <asm/arch-rockchip/hardware.h>
  20 #include <asm/arch-rockchip/sdram_common.h>
  21 #include <asm/arch-rockchip/sdram_rk3399.h>
  22 #include <linux/err.h>
  23 #include <time.h>
  24
  25 #define PRESET_SGRF_HOLD(n)     ((0x1 << (6 + 16)) | ((n) << 6))
  26 #define PRESET_GPIO0_HOLD(n)    ((0x1 << (7 + 16)) | ((n) << 7))
  27 #define PRESET_GPIO1_HOLD(n)    ((0x1 << (8 + 16)) | ((n) << 8))
  28
  29 #define PHY_DRV_ODT_HI_Z        0x0
  30 #define PHY_DRV_ODT_240         0x1
  31 #define PHY_DRV_ODT_120         0x8
  32 #define PHY_DRV_ODT_80          0x9
  33 #define PHY_DRV_ODT_60          0xc
  34 #define PHY_DRV_ODT_48          0xd
  35 #define PHY_DRV_ODT_40          0xe
  36 #define PHY_DRV_ODT_34_3        0xf
  37
  38 #define PHY_BOOSTP_EN           0x1
  39 #define PHY_BOOSTN_EN           0x1
  40 #define PHY_SLEWP_EN            0x1
  41 #define PHY_SLEWN_EN            0x1
  42 #define PHY_RX_CM_INPUT         0x1
  43 #define CS0_MR22_VAL            0
  44 #define CS1_MR22_VAL            3
  45
  46 #define CRU_SFTRST_DDR_CTRL(ch, n)      ((0x1 << (8 + 16 + (ch) * 4)) | \
  47                                         ((n) << (8 + (ch) * 4)))
  48 #define CRU_SFTRST_DDR_PHY(ch, n)       ((0x1 << (9 + 16 + (ch) * 4)) | \
  49                                         ((n) << (9 + (ch) * 4)))
  50 struct chan_info {
  51         struct rk3399_ddr_pctl_regs *pctl;
  52         struct rk3399_ddr_pi_regs *pi;
  53         struct rk3399_ddr_publ_regs *publ;
  54         struct rk3399_msch_regs *msch;
  55 };
  56
  57 struct dram_info {
  58 #if defined(CONFIG_TPL_BUILD) || \
  59         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  60         u32 pwrup_srefresh_exit[2];
  61         struct chan_info chan[2];
  62         struct clk ddr_clk;
  63         struct rk3399_cru *cru;
  64         struct rk3399_grf_regs *grf;
  65         struct rk3399_pmucru *pmucru;
  66         struct rk3399_pmusgrf_regs *pmusgrf;
  67         struct rk3399_ddr_cic_regs *cic;
  68 #endif
  69         struct ram_info info;
  70         struct rk3399_pmugrf_regs *pmugrf;
  71 };
  72
  73 #if defined(CONFIG_TPL_BUILD) || \
  74         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  75
  76 struct rockchip_dmc_plat {
  77 #if CONFIG_IS_ENABLED(OF_PLATDATA)
  78         struct dtd_rockchip_rk3399_dmc dtplat;
  79 #else
  80         struct rk3399_sdram_params sdram_params;
  81 #endif
  82         struct regmap *map;
  83 };
  84
  85 struct io_setting {
  86         u32 mhz;
  87         u32 mr5;
  88         /* dram side */
  89         u32 dq_odt;
  90         u32 ca_odt;
  91         u32 pdds;
  92         u32 dq_vref;
  93         u32 ca_vref;
  94         /* phy side */
  95         u32 rd_odt;
  96         u32 wr_dq_drv;
  97         u32 wr_ca_drv;
  98         u32 wr_ckcs_drv;
  99         u32 rd_odt_en;
 100         u32 rd_vref;
 101 } lpddr4_io_setting[] = {
 102         {
 103                 50 * MHz,
 104                 0,
 105                 /* dram side */
 106                 0,      /* dq_odt; */
 107                 0,      /* ca_odt; */
 108                 6,      /* pdds; */
 109                 0x72,   /* dq_vref; */
 110                 0x72,   /* ca_vref; */
 111                 /* phy side */
 112                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 113                 PHY_DRV_ODT_40, /* wr_dq_drv; */
 114                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 115                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 116                 0,      /* rd_odt_en;*/
 117                 41,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 118         },
 119         {
 120                 600 * MHz,
 121                 0,
 122                 /* dram side */
 123                 1,      /* dq_odt; */
 124                 0,      /* ca_odt; */
 125                 6,      /* pdds; */
 126                 0x72,   /* dq_vref; */
 127                 0x72,   /* ca_vref; */
 128                 /* phy side */
 129                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 130                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 131                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 132                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 133                 0,      /* rd_odt_en; */
 134                 32,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 135         },
 136         {
 137                 800 * MHz,
 138                 0,
 139                 /* dram side */
 140                 1,      /* dq_odt; */
 141                 0,      /* ca_odt; */
 142                 1,      /* pdds; */
 143                 0x72,   /* dq_vref; */
 144                 0x72,   /* ca_vref; */
 145                 /* phy side */
 146                 PHY_DRV_ODT_40, /* rd_odt; */
 147                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 148                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 149                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 150                 1,      /* rd_odt_en; */
 151                 17,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 152         },
 153         {
 154                 933 * MHz,
 155                 0,
 156                 /* dram side */
 157                 3,      /* dq_odt; */
 158                 0,      /* ca_odt; */
 159                 6,      /* pdds; */
 160                 0x59,   /* dq_vref; 32% */
 161                 0x72,   /* ca_vref; */
 162                 /* phy side */
 163                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 164                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 165                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 166                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 167                 0,      /* rd_odt_en; */
 168                 32,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 169         },
 170         {
 171                 1066 * MHz,
 172                 0,
 173                 /* dram side */
 174                 6,      /* dq_odt; */
 175                 0,      /* ca_odt; */
 176                 1,      /* pdds; */
 177                 0x10,   /* dq_vref; */
 178                 0x72,   /* ca_vref; */
 179                 /* phy side */
 180                 PHY_DRV_ODT_40, /* rd_odt; */
 181                 PHY_DRV_ODT_60, /* wr_dq_drv; */
 182                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 183                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 184                 1,      /* rd_odt_en; */
 185                 17,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 186         },
 187 };
 188
 189 /**
 190  * phy = 0, PHY boot freq
 191  * phy = 1, PHY index 0
 192  * phy = 2, PHY index 1
 193  */
 194 static struct io_setting *
 195 lpddr4_get_io_settings(const struct rk3399_sdram_params *params, u32 mr5)
 196 {
 197         struct io_setting *io = NULL;
 198         u32 n;
 199
 200         for (n = 0; n < ARRAY_SIZE(lpddr4_io_setting); n++) {
 201                 io = &lpddr4_io_setting[n];
 202
 203                 if (io->mr5 != 0) {
 204                         if (io->mhz >= params->base.ddr_freq &&
 205                             io->mr5 == mr5)
 206                                 break;
 207                 } else {
 208                         if (io->mhz >= params->base.ddr_freq)
 209                                 break;
 210                 }
 211         }
 212
 213         return io;
 214 }
 215
 216 static void *get_ddrc0_con(struct dram_info *dram, u8 channel)
 217 {
 218         return (channel == 0) ? &dram->grf->ddrc0_con0 : &dram->grf->ddrc0_con1;
 219 }
 220
 221 static void copy_to_reg(u32 *dest, const u32 *src, u32 n)
 222 {
 223         int i;
 224
 225         for (i = 0; i < n / sizeof(u32); i++) {
 226                 writel(*src, dest);
 227                 src++;
 228                 dest++;
 229         }
 230 }
 231
 232 static void rkclk_ddr_reset(struct rk3399_cru *cru, u32 channel, u32 ctl,
 233                             u32 phy)
 234 {
 235         channel &= 0x1;
 236         ctl &= 0x1;
 237         phy &= 0x1;
 238         writel(CRU_SFTRST_DDR_CTRL(channel, ctl) |
 239                                    CRU_SFTRST_DDR_PHY(channel, phy),
 240                                    &cru->softrst_con[4]);
 241 }
 242
 243 static void phy_pctrl_reset(struct rk3399_cru *cru,  u32 channel)
 244 {
 245         rkclk_ddr_reset(cru, channel, 1, 1);
 246         udelay(10);
 247
 248         rkclk_ddr_reset(cru, channel, 1, 0);
 249         udelay(10);
 250
 251         rkclk_ddr_reset(cru, channel, 0, 0);
 252         udelay(10);
 253 }
 254
 255 static void phy_dll_bypass_set(struct rk3399_ddr_publ_regs *ddr_publ_regs,
 256                                u32 freq)
 257 {
 258         u32 *denali_phy = ddr_publ_regs->denali_phy;
 259
 260         /* From IP spec, only freq small than 125 can enter dll bypass mode */
 261         if (freq <= 125) {
 262                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 263                 setbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 264                 setbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 265                 setbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 266                 setbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 267
 268                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 269                 setbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 270                 setbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 271                 setbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 272         } else {
 273                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 274                 clrbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 275                 clrbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 276                 clrbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 277                 clrbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 278
 279                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 280                 clrbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 281                 clrbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 282                 clrbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 283         }
 284 }
 285
 286 static void set_memory_map(const struct chan_info *chan, u32 channel,
 287                            const struct rk3399_sdram_params *params)
 288 {
 289         const struct rk3399_sdram_channel *sdram_ch = &params->ch[channel];
 290         u32 *denali_ctl = chan->pctl->denali_ctl;
 291         u32 *denali_pi = chan->pi->denali_pi;
 292         u32 cs_map;
 293         u32 reduc;
 294         u32 row;
 295
 296         /* Get row number from ddrconfig setting */
 297         if (sdram_ch->cap_info.ddrconfig < 2 ||
 298             sdram_ch->cap_info.ddrconfig == 4)
 299                 row = 16;
 300         else if (sdram_ch->cap_info.ddrconfig == 3)
 301                 row = 14;
 302         else
 303                 row = 15;
 304
 305         cs_map = (sdram_ch->cap_info.rank > 1) ? 3 : 1;
 306         reduc = (sdram_ch->cap_info.bw == 2) ? 0 : 1;
 307
 308         /* Set the dram configuration to ctrl */
 309         clrsetbits_le32(&denali_ctl[191], 0xF, (12 - sdram_ch->cap_info.col));
 310         clrsetbits_le32(&denali_ctl[190], (0x3 << 16) | (0x7 << 24),
 311                         ((3 - sdram_ch->cap_info.bk) << 16) |
 312                         ((16 - row) << 24));
 313
 314         clrsetbits_le32(&denali_ctl[196], 0x3 | (1 << 16),
 315                         cs_map | (reduc << 16));
 316
 317         /* PI_199 PI_COL_DIFF:RW:0:4 */
 318         clrsetbits_le32(&denali_pi[199], 0xF, (12 - sdram_ch->cap_info.col));
 319
 320         /* PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2 */
 321         clrsetbits_le32(&denali_pi[155], (0x3 << 16) | (0x7 << 24),
 322                         ((3 - sdram_ch->cap_info.bk) << 16) |
 323                         ((16 - row) << 24));
 324
 325         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 326                 if (cs_map == 1)
 327                         cs_map = 0x5;
 328                 else if (cs_map == 2)
 329                         cs_map = 0xa;
 330                 else
 331                         cs_map = 0xF;
 332         }
 333
 334         /* PI_41 PI_CS_MAP:RW:24:4 */
 335         clrsetbits_le32(&denali_pi[41], 0xf << 24, cs_map << 24);
 336         if (sdram_ch->cap_info.rank == 1 && params->base.dramtype == DDR3)
 337                 writel(0x2EC7FFFF, &denali_pi[34]);
 338 }
 339
 340 static int phy_io_config(const struct chan_info *chan,
 341                          const struct rk3399_sdram_params *params)
 342 {
 343         u32 *denali_phy = chan->publ->denali_phy;
 344         u32 vref_mode_dq, vref_value_dq, vref_mode_ac, vref_value_ac;
 345         u32 mode_sel;
 346         u32 reg_value;
 347         u32 drv_value, odt_value;
 348         u32 speed;
 349
 350         /* vref setting */
 351         if (params->base.dramtype == LPDDR4) {
 352                 /* LPDDR4 */
 353                 vref_mode_dq = 0x6;
 354                 vref_value_dq = 0x1f;
 355                 vref_mode_ac = 0x6;
 356                 vref_value_ac = 0x1f;
 357                 mode_sel = 0x6;
 358         } else if (params->base.dramtype == LPDDR3) {
 359                 if (params->base.odt == 1) {
 360                         vref_mode_dq = 0x5;  /* LPDDR3 ODT */
 361                         drv_value = (readl(&denali_phy[6]) >> 12) & 0xf;
 362                         odt_value = (readl(&denali_phy[6]) >> 4) & 0xf;
 363                         if (drv_value == PHY_DRV_ODT_48) {
 364                                 switch (odt_value) {
 365                                 case PHY_DRV_ODT_240:
 366                                         vref_value_dq = 0x16;
 367                                         break;
 368                                 case PHY_DRV_ODT_120:
 369                                         vref_value_dq = 0x26;
 370                                         break;
 371                                 case PHY_DRV_ODT_60:
 372                                         vref_value_dq = 0x36;
 373                                         break;
 374                                 default:
 375                                         debug("Invalid ODT value.\n");
 376                                         return -EINVAL;
 377                                 }
 378                         } else if (drv_value == PHY_DRV_ODT_40) {
 379                                 switch (odt_value) {
 380                                 case PHY_DRV_ODT_240:
 381                                         vref_value_dq = 0x19;
 382                                         break;
 383                                 case PHY_DRV_ODT_120:
 384                                         vref_value_dq = 0x23;
 385                                         break;
 386                                 case PHY_DRV_ODT_60:
 387                                         vref_value_dq = 0x31;
 388                                         break;
 389                                 default:
 390                                         debug("Invalid ODT value.\n");
 391                                         return -EINVAL;
 392                                 }
 393                         } else if (drv_value == PHY_DRV_ODT_34_3) {
 394                                 switch (odt_value) {
 395                                 case PHY_DRV_ODT_240:
 396                                         vref_value_dq = 0x17;
 397                                         break;
 398                                 case PHY_DRV_ODT_120:
 399                                         vref_value_dq = 0x20;
 400                                         break;
 401                                 case PHY_DRV_ODT_60:
 402                                         vref_value_dq = 0x2e;
 403                                         break;
 404                                 default:
 405                                         debug("Invalid ODT value.\n");
 406                                         return -EINVAL;
 407                                 }
 408                         } else {
 409                                 debug("Invalid DRV value.\n");
 410                                 return -EINVAL;
 411                         }
 412                 } else {
 413                         vref_mode_dq = 0x2;  /* LPDDR3 */
 414                         vref_value_dq = 0x1f;
 415                 }
 416                 vref_mode_ac = 0x2;
 417                 vref_value_ac = 0x1f;
 418                 mode_sel = 0x0;
 419         } else if (params->base.dramtype == DDR3) {
 420                 /* DDR3L */
 421                 vref_mode_dq = 0x1;
 422                 vref_value_dq = 0x1f;
 423                 vref_mode_ac = 0x1;
 424                 vref_value_ac = 0x1f;
 425                 mode_sel = 0x1;
 426         } else {
 427                 debug("Unknown DRAM type.\n");
 428                 return -EINVAL;
 429         }
 430
 431         reg_value = (vref_mode_dq << 9) | (0x1 << 8) | vref_value_dq;
 432
 433         /* PHY_913 PHY_PAD_VREF_CTRL_DQ_0 12bits offset_8 */
 434         clrsetbits_le32(&denali_phy[913], 0xfff << 8, reg_value << 8);
 435         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_1 12bits offset_0 */
 436         clrsetbits_le32(&denali_phy[914], 0xfff, reg_value);
 437         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_2 12bits offset_16 */
 438         clrsetbits_le32(&denali_phy[914], 0xfff << 16, reg_value << 16);
 439         /* PHY_915 PHY_PAD_VREF_CTRL_DQ_3 12bits offset_0 */
 440         clrsetbits_le32(&denali_phy[915], 0xfff, reg_value);
 441
 442         reg_value = (vref_mode_ac << 9) | (0x1 << 8) | vref_value_ac;
 443
 444         /* PHY_915 PHY_PAD_VREF_CTRL_AC 12bits offset_16 */
 445         clrsetbits_le32(&denali_phy[915], 0xfff << 16, reg_value << 16);
 446
 447         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 448         clrsetbits_le32(&denali_phy[924], 0x7 << 15, mode_sel << 15);
 449         /* PHY_926 PHY_PAD_DATA_DRIVE */
 450         clrsetbits_le32(&denali_phy[926], 0x7 << 6, mode_sel << 6);
 451         /* PHY_927 PHY_PAD_DQS_DRIVE */
 452         clrsetbits_le32(&denali_phy[927], 0x7 << 6, mode_sel << 6);
 453         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 454         clrsetbits_le32(&denali_phy[928], 0x7 << 14, mode_sel << 14);
 455         /* PHY_929 PHY_PAD_CLK_DRIVE */
 456         clrsetbits_le32(&denali_phy[929], 0x7 << 14, mode_sel << 14);
 457         /* PHY_935 PHY_PAD_CKE_DRIVE */
 458         clrsetbits_le32(&denali_phy[935], 0x7 << 14, mode_sel << 14);
 459         /* PHY_937 PHY_PAD_RST_DRIVE */
 460         clrsetbits_le32(&denali_phy[937], 0x7 << 14, mode_sel << 14);
 461         /* PHY_939 PHY_PAD_CS_DRIVE */
 462         clrsetbits_le32(&denali_phy[939], 0x7 << 14, mode_sel << 14);
 463
 464         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 465                 /* BOOSTP_EN & BOOSTN_EN */
 466                 reg_value = ((PHY_BOOSTP_EN << 4) | PHY_BOOSTN_EN);
 467                 /* PHY_925 PHY_PAD_FDBK_DRIVE2 */
 468                 clrsetbits_le32(&denali_phy[925], 0xff << 8, reg_value << 8);
 469                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 470                 clrsetbits_le32(&denali_phy[926], 0xff << 12, reg_value << 12);
 471                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 472                 clrsetbits_le32(&denali_phy[927], 0xff << 14, reg_value << 14);
 473                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 474                 clrsetbits_le32(&denali_phy[928], 0xff << 20, reg_value << 20);
 475                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 476                 clrsetbits_le32(&denali_phy[929], 0xff << 22, reg_value << 22);
 477                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 478                 clrsetbits_le32(&denali_phy[935], 0xff << 20, reg_value << 20);
 479                 /* PHY_937 PHY_PAD_RST_DRIVE */
 480                 clrsetbits_le32(&denali_phy[937], 0xff << 20, reg_value << 20);
 481                 /* PHY_939 PHY_PAD_CS_DRIVE */
 482                 clrsetbits_le32(&denali_phy[939], 0xff << 20, reg_value << 20);
 483
 484                 /* SLEWP_EN & SLEWN_EN */
 485                 reg_value = ((PHY_SLEWP_EN << 3) | PHY_SLEWN_EN);
 486                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 487                 clrsetbits_le32(&denali_phy[924], 0x3f << 8, reg_value << 8);
 488                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 489                 clrsetbits_le32(&denali_phy[926], 0x3f, reg_value);
 490                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 491                 clrsetbits_le32(&denali_phy[927], 0x3f, reg_value);
 492                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 493                 clrsetbits_le32(&denali_phy[928], 0x3f << 8, reg_value << 8);
 494                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 495                 clrsetbits_le32(&denali_phy[929], 0x3f << 8, reg_value << 8);
 496                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 497                 clrsetbits_le32(&denali_phy[935], 0x3f << 8, reg_value << 8);
 498                 /* PHY_937 PHY_PAD_RST_DRIVE */
 499                 clrsetbits_le32(&denali_phy[937], 0x3f << 8, reg_value << 8);
 500                 /* PHY_939 PHY_PAD_CS_DRIVE */
 501                 clrsetbits_le32(&denali_phy[939], 0x3f << 8, reg_value << 8);
 502         }
 503
 504         /* speed setting */
 505         if (params->base.ddr_freq < 400)
 506                 speed = 0x0;
 507         else if (params->base.ddr_freq < 800)
 508                 speed = 0x1;
 509         else if (params->base.ddr_freq < 1200)
 510                 speed = 0x2;
 511         else
 512                 speed = 0x3;
 513
 514         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 515         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
 516         /* PHY_926 PHY_PAD_DATA_DRIVE */
 517         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
 518         /* PHY_927 PHY_PAD_DQS_DRIVE */
 519         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
 520         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 521         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
 522         /* PHY_929 PHY_PAD_CLK_DRIVE */
 523         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
 524         /* PHY_935 PHY_PAD_CKE_DRIVE */
 525         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
 526         /* PHY_937 PHY_PAD_RST_DRIVE */
 527         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
 528         /* PHY_939 PHY_PAD_CS_DRIVE */
 529         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
 530
 531         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 532                 /* RX_CM_INPUT */
 533                 reg_value = PHY_RX_CM_INPUT;
 534                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 535                 clrsetbits_le32(&denali_phy[924], 0x1 << 14, reg_value << 14);
 536                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 537                 clrsetbits_le32(&denali_phy[926], 0x1 << 11, reg_value << 11);
 538                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 539                 clrsetbits_le32(&denali_phy[927], 0x1 << 13, reg_value << 13);
 540                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 541                 clrsetbits_le32(&denali_phy[928], 0x1 << 19, reg_value << 19);
 542                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 543                 clrsetbits_le32(&denali_phy[929], 0x1 << 21, reg_value << 21);
 544                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 545                 clrsetbits_le32(&denali_phy[935], 0x1 << 19, reg_value << 19);
 546                 /* PHY_937 PHY_PAD_RST_DRIVE */
 547                 clrsetbits_le32(&denali_phy[937], 0x1 << 19, reg_value << 19);
 548                 /* PHY_939 PHY_PAD_CS_DRIVE */
 549                 clrsetbits_le32(&denali_phy[939], 0x1 << 19, reg_value << 19);
 550         }
 551
 552         return 0;
 553 }
 554
 555 static void set_ds_odt(const struct chan_info *chan,
 556                        const struct rk3399_sdram_params *params, u32 mr5)
 557 {
 558         u32 *denali_phy = chan->publ->denali_phy;
 559         u32 *denali_ctl = chan->pctl->denali_ctl;
 560         u32 tsel_idle_en, tsel_wr_en, tsel_rd_en;
 561         u32 tsel_idle_select_p, tsel_rd_select_p;
 562         u32 tsel_idle_select_n, tsel_rd_select_n;
 563         u32 tsel_wr_select_dq_p, tsel_wr_select_ca_p;
 564         u32 tsel_wr_select_dq_n, tsel_wr_select_ca_n;
 565         u32 tsel_ckcs_select_p, tsel_ckcs_select_n;
 566         struct io_setting *io = NULL;
 567         u32 soc_odt = 0;
 568         u32 reg_value;
 569
 570         if (params->base.dramtype == LPDDR4) {
 571                 io = lpddr4_get_io_settings(params, mr5);
 572
 573                 tsel_rd_select_p = PHY_DRV_ODT_HI_Z;
 574                 tsel_rd_select_n = io->rd_odt;
 575
 576                 tsel_idle_select_p = PHY_DRV_ODT_HI_Z;
 577                 tsel_idle_select_n = PHY_DRV_ODT_240;
 578
 579                 tsel_wr_select_dq_p = io->wr_dq_drv;
 580                 tsel_wr_select_dq_n = PHY_DRV_ODT_40;
 581
 582                 tsel_wr_select_ca_p = io->wr_ca_drv;
 583                 tsel_wr_select_ca_n = PHY_DRV_ODT_40;
 584
 585                 tsel_ckcs_select_p = io->wr_ckcs_drv;
 586                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 587                 switch (tsel_rd_select_n) {
 588                 case PHY_DRV_ODT_240:
 589                         soc_odt = 1;
 590                         break;
 591                 case PHY_DRV_ODT_120:
 592                         soc_odt = 2;
 593                         break;
 594                 case PHY_DRV_ODT_80:
 595                         soc_odt = 3;
 596                         break;
 597                 case PHY_DRV_ODT_60:
 598                         soc_odt = 4;
 599                         break;
 600                 case PHY_DRV_ODT_48:
 601                         soc_odt = 5;
 602                         break;
 603                 case PHY_DRV_ODT_40:
 604                         soc_odt = 6;
 605                         break;
 606                 case PHY_DRV_ODT_34_3:
 607                         soc_odt = 6;
 608                         printf("%s: Unable to support LPDDR4 MR22 Soc ODT\n",
 609                                __func__);
 610                         break;
 611                 case PHY_DRV_ODT_HI_Z:
 612                 default:
 613                         soc_odt = 0;
 614                         break;
 615                 }
 616         } else if (params->base.dramtype == LPDDR3) {
 617                 tsel_rd_select_p = PHY_DRV_ODT_240;
 618                 tsel_rd_select_n = PHY_DRV_ODT_HI_Z;
 619
 620                 tsel_idle_select_p = PHY_DRV_ODT_240;
 621                 tsel_idle_select_n = PHY_DRV_ODT_HI_Z;
 622
 623                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 624                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 625
 626                 tsel_wr_select_ca_p = PHY_DRV_ODT_48;
 627                 tsel_wr_select_ca_n = PHY_DRV_ODT_48;
 628
 629                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 630                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 631         } else {
 632                 tsel_rd_select_p = PHY_DRV_ODT_240;
 633                 tsel_rd_select_n = PHY_DRV_ODT_240;
 634
 635                 tsel_idle_select_p = PHY_DRV_ODT_240;
 636                 tsel_idle_select_n = PHY_DRV_ODT_240;
 637
 638                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 639                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 640
 641                 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3;
 642                 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3;
 643
 644                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 645                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 646         }
 647
 648         if (params->base.odt == 1)
 649                 tsel_rd_en = 1;
 650         else
 651                 tsel_rd_en = 0;
 652
 653         tsel_wr_en = 0;
 654         tsel_idle_en = 0;
 655
 656         /* F0_0 */
 657         clrsetbits_le32(&denali_ctl[145], 0xFF << 16,
 658                         (soc_odt | (CS0_MR22_VAL << 3)) << 16);
 659         /* F2_0, F1_0 */
 660         clrsetbits_le32(&denali_ctl[146], 0xFF00FF,
 661                         ((soc_odt | (CS0_MR22_VAL << 3)) << 16) |
 662                         (soc_odt | (CS0_MR22_VAL << 3)));
 663         /* F0_1 */
 664         clrsetbits_le32(&denali_ctl[159], 0xFF << 16,
 665                         (soc_odt | (CS1_MR22_VAL << 3)) << 16);
 666         /* F2_1, F1_1 */
 667         clrsetbits_le32(&denali_ctl[160], 0xFF00FF,
 668                         ((soc_odt | (CS1_MR22_VAL << 3)) << 16) |
 669                         (soc_odt | (CS1_MR22_VAL << 3)));
 670
 671         /*
 672          * phy_dq_tsel_select_X 24bits DENALI_PHY_6/134/262/390 offset_0
 673          * sets termination values for read/idle cycles and drive strength
 674          * for write cycles for DQ/DM
 675          */
 676         reg_value = tsel_rd_select_n | (tsel_rd_select_p << 0x4) |
 677                     (tsel_wr_select_dq_n << 8) | (tsel_wr_select_dq_p << 12) |
 678                     (tsel_idle_select_n << 16) | (tsel_idle_select_p << 20);
 679         clrsetbits_le32(&denali_phy[6], 0xffffff, reg_value);
 680         clrsetbits_le32(&denali_phy[134], 0xffffff, reg_value);
 681         clrsetbits_le32(&denali_phy[262], 0xffffff, reg_value);
 682         clrsetbits_le32(&denali_phy[390], 0xffffff, reg_value);
 683
 684         /*
 685          * phy_dqs_tsel_select_X 24bits DENALI_PHY_7/135/263/391 offset_0
 686          * sets termination values for read/idle cycles and drive strength
 687          * for write cycles for DQS
 688          */
 689         clrsetbits_le32(&denali_phy[7], 0xffffff, reg_value);
 690         clrsetbits_le32(&denali_phy[135], 0xffffff, reg_value);
 691         clrsetbits_le32(&denali_phy[263], 0xffffff, reg_value);
 692         clrsetbits_le32(&denali_phy[391], 0xffffff, reg_value);
 693
 694         /* phy_adr_tsel_select_ 8bits DENALI_PHY_544/672/800 offset_0 */
 695         reg_value = tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 0x4);
 696         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 697                 /* LPDDR4 these register read always return 0, so
 698                  * can not use clrsetbits_le32(), need to write32
 699                  */
 700                 writel((0x300 << 8) | reg_value, &denali_phy[544]);
 701                 writel((0x300 << 8) | reg_value, &denali_phy[672]);
 702                 writel((0x300 << 8) | reg_value, &denali_phy[800]);
 703         } else {
 704                 clrsetbits_le32(&denali_phy[544], 0xff, reg_value);
 705                 clrsetbits_le32(&denali_phy[672], 0xff, reg_value);
 706                 clrsetbits_le32(&denali_phy[800], 0xff, reg_value);
 707         }
 708
 709         /* phy_pad_addr_drive 8bits DENALI_PHY_928 offset_0 */
 710         clrsetbits_le32(&denali_phy[928], 0xff, reg_value);
 711
 712         /* phy_pad_rst_drive 8bits DENALI_PHY_937 offset_0 */
 713         clrsetbits_le32(&denali_phy[937], 0xff, reg_value);
 714
 715         /* phy_pad_cke_drive 8bits DENALI_PHY_935 offset_0 */
 716         clrsetbits_le32(&denali_phy[935], 0xff, reg_value);
 717
 718         /* phy_pad_cs_drive 8bits DENALI_PHY_939 offset_0 */
 719         clrsetbits_le32(&denali_phy[939], 0xff,
 720                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 721
 722         /* phy_pad_clk_drive 8bits DENALI_PHY_929 offset_0 */
 723         clrsetbits_le32(&denali_phy[929], 0xff,
 724                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 725
 726         /* phy_pad_fdbk_drive 23bit DENALI_PHY_924/925 */
 727         clrsetbits_le32(&denali_phy[924], 0xff,
 728                         tsel_wr_select_dq_n | (tsel_wr_select_dq_p << 4));
 729         clrsetbits_le32(&denali_phy[925], 0xff,
 730                         tsel_rd_select_n | (tsel_rd_select_p << 4));
 731
 732         /* phy_dq_tsel_enable_X 3bits DENALI_PHY_5/133/261/389 offset_16 */
 733         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 734                 << 16;
 735         clrsetbits_le32(&denali_phy[5], 0x7 << 16, reg_value);
 736         clrsetbits_le32(&denali_phy[133], 0x7 << 16, reg_value);
 737         clrsetbits_le32(&denali_phy[261], 0x7 << 16, reg_value);
 738         clrsetbits_le32(&denali_phy[389], 0x7 << 16, reg_value);
 739
 740         /* phy_dqs_tsel_enable_X 3bits DENALI_PHY_6/134/262/390 offset_24 */
 741         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 742                 << 24;
 743         clrsetbits_le32(&denali_phy[6], 0x7 << 24, reg_value);
 744         clrsetbits_le32(&denali_phy[134], 0x7 << 24, reg_value);
 745         clrsetbits_le32(&denali_phy[262], 0x7 << 24, reg_value);
 746         clrsetbits_le32(&denali_phy[390], 0x7 << 24, reg_value);
 747
 748         /* phy_adr_tsel_enable_ 1bit DENALI_PHY_518/646/774 offset_8 */
 749         reg_value = tsel_wr_en << 8;
 750         clrsetbits_le32(&denali_phy[518], 0x1 << 8, reg_value);
 751         clrsetbits_le32(&denali_phy[646], 0x1 << 8, reg_value);
 752         clrsetbits_le32(&denali_phy[774], 0x1 << 8, reg_value);
 753
 754         /* phy_pad_addr_term tsel 1bit DENALI_PHY_933 offset_17 */
 755         reg_value = tsel_wr_en << 17;
 756         clrsetbits_le32(&denali_phy[933], 0x1 << 17, reg_value);
 757         /*
 758          * pad_rst/cke/cs/clk_term tsel 1bits
 759          * DENALI_PHY_938/936/940/934 offset_17
 760          */
 761         clrsetbits_le32(&denali_phy[938], 0x1 << 17, reg_value);
 762         clrsetbits_le32(&denali_phy[936], 0x1 << 17, reg_value);
 763         clrsetbits_le32(&denali_phy[940], 0x1 << 17, reg_value);
 764         clrsetbits_le32(&denali_phy[934], 0x1 << 17, reg_value);
 765
 766         /* phy_pad_fdbk_term 1bit DENALI_PHY_930 offset_17 */
 767         clrsetbits_le32(&denali_phy[930], 0x1 << 17, reg_value);
 768
 769         phy_io_config(chan, params);
 770 }
 771
 772 static void pctl_start(struct dram_info *dram, u8 channel)
 773 {
 774         const struct chan_info *chan = &dram->chan[channel];
 775         u32 *denali_ctl = chan->pctl->denali_ctl;
 776         u32 *denali_phy = chan->publ->denali_phy;
 777         u32 *ddrc0_con = get_ddrc0_con(dram, channel);
 778         u32 count = 0;
 779         u32 byte, tmp;
 780
 781         writel(0x01000000, &ddrc0_con);
 782
 783         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 0x2 << 24);
 784
 785         while (!(readl(&denali_ctl[203]) & (1 << 3))) {
 786                 if (count > 1000) {
 787                         printf("%s: Failed to init pctl for channel %d\n",
 788                                __func__, channel);
 789                         while (1)
 790                                 ;
 791                 }
 792
 793                 udelay(1);
 794                 count++;
 795         }
 796
 797         writel(0x01000100, &ddrc0_con);
 798
 799         for (byte = 0; byte < 4; byte++) {
 800                 tmp = 0x820;
 801                 writel((tmp << 16) | tmp, &denali_phy[53 + (128 * byte)]);
 802                 writel((tmp << 16) | tmp, &denali_phy[54 + (128 * byte)]);
 803                 writel((tmp << 16) | tmp, &denali_phy[55 + (128 * byte)]);
 804                 writel((tmp << 16) | tmp, &denali_phy[56 + (128 * byte)]);
 805                 writel((tmp << 16) | tmp, &denali_phy[57 + (128 * byte)]);
 806
 807                 clrsetbits_le32(&denali_phy[58 + (128 * byte)], 0xffff, tmp);
 808         }
 809
 810         clrsetbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT,
 811                         dram->pwrup_srefresh_exit[channel]);
 812 }
 813
 814 static int pctl_cfg(struct dram_info *dram, const struct chan_info *chan,
 815                     u32 channel, struct rk3399_sdram_params *params)
 816 {
 817         u32 *denali_ctl = chan->pctl->denali_ctl;
 818         u32 *denali_pi = chan->pi->denali_pi;
 819         u32 *denali_phy = chan->publ->denali_phy;
 820         const u32 *params_ctl = params->pctl_regs.denali_ctl;
 821         const u32 *params_phy = params->phy_regs.denali_phy;
 822         u32 tmp, tmp1, tmp2;
 823
 824         /*
 825          * work around controller bug:
 826          * Do not program DRAM_CLASS until NO_PHY_IND_TRAIN_INT is programmed
 827          */
 828         copy_to_reg(&denali_ctl[1], &params_ctl[1],
 829                     sizeof(struct rk3399_ddr_pctl_regs) - 4);
 830         writel(params_ctl[0], &denali_ctl[0]);
 831
 832         /*
 833          * two channel init at the same time, then ZQ Cal Start
 834          * at the same time, it will use the same RZQ, but cannot
 835          * start at the same time.
 836          *
 837          * So, increase tINIT3 for channel 1, will avoid two
 838          * channel ZQ Cal Start at the same time
 839          */
 840         if (params->base.dramtype == LPDDR4 && channel == 1) {
 841                 tmp = ((params->base.ddr_freq * MHz + 999) / 1000);
 842                 tmp1 = readl(&denali_ctl[14]);
 843                 writel(tmp + tmp1, &denali_ctl[14]);
 844         }
 845
 846         copy_to_reg(denali_pi, &params->pi_regs.denali_pi[0],
 847                     sizeof(struct rk3399_ddr_pi_regs));
 848
 849         /* rank count need to set for init */
 850         set_memory_map(chan, channel, params);
 851
 852         writel(params->phy_regs.denali_phy[910], &denali_phy[910]);
 853         writel(params->phy_regs.denali_phy[911], &denali_phy[911]);
 854         writel(params->phy_regs.denali_phy[912], &denali_phy[912]);
 855
 856         if (IS_ENABLED(CONFIG_RAM_RK3399_LPDDR4)) {
 857                 writel(params->phy_regs.denali_phy[898], &denali_phy[898]);
 858                 writel(params->phy_regs.denali_phy[919], &denali_phy[919]);
 859         }
 860
 861         dram->pwrup_srefresh_exit[channel] = readl(&denali_ctl[68]) &
 862                                              PWRUP_SREFRESH_EXIT;
 863         clrbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT);
 864
 865         /* PHY_DLL_RST_EN */
 866         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 1 << 24);
 867
 868         setbits_le32(&denali_pi[0], START);
 869         setbits_le32(&denali_ctl[0], START);
 870
 871         /**
 872          * LPDDR4 use PLL bypass mode for init
 873          * not need to wait for the PLL to lock
 874          */
 875         if (params->base.dramtype != LPDDR4) {
 876                 /* Waiting for phy DLL lock */
 877                 while (1) {
 878                         tmp = readl(&denali_phy[920]);
 879                         tmp1 = readl(&denali_phy[921]);
 880                         tmp2 = readl(&denali_phy[922]);
 881                         if ((((tmp >> 16) & 0x1) == 0x1) &&
 882                             (((tmp1 >> 16) & 0x1) == 0x1) &&
 883                             (((tmp1 >> 0) & 0x1) == 0x1) &&
 884                             (((tmp2 >> 0) & 0x1) == 0x1))
 885                                 break;
 886                 }
 887         }
 888
 889         copy_to_reg(&denali_phy[896], &params_phy[896], (958 - 895) * 4);
 890         copy_to_reg(&denali_phy[0], &params_phy[0], (90 - 0 + 1) * 4);
 891         copy_to_reg(&denali_phy[128], &params_phy[128], (218 - 128 + 1) * 4);
 892         copy_to_reg(&denali_phy[256], &params_phy[256], (346 - 256 + 1) * 4);
 893         copy_to_reg(&denali_phy[384], &params_phy[384], (474 - 384 + 1) * 4);
 894         copy_to_reg(&denali_phy[512], &params_phy[512], (549 - 512 + 1) * 4);
 895         copy_to_reg(&denali_phy[640], &params_phy[640], (677 - 640 + 1) * 4);
 896         copy_to_reg(&denali_phy[768], &params_phy[768], (805 - 768 + 1) * 4);
 897         set_ds_odt(chan, params, 0);
 898
 899         /*
 900          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_84/212/340/468 offset_8
 901          * dqs_tsel_wr_end[7:4] add Half cycle
 902          */
 903         tmp = (readl(&denali_phy[84]) >> 8) & 0xff;
 904         clrsetbits_le32(&denali_phy[84], 0xff << 8, (tmp + 0x10) << 8);
 905         tmp = (readl(&denali_phy[212]) >> 8) & 0xff;
 906         clrsetbits_le32(&denali_phy[212], 0xff << 8, (tmp + 0x10) << 8);
 907         tmp = (readl(&denali_phy[340]) >> 8) & 0xff;
 908         clrsetbits_le32(&denali_phy[340], 0xff << 8, (tmp + 0x10) << 8);
 909         tmp = (readl(&denali_phy[468]) >> 8) & 0xff;
 910         clrsetbits_le32(&denali_phy[468], 0xff << 8, (tmp + 0x10) << 8);
 911
 912         /*
 913          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_83/211/339/467 offset_8
 914          * dq_tsel_wr_end[7:4] add Half cycle
 915          */
 916         tmp = (readl(&denali_phy[83]) >> 16) & 0xff;
 917         clrsetbits_le32(&denali_phy[83], 0xff << 16, (tmp + 0x10) << 16);
 918         tmp = (readl(&denali_phy[211]) >> 16) & 0xff;
 919         clrsetbits_le32(&denali_phy[211], 0xff << 16, (tmp + 0x10) << 16);
 920         tmp = (readl(&denali_phy[339]) >> 16) & 0xff;
 921         clrsetbits_le32(&denali_phy[339], 0xff << 16, (tmp + 0x10) << 16);
 922         tmp = (readl(&denali_phy[467]) >> 16) & 0xff;
 923         clrsetbits_le32(&denali_phy[467], 0xff << 16, (tmp + 0x10) << 16);
 924
 925         return 0;
 926 }
 927
 928 static void select_per_cs_training_index(const struct chan_info *chan,
 929                                          u32 rank)
 930 {
 931         u32 *denali_phy = chan->publ->denali_phy;
 932
 933         /* PHY_84 PHY_PER_CS_TRAINING_EN_0 1bit offset_16 */
 934         if ((readl(&denali_phy[84]) >> 16) & 1) {
 935                 /*
 936                  * PHY_8/136/264/392
 937                  * phy_per_cs_training_index_X 1bit offset_24
 938                  */
 939                 clrsetbits_le32(&denali_phy[8], 0x1 << 24, rank << 24);
 940                 clrsetbits_le32(&denali_phy[136], 0x1 << 24, rank << 24);
 941                 clrsetbits_le32(&denali_phy[264], 0x1 << 24, rank << 24);
 942                 clrsetbits_le32(&denali_phy[392], 0x1 << 24, rank << 24);
 943         }
 944 }
 945
 946 static void override_write_leveling_value(const struct chan_info *chan)
 947 {
 948         u32 *denali_ctl = chan->pctl->denali_ctl;
 949         u32 *denali_phy = chan->publ->denali_phy;
 950         u32 byte;
 951
 952         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
 953         setbits_le32(&denali_phy[896], 1);
 954
 955         /*
 956          * PHY_8/136/264/392
 957          * phy_per_cs_training_multicast_en_X 1bit offset_16
 958          */
 959         clrsetbits_le32(&denali_phy[8], 0x1 << 16, 1 << 16);
 960         clrsetbits_le32(&denali_phy[136], 0x1 << 16, 1 << 16);
 961         clrsetbits_le32(&denali_phy[264], 0x1 << 16, 1 << 16);
 962         clrsetbits_le32(&denali_phy[392], 0x1 << 16, 1 << 16);
 963
 964         for (byte = 0; byte < 4; byte++)
 965                 clrsetbits_le32(&denali_phy[63 + (128 * byte)], 0xffff << 16,
 966                                 0x200 << 16);
 967
 968         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
 969         clrbits_le32(&denali_phy[896], 1);
 970
 971         /* CTL_200 ctrlupd_req 1bit offset_8 */
 972         clrsetbits_le32(&denali_ctl[200], 0x1 << 8, 0x1 << 8);
 973 }
 974
 975 static int data_training_ca(const struct chan_info *chan, u32 channel,
 976                             const struct rk3399_sdram_params *params)
 977 {
 978         u32 *denali_pi = chan->pi->denali_pi;
 979         u32 *denali_phy = chan->publ->denali_phy;
 980         u32 i, tmp;
 981         u32 obs_0, obs_1, obs_2, obs_err = 0;
 982         u32 rank = params->ch[channel].cap_info.rank;
 983         u32 rank_mask;
 984
 985         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 986         writel(0x00003f7c, (&denali_pi[175]));
 987
 988         if (params->base.dramtype == LPDDR4)
 989                 rank_mask = (rank == 1) ? 0x5 : 0xf;
 990         else
 991                 rank_mask = (rank == 1) ? 0x1 : 0x3;
 992
 993         for (i = 0; i < 4; i++) {
 994                 if (!(rank_mask & (1 << i)))
 995                         continue;
 996
 997                 select_per_cs_training_index(chan, i);
 998
 999                 /* PI_100 PI_CALVL_EN:RW:8:2 */
1000                 clrsetbits_le32(&denali_pi[100], 0x3 << 8, 0x2 << 8);
1001
1002                 /* PI_92 PI_CALVL_REQ:WR:16:1,PI_CALVL_CS:RW:24:2 */
1003                 clrsetbits_le32(&denali_pi[92],
1004                                 (0x1 << 16) | (0x3 << 24),
1005                                 (0x1 << 16) | (i << 24));
1006
1007                 /* Waiting for training complete */
1008                 while (1) {
1009                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1010                         tmp = readl(&denali_pi[174]) >> 8;
1011                         /*
1012                          * check status obs
1013                          * PHY_532/660/789 phy_adr_calvl_obs1_:0:32
1014                          */
1015                         obs_0 = readl(&denali_phy[532]);
1016                         obs_1 = readl(&denali_phy[660]);
1017                         obs_2 = readl(&denali_phy[788]);
1018                         if (((obs_0 >> 30) & 0x3) ||
1019                             ((obs_1 >> 30) & 0x3) ||
1020                             ((obs_2 >> 30) & 0x3))
1021                                 obs_err = 1;
1022                         if ((((tmp >> 11) & 0x1) == 0x1) &&
1023                             (((tmp >> 13) & 0x1) == 0x1) &&
1024                             (((tmp >> 5) & 0x1) == 0x0) &&
1025                             obs_err == 0)
1026                                 break;
1027                         else if ((((tmp >> 5) & 0x1) == 0x1) ||
1028                                  (obs_err == 1))
1029                                 return -EIO;
1030                 }
1031
1032                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1033                 writel(0x00003f7c, (&denali_pi[175]));
1034         }
1035
1036         clrbits_le32(&denali_pi[100], 0x3 << 8);
1037
1038         return 0;
1039 }
1040
1041 static int data_training_wl(const struct chan_info *chan, u32 channel,
1042                             const struct rk3399_sdram_params *params)
1043 {
1044         u32 *denali_pi = chan->pi->denali_pi;
1045         u32 *denali_phy = chan->publ->denali_phy;
1046         u32 i, tmp;
1047         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1048         u32 rank = params->ch[channel].cap_info.rank;
1049
1050         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1051         writel(0x00003f7c, (&denali_pi[175]));
1052
1053         for (i = 0; i < rank; i++) {
1054                 select_per_cs_training_index(chan, i);
1055
1056                 /* PI_60 PI_WRLVL_EN:RW:8:2 */
1057                 clrsetbits_le32(&denali_pi[60], 0x3 << 8, 0x2 << 8);
1058
1059                 /* PI_59 PI_WRLVL_REQ:WR:8:1,PI_WRLVL_CS:RW:16:2 */
1060                 clrsetbits_le32(&denali_pi[59],
1061                                 (0x1 << 8) | (0x3 << 16),
1062                                 (0x1 << 8) | (i << 16));
1063
1064                 /* Waiting for training complete */
1065                 while (1) {
1066                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1067                         tmp = readl(&denali_pi[174]) >> 8;
1068
1069                         /*
1070                          * check status obs, if error maybe can not
1071                          * get leveling done PHY_40/168/296/424
1072                          * phy_wrlvl_status_obs_X:0:13
1073                          */
1074                         obs_0 = readl(&denali_phy[40]);
1075                         obs_1 = readl(&denali_phy[168]);
1076                         obs_2 = readl(&denali_phy[296]);
1077                         obs_3 = readl(&denali_phy[424]);
1078                         if (((obs_0 >> 12) & 0x1) ||
1079                             ((obs_1 >> 12) & 0x1) ||
1080                             ((obs_2 >> 12) & 0x1) ||
1081                             ((obs_3 >> 12) & 0x1))
1082                                 obs_err = 1;
1083                         if ((((tmp >> 10) & 0x1) == 0x1) &&
1084                             (((tmp >> 13) & 0x1) == 0x1) &&
1085                             (((tmp >> 4) & 0x1) == 0x0) &&
1086                             obs_err == 0)
1087                                 break;
1088                         else if ((((tmp >> 4) & 0x1) == 0x1) ||
1089                                  (obs_err == 1))
1090                                 return -EIO;
1091                 }
1092
1093                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1094                 writel(0x00003f7c, (&denali_pi[175]));
1095         }
1096
1097         override_write_leveling_value(chan);
1098         clrbits_le32(&denali_pi[60], 0x3 << 8);
1099
1100         return 0;
1101 }
1102
1103 static int data_training_rg(const struct chan_info *chan, u32 channel,
1104                             const struct rk3399_sdram_params *params)
1105 {
1106         u32 *denali_pi = chan->pi->denali_pi;
1107         u32 *denali_phy = chan->publ->denali_phy;
1108         u32 i, tmp;
1109         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1110         u32 rank = params->ch[channel].cap_info.rank;
1111
1112         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1113         writel(0x00003f7c, (&denali_pi[175]));
1114
1115         for (i = 0; i < rank; i++) {
1116                 select_per_cs_training_index(chan, i);
1117
1118                 /* PI_80 PI_RDLVL_GATE_EN:RW:24:2 */
1119                 clrsetbits_le32(&denali_pi[80], 0x3 << 24, 0x2 << 24);
1120
1121                 /*
1122                  * PI_74 PI_RDLVL_GATE_REQ:WR:16:1
1123                  * PI_RDLVL_CS:RW:24:2
1124                  */
1125                 clrsetbits_le32(&denali_pi[74],
1126                                 (0x1 << 16) | (0x3 << 24),
1127                                 (0x1 << 16) | (i << 24));
1128
1129                 /* Waiting for training complete */
1130                 while (1) {
1131                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1132                         tmp = readl(&denali_pi[174]) >> 8;
1133
1134                         /*
1135                          * check status obs
1136                          * PHY_43/171/299/427
1137                          *     PHY_GTLVL_STATUS_OBS_x:16:8
1138                          */
1139                         obs_0 = readl(&denali_phy[43]);
1140                         obs_1 = readl(&denali_phy[171]);
1141                         obs_2 = readl(&denali_phy[299]);
1142                         obs_3 = readl(&denali_phy[427]);
1143                         if (((obs_0 >> (16 + 6)) & 0x3) ||
1144                             ((obs_1 >> (16 + 6)) & 0x3) ||
1145                             ((obs_2 >> (16 + 6)) & 0x3) ||
1146                             ((obs_3 >> (16 + 6)) & 0x3))
1147                                 obs_err = 1;
1148                         if ((((tmp >> 9) & 0x1) == 0x1) &&
1149                             (((tmp >> 13) & 0x1) == 0x1) &&
1150                             (((tmp >> 3) & 0x1) == 0x0) &&
1151                             obs_err == 0)
1152                                 break;
1153                         else if ((((tmp >> 3) & 0x1) == 0x1) ||
1154                                  (obs_err == 1))
1155                                 return -EIO;
1156                 }
1157
1158                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1159                 writel(0x00003f7c, (&denali_pi[175]));
1160         }
1161
1162         clrbits_le32(&denali_pi[80], 0x3 << 24);
1163
1164         return 0;
1165 }
1166
1167 static int data_training_rl(const struct chan_info *chan, u32 channel,
1168                             const struct rk3399_sdram_params *params)
1169 {
1170         u32 *denali_pi = chan->pi->denali_pi;
1171         u32 i, tmp;
1172         u32 rank = params->ch[channel].cap_info.rank;
1173
1174         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1175         writel(0x00003f7c, (&denali_pi[175]));
1176
1177         for (i = 0; i < rank; i++) {
1178                 select_per_cs_training_index(chan, i);
1179
1180                 /* PI_80 PI_RDLVL_EN:RW:16:2 */
1181                 clrsetbits_le32(&denali_pi[80], 0x3 << 16, 0x2 << 16);
1182
1183                 /* PI_74 PI_RDLVL_REQ:WR:8:1,PI_RDLVL_CS:RW:24:2 */
1184                 clrsetbits_le32(&denali_pi[74],
1185                                 (0x1 << 8) | (0x3 << 24),
1186                                 (0x1 << 8) | (i << 24));
1187
1188                 /* Waiting for training complete */
1189                 while (1) {
1190                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1191                         tmp = readl(&denali_pi[174]) >> 8;
1192
1193                         /*
1194                          * make sure status obs not report error bit
1195                          * PHY_46/174/302/430
1196                          *     phy_rdlvl_status_obs_X:16:8
1197                          */
1198                         if ((((tmp >> 8) & 0x1) == 0x1) &&
1199                             (((tmp >> 13) & 0x1) == 0x1) &&
1200                             (((tmp >> 2) & 0x1) == 0x0))
1201                                 break;
1202                         else if (((tmp >> 2) & 0x1) == 0x1)
1203                                 return -EIO;
1204                 }
1205
1206                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1207                 writel(0x00003f7c, (&denali_pi[175]));
1208         }
1209
1210         clrbits_le32(&denali_pi[80], 0x3 << 16);
1211
1212         return 0;
1213 }
1214
1215 static int data_training_wdql(const struct chan_info *chan, u32 channel,
1216                               const struct rk3399_sdram_params *params)
1217 {
1218         u32 *denali_pi = chan->pi->denali_pi;
1219         u32 i, tmp;
1220         u32 rank = params->ch[channel].cap_info.rank;
1221         u32 rank_mask;
1222
1223         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1224         writel(0x00003f7c, (&denali_pi[175]));
1225
1226         if (params->base.dramtype == LPDDR4)
1227                 rank_mask = (rank == 1) ? 0x5 : 0xf;
1228         else
1229                 rank_mask = (rank == 1) ? 0x1 : 0x3;
1230
1231         for (i = 0; i < 4; i++) {
1232                 if (!(rank_mask & (1 << i)))
1233                         continue;
1234
1235                 select_per_cs_training_index(chan, i);
1236
1237                 /*
1238                  * disable PI_WDQLVL_VREF_EN before wdq leveling?
1239                  * PI_181 PI_WDQLVL_VREF_EN:RW:8:1
1240                  */
1241                 clrbits_le32(&denali_pi[181], 0x1 << 8);
1242
1243                 /* PI_124 PI_WDQLVL_EN:RW:16:2 */
1244                 clrsetbits_le32(&denali_pi[124], 0x3 << 16, 0x2 << 16);
1245
1246                 /* PI_121 PI_WDQLVL_REQ:WR:8:1,PI_WDQLVL_CS:RW:16:2 */
1247                 clrsetbits_le32(&denali_pi[121],
1248                                 (0x1 << 8) | (0x3 << 16),
1249                                 (0x1 << 8) | (i << 16));
1250
1251                 /* Waiting for training complete */
1252                 while (1) {
1253                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1254                         tmp = readl(&denali_pi[174]) >> 8;
1255                         if ((((tmp >> 12) & 0x1) == 0x1) &&
1256                             (((tmp >> 13) & 0x1) == 0x1) &&
1257                             (((tmp >> 6) & 0x1) == 0x0))
1258                                 break;
1259                         else if (((tmp >> 6) & 0x1) == 0x1)
1260                                 return -EIO;
1261                 }
1262
1263                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1264                 writel(0x00003f7c, (&denali_pi[175]));
1265         }
1266
1267         clrbits_le32(&denali_pi[124], 0x3 << 16);
1268
1269         return 0;
1270 }
1271
1272 static int data_training(const struct chan_info *chan, u32 channel,
1273                          const struct rk3399_sdram_params *params,
1274                          u32 training_flag)
1275 {
1276         u32 *denali_phy = chan->publ->denali_phy;
1277         int ret;
1278
1279         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1280         setbits_le32(&denali_phy[927], (1 << 22));
1281
1282         if (training_flag == PI_FULL_TRAINING) {
1283                 if (params->base.dramtype == LPDDR4) {
1284                         training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING |
1285                                         PI_READ_GATE_TRAINING |
1286                                         PI_READ_LEVELING | PI_WDQ_LEVELING;
1287                 } else if (params->base.dramtype == LPDDR3) {
1288                         training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING |
1289                                         PI_READ_GATE_TRAINING;
1290                 } else if (params->base.dramtype == DDR3) {
1291                         training_flag = PI_WRITE_LEVELING |
1292                                         PI_READ_GATE_TRAINING |
1293                                         PI_READ_LEVELING;
1294                 }
1295         }
1296
1297         /* ca training(LPDDR4,LPDDR3 support) */
1298         if ((training_flag & PI_CA_TRAINING) == PI_CA_TRAINING) {
1299                 ret = data_training_ca(chan, channel, params);
1300                 if (ret < 0) {
1301                         debug("%s: data training ca failed\n", __func__);
1302                         return ret;
1303                 }
1304         }
1305
1306         /* write leveling(LPDDR4,LPDDR3,DDR3 support) */
1307         if ((training_flag & PI_WRITE_LEVELING) == PI_WRITE_LEVELING) {
1308                 ret = data_training_wl(chan, channel, params);
1309                 if (ret < 0) {
1310                         debug("%s: data training wl failed\n", __func__);
1311                         return ret;
1312                 }
1313         }
1314
1315         /* read gate training(LPDDR4,LPDDR3,DDR3 support) */
1316         if ((training_flag & PI_READ_GATE_TRAINING) == PI_READ_GATE_TRAINING) {
1317                 ret = data_training_rg(chan, channel, params);
1318                 if (ret < 0) {
1319                         debug("%s: data training rg failed\n", __func__);
1320                         return ret;
1321                 }
1322         }
1323
1324         /* read leveling(LPDDR4,LPDDR3,DDR3 support) */
1325         if ((training_flag & PI_READ_LEVELING) == PI_READ_LEVELING) {
1326                 ret = data_training_rl(chan, channel, params);
1327                 if (ret < 0) {
1328                         debug("%s: data training rl failed\n", __func__);
1329                         return ret;
1330                 }
1331         }
1332
1333         /* wdq leveling(LPDDR4 support) */
1334         if ((training_flag & PI_WDQ_LEVELING) == PI_WDQ_LEVELING) {
1335                 ret = data_training_wdql(chan, channel, params);
1336                 if (ret < 0) {
1337                         debug("%s: data training wdql failed\n", __func__);
1338                         return ret;
1339                 }
1340         }
1341
1342         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1343         clrbits_le32(&denali_phy[927], (1 << 22));
1344
1345         return 0;
1346 }
1347
1348 static void set_ddrconfig(const struct chan_info *chan,
1349                           const struct rk3399_sdram_params *params,
1350                           unsigned char channel, u32 ddrconfig)
1351 {
1352         /* only need to set ddrconfig */
1353         struct rk3399_msch_regs *ddr_msch_regs = chan->msch;
1354         unsigned int cs0_cap = 0;
1355         unsigned int cs1_cap = 0;
1356
1357         cs0_cap = (1 << (params->ch[channel].cap_info.cs0_row
1358                         + params->ch[channel].cap_info.col
1359                         + params->ch[channel].cap_info.bk
1360                         + params->ch[channel].cap_info.bw - 20));
1361         if (params->ch[channel].cap_info.rank > 1)
1362                 cs1_cap = cs0_cap >> (params->ch[channel].cap_info.cs0_row
1363                                 - params->ch[channel].cap_info.cs1_row);
1364         if (params->ch[channel].cap_info.row_3_4) {
1365                 cs0_cap = cs0_cap * 3 / 4;
1366                 cs1_cap = cs1_cap * 3 / 4;
1367         }
1368
1369         writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf);
1370         writel(((cs0_cap / 32) & 0xff) | (((cs1_cap / 32) & 0xff) << 8),
1371                &ddr_msch_regs->ddrsize);
1372 }
1373
1374 static void dram_all_config(struct dram_info *dram,
1375                             const struct rk3399_sdram_params *params)
1376 {
1377         u32 sys_reg2 = 0;
1378         u32 sys_reg3 = 0;
1379         unsigned int channel, idx;
1380
1381         sys_reg2 |= SYS_REG_ENC_DDRTYPE(params->base.dramtype);
1382         sys_reg2 |= SYS_REG_ENC_NUM_CH(params->base.num_channels);
1383
1384         for (channel = 0, idx = 0;
1385              (idx < params->base.num_channels) && (channel < 2);
1386              channel++) {
1387                 const struct rk3399_sdram_channel *info = &params->ch[channel];
1388                 struct rk3399_msch_regs *ddr_msch_regs;
1389                 const struct rk3399_msch_timings *noc_timing;
1390
1391                 if (params->ch[channel].cap_info.col == 0)
1392                         continue;
1393                 idx++;
1394                 sys_reg2 |= SYS_REG_ENC_ROW_3_4(info->cap_info.row_3_4, channel);
1395                 sys_reg2 |= SYS_REG_ENC_CHINFO(channel);
1396                 sys_reg2 |= SYS_REG_ENC_RANK(info->cap_info.rank, channel);
1397                 sys_reg2 |= SYS_REG_ENC_COL(info->cap_info.col, channel);
1398                 sys_reg2 |= SYS_REG_ENC_BK(info->cap_info.bk, channel);
1399                 sys_reg2 |= SYS_REG_ENC_BW(info->cap_info.bw, channel);
1400                 sys_reg2 |= SYS_REG_ENC_DBW(info->cap_info.dbw, channel);
1401                 SYS_REG_ENC_CS0_ROW(info->cap_info.cs0_row, sys_reg2, sys_reg3, channel);
1402                 if (info->cap_info.cs1_row)
1403                         SYS_REG_ENC_CS1_ROW(info->cap_info.cs1_row, sys_reg2,
1404                                             sys_reg3, channel);
1405                 sys_reg3 |= SYS_REG_ENC_CS1_COL(info->cap_info.col, channel);
1406                 sys_reg3 |= SYS_REG_ENC_VERSION(DDR_SYS_REG_VERSION);
1407
1408                 ddr_msch_regs = dram->chan[channel].msch;
1409                 noc_timing = &params->ch[channel].noc_timings;
1410                 writel(noc_timing->ddrtiminga0,
1411                        &ddr_msch_regs->ddrtiminga0);
1412                 writel(noc_timing->ddrtimingb0,
1413                        &ddr_msch_regs->ddrtimingb0);
1414                 writel(noc_timing->ddrtimingc0.d32,
1415                        &ddr_msch_regs->ddrtimingc0);
1416                 writel(noc_timing->devtodev0,
1417                        &ddr_msch_regs->devtodev0);
1418                 writel(noc_timing->ddrmode.d32,
1419                        &ddr_msch_regs->ddrmode);
1420
1421                 /**
1422                  * rank 1 memory clock disable (dfi_dram_clk_disable = 1)
1423                  *
1424                  * The hardware for LPDDR4 with
1425                  * - CLK0P/N connect to lower 16-bits
1426                  * - CLK1P/N connect to higher 16-bits
1427                  *
1428                  * dfi dram clk is configured via CLK1P/N, so disabling
1429                  * dfi dram clk will disable the CLK1P/N as well for lpddr4.
1430                  */
1431                 if (params->ch[channel].cap_info.rank == 1 &&
1432                     params->base.dramtype != LPDDR4)
1433                         setbits_le32(&dram->chan[channel].pctl->denali_ctl[276],
1434                                      1 << 17);
1435         }
1436
1437         writel(sys_reg2, &dram->pmugrf->os_reg2);
1438         writel(sys_reg3, &dram->pmugrf->os_reg3);
1439         rk_clrsetreg(&dram->pmusgrf->soc_con4, 0x1f << 10,
1440                      params->base.stride << 10);
1441
1442         /* reboot hold register set */
1443         writel(PRESET_SGRF_HOLD(0) | PRESET_GPIO0_HOLD(1) |
1444                 PRESET_GPIO1_HOLD(1),
1445                 &dram->pmucru->pmucru_rstnhold_con[1]);
1446         clrsetbits_le32(&dram->cru->glb_rst_con, 0x3, 0x3);
1447 }
1448
1449 static int switch_to_phy_index1(struct dram_info *dram,
1450                                 const struct rk3399_sdram_params *params)
1451 {
1452         u32 channel;
1453         u32 *denali_phy;
1454         u32 ch_count = params->base.num_channels;
1455         int ret;
1456         int i = 0;
1457
1458         writel(RK_CLRSETBITS(0x03 << 4 | 1 << 2 | 1,
1459                              1 << 4 | 1 << 2 | 1),
1460                         &dram->cic->cic_ctrl0);
1461         while (!(readl(&dram->cic->cic_status0) & (1 << 2))) {
1462                 mdelay(10);
1463                 i++;
1464                 if (i > 10) {
1465                         debug("index1 frequency change overtime\n");
1466                         return -ETIME;
1467                 }
1468         }
1469
1470         i = 0;
1471         writel(RK_CLRSETBITS(1 << 1, 1 << 1), &dram->cic->cic_ctrl0);
1472         while (!(readl(&dram->cic->cic_status0) & (1 << 0))) {
1473                 mdelay(10);
1474                 i++;
1475                 if (i > 10) {
1476                         debug("index1 frequency done overtime\n");
1477                         return -ETIME;
1478                 }
1479         }
1480
1481         for (channel = 0; channel < ch_count; channel++) {
1482                 denali_phy = dram->chan[channel].publ->denali_phy;
1483                 clrsetbits_le32(&denali_phy[896], (0x3 << 8) | 1, 1 << 8);
1484                 ret = data_training(&dram->chan[channel], channel,
1485                                     params, PI_FULL_TRAINING);
1486                 if (ret < 0) {
1487                         debug("index1 training failed\n");
1488                         return ret;
1489                 }
1490         }
1491
1492         return 0;
1493 }
1494
1495 static unsigned char calculate_stride(struct rk3399_sdram_params *params)
1496 {
1497         unsigned int stride = params->base.stride;
1498         unsigned int channel, chinfo = 0;
1499         unsigned int ch_cap[2] = {0, 0};
1500         u64 cap;
1501
1502         for (channel = 0; channel < 2; channel++) {
1503                 unsigned int cs0_cap = 0;
1504                 unsigned int cs1_cap = 0;
1505                 struct sdram_cap_info *cap_info = &params->ch[channel].cap_info;
1506
1507                 if (cap_info->col == 0)
1508                         continue;
1509
1510                 cs0_cap = (1 << (cap_info->cs0_row + cap_info->col +
1511                                  cap_info->bk + cap_info->bw - 20));
1512                 if (cap_info->rank > 1)
1513                         cs1_cap = cs0_cap >> (cap_info->cs0_row
1514                                               - cap_info->cs1_row);
1515                 if (cap_info->row_3_4) {
1516                         cs0_cap = cs0_cap * 3 / 4;
1517                         cs1_cap = cs1_cap * 3 / 4;
1518                 }
1519                 ch_cap[channel] = cs0_cap + cs1_cap;
1520                 chinfo |= 1 << channel;
1521         }
1522
1523         /* stride calculation for 1 channel */
1524         if (params->base.num_channels == 1 && chinfo & 1)
1525                 return 0x17;    /* channel a */
1526
1527         /* stride calculation for 2 channels, default gstride type is 256B */
1528         if (ch_cap[0] == ch_cap[1]) {
1529                 cap = ch_cap[0] + ch_cap[1];
1530                 switch (cap) {
1531                 /* 512MB */
1532                 case 512:
1533                         stride = 0;
1534                         break;
1535                 /* 1GB */
1536                 case 1024:
1537                         stride = 0x5;
1538                         break;
1539                 /*
1540                  * 768MB + 768MB same as total 2GB memory
1541                  * useful space: 0-768MB 1GB-1792MB
1542                  */
1543                 case 1536:
1544                 /* 2GB */
1545                 case 2048:
1546                         stride = 0x9;
1547                         break;
1548                 /* 1536MB + 1536MB */
1549                 case 3072:
1550                         stride = 0x11;
1551                         break;
1552                 /* 4GB */
1553                 case 4096:
1554                         stride = 0xD;
1555                         break;
1556                 default:
1557                         printf("%s: Unable to calculate stride for ", __func__);
1558                         print_size((cap * (1 << 20)), " capacity\n");
1559                         break;
1560                 }
1561         }
1562
1563         sdram_print_stride(stride);
1564
1565         return stride;
1566 }
1567
1568 static void clear_channel_params(struct rk3399_sdram_params *params, u8 channel)
1569 {
1570         params->ch[channel].cap_info.rank = 0;
1571         params->ch[channel].cap_info.col = 0;
1572         params->ch[channel].cap_info.bk = 0;
1573         params->ch[channel].cap_info.bw = 32;
1574         params->ch[channel].cap_info.dbw = 32;
1575         params->ch[channel].cap_info.row_3_4 = 0;
1576         params->ch[channel].cap_info.cs0_row = 0;
1577         params->ch[channel].cap_info.cs1_row = 0;
1578         params->ch[channel].cap_info.ddrconfig = 0;
1579 }
1580
1581 static int pctl_init(struct dram_info *dram, struct rk3399_sdram_params *params)
1582 {
1583         int channel;
1584         int ret;
1585
1586         for (channel = 0; channel < 2; channel++) {
1587                 const struct chan_info *chan = &dram->chan[channel];
1588                 struct rk3399_cru *cru = dram->cru;
1589                 struct rk3399_ddr_publ_regs *publ = chan->publ;
1590
1591                 phy_pctrl_reset(cru, channel);
1592                 phy_dll_bypass_set(publ, params->base.ddr_freq);
1593
1594                 ret = pctl_cfg(dram, chan, channel, params);
1595                 if (ret < 0) {
1596                         printf("%s: pctl config failed\n", __func__);
1597                         return ret;
1598                 }
1599
1600                 /* start to trigger initialization */
1601                 pctl_start(dram, channel);
1602         }
1603
1604         return 0;
1605 }
1606
1607 static int sdram_init(struct dram_info *dram,
1608                       struct rk3399_sdram_params *params)
1609 {
1610         unsigned char dramtype = params->base.dramtype;
1611         unsigned int ddr_freq = params->base.ddr_freq;
1612         u32 training_flag = PI_READ_GATE_TRAINING;
1613         int channel, ch, rank;
1614         int ret;
1615
1616         debug("Starting SDRAM initialization...\n");
1617
1618         if ((dramtype == DDR3 && ddr_freq > 933) ||
1619             (dramtype == LPDDR3 && ddr_freq > 933) ||
1620             (dramtype == LPDDR4 && ddr_freq > 800)) {
1621                 debug("SDRAM frequency is to high!");
1622                 return -E2BIG;
1623         }
1624
1625         for (ch = 0; ch < 2; ch++) {
1626                 params->ch[ch].cap_info.rank = 2;
1627                 for (rank = 2; rank != 0; rank--) {
1628                         ret = pctl_init(dram, params);
1629                         if (ret < 0) {
1630                                 printf("%s: pctl init failed\n", __func__);
1631                                 return ret;
1632                         }
1633
1634                         /* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */
1635                         if (dramtype == LPDDR3)
1636                                 udelay(10);
1637
1638                         params->ch[ch].cap_info.rank = rank;
1639
1640                         /*
1641                          * LPDDR3 CA training msut be trigger before
1642                          * other training.
1643                          * DDR3 is not have CA training.
1644                          */
1645                         if (params->base.dramtype == LPDDR3)
1646                                 training_flag |= PI_CA_TRAINING;
1647
1648                         if (!(data_training(&dram->chan[ch], ch,
1649                                             params, training_flag)))
1650                                 break;
1651                 }
1652                 /* Computed rank with associated channel number */
1653                 params->ch[ch].cap_info.rank = rank;
1654         }
1655
1656         params->base.num_channels = 0;
1657         for (channel = 0; channel < 2; channel++) {
1658                 const struct chan_info *chan = &dram->chan[channel];
1659                 struct sdram_cap_info *cap_info = &params->ch[channel].cap_info;
1660                 u8 training_flag = PI_FULL_TRAINING;
1661
1662                 if (cap_info->rank == 0) {
1663                         clear_channel_params(params, channel);
1664                         continue;
1665                 } else {
1666                         params->base.num_channels++;
1667                 }
1668
1669                 debug("Channel ");
1670                 debug(channel ? "1: " : "0: ");
1671
1672                 /* LPDDR3 should have write and read gate training */
1673                 if (params->base.dramtype == LPDDR3)
1674                         training_flag = PI_WRITE_LEVELING |
1675                                         PI_READ_GATE_TRAINING;
1676
1677                 if (params->base.dramtype != LPDDR4) {
1678                         ret = data_training(dram, channel, params,
1679                                             training_flag);
1680                         if (!ret) {
1681                                 debug("%s: data train failed for channel %d\n",
1682                                       __func__, ret);
1683                                 continue;
1684                         }
1685                 }
1686
1687                 sdram_print_ddr_info(cap_info, &params->base);
1688
1689                 set_ddrconfig(chan, params, channel, cap_info->ddrconfig);
1690         }
1691
1692         if (params->base.num_channels == 0) {
1693                 printf("%s: ", __func__);
1694                 sdram_print_dram_type(params->base.dramtype);
1695                 printf(" - %dMHz failed!\n", params->base.ddr_freq);
1696                 return -EINVAL;
1697         }
1698
1699         params->base.stride = calculate_stride(params);
1700         dram_all_config(dram, params);
1701         switch_to_phy_index1(dram, params);
1702
1703         debug("Finish SDRAM initialization...\n");
1704         return 0;
1705 }
1706
1707 static int rk3399_dmc_ofdata_to_platdata(struct udevice *dev)
1708 {
1709 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
1710         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
1711         int ret;
1712
1713         ret = dev_read_u32_array(dev, "rockchip,sdram-params",
1714                                  (u32 *)&plat->sdram_params,
1715                                  sizeof(plat->sdram_params) / sizeof(u32));
1716         if (ret) {
1717                 printf("%s: Cannot read rockchip,sdram-params %d\n",
1718                        __func__, ret);
1719                 return ret;
1720         }
1721         ret = regmap_init_mem(dev_ofnode(dev), &plat->map);
1722         if (ret)
1723                 printf("%s: regmap failed %d\n", __func__, ret);
1724
1725 #endif
1726         return 0;
1727 }
1728
1729 #if CONFIG_IS_ENABLED(OF_PLATDATA)
1730 static int conv_of_platdata(struct udevice *dev)
1731 {
1732         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
1733         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
1734         int ret;
1735
1736         ret = regmap_init_mem_platdata(dev, dtplat->reg,
1737                                        ARRAY_SIZE(dtplat->reg) / 2,
1738                                        &plat->map);
1739         if (ret)
1740                 return ret;
1741
1742         return 0;
1743 }
1744 #endif
1745
1746 static int rk3399_dmc_init(struct udevice *dev)
1747 {
1748         struct dram_info *priv = dev_get_priv(dev);
1749         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
1750         int ret;
1751 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
1752         struct rk3399_sdram_params *params = &plat->sdram_params;
1753 #else
1754         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
1755         struct rk3399_sdram_params *params =
1756                                         (void *)dtplat->rockchip_sdram_params;
1757
1758         ret = conv_of_platdata(dev);
1759         if (ret)
1760                 return ret;
1761 #endif
1762
1763         priv->cic = syscon_get_first_range(ROCKCHIP_SYSCON_CIC);
1764         priv->grf = syscon_get_first_range(ROCKCHIP_SYSCON_GRF);
1765         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
1766         priv->pmusgrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUSGRF);
1767         priv->pmucru = rockchip_get_pmucru();
1768         priv->cru = rockchip_get_cru();
1769         priv->chan[0].pctl = regmap_get_range(plat->map, 0);
1770         priv->chan[0].pi = regmap_get_range(plat->map, 1);
1771         priv->chan[0].publ = regmap_get_range(plat->map, 2);
1772         priv->chan[0].msch = regmap_get_range(plat->map, 3);
1773         priv->chan[1].pctl = regmap_get_range(plat->map, 4);
1774         priv->chan[1].pi = regmap_get_range(plat->map, 5);
1775         priv->chan[1].publ = regmap_get_range(plat->map, 6);
1776         priv->chan[1].msch = regmap_get_range(plat->map, 7);
1777
1778         debug("con reg %p %p %p %p %p %p %p %p\n",
1779               priv->chan[0].pctl, priv->chan[0].pi,
1780               priv->chan[0].publ, priv->chan[0].msch,
1781               priv->chan[1].pctl, priv->chan[1].pi,
1782               priv->chan[1].publ, priv->chan[1].msch);
1783         debug("cru %p, cic %p, grf %p, sgrf %p, pmucru %p\n", priv->cru,
1784               priv->cic, priv->pmugrf, priv->pmusgrf, priv->pmucru);
1785
1786 #if CONFIG_IS_ENABLED(OF_PLATDATA)
1787         ret = clk_get_by_index_platdata(dev, 0, dtplat->clocks, &priv->ddr_clk);
1788 #else
1789         ret = clk_get_by_index(dev, 0, &priv->ddr_clk);
1790 #endif
1791         if (ret) {
1792                 printf("%s clk get failed %d\n", __func__, ret);
1793                 return ret;
1794         }
1795
1796         ret = clk_set_rate(&priv->ddr_clk, params->base.ddr_freq * MHz);
1797         if (ret < 0) {
1798                 printf("%s clk set failed %d\n", __func__, ret);
1799                 return ret;
1800         }
1801
1802         ret = sdram_init(priv, params);
1803         if (ret < 0) {
1804                 printf("%s DRAM init failed %d\n", __func__, ret);
1805                 return ret;
1806         }
1807
1808         return 0;
1809 }
1810 #endif
1811
1812 static int rk3399_dmc_probe(struct udevice *dev)
1813 {
1814 #if defined(CONFIG_TPL_BUILD) || \
1815         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
1816         if (rk3399_dmc_init(dev))
1817                 return 0;
1818 #else
1819         struct dram_info *priv = dev_get_priv(dev);
1820
1821         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
1822         debug("%s: pmugrf = %p\n", __func__, priv->pmugrf);
1823         priv->info.base = CONFIG_SYS_SDRAM_BASE;
1824         priv->info.size =
1825                 rockchip_sdram_size((phys_addr_t)&priv->pmugrf->os_reg2);
1826 #endif
1827         return 0;
1828 }
1829
1830 static int rk3399_dmc_get_info(struct udevice *dev, struct ram_info *info)
1831 {
1832         struct dram_info *priv = dev_get_priv(dev);
1833
1834         *info = priv->info;
1835
1836         return 0;
1837 }
1838
1839 static struct ram_ops rk3399_dmc_ops = {
1840         .get_info = rk3399_dmc_get_info,
1841 };
1842
1843 static const struct udevice_id rk3399_dmc_ids[] = {
1844         { .compatible = "rockchip,rk3399-dmc" },
1845         { }
1846 };
1847
1848 U_BOOT_DRIVER(dmc_rk3399) = {
1849         .name = "rockchip_rk3399_dmc",
1850         .id = UCLASS_RAM,
1851         .of_match = rk3399_dmc_ids,
1852         .ops = &rk3399_dmc_ops,
1853 #if defined(CONFIG_TPL_BUILD) || \
1854         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
1855         .ofdata_to_platdata = rk3399_dmc_ofdata_to_platdata,
1856 #endif
1857         .probe = rk3399_dmc_probe,
1858         .priv_auto_alloc_size = sizeof(struct dram_info),
1859 #if defined(CONFIG_TPL_BUILD) || \
1860         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
1861         .platdata_auto_alloc_size = sizeof(struct rockchip_dmc_plat),
1862 #endif
1863 };