drivers/ram/rockchip/sdram_rk3399.c

   1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
   2 /*
   3  * (C) Copyright 2016-2017 Rockchip Inc.
   4  *
   5  * Adapted from coreboot.
   6  */
   7
   8 #include <common.h>
   9 #include <clk.h>
  10 #include <dm.h>
  11 #include <dt-structs.h>
  12 #include <ram.h>
  13 #include <regmap.h>
  14 #include <syscon.h>
  15 #include <asm/io.h>
  16 #include <asm/arch-rockchip/clock.h>
  17 #include <asm/arch-rockchip/cru_rk3399.h>
  18 #include <asm/arch-rockchip/grf_rk3399.h>
  19 #include <asm/arch-rockchip/pmu_rk3399.h>
  20 #include <asm/arch-rockchip/hardware.h>
  21 #include <asm/arch-rockchip/sdram.h>
  22 #include <asm/arch-rockchip/sdram_rk3399.h>
  23 #include <linux/err.h>
  24 #include <time.h>
  25
  26 #define PRESET_SGRF_HOLD(n)     ((0x1 << (6 + 16)) | ((n) << 6))
  27 #define PRESET_GPIO0_HOLD(n)    ((0x1 << (7 + 16)) | ((n) << 7))
  28 #define PRESET_GPIO1_HOLD(n)    ((0x1 << (8 + 16)) | ((n) << 8))
  29
  30 #define PHY_DRV_ODT_HI_Z        0x0
  31 #define PHY_DRV_ODT_240         0x1
  32 #define PHY_DRV_ODT_120         0x8
  33 #define PHY_DRV_ODT_80          0x9
  34 #define PHY_DRV_ODT_60          0xc
  35 #define PHY_DRV_ODT_48          0xd
  36 #define PHY_DRV_ODT_40          0xe
  37 #define PHY_DRV_ODT_34_3        0xf
  38
  39 #define PHY_BOOSTP_EN           0x1
  40 #define PHY_BOOSTN_EN           0x1
  41 #define PHY_SLEWP_EN            0x1
  42 #define PHY_SLEWN_EN            0x1
  43 #define PHY_RX_CM_INPUT         0x1
  44 #define CS0_MR22_VAL            0
  45 #define CS1_MR22_VAL            3
  46
  47 #define CRU_SFTRST_DDR_CTRL(ch, n)      ((0x1 << (8 + 16 + (ch) * 4)) | \
  48                                         ((n) << (8 + (ch) * 4)))
  49 #define CRU_SFTRST_DDR_PHY(ch, n)       ((0x1 << (9 + 16 + (ch) * 4)) | \
  50                                         ((n) << (9 + (ch) * 4)))
  51 struct chan_info {
  52         struct rk3399_ddr_pctl_regs *pctl;
  53         struct rk3399_ddr_pi_regs *pi;
  54         struct rk3399_ddr_publ_regs *publ;
  55         struct msch_regs *msch;
  56 };
  57
  58 struct dram_info {
  59 #if defined(CONFIG_TPL_BUILD) || \
  60         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  61         u32 pwrup_srefresh_exit[2];
  62         struct chan_info chan[2];
  63         struct clk ddr_clk;
  64         struct rk3399_cru *cru;
  65         struct rk3399_grf_regs *grf;
  66         struct rk3399_pmu_regs *pmu;
  67         struct rk3399_pmucru *pmucru;
  68         struct rk3399_pmusgrf_regs *pmusgrf;
  69         struct rk3399_ddr_cic_regs *cic;
  70         const struct sdram_rk3399_ops *ops;
  71 #endif
  72         struct ram_info info;
  73         struct rk3399_pmugrf_regs *pmugrf;
  74 };
  75
  76 struct sdram_rk3399_ops {
  77         int (*data_training_first)(struct dram_info *dram, u32 channel, u8 rank,
  78                                    struct rk3399_sdram_params *sdram);
  79         int (*set_rate_index)(struct dram_info *dram,
  80                               struct rk3399_sdram_params *params);
  81 };
  82
  83 #if defined(CONFIG_TPL_BUILD) || \
  84         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  85
  86 struct rockchip_dmc_plat {
  87 #if CONFIG_IS_ENABLED(OF_PLATDATA)
  88         struct dtd_rockchip_rk3399_dmc dtplat;
  89 #else
  90         struct rk3399_sdram_params sdram_params;
  91 #endif
  92         struct regmap *map;
  93 };
  94
  95 struct io_setting {
  96         u32 mhz;
  97         u32 mr5;
  98         /* dram side */
  99         u32 dq_odt;
 100         u32 ca_odt;
 101         u32 pdds;
 102         u32 dq_vref;
 103         u32 ca_vref;
 104         /* phy side */
 105         u32 rd_odt;
 106         u32 wr_dq_drv;
 107         u32 wr_ca_drv;
 108         u32 wr_ckcs_drv;
 109         u32 rd_odt_en;
 110         u32 rd_vref;
 111 } lpddr4_io_setting[] = {
 112         {
 113                 50 * MHz,
 114                 0,
 115                 /* dram side */
 116                 0,      /* dq_odt; */
 117                 0,      /* ca_odt; */
 118                 6,      /* pdds; */
 119                 0x72,   /* dq_vref; */
 120                 0x72,   /* ca_vref; */
 121                 /* phy side */
 122                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 123                 PHY_DRV_ODT_40, /* wr_dq_drv; */
 124                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 125                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 126                 0,      /* rd_odt_en;*/
 127                 41,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 128         },
 129         {
 130                 600 * MHz,
 131                 0,
 132                 /* dram side */
 133                 1,      /* dq_odt; */
 134                 0,      /* ca_odt; */
 135                 6,      /* pdds; */
 136                 0x72,   /* dq_vref; */
 137                 0x72,   /* ca_vref; */
 138                 /* phy side */
 139                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 140                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 141                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 142                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 143                 0,      /* rd_odt_en; */
 144                 32,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 145         },
 146         {
 147                 800 * MHz,
 148                 0,
 149                 /* dram side */
 150                 1,      /* dq_odt; */
 151                 0,      /* ca_odt; */
 152                 1,      /* pdds; */
 153                 0x72,   /* dq_vref; */
 154                 0x72,   /* ca_vref; */
 155                 /* phy side */
 156                 PHY_DRV_ODT_40, /* rd_odt; */
 157                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 158                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 159                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 160                 1,      /* rd_odt_en; */
 161                 17,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 162         },
 163         {
 164                 933 * MHz,
 165                 0,
 166                 /* dram side */
 167                 3,      /* dq_odt; */
 168                 0,      /* ca_odt; */
 169                 6,      /* pdds; */
 170                 0x59,   /* dq_vref; 32% */
 171                 0x72,   /* ca_vref; */
 172                 /* phy side */
 173                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 174                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 175                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 176                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 177                 0,      /* rd_odt_en; */
 178                 32,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 179         },
 180         {
 181                 1066 * MHz,
 182                 0,
 183                 /* dram side */
 184                 6,      /* dq_odt; */
 185                 0,      /* ca_odt; */
 186                 1,      /* pdds; */
 187                 0x10,   /* dq_vref; */
 188                 0x72,   /* ca_vref; */
 189                 /* phy side */
 190                 PHY_DRV_ODT_40, /* rd_odt; */
 191                 PHY_DRV_ODT_60, /* wr_dq_drv; */
 192                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 193                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 194                 1,      /* rd_odt_en; */
 195                 17,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 196         },
 197 };
 198
 199 static struct io_setting *
 200 lpddr4_get_io_settings(const struct rk3399_sdram_params *params, u32 mr5)
 201 {
 202         struct io_setting *io = NULL;
 203         u32 n;
 204
 205         for (n = 0; n < ARRAY_SIZE(lpddr4_io_setting); n++) {
 206                 io = &lpddr4_io_setting[n];
 207
 208                 if (io->mr5 != 0) {
 209                         if (io->mhz >= params->base.ddr_freq &&
 210                             io->mr5 == mr5)
 211                                 break;
 212                 } else {
 213                         if (io->mhz >= params->base.ddr_freq)
 214                                 break;
 215                 }
 216         }
 217
 218         return io;
 219 }
 220
 221 static void *get_denali_ctl(const struct chan_info *chan,
 222                             struct rk3399_sdram_params *params, bool reg)
 223 {
 224         return reg ? &chan->pctl->denali_ctl : &params->pctl_regs.denali_ctl;
 225 }
 226
 227 static void *get_denali_phy(const struct chan_info *chan,
 228                             struct rk3399_sdram_params *params, bool reg)
 229 {
 230         return reg ? &chan->publ->denali_phy : &params->phy_regs.denali_phy;
 231 }
 232
 233 static void *get_ddrc0_con(struct dram_info *dram, u8 channel)
 234 {
 235         return (channel == 0) ? &dram->grf->ddrc0_con0 : &dram->grf->ddrc0_con1;
 236 }
 237
 238 static void rkclk_ddr_reset(struct rk3399_cru *cru, u32 channel, u32 ctl,
 239                             u32 phy)
 240 {
 241         channel &= 0x1;
 242         ctl &= 0x1;
 243         phy &= 0x1;
 244         writel(CRU_SFTRST_DDR_CTRL(channel, ctl) |
 245                                    CRU_SFTRST_DDR_PHY(channel, phy),
 246                                    &cru->softrst_con[4]);
 247 }
 248
 249 static void phy_pctrl_reset(struct rk3399_cru *cru,  u32 channel)
 250 {
 251         rkclk_ddr_reset(cru, channel, 1, 1);
 252         udelay(10);
 253
 254         rkclk_ddr_reset(cru, channel, 1, 0);
 255         udelay(10);
 256
 257         rkclk_ddr_reset(cru, channel, 0, 0);
 258         udelay(10);
 259 }
 260
 261 static void phy_dll_bypass_set(struct rk3399_ddr_publ_regs *ddr_publ_regs,
 262                                u32 freq)
 263 {
 264         u32 *denali_phy = ddr_publ_regs->denali_phy;
 265
 266         /* From IP spec, only freq small than 125 can enter dll bypass mode */
 267         if (freq <= 125) {
 268                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 269                 setbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 270                 setbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 271                 setbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 272                 setbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 273
 274                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 275                 setbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 276                 setbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 277                 setbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 278         } else {
 279                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 280                 clrbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 281                 clrbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 282                 clrbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 283                 clrbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 284
 285                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 286                 clrbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 287                 clrbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 288                 clrbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 289         }
 290 }
 291
 292 static void set_memory_map(const struct chan_info *chan, u32 channel,
 293                            const struct rk3399_sdram_params *params)
 294 {
 295         const struct rk3399_sdram_channel *sdram_ch = &params->ch[channel];
 296         u32 *denali_ctl = chan->pctl->denali_ctl;
 297         u32 *denali_pi = chan->pi->denali_pi;
 298         u32 cs_map;
 299         u32 reduc;
 300         u32 row;
 301
 302         /* Get row number from ddrconfig setting */
 303         if (sdram_ch->cap_info.ddrconfig < 2 ||
 304             sdram_ch->cap_info.ddrconfig == 4)
 305                 row = 16;
 306         else if (sdram_ch->cap_info.ddrconfig == 3)
 307                 row = 14;
 308         else
 309                 row = 15;
 310
 311         cs_map = (sdram_ch->cap_info.rank > 1) ? 3 : 1;
 312         reduc = (sdram_ch->cap_info.bw == 2) ? 0 : 1;
 313
 314         /* Set the dram configuration to ctrl */
 315         clrsetbits_le32(&denali_ctl[191], 0xF, (12 - sdram_ch->cap_info.col));
 316         clrsetbits_le32(&denali_ctl[190], (0x3 << 16) | (0x7 << 24),
 317                         ((3 - sdram_ch->cap_info.bk) << 16) |
 318                         ((16 - row) << 24));
 319
 320         clrsetbits_le32(&denali_ctl[196], 0x3 | (1 << 16),
 321                         cs_map | (reduc << 16));
 322
 323         /* PI_199 PI_COL_DIFF:RW:0:4 */
 324         clrsetbits_le32(&denali_pi[199], 0xF, (12 - sdram_ch->cap_info.col));
 325
 326         /* PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2 */
 327         clrsetbits_le32(&denali_pi[155], (0x3 << 16) | (0x7 << 24),
 328                         ((3 - sdram_ch->cap_info.bk) << 16) |
 329                         ((16 - row) << 24));
 330
 331         if (params->base.dramtype == LPDDR4) {
 332                 if (cs_map == 1)
 333                         cs_map = 0x5;
 334                 else if (cs_map == 2)
 335                         cs_map = 0xa;
 336                 else
 337                         cs_map = 0xF;
 338         }
 339
 340         /* PI_41 PI_CS_MAP:RW:24:4 */
 341         clrsetbits_le32(&denali_pi[41], 0xf << 24, cs_map << 24);
 342         if (sdram_ch->cap_info.rank == 1 && params->base.dramtype == DDR3)
 343                 writel(0x2EC7FFFF, &denali_pi[34]);
 344 }
 345
 346 static int phy_io_config(const struct chan_info *chan,
 347                          const struct rk3399_sdram_params *params, u32 mr5)
 348 {
 349         u32 *denali_phy = chan->publ->denali_phy;
 350         u32 vref_mode_dq, vref_value_dq, vref_mode_ac, vref_value_ac;
 351         u32 mode_sel;
 352         u32 reg_value;
 353         u32 drv_value, odt_value;
 354         u32 speed;
 355
 356         /* vref setting & mode setting */
 357         if (params->base.dramtype == LPDDR4) {
 358                 struct io_setting *io = lpddr4_get_io_settings(params, mr5);
 359                 u32 rd_vref = io->rd_vref * 1000;
 360
 361                 if (rd_vref < 36700) {
 362                         /* MODE_LV[2:0] = LPDDR4 (Range 2)*/
 363                         vref_mode_dq = 0x7;
 364                         /* MODE[2:0]= LPDDR4 Range 2(0.4*VDDQ) */
 365                         mode_sel = 0x5;
 366                         vref_value_dq = (rd_vref - 3300) / 521;
 367                 } else {
 368                         /* MODE_LV[2:0] = LPDDR4 (Range 1)*/
 369                         vref_mode_dq = 0x6;
 370                         /* MODE[2:0]= LPDDR4 Range 1(0.33*VDDQ) */
 371                         mode_sel = 0x4;
 372                         vref_value_dq = (rd_vref - 15300) / 521;
 373                 }
 374                 vref_mode_ac = 0x6;
 375                 /* VDDQ/3/2=16.8% */
 376                 vref_value_ac = 0x3;
 377         } else if (params->base.dramtype == LPDDR3) {
 378                 if (params->base.odt == 1) {
 379                         vref_mode_dq = 0x5;  /* LPDDR3 ODT */
 380                         drv_value = (readl(&denali_phy[6]) >> 12) & 0xf;
 381                         odt_value = (readl(&denali_phy[6]) >> 4) & 0xf;
 382                         if (drv_value == PHY_DRV_ODT_48) {
 383                                 switch (odt_value) {
 384                                 case PHY_DRV_ODT_240:
 385                                         vref_value_dq = 0x16;
 386                                         break;
 387                                 case PHY_DRV_ODT_120:
 388                                         vref_value_dq = 0x26;
 389                                         break;
 390                                 case PHY_DRV_ODT_60:
 391                                         vref_value_dq = 0x36;
 392                                         break;
 393                                 default:
 394                                         debug("Invalid ODT value.\n");
 395                                         return -EINVAL;
 396                                 }
 397                         } else if (drv_value == PHY_DRV_ODT_40) {
 398                                 switch (odt_value) {
 399                                 case PHY_DRV_ODT_240:
 400                                         vref_value_dq = 0x19;
 401                                         break;
 402                                 case PHY_DRV_ODT_120:
 403                                         vref_value_dq = 0x23;
 404                                         break;
 405                                 case PHY_DRV_ODT_60:
 406                                         vref_value_dq = 0x31;
 407                                         break;
 408                                 default:
 409                                         debug("Invalid ODT value.\n");
 410                                         return -EINVAL;
 411                                 }
 412                         } else if (drv_value == PHY_DRV_ODT_34_3) {
 413                                 switch (odt_value) {
 414                                 case PHY_DRV_ODT_240:
 415                                         vref_value_dq = 0x17;
 416                                         break;
 417                                 case PHY_DRV_ODT_120:
 418                                         vref_value_dq = 0x20;
 419                                         break;
 420                                 case PHY_DRV_ODT_60:
 421                                         vref_value_dq = 0x2e;
 422                                         break;
 423                                 default:
 424                                         debug("Invalid ODT value.\n");
 425                                         return -EINVAL;
 426                                 }
 427                         } else {
 428                                 debug("Invalid DRV value.\n");
 429                                 return -EINVAL;
 430                         }
 431                 } else {
 432                         vref_mode_dq = 0x2;  /* LPDDR3 */
 433                         vref_value_dq = 0x1f;
 434                 }
 435                 vref_mode_ac = 0x2;
 436                 vref_value_ac = 0x1f;
 437                 mode_sel = 0x0;
 438         } else if (params->base.dramtype == DDR3) {
 439                 /* DDR3L */
 440                 vref_mode_dq = 0x1;
 441                 vref_value_dq = 0x1f;
 442                 vref_mode_ac = 0x1;
 443                 vref_value_ac = 0x1f;
 444                 mode_sel = 0x1;
 445         } else {
 446                 debug("Unknown DRAM type.\n");
 447                 return -EINVAL;
 448         }
 449
 450         reg_value = (vref_mode_dq << 9) | (0x1 << 8) | vref_value_dq;
 451
 452         /* PHY_913 PHY_PAD_VREF_CTRL_DQ_0 12bits offset_8 */
 453         clrsetbits_le32(&denali_phy[913], 0xfff << 8, reg_value << 8);
 454         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_1 12bits offset_0 */
 455         clrsetbits_le32(&denali_phy[914], 0xfff, reg_value);
 456         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_2 12bits offset_16 */
 457         clrsetbits_le32(&denali_phy[914], 0xfff << 16, reg_value << 16);
 458         /* PHY_915 PHY_PAD_VREF_CTRL_DQ_3 12bits offset_0 */
 459         clrsetbits_le32(&denali_phy[915], 0xfff, reg_value);
 460
 461         reg_value = (vref_mode_ac << 9) | (0x1 << 8) | vref_value_ac;
 462
 463         /* PHY_915 PHY_PAD_VREF_CTRL_AC 12bits offset_16 */
 464         clrsetbits_le32(&denali_phy[915], 0xfff << 16, reg_value << 16);
 465
 466         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 467         clrsetbits_le32(&denali_phy[924], 0x7 << 15, mode_sel << 15);
 468         /* PHY_926 PHY_PAD_DATA_DRIVE */
 469         clrsetbits_le32(&denali_phy[926], 0x7 << 6, mode_sel << 6);
 470         /* PHY_927 PHY_PAD_DQS_DRIVE */
 471         clrsetbits_le32(&denali_phy[927], 0x7 << 6, mode_sel << 6);
 472         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 473         clrsetbits_le32(&denali_phy[928], 0x7 << 14, mode_sel << 14);
 474         /* PHY_929 PHY_PAD_CLK_DRIVE */
 475         clrsetbits_le32(&denali_phy[929], 0x7 << 14, mode_sel << 14);
 476         /* PHY_935 PHY_PAD_CKE_DRIVE */
 477         clrsetbits_le32(&denali_phy[935], 0x7 << 14, mode_sel << 14);
 478         /* PHY_937 PHY_PAD_RST_DRIVE */
 479         clrsetbits_le32(&denali_phy[937], 0x7 << 14, mode_sel << 14);
 480         /* PHY_939 PHY_PAD_CS_DRIVE */
 481         clrsetbits_le32(&denali_phy[939], 0x7 << 14, mode_sel << 14);
 482
 483         if (params->base.dramtype == LPDDR4) {
 484                 /* BOOSTP_EN & BOOSTN_EN */
 485                 reg_value = ((PHY_BOOSTP_EN << 4) | PHY_BOOSTN_EN);
 486                 /* PHY_925 PHY_PAD_FDBK_DRIVE2 */
 487                 clrsetbits_le32(&denali_phy[925], 0xff << 8, reg_value << 8);
 488                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 489                 clrsetbits_le32(&denali_phy[926], 0xff << 12, reg_value << 12);
 490                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 491                 clrsetbits_le32(&denali_phy[927], 0xff << 14, reg_value << 14);
 492                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 493                 clrsetbits_le32(&denali_phy[928], 0xff << 20, reg_value << 20);
 494                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 495                 clrsetbits_le32(&denali_phy[929], 0xff << 22, reg_value << 22);
 496                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 497                 clrsetbits_le32(&denali_phy[935], 0xff << 20, reg_value << 20);
 498                 /* PHY_937 PHY_PAD_RST_DRIVE */
 499                 clrsetbits_le32(&denali_phy[937], 0xff << 20, reg_value << 20);
 500                 /* PHY_939 PHY_PAD_CS_DRIVE */
 501                 clrsetbits_le32(&denali_phy[939], 0xff << 20, reg_value << 20);
 502
 503                 /* SLEWP_EN & SLEWN_EN */
 504                 reg_value = ((PHY_SLEWP_EN << 3) | PHY_SLEWN_EN);
 505                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 506                 clrsetbits_le32(&denali_phy[924], 0x3f << 8, reg_value << 8);
 507                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 508                 clrsetbits_le32(&denali_phy[926], 0x3f, reg_value);
 509                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 510                 clrsetbits_le32(&denali_phy[927], 0x3f, reg_value);
 511                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 512                 clrsetbits_le32(&denali_phy[928], 0x3f << 8, reg_value << 8);
 513                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 514                 clrsetbits_le32(&denali_phy[929], 0x3f << 8, reg_value << 8);
 515                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 516                 clrsetbits_le32(&denali_phy[935], 0x3f << 8, reg_value << 8);
 517                 /* PHY_937 PHY_PAD_RST_DRIVE */
 518                 clrsetbits_le32(&denali_phy[937], 0x3f << 8, reg_value << 8);
 519                 /* PHY_939 PHY_PAD_CS_DRIVE */
 520                 clrsetbits_le32(&denali_phy[939], 0x3f << 8, reg_value << 8);
 521         }
 522
 523         /* speed setting */
 524         if (params->base.ddr_freq < 400)
 525                 speed = 0x0;
 526         else if (params->base.ddr_freq < 800)
 527                 speed = 0x1;
 528         else if (params->base.ddr_freq < 1200)
 529                 speed = 0x2;
 530         else
 531                 speed = 0x3;
 532
 533         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 534         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
 535         /* PHY_926 PHY_PAD_DATA_DRIVE */
 536         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
 537         /* PHY_927 PHY_PAD_DQS_DRIVE */
 538         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
 539         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 540         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
 541         /* PHY_929 PHY_PAD_CLK_DRIVE */
 542         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
 543         /* PHY_935 PHY_PAD_CKE_DRIVE */
 544         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
 545         /* PHY_937 PHY_PAD_RST_DRIVE */
 546         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
 547         /* PHY_939 PHY_PAD_CS_DRIVE */
 548         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
 549
 550         if (params->base.dramtype == LPDDR4) {
 551                 /* RX_CM_INPUT */
 552                 reg_value = PHY_RX_CM_INPUT;
 553                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 554                 clrsetbits_le32(&denali_phy[924], 0x1 << 14, reg_value << 14);
 555                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 556                 clrsetbits_le32(&denali_phy[926], 0x1 << 11, reg_value << 11);
 557                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 558                 clrsetbits_le32(&denali_phy[927], 0x1 << 13, reg_value << 13);
 559                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 560                 clrsetbits_le32(&denali_phy[928], 0x1 << 19, reg_value << 19);
 561                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 562                 clrsetbits_le32(&denali_phy[929], 0x1 << 21, reg_value << 21);
 563                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 564                 clrsetbits_le32(&denali_phy[935], 0x1 << 19, reg_value << 19);
 565                 /* PHY_937 PHY_PAD_RST_DRIVE */
 566                 clrsetbits_le32(&denali_phy[937], 0x1 << 19, reg_value << 19);
 567                 /* PHY_939 PHY_PAD_CS_DRIVE */
 568                 clrsetbits_le32(&denali_phy[939], 0x1 << 19, reg_value << 19);
 569         }
 570
 571         return 0;
 572 }
 573
 574 static void set_ds_odt(const struct chan_info *chan,
 575                        struct rk3399_sdram_params *params,
 576                        bool ctl_phy_reg, u32 mr5)
 577 {
 578         u32 *denali_phy = get_denali_phy(chan, params, ctl_phy_reg);
 579         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
 580         u32 tsel_idle_en, tsel_wr_en, tsel_rd_en;
 581         u32 tsel_idle_select_p, tsel_rd_select_p;
 582         u32 tsel_idle_select_n, tsel_rd_select_n;
 583         u32 tsel_wr_select_dq_p, tsel_wr_select_ca_p;
 584         u32 tsel_wr_select_dq_n, tsel_wr_select_ca_n;
 585         u32 tsel_ckcs_select_p, tsel_ckcs_select_n;
 586         struct io_setting *io = NULL;
 587         u32 soc_odt = 0;
 588         u32 reg_value;
 589
 590         if (params->base.dramtype == LPDDR4) {
 591                 io = lpddr4_get_io_settings(params, mr5);
 592
 593                 tsel_rd_select_p = PHY_DRV_ODT_HI_Z;
 594                 tsel_rd_select_n = io->rd_odt;
 595
 596                 tsel_idle_select_p = PHY_DRV_ODT_HI_Z;
 597                 tsel_idle_select_n = PHY_DRV_ODT_240;
 598
 599                 tsel_wr_select_dq_p = io->wr_dq_drv;
 600                 tsel_wr_select_dq_n = PHY_DRV_ODT_40;
 601
 602                 tsel_wr_select_ca_p = io->wr_ca_drv;
 603                 tsel_wr_select_ca_n = PHY_DRV_ODT_40;
 604
 605                 tsel_ckcs_select_p = io->wr_ckcs_drv;
 606                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 607                 switch (tsel_rd_select_n) {
 608                 case PHY_DRV_ODT_240:
 609                         soc_odt = 1;
 610                         break;
 611                 case PHY_DRV_ODT_120:
 612                         soc_odt = 2;
 613                         break;
 614                 case PHY_DRV_ODT_80:
 615                         soc_odt = 3;
 616                         break;
 617                 case PHY_DRV_ODT_60:
 618                         soc_odt = 4;
 619                         break;
 620                 case PHY_DRV_ODT_48:
 621                         soc_odt = 5;
 622                         break;
 623                 case PHY_DRV_ODT_40:
 624                         soc_odt = 6;
 625                         break;
 626                 case PHY_DRV_ODT_34_3:
 627                         soc_odt = 6;
 628                         printf("%s: Unable to support LPDDR4 MR22 Soc ODT\n",
 629                                __func__);
 630                         break;
 631                 case PHY_DRV_ODT_HI_Z:
 632                 default:
 633                         soc_odt = 0;
 634                         break;
 635                 }
 636         } else if (params->base.dramtype == LPDDR3) {
 637                 tsel_rd_select_p = PHY_DRV_ODT_240;
 638                 tsel_rd_select_n = PHY_DRV_ODT_HI_Z;
 639
 640                 tsel_idle_select_p = PHY_DRV_ODT_240;
 641                 tsel_idle_select_n = PHY_DRV_ODT_HI_Z;
 642
 643                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 644                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 645
 646                 tsel_wr_select_ca_p = PHY_DRV_ODT_48;
 647                 tsel_wr_select_ca_n = PHY_DRV_ODT_48;
 648
 649                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 650                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 651         } else {
 652                 tsel_rd_select_p = PHY_DRV_ODT_240;
 653                 tsel_rd_select_n = PHY_DRV_ODT_240;
 654
 655                 tsel_idle_select_p = PHY_DRV_ODT_240;
 656                 tsel_idle_select_n = PHY_DRV_ODT_240;
 657
 658                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 659                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 660
 661                 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3;
 662                 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3;
 663
 664                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 665                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 666         }
 667
 668         if (params->base.odt == 1) {
 669                 tsel_rd_en = 1;
 670
 671                 if (params->base.dramtype == LPDDR4)
 672                         tsel_rd_en = io->rd_odt_en;
 673         } else {
 674                 tsel_rd_en = 0;
 675         }
 676
 677         tsel_wr_en = 0;
 678         tsel_idle_en = 0;
 679
 680         /* F0_0 */
 681         clrsetbits_le32(&denali_ctl[145], 0xFF << 16,
 682                         (soc_odt | (CS0_MR22_VAL << 3)) << 16);
 683         /* F2_0, F1_0 */
 684         clrsetbits_le32(&denali_ctl[146], 0xFF00FF,
 685                         ((soc_odt | (CS0_MR22_VAL << 3)) << 16) |
 686                         (soc_odt | (CS0_MR22_VAL << 3)));
 687         /* F0_1 */
 688         clrsetbits_le32(&denali_ctl[159], 0xFF << 16,
 689                         (soc_odt | (CS1_MR22_VAL << 3)) << 16);
 690         /* F2_1, F1_1 */
 691         clrsetbits_le32(&denali_ctl[160], 0xFF00FF,
 692                         ((soc_odt | (CS1_MR22_VAL << 3)) << 16) |
 693                         (soc_odt | (CS1_MR22_VAL << 3)));
 694
 695         /*
 696          * phy_dq_tsel_select_X 24bits DENALI_PHY_6/134/262/390 offset_0
 697          * sets termination values for read/idle cycles and drive strength
 698          * for write cycles for DQ/DM
 699          */
 700         reg_value = tsel_rd_select_n | (tsel_rd_select_p << 0x4) |
 701                     (tsel_wr_select_dq_n << 8) | (tsel_wr_select_dq_p << 12) |
 702                     (tsel_idle_select_n << 16) | (tsel_idle_select_p << 20);
 703         clrsetbits_le32(&denali_phy[6], 0xffffff, reg_value);
 704         clrsetbits_le32(&denali_phy[134], 0xffffff, reg_value);
 705         clrsetbits_le32(&denali_phy[262], 0xffffff, reg_value);
 706         clrsetbits_le32(&denali_phy[390], 0xffffff, reg_value);
 707
 708         /*
 709          * phy_dqs_tsel_select_X 24bits DENALI_PHY_7/135/263/391 offset_0
 710          * sets termination values for read/idle cycles and drive strength
 711          * for write cycles for DQS
 712          */
 713         clrsetbits_le32(&denali_phy[7], 0xffffff, reg_value);
 714         clrsetbits_le32(&denali_phy[135], 0xffffff, reg_value);
 715         clrsetbits_le32(&denali_phy[263], 0xffffff, reg_value);
 716         clrsetbits_le32(&denali_phy[391], 0xffffff, reg_value);
 717
 718         /* phy_adr_tsel_select_ 8bits DENALI_PHY_544/672/800 offset_0 */
 719         reg_value = tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 0x4);
 720         if (params->base.dramtype == LPDDR4) {
 721                 /* LPDDR4 these register read always return 0, so
 722                  * can not use clrsetbits_le32(), need to write32
 723                  */
 724                 writel((0x300 << 8) | reg_value, &denali_phy[544]);
 725                 writel((0x300 << 8) | reg_value, &denali_phy[672]);
 726                 writel((0x300 << 8) | reg_value, &denali_phy[800]);
 727         } else {
 728                 clrsetbits_le32(&denali_phy[544], 0xff, reg_value);
 729                 clrsetbits_le32(&denali_phy[672], 0xff, reg_value);
 730                 clrsetbits_le32(&denali_phy[800], 0xff, reg_value);
 731         }
 732
 733         /* phy_pad_addr_drive 8bits DENALI_PHY_928 offset_0 */
 734         clrsetbits_le32(&denali_phy[928], 0xff, reg_value);
 735
 736         /* phy_pad_rst_drive 8bits DENALI_PHY_937 offset_0 */
 737         if (!ctl_phy_reg)
 738                 clrsetbits_le32(&denali_phy[937], 0xff, reg_value);
 739
 740         /* phy_pad_cke_drive 8bits DENALI_PHY_935 offset_0 */
 741         clrsetbits_le32(&denali_phy[935], 0xff, reg_value);
 742
 743         /* phy_pad_cs_drive 8bits DENALI_PHY_939 offset_0 */
 744         clrsetbits_le32(&denali_phy[939], 0xff,
 745                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 746
 747         /* phy_pad_clk_drive 8bits DENALI_PHY_929 offset_0 */
 748         clrsetbits_le32(&denali_phy[929], 0xff,
 749                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 750
 751         /* phy_pad_fdbk_drive 23bit DENALI_PHY_924/925 */
 752         clrsetbits_le32(&denali_phy[924], 0xff,
 753                         tsel_wr_select_dq_n | (tsel_wr_select_dq_p << 4));
 754         clrsetbits_le32(&denali_phy[925], 0xff,
 755                         tsel_rd_select_n | (tsel_rd_select_p << 4));
 756
 757         /* phy_dq_tsel_enable_X 3bits DENALI_PHY_5/133/261/389 offset_16 */
 758         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 759                 << 16;
 760         clrsetbits_le32(&denali_phy[5], 0x7 << 16, reg_value);
 761         clrsetbits_le32(&denali_phy[133], 0x7 << 16, reg_value);
 762         clrsetbits_le32(&denali_phy[261], 0x7 << 16, reg_value);
 763         clrsetbits_le32(&denali_phy[389], 0x7 << 16, reg_value);
 764
 765         /* phy_dqs_tsel_enable_X 3bits DENALI_PHY_6/134/262/390 offset_24 */
 766         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 767                 << 24;
 768         clrsetbits_le32(&denali_phy[6], 0x7 << 24, reg_value);
 769         clrsetbits_le32(&denali_phy[134], 0x7 << 24, reg_value);
 770         clrsetbits_le32(&denali_phy[262], 0x7 << 24, reg_value);
 771         clrsetbits_le32(&denali_phy[390], 0x7 << 24, reg_value);
 772
 773         /* phy_adr_tsel_enable_ 1bit DENALI_PHY_518/646/774 offset_8 */
 774         reg_value = tsel_wr_en << 8;
 775         clrsetbits_le32(&denali_phy[518], 0x1 << 8, reg_value);
 776         clrsetbits_le32(&denali_phy[646], 0x1 << 8, reg_value);
 777         clrsetbits_le32(&denali_phy[774], 0x1 << 8, reg_value);
 778
 779         /* phy_pad_addr_term tsel 1bit DENALI_PHY_933 offset_17 */
 780         reg_value = tsel_wr_en << 17;
 781         clrsetbits_le32(&denali_phy[933], 0x1 << 17, reg_value);
 782         /*
 783          * pad_rst/cke/cs/clk_term tsel 1bits
 784          * DENALI_PHY_938/936/940/934 offset_17
 785          */
 786         clrsetbits_le32(&denali_phy[938], 0x1 << 17, reg_value);
 787         clrsetbits_le32(&denali_phy[936], 0x1 << 17, reg_value);
 788         clrsetbits_le32(&denali_phy[940], 0x1 << 17, reg_value);
 789         clrsetbits_le32(&denali_phy[934], 0x1 << 17, reg_value);
 790
 791         /* phy_pad_fdbk_term 1bit DENALI_PHY_930 offset_17 */
 792         clrsetbits_le32(&denali_phy[930], 0x1 << 17, reg_value);
 793
 794         phy_io_config(chan, params, mr5);
 795 }
 796
 797 static void pctl_start(struct dram_info *dram, u8 channel)
 798 {
 799         const struct chan_info *chan = &dram->chan[channel];
 800         u32 *denali_ctl = chan->pctl->denali_ctl;
 801         u32 *denali_phy = chan->publ->denali_phy;
 802         u32 *ddrc0_con = get_ddrc0_con(dram, channel);
 803         u32 count = 0;
 804         u32 byte, tmp;
 805
 806         writel(0x01000000, &ddrc0_con);
 807
 808         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 0x2 << 24);
 809
 810         while (!(readl(&denali_ctl[203]) & (1 << 3))) {
 811                 if (count > 1000) {
 812                         printf("%s: Failed to init pctl for channel %d\n",
 813                                __func__, channel);
 814                         while (1)
 815                                 ;
 816                 }
 817
 818                 udelay(1);
 819                 count++;
 820         }
 821
 822         writel(0x01000100, &ddrc0_con);
 823
 824         for (byte = 0; byte < 4; byte++) {
 825                 tmp = 0x820;
 826                 writel((tmp << 16) | tmp, &denali_phy[53 + (128 * byte)]);
 827                 writel((tmp << 16) | tmp, &denali_phy[54 + (128 * byte)]);
 828                 writel((tmp << 16) | tmp, &denali_phy[55 + (128 * byte)]);
 829                 writel((tmp << 16) | tmp, &denali_phy[56 + (128 * byte)]);
 830                 writel((tmp << 16) | tmp, &denali_phy[57 + (128 * byte)]);
 831
 832                 clrsetbits_le32(&denali_phy[58 + (128 * byte)], 0xffff, tmp);
 833         }
 834
 835         clrsetbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT,
 836                         dram->pwrup_srefresh_exit[channel]);
 837 }
 838
 839 static int pctl_cfg(struct dram_info *dram, const struct chan_info *chan,
 840                     u32 channel, struct rk3399_sdram_params *params)
 841 {
 842         u32 *denali_ctl = chan->pctl->denali_ctl;
 843         u32 *denali_pi = chan->pi->denali_pi;
 844         u32 *denali_phy = chan->publ->denali_phy;
 845         const u32 *params_ctl = params->pctl_regs.denali_ctl;
 846         const u32 *params_phy = params->phy_regs.denali_phy;
 847         u32 tmp, tmp1, tmp2;
 848
 849         /*
 850          * work around controller bug:
 851          * Do not program DRAM_CLASS until NO_PHY_IND_TRAIN_INT is programmed
 852          */
 853         sdram_copy_to_reg(&denali_ctl[1], &params_ctl[1],
 854                           sizeof(struct rk3399_ddr_pctl_regs) - 4);
 855         writel(params_ctl[0], &denali_ctl[0]);
 856
 857         /*
 858          * two channel init at the same time, then ZQ Cal Start
 859          * at the same time, it will use the same RZQ, but cannot
 860          * start at the same time.
 861          *
 862          * So, increase tINIT3 for channel 1, will avoid two
 863          * channel ZQ Cal Start at the same time
 864          */
 865         if (params->base.dramtype == LPDDR4 && channel == 1) {
 866                 tmp = ((params->base.ddr_freq * MHz + 999) / 1000);
 867                 tmp1 = readl(&denali_ctl[14]);
 868                 writel(tmp + tmp1, &denali_ctl[14]);
 869         }
 870
 871         sdram_copy_to_reg(denali_pi, &params->pi_regs.denali_pi[0],
 872                           sizeof(struct rk3399_ddr_pi_regs));
 873
 874         /* rank count need to set for init */
 875         set_memory_map(chan, channel, params);
 876
 877         writel(params->phy_regs.denali_phy[910], &denali_phy[910]);
 878         writel(params->phy_regs.denali_phy[911], &denali_phy[911]);
 879         writel(params->phy_regs.denali_phy[912], &denali_phy[912]);
 880
 881         if (params->base.dramtype == LPDDR4) {
 882                 writel(params->phy_regs.denali_phy[898], &denali_phy[898]);
 883                 writel(params->phy_regs.denali_phy[919], &denali_phy[919]);
 884         }
 885
 886         dram->pwrup_srefresh_exit[channel] = readl(&denali_ctl[68]) &
 887                                              PWRUP_SREFRESH_EXIT;
 888         clrbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT);
 889
 890         /* PHY_DLL_RST_EN */
 891         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 1 << 24);
 892
 893         setbits_le32(&denali_pi[0], START);
 894         setbits_le32(&denali_ctl[0], START);
 895
 896         /**
 897          * LPDDR4 use PLL bypass mode for init
 898          * not need to wait for the PLL to lock
 899          */
 900         if (params->base.dramtype != LPDDR4) {
 901                 /* Waiting for phy DLL lock */
 902                 while (1) {
 903                         tmp = readl(&denali_phy[920]);
 904                         tmp1 = readl(&denali_phy[921]);
 905                         tmp2 = readl(&denali_phy[922]);
 906                         if ((((tmp >> 16) & 0x1) == 0x1) &&
 907                             (((tmp1 >> 16) & 0x1) == 0x1) &&
 908                             (((tmp1 >> 0) & 0x1) == 0x1) &&
 909                             (((tmp2 >> 0) & 0x1) == 0x1))
 910                                 break;
 911                 }
 912         }
 913
 914         sdram_copy_to_reg(&denali_phy[896], &params_phy[896], (958 - 895) * 4);
 915         sdram_copy_to_reg(&denali_phy[0], &params_phy[0], (90 - 0 + 1) * 4);
 916         sdram_copy_to_reg(&denali_phy[128], &params_phy[128],
 917                           (218 - 128 + 1) * 4);
 918         sdram_copy_to_reg(&denali_phy[256], &params_phy[256],
 919                           (346 - 256 + 1) * 4);
 920         sdram_copy_to_reg(&denali_phy[384], &params_phy[384],
 921                           (474 - 384 + 1) * 4);
 922         sdram_copy_to_reg(&denali_phy[512], &params_phy[512],
 923                           (549 - 512 + 1) * 4);
 924         sdram_copy_to_reg(&denali_phy[640], &params_phy[640],
 925                           (677 - 640 + 1) * 4);
 926         sdram_copy_to_reg(&denali_phy[768], &params_phy[768],
 927                           (805 - 768 + 1) * 4);
 928
 929         set_ds_odt(chan, params, true, 0);
 930
 931         /*
 932          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_84/212/340/468 offset_8
 933          * dqs_tsel_wr_end[7:4] add Half cycle
 934          */
 935         tmp = (readl(&denali_phy[84]) >> 8) & 0xff;
 936         clrsetbits_le32(&denali_phy[84], 0xff << 8, (tmp + 0x10) << 8);
 937         tmp = (readl(&denali_phy[212]) >> 8) & 0xff;
 938         clrsetbits_le32(&denali_phy[212], 0xff << 8, (tmp + 0x10) << 8);
 939         tmp = (readl(&denali_phy[340]) >> 8) & 0xff;
 940         clrsetbits_le32(&denali_phy[340], 0xff << 8, (tmp + 0x10) << 8);
 941         tmp = (readl(&denali_phy[468]) >> 8) & 0xff;
 942         clrsetbits_le32(&denali_phy[468], 0xff << 8, (tmp + 0x10) << 8);
 943
 944         /*
 945          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_83/211/339/467 offset_8
 946          * dq_tsel_wr_end[7:4] add Half cycle
 947          */
 948         tmp = (readl(&denali_phy[83]) >> 16) & 0xff;
 949         clrsetbits_le32(&denali_phy[83], 0xff << 16, (tmp + 0x10) << 16);
 950         tmp = (readl(&denali_phy[211]) >> 16) & 0xff;
 951         clrsetbits_le32(&denali_phy[211], 0xff << 16, (tmp + 0x10) << 16);
 952         tmp = (readl(&denali_phy[339]) >> 16) & 0xff;
 953         clrsetbits_le32(&denali_phy[339], 0xff << 16, (tmp + 0x10) << 16);
 954         tmp = (readl(&denali_phy[467]) >> 16) & 0xff;
 955         clrsetbits_le32(&denali_phy[467], 0xff << 16, (tmp + 0x10) << 16);
 956
 957         return 0;
 958 }
 959
 960 static void select_per_cs_training_index(const struct chan_info *chan,
 961                                          u32 rank)
 962 {
 963         u32 *denali_phy = chan->publ->denali_phy;
 964
 965         /* PHY_84 PHY_PER_CS_TRAINING_EN_0 1bit offset_16 */
 966         if ((readl(&denali_phy[84]) >> 16) & 1) {
 967                 /*
 968                  * PHY_8/136/264/392
 969                  * phy_per_cs_training_index_X 1bit offset_24
 970                  */
 971                 clrsetbits_le32(&denali_phy[8], 0x1 << 24, rank << 24);
 972                 clrsetbits_le32(&denali_phy[136], 0x1 << 24, rank << 24);
 973                 clrsetbits_le32(&denali_phy[264], 0x1 << 24, rank << 24);
 974                 clrsetbits_le32(&denali_phy[392], 0x1 << 24, rank << 24);
 975         }
 976 }
 977
 978 static void override_write_leveling_value(const struct chan_info *chan)
 979 {
 980         u32 *denali_ctl = chan->pctl->denali_ctl;
 981         u32 *denali_phy = chan->publ->denali_phy;
 982         u32 byte;
 983
 984         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
 985         setbits_le32(&denali_phy[896], 1);
 986
 987         /*
 988          * PHY_8/136/264/392
 989          * phy_per_cs_training_multicast_en_X 1bit offset_16
 990          */
 991         clrsetbits_le32(&denali_phy[8], 0x1 << 16, 1 << 16);
 992         clrsetbits_le32(&denali_phy[136], 0x1 << 16, 1 << 16);
 993         clrsetbits_le32(&denali_phy[264], 0x1 << 16, 1 << 16);
 994         clrsetbits_le32(&denali_phy[392], 0x1 << 16, 1 << 16);
 995
 996         for (byte = 0; byte < 4; byte++)
 997                 clrsetbits_le32(&denali_phy[63 + (128 * byte)], 0xffff << 16,
 998                                 0x200 << 16);
 999
1000         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
1001         clrbits_le32(&denali_phy[896], 1);
1002
1003         /* CTL_200 ctrlupd_req 1bit offset_8 */
1004         clrsetbits_le32(&denali_ctl[200], 0x1 << 8, 0x1 << 8);
1005 }
1006
1007 static int data_training_ca(const struct chan_info *chan, u32 channel,
1008                             const struct rk3399_sdram_params *params)
1009 {
1010         u32 *denali_pi = chan->pi->denali_pi;
1011         u32 *denali_phy = chan->publ->denali_phy;
1012         u32 i, tmp;
1013         u32 obs_0, obs_1, obs_2, obs_err = 0;
1014         u32 rank = params->ch[channel].cap_info.rank;
1015         u32 rank_mask;
1016
1017         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1018         writel(0x00003f7c, (&denali_pi[175]));
1019
1020         if (params->base.dramtype == LPDDR4)
1021                 rank_mask = (rank == 1) ? 0x5 : 0xf;
1022         else
1023                 rank_mask = (rank == 1) ? 0x1 : 0x3;
1024
1025         for (i = 0; i < 4; i++) {
1026                 if (!(rank_mask & (1 << i)))
1027                         continue;
1028
1029                 select_per_cs_training_index(chan, i);
1030
1031                 /* PI_100 PI_CALVL_EN:RW:8:2 */
1032                 clrsetbits_le32(&denali_pi[100], 0x3 << 8, 0x2 << 8);
1033
1034                 /* PI_92 PI_CALVL_REQ:WR:16:1,PI_CALVL_CS:RW:24:2 */
1035                 clrsetbits_le32(&denali_pi[92],
1036                                 (0x1 << 16) | (0x3 << 24),
1037                                 (0x1 << 16) | (i << 24));
1038
1039                 /* Waiting for training complete */
1040                 while (1) {
1041                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1042                         tmp = readl(&denali_pi[174]) >> 8;
1043                         /*
1044                          * check status obs
1045                          * PHY_532/660/789 phy_adr_calvl_obs1_:0:32
1046                          */
1047                         obs_0 = readl(&denali_phy[532]);
1048                         obs_1 = readl(&denali_phy[660]);
1049                         obs_2 = readl(&denali_phy[788]);
1050                         if (((obs_0 >> 30) & 0x3) ||
1051                             ((obs_1 >> 30) & 0x3) ||
1052                             ((obs_2 >> 30) & 0x3))
1053                                 obs_err = 1;
1054                         if ((((tmp >> 11) & 0x1) == 0x1) &&
1055                             (((tmp >> 13) & 0x1) == 0x1) &&
1056                             (((tmp >> 5) & 0x1) == 0x0) &&
1057                             obs_err == 0)
1058                                 break;
1059                         else if ((((tmp >> 5) & 0x1) == 0x1) ||
1060                                  (obs_err == 1))
1061                                 return -EIO;
1062                 }
1063
1064                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1065                 writel(0x00003f7c, (&denali_pi[175]));
1066         }
1067
1068         clrbits_le32(&denali_pi[100], 0x3 << 8);
1069
1070         return 0;
1071 }
1072
1073 static int data_training_wl(const struct chan_info *chan, u32 channel,
1074                             const struct rk3399_sdram_params *params)
1075 {
1076         u32 *denali_pi = chan->pi->denali_pi;
1077         u32 *denali_phy = chan->publ->denali_phy;
1078         u32 i, tmp;
1079         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1080         u32 rank = params->ch[channel].cap_info.rank;
1081
1082         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1083         writel(0x00003f7c, (&denali_pi[175]));
1084
1085         for (i = 0; i < rank; i++) {
1086                 select_per_cs_training_index(chan, i);
1087
1088                 /* PI_60 PI_WRLVL_EN:RW:8:2 */
1089                 clrsetbits_le32(&denali_pi[60], 0x3 << 8, 0x2 << 8);
1090
1091                 /* PI_59 PI_WRLVL_REQ:WR:8:1,PI_WRLVL_CS:RW:16:2 */
1092                 clrsetbits_le32(&denali_pi[59],
1093                                 (0x1 << 8) | (0x3 << 16),
1094                                 (0x1 << 8) | (i << 16));
1095
1096                 /* Waiting for training complete */
1097                 while (1) {
1098                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1099                         tmp = readl(&denali_pi[174]) >> 8;
1100
1101                         /*
1102                          * check status obs, if error maybe can not
1103                          * get leveling done PHY_40/168/296/424
1104                          * phy_wrlvl_status_obs_X:0:13
1105                          */
1106                         obs_0 = readl(&denali_phy[40]);
1107                         obs_1 = readl(&denali_phy[168]);
1108                         obs_2 = readl(&denali_phy[296]);
1109                         obs_3 = readl(&denali_phy[424]);
1110                         if (((obs_0 >> 12) & 0x1) ||
1111                             ((obs_1 >> 12) & 0x1) ||
1112                             ((obs_2 >> 12) & 0x1) ||
1113                             ((obs_3 >> 12) & 0x1))
1114                                 obs_err = 1;
1115                         if ((((tmp >> 10) & 0x1) == 0x1) &&
1116                             (((tmp >> 13) & 0x1) == 0x1) &&
1117                             (((tmp >> 4) & 0x1) == 0x0) &&
1118                             obs_err == 0)
1119                                 break;
1120                         else if ((((tmp >> 4) & 0x1) == 0x1) ||
1121                                  (obs_err == 1))
1122                                 return -EIO;
1123                 }
1124
1125                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1126                 writel(0x00003f7c, (&denali_pi[175]));
1127         }
1128
1129         override_write_leveling_value(chan);
1130         clrbits_le32(&denali_pi[60], 0x3 << 8);
1131
1132         return 0;
1133 }
1134
1135 static int data_training_rg(const struct chan_info *chan, u32 channel,
1136                             const struct rk3399_sdram_params *params)
1137 {
1138         u32 *denali_pi = chan->pi->denali_pi;
1139         u32 *denali_phy = chan->publ->denali_phy;
1140         u32 i, tmp;
1141         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1142         u32 rank = params->ch[channel].cap_info.rank;
1143
1144         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1145         writel(0x00003f7c, (&denali_pi[175]));
1146
1147         for (i = 0; i < rank; i++) {
1148                 select_per_cs_training_index(chan, i);
1149
1150                 /* PI_80 PI_RDLVL_GATE_EN:RW:24:2 */
1151                 clrsetbits_le32(&denali_pi[80], 0x3 << 24, 0x2 << 24);
1152
1153                 /*
1154                  * PI_74 PI_RDLVL_GATE_REQ:WR:16:1
1155                  * PI_RDLVL_CS:RW:24:2
1156                  */
1157                 clrsetbits_le32(&denali_pi[74],
1158                                 (0x1 << 16) | (0x3 << 24),
1159                                 (0x1 << 16) | (i << 24));
1160
1161                 /* Waiting for training complete */
1162                 while (1) {
1163                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1164                         tmp = readl(&denali_pi[174]) >> 8;
1165
1166                         /*
1167                          * check status obs
1168                          * PHY_43/171/299/427
1169                          *     PHY_GTLVL_STATUS_OBS_x:16:8
1170                          */
1171                         obs_0 = readl(&denali_phy[43]);
1172                         obs_1 = readl(&denali_phy[171]);
1173                         obs_2 = readl(&denali_phy[299]);
1174                         obs_3 = readl(&denali_phy[427]);
1175                         if (((obs_0 >> (16 + 6)) & 0x3) ||
1176                             ((obs_1 >> (16 + 6)) & 0x3) ||
1177                             ((obs_2 >> (16 + 6)) & 0x3) ||
1178                             ((obs_3 >> (16 + 6)) & 0x3))
1179                                 obs_err = 1;
1180                         if ((((tmp >> 9) & 0x1) == 0x1) &&
1181                             (((tmp >> 13) & 0x1) == 0x1) &&
1182                             (((tmp >> 3) & 0x1) == 0x0) &&
1183                             obs_err == 0)
1184                                 break;
1185                         else if ((((tmp >> 3) & 0x1) == 0x1) ||
1186                                  (obs_err == 1))
1187                                 return -EIO;
1188                 }
1189
1190                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1191                 writel(0x00003f7c, (&denali_pi[175]));
1192         }
1193
1194         clrbits_le32(&denali_pi[80], 0x3 << 24);
1195
1196         return 0;
1197 }
1198
1199 static int data_training_rl(const struct chan_info *chan, u32 channel,
1200                             const struct rk3399_sdram_params *params)
1201 {
1202         u32 *denali_pi = chan->pi->denali_pi;
1203         u32 i, tmp;
1204         u32 rank = params->ch[channel].cap_info.rank;
1205
1206         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1207         writel(0x00003f7c, (&denali_pi[175]));
1208
1209         for (i = 0; i < rank; i++) {
1210                 select_per_cs_training_index(chan, i);
1211
1212                 /* PI_80 PI_RDLVL_EN:RW:16:2 */
1213                 clrsetbits_le32(&denali_pi[80], 0x3 << 16, 0x2 << 16);
1214
1215                 /* PI_74 PI_RDLVL_REQ:WR:8:1,PI_RDLVL_CS:RW:24:2 */
1216                 clrsetbits_le32(&denali_pi[74],
1217                                 (0x1 << 8) | (0x3 << 24),
1218                                 (0x1 << 8) | (i << 24));
1219
1220                 /* Waiting for training complete */
1221                 while (1) {
1222                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1223                         tmp = readl(&denali_pi[174]) >> 8;
1224
1225                         /*
1226                          * make sure status obs not report error bit
1227                          * PHY_46/174/302/430
1228                          *     phy_rdlvl_status_obs_X:16:8
1229                          */
1230                         if ((((tmp >> 8) & 0x1) == 0x1) &&
1231                             (((tmp >> 13) & 0x1) == 0x1) &&
1232                             (((tmp >> 2) & 0x1) == 0x0))
1233                                 break;
1234                         else if (((tmp >> 2) & 0x1) == 0x1)
1235                                 return -EIO;
1236                 }
1237
1238                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1239                 writel(0x00003f7c, (&denali_pi[175]));
1240         }
1241
1242         clrbits_le32(&denali_pi[80], 0x3 << 16);
1243
1244         return 0;
1245 }
1246
1247 static int data_training_wdql(const struct chan_info *chan, u32 channel,
1248                               const struct rk3399_sdram_params *params)
1249 {
1250         u32 *denali_pi = chan->pi->denali_pi;
1251         u32 i, tmp;
1252         u32 rank = params->ch[channel].cap_info.rank;
1253         u32 rank_mask;
1254
1255         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1256         writel(0x00003f7c, (&denali_pi[175]));
1257
1258         if (params->base.dramtype == LPDDR4)
1259                 rank_mask = (rank == 1) ? 0x5 : 0xf;
1260         else
1261                 rank_mask = (rank == 1) ? 0x1 : 0x3;
1262
1263         for (i = 0; i < 4; i++) {
1264                 if (!(rank_mask & (1 << i)))
1265                         continue;
1266
1267                 select_per_cs_training_index(chan, i);
1268
1269                 /*
1270                  * disable PI_WDQLVL_VREF_EN before wdq leveling?
1271                  * PI_181 PI_WDQLVL_VREF_EN:RW:8:1
1272                  */
1273                 clrbits_le32(&denali_pi[181], 0x1 << 8);
1274
1275                 /* PI_124 PI_WDQLVL_EN:RW:16:2 */
1276                 clrsetbits_le32(&denali_pi[124], 0x3 << 16, 0x2 << 16);
1277
1278                 /* PI_121 PI_WDQLVL_REQ:WR:8:1,PI_WDQLVL_CS:RW:16:2 */
1279                 clrsetbits_le32(&denali_pi[121],
1280                                 (0x1 << 8) | (0x3 << 16),
1281                                 (0x1 << 8) | (i << 16));
1282
1283                 /* Waiting for training complete */
1284                 while (1) {
1285                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1286                         tmp = readl(&denali_pi[174]) >> 8;
1287                         if ((((tmp >> 12) & 0x1) == 0x1) &&
1288                             (((tmp >> 13) & 0x1) == 0x1) &&
1289                             (((tmp >> 6) & 0x1) == 0x0))
1290                                 break;
1291                         else if (((tmp >> 6) & 0x1) == 0x1)
1292                                 return -EIO;
1293                 }
1294
1295                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1296                 writel(0x00003f7c, (&denali_pi[175]));
1297         }
1298
1299         clrbits_le32(&denali_pi[124], 0x3 << 16);
1300
1301         return 0;
1302 }
1303
1304 static int data_training(struct dram_info *dram, u32 channel,
1305                          const struct rk3399_sdram_params *params,
1306                          u32 training_flag)
1307 {
1308         struct chan_info *chan = &dram->chan[channel];
1309         u32 *denali_phy = chan->publ->denali_phy;
1310         int ret;
1311
1312         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1313         setbits_le32(&denali_phy[927], (1 << 22));
1314
1315         if (training_flag == PI_FULL_TRAINING) {
1316                 if (params->base.dramtype == LPDDR4) {
1317                         training_flag = PI_WRITE_LEVELING |
1318                                         PI_READ_GATE_TRAINING |
1319                                         PI_READ_LEVELING | PI_WDQ_LEVELING;
1320                 } else if (params->base.dramtype == LPDDR3) {
1321                         training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING |
1322                                         PI_READ_GATE_TRAINING;
1323                 } else if (params->base.dramtype == DDR3) {
1324                         training_flag = PI_WRITE_LEVELING |
1325                                         PI_READ_GATE_TRAINING |
1326                                         PI_READ_LEVELING;
1327                 }
1328         }
1329
1330         /* ca training(LPDDR4,LPDDR3 support) */
1331         if ((training_flag & PI_CA_TRAINING) == PI_CA_TRAINING) {
1332                 ret = data_training_ca(chan, channel, params);
1333                 if (ret < 0) {
1334                         debug("%s: data training ca failed\n", __func__);
1335                         return ret;
1336                 }
1337         }
1338
1339         /* write leveling(LPDDR4,LPDDR3,DDR3 support) */
1340         if ((training_flag & PI_WRITE_LEVELING) == PI_WRITE_LEVELING) {
1341                 ret = data_training_wl(chan, channel, params);
1342                 if (ret < 0) {
1343                         debug("%s: data training wl failed\n", __func__);
1344                         return ret;
1345                 }
1346         }
1347
1348         /* read gate training(LPDDR4,LPDDR3,DDR3 support) */
1349         if ((training_flag & PI_READ_GATE_TRAINING) == PI_READ_GATE_TRAINING) {
1350                 ret = data_training_rg(chan, channel, params);
1351                 if (ret < 0) {
1352                         debug("%s: data training rg failed\n", __func__);
1353                         return ret;
1354                 }
1355         }
1356
1357         /* read leveling(LPDDR4,LPDDR3,DDR3 support) */
1358         if ((training_flag & PI_READ_LEVELING) == PI_READ_LEVELING) {
1359                 ret = data_training_rl(chan, channel, params);
1360                 if (ret < 0) {
1361                         debug("%s: data training rl failed\n", __func__);
1362                         return ret;
1363                 }
1364         }
1365
1366         /* wdq leveling(LPDDR4 support) */
1367         if ((training_flag & PI_WDQ_LEVELING) == PI_WDQ_LEVELING) {
1368                 ret = data_training_wdql(chan, channel, params);
1369                 if (ret < 0) {
1370                         debug("%s: data training wdql failed\n", __func__);
1371                         return ret;
1372                 }
1373         }
1374
1375         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1376         clrbits_le32(&denali_phy[927], (1 << 22));
1377
1378         return 0;
1379 }
1380
1381 static void set_ddrconfig(const struct chan_info *chan,
1382                           const struct rk3399_sdram_params *params,
1383                           unsigned char channel, u32 ddrconfig)
1384 {
1385         /* only need to set ddrconfig */
1386         struct msch_regs *ddr_msch_regs = chan->msch;
1387         unsigned int cs0_cap = 0;
1388         unsigned int cs1_cap = 0;
1389
1390         cs0_cap = (1 << (params->ch[channel].cap_info.cs0_row
1391                         + params->ch[channel].cap_info.col
1392                         + params->ch[channel].cap_info.bk
1393                         + params->ch[channel].cap_info.bw - 20));
1394         if (params->ch[channel].cap_info.rank > 1)
1395                 cs1_cap = cs0_cap >> (params->ch[channel].cap_info.cs0_row
1396                                 - params->ch[channel].cap_info.cs1_row);
1397         if (params->ch[channel].cap_info.row_3_4) {
1398                 cs0_cap = cs0_cap * 3 / 4;
1399                 cs1_cap = cs1_cap * 3 / 4;
1400         }
1401
1402         writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf);
1403         writel(((cs0_cap / 32) & 0xff) | (((cs1_cap / 32) & 0xff) << 8),
1404                &ddr_msch_regs->ddrsize);
1405 }
1406
1407 static void sdram_msch_config(struct msch_regs *msch,
1408                               struct sdram_msch_timings *noc_timings)
1409 {
1410         writel(noc_timings->ddrtiminga0.d32,
1411                &msch->ddrtiminga0.d32);
1412         writel(noc_timings->ddrtimingb0.d32,
1413                &msch->ddrtimingb0.d32);
1414         writel(noc_timings->ddrtimingc0.d32,
1415                &msch->ddrtimingc0.d32);
1416         writel(noc_timings->devtodev0.d32,
1417                &msch->devtodev0.d32);
1418         writel(noc_timings->ddrmode.d32,
1419                &msch->ddrmode.d32);
1420 }
1421
1422 static void dram_all_config(struct dram_info *dram,
1423                             struct rk3399_sdram_params *params)
1424 {
1425         u32 sys_reg2 = 0;
1426         u32 sys_reg3 = 0;
1427         unsigned int channel, idx;
1428
1429         for (channel = 0, idx = 0;
1430              (idx < params->base.num_channels) && (channel < 2);
1431              channel++) {
1432                 struct msch_regs *ddr_msch_regs;
1433                 struct sdram_msch_timings *noc_timing;
1434
1435                 if (params->ch[channel].cap_info.col == 0)
1436                         continue;
1437                 idx++;
1438                 sdram_org_config(&params->ch[channel].cap_info,
1439                                  &params->base, &sys_reg2,
1440                                  &sys_reg3, channel);
1441                 ddr_msch_regs = dram->chan[channel].msch;
1442                 noc_timing = &params->ch[channel].noc_timings;
1443                 sdram_msch_config(ddr_msch_regs, noc_timing);
1444
1445                 /**
1446                  * rank 1 memory clock disable (dfi_dram_clk_disable = 1)
1447                  *
1448                  * The hardware for LPDDR4 with
1449                  * - CLK0P/N connect to lower 16-bits
1450                  * - CLK1P/N connect to higher 16-bits
1451                  *
1452                  * dfi dram clk is configured via CLK1P/N, so disabling
1453                  * dfi dram clk will disable the CLK1P/N as well for lpddr4.
1454                  */
1455                 if (params->ch[channel].cap_info.rank == 1 &&
1456                     params->base.dramtype != LPDDR4)
1457                         setbits_le32(&dram->chan[channel].pctl->denali_ctl[276],
1458                                      1 << 17);
1459         }
1460
1461         writel(sys_reg2, &dram->pmugrf->os_reg2);
1462         writel(sys_reg3, &dram->pmugrf->os_reg3);
1463         rk_clrsetreg(&dram->pmusgrf->soc_con4, 0x1f << 10,
1464                      params->base.stride << 10);
1465
1466         /* reboot hold register set */
1467         writel(PRESET_SGRF_HOLD(0) | PRESET_GPIO0_HOLD(1) |
1468                 PRESET_GPIO1_HOLD(1),
1469                 &dram->pmucru->pmucru_rstnhold_con[1]);
1470         clrsetbits_le32(&dram->cru->glb_rst_con, 0x3, 0x3);
1471 }
1472
1473 static void set_cap_relate_config(const struct chan_info *chan,
1474                                   struct rk3399_sdram_params *params,
1475                                   unsigned int channel)
1476 {
1477         u32 *denali_ctl = chan->pctl->denali_ctl;
1478         u32 tmp;
1479         struct sdram_msch_timings *noc_timing;
1480
1481         if (params->base.dramtype == LPDDR3) {
1482                 tmp = (8 << params->ch[channel].cap_info.bw) /
1483                         (8 << params->ch[channel].cap_info.dbw);
1484
1485                 /**
1486                  * memdata_ratio
1487                  * 1 -> 0, 2 -> 1, 4 -> 2
1488                  */
1489                 clrsetbits_le32(&denali_ctl[197], 0x7,
1490                                 (tmp >> 1));
1491                 clrsetbits_le32(&denali_ctl[198], 0x7 << 8,
1492                                 (tmp >> 1) << 8);
1493         }
1494
1495         noc_timing = &params->ch[channel].noc_timings;
1496
1497         /*
1498          * noc timing bw relate timing is 32 bit, and real bw is 16bit
1499          * actually noc reg is setting at function dram_all_config
1500          */
1501         if (params->ch[channel].cap_info.bw == 16 &&
1502             noc_timing->ddrmode.b.mwrsize == 2) {
1503                 if (noc_timing->ddrmode.b.burstsize)
1504                         noc_timing->ddrmode.b.burstsize -= 1;
1505                 noc_timing->ddrmode.b.mwrsize -= 1;
1506                 noc_timing->ddrtimingc0.b.burstpenalty *= 2;
1507                 noc_timing->ddrtimingc0.b.wrtomwr *= 2;
1508         }
1509 }
1510
1511 static u32 calculate_ddrconfig(struct rk3399_sdram_params *params, u32 channel)
1512 {
1513         unsigned int cs0_row = params->ch[channel].cap_info.cs0_row;
1514         unsigned int col = params->ch[channel].cap_info.col;
1515         unsigned int bw = params->ch[channel].cap_info.bw;
1516         u16  ddr_cfg_2_rbc[] = {
1517                 /*
1518                  * [6]    highest bit col
1519                  * [5:3]  max row(14+n)
1520                  * [2]    insertion row
1521                  * [1:0]  col(9+n),col, data bus 32bit
1522                  *
1523                  * highbitcol, max_row, insertion_row,  col
1524                  */
1525                 ((0 << 6) | (2 << 3) | (0 << 2) | 0), /* 0 */
1526                 ((0 << 6) | (2 << 3) | (0 << 2) | 1), /* 1 */
1527                 ((0 << 6) | (1 << 3) | (0 << 2) | 2), /* 2 */
1528                 ((0 << 6) | (0 << 3) | (0 << 2) | 3), /* 3 */
1529                 ((0 << 6) | (2 << 3) | (1 << 2) | 1), /* 4 */
1530                 ((0 << 6) | (1 << 3) | (1 << 2) | 2), /* 5 */
1531                 ((1 << 6) | (0 << 3) | (0 << 2) | 2), /* 6 */
1532                 ((1 << 6) | (1 << 3) | (0 << 2) | 2), /* 7 */
1533         };
1534         u32 i;
1535
1536         col -= (bw == 2) ? 0 : 1;
1537         col -= 9;
1538
1539         for (i = 0; i < 4; i++) {
1540                 if ((col == (ddr_cfg_2_rbc[i] & 0x3)) &&
1541                     (cs0_row <= (((ddr_cfg_2_rbc[i] >> 3) & 0x7) + 14)))
1542                         break;
1543         }
1544
1545         if (i >= 4)
1546                 i = -EINVAL;
1547
1548         return i;
1549 }
1550
1551 #if !defined(CONFIG_RAM_RK3399_LPDDR4)
1552 static int data_training_first(struct dram_info *dram, u32 channel, u8 rank,
1553                                struct rk3399_sdram_params *params)
1554 {
1555         u8 training_flag = PI_READ_GATE_TRAINING;
1556
1557         /*
1558          * LPDDR3 CA training msut be trigger before
1559          * other training.
1560          * DDR3 is not have CA training.
1561          */
1562
1563         if (params->base.dramtype == LPDDR3)
1564                 training_flag |= PI_CA_TRAINING;
1565
1566         return data_training(dram, channel, params, training_flag);
1567 }
1568
1569 static int switch_to_phy_index1(struct dram_info *dram,
1570                                 struct rk3399_sdram_params *params)
1571 {
1572         u32 channel;
1573         u32 *denali_phy;
1574         u32 ch_count = params->base.num_channels;
1575         int ret;
1576         int i = 0;
1577
1578         writel(RK_CLRSETBITS(0x03 << 4 | 1 << 2 | 1,
1579                              1 << 4 | 1 << 2 | 1),
1580                         &dram->cic->cic_ctrl0);
1581         while (!(readl(&dram->cic->cic_status0) & (1 << 2))) {
1582                 mdelay(10);
1583                 i++;
1584                 if (i > 10) {
1585                         debug("index1 frequency change overtime\n");
1586                         return -ETIME;
1587                 }
1588         }
1589
1590         i = 0;
1591         writel(RK_CLRSETBITS(1 << 1, 1 << 1), &dram->cic->cic_ctrl0);
1592         while (!(readl(&dram->cic->cic_status0) & (1 << 0))) {
1593                 mdelay(10);
1594                 i++;
1595                 if (i > 10) {
1596                         debug("index1 frequency done overtime\n");
1597                         return -ETIME;
1598                 }
1599         }
1600
1601         for (channel = 0; channel < ch_count; channel++) {
1602                 denali_phy = dram->chan[channel].publ->denali_phy;
1603                 clrsetbits_le32(&denali_phy[896], (0x3 << 8) | 1, 1 << 8);
1604                 ret = data_training(dram, channel, params, PI_FULL_TRAINING);
1605                 if (ret < 0) {
1606                         debug("index1 training failed\n");
1607                         return ret;
1608                 }
1609         }
1610
1611         return 0;
1612 }
1613
1614 #else
1615
1616 struct rk3399_sdram_params dfs_cfgs_lpddr4[] = {
1617 #include "sdram-rk3399-lpddr4-400.inc"
1618 #include "sdram-rk3399-lpddr4-800.inc"
1619 };
1620
1621 static void *get_denali_pi(const struct chan_info *chan,
1622                            struct rk3399_sdram_params *params, bool reg)
1623 {
1624         return reg ? &chan->pi->denali_pi : &params->pi_regs.denali_pi;
1625 }
1626
1627 static u32 lpddr4_get_phy_fn(struct rk3399_sdram_params *params, u32 ctl_fn)
1628 {
1629         u32 lpddr4_phy_fn[] = {1, 0, 0xb};
1630
1631         return lpddr4_phy_fn[ctl_fn];
1632 }
1633
1634 static u32 lpddr4_get_ctl_fn(struct rk3399_sdram_params *params, u32 phy_fn)
1635 {
1636         u32 lpddr4_ctl_fn[] = {1, 0, 2};
1637
1638         return lpddr4_ctl_fn[phy_fn];
1639 }
1640
1641 static u32 get_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf)
1642 {
1643         return ((readl(&pmusgrf->soc_con4) >> 10) & 0x1F);
1644 }
1645
1646 static void set_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf, u32 stride)
1647 {
1648         rk_clrsetreg(&pmusgrf->soc_con4, 0x1f << 10, stride << 10);
1649 }
1650
1651 /**
1652  * read mr_num mode register
1653  * rank = 1: cs0
1654  * rank = 2: cs1
1655  */
1656 static int read_mr(struct rk3399_ddr_pctl_regs *ddr_pctl_regs, u32 rank,
1657                    u32 mr_num, u32 *buf)
1658 {
1659         s32 timeout = 100;
1660
1661         writel(((1 << 16) | (((rank == 2) ? 1 : 0) << 8) | mr_num) << 8,
1662                &ddr_pctl_regs->denali_ctl[118]);
1663
1664         while (0 == (readl(&ddr_pctl_regs->denali_ctl[203]) &
1665                         ((1 << 21) | (1 << 12)))) {
1666                 udelay(1);
1667
1668                 if (timeout <= 0) {
1669                         printf("%s: pctl timeout!\n", __func__);
1670                         return -ETIMEDOUT;
1671                 }
1672
1673                 timeout--;
1674         }
1675
1676         if (!(readl(&ddr_pctl_regs->denali_ctl[203]) & (1 << 12))) {
1677                 *buf = readl(&ddr_pctl_regs->denali_ctl[119]) & 0xFF;
1678         } else {
1679                 printf("%s: read mr failed with 0x%x status\n", __func__,
1680                        readl(&ddr_pctl_regs->denali_ctl[17]) & 0x3);
1681                 *buf = 0;
1682         }
1683
1684         setbits_le32(&ddr_pctl_regs->denali_ctl[205], (1 << 21) | (1 << 12));
1685
1686         return 0;
1687 }
1688
1689 static int lpddr4_mr_detect(struct dram_info *dram, u32 channel, u8 rank,
1690                             struct rk3399_sdram_params *params)
1691 {
1692         u64 cs0_cap;
1693         u32 stride;
1694         u32 cs = 0, col = 0, bk = 0, bw = 0, row_3_4 = 0;
1695         u32 cs0_row = 0, cs1_row = 0, ddrconfig = 0;
1696         u32 mr5, mr12, mr14;
1697         struct chan_info *chan = &dram->chan[channel];
1698         struct rk3399_ddr_pctl_regs *ddr_pctl_regs = chan->pctl;
1699         void __iomem *addr = NULL;
1700         int ret = 0;
1701         u32 val;
1702
1703         stride = get_ddr_stride(dram->pmusgrf);
1704
1705         if (params->ch[channel].cap_info.col == 0) {
1706                 ret = -EPERM;
1707                 goto end;
1708         }
1709
1710         cs = params->ch[channel].cap_info.rank;
1711         col = params->ch[channel].cap_info.col;
1712         bk = params->ch[channel].cap_info.bk;
1713         bw = params->ch[channel].cap_info.bw;
1714         row_3_4 = params->ch[channel].cap_info.row_3_4;
1715         cs0_row = params->ch[channel].cap_info.cs0_row;
1716         cs1_row = params->ch[channel].cap_info.cs1_row;
1717         ddrconfig = params->ch[channel].cap_info.ddrconfig;
1718
1719         /* 2GB */
1720         params->ch[channel].cap_info.rank = 2;
1721         params->ch[channel].cap_info.col = 10;
1722         params->ch[channel].cap_info.bk = 3;
1723         params->ch[channel].cap_info.bw = 2;
1724         params->ch[channel].cap_info.row_3_4 = 0;
1725         params->ch[channel].cap_info.cs0_row = 15;
1726         params->ch[channel].cap_info.cs1_row = 15;
1727         params->ch[channel].cap_info.ddrconfig = 1;
1728
1729         set_memory_map(chan, channel, params);
1730         params->ch[channel].cap_info.ddrconfig =
1731                         calculate_ddrconfig(params, channel);
1732         set_ddrconfig(chan, params, channel,
1733                       params->ch[channel].cap_info.ddrconfig);
1734         set_cap_relate_config(chan, params, channel);
1735
1736         cs0_cap = (1 << (params->ch[channel].cap_info.bw
1737                         + params->ch[channel].cap_info.col
1738                         + params->ch[channel].cap_info.bk
1739                         + params->ch[channel].cap_info.cs0_row));
1740
1741         if (params->ch[channel].cap_info.row_3_4)
1742                 cs0_cap = cs0_cap * 3 / 4;
1743
1744         if (channel == 0)
1745                 set_ddr_stride(dram->pmusgrf, 0x17);
1746         else
1747                 set_ddr_stride(dram->pmusgrf, 0x18);
1748
1749         /* read and write data to DRAM, avoid be optimized by compiler. */
1750         if (rank == 1)
1751                 addr = (void __iomem *)0x100;
1752         else if (rank == 2)
1753                 addr = (void __iomem *)(cs0_cap + 0x100);
1754
1755         val = readl(addr);
1756         writel(val + 1, addr);
1757
1758         read_mr(ddr_pctl_regs, rank, 5, &mr5);
1759         read_mr(ddr_pctl_regs, rank, 12, &mr12);
1760         read_mr(ddr_pctl_regs, rank, 14, &mr14);
1761
1762         if (mr5 == 0 || mr12 != 0x4d || mr14 != 0x4d) {
1763                 ret = -EINVAL;
1764                 goto end;
1765         }
1766 end:
1767         params->ch[channel].cap_info.rank = cs;
1768         params->ch[channel].cap_info.col = col;
1769         params->ch[channel].cap_info.bk = bk;
1770         params->ch[channel].cap_info.bw = bw;
1771         params->ch[channel].cap_info.row_3_4 = row_3_4;
1772         params->ch[channel].cap_info.cs0_row = cs0_row;
1773         params->ch[channel].cap_info.cs1_row = cs1_row;
1774         params->ch[channel].cap_info.ddrconfig = ddrconfig;
1775
1776         set_ddr_stride(dram->pmusgrf, stride);
1777
1778         return ret;
1779 }
1780
1781 static void set_lpddr4_dq_odt(const struct chan_info *chan,
1782                               struct rk3399_sdram_params *params, u32 ctl_fn,
1783                               bool en, bool ctl_phy_reg, u32 mr5)
1784 {
1785         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1786         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1787         struct io_setting *io;
1788         u32 reg_value;
1789
1790         io = lpddr4_get_io_settings(params, mr5);
1791         if (en)
1792                 reg_value = io->dq_odt;
1793         else
1794                 reg_value = 0;
1795
1796         switch (ctl_fn) {
1797         case 0:
1798                 clrsetbits_le32(&denali_ctl[139], 0x7 << 24, reg_value << 24);
1799                 clrsetbits_le32(&denali_ctl[153], 0x7 << 24, reg_value << 24);
1800
1801                 clrsetbits_le32(&denali_pi[132], 0x7 << 0, (reg_value << 0));
1802                 clrsetbits_le32(&denali_pi[139], 0x7 << 16, (reg_value << 16));
1803                 clrsetbits_le32(&denali_pi[147], 0x7 << 0, (reg_value << 0));
1804                 clrsetbits_le32(&denali_pi[154], 0x7 << 16, (reg_value << 16));
1805                 break;
1806         case 1:
1807                 clrsetbits_le32(&denali_ctl[140], 0x7 << 0, reg_value << 0);
1808                 clrsetbits_le32(&denali_ctl[154], 0x7 << 0, reg_value << 0);
1809
1810                 clrsetbits_le32(&denali_pi[129], 0x7 << 16, (reg_value << 16));
1811                 clrsetbits_le32(&denali_pi[137], 0x7 << 0, (reg_value << 0));
1812                 clrsetbits_le32(&denali_pi[144], 0x7 << 16, (reg_value << 16));
1813                 clrsetbits_le32(&denali_pi[152], 0x7 << 0, (reg_value << 0));
1814                 break;
1815         case 2:
1816         default:
1817                 clrsetbits_le32(&denali_ctl[140], 0x7 << 8, (reg_value << 8));
1818                 clrsetbits_le32(&denali_ctl[154], 0x7 << 8, (reg_value << 8));
1819
1820                 clrsetbits_le32(&denali_pi[127], 0x7 << 0, (reg_value << 0));
1821                 clrsetbits_le32(&denali_pi[134], 0x7 << 16, (reg_value << 16));
1822                 clrsetbits_le32(&denali_pi[142], 0x7 << 0, (reg_value << 0));
1823                 clrsetbits_le32(&denali_pi[149], 0x7 << 16, (reg_value << 16));
1824                 break;
1825         }
1826 }
1827
1828 static void set_lpddr4_ca_odt(const struct chan_info *chan,
1829                               struct rk3399_sdram_params *params, u32 ctl_fn,
1830                               bool en, bool ctl_phy_reg, u32 mr5)
1831 {
1832         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1833         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1834         struct io_setting *io;
1835         u32 reg_value;
1836
1837         io = lpddr4_get_io_settings(params, mr5);
1838         if (en)
1839                 reg_value = io->ca_odt;
1840         else
1841                 reg_value = 0;
1842
1843         switch (ctl_fn) {
1844         case 0:
1845                 clrsetbits_le32(&denali_ctl[139], 0x7 << 28, reg_value << 28);
1846                 clrsetbits_le32(&denali_ctl[153], 0x7 << 28, reg_value << 28);
1847
1848                 clrsetbits_le32(&denali_pi[132], 0x7 << 4, reg_value << 4);
1849                 clrsetbits_le32(&denali_pi[139], 0x7 << 20, reg_value << 20);
1850                 clrsetbits_le32(&denali_pi[147], 0x7 << 4, reg_value << 4);
1851                 clrsetbits_le32(&denali_pi[154], 0x7 << 20, reg_value << 20);
1852                 break;
1853         case 1:
1854                 clrsetbits_le32(&denali_ctl[140], 0x7 << 4, reg_value << 4);
1855                 clrsetbits_le32(&denali_ctl[154], 0x7 << 4, reg_value << 4);
1856
1857                 clrsetbits_le32(&denali_pi[129], 0x7 << 20, reg_value << 20);
1858                 clrsetbits_le32(&denali_pi[137], 0x7 << 4, reg_value << 4);
1859                 clrsetbits_le32(&denali_pi[144], 0x7 << 20, reg_value << 20);
1860                 clrsetbits_le32(&denali_pi[152], 0x7 << 4, reg_value << 4);
1861                 break;
1862         case 2:
1863         default:
1864                 clrsetbits_le32(&denali_ctl[140], 0x7 << 12, (reg_value << 12));
1865                 clrsetbits_le32(&denali_ctl[154], 0x7 << 12, (reg_value << 12));
1866
1867                 clrsetbits_le32(&denali_pi[127], 0x7 << 4, reg_value << 4);
1868                 clrsetbits_le32(&denali_pi[134], 0x7 << 20, reg_value << 20);
1869                 clrsetbits_le32(&denali_pi[142], 0x7 << 4, reg_value << 4);
1870                 clrsetbits_le32(&denali_pi[149], 0x7 << 20, reg_value << 20);
1871                 break;
1872         }
1873 }
1874
1875 static void set_lpddr4_MR3(const struct chan_info *chan,
1876                            struct rk3399_sdram_params *params, u32 ctl_fn,
1877                            bool ctl_phy_reg, u32 mr5)
1878 {
1879         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1880         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1881         struct io_setting *io;
1882         u32 reg_value;
1883
1884         io = lpddr4_get_io_settings(params, mr5);
1885
1886         reg_value = ((io->pdds << 3) | 1);
1887
1888         switch (ctl_fn) {
1889         case 0:
1890                 clrsetbits_le32(&denali_ctl[138], 0xFFFF, reg_value);
1891                 clrsetbits_le32(&denali_ctl[152], 0xFFFF, reg_value);
1892
1893                 clrsetbits_le32(&denali_pi[131], 0xFFFF << 16, reg_value << 16);
1894                 clrsetbits_le32(&denali_pi[139], 0xFFFF, reg_value);
1895                 clrsetbits_le32(&denali_pi[146], 0xFFFF << 16, reg_value << 16);
1896                 clrsetbits_le32(&denali_pi[154], 0xFFFF, reg_value);
1897                 break;
1898         case 1:
1899                 clrsetbits_le32(&denali_ctl[138], 0xFFFF << 16,
1900                                 reg_value << 16);
1901                 clrsetbits_le32(&denali_ctl[152], 0xFFFF << 16,
1902                                 reg_value << 16);
1903
1904                 clrsetbits_le32(&denali_pi[129], 0xFFFF, reg_value);
1905                 clrsetbits_le32(&denali_pi[136], 0xFFFF << 16, reg_value << 16);
1906                 clrsetbits_le32(&denali_pi[144], 0xFFFF, reg_value);
1907                 clrsetbits_le32(&denali_pi[151], 0xFFFF << 16, reg_value << 16);
1908                 break;
1909         case 2:
1910         default:
1911                 clrsetbits_le32(&denali_ctl[139], 0xFFFF, reg_value);
1912                 clrsetbits_le32(&denali_ctl[153], 0xFFFF, reg_value);
1913
1914                 clrsetbits_le32(&denali_pi[126], 0xFFFF << 16, reg_value << 16);
1915                 clrsetbits_le32(&denali_pi[134], 0xFFFF, reg_value);
1916                 clrsetbits_le32(&denali_pi[141], 0xFFFF << 16, reg_value << 16);
1917                 clrsetbits_le32(&denali_pi[149], 0xFFFF, reg_value);
1918                 break;
1919         }
1920 }
1921
1922 static void set_lpddr4_MR12(const struct chan_info *chan,
1923                             struct rk3399_sdram_params *params, u32 ctl_fn,
1924                             bool ctl_phy_reg, u32 mr5)
1925 {
1926         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1927         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1928         struct io_setting *io;
1929         u32 reg_value;
1930
1931         io = lpddr4_get_io_settings(params, mr5);
1932
1933         reg_value = io->ca_vref;
1934
1935         switch (ctl_fn) {
1936         case 0:
1937                 clrsetbits_le32(&denali_ctl[140], 0xFFFF << 16,
1938                                 reg_value << 16);
1939                 clrsetbits_le32(&denali_ctl[154], 0xFFFF << 16,
1940                                 reg_value << 16);
1941
1942                 clrsetbits_le32(&denali_pi[132], 0xFF << 8, reg_value << 8);
1943                 clrsetbits_le32(&denali_pi[139], 0xFF << 24, reg_value << 24);
1944                 clrsetbits_le32(&denali_pi[147], 0xFF << 8, reg_value << 8);
1945                 clrsetbits_le32(&denali_pi[154], 0xFF << 24, reg_value << 24);
1946                 break;
1947         case 1:
1948                 clrsetbits_le32(&denali_ctl[141], 0xFFFF, reg_value);
1949                 clrsetbits_le32(&denali_ctl[155], 0xFFFF, reg_value);
1950
1951                 clrsetbits_le32(&denali_pi[129], 0xFF << 24, reg_value << 24);
1952                 clrsetbits_le32(&denali_pi[137], 0xFF << 8, reg_value << 8);
1953                 clrsetbits_le32(&denali_pi[144], 0xFF << 24, reg_value << 24);
1954                 clrsetbits_le32(&denali_pi[152], 0xFF << 8, reg_value << 8);
1955                 break;
1956         case 2:
1957         default:
1958                 clrsetbits_le32(&denali_ctl[141], 0xFFFF << 16,
1959                                 reg_value << 16);
1960                 clrsetbits_le32(&denali_ctl[155], 0xFFFF << 16,
1961                                 reg_value << 16);
1962
1963                 clrsetbits_le32(&denali_pi[127], 0xFF << 8, reg_value << 8);
1964                 clrsetbits_le32(&denali_pi[134], 0xFF << 24, reg_value << 24);
1965                 clrsetbits_le32(&denali_pi[142], 0xFF << 8, reg_value << 8);
1966                 clrsetbits_le32(&denali_pi[149], 0xFF << 24, reg_value << 24);
1967                 break;
1968         }
1969 }
1970
1971 static void set_lpddr4_MR14(const struct chan_info *chan,
1972                             struct rk3399_sdram_params *params, u32 ctl_fn,
1973                             bool ctl_phy_reg, u32 mr5)
1974 {
1975         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1976         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1977         struct io_setting *io;
1978         u32 reg_value;
1979
1980         io = lpddr4_get_io_settings(params, mr5);
1981
1982         reg_value = io->dq_vref;
1983
1984         switch (ctl_fn) {
1985         case 0:
1986                 clrsetbits_le32(&denali_ctl[142], 0xFFFF << 16,
1987                                 reg_value << 16);
1988                 clrsetbits_le32(&denali_ctl[156], 0xFFFF << 16,
1989                                 reg_value << 16);
1990
1991                 clrsetbits_le32(&denali_pi[132], 0xFF << 16, reg_value << 16);
1992                 clrsetbits_le32(&denali_pi[140], 0xFF << 0, reg_value << 0);
1993                 clrsetbits_le32(&denali_pi[147], 0xFF << 16, reg_value << 16);
1994                 clrsetbits_le32(&denali_pi[155], 0xFF << 0, reg_value << 0);
1995                 break;
1996         case 1:
1997                 clrsetbits_le32(&denali_ctl[143], 0xFFFF, reg_value);
1998                 clrsetbits_le32(&denali_ctl[157], 0xFFFF, reg_value);
1999
2000                 clrsetbits_le32(&denali_pi[130], 0xFF << 0, reg_value << 0);
2001                 clrsetbits_le32(&denali_pi[137], 0xFF << 16, reg_value << 16);
2002                 clrsetbits_le32(&denali_pi[145], 0xFF << 0, reg_value << 0);
2003                 clrsetbits_le32(&denali_pi[152], 0xFF << 16, reg_value << 16);
2004                 break;
2005         case 2:
2006         default:
2007                 clrsetbits_le32(&denali_ctl[143], 0xFFFF << 16,
2008                                 reg_value << 16);
2009                 clrsetbits_le32(&denali_ctl[157], 0xFFFF << 16,
2010                                 reg_value << 16);
2011
2012                 clrsetbits_le32(&denali_pi[127], 0xFF << 16, reg_value << 16);
2013                 clrsetbits_le32(&denali_pi[135], 0xFF << 0, reg_value << 0);
2014                 clrsetbits_le32(&denali_pi[142], 0xFF << 16, reg_value << 16);
2015                 clrsetbits_le32(&denali_pi[150], 0xFF << 0, reg_value << 0);
2016                 break;
2017         }
2018 }
2019
2020 static void lpddr4_copy_phy(struct dram_info *dram,
2021                             struct rk3399_sdram_params *params, u32 phy_fn,
2022                             struct rk3399_sdram_params *params_cfg,
2023                             u32 channel)
2024 {
2025         u32 *denali_ctl, *denali_phy;
2026         u32 *denali_phy_params;
2027         u32 speed = 0;
2028         u32 ctl_fn, mr5;
2029
2030         denali_ctl = dram->chan[channel].pctl->denali_ctl;
2031         denali_phy = dram->chan[channel].publ->denali_phy;
2032         denali_phy_params = params_cfg->phy_regs.denali_phy;
2033
2034         /* switch index */
2035         clrsetbits_le32(&denali_phy_params[896], 0x3 << 8,
2036                         phy_fn << 8);
2037         writel(denali_phy_params[896], &denali_phy[896]);
2038
2039         /* phy_pll_ctrl_ca, phy_pll_ctrl */
2040         writel(denali_phy_params[911], &denali_phy[911]);
2041
2042         /* phy_low_freq_sel */
2043         clrsetbits_le32(&denali_phy[913], 0x1,
2044                         denali_phy_params[913] & 0x1);
2045
2046         /* phy_grp_slave_delay_x, phy_cslvl_dly_step */
2047         writel(denali_phy_params[916], &denali_phy[916]);
2048         writel(denali_phy_params[917], &denali_phy[917]);
2049         writel(denali_phy_params[918], &denali_phy[918]);
2050
2051         /* phy_adrz_sw_wraddr_shift_x  */
2052         writel(denali_phy_params[512], &denali_phy[512]);
2053         clrsetbits_le32(&denali_phy[513], 0xffff,
2054                         denali_phy_params[513] & 0xffff);
2055         writel(denali_phy_params[640], &denali_phy[640]);
2056         clrsetbits_le32(&denali_phy[641], 0xffff,
2057                         denali_phy_params[641] & 0xffff);
2058         writel(denali_phy_params[768], &denali_phy[768]);
2059         clrsetbits_le32(&denali_phy[769], 0xffff,
2060                         denali_phy_params[769] & 0xffff);
2061
2062         writel(denali_phy_params[544], &denali_phy[544]);
2063         writel(denali_phy_params[545], &denali_phy[545]);
2064         writel(denali_phy_params[546], &denali_phy[546]);
2065         writel(denali_phy_params[547], &denali_phy[547]);
2066
2067         writel(denali_phy_params[672], &denali_phy[672]);
2068         writel(denali_phy_params[673], &denali_phy[673]);
2069         writel(denali_phy_params[674], &denali_phy[674]);
2070         writel(denali_phy_params[675], &denali_phy[675]);
2071
2072         writel(denali_phy_params[800], &denali_phy[800]);
2073         writel(denali_phy_params[801], &denali_phy[801]);
2074         writel(denali_phy_params[802], &denali_phy[802]);
2075         writel(denali_phy_params[803], &denali_phy[803]);
2076
2077         /*
2078          * phy_adr_master_delay_start_x
2079          * phy_adr_master_delay_step_x
2080          * phy_adr_master_delay_wait_x
2081          */
2082         writel(denali_phy_params[548], &denali_phy[548]);
2083         writel(denali_phy_params[676], &denali_phy[676]);
2084         writel(denali_phy_params[804], &denali_phy[804]);
2085
2086         /* phy_adr_calvl_dly_step_x */
2087         writel(denali_phy_params[549], &denali_phy[549]);
2088         writel(denali_phy_params[677], &denali_phy[677]);
2089         writel(denali_phy_params[805], &denali_phy[805]);
2090
2091         /*
2092          * phy_clk_wrdm_slave_delay_x
2093          * phy_clk_wrdqz_slave_delay_x
2094          * phy_clk_wrdqs_slave_delay_x
2095          */
2096         sdram_copy_to_reg((u32 *)&denali_phy[59],
2097                           (u32 *)&denali_phy_params[59], (63 - 58) * 4);
2098         sdram_copy_to_reg((u32 *)&denali_phy[187],
2099                           (u32 *)&denali_phy_params[187], (191 - 186) * 4);
2100         sdram_copy_to_reg((u32 *)&denali_phy[315],
2101                           (u32 *)&denali_phy_params[315], (319 - 314) * 4);
2102         sdram_copy_to_reg((u32 *)&denali_phy[443],
2103                           (u32 *)&denali_phy_params[443], (447 - 442) * 4);
2104
2105         /*
2106          * phy_dqs_tsel_wr_timing_x 8bits denali_phy_84/212/340/468 offset_8
2107          * dqs_tsel_wr_end[7:4] add half cycle
2108          * phy_dq_tsel_wr_timing_x 8bits denali_phy_83/211/339/467 offset_8
2109          * dq_tsel_wr_end[7:4] add half cycle
2110          */
2111         writel(denali_phy_params[83] + (0x10 << 16), &denali_phy[83]);
2112         writel(denali_phy_params[84] + (0x10 << 8), &denali_phy[84]);
2113         writel(denali_phy_params[85], &denali_phy[85]);
2114
2115         writel(denali_phy_params[211] + (0x10 << 16), &denali_phy[211]);
2116         writel(denali_phy_params[212] + (0x10 << 8), &denali_phy[212]);
2117         writel(denali_phy_params[213], &denali_phy[213]);
2118
2119         writel(denali_phy_params[339] + (0x10 << 16), &denali_phy[339]);
2120         writel(denali_phy_params[340] + (0x10 << 8), &denali_phy[340]);
2121         writel(denali_phy_params[341], &denali_phy[341]);
2122
2123         writel(denali_phy_params[467] + (0x10 << 16), &denali_phy[467]);
2124         writel(denali_phy_params[468] + (0x10 << 8), &denali_phy[468]);
2125         writel(denali_phy_params[469], &denali_phy[469]);
2126
2127         /*
2128          * phy_gtlvl_resp_wait_cnt_x
2129          * phy_gtlvl_dly_step_x
2130          * phy_wrlvl_resp_wait_cnt_x
2131          * phy_gtlvl_final_step_x
2132          * phy_gtlvl_back_step_x
2133          * phy_rdlvl_dly_step_x
2134          *
2135          * phy_master_delay_step_x
2136          * phy_master_delay_wait_x
2137          * phy_wrlvl_dly_step_x
2138          * phy_rptr_update_x
2139          * phy_wdqlvl_dly_step_x
2140          */
2141         writel(denali_phy_params[87], &denali_phy[87]);
2142         writel(denali_phy_params[88], &denali_phy[88]);
2143         writel(denali_phy_params[89], &denali_phy[89]);
2144         writel(denali_phy_params[90], &denali_phy[90]);
2145
2146         writel(denali_phy_params[215], &denali_phy[215]);
2147         writel(denali_phy_params[216], &denali_phy[216]);
2148         writel(denali_phy_params[217], &denali_phy[217]);
2149         writel(denali_phy_params[218], &denali_phy[218]);
2150
2151         writel(denali_phy_params[343], &denali_phy[343]);
2152         writel(denali_phy_params[344], &denali_phy[344]);
2153         writel(denali_phy_params[345], &denali_phy[345]);
2154         writel(denali_phy_params[346], &denali_phy[346]);
2155
2156         writel(denali_phy_params[471], &denali_phy[471]);
2157         writel(denali_phy_params[472], &denali_phy[472]);
2158         writel(denali_phy_params[473], &denali_phy[473]);
2159         writel(denali_phy_params[474], &denali_phy[474]);
2160
2161         /*
2162          * phy_gtlvl_lat_adj_start_x
2163          * phy_gtlvl_rddqs_slv_dly_start_x
2164          * phy_rdlvl_rddqs_dq_slv_dly_start_x
2165          * phy_wdqlvl_dqdm_slv_dly_start_x
2166          */
2167         writel(denali_phy_params[80], &denali_phy[80]);
2168         writel(denali_phy_params[81], &denali_phy[81]);
2169
2170         writel(denali_phy_params[208], &denali_phy[208]);
2171         writel(denali_phy_params[209], &denali_phy[209]);
2172
2173         writel(denali_phy_params[336], &denali_phy[336]);
2174         writel(denali_phy_params[337], &denali_phy[337]);
2175
2176         writel(denali_phy_params[464], &denali_phy[464]);
2177         writel(denali_phy_params[465], &denali_phy[465]);
2178
2179         /*
2180          * phy_master_delay_start_x
2181          * phy_sw_master_mode_x
2182          * phy_rddata_en_tsel_dly_x
2183          */
2184         writel(denali_phy_params[86], &denali_phy[86]);
2185         writel(denali_phy_params[214], &denali_phy[214]);
2186         writel(denali_phy_params[342], &denali_phy[342]);
2187         writel(denali_phy_params[470], &denali_phy[470]);
2188
2189         /*
2190          * phy_rddqz_slave_delay_x
2191          * phy_rddqs_dqz_fall_slave_delay_x
2192          * phy_rddqs_dqz_rise_slave_delay_x
2193          * phy_rddqs_dm_fall_slave_delay_x
2194          * phy_rddqs_dm_rise_slave_delay_x
2195          * phy_rddqs_gate_slave_delay_x
2196          * phy_wrlvl_delay_early_threshold_x
2197          * phy_write_path_lat_add_x
2198          * phy_rddqs_latency_adjust_x
2199          * phy_wrlvl_delay_period_threshold_x
2200          * phy_wrlvl_early_force_zero_x
2201          */
2202         sdram_copy_to_reg((u32 *)&denali_phy[64],
2203                           (u32 *)&denali_phy_params[64], (67 - 63) * 4);
2204         clrsetbits_le32(&denali_phy[68], 0xfffffc00,
2205                         denali_phy_params[68] & 0xfffffc00);
2206         sdram_copy_to_reg((u32 *)&denali_phy[69],
2207                           (u32 *)&denali_phy_params[69], (79 - 68) * 4);
2208         sdram_copy_to_reg((u32 *)&denali_phy[192],
2209                           (u32 *)&denali_phy_params[192], (195 - 191) * 4);
2210         clrsetbits_le32(&denali_phy[196], 0xfffffc00,
2211                         denali_phy_params[196] & 0xfffffc00);
2212         sdram_copy_to_reg((u32 *)&denali_phy[197],
2213                           (u32 *)&denali_phy_params[197], (207 - 196) * 4);
2214         sdram_copy_to_reg((u32 *)&denali_phy[320],
2215                           (u32 *)&denali_phy_params[320], (323 - 319) * 4);
2216         clrsetbits_le32(&denali_phy[324], 0xfffffc00,
2217                         denali_phy_params[324] & 0xfffffc00);
2218         sdram_copy_to_reg((u32 *)&denali_phy[325],
2219                           (u32 *)&denali_phy_params[325], (335 - 324) * 4);
2220         sdram_copy_to_reg((u32 *)&denali_phy[448],
2221                           (u32 *)&denali_phy_params[448], (451 - 447) * 4);
2222         clrsetbits_le32(&denali_phy[452], 0xfffffc00,
2223                         denali_phy_params[452] & 0xfffffc00);
2224         sdram_copy_to_reg((u32 *)&denali_phy[453],
2225                           (u32 *)&denali_phy_params[453], (463 - 452) * 4);
2226
2227         /* phy_two_cyc_preamble_x */
2228         clrsetbits_le32(&denali_phy[7], 0x3 << 24,
2229                         denali_phy_params[7] & (0x3 << 24));
2230         clrsetbits_le32(&denali_phy[135], 0x3 << 24,
2231                         denali_phy_params[135] & (0x3 << 24));
2232         clrsetbits_le32(&denali_phy[263], 0x3 << 24,
2233                         denali_phy_params[263] & (0x3 << 24));
2234         clrsetbits_le32(&denali_phy[391], 0x3 << 24,
2235                         denali_phy_params[391] & (0x3 << 24));
2236
2237         /* speed */
2238         if (params_cfg->base.ddr_freq < 400)
2239                 speed = 0x0;
2240         else if (params_cfg->base.ddr_freq < 800)
2241                 speed = 0x1;
2242         else if (params_cfg->base.ddr_freq < 1200)
2243                 speed = 0x2;
2244
2245         /* phy_924 phy_pad_fdbk_drive */
2246         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
2247         /* phy_926 phy_pad_data_drive */
2248         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
2249         /* phy_927 phy_pad_dqs_drive */
2250         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
2251         /* phy_928 phy_pad_addr_drive */
2252         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
2253         /* phy_929 phy_pad_clk_drive */
2254         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
2255         /* phy_935 phy_pad_cke_drive */
2256         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
2257         /* phy_937 phy_pad_rst_drive */
2258         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
2259         /* phy_939 phy_pad_cs_drive */
2260         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
2261
2262         read_mr(dram->chan[channel].pctl, 1, 5, &mr5);
2263         set_ds_odt(&dram->chan[channel], params_cfg, true, mr5);
2264
2265         ctl_fn = lpddr4_get_ctl_fn(params_cfg, phy_fn);
2266         set_lpddr4_dq_odt(&dram->chan[channel], params_cfg,
2267                           ctl_fn, true, true, mr5);
2268         set_lpddr4_ca_odt(&dram->chan[channel], params_cfg,
2269                           ctl_fn, true, true, mr5);
2270         set_lpddr4_MR3(&dram->chan[channel], params_cfg,
2271                        ctl_fn, true, mr5);
2272         set_lpddr4_MR12(&dram->chan[channel], params_cfg,
2273                         ctl_fn, true, mr5);
2274         set_lpddr4_MR14(&dram->chan[channel], params_cfg,
2275                         ctl_fn, true, mr5);
2276
2277         /*
2278          * if phy_sw_master_mode_x not bypass mode,
2279          * clear phy_slice_pwr_rdc_disable.
2280          * note: need use timings, not ddr_publ_regs
2281          */
2282         if (!((denali_phy_params[86] >> 8) & (1 << 2))) {
2283                 clrbits_le32(&denali_phy[10], 1 << 16);
2284                 clrbits_le32(&denali_phy[138], 1 << 16);
2285                 clrbits_le32(&denali_phy[266], 1 << 16);
2286                 clrbits_le32(&denali_phy[394], 1 << 16);
2287         }
2288
2289         /*
2290          * when PHY_PER_CS_TRAINING_EN=1, W2W_DIFFCS_DLY_Fx can't
2291          * smaller than 8
2292          * NOTE: need use timings, not ddr_publ_regs
2293          */
2294         if ((denali_phy_params[84] >> 16) & 1) {
2295                 if (((readl(&denali_ctl[217 + ctl_fn]) >>
2296                         16) & 0x1f) < 8)
2297                         clrsetbits_le32(&denali_ctl[217 + ctl_fn],
2298                                         0x1f << 16,
2299                                         8 << 16);
2300         }
2301 }
2302
2303 static void lpddr4_set_phy(struct dram_info *dram,
2304                            struct rk3399_sdram_params *params, u32 phy_fn,
2305                            struct rk3399_sdram_params *params_cfg)
2306 {
2307         u32 channel;
2308
2309         for (channel = 0; channel < 2; channel++)
2310                 lpddr4_copy_phy(dram, params, phy_fn, params_cfg,
2311                                 channel);
2312 }
2313
2314 static int lpddr4_set_ctl(struct dram_info *dram,
2315                           struct rk3399_sdram_params *params,
2316                           u32 fn, u32 hz)
2317 {
2318         u32 channel;
2319         int ret_clk, ret;
2320
2321         /* cci idle req stall */
2322         writel(0x70007, &dram->grf->soc_con0);
2323
2324         /* enable all clk */
2325         setbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));
2326
2327         /* idle */
2328         setbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));
2329         while ((readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))
2330                != (0x3 << 18))
2331                 ;
2332
2333         /* change freq */
2334         writel((((0x3 << 4) | (1 << 2) | 1) << 16) |
2335                 (fn << 4) | (1 << 2) | 1, &dram->cic->cic_ctrl0);
2336         while (!(readl(&dram->cic->cic_status0) & (1 << 2)))
2337                 ;
2338
2339         ret_clk = clk_set_rate(&dram->ddr_clk, hz);
2340         if (ret_clk < 0) {
2341                 printf("%s clk set failed %d\n", __func__, ret_clk);
2342                 return ret_clk;
2343         }
2344
2345         writel(0x20002, &dram->cic->cic_ctrl0);
2346         while (!(readl(&dram->cic->cic_status0) & (1 << 0)))
2347                 ;
2348
2349         /* deidle */
2350         clrbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));
2351         while (readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))
2352                 ;
2353
2354         /* clear enable all clk */
2355         clrbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));
2356
2357         /* lpddr4 ctl2 can not do training, all training will fail */
2358         if (!(params->base.dramtype == LPDDR4 && fn == 2)) {
2359                 for (channel = 0; channel < 2; channel++) {
2360                         if (!(params->ch[channel].cap_info.col))
2361                                 continue;
2362                         ret = data_training(dram, channel, params,
2363                                             PI_FULL_TRAINING);
2364                         if (ret)
2365                                 printf("%s: channel %d training failed!\n",
2366                                        __func__, channel);
2367                         else
2368                                 debug("%s: channel %d training pass\n",
2369                                       __func__, channel);
2370                 }
2371         }
2372
2373         return 0;
2374 }
2375
2376 static int lpddr4_set_rate(struct dram_info *dram,
2377                            struct rk3399_sdram_params *params)
2378 {
2379         u32 ctl_fn;
2380         u32 phy_fn;
2381
2382         for (ctl_fn = 0; ctl_fn < 2; ctl_fn++) {
2383                 phy_fn = lpddr4_get_phy_fn(params, ctl_fn);
2384
2385                 lpddr4_set_phy(dram, params, phy_fn, &dfs_cfgs_lpddr4[ctl_fn]);
2386                 lpddr4_set_ctl(dram, params, ctl_fn,
2387                                dfs_cfgs_lpddr4[ctl_fn].base.ddr_freq);
2388
2389                 printf("%s: change freq to %d mhz %d, %d\n", __func__,
2390                        dfs_cfgs_lpddr4[ctl_fn].base.ddr_freq, ctl_fn, phy_fn);
2391         }
2392
2393         return 0;
2394 }
2395 #endif /* CONFIG_RAM_RK3399_LPDDR4 */
2396
2397 static unsigned char calculate_stride(struct rk3399_sdram_params *params)
2398 {
2399         unsigned int stride = params->base.stride;
2400         unsigned int channel, chinfo = 0;
2401         unsigned int ch_cap[2] = {0, 0};
2402         u64 cap;
2403
2404         for (channel = 0; channel < 2; channel++) {
2405                 unsigned int cs0_cap = 0;
2406                 unsigned int cs1_cap = 0;
2407                 struct sdram_cap_info *cap_info = &params->ch[channel].cap_info;
2408
2409                 if (cap_info->col == 0)
2410                         continue;
2411
2412                 cs0_cap = (1 << (cap_info->cs0_row + cap_info->col +
2413                                  cap_info->bk + cap_info->bw - 20));
2414                 if (cap_info->rank > 1)
2415                         cs1_cap = cs0_cap >> (cap_info->cs0_row
2416                                               - cap_info->cs1_row);
2417                 if (cap_info->row_3_4) {
2418                         cs0_cap = cs0_cap * 3 / 4;
2419                         cs1_cap = cs1_cap * 3 / 4;
2420                 }
2421                 ch_cap[channel] = cs0_cap + cs1_cap;
2422                 chinfo |= 1 << channel;
2423         }
2424
2425         /* stride calculation for 1 channel */
2426         if (params->base.num_channels == 1 && chinfo & 1)
2427                 return 0x17;    /* channel a */
2428
2429         /* stride calculation for 2 channels, default gstride type is 256B */
2430         if (ch_cap[0] == ch_cap[1]) {
2431                 cap = ch_cap[0] + ch_cap[1];
2432                 switch (cap) {
2433                 /* 512MB */
2434                 case 512:
2435                         stride = 0;
2436                         break;
2437                 /* 1GB */
2438                 case 1024:
2439                         stride = 0x5;
2440                         break;
2441                 /*
2442                  * 768MB + 768MB same as total 2GB memory
2443                  * useful space: 0-768MB 1GB-1792MB
2444                  */
2445                 case 1536:
2446                 /* 2GB */
2447                 case 2048:
2448                         stride = 0x9;
2449                         break;
2450                 /* 1536MB + 1536MB */
2451                 case 3072:
2452                         stride = 0x11;
2453                         break;
2454                 /* 4GB */
2455                 case 4096:
2456                         stride = 0xD;
2457                         break;
2458                 default:
2459                         printf("%s: Unable to calculate stride for ", __func__);
2460                         print_size((cap * (1 << 20)), " capacity\n");
2461                         break;
2462                 }
2463         }
2464
2465         sdram_print_stride(stride);
2466
2467         return stride;
2468 }
2469
2470 static void clear_channel_params(struct rk3399_sdram_params *params, u8 channel)
2471 {
2472         params->ch[channel].cap_info.rank = 0;
2473         params->ch[channel].cap_info.col = 0;
2474         params->ch[channel].cap_info.bk = 0;
2475         params->ch[channel].cap_info.bw = 32;
2476         params->ch[channel].cap_info.dbw = 32;
2477         params->ch[channel].cap_info.row_3_4 = 0;
2478         params->ch[channel].cap_info.cs0_row = 0;
2479         params->ch[channel].cap_info.cs1_row = 0;
2480         params->ch[channel].cap_info.ddrconfig = 0;
2481 }
2482
2483 static int pctl_init(struct dram_info *dram, struct rk3399_sdram_params *params)
2484 {
2485         int channel;
2486         int ret;
2487
2488         for (channel = 0; channel < 2; channel++) {
2489                 const struct chan_info *chan = &dram->chan[channel];
2490                 struct rk3399_cru *cru = dram->cru;
2491                 struct rk3399_ddr_publ_regs *publ = chan->publ;
2492
2493                 phy_pctrl_reset(cru, channel);
2494                 phy_dll_bypass_set(publ, params->base.ddr_freq);
2495
2496                 ret = pctl_cfg(dram, chan, channel, params);
2497                 if (ret < 0) {
2498                         printf("%s: pctl config failed\n", __func__);
2499                         return ret;
2500                 }
2501
2502                 /* start to trigger initialization */
2503                 pctl_start(dram, channel);
2504         }
2505
2506         return 0;
2507 }
2508
2509 static int sdram_init(struct dram_info *dram,
2510                       struct rk3399_sdram_params *params)
2511 {
2512         unsigned char dramtype = params->base.dramtype;
2513         unsigned int ddr_freq = params->base.ddr_freq;
2514         int channel, ch, rank;
2515         int ret;
2516
2517         debug("Starting SDRAM initialization...\n");
2518
2519         if ((dramtype == DDR3 && ddr_freq > 933) ||
2520             (dramtype == LPDDR3 && ddr_freq > 933) ||
2521             (dramtype == LPDDR4 && ddr_freq > 800)) {
2522                 debug("SDRAM frequency is to high!");
2523                 return -E2BIG;
2524         }
2525
2526         for (ch = 0; ch < 2; ch++) {
2527                 params->ch[ch].cap_info.rank = 2;
2528                 for (rank = 2; rank != 0; rank--) {
2529                         ret = pctl_init(dram, params);
2530                         if (ret < 0) {
2531                                 printf("%s: pctl init failed\n", __func__);
2532                                 return ret;
2533                         }
2534
2535                         /* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */
2536                         if (dramtype == LPDDR3)
2537                                 udelay(10);
2538
2539                         params->ch[ch].cap_info.rank = rank;
2540
2541                         ret = dram->ops->data_training_first(dram, ch,
2542                                                              rank, params);
2543                         if (!ret) {
2544                                 debug("%s: data trained for rank %d, ch %d\n",
2545                                       __func__, rank, ch);
2546                                 break;
2547                         }
2548                 }
2549                 /* Computed rank with associated channel number */
2550                 params->ch[ch].cap_info.rank = rank;
2551         }
2552
2553         params->base.num_channels = 0;
2554         for (channel = 0; channel < 2; channel++) {
2555                 const struct chan_info *chan = &dram->chan[channel];
2556                 struct sdram_cap_info *cap_info = &params->ch[channel].cap_info;
2557                 u8 training_flag = PI_FULL_TRAINING;
2558
2559                 if (cap_info->rank == 0) {
2560                         clear_channel_params(params, channel);
2561                         continue;
2562                 } else {
2563                         params->base.num_channels++;
2564                 }
2565
2566                 printf("Channel ");
2567                 printf(channel ? "1: " : "0: ");
2568
2569                 /* LPDDR3 should have write and read gate training */
2570                 if (params->base.dramtype == LPDDR3)
2571                         training_flag = PI_WRITE_LEVELING |
2572                                         PI_READ_GATE_TRAINING;
2573
2574                 if (params->base.dramtype != LPDDR4) {
2575                         ret = data_training(dram, channel, params,
2576                                             training_flag);
2577                         if (!ret) {
2578                                 debug("%s: data train failed for channel %d\n",
2579                                       __func__, ret);
2580                                 continue;
2581                         }
2582                 }
2583
2584                 sdram_print_ddr_info(cap_info, &params->base);
2585                 set_memory_map(chan, channel, params);
2586                 cap_info->ddrconfig = calculate_ddrconfig(params, channel);
2587
2588                 set_ddrconfig(chan, params, channel, cap_info->ddrconfig);
2589                 set_cap_relate_config(chan, params, channel);
2590         }
2591
2592         if (params->base.num_channels == 0) {
2593                 printf("%s: ", __func__);
2594                 sdram_print_dram_type(params->base.dramtype);
2595                 printf(" - %dMHz failed!\n", params->base.ddr_freq);
2596                 return -EINVAL;
2597         }
2598
2599         params->base.stride = calculate_stride(params);
2600         dram_all_config(dram, params);
2601
2602         dram->ops->set_rate_index(dram, params);
2603
2604         debug("Finish SDRAM initialization...\n");
2605         return 0;
2606 }
2607
2608 static int rk3399_dmc_ofdata_to_platdata(struct udevice *dev)
2609 {
2610 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
2611         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
2612         int ret;
2613
2614         ret = dev_read_u32_array(dev, "rockchip,sdram-params",
2615                                  (u32 *)&plat->sdram_params,
2616                                  sizeof(plat->sdram_params) / sizeof(u32));
2617         if (ret) {
2618                 printf("%s: Cannot read rockchip,sdram-params %d\n",
2619                        __func__, ret);
2620                 return ret;
2621         }
2622         ret = regmap_init_mem(dev_ofnode(dev), &plat->map);
2623         if (ret)
2624                 printf("%s: regmap failed %d\n", __func__, ret);
2625
2626 #endif
2627         return 0;
2628 }
2629
2630 #if CONFIG_IS_ENABLED(OF_PLATDATA)
2631 static int conv_of_platdata(struct udevice *dev)
2632 {
2633         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
2634         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
2635         int ret;
2636
2637         ret = regmap_init_mem_platdata(dev, dtplat->reg,
2638                                        ARRAY_SIZE(dtplat->reg) / 2,
2639                                        &plat->map);
2640         if (ret)
2641                 return ret;
2642
2643         return 0;
2644 }
2645 #endif
2646
2647 static const struct sdram_rk3399_ops rk3399_ops = {
2648 #if !defined(CONFIG_RAM_RK3399_LPDDR4)
2649         .data_training_first = data_training_first,
2650         .set_rate_index = switch_to_phy_index1,
2651 #else
2652         .data_training_first = lpddr4_mr_detect,
2653         .set_rate_index = lpddr4_set_rate,
2654 #endif
2655 };
2656
2657 static int rk3399_dmc_init(struct udevice *dev)
2658 {
2659         struct dram_info *priv = dev_get_priv(dev);
2660         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
2661         int ret;
2662 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
2663         struct rk3399_sdram_params *params = &plat->sdram_params;
2664 #else
2665         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
2666         struct rk3399_sdram_params *params =
2667                                         (void *)dtplat->rockchip_sdram_params;
2668
2669         ret = conv_of_platdata(dev);
2670         if (ret)
2671                 return ret;
2672 #endif
2673
2674         priv->ops = &rk3399_ops;
2675         priv->cic = syscon_get_first_range(ROCKCHIP_SYSCON_CIC);
2676         priv->grf = syscon_get_first_range(ROCKCHIP_SYSCON_GRF);
2677         priv->pmu = syscon_get_first_range(ROCKCHIP_SYSCON_PMU);
2678         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
2679         priv->pmusgrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUSGRF);
2680         priv->pmucru = rockchip_get_pmucru();
2681         priv->cru = rockchip_get_cru();
2682         priv->chan[0].pctl = regmap_get_range(plat->map, 0);
2683         priv->chan[0].pi = regmap_get_range(plat->map, 1);
2684         priv->chan[0].publ = regmap_get_range(plat->map, 2);
2685         priv->chan[0].msch = regmap_get_range(plat->map, 3);
2686         priv->chan[1].pctl = regmap_get_range(plat->map, 4);
2687         priv->chan[1].pi = regmap_get_range(plat->map, 5);
2688         priv->chan[1].publ = regmap_get_range(plat->map, 6);
2689         priv->chan[1].msch = regmap_get_range(plat->map, 7);
2690
2691         debug("con reg %p %p %p %p %p %p %p %p\n",
2692               priv->chan[0].pctl, priv->chan[0].pi,
2693               priv->chan[0].publ, priv->chan[0].msch,
2694               priv->chan[1].pctl, priv->chan[1].pi,
2695               priv->chan[1].publ, priv->chan[1].msch);
2696         debug("cru %p, cic %p, grf %p, sgrf %p, pmucru %p, pmu %p\n", priv->cru,
2697               priv->cic, priv->pmugrf, priv->pmusgrf, priv->pmucru, priv->pmu);
2698
2699 #if CONFIG_IS_ENABLED(OF_PLATDATA)
2700         ret = clk_get_by_index_platdata(dev, 0, dtplat->clocks, &priv->ddr_clk);
2701 #else
2702         ret = clk_get_by_index(dev, 0, &priv->ddr_clk);
2703 #endif
2704         if (ret) {
2705                 printf("%s clk get failed %d\n", __func__, ret);
2706                 return ret;
2707         }
2708
2709         ret = clk_set_rate(&priv->ddr_clk, params->base.ddr_freq * MHz);
2710         if (ret < 0) {
2711                 printf("%s clk set failed %d\n", __func__, ret);
2712                 return ret;
2713         }
2714
2715         ret = sdram_init(priv, params);
2716         if (ret < 0) {
2717                 printf("%s DRAM init failed %d\n", __func__, ret);
2718                 return ret;
2719         }
2720
2721         return 0;
2722 }
2723 #endif
2724
2725 static int rk3399_dmc_probe(struct udevice *dev)
2726 {
2727 #if defined(CONFIG_TPL_BUILD) || \
2728         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
2729         if (rk3399_dmc_init(dev))
2730                 return 0;
2731 #else
2732         struct dram_info *priv = dev_get_priv(dev);
2733
2734         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
2735         debug("%s: pmugrf = %p\n", __func__, priv->pmugrf);
2736         priv->info.base = CONFIG_SYS_SDRAM_BASE;
2737         priv->info.size =
2738                 rockchip_sdram_size((phys_addr_t)&priv->pmugrf->os_reg2);
2739 #endif
2740         return 0;
2741 }
2742
2743 static int rk3399_dmc_get_info(struct udevice *dev, struct ram_info *info)
2744 {
2745         struct dram_info *priv = dev_get_priv(dev);
2746
2747         *info = priv->info;
2748
2749         return 0;
2750 }
2751
2752 static struct ram_ops rk3399_dmc_ops = {
2753         .get_info = rk3399_dmc_get_info,
2754 };
2755
2756 static const struct udevice_id rk3399_dmc_ids[] = {
2757         { .compatible = "rockchip,rk3399-dmc" },
2758         { }
2759 };
2760
2761 U_BOOT_DRIVER(dmc_rk3399) = {
2762         .name = "rockchip_rk3399_dmc",
2763         .id = UCLASS_RAM,
2764         .of_match = rk3399_dmc_ids,
2765         .ops = &rk3399_dmc_ops,
2766 #if defined(CONFIG_TPL_BUILD) || \
2767         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
2768         .ofdata_to_platdata = rk3399_dmc_ofdata_to_platdata,
2769 #endif
2770         .probe = rk3399_dmc_probe,
2771         .priv_auto_alloc_size = sizeof(struct dram_info),
2772 #if defined(CONFIG_TPL_BUILD) || \
2773         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
2774         .platdata_auto_alloc_size = sizeof(struct rockchip_dmc_plat),
2775 #endif
2776 };