drivers/ram/rockchip/sdram_rk3399.c

   1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
   2 /*
   3  * (C) Copyright 2016-2017 Rockchip Inc.
   4  *
   5  * Adapted from coreboot.
   6  */
   7
   8 #include <common.h>
   9 #include <clk.h>
  10 #include <dm.h>
  11 #include <dt-structs.h>
  12 #include <ram.h>
  13 #include <regmap.h>
  14 #include <syscon.h>
  15 #include <asm/io.h>
  16 #include <asm/arch-rockchip/clock.h>
  17 #include <asm/arch-rockchip/cru.h>
  18 #include <asm/arch-rockchip/grf_rk3399.h>
  19 #include <asm/arch-rockchip/pmu_rk3399.h>
  20 #include <asm/arch-rockchip/hardware.h>
  21 #include <asm/arch-rockchip/sdram.h>
  22 #include <asm/arch-rockchip/sdram_rk3399.h>
  23 #include <linux/err.h>
  24 #include <time.h>
  25
  26 #define PRESET_SGRF_HOLD(n)     ((0x1 << (6 + 16)) | ((n) << 6))
  27 #define PRESET_GPIO0_HOLD(n)    ((0x1 << (7 + 16)) | ((n) << 7))
  28 #define PRESET_GPIO1_HOLD(n)    ((0x1 << (8 + 16)) | ((n) << 8))
  29
  30 #define PHY_DRV_ODT_HI_Z        0x0
  31 #define PHY_DRV_ODT_240         0x1
  32 #define PHY_DRV_ODT_120         0x8
  33 #define PHY_DRV_ODT_80          0x9
  34 #define PHY_DRV_ODT_60          0xc
  35 #define PHY_DRV_ODT_48          0xd
  36 #define PHY_DRV_ODT_40          0xe
  37 #define PHY_DRV_ODT_34_3        0xf
  38
  39 #define PHY_BOOSTP_EN           0x1
  40 #define PHY_BOOSTN_EN           0x1
  41 #define PHY_SLEWP_EN            0x1
  42 #define PHY_SLEWN_EN            0x1
  43 #define PHY_RX_CM_INPUT         0x1
  44 #define CS0_MR22_VAL            0
  45 #define CS1_MR22_VAL            3
  46
  47 /* LPDDR3 DRAM DS */
  48 #define LPDDR3_DS_34            0x1
  49 #define LPDDR3_DS_40            0x2
  50 #define LPDDR3_DS_48            0x3
  51
  52 #define CRU_SFTRST_DDR_CTRL(ch, n)      ((0x1 << (8 + 16 + (ch) * 4)) | \
  53                                         ((n) << (8 + (ch) * 4)))
  54 #define CRU_SFTRST_DDR_PHY(ch, n)       ((0x1 << (9 + 16 + (ch) * 4)) | \
  55                                         ((n) << (9 + (ch) * 4)))
  56 struct chan_info {
  57         struct rk3399_ddr_pctl_regs *pctl;
  58         struct rk3399_ddr_pi_regs *pi;
  59         struct rk3399_ddr_publ_regs *publ;
  60         struct msch_regs *msch;
  61 };
  62
  63 struct dram_info {
  64 #if defined(CONFIG_TPL_BUILD) || \
  65         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  66         u32 pwrup_srefresh_exit[2];
  67         struct chan_info chan[2];
  68         struct clk ddr_clk;
  69         struct rockchip_cru *cru;
  70         struct rk3399_grf_regs *grf;
  71         struct rk3399_pmu_regs *pmu;
  72         struct rk3399_pmucru *pmucru;
  73         struct rk3399_pmusgrf_regs *pmusgrf;
  74         struct rk3399_ddr_cic_regs *cic;
  75         const struct sdram_rk3399_ops *ops;
  76 #endif
  77         struct ram_info info;
  78         struct rk3399_pmugrf_regs *pmugrf;
  79 };
  80
  81 struct sdram_rk3399_ops {
  82         int (*data_training_first)(struct dram_info *dram, u32 channel, u8 rank,
  83                                    struct rk3399_sdram_params *sdram);
  84         int (*set_rate_index)(struct dram_info *dram,
  85                               struct rk3399_sdram_params *params);
  86         void (*modify_param)(const struct chan_info *chan,
  87                              struct rk3399_sdram_params *params);
  88         struct rk3399_sdram_params *
  89                 (*get_phy_index_params)(u32 phy_fn,
  90                                         struct rk3399_sdram_params *params);
  91 };
  92
  93 #if defined(CONFIG_TPL_BUILD) || \
  94         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  95
  96 struct rockchip_dmc_plat {
  97 #if CONFIG_IS_ENABLED(OF_PLATDATA)
  98         struct dtd_rockchip_rk3399_dmc dtplat;
  99 #else
 100         struct rk3399_sdram_params sdram_params;
 101 #endif
 102         struct regmap *map;
 103 };
 104
 105 struct io_setting {
 106         u32 mhz;
 107         u32 mr5;
 108         /* dram side */
 109         u32 dq_odt;
 110         u32 ca_odt;
 111         u32 pdds;
 112         u32 dq_vref;
 113         u32 ca_vref;
 114         /* phy side */
 115         u32 rd_odt;
 116         u32 wr_dq_drv;
 117         u32 wr_ca_drv;
 118         u32 wr_ckcs_drv;
 119         u32 rd_odt_en;
 120         u32 rd_vref;
 121 } lpddr4_io_setting[] = {
 122         {
 123                 50 * MHz,
 124                 0,
 125                 /* dram side */
 126                 0,      /* dq_odt; */
 127                 0,      /* ca_odt; */
 128                 6,      /* pdds; */
 129                 0x72,   /* dq_vref; */
 130                 0x72,   /* ca_vref; */
 131                 /* phy side */
 132                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 133                 PHY_DRV_ODT_40, /* wr_dq_drv; */
 134                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 135                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 136                 0,      /* rd_odt_en;*/
 137                 41,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 138         },
 139         {
 140                 600 * MHz,
 141                 0,
 142                 /* dram side */
 143                 1,      /* dq_odt; */
 144                 0,      /* ca_odt; */
 145                 6,      /* pdds; */
 146                 0x72,   /* dq_vref; */
 147                 0x72,   /* ca_vref; */
 148                 /* phy side */
 149                 PHY_DRV_ODT_HI_Z,       /* rd_odt; */
 150                 PHY_DRV_ODT_48, /* wr_dq_drv; */
 151                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 152                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 153                 0,      /* rd_odt_en; */
 154                 32,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 155         },
 156         {
 157                 933 * MHz,
 158                 0,
 159                 /* dram side */
 160                 1,      /* dq_odt; */
 161                 0,      /* ca_odt; */
 162                 3,      /* pdds; */
 163                 0x72,   /* dq_vref; */
 164                 0x72,   /* ca_vref; */
 165                 /* phy side */
 166                 PHY_DRV_ODT_80, /* rd_odt; */
 167                 PHY_DRV_ODT_40, /* wr_dq_drv; */
 168                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 169                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 170                 1,      /* rd_odt_en; */
 171                 20,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 172         },
 173         {
 174                 1066 * MHz,
 175                 0,
 176                 /* dram side */
 177                 6,      /* dq_odt; */
 178                 0,      /* ca_odt; */
 179                 3,      /* pdds; */
 180                 0x10,   /* dq_vref; */
 181                 0x72,   /* ca_vref; */
 182                 /* phy side */
 183                 PHY_DRV_ODT_80, /* rd_odt; */
 184                 PHY_DRV_ODT_60, /* wr_dq_drv; */
 185                 PHY_DRV_ODT_40, /* wr_ca_drv; */
 186                 PHY_DRV_ODT_40, /* wr_ckcs_drv; */
 187                 1,      /* rd_odt_en; */
 188                 20,     /* rd_vref; (unit %, range 3.3% - 48.7%) */
 189         },
 190 };
 191
 192 static struct io_setting *
 193 lpddr4_get_io_settings(const struct rk3399_sdram_params *params, u32 mr5)
 194 {
 195         struct io_setting *io = NULL;
 196         u32 n;
 197
 198         for (n = 0; n < ARRAY_SIZE(lpddr4_io_setting); n++) {
 199                 io = &lpddr4_io_setting[n];
 200
 201                 if (io->mr5 != 0) {
 202                         if (io->mhz >= params->base.ddr_freq &&
 203                             io->mr5 == mr5)
 204                                 break;
 205                 } else {
 206                         if (io->mhz >= params->base.ddr_freq)
 207                                 break;
 208                 }
 209         }
 210
 211         return io;
 212 }
 213
 214 static void *get_denali_ctl(const struct chan_info *chan,
 215                             struct rk3399_sdram_params *params, bool reg)
 216 {
 217         return reg ? &chan->pctl->denali_ctl : &params->pctl_regs.denali_ctl;
 218 }
 219
 220 static void *get_denali_phy(const struct chan_info *chan,
 221                             struct rk3399_sdram_params *params, bool reg)
 222 {
 223         return reg ? &chan->publ->denali_phy : &params->phy_regs.denali_phy;
 224 }
 225
 226 static void *get_ddrc0_con(struct dram_info *dram, u8 channel)
 227 {
 228         return (channel == 0) ? &dram->grf->ddrc0_con0 : &dram->grf->ddrc1_con0;
 229 }
 230
 231 static void rkclk_ddr_reset(struct rockchip_cru *cru, u32 channel, u32 ctl,
 232                             u32 phy)
 233 {
 234         channel &= 0x1;
 235         ctl &= 0x1;
 236         phy &= 0x1;
 237         writel(CRU_SFTRST_DDR_CTRL(channel, ctl) |
 238                                    CRU_SFTRST_DDR_PHY(channel, phy),
 239                                    &cru->softrst_con[4]);
 240 }
 241
 242 static void phy_pctrl_reset(struct rockchip_cru *cru,  u32 channel)
 243 {
 244         rkclk_ddr_reset(cru, channel, 1, 1);
 245         udelay(10);
 246
 247         rkclk_ddr_reset(cru, channel, 1, 0);
 248         udelay(10);
 249
 250         rkclk_ddr_reset(cru, channel, 0, 0);
 251         udelay(10);
 252 }
 253
 254 static void phy_dll_bypass_set(struct rk3399_ddr_publ_regs *ddr_publ_regs,
 255                                u32 freq)
 256 {
 257         u32 *denali_phy = ddr_publ_regs->denali_phy;
 258
 259         /* From IP spec, only freq small than 125 can enter dll bypass mode */
 260         if (freq <= 125) {
 261                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 262                 setbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 263                 setbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 264                 setbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 265                 setbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 266
 267                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 268                 setbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 269                 setbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 270                 setbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 271         } else {
 272                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 273                 clrbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 274                 clrbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 275                 clrbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 276                 clrbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 277
 278                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 279                 clrbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 280                 clrbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 281                 clrbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 282         }
 283 }
 284
 285 static void set_memory_map(const struct chan_info *chan, u32 channel,
 286                            const struct rk3399_sdram_params *params)
 287 {
 288         const struct rk3399_sdram_channel *sdram_ch = &params->ch[channel];
 289         u32 *denali_ctl = chan->pctl->denali_ctl;
 290         u32 *denali_pi = chan->pi->denali_pi;
 291         u32 cs_map;
 292         u32 reduc;
 293         u32 row;
 294
 295         /* Get row number from ddrconfig setting */
 296         if (sdram_ch->cap_info.ddrconfig < 2 ||
 297             sdram_ch->cap_info.ddrconfig == 4)
 298                 row = 16;
 299         else if (sdram_ch->cap_info.ddrconfig == 3 ||
 300                  sdram_ch->cap_info.ddrconfig == 5)
 301                 row = 14;
 302         else
 303                 row = 15;
 304
 305         cs_map = (sdram_ch->cap_info.rank > 1) ? 3 : 1;
 306         reduc = (sdram_ch->cap_info.bw == 2) ? 0 : 1;
 307
 308         /* Set the dram configuration to ctrl */
 309         clrsetbits_le32(&denali_ctl[191], 0xF, (12 - sdram_ch->cap_info.col));
 310         clrsetbits_le32(&denali_ctl[190], (0x3 << 16) | (0x7 << 24),
 311                         ((3 - sdram_ch->cap_info.bk) << 16) |
 312                         ((16 - row) << 24));
 313
 314         clrsetbits_le32(&denali_ctl[196], 0x3 | (1 << 16),
 315                         cs_map | (reduc << 16));
 316
 317         /* PI_199 PI_COL_DIFF:RW:0:4 */
 318         clrsetbits_le32(&denali_pi[199], 0xF, (12 - sdram_ch->cap_info.col));
 319
 320         /* PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2 */
 321         clrsetbits_le32(&denali_pi[155], (0x3 << 16) | (0x7 << 24),
 322                         ((3 - sdram_ch->cap_info.bk) << 16) |
 323                         ((16 - row) << 24));
 324
 325         if (params->base.dramtype == LPDDR4) {
 326                 if (cs_map == 1)
 327                         cs_map = 0x5;
 328                 else if (cs_map == 2)
 329                         cs_map = 0xa;
 330                 else
 331                         cs_map = 0xF;
 332         }
 333
 334         /* PI_41 PI_CS_MAP:RW:24:4 */
 335         clrsetbits_le32(&denali_pi[41], 0xf << 24, cs_map << 24);
 336         if (sdram_ch->cap_info.rank == 1 && params->base.dramtype == DDR3)
 337                 writel(0x2EC7FFFF, &denali_pi[34]);
 338 }
 339
 340 static int phy_io_config(u32 *denali_phy, u32 *denali_ctl,
 341                          const struct rk3399_sdram_params *params, u32 mr5)
 342 {
 343         u32 vref_mode_dq, vref_value_dq, vref_mode_ac, vref_value_ac;
 344         u32 mode_sel;
 345         u32 speed;
 346         u32 reg_value;
 347         u32 ds_value, odt_value;
 348
 349         /* vref setting & mode setting */
 350         if (params->base.dramtype == LPDDR4) {
 351                 struct io_setting *io = lpddr4_get_io_settings(params, mr5);
 352                 u32 rd_vref = io->rd_vref * 1000;
 353
 354                 if (rd_vref < 36700) {
 355                         /* MODE_LV[2:0] = LPDDR4 (Range 2)*/
 356                         vref_mode_dq = 0x7;
 357                         /* MODE[2:0]= LPDDR4 Range 2(0.4*VDDQ) */
 358                         mode_sel = 0x5;
 359                         vref_value_dq = (rd_vref - 3300) / 521;
 360                 } else {
 361                         /* MODE_LV[2:0] = LPDDR4 (Range 1)*/
 362                         vref_mode_dq = 0x6;
 363                         /* MODE[2:0]= LPDDR4 Range 1(0.33*VDDQ) */
 364                         mode_sel = 0x4;
 365                         vref_value_dq = (rd_vref - 15300) / 521;
 366                 }
 367                 vref_mode_ac = 0x6;
 368                 /* VDDQ/3/2=16.8% */
 369                 vref_value_ac = 0x3;
 370         } else if (params->base.dramtype == LPDDR3) {
 371                 if (params->base.odt == 1) {
 372                         vref_mode_dq = 0x5;  /* LPDDR3 ODT */
 373                         ds_value = readl(&denali_ctl[138]) & 0xf;
 374                         odt_value = (readl(&denali_phy[6]) >> 4) & 0xf;
 375                         if (ds_value == LPDDR3_DS_48) {
 376                                 switch (odt_value) {
 377                                 case PHY_DRV_ODT_240:
 378                                         vref_value_dq = 0x1B;
 379                                         break;
 380                                 case PHY_DRV_ODT_120:
 381                                         vref_value_dq = 0x26;
 382                                         break;
 383                                 case PHY_DRV_ODT_60:
 384                                         vref_value_dq = 0x36;
 385                                         break;
 386                                 default:
 387                                         debug("Invalid ODT value.\n");
 388                                         return -EINVAL;
 389                                 }
 390                         } else if (ds_value == LPDDR3_DS_40) {
 391                                 switch (odt_value) {
 392                                 case PHY_DRV_ODT_240:
 393                                         vref_value_dq = 0x19;
 394                                         break;
 395                                 case PHY_DRV_ODT_120:
 396                                         vref_value_dq = 0x23;
 397                                         break;
 398                                 case PHY_DRV_ODT_60:
 399                                         vref_value_dq = 0x31;
 400                                         break;
 401                                 default:
 402                                         debug("Invalid ODT value.\n");
 403                                         return -EINVAL;
 404                                 }
 405                         } else if (ds_value == LPDDR3_DS_34) {
 406                                 switch (odt_value) {
 407                                 case PHY_DRV_ODT_240:
 408                                         vref_value_dq = 0x17;
 409                                         break;
 410                                 case PHY_DRV_ODT_120:
 411                                         vref_value_dq = 0x20;
 412                                         break;
 413                                 case PHY_DRV_ODT_60:
 414                                         vref_value_dq = 0x2e;
 415                                         break;
 416                                 default:
 417                                         debug("Invalid ODT value.\n");
 418                                         return -EINVAL;
 419                                 }
 420                         } else {
 421                                 debug("Invalid DRV value.\n");
 422                                 return -EINVAL;
 423                         }
 424                 } else {
 425                         vref_mode_dq = 0x2;  /* LPDDR3 */
 426                         vref_value_dq = 0x1f;
 427                 }
 428                 vref_mode_ac = 0x2;
 429                 vref_value_ac = 0x1f;
 430                 mode_sel = 0x0;
 431         } else if (params->base.dramtype == DDR3) {
 432                 /* DDR3L */
 433                 vref_mode_dq = 0x1;
 434                 vref_value_dq = 0x1f;
 435                 vref_mode_ac = 0x1;
 436                 vref_value_ac = 0x1f;
 437                 mode_sel = 0x1;
 438         } else {
 439                 debug("Unknown DRAM type.\n");
 440                 return -EINVAL;
 441         }
 442
 443         reg_value = (vref_mode_dq << 9) | (0x1 << 8) | vref_value_dq;
 444
 445         /* PHY_913 PHY_PAD_VREF_CTRL_DQ_0 12bits offset_8 */
 446         clrsetbits_le32(&denali_phy[913], 0xfff << 8, reg_value << 8);
 447         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_1 12bits offset_0 */
 448         clrsetbits_le32(&denali_phy[914], 0xfff, reg_value);
 449         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_2 12bits offset_16 */
 450         clrsetbits_le32(&denali_phy[914], 0xfff << 16, reg_value << 16);
 451         /* PHY_915 PHY_PAD_VREF_CTRL_DQ_3 12bits offset_0 */
 452         clrsetbits_le32(&denali_phy[915], 0xfff, reg_value);
 453
 454         reg_value = (vref_mode_ac << 9) | (0x1 << 8) | vref_value_ac;
 455
 456         /* PHY_915 PHY_PAD_VREF_CTRL_AC 12bits offset_16 */
 457         clrsetbits_le32(&denali_phy[915], 0xfff << 16, reg_value << 16);
 458
 459         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 460         clrsetbits_le32(&denali_phy[924], 0x7 << 15, mode_sel << 15);
 461         /* PHY_926 PHY_PAD_DATA_DRIVE */
 462         clrsetbits_le32(&denali_phy[926], 0x7 << 6, mode_sel << 6);
 463         /* PHY_927 PHY_PAD_DQS_DRIVE */
 464         clrsetbits_le32(&denali_phy[927], 0x7 << 6, mode_sel << 6);
 465         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 466         clrsetbits_le32(&denali_phy[928], 0x7 << 14, mode_sel << 14);
 467         /* PHY_929 PHY_PAD_CLK_DRIVE */
 468         clrsetbits_le32(&denali_phy[929], 0x7 << 14, mode_sel << 14);
 469         /* PHY_935 PHY_PAD_CKE_DRIVE */
 470         clrsetbits_le32(&denali_phy[935], 0x7 << 14, mode_sel << 14);
 471         /* PHY_937 PHY_PAD_RST_DRIVE */
 472         clrsetbits_le32(&denali_phy[937], 0x7 << 14, mode_sel << 14);
 473         /* PHY_939 PHY_PAD_CS_DRIVE */
 474         clrsetbits_le32(&denali_phy[939], 0x7 << 14, mode_sel << 14);
 475
 476         if (params->base.dramtype == LPDDR4) {
 477                 /* BOOSTP_EN & BOOSTN_EN */
 478                 reg_value = ((PHY_BOOSTP_EN << 4) | PHY_BOOSTN_EN);
 479                 /* PHY_925 PHY_PAD_FDBK_DRIVE2 */
 480                 clrsetbits_le32(&denali_phy[925], 0xff << 8, reg_value << 8);
 481                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 482                 clrsetbits_le32(&denali_phy[926], 0xff << 12, reg_value << 12);
 483                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 484                 clrsetbits_le32(&denali_phy[927], 0xff << 14, reg_value << 14);
 485                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 486                 clrsetbits_le32(&denali_phy[928], 0xff << 20, reg_value << 20);
 487                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 488                 clrsetbits_le32(&denali_phy[929], 0xff << 22, reg_value << 22);
 489                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 490                 clrsetbits_le32(&denali_phy[935], 0xff << 20, reg_value << 20);
 491                 /* PHY_937 PHY_PAD_RST_DRIVE */
 492                 clrsetbits_le32(&denali_phy[937], 0xff << 20, reg_value << 20);
 493                 /* PHY_939 PHY_PAD_CS_DRIVE */
 494                 clrsetbits_le32(&denali_phy[939], 0xff << 20, reg_value << 20);
 495
 496                 /* SLEWP_EN & SLEWN_EN */
 497                 reg_value = ((PHY_SLEWP_EN << 3) | PHY_SLEWN_EN);
 498                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 499                 clrsetbits_le32(&denali_phy[924], 0x3f << 8, reg_value << 8);
 500                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 501                 clrsetbits_le32(&denali_phy[926], 0x3f, reg_value);
 502                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 503                 clrsetbits_le32(&denali_phy[927], 0x3f, reg_value);
 504                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 505                 clrsetbits_le32(&denali_phy[928], 0x3f << 8, reg_value << 8);
 506                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 507                 clrsetbits_le32(&denali_phy[929], 0x3f << 8, reg_value << 8);
 508                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 509                 clrsetbits_le32(&denali_phy[935], 0x3f << 8, reg_value << 8);
 510                 /* PHY_937 PHY_PAD_RST_DRIVE */
 511                 clrsetbits_le32(&denali_phy[937], 0x3f << 8, reg_value << 8);
 512                 /* PHY_939 PHY_PAD_CS_DRIVE */
 513                 clrsetbits_le32(&denali_phy[939], 0x3f << 8, reg_value << 8);
 514         }
 515
 516         /* speed setting */
 517         speed = 0x2;
 518
 519         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 520         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
 521         /* PHY_926 PHY_PAD_DATA_DRIVE */
 522         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
 523         /* PHY_927 PHY_PAD_DQS_DRIVE */
 524         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
 525         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 526         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
 527         /* PHY_929 PHY_PAD_CLK_DRIVE */
 528         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
 529         /* PHY_935 PHY_PAD_CKE_DRIVE */
 530         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
 531         /* PHY_937 PHY_PAD_RST_DRIVE */
 532         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
 533         /* PHY_939 PHY_PAD_CS_DRIVE */
 534         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
 535
 536         if (params->base.dramtype == LPDDR4) {
 537                 /* RX_CM_INPUT */
 538                 reg_value = PHY_RX_CM_INPUT;
 539                 /* PHY_924 PHY_PAD_FDBK_DRIVE */
 540                 clrsetbits_le32(&denali_phy[924], 0x1 << 14, reg_value << 14);
 541                 /* PHY_926 PHY_PAD_DATA_DRIVE */
 542                 clrsetbits_le32(&denali_phy[926], 0x1 << 11, reg_value << 11);
 543                 /* PHY_927 PHY_PAD_DQS_DRIVE */
 544                 clrsetbits_le32(&denali_phy[927], 0x1 << 13, reg_value << 13);
 545                 /* PHY_928 PHY_PAD_ADDR_DRIVE */
 546                 clrsetbits_le32(&denali_phy[928], 0x1 << 19, reg_value << 19);
 547                 /* PHY_929 PHY_PAD_CLK_DRIVE */
 548                 clrsetbits_le32(&denali_phy[929], 0x1 << 21, reg_value << 21);
 549                 /* PHY_935 PHY_PAD_CKE_DRIVE */
 550                 clrsetbits_le32(&denali_phy[935], 0x1 << 19, reg_value << 19);
 551                 /* PHY_937 PHY_PAD_RST_DRIVE */
 552                 clrsetbits_le32(&denali_phy[937], 0x1 << 19, reg_value << 19);
 553                 /* PHY_939 PHY_PAD_CS_DRIVE */
 554                 clrsetbits_le32(&denali_phy[939], 0x1 << 19, reg_value << 19);
 555         }
 556
 557         return 0;
 558 }
 559
 560 static void set_ds_odt(const struct chan_info *chan,
 561                        struct rk3399_sdram_params *params,
 562                        bool ctl_phy_reg, u32 mr5)
 563 {
 564         u32 *denali_phy = get_denali_phy(chan, params, ctl_phy_reg);
 565         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
 566         u32 tsel_idle_en, tsel_wr_en, tsel_rd_en;
 567         u32 tsel_idle_select_p, tsel_rd_select_p;
 568         u32 tsel_idle_select_n, tsel_rd_select_n;
 569         u32 tsel_wr_select_dq_p, tsel_wr_select_ca_p;
 570         u32 tsel_wr_select_dq_n, tsel_wr_select_ca_n;
 571         u32 tsel_ckcs_select_p, tsel_ckcs_select_n;
 572         struct io_setting *io = NULL;
 573         u32 soc_odt = 0;
 574         u32 reg_value;
 575
 576         if (params->base.dramtype == LPDDR4) {
 577                 io = lpddr4_get_io_settings(params, mr5);
 578
 579                 tsel_rd_select_p = PHY_DRV_ODT_HI_Z;
 580                 tsel_rd_select_n = io->rd_odt;
 581
 582                 tsel_idle_select_p = PHY_DRV_ODT_HI_Z;
 583                 tsel_idle_select_n = PHY_DRV_ODT_HI_Z;
 584
 585                 tsel_wr_select_dq_p = io->wr_dq_drv;
 586                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 587
 588                 tsel_wr_select_ca_p = io->wr_ca_drv;
 589                 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3;
 590
 591                 tsel_ckcs_select_p = io->wr_ckcs_drv;
 592                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 593
 594                 switch (tsel_rd_select_n) {
 595                 case PHY_DRV_ODT_240:
 596                         soc_odt = 1;
 597                         break;
 598                 case PHY_DRV_ODT_120:
 599                         soc_odt = 2;
 600                         break;
 601                 case PHY_DRV_ODT_80:
 602                         soc_odt = 3;
 603                         break;
 604                 case PHY_DRV_ODT_60:
 605                         soc_odt = 4;
 606                         break;
 607                 case PHY_DRV_ODT_48:
 608                         soc_odt = 5;
 609                         break;
 610                 case PHY_DRV_ODT_40:
 611                         soc_odt = 6;
 612                         break;
 613                 case PHY_DRV_ODT_34_3:
 614                         soc_odt = 6;
 615                         printf("%s: Unable to support LPDDR4 MR22 Soc ODT\n",
 616                                __func__);
 617                         break;
 618                 case PHY_DRV_ODT_HI_Z:
 619                 default:
 620                         soc_odt = 0;
 621                         break;
 622                 }
 623         } else if (params->base.dramtype == LPDDR3) {
 624                 tsel_rd_select_p = PHY_DRV_ODT_240;
 625                 tsel_rd_select_n = PHY_DRV_ODT_HI_Z;
 626
 627                 tsel_idle_select_p = PHY_DRV_ODT_240;
 628                 tsel_idle_select_n = PHY_DRV_ODT_HI_Z;
 629
 630                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 631                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 632
 633                 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3;
 634                 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3;
 635
 636                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 637                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 638         } else {
 639                 tsel_rd_select_p = PHY_DRV_ODT_240;
 640                 tsel_rd_select_n = PHY_DRV_ODT_240;
 641
 642                 tsel_idle_select_p = PHY_DRV_ODT_240;
 643                 tsel_idle_select_n = PHY_DRV_ODT_240;
 644
 645                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 646                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 647
 648                 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3;
 649                 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3;
 650
 651                 tsel_ckcs_select_p = PHY_DRV_ODT_34_3;
 652                 tsel_ckcs_select_n = PHY_DRV_ODT_34_3;
 653         }
 654
 655         if (params->base.odt == 1) {
 656                 tsel_rd_en = 1;
 657
 658                 if (params->base.dramtype == LPDDR4)
 659                         tsel_rd_en = io->rd_odt_en;
 660         } else {
 661                 tsel_rd_en = 0;
 662         }
 663
 664         tsel_wr_en = 0;
 665         tsel_idle_en = 0;
 666
 667         /* F0_0 */
 668         clrsetbits_le32(&denali_ctl[145], 0xFF << 16,
 669                         (soc_odt | (CS0_MR22_VAL << 3)) << 16);
 670         /* F2_0, F1_0 */
 671         clrsetbits_le32(&denali_ctl[146], 0xFF00FF,
 672                         ((soc_odt | (CS0_MR22_VAL << 3)) << 16) |
 673                         (soc_odt | (CS0_MR22_VAL << 3)));
 674         /* F0_1 */
 675         clrsetbits_le32(&denali_ctl[159], 0xFF << 16,
 676                         (soc_odt | (CS1_MR22_VAL << 3)) << 16);
 677         /* F2_1, F1_1 */
 678         clrsetbits_le32(&denali_ctl[160], 0xFF00FF,
 679                         ((soc_odt | (CS1_MR22_VAL << 3)) << 16) |
 680                         (soc_odt | (CS1_MR22_VAL << 3)));
 681
 682         /*
 683          * phy_dq_tsel_select_X 24bits DENALI_PHY_6/134/262/390 offset_0
 684          * sets termination values for read/idle cycles and drive strength
 685          * for write cycles for DQ/DM
 686          */
 687         reg_value = tsel_rd_select_n | (tsel_rd_select_p << 0x4) |
 688                     (tsel_wr_select_dq_n << 8) | (tsel_wr_select_dq_p << 12) |
 689                     (tsel_idle_select_n << 16) | (tsel_idle_select_p << 20);
 690         clrsetbits_le32(&denali_phy[6], 0xffffff, reg_value);
 691         clrsetbits_le32(&denali_phy[134], 0xffffff, reg_value);
 692         clrsetbits_le32(&denali_phy[262], 0xffffff, reg_value);
 693         clrsetbits_le32(&denali_phy[390], 0xffffff, reg_value);
 694
 695         /*
 696          * phy_dqs_tsel_select_X 24bits DENALI_PHY_7/135/263/391 offset_0
 697          * sets termination values for read/idle cycles and drive strength
 698          * for write cycles for DQS
 699          */
 700         clrsetbits_le32(&denali_phy[7], 0xffffff, reg_value);
 701         clrsetbits_le32(&denali_phy[135], 0xffffff, reg_value);
 702         clrsetbits_le32(&denali_phy[263], 0xffffff, reg_value);
 703         clrsetbits_le32(&denali_phy[391], 0xffffff, reg_value);
 704
 705         /* phy_adr_tsel_select_ 8bits DENALI_PHY_544/672/800 offset_0 */
 706         reg_value = tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 0x4);
 707         if (params->base.dramtype == LPDDR4) {
 708                 /* LPDDR4 these register read always return 0, so
 709                  * can not use clrsetbits_le32(), need to write32
 710                  */
 711                 writel((0x300 << 8) | reg_value, &denali_phy[544]);
 712                 writel((0x300 << 8) | reg_value, &denali_phy[672]);
 713                 writel((0x300 << 8) | reg_value, &denali_phy[800]);
 714         } else {
 715                 clrsetbits_le32(&denali_phy[544], 0xff, reg_value);
 716                 clrsetbits_le32(&denali_phy[672], 0xff, reg_value);
 717                 clrsetbits_le32(&denali_phy[800], 0xff, reg_value);
 718         }
 719
 720         /* phy_pad_addr_drive 8bits DENALI_PHY_928 offset_0 */
 721         clrsetbits_le32(&denali_phy[928], 0xff, reg_value);
 722
 723         /* phy_pad_rst_drive 8bits DENALI_PHY_937 offset_0 */
 724         if (!ctl_phy_reg)
 725                 clrsetbits_le32(&denali_phy[937], 0xff, reg_value);
 726
 727         /* phy_pad_cke_drive 8bits DENALI_PHY_935 offset_0 */
 728         clrsetbits_le32(&denali_phy[935], 0xff, reg_value);
 729
 730         /* phy_pad_cs_drive 8bits DENALI_PHY_939 offset_0 */
 731         clrsetbits_le32(&denali_phy[939], 0xff,
 732                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 733
 734         /* phy_pad_clk_drive 8bits DENALI_PHY_929 offset_0 */
 735         clrsetbits_le32(&denali_phy[929], 0xff,
 736                         tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4));
 737
 738         /* phy_pad_fdbk_drive 23bit DENALI_PHY_924/925 */
 739         clrsetbits_le32(&denali_phy[924], 0xff,
 740                         tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 4));
 741         clrsetbits_le32(&denali_phy[925], 0xff,
 742                         tsel_wr_select_dq_n | (tsel_wr_select_dq_p << 4));
 743
 744         /* phy_dq_tsel_enable_X 3bits DENALI_PHY_5/133/261/389 offset_16 */
 745         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 746                 << 16;
 747         clrsetbits_le32(&denali_phy[5], 0x7 << 16, reg_value);
 748         clrsetbits_le32(&denali_phy[133], 0x7 << 16, reg_value);
 749         clrsetbits_le32(&denali_phy[261], 0x7 << 16, reg_value);
 750         clrsetbits_le32(&denali_phy[389], 0x7 << 16, reg_value);
 751
 752         /* phy_dqs_tsel_enable_X 3bits DENALI_PHY_6/134/262/390 offset_24 */
 753         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 754                 << 24;
 755         clrsetbits_le32(&denali_phy[6], 0x7 << 24, reg_value);
 756         clrsetbits_le32(&denali_phy[134], 0x7 << 24, reg_value);
 757         clrsetbits_le32(&denali_phy[262], 0x7 << 24, reg_value);
 758         clrsetbits_le32(&denali_phy[390], 0x7 << 24, reg_value);
 759
 760         /* phy_adr_tsel_enable_ 1bit DENALI_PHY_518/646/774 offset_8 */
 761         reg_value = tsel_wr_en << 8;
 762         clrsetbits_le32(&denali_phy[518], 0x1 << 8, reg_value);
 763         clrsetbits_le32(&denali_phy[646], 0x1 << 8, reg_value);
 764         clrsetbits_le32(&denali_phy[774], 0x1 << 8, reg_value);
 765
 766         /* phy_pad_addr_term tsel 1bit DENALI_PHY_933 offset_17 */
 767         reg_value = tsel_wr_en << 17;
 768         clrsetbits_le32(&denali_phy[933], 0x1 << 17, reg_value);
 769         /*
 770          * pad_rst/cke/cs/clk_term tsel 1bits
 771          * DENALI_PHY_938/936/940/934 offset_17
 772          */
 773         clrsetbits_le32(&denali_phy[938], 0x1 << 17, reg_value);
 774         clrsetbits_le32(&denali_phy[936], 0x1 << 17, reg_value);
 775         clrsetbits_le32(&denali_phy[940], 0x1 << 17, reg_value);
 776         clrsetbits_le32(&denali_phy[934], 0x1 << 17, reg_value);
 777
 778         /* phy_pad_fdbk_term 1bit DENALI_PHY_930 offset_17 */
 779         clrsetbits_le32(&denali_phy[930], 0x1 << 17, reg_value);
 780
 781         phy_io_config(denali_phy, denali_ctl, params, mr5);
 782 }
 783
 784 static void pctl_start(struct dram_info *dram,
 785                        struct rk3399_sdram_params *params,
 786                        u32 channel_mask)
 787 {
 788         const struct chan_info *chan_0 = &dram->chan[0];
 789         const struct chan_info *chan_1 = &dram->chan[1];
 790
 791         u32 *denali_ctl_0 = chan_0->pctl->denali_ctl;
 792         u32 *denali_phy_0 = chan_0->publ->denali_phy;
 793         u32 *ddrc0_con_0 = get_ddrc0_con(dram, 0);
 794         u32 *denali_ctl_1 = chan_1->pctl->denali_ctl;
 795         u32 *denali_phy_1 = chan_1->publ->denali_phy;
 796         u32 *ddrc1_con_0 = get_ddrc0_con(dram, 1);
 797         u32 count = 0;
 798         u32 byte, tmp;
 799
 800         /* PHY_DLL_RST_EN */
 801         if (channel_mask & 1) {
 802                 writel(0x01000000, &ddrc0_con_0);
 803                 clrsetbits_le32(&denali_phy_0[957], 0x3 << 24, 0x2 << 24);
 804         }
 805
 806         if (channel_mask & 1) {
 807                 count = 0;
 808                 while (!(readl(&denali_ctl_0[203]) & (1 << 3))) {
 809                         if (count > 1000) {
 810                                 printf("%s: Failed to init pctl channel 0\n",
 811                                        __func__);
 812                                 while (1)
 813                                         ;
 814                         }
 815                         udelay(1);
 816                         count++;
 817                 }
 818
 819                 writel(0x01000100, &ddrc0_con_0);
 820                 for (byte = 0; byte < 4; byte++)        {
 821                         tmp = 0x820;
 822                         writel((tmp << 16) | tmp,
 823                                &denali_phy_0[53 + (128 * byte)]);
 824                         writel((tmp << 16) | tmp,
 825                                &denali_phy_0[54 + (128 * byte)]);
 826                         writel((tmp << 16) | tmp,
 827                                &denali_phy_0[55 + (128 * byte)]);
 828                         writel((tmp << 16) | tmp,
 829                                &denali_phy_0[56 + (128 * byte)]);
 830                         writel((tmp << 16) | tmp,
 831                                &denali_phy_0[57 + (128 * byte)]);
 832                         clrsetbits_le32(&denali_phy_0[58 + (128 * byte)],
 833                                         0xffff, tmp);
 834                 }
 835                 clrsetbits_le32(&denali_ctl_0[68], PWRUP_SREFRESH_EXIT,
 836                                 dram->pwrup_srefresh_exit[0]);
 837         }
 838
 839         if (channel_mask & 2) {
 840                 writel(0x01000000, &ddrc1_con_0);
 841                 clrsetbits_le32(&denali_phy_1[957], 0x3 << 24, 0x2 << 24);
 842         }
 843         if (channel_mask & 2) {
 844                 count = 0;
 845                 while (!(readl(&denali_ctl_1[203]) & (1 << 3))) {
 846                         if (count > 1000) {
 847                                 printf("%s: Failed to init pctl channel 1\n",
 848                                        __func__);
 849                                 while (1)
 850                                         ;
 851                         }
 852                         udelay(1);
 853                         count++;
 854                 }
 855
 856                 writel(0x01000100, &ddrc1_con_0);
 857                 for (byte = 0; byte < 4; byte++)        {
 858                         tmp = 0x820;
 859                         writel((tmp << 16) | tmp,
 860                                &denali_phy_1[53 + (128 * byte)]);
 861                         writel((tmp << 16) | tmp,
 862                                &denali_phy_1[54 + (128 * byte)]);
 863                         writel((tmp << 16) | tmp,
 864                                &denali_phy_1[55 + (128 * byte)]);
 865                         writel((tmp << 16) | tmp,
 866                                &denali_phy_1[56 + (128 * byte)]);
 867                         writel((tmp << 16) | tmp,
 868                                &denali_phy_1[57 + (128 * byte)]);
 869                         clrsetbits_le32(&denali_phy_1[58 + (128 * byte)],
 870                                         0xffff, tmp);
 871                 }
 872
 873                 clrsetbits_le32(&denali_ctl_1[68], PWRUP_SREFRESH_EXIT,
 874                                 dram->pwrup_srefresh_exit[1]);
 875
 876                 /*
 877                  * restore channel 1 RESET original setting
 878                  * to avoid 240ohm too weak to prevent ESD test
 879                  */
 880                 if (params->base.dramtype == LPDDR4)
 881                         clrsetbits_le32(&denali_phy_1[937], 0xff,
 882                                         params->phy_regs.denali_phy[937] &
 883                                         0xFF);
 884         }
 885 }
 886
 887 static int pctl_cfg(struct dram_info *dram, const struct chan_info *chan,
 888                     u32 channel, struct rk3399_sdram_params *params)
 889 {
 890         u32 *denali_ctl = chan->pctl->denali_ctl;
 891         u32 *denali_pi = chan->pi->denali_pi;
 892         u32 *denali_phy = chan->publ->denali_phy;
 893         const u32 *params_ctl = params->pctl_regs.denali_ctl;
 894         const u32 *params_phy = params->phy_regs.denali_phy;
 895         u32 tmp, tmp1, tmp2;
 896         struct rk3399_sdram_params *params_cfg;
 897         u32 byte;
 898
 899         dram->ops->modify_param(chan, params);
 900         /*
 901          * work around controller bug:
 902          * Do not program DRAM_CLASS until NO_PHY_IND_TRAIN_INT is programmed
 903          */
 904         sdram_copy_to_reg(&denali_ctl[1], &params_ctl[1],
 905                           sizeof(struct rk3399_ddr_pctl_regs) - 4);
 906         writel(params_ctl[0], &denali_ctl[0]);
 907
 908         /*
 909          * two channel init at the same time, then ZQ Cal Start
 910          * at the same time, it will use the same RZQ, but cannot
 911          * start at the same time.
 912          *
 913          * So, increase tINIT3 for channel 1, will avoid two
 914          * channel ZQ Cal Start at the same time
 915          */
 916         if (params->base.dramtype == LPDDR4 && channel == 1) {
 917                 tmp = ((params->base.ddr_freq * MHz + 999) / 1000);
 918                 tmp1 = readl(&denali_ctl[14]);
 919                 writel(tmp + tmp1, &denali_ctl[14]);
 920         }
 921
 922         sdram_copy_to_reg(denali_pi, &params->pi_regs.denali_pi[0],
 923                           sizeof(struct rk3399_ddr_pi_regs));
 924
 925         /* rank count need to set for init */
 926         set_memory_map(chan, channel, params);
 927
 928         writel(params->phy_regs.denali_phy[910], &denali_phy[910]);
 929         writel(params->phy_regs.denali_phy[911], &denali_phy[911]);
 930         writel(params->phy_regs.denali_phy[912], &denali_phy[912]);
 931
 932         if (params->base.dramtype == LPDDR4) {
 933                 writel(params->phy_regs.denali_phy[898], &denali_phy[898]);
 934                 writel(params->phy_regs.denali_phy[919], &denali_phy[919]);
 935         }
 936
 937         dram->pwrup_srefresh_exit[channel] = readl(&denali_ctl[68]) &
 938                                              PWRUP_SREFRESH_EXIT;
 939         clrbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT);
 940
 941         /* PHY_DLL_RST_EN */
 942         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 1 << 24);
 943
 944         setbits_le32(&denali_pi[0], START);
 945         setbits_le32(&denali_ctl[0], START);
 946
 947         /**
 948          * LPDDR4 use PLL bypass mode for init
 949          * not need to wait for the PLL to lock
 950          */
 951         if (params->base.dramtype != LPDDR4) {
 952                 /* Waiting for phy DLL lock */
 953                 while (1) {
 954                         tmp = readl(&denali_phy[920]);
 955                         tmp1 = readl(&denali_phy[921]);
 956                         tmp2 = readl(&denali_phy[922]);
 957                         if ((((tmp >> 16) & 0x1) == 0x1) &&
 958                             (((tmp1 >> 16) & 0x1) == 0x1) &&
 959                             (((tmp1 >> 0) & 0x1) == 0x1) &&
 960                             (((tmp2 >> 0) & 0x1) == 0x1))
 961                                 break;
 962                 }
 963         }
 964
 965         sdram_copy_to_reg(&denali_phy[896], &params_phy[896], (958 - 895) * 4);
 966         sdram_copy_to_reg(&denali_phy[0], &params_phy[0], (90 - 0 + 1) * 4);
 967         sdram_copy_to_reg(&denali_phy[128], &params_phy[128],
 968                           (218 - 128 + 1) * 4);
 969         sdram_copy_to_reg(&denali_phy[256], &params_phy[256],
 970                           (346 - 256 + 1) * 4);
 971         sdram_copy_to_reg(&denali_phy[384], &params_phy[384],
 972                           (474 - 384 + 1) * 4);
 973         sdram_copy_to_reg(&denali_phy[512], &params_phy[512],
 974                           (549 - 512 + 1) * 4);
 975         sdram_copy_to_reg(&denali_phy[640], &params_phy[640],
 976                           (677 - 640 + 1) * 4);
 977         sdram_copy_to_reg(&denali_phy[768], &params_phy[768],
 978                           (805 - 768 + 1) * 4);
 979
 980         if (params->base.dramtype == LPDDR4)
 981                 params_cfg = dram->ops->get_phy_index_params(1, params);
 982         else
 983                 params_cfg = dram->ops->get_phy_index_params(0, params);
 984
 985         clrsetbits_le32(&params_cfg->phy_regs.denali_phy[896], 0x3 << 8,
 986                         0 << 8);
 987         writel(params_cfg->phy_regs.denali_phy[896], &denali_phy[896]);
 988
 989         writel(params->phy_regs.denali_phy[83] + (0x10 << 16),
 990                &denali_phy[83]);
 991         writel(params->phy_regs.denali_phy[84] + (0x10 << 8),
 992                &denali_phy[84]);
 993         writel(params->phy_regs.denali_phy[211] + (0x10 << 16),
 994                &denali_phy[211]);
 995         writel(params->phy_regs.denali_phy[212] + (0x10 << 8),
 996                &denali_phy[212]);
 997         writel(params->phy_regs.denali_phy[339] + (0x10 << 16),
 998                &denali_phy[339]);
 999         writel(params->phy_regs.denali_phy[340] + (0x10 << 8),
1000                &denali_phy[340]);
1001         writel(params->phy_regs.denali_phy[467] + (0x10 << 16),
1002                &denali_phy[467]);
1003         writel(params->phy_regs.denali_phy[468] + (0x10 << 8),
1004                &denali_phy[468]);
1005
1006         if (params->base.dramtype == LPDDR4) {
1007                 /*
1008                  * to improve write dqs and dq phase from 1.5ns to 3.5ns
1009                  * at 50MHz. this's the measure result from oscilloscope
1010                  * of dqs and dq write signal.
1011                  */
1012                 for (byte = 0; byte < 4; byte++) {
1013                         tmp = 0x680;
1014                         clrsetbits_le32(&denali_phy[1 + (128 * byte)],
1015                                         0xfff << 8, tmp << 8);
1016                 }
1017                 /*
1018                  * to workaround 366ball two channel's RESET connect to
1019                  * one RESET signal of die
1020                  */
1021                 if (channel == 1)
1022                         clrsetbits_le32(&denali_phy[937], 0xff,
1023                                         PHY_DRV_ODT_240 |
1024                                         (PHY_DRV_ODT_240 << 0x4));
1025         }
1026
1027         return 0;
1028 }
1029
1030 static void select_per_cs_training_index(const struct chan_info *chan,
1031                                          u32 rank)
1032 {
1033         u32 *denali_phy = chan->publ->denali_phy;
1034
1035         /* PHY_84 PHY_PER_CS_TRAINING_EN_0 1bit offset_16 */
1036         if ((readl(&denali_phy[84]) >> 16) & 1) {
1037                 /*
1038                  * PHY_8/136/264/392
1039                  * phy_per_cs_training_index_X 1bit offset_24
1040                  */
1041                 clrsetbits_le32(&denali_phy[8], 0x1 << 24, rank << 24);
1042                 clrsetbits_le32(&denali_phy[136], 0x1 << 24, rank << 24);
1043                 clrsetbits_le32(&denali_phy[264], 0x1 << 24, rank << 24);
1044                 clrsetbits_le32(&denali_phy[392], 0x1 << 24, rank << 24);
1045         }
1046 }
1047
1048 static void override_write_leveling_value(const struct chan_info *chan)
1049 {
1050         u32 *denali_ctl = chan->pctl->denali_ctl;
1051         u32 *denali_phy = chan->publ->denali_phy;
1052         u32 byte;
1053
1054         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
1055         setbits_le32(&denali_phy[896], 1);
1056
1057         /*
1058          * PHY_8/136/264/392
1059          * phy_per_cs_training_multicast_en_X 1bit offset_16
1060          */
1061         clrsetbits_le32(&denali_phy[8], 0x1 << 16, 1 << 16);
1062         clrsetbits_le32(&denali_phy[136], 0x1 << 16, 1 << 16);
1063         clrsetbits_le32(&denali_phy[264], 0x1 << 16, 1 << 16);
1064         clrsetbits_le32(&denali_phy[392], 0x1 << 16, 1 << 16);
1065
1066         for (byte = 0; byte < 4; byte++)
1067                 clrsetbits_le32(&denali_phy[63 + (128 * byte)], 0xffff << 16,
1068                                 0x200 << 16);
1069
1070         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
1071         clrbits_le32(&denali_phy[896], 1);
1072
1073         /* CTL_200 ctrlupd_req 1bit offset_8 */
1074         clrsetbits_le32(&denali_ctl[200], 0x1 << 8, 0x1 << 8);
1075 }
1076
1077 static int data_training_ca(const struct chan_info *chan, u32 channel,
1078                             const struct rk3399_sdram_params *params)
1079 {
1080         u32 *denali_pi = chan->pi->denali_pi;
1081         u32 *denali_phy = chan->publ->denali_phy;
1082         u32 i, tmp;
1083         u32 obs_0, obs_1, obs_2, obs_err = 0;
1084         u32 rank = params->ch[channel].cap_info.rank;
1085         u32 rank_mask;
1086
1087         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1088         writel(0x00003f7c, (&denali_pi[175]));
1089
1090         if (params->base.dramtype == LPDDR4)
1091                 rank_mask = (rank == 1) ? 0x5 : 0xf;
1092         else
1093                 rank_mask = (rank == 1) ? 0x1 : 0x3;
1094
1095         for (i = 0; i < 4; i++) {
1096                 if (!(rank_mask & (1 << i)))
1097                         continue;
1098
1099                 select_per_cs_training_index(chan, i);
1100
1101                 /* PI_100 PI_CALVL_EN:RW:8:2 */
1102                 clrsetbits_le32(&denali_pi[100], 0x3 << 8, 0x2 << 8);
1103
1104                 /* PI_92 PI_CALVL_REQ:WR:16:1,PI_CALVL_CS:RW:24:2 */
1105                 clrsetbits_le32(&denali_pi[92],
1106                                 (0x1 << 16) | (0x3 << 24),
1107                                 (0x1 << 16) | (i << 24));
1108
1109                 /* Waiting for training complete */
1110                 while (1) {
1111                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1112                         tmp = readl(&denali_pi[174]) >> 8;
1113                         /*
1114                          * check status obs
1115                          * PHY_532/660/789 phy_adr_calvl_obs1_:0:32
1116                          */
1117                         obs_0 = readl(&denali_phy[532]);
1118                         obs_1 = readl(&denali_phy[660]);
1119                         obs_2 = readl(&denali_phy[788]);
1120                         if (((obs_0 >> 30) & 0x3) ||
1121                             ((obs_1 >> 30) & 0x3) ||
1122                             ((obs_2 >> 30) & 0x3))
1123                                 obs_err = 1;
1124                         if ((((tmp >> 11) & 0x1) == 0x1) &&
1125                             (((tmp >> 13) & 0x1) == 0x1) &&
1126                             (((tmp >> 5) & 0x1) == 0x0) &&
1127                             obs_err == 0)
1128                                 break;
1129                         else if ((((tmp >> 5) & 0x1) == 0x1) ||
1130                                  (obs_err == 1))
1131                                 return -EIO;
1132                 }
1133
1134                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1135                 writel(0x00003f7c, (&denali_pi[175]));
1136         }
1137
1138         clrbits_le32(&denali_pi[100], 0x3 << 8);
1139
1140         return 0;
1141 }
1142
1143 static int data_training_wl(const struct chan_info *chan, u32 channel,
1144                             const struct rk3399_sdram_params *params)
1145 {
1146         u32 *denali_pi = chan->pi->denali_pi;
1147         u32 *denali_phy = chan->publ->denali_phy;
1148         u32 i, tmp;
1149         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1150         u32 rank = params->ch[channel].cap_info.rank;
1151
1152         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1153         writel(0x00003f7c, (&denali_pi[175]));
1154
1155         for (i = 0; i < rank; i++) {
1156                 select_per_cs_training_index(chan, i);
1157
1158                 /* PI_60 PI_WRLVL_EN:RW:8:2 */
1159                 clrsetbits_le32(&denali_pi[60], 0x3 << 8, 0x2 << 8);
1160
1161                 /* PI_59 PI_WRLVL_REQ:WR:8:1,PI_WRLVL_CS:RW:16:2 */
1162                 clrsetbits_le32(&denali_pi[59],
1163                                 (0x1 << 8) | (0x3 << 16),
1164                                 (0x1 << 8) | (i << 16));
1165
1166                 /* Waiting for training complete */
1167                 while (1) {
1168                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1169                         tmp = readl(&denali_pi[174]) >> 8;
1170
1171                         /*
1172                          * check status obs, if error maybe can not
1173                          * get leveling done PHY_40/168/296/424
1174                          * phy_wrlvl_status_obs_X:0:13
1175                          */
1176                         obs_0 = readl(&denali_phy[40]);
1177                         obs_1 = readl(&denali_phy[168]);
1178                         obs_2 = readl(&denali_phy[296]);
1179                         obs_3 = readl(&denali_phy[424]);
1180                         if (((obs_0 >> 12) & 0x1) ||
1181                             ((obs_1 >> 12) & 0x1) ||
1182                             ((obs_2 >> 12) & 0x1) ||
1183                             ((obs_3 >> 12) & 0x1))
1184                                 obs_err = 1;
1185                         if ((((tmp >> 10) & 0x1) == 0x1) &&
1186                             (((tmp >> 13) & 0x1) == 0x1) &&
1187                             (((tmp >> 4) & 0x1) == 0x0) &&
1188                             obs_err == 0)
1189                                 break;
1190                         else if ((((tmp >> 4) & 0x1) == 0x1) ||
1191                                  (obs_err == 1))
1192                                 return -EIO;
1193                 }
1194
1195                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1196                 writel(0x00003f7c, (&denali_pi[175]));
1197         }
1198
1199         override_write_leveling_value(chan);
1200         clrbits_le32(&denali_pi[60], 0x3 << 8);
1201
1202         return 0;
1203 }
1204
1205 static int data_training_rg(const struct chan_info *chan, u32 channel,
1206                             const struct rk3399_sdram_params *params)
1207 {
1208         u32 *denali_pi = chan->pi->denali_pi;
1209         u32 *denali_phy = chan->publ->denali_phy;
1210         u32 i, tmp;
1211         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
1212         u32 rank = params->ch[channel].cap_info.rank;
1213
1214         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1215         writel(0x00003f7c, (&denali_pi[175]));
1216
1217         for (i = 0; i < rank; i++) {
1218                 select_per_cs_training_index(chan, i);
1219
1220                 /* PI_80 PI_RDLVL_GATE_EN:RW:24:2 */
1221                 clrsetbits_le32(&denali_pi[80], 0x3 << 24, 0x2 << 24);
1222
1223                 /*
1224                  * PI_74 PI_RDLVL_GATE_REQ:WR:16:1
1225                  * PI_RDLVL_CS:RW:24:2
1226                  */
1227                 clrsetbits_le32(&denali_pi[74],
1228                                 (0x1 << 16) | (0x3 << 24),
1229                                 (0x1 << 16) | (i << 24));
1230
1231                 /* Waiting for training complete */
1232                 while (1) {
1233                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1234                         tmp = readl(&denali_pi[174]) >> 8;
1235
1236                         /*
1237                          * check status obs
1238                          * PHY_43/171/299/427
1239                          *     PHY_GTLVL_STATUS_OBS_x:16:8
1240                          */
1241                         obs_0 = readl(&denali_phy[43]);
1242                         obs_1 = readl(&denali_phy[171]);
1243                         obs_2 = readl(&denali_phy[299]);
1244                         obs_3 = readl(&denali_phy[427]);
1245                         if (((obs_0 >> (16 + 6)) & 0x3) ||
1246                             ((obs_1 >> (16 + 6)) & 0x3) ||
1247                             ((obs_2 >> (16 + 6)) & 0x3) ||
1248                             ((obs_3 >> (16 + 6)) & 0x3))
1249                                 obs_err = 1;
1250                         if ((((tmp >> 9) & 0x1) == 0x1) &&
1251                             (((tmp >> 13) & 0x1) == 0x1) &&
1252                             (((tmp >> 3) & 0x1) == 0x0) &&
1253                             obs_err == 0)
1254                                 break;
1255                         else if ((((tmp >> 3) & 0x1) == 0x1) ||
1256                                  (obs_err == 1))
1257                                 return -EIO;
1258                 }
1259
1260                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1261                 writel(0x00003f7c, (&denali_pi[175]));
1262         }
1263
1264         clrbits_le32(&denali_pi[80], 0x3 << 24);
1265
1266         return 0;
1267 }
1268
1269 static int data_training_rl(const struct chan_info *chan, u32 channel,
1270                             const struct rk3399_sdram_params *params)
1271 {
1272         u32 *denali_pi = chan->pi->denali_pi;
1273         u32 i, tmp;
1274         u32 rank = params->ch[channel].cap_info.rank;
1275
1276         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1277         writel(0x00003f7c, (&denali_pi[175]));
1278
1279         for (i = 0; i < rank; i++) {
1280                 select_per_cs_training_index(chan, i);
1281
1282                 /* PI_80 PI_RDLVL_EN:RW:16:2 */
1283                 clrsetbits_le32(&denali_pi[80], 0x3 << 16, 0x2 << 16);
1284
1285                 /* PI_74 PI_RDLVL_REQ:WR:8:1,PI_RDLVL_CS:RW:24:2 */
1286                 clrsetbits_le32(&denali_pi[74],
1287                                 (0x1 << 8) | (0x3 << 24),
1288                                 (0x1 << 8) | (i << 24));
1289
1290                 /* Waiting for training complete */
1291                 while (1) {
1292                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1293                         tmp = readl(&denali_pi[174]) >> 8;
1294
1295                         /*
1296                          * make sure status obs not report error bit
1297                          * PHY_46/174/302/430
1298                          *     phy_rdlvl_status_obs_X:16:8
1299                          */
1300                         if ((((tmp >> 8) & 0x1) == 0x1) &&
1301                             (((tmp >> 13) & 0x1) == 0x1) &&
1302                             (((tmp >> 2) & 0x1) == 0x0))
1303                                 break;
1304                         else if (((tmp >> 2) & 0x1) == 0x1)
1305                                 return -EIO;
1306                 }
1307
1308                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1309                 writel(0x00003f7c, (&denali_pi[175]));
1310         }
1311
1312         clrbits_le32(&denali_pi[80], 0x3 << 16);
1313
1314         return 0;
1315 }
1316
1317 static int data_training_wdql(const struct chan_info *chan, u32 channel,
1318                               const struct rk3399_sdram_params *params)
1319 {
1320         u32 *denali_pi = chan->pi->denali_pi;
1321         u32 i, tmp;
1322         u32 rank = params->ch[channel].cap_info.rank;
1323         u32 rank_mask;
1324
1325         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1326         writel(0x00003f7c, (&denali_pi[175]));
1327
1328         if (params->base.dramtype == LPDDR4)
1329                 rank_mask = (rank == 1) ? 0x5 : 0xf;
1330         else
1331                 rank_mask = (rank == 1) ? 0x1 : 0x3;
1332
1333         for (i = 0; i < 4; i++) {
1334                 if (!(rank_mask & (1 << i)))
1335                         continue;
1336
1337                 select_per_cs_training_index(chan, i);
1338
1339                 /*
1340                  * disable PI_WDQLVL_VREF_EN before wdq leveling?
1341                  * PI_117 PI_WDQLVL_VREF_EN:RW:8:1
1342                  */
1343                 clrbits_le32(&denali_pi[117], 0x1 << 8);
1344                 /* PI_124 PI_WDQLVL_EN:RW:16:2 */
1345                 clrsetbits_le32(&denali_pi[124], 0x3 << 16, 0x2 << 16);
1346
1347                 /* PI_121 PI_WDQLVL_REQ:WR:8:1,PI_WDQLVL_CS:RW:16:2 */
1348                 clrsetbits_le32(&denali_pi[121],
1349                                 (0x1 << 8) | (0x3 << 16),
1350                                 (0x1 << 8) | (i << 16));
1351
1352                 /* Waiting for training complete */
1353                 while (1) {
1354                         /* PI_174 PI_INT_STATUS:RD:8:18 */
1355                         tmp = readl(&denali_pi[174]) >> 8;
1356                         if ((((tmp >> 12) & 0x1) == 0x1) &&
1357                             (((tmp >> 13) & 0x1) == 0x1) &&
1358                             (((tmp >> 6) & 0x1) == 0x0))
1359                                 break;
1360                         else if (((tmp >> 6) & 0x1) == 0x1)
1361                                 return -EIO;
1362                 }
1363
1364                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
1365                 writel(0x00003f7c, (&denali_pi[175]));
1366         }
1367
1368         clrbits_le32(&denali_pi[124], 0x3 << 16);
1369
1370         return 0;
1371 }
1372
1373 static int data_training(struct dram_info *dram, u32 channel,
1374                          const struct rk3399_sdram_params *params,
1375                          u32 training_flag)
1376 {
1377         struct chan_info *chan = &dram->chan[channel];
1378         u32 *denali_phy = chan->publ->denali_phy;
1379         int ret;
1380
1381         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1382         setbits_le32(&denali_phy[927], (1 << 22));
1383
1384         if (training_flag == PI_FULL_TRAINING) {
1385                 if (params->base.dramtype == LPDDR4) {
1386                         training_flag = PI_WRITE_LEVELING |
1387                                         PI_READ_GATE_TRAINING |
1388                                         PI_READ_LEVELING | PI_WDQ_LEVELING;
1389                 } else if (params->base.dramtype == LPDDR3) {
1390                         training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING |
1391                                         PI_READ_GATE_TRAINING;
1392                 } else if (params->base.dramtype == DDR3) {
1393                         training_flag = PI_WRITE_LEVELING |
1394                                         PI_READ_GATE_TRAINING |
1395                                         PI_READ_LEVELING;
1396                 }
1397         }
1398
1399         /* ca training(LPDDR4,LPDDR3 support) */
1400         if ((training_flag & PI_CA_TRAINING) == PI_CA_TRAINING) {
1401                 ret = data_training_ca(chan, channel, params);
1402                 if (ret < 0) {
1403                         debug("%s: data training ca failed\n", __func__);
1404                         return ret;
1405                 }
1406         }
1407
1408         /* write leveling(LPDDR4,LPDDR3,DDR3 support) */
1409         if ((training_flag & PI_WRITE_LEVELING) == PI_WRITE_LEVELING) {
1410                 ret = data_training_wl(chan, channel, params);
1411                 if (ret < 0) {
1412                         debug("%s: data training wl failed\n", __func__);
1413                         return ret;
1414                 }
1415         }
1416
1417         /* read gate training(LPDDR4,LPDDR3,DDR3 support) */
1418         if ((training_flag & PI_READ_GATE_TRAINING) == PI_READ_GATE_TRAINING) {
1419                 ret = data_training_rg(chan, channel, params);
1420                 if (ret < 0) {
1421                         debug("%s: data training rg failed\n", __func__);
1422                         return ret;
1423                 }
1424         }
1425
1426         /* read leveling(LPDDR4,LPDDR3,DDR3 support) */
1427         if ((training_flag & PI_READ_LEVELING) == PI_READ_LEVELING) {
1428                 ret = data_training_rl(chan, channel, params);
1429                 if (ret < 0) {
1430                         debug("%s: data training rl failed\n", __func__);
1431                         return ret;
1432                 }
1433         }
1434
1435         /* wdq leveling(LPDDR4 support) */
1436         if ((training_flag & PI_WDQ_LEVELING) == PI_WDQ_LEVELING) {
1437                 ret = data_training_wdql(chan, channel, params);
1438                 if (ret < 0) {
1439                         debug("%s: data training wdql failed\n", __func__);
1440                         return ret;
1441                 }
1442         }
1443
1444         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1445         clrbits_le32(&denali_phy[927], (1 << 22));
1446
1447         return 0;
1448 }
1449
1450 static void set_ddrconfig(const struct chan_info *chan,
1451                           const struct rk3399_sdram_params *params,
1452                           unsigned char channel, u32 ddrconfig)
1453 {
1454         /* only need to set ddrconfig */
1455         struct msch_regs *ddr_msch_regs = chan->msch;
1456         unsigned int cs0_cap = 0;
1457         unsigned int cs1_cap = 0;
1458
1459         cs0_cap = (1 << (params->ch[channel].cap_info.cs0_row
1460                         + params->ch[channel].cap_info.col
1461                         + params->ch[channel].cap_info.bk
1462                         + params->ch[channel].cap_info.bw - 20));
1463         if (params->ch[channel].cap_info.rank > 1)
1464                 cs1_cap = cs0_cap >> (params->ch[channel].cap_info.cs0_row
1465                                 - params->ch[channel].cap_info.cs1_row);
1466         if (params->ch[channel].cap_info.row_3_4) {
1467                 cs0_cap = cs0_cap * 3 / 4;
1468                 cs1_cap = cs1_cap * 3 / 4;
1469         }
1470
1471         writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf);
1472         writel(((cs0_cap / 32) & 0xff) | (((cs1_cap / 32) & 0xff) << 8),
1473                &ddr_msch_regs->ddrsize);
1474 }
1475
1476 static void sdram_msch_config(struct msch_regs *msch,
1477                               struct sdram_msch_timings *noc_timings)
1478 {
1479         writel(noc_timings->ddrtiminga0.d32,
1480                &msch->ddrtiminga0.d32);
1481         writel(noc_timings->ddrtimingb0.d32,
1482                &msch->ddrtimingb0.d32);
1483         writel(noc_timings->ddrtimingc0.d32,
1484                &msch->ddrtimingc0.d32);
1485         writel(noc_timings->devtodev0.d32,
1486                &msch->devtodev0.d32);
1487         writel(noc_timings->ddrmode.d32,
1488                &msch->ddrmode.d32);
1489 }
1490
1491 static void dram_all_config(struct dram_info *dram,
1492                             struct rk3399_sdram_params *params)
1493 {
1494         u32 sys_reg2 = 0;
1495         u32 sys_reg3 = 0;
1496         unsigned int channel, idx;
1497
1498         for (channel = 0, idx = 0;
1499              (idx < params->base.num_channels) && (channel < 2);
1500              channel++) {
1501                 struct msch_regs *ddr_msch_regs;
1502                 struct sdram_msch_timings *noc_timing;
1503
1504                 if (params->ch[channel].cap_info.col == 0)
1505                         continue;
1506                 idx++;
1507                 sdram_org_config(&params->ch[channel].cap_info,
1508                                  &params->base, &sys_reg2,
1509                                  &sys_reg3, channel);
1510                 ddr_msch_regs = dram->chan[channel].msch;
1511                 noc_timing = &params->ch[channel].noc_timings;
1512                 sdram_msch_config(ddr_msch_regs, noc_timing);
1513
1514                 /**
1515                  * rank 1 memory clock disable (dfi_dram_clk_disable = 1)
1516                  *
1517                  * The hardware for LPDDR4 with
1518                  * - CLK0P/N connect to lower 16-bits
1519                  * - CLK1P/N connect to higher 16-bits
1520                  *
1521                  * dfi dram clk is configured via CLK1P/N, so disabling
1522                  * dfi dram clk will disable the CLK1P/N as well for lpddr4.
1523                  */
1524                 if (params->ch[channel].cap_info.rank == 1 &&
1525                     params->base.dramtype != LPDDR4)
1526                         setbits_le32(&dram->chan[channel].pctl->denali_ctl[276],
1527                                      1 << 17);
1528         }
1529
1530         writel(sys_reg2, &dram->pmugrf->os_reg2);
1531         writel(sys_reg3, &dram->pmugrf->os_reg3);
1532         rk_clrsetreg(&dram->pmusgrf->soc_con4, 0x1f << 10,
1533                      params->base.stride << 10);
1534
1535         /* reboot hold register set */
1536         writel(PRESET_SGRF_HOLD(0) | PRESET_GPIO0_HOLD(1) |
1537                 PRESET_GPIO1_HOLD(1),
1538                 &dram->pmucru->pmucru_rstnhold_con[1]);
1539         clrsetbits_le32(&dram->cru->glb_rst_con, 0x3, 0x3);
1540 }
1541
1542 static void set_cap_relate_config(const struct chan_info *chan,
1543                                   struct rk3399_sdram_params *params,
1544                                   unsigned int channel)
1545 {
1546         u32 *denali_ctl = chan->pctl->denali_ctl;
1547         u32 tmp;
1548         struct sdram_msch_timings *noc_timing;
1549
1550         if (params->base.dramtype == LPDDR3) {
1551                 tmp = (8 << params->ch[channel].cap_info.bw) /
1552                         (8 << params->ch[channel].cap_info.dbw);
1553
1554                 /**
1555                  * memdata_ratio
1556                  * 1 -> 0, 2 -> 1, 4 -> 2
1557                  */
1558                 clrsetbits_le32(&denali_ctl[197], 0x7,
1559                                 (tmp >> 1));
1560                 clrsetbits_le32(&denali_ctl[198], 0x7 << 8,
1561                                 (tmp >> 1) << 8);
1562         }
1563
1564         noc_timing = &params->ch[channel].noc_timings;
1565
1566         /*
1567          * noc timing bw relate timing is 32 bit, and real bw is 16bit
1568          * actually noc reg is setting at function dram_all_config
1569          */
1570         if (params->ch[channel].cap_info.bw == 16 &&
1571             noc_timing->ddrmode.b.mwrsize == 2) {
1572                 if (noc_timing->ddrmode.b.burstsize)
1573                         noc_timing->ddrmode.b.burstsize -= 1;
1574                 noc_timing->ddrmode.b.mwrsize -= 1;
1575                 noc_timing->ddrtimingc0.b.burstpenalty *= 2;
1576                 noc_timing->ddrtimingc0.b.wrtomwr *= 2;
1577         }
1578 }
1579
1580 static u32 calculate_ddrconfig(struct rk3399_sdram_params *params, u32 channel)
1581 {
1582         unsigned int cs0_row = params->ch[channel].cap_info.cs0_row;
1583         unsigned int col = params->ch[channel].cap_info.col;
1584         unsigned int bw = params->ch[channel].cap_info.bw;
1585         u16  ddr_cfg_2_rbc[] = {
1586                 /*
1587                  * [6]    highest bit col
1588                  * [5:3]  max row(14+n)
1589                  * [2]    insertion row
1590                  * [1:0]  col(9+n),col, data bus 32bit
1591                  *
1592                  * highbitcol, max_row, insertion_row,  col
1593                  */
1594                 ((0 << 6) | (2 << 3) | (0 << 2) | 0), /* 0 */
1595                 ((0 << 6) | (2 << 3) | (0 << 2) | 1), /* 1 */
1596                 ((0 << 6) | (1 << 3) | (0 << 2) | 2), /* 2 */
1597                 ((0 << 6) | (0 << 3) | (0 << 2) | 3), /* 3 */
1598                 ((0 << 6) | (2 << 3) | (1 << 2) | 1), /* 4 */
1599                 ((0 << 6) | (1 << 3) | (1 << 2) | 2), /* 5 */
1600                 ((1 << 6) | (0 << 3) | (0 << 2) | 2), /* 6 */
1601                 ((1 << 6) | (1 << 3) | (0 << 2) | 2), /* 7 */
1602         };
1603         u32 i;
1604
1605         col -= (bw == 2) ? 0 : 1;
1606         col -= 9;
1607
1608         for (i = 0; i < 4; i++) {
1609                 if ((col == (ddr_cfg_2_rbc[i] & 0x3)) &&
1610                     (cs0_row <= (((ddr_cfg_2_rbc[i] >> 3) & 0x7) + 14)))
1611                         break;
1612         }
1613
1614         if (i >= 4)
1615                 i = -EINVAL;
1616
1617         return i;
1618 }
1619
1620 static void set_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf, u32 stride)
1621 {
1622         rk_clrsetreg(&pmusgrf->soc_con4, 0x1f << 10, stride << 10);
1623 }
1624
1625 #if !defined(CONFIG_RAM_RK3399_LPDDR4)
1626 static int data_training_first(struct dram_info *dram, u32 channel, u8 rank,
1627                                struct rk3399_sdram_params *params)
1628 {
1629         u8 training_flag = PI_READ_GATE_TRAINING;
1630
1631         /*
1632          * LPDDR3 CA training msut be trigger before
1633          * other training.
1634          * DDR3 is not have CA training.
1635          */
1636
1637         if (params->base.dramtype == LPDDR3)
1638                 training_flag |= PI_CA_TRAINING;
1639
1640         return data_training(dram, channel, params, training_flag);
1641 }
1642
1643 static int switch_to_phy_index1(struct dram_info *dram,
1644                                 struct rk3399_sdram_params *params)
1645 {
1646         u32 channel;
1647         u32 *denali_phy;
1648         u32 ch_count = params->base.num_channels;
1649         int ret;
1650         int i = 0;
1651
1652         writel(RK_CLRSETBITS(0x03 << 4 | 1 << 2 | 1,
1653                              1 << 4 | 1 << 2 | 1),
1654                         &dram->cic->cic_ctrl0);
1655         while (!(readl(&dram->cic->cic_status0) & (1 << 2))) {
1656                 mdelay(10);
1657                 i++;
1658                 if (i > 10) {
1659                         debug("index1 frequency change overtime\n");
1660                         return -ETIME;
1661                 }
1662         }
1663
1664         i = 0;
1665         writel(RK_CLRSETBITS(1 << 1, 1 << 1), &dram->cic->cic_ctrl0);
1666         while (!(readl(&dram->cic->cic_status0) & (1 << 0))) {
1667                 mdelay(10);
1668                 i++;
1669                 if (i > 10) {
1670                         debug("index1 frequency done overtime\n");
1671                         return -ETIME;
1672                 }
1673         }
1674
1675         for (channel = 0; channel < ch_count; channel++) {
1676                 denali_phy = dram->chan[channel].publ->denali_phy;
1677                 clrsetbits_le32(&denali_phy[896], (0x3 << 8) | 1, 1 << 8);
1678                 ret = data_training(dram, channel, params, PI_FULL_TRAINING);
1679                 if (ret < 0) {
1680                         debug("index1 training failed\n");
1681                         return ret;
1682                 }
1683         }
1684
1685         return 0;
1686 }
1687
1688 struct rk3399_sdram_params
1689         *get_phy_index_params(u32 phy_fn,
1690                               struct rk3399_sdram_params *params)
1691 {
1692         if (phy_fn == 0)
1693                 return params;
1694         else
1695                 return NULL;
1696 }
1697
1698 void modify_param(const struct chan_info *chan,
1699                   struct rk3399_sdram_params *params)
1700 {
1701         struct rk3399_sdram_params *params_cfg;
1702         u32 *denali_pi_params;
1703
1704         denali_pi_params = params->pi_regs.denali_pi;
1705
1706         /* modify PHY F0/F1/F2 params */
1707         params_cfg = get_phy_index_params(0, params);
1708         set_ds_odt(chan, params_cfg, false, 0);
1709
1710         clrsetbits_le32(&denali_pi_params[45], 0x1 << 24, 0x1 << 24);
1711         clrsetbits_le32(&denali_pi_params[61], 0x1 << 24, 0x1 << 24);
1712         clrsetbits_le32(&denali_pi_params[76], 0x1 << 24, 0x1 << 24);
1713         clrsetbits_le32(&denali_pi_params[77], 0x1, 0x1);
1714 }
1715 #else
1716
1717 struct rk3399_sdram_params dfs_cfgs_lpddr4[] = {
1718 #include "sdram-rk3399-lpddr4-400.inc"
1719 #include "sdram-rk3399-lpddr4-800.inc"
1720 };
1721
1722 static struct rk3399_sdram_params
1723         *lpddr4_get_phy_index_params(u32 phy_fn,
1724                                      struct rk3399_sdram_params *params)
1725 {
1726         if (phy_fn == 0)
1727                 return params;
1728         else if (phy_fn == 1)
1729                 return &dfs_cfgs_lpddr4[1];
1730         else if (phy_fn == 2)
1731                 return &dfs_cfgs_lpddr4[0];
1732         else
1733                 return NULL;
1734 }
1735
1736 static void *get_denali_pi(const struct chan_info *chan,
1737                            struct rk3399_sdram_params *params, bool reg)
1738 {
1739         return reg ? &chan->pi->denali_pi : &params->pi_regs.denali_pi;
1740 }
1741
1742 static u32 lpddr4_get_phy_fn(struct rk3399_sdram_params *params, u32 ctl_fn)
1743 {
1744         u32 lpddr4_phy_fn[] = {1, 0, 0xb};
1745
1746         return lpddr4_phy_fn[ctl_fn];
1747 }
1748
1749 static u32 lpddr4_get_ctl_fn(struct rk3399_sdram_params *params, u32 phy_fn)
1750 {
1751         u32 lpddr4_ctl_fn[] = {1, 0, 2};
1752
1753         return lpddr4_ctl_fn[phy_fn];
1754 }
1755
1756 static u32 get_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf)
1757 {
1758         return ((readl(&pmusgrf->soc_con4) >> 10) & 0x1F);
1759 }
1760
1761 /*
1762  * read mr_num mode register
1763  * rank = 1: cs0
1764  * rank = 2: cs1
1765  */
1766 static int read_mr(struct rk3399_ddr_pctl_regs *ddr_pctl_regs, u32 rank,
1767                    u32 mr_num, u32 *buf)
1768 {
1769         s32 timeout = 100;
1770
1771         writel(((1 << 16) | (((rank == 2) ? 1 : 0) << 8) | mr_num) << 8,
1772                &ddr_pctl_regs->denali_ctl[118]);
1773
1774         while (0 == (readl(&ddr_pctl_regs->denali_ctl[203]) &
1775                         ((1 << 21) | (1 << 12)))) {
1776                 udelay(1);
1777
1778                 if (timeout <= 0) {
1779                         printf("%s: pctl timeout!\n", __func__);
1780                         return -ETIMEDOUT;
1781                 }
1782
1783                 timeout--;
1784         }
1785
1786         if (!(readl(&ddr_pctl_regs->denali_ctl[203]) & (1 << 12))) {
1787                 *buf = readl(&ddr_pctl_regs->denali_ctl[119]) & 0xFF;
1788         } else {
1789                 printf("%s: read mr failed with 0x%x status\n", __func__,
1790                        readl(&ddr_pctl_regs->denali_ctl[17]) & 0x3);
1791                 *buf = 0;
1792         }
1793
1794         setbits_le32(&ddr_pctl_regs->denali_ctl[205], (1 << 21) | (1 << 12));
1795
1796         return 0;
1797 }
1798
1799 static int lpddr4_mr_detect(struct dram_info *dram, u32 channel, u8 rank,
1800                             struct rk3399_sdram_params *params)
1801 {
1802         u64 cs0_cap;
1803         u32 stride;
1804         u32 cs = 0, col = 0, bk = 0, bw = 0, row_3_4 = 0;
1805         u32 cs0_row = 0, cs1_row = 0, ddrconfig = 0;
1806         u32 mr5, mr12, mr14;
1807         struct chan_info *chan = &dram->chan[channel];
1808         struct rk3399_ddr_pctl_regs *ddr_pctl_regs = chan->pctl;
1809         void __iomem *addr = NULL;
1810         int ret = 0;
1811         u32 val;
1812
1813         stride = get_ddr_stride(dram->pmusgrf);
1814
1815         if (params->ch[channel].cap_info.col == 0) {
1816                 ret = -EPERM;
1817                 goto end;
1818         }
1819
1820         cs = params->ch[channel].cap_info.rank;
1821         col = params->ch[channel].cap_info.col;
1822         bk = params->ch[channel].cap_info.bk;
1823         bw = params->ch[channel].cap_info.bw;
1824         row_3_4 = params->ch[channel].cap_info.row_3_4;
1825         cs0_row = params->ch[channel].cap_info.cs0_row;
1826         cs1_row = params->ch[channel].cap_info.cs1_row;
1827         ddrconfig = params->ch[channel].cap_info.ddrconfig;
1828
1829         /* 2GB */
1830         params->ch[channel].cap_info.rank = 2;
1831         params->ch[channel].cap_info.col = 10;
1832         params->ch[channel].cap_info.bk = 3;
1833         params->ch[channel].cap_info.bw = 2;
1834         params->ch[channel].cap_info.row_3_4 = 0;
1835         params->ch[channel].cap_info.cs0_row = 15;
1836         params->ch[channel].cap_info.cs1_row = 15;
1837         params->ch[channel].cap_info.ddrconfig = 1;
1838
1839         set_memory_map(chan, channel, params);
1840         params->ch[channel].cap_info.ddrconfig =
1841                         calculate_ddrconfig(params, channel);
1842         set_ddrconfig(chan, params, channel,
1843                       params->ch[channel].cap_info.ddrconfig);
1844         set_cap_relate_config(chan, params, channel);
1845
1846         cs0_cap = (1 << (params->ch[channel].cap_info.bw
1847                         + params->ch[channel].cap_info.col
1848                         + params->ch[channel].cap_info.bk
1849                         + params->ch[channel].cap_info.cs0_row));
1850
1851         if (params->ch[channel].cap_info.row_3_4)
1852                 cs0_cap = cs0_cap * 3 / 4;
1853
1854         if (channel == 0)
1855                 set_ddr_stride(dram->pmusgrf, 0x17);
1856         else
1857                 set_ddr_stride(dram->pmusgrf, 0x18);
1858
1859         /* read and write data to DRAM, avoid be optimized by compiler. */
1860         if (rank == 1)
1861                 addr = (void __iomem *)0x100;
1862         else if (rank == 2)
1863                 addr = (void __iomem *)(cs0_cap + 0x100);
1864
1865         val = readl(addr);
1866         writel(val + 1, addr);
1867
1868         read_mr(ddr_pctl_regs, rank, 5, &mr5);
1869         read_mr(ddr_pctl_regs, rank, 12, &mr12);
1870         read_mr(ddr_pctl_regs, rank, 14, &mr14);
1871
1872         if (mr5 == 0 || mr12 != 0x4d || mr14 != 0x4d) {
1873                 ret = -EINVAL;
1874                 goto end;
1875         }
1876 end:
1877         params->ch[channel].cap_info.rank = cs;
1878         params->ch[channel].cap_info.col = col;
1879         params->ch[channel].cap_info.bk = bk;
1880         params->ch[channel].cap_info.bw = bw;
1881         params->ch[channel].cap_info.row_3_4 = row_3_4;
1882         params->ch[channel].cap_info.cs0_row = cs0_row;
1883         params->ch[channel].cap_info.cs1_row = cs1_row;
1884         params->ch[channel].cap_info.ddrconfig = ddrconfig;
1885
1886         set_ddr_stride(dram->pmusgrf, stride);
1887
1888         return ret;
1889 }
1890
1891 static void set_lpddr4_dq_odt(const struct chan_info *chan,
1892                               struct rk3399_sdram_params *params, u32 ctl_fn,
1893                               bool en, bool ctl_phy_reg, u32 mr5)
1894 {
1895         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1896         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1897         struct io_setting *io;
1898         u32 reg_value;
1899
1900         io = lpddr4_get_io_settings(params, mr5);
1901         if (en)
1902                 reg_value = io->dq_odt;
1903         else
1904                 reg_value = 0;
1905
1906         switch (ctl_fn) {
1907         case 0:
1908                 clrsetbits_le32(&denali_ctl[139], 0x7 << 24, reg_value << 24);
1909                 clrsetbits_le32(&denali_ctl[153], 0x7 << 24, reg_value << 24);
1910
1911                 clrsetbits_le32(&denali_pi[132], 0x7 << 0, (reg_value << 0));
1912                 clrsetbits_le32(&denali_pi[139], 0x7 << 16, (reg_value << 16));
1913                 clrsetbits_le32(&denali_pi[147], 0x7 << 0, (reg_value << 0));
1914                 clrsetbits_le32(&denali_pi[154], 0x7 << 16, (reg_value << 16));
1915                 break;
1916         case 1:
1917                 clrsetbits_le32(&denali_ctl[140], 0x7 << 0, reg_value << 0);
1918                 clrsetbits_le32(&denali_ctl[154], 0x7 << 0, reg_value << 0);
1919
1920                 clrsetbits_le32(&denali_pi[129], 0x7 << 16, (reg_value << 16));
1921                 clrsetbits_le32(&denali_pi[137], 0x7 << 0, (reg_value << 0));
1922                 clrsetbits_le32(&denali_pi[144], 0x7 << 16, (reg_value << 16));
1923                 clrsetbits_le32(&denali_pi[152], 0x7 << 0, (reg_value << 0));
1924                 break;
1925         case 2:
1926         default:
1927                 clrsetbits_le32(&denali_ctl[140], 0x7 << 8, (reg_value << 8));
1928                 clrsetbits_le32(&denali_ctl[154], 0x7 << 8, (reg_value << 8));
1929
1930                 clrsetbits_le32(&denali_pi[127], 0x7 << 0, (reg_value << 0));
1931                 clrsetbits_le32(&denali_pi[134], 0x7 << 16, (reg_value << 16));
1932                 clrsetbits_le32(&denali_pi[142], 0x7 << 0, (reg_value << 0));
1933                 clrsetbits_le32(&denali_pi[149], 0x7 << 16, (reg_value << 16));
1934                 break;
1935         }
1936 }
1937
1938 static void set_lpddr4_ca_odt(const struct chan_info *chan,
1939                               struct rk3399_sdram_params *params, u32 ctl_fn,
1940                               bool en, bool ctl_phy_reg, u32 mr5)
1941 {
1942         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1943         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1944         struct io_setting *io;
1945         u32 reg_value;
1946
1947         io = lpddr4_get_io_settings(params, mr5);
1948         if (en)
1949                 reg_value = io->ca_odt;
1950         else
1951                 reg_value = 0;
1952
1953         switch (ctl_fn) {
1954         case 0:
1955                 clrsetbits_le32(&denali_ctl[139], 0x7 << 28, reg_value << 28);
1956                 clrsetbits_le32(&denali_ctl[153], 0x7 << 28, reg_value << 28);
1957
1958                 clrsetbits_le32(&denali_pi[132], 0x7 << 4, reg_value << 4);
1959                 clrsetbits_le32(&denali_pi[139], 0x7 << 20, reg_value << 20);
1960                 clrsetbits_le32(&denali_pi[147], 0x7 << 4, reg_value << 4);
1961                 clrsetbits_le32(&denali_pi[154], 0x7 << 20, reg_value << 20);
1962                 break;
1963         case 1:
1964                 clrsetbits_le32(&denali_ctl[140], 0x7 << 4, reg_value << 4);
1965                 clrsetbits_le32(&denali_ctl[154], 0x7 << 4, reg_value << 4);
1966
1967                 clrsetbits_le32(&denali_pi[129], 0x7 << 20, reg_value << 20);
1968                 clrsetbits_le32(&denali_pi[137], 0x7 << 4, reg_value << 4);
1969                 clrsetbits_le32(&denali_pi[144], 0x7 << 20, reg_value << 20);
1970                 clrsetbits_le32(&denali_pi[152], 0x7 << 4, reg_value << 4);
1971                 break;
1972         case 2:
1973         default:
1974                 clrsetbits_le32(&denali_ctl[140], 0x7 << 12, (reg_value << 12));
1975                 clrsetbits_le32(&denali_ctl[154], 0x7 << 12, (reg_value << 12));
1976
1977                 clrsetbits_le32(&denali_pi[127], 0x7 << 4, reg_value << 4);
1978                 clrsetbits_le32(&denali_pi[134], 0x7 << 20, reg_value << 20);
1979                 clrsetbits_le32(&denali_pi[142], 0x7 << 4, reg_value << 4);
1980                 clrsetbits_le32(&denali_pi[149], 0x7 << 20, reg_value << 20);
1981                 break;
1982         }
1983 }
1984
1985 static void set_lpddr4_MR3(const struct chan_info *chan,
1986                            struct rk3399_sdram_params *params, u32 ctl_fn,
1987                            bool ctl_phy_reg, u32 mr5)
1988 {
1989         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
1990         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
1991         struct io_setting *io;
1992         u32 reg_value;
1993
1994         io = lpddr4_get_io_settings(params, mr5);
1995
1996         reg_value = ((io->pdds << 3) | 1);
1997
1998         switch (ctl_fn) {
1999         case 0:
2000                 clrsetbits_le32(&denali_ctl[138], 0xFFFF, reg_value);
2001                 clrsetbits_le32(&denali_ctl[152], 0xFFFF, reg_value);
2002
2003                 clrsetbits_le32(&denali_pi[131], 0xFFFF << 16, reg_value << 16);
2004                 clrsetbits_le32(&denali_pi[139], 0xFFFF, reg_value);
2005                 clrsetbits_le32(&denali_pi[146], 0xFFFF << 16, reg_value << 16);
2006                 clrsetbits_le32(&denali_pi[154], 0xFFFF, reg_value);
2007                 break;
2008         case 1:
2009                 clrsetbits_le32(&denali_ctl[138], 0xFFFF << 16,
2010                                 reg_value << 16);
2011                 clrsetbits_le32(&denali_ctl[152], 0xFFFF << 16,
2012                                 reg_value << 16);
2013
2014                 clrsetbits_le32(&denali_pi[129], 0xFFFF, reg_value);
2015                 clrsetbits_le32(&denali_pi[136], 0xFFFF << 16, reg_value << 16);
2016                 clrsetbits_le32(&denali_pi[144], 0xFFFF, reg_value);
2017                 clrsetbits_le32(&denali_pi[151], 0xFFFF << 16, reg_value << 16);
2018                 break;
2019         case 2:
2020         default:
2021                 clrsetbits_le32(&denali_ctl[139], 0xFFFF, reg_value);
2022                 clrsetbits_le32(&denali_ctl[153], 0xFFFF, reg_value);
2023
2024                 clrsetbits_le32(&denali_pi[126], 0xFFFF << 16, reg_value << 16);
2025                 clrsetbits_le32(&denali_pi[134], 0xFFFF, reg_value);
2026                 clrsetbits_le32(&denali_pi[141], 0xFFFF << 16, reg_value << 16);
2027                 clrsetbits_le32(&denali_pi[149], 0xFFFF, reg_value);
2028                 break;
2029         }
2030 }
2031
2032 static void set_lpddr4_MR12(const struct chan_info *chan,
2033                             struct rk3399_sdram_params *params, u32 ctl_fn,
2034                             bool ctl_phy_reg, u32 mr5)
2035 {
2036         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
2037         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
2038         struct io_setting *io;
2039         u32 reg_value;
2040
2041         io = lpddr4_get_io_settings(params, mr5);
2042
2043         reg_value = io->ca_vref;
2044
2045         switch (ctl_fn) {
2046         case 0:
2047                 clrsetbits_le32(&denali_ctl[140], 0xFFFF << 16,
2048                                 reg_value << 16);
2049                 clrsetbits_le32(&denali_ctl[154], 0xFFFF << 16,
2050                                 reg_value << 16);
2051
2052                 clrsetbits_le32(&denali_pi[132], 0xFF << 8, reg_value << 8);
2053                 clrsetbits_le32(&denali_pi[139], 0xFF << 24, reg_value << 24);
2054                 clrsetbits_le32(&denali_pi[147], 0xFF << 8, reg_value << 8);
2055                 clrsetbits_le32(&denali_pi[154], 0xFF << 24, reg_value << 24);
2056                 break;
2057         case 1:
2058                 clrsetbits_le32(&denali_ctl[141], 0xFFFF, reg_value);
2059                 clrsetbits_le32(&denali_ctl[155], 0xFFFF, reg_value);
2060
2061                 clrsetbits_le32(&denali_pi[129], 0xFF << 24, reg_value << 24);
2062                 clrsetbits_le32(&denali_pi[137], 0xFF << 8, reg_value << 8);
2063                 clrsetbits_le32(&denali_pi[144], 0xFF << 24, reg_value << 24);
2064                 clrsetbits_le32(&denali_pi[152], 0xFF << 8, reg_value << 8);
2065                 break;
2066         case 2:
2067         default:
2068                 clrsetbits_le32(&denali_ctl[141], 0xFFFF << 16,
2069                                 reg_value << 16);
2070                 clrsetbits_le32(&denali_ctl[155], 0xFFFF << 16,
2071                                 reg_value << 16);
2072
2073                 clrsetbits_le32(&denali_pi[127], 0xFF << 8, reg_value << 8);
2074                 clrsetbits_le32(&denali_pi[134], 0xFF << 24, reg_value << 24);
2075                 clrsetbits_le32(&denali_pi[142], 0xFF << 8, reg_value << 8);
2076                 clrsetbits_le32(&denali_pi[149], 0xFF << 24, reg_value << 24);
2077                 break;
2078         }
2079 }
2080
2081 static void set_lpddr4_MR14(const struct chan_info *chan,
2082                             struct rk3399_sdram_params *params, u32 ctl_fn,
2083                             bool ctl_phy_reg, u32 mr5)
2084 {
2085         u32 *denali_ctl = get_denali_ctl(chan, params, ctl_phy_reg);
2086         u32 *denali_pi = get_denali_pi(chan, params, ctl_phy_reg);
2087         struct io_setting *io;
2088         u32 reg_value;
2089
2090         io = lpddr4_get_io_settings(params, mr5);
2091
2092         reg_value = io->dq_vref;
2093
2094         switch (ctl_fn) {
2095         case 0:
2096                 clrsetbits_le32(&denali_ctl[142], 0xFFFF << 16,
2097                                 reg_value << 16);
2098                 clrsetbits_le32(&denali_ctl[156], 0xFFFF << 16,
2099                                 reg_value << 16);
2100
2101                 clrsetbits_le32(&denali_pi[132], 0xFF << 16, reg_value << 16);
2102                 clrsetbits_le32(&denali_pi[140], 0xFF << 0, reg_value << 0);
2103                 clrsetbits_le32(&denali_pi[147], 0xFF << 16, reg_value << 16);
2104                 clrsetbits_le32(&denali_pi[155], 0xFF << 0, reg_value << 0);
2105                 break;
2106         case 1:
2107                 clrsetbits_le32(&denali_ctl[143], 0xFFFF, reg_value);
2108                 clrsetbits_le32(&denali_ctl[157], 0xFFFF, reg_value);
2109
2110                 clrsetbits_le32(&denali_pi[130], 0xFF << 0, reg_value << 0);
2111                 clrsetbits_le32(&denali_pi[137], 0xFF << 16, reg_value << 16);
2112                 clrsetbits_le32(&denali_pi[145], 0xFF << 0, reg_value << 0);
2113                 clrsetbits_le32(&denali_pi[152], 0xFF << 16, reg_value << 16);
2114                 break;
2115         case 2:
2116         default:
2117                 clrsetbits_le32(&denali_ctl[143], 0xFFFF << 16,
2118                                 reg_value << 16);
2119                 clrsetbits_le32(&denali_ctl[157], 0xFFFF << 16,
2120                                 reg_value << 16);
2121
2122                 clrsetbits_le32(&denali_pi[127], 0xFF << 16, reg_value << 16);
2123                 clrsetbits_le32(&denali_pi[135], 0xFF << 0, reg_value << 0);
2124                 clrsetbits_le32(&denali_pi[142], 0xFF << 16, reg_value << 16);
2125                 clrsetbits_le32(&denali_pi[150], 0xFF << 0, reg_value << 0);
2126                 break;
2127         }
2128 }
2129
2130 void lpddr4_modify_param(const struct chan_info *chan,
2131                          struct rk3399_sdram_params *params)
2132 {
2133         struct rk3399_sdram_params *params_cfg;
2134         u32 *denali_ctl_params;
2135         u32 *denali_pi_params;
2136         u32 *denali_phy_params;
2137
2138         denali_ctl_params = params->pctl_regs.denali_ctl;
2139         denali_pi_params = params->pi_regs.denali_pi;
2140         denali_phy_params = params->phy_regs.denali_phy;
2141
2142         set_lpddr4_dq_odt(chan, params, 2, true, false, 0);
2143         set_lpddr4_ca_odt(chan, params, 2, true, false, 0);
2144         set_lpddr4_MR3(chan, params, 2, false, 0);
2145         set_lpddr4_MR12(chan, params, 2, false, 0);
2146         set_lpddr4_MR14(chan, params, 2, false, 0);
2147         params_cfg = lpddr4_get_phy_index_params(0, params);
2148         set_ds_odt(chan, params_cfg, false, 0);
2149         /* read two cycle preamble */
2150         clrsetbits_le32(&denali_ctl_params[200], 0x3 << 24, 0x3 << 24);
2151         clrsetbits_le32(&denali_phy_params[7], 0x3 << 24, 0x3 << 24);
2152         clrsetbits_le32(&denali_phy_params[135], 0x3 << 24, 0x3 << 24);
2153         clrsetbits_le32(&denali_phy_params[263], 0x3 << 24, 0x3 << 24);
2154         clrsetbits_le32(&denali_phy_params[391], 0x3 << 24, 0x3 << 24);
2155
2156         /* boot frequency two cycle preamble */
2157         clrsetbits_le32(&denali_phy_params[2], 0x3 << 16, 0x3 << 16);
2158         clrsetbits_le32(&denali_phy_params[130], 0x3 << 16, 0x3 << 16);
2159         clrsetbits_le32(&denali_phy_params[258], 0x3 << 16, 0x3 << 16);
2160         clrsetbits_le32(&denali_phy_params[386], 0x3 << 16, 0x3 << 16);
2161
2162         clrsetbits_le32(&denali_pi_params[45], 0x3 << 8, 0x3 << 8);
2163         clrsetbits_le32(&denali_pi_params[58], 0x1, 0x1);
2164
2165         /*
2166          * bypass mode need PHY_SLICE_PWR_RDC_DISABLE_x = 1,
2167          * boot frequency mode use bypass mode
2168          */
2169         setbits_le32(&denali_phy_params[10], 1 << 16);
2170         setbits_le32(&denali_phy_params[138], 1 << 16);
2171         setbits_le32(&denali_phy_params[266], 1 << 16);
2172         setbits_le32(&denali_phy_params[394], 1 << 16);
2173
2174         clrsetbits_le32(&denali_pi_params[45], 0x1 << 24, 0x1 << 24);
2175         clrsetbits_le32(&denali_pi_params[61], 0x1 << 24, 0x1 << 24);
2176         clrsetbits_le32(&denali_pi_params[76], 0x1 << 24, 0x1 << 24);
2177         clrsetbits_le32(&denali_pi_params[77], 0x1, 0x1);
2178 }
2179
2180 static void lpddr4_copy_phy(struct dram_info *dram,
2181                             struct rk3399_sdram_params *params, u32 phy_fn,
2182                             struct rk3399_sdram_params *params_cfg,
2183                             u32 channel)
2184 {
2185         u32 *denali_ctl, *denali_phy;
2186         u32 *denali_phy_params;
2187         u32 speed = 0;
2188         u32 ctl_fn, mr5;
2189
2190         denali_ctl = dram->chan[channel].pctl->denali_ctl;
2191         denali_phy = dram->chan[channel].publ->denali_phy;
2192         denali_phy_params = params_cfg->phy_regs.denali_phy;
2193
2194         /* switch index */
2195         clrsetbits_le32(&denali_phy_params[896], 0x3 << 8,
2196                         phy_fn << 8);
2197         writel(denali_phy_params[896], &denali_phy[896]);
2198
2199         /* phy_pll_ctrl_ca, phy_pll_ctrl */
2200         writel(denali_phy_params[911], &denali_phy[911]);
2201
2202         /* phy_low_freq_sel */
2203         clrsetbits_le32(&denali_phy[913], 0x1,
2204                         denali_phy_params[913] & 0x1);
2205
2206         /* phy_grp_slave_delay_x, phy_cslvl_dly_step */
2207         writel(denali_phy_params[916], &denali_phy[916]);
2208         writel(denali_phy_params[917], &denali_phy[917]);
2209         writel(denali_phy_params[918], &denali_phy[918]);
2210
2211         /* phy_adrz_sw_wraddr_shift_x  */
2212         writel(denali_phy_params[512], &denali_phy[512]);
2213         clrsetbits_le32(&denali_phy[513], 0xffff,
2214                         denali_phy_params[513] & 0xffff);
2215         writel(denali_phy_params[640], &denali_phy[640]);
2216         clrsetbits_le32(&denali_phy[641], 0xffff,
2217                         denali_phy_params[641] & 0xffff);
2218         writel(denali_phy_params[768], &denali_phy[768]);
2219         clrsetbits_le32(&denali_phy[769], 0xffff,
2220                         denali_phy_params[769] & 0xffff);
2221
2222         writel(denali_phy_params[544], &denali_phy[544]);
2223         writel(denali_phy_params[545], &denali_phy[545]);
2224         writel(denali_phy_params[546], &denali_phy[546]);
2225         writel(denali_phy_params[547], &denali_phy[547]);
2226
2227         writel(denali_phy_params[672], &denali_phy[672]);
2228         writel(denali_phy_params[673], &denali_phy[673]);
2229         writel(denali_phy_params[674], &denali_phy[674]);
2230         writel(denali_phy_params[675], &denali_phy[675]);
2231
2232         writel(denali_phy_params[800], &denali_phy[800]);
2233         writel(denali_phy_params[801], &denali_phy[801]);
2234         writel(denali_phy_params[802], &denali_phy[802]);
2235         writel(denali_phy_params[803], &denali_phy[803]);
2236
2237         /*
2238          * phy_adr_master_delay_start_x
2239          * phy_adr_master_delay_step_x
2240          * phy_adr_master_delay_wait_x
2241          */
2242         writel(denali_phy_params[548], &denali_phy[548]);
2243         writel(denali_phy_params[676], &denali_phy[676]);
2244         writel(denali_phy_params[804], &denali_phy[804]);
2245
2246         /* phy_adr_calvl_dly_step_x */
2247         writel(denali_phy_params[549], &denali_phy[549]);
2248         writel(denali_phy_params[677], &denali_phy[677]);
2249         writel(denali_phy_params[805], &denali_phy[805]);
2250
2251         /*
2252          * phy_clk_wrdm_slave_delay_x
2253          * phy_clk_wrdqz_slave_delay_x
2254          * phy_clk_wrdqs_slave_delay_x
2255          */
2256         sdram_copy_to_reg((u32 *)&denali_phy[59],
2257                           (u32 *)&denali_phy_params[59], (63 - 58) * 4);
2258         sdram_copy_to_reg((u32 *)&denali_phy[187],
2259                           (u32 *)&denali_phy_params[187], (191 - 186) * 4);
2260         sdram_copy_to_reg((u32 *)&denali_phy[315],
2261                           (u32 *)&denali_phy_params[315], (319 - 314) * 4);
2262         sdram_copy_to_reg((u32 *)&denali_phy[443],
2263                           (u32 *)&denali_phy_params[443], (447 - 442) * 4);
2264
2265         /*
2266          * phy_dqs_tsel_wr_timing_x 8bits denali_phy_84/212/340/468 offset_8
2267          * dqs_tsel_wr_end[7:4] add half cycle
2268          * phy_dq_tsel_wr_timing_x 8bits denali_phy_83/211/339/467 offset_8
2269          * dq_tsel_wr_end[7:4] add half cycle
2270          */
2271         writel(denali_phy_params[83] + (0x10 << 16), &denali_phy[83]);
2272         writel(denali_phy_params[84] + (0x10 << 8), &denali_phy[84]);
2273         writel(denali_phy_params[85], &denali_phy[85]);
2274
2275         writel(denali_phy_params[211] + (0x10 << 16), &denali_phy[211]);
2276         writel(denali_phy_params[212] + (0x10 << 8), &denali_phy[212]);
2277         writel(denali_phy_params[213], &denali_phy[213]);
2278
2279         writel(denali_phy_params[339] + (0x10 << 16), &denali_phy[339]);
2280         writel(denali_phy_params[340] + (0x10 << 8), &denali_phy[340]);
2281         writel(denali_phy_params[341], &denali_phy[341]);
2282
2283         writel(denali_phy_params[467] + (0x10 << 16), &denali_phy[467]);
2284         writel(denali_phy_params[468] + (0x10 << 8), &denali_phy[468]);
2285         writel(denali_phy_params[469], &denali_phy[469]);
2286
2287         /*
2288          * phy_gtlvl_resp_wait_cnt_x
2289          * phy_gtlvl_dly_step_x
2290          * phy_wrlvl_resp_wait_cnt_x
2291          * phy_gtlvl_final_step_x
2292          * phy_gtlvl_back_step_x
2293          * phy_rdlvl_dly_step_x
2294          *
2295          * phy_master_delay_step_x
2296          * phy_master_delay_wait_x
2297          * phy_wrlvl_dly_step_x
2298          * phy_rptr_update_x
2299          * phy_wdqlvl_dly_step_x
2300          */
2301         writel(denali_phy_params[87], &denali_phy[87]);
2302         writel(denali_phy_params[88], &denali_phy[88]);
2303         writel(denali_phy_params[89], &denali_phy[89]);
2304         writel(denali_phy_params[90], &denali_phy[90]);
2305
2306         writel(denali_phy_params[215], &denali_phy[215]);
2307         writel(denali_phy_params[216], &denali_phy[216]);
2308         writel(denali_phy_params[217], &denali_phy[217]);
2309         writel(denali_phy_params[218], &denali_phy[218]);
2310
2311         writel(denali_phy_params[343], &denali_phy[343]);
2312         writel(denali_phy_params[344], &denali_phy[344]);
2313         writel(denali_phy_params[345], &denali_phy[345]);
2314         writel(denali_phy_params[346], &denali_phy[346]);
2315
2316         writel(denali_phy_params[471], &denali_phy[471]);
2317         writel(denali_phy_params[472], &denali_phy[472]);
2318         writel(denali_phy_params[473], &denali_phy[473]);
2319         writel(denali_phy_params[474], &denali_phy[474]);
2320
2321         /*
2322          * phy_gtlvl_lat_adj_start_x
2323          * phy_gtlvl_rddqs_slv_dly_start_x
2324          * phy_rdlvl_rddqs_dq_slv_dly_start_x
2325          * phy_wdqlvl_dqdm_slv_dly_start_x
2326          */
2327         writel(denali_phy_params[80], &denali_phy[80]);
2328         writel(denali_phy_params[81], &denali_phy[81]);
2329
2330         writel(denali_phy_params[208], &denali_phy[208]);
2331         writel(denali_phy_params[209], &denali_phy[209]);
2332
2333         writel(denali_phy_params[336], &denali_phy[336]);
2334         writel(denali_phy_params[337], &denali_phy[337]);
2335
2336         writel(denali_phy_params[464], &denali_phy[464]);
2337         writel(denali_phy_params[465], &denali_phy[465]);
2338
2339         /*
2340          * phy_master_delay_start_x
2341          * phy_sw_master_mode_x
2342          * phy_rddata_en_tsel_dly_x
2343          */
2344         writel(denali_phy_params[86], &denali_phy[86]);
2345         writel(denali_phy_params[214], &denali_phy[214]);
2346         writel(denali_phy_params[342], &denali_phy[342]);
2347         writel(denali_phy_params[470], &denali_phy[470]);
2348
2349         /*
2350          * phy_rddqz_slave_delay_x
2351          * phy_rddqs_dqz_fall_slave_delay_x
2352          * phy_rddqs_dqz_rise_slave_delay_x
2353          * phy_rddqs_dm_fall_slave_delay_x
2354          * phy_rddqs_dm_rise_slave_delay_x
2355          * phy_rddqs_gate_slave_delay_x
2356          * phy_wrlvl_delay_early_threshold_x
2357          * phy_write_path_lat_add_x
2358          * phy_rddqs_latency_adjust_x
2359          * phy_wrlvl_delay_period_threshold_x
2360          * phy_wrlvl_early_force_zero_x
2361          */
2362         sdram_copy_to_reg((u32 *)&denali_phy[64],
2363                           (u32 *)&denali_phy_params[64], (67 - 63) * 4);
2364         clrsetbits_le32(&denali_phy[68], 0xfffffc00,
2365                         denali_phy_params[68] & 0xfffffc00);
2366         sdram_copy_to_reg((u32 *)&denali_phy[69],
2367                           (u32 *)&denali_phy_params[69], (79 - 68) * 4);
2368         sdram_copy_to_reg((u32 *)&denali_phy[192],
2369                           (u32 *)&denali_phy_params[192], (195 - 191) * 4);
2370         clrsetbits_le32(&denali_phy[196], 0xfffffc00,
2371                         denali_phy_params[196] & 0xfffffc00);
2372         sdram_copy_to_reg((u32 *)&denali_phy[197],
2373                           (u32 *)&denali_phy_params[197], (207 - 196) * 4);
2374         sdram_copy_to_reg((u32 *)&denali_phy[320],
2375                           (u32 *)&denali_phy_params[320], (323 - 319) * 4);
2376         clrsetbits_le32(&denali_phy[324], 0xfffffc00,
2377                         denali_phy_params[324] & 0xfffffc00);
2378         sdram_copy_to_reg((u32 *)&denali_phy[325],
2379                           (u32 *)&denali_phy_params[325], (335 - 324) * 4);
2380         sdram_copy_to_reg((u32 *)&denali_phy[448],
2381                           (u32 *)&denali_phy_params[448], (451 - 447) * 4);
2382         clrsetbits_le32(&denali_phy[452], 0xfffffc00,
2383                         denali_phy_params[452] & 0xfffffc00);
2384         sdram_copy_to_reg((u32 *)&denali_phy[453],
2385                           (u32 *)&denali_phy_params[453], (463 - 452) * 4);
2386
2387         /* phy_two_cyc_preamble_x */
2388         clrsetbits_le32(&denali_phy[7], 0x3 << 24,
2389                         denali_phy_params[7] & (0x3 << 24));
2390         clrsetbits_le32(&denali_phy[135], 0x3 << 24,
2391                         denali_phy_params[135] & (0x3 << 24));
2392         clrsetbits_le32(&denali_phy[263], 0x3 << 24,
2393                         denali_phy_params[263] & (0x3 << 24));
2394         clrsetbits_le32(&denali_phy[391], 0x3 << 24,
2395                         denali_phy_params[391] & (0x3 << 24));
2396
2397         /* speed */
2398         if (params_cfg->base.ddr_freq < 400)
2399                 speed = 0x0;
2400         else if (params_cfg->base.ddr_freq < 800)
2401                 speed = 0x1;
2402         else if (params_cfg->base.ddr_freq < 1200)
2403                 speed = 0x2;
2404
2405         /* phy_924 phy_pad_fdbk_drive */
2406         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
2407         /* phy_926 phy_pad_data_drive */
2408         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
2409         /* phy_927 phy_pad_dqs_drive */
2410         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
2411         /* phy_928 phy_pad_addr_drive */
2412         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
2413         /* phy_929 phy_pad_clk_drive */
2414         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
2415         /* phy_935 phy_pad_cke_drive */
2416         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
2417         /* phy_937 phy_pad_rst_drive */
2418         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
2419         /* phy_939 phy_pad_cs_drive */
2420         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
2421
2422         if (params_cfg->base.dramtype == LPDDR4) {
2423                 read_mr(dram->chan[channel].pctl, 1, 5, &mr5);
2424                 set_ds_odt(&dram->chan[channel], params_cfg, true, mr5);
2425
2426                 ctl_fn = lpddr4_get_ctl_fn(params_cfg, phy_fn);
2427                 set_lpddr4_dq_odt(&dram->chan[channel], params_cfg,
2428                                   ctl_fn, true, true, mr5);
2429                 set_lpddr4_ca_odt(&dram->chan[channel], params_cfg,
2430                                   ctl_fn, true, true, mr5);
2431                 set_lpddr4_MR3(&dram->chan[channel], params_cfg,
2432                                ctl_fn, true, mr5);
2433                 set_lpddr4_MR12(&dram->chan[channel], params_cfg,
2434                                 ctl_fn, true, mr5);
2435                 set_lpddr4_MR14(&dram->chan[channel], params_cfg,
2436                                 ctl_fn, true, mr5);
2437
2438                 /*
2439                  * if phy_sw_master_mode_x not bypass mode,
2440                  * clear phy_slice_pwr_rdc_disable.
2441                  * note: need use timings, not ddr_publ_regs
2442                  */
2443                 if (!((denali_phy_params[86] >> 8) & (1 << 2))) {
2444                         clrbits_le32(&denali_phy[10], 1 << 16);
2445                         clrbits_le32(&denali_phy[138], 1 << 16);
2446                         clrbits_le32(&denali_phy[266], 1 << 16);
2447                         clrbits_le32(&denali_phy[394], 1 << 16);
2448                 }
2449
2450                 /*
2451                  * when PHY_PER_CS_TRAINING_EN=1, W2W_DIFFCS_DLY_Fx can't
2452                  * smaller than 8
2453                  * NOTE: need use timings, not ddr_publ_regs
2454                  */
2455                 if ((denali_phy_params[84] >> 16) & 1) {
2456                         if (((readl(&denali_ctl[217 + ctl_fn]) >>
2457                                 16) & 0x1f) < 8)
2458                                 clrsetbits_le32(&denali_ctl[217 + ctl_fn],
2459                                                 0x1f << 16,
2460                                                 8 << 16);
2461                 }
2462         }
2463 }
2464
2465 static void lpddr4_set_phy(struct dram_info *dram,
2466                            struct rk3399_sdram_params *params, u32 phy_fn,
2467                            struct rk3399_sdram_params *params_cfg)
2468 {
2469         u32 channel;
2470
2471         for (channel = 0; channel < 2; channel++)
2472                 lpddr4_copy_phy(dram, params, phy_fn, params_cfg,
2473                                 channel);
2474 }
2475
2476 static int lpddr4_set_ctl(struct dram_info *dram,
2477                           struct rk3399_sdram_params *params,
2478                           u32 fn, u32 hz)
2479 {
2480         u32 channel;
2481         int ret_clk, ret;
2482
2483         /* cci idle req stall */
2484         writel(0x70007, &dram->grf->soc_con0);
2485
2486         /* enable all clk */
2487         setbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));
2488
2489         /* idle */
2490         setbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));
2491         while ((readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))
2492                != (0x3 << 18))
2493                 ;
2494
2495         /* change freq */
2496         writel((((0x3 << 4) | (1 << 2) | 1) << 16) |
2497                 (fn << 4) | (1 << 2) | 1, &dram->cic->cic_ctrl0);
2498         while (!(readl(&dram->cic->cic_status0) & (1 << 2)))
2499                 ;
2500
2501         ret_clk = clk_set_rate(&dram->ddr_clk, hz);
2502         if (ret_clk < 0) {
2503                 printf("%s clk set failed %d\n", __func__, ret_clk);
2504                 return ret_clk;
2505         }
2506
2507         writel(0x20002, &dram->cic->cic_ctrl0);
2508         while (!(readl(&dram->cic->cic_status0) & (1 << 0)))
2509                 ;
2510
2511         /* deidle */
2512         clrbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));
2513         while (readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))
2514                 ;
2515
2516         /* clear enable all clk */
2517         clrbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));
2518
2519         /* lpddr4 ctl2 can not do training, all training will fail */
2520         if (!(params->base.dramtype == LPDDR4 && fn == 2)) {
2521                 for (channel = 0; channel < 2; channel++) {
2522                         if (!(params->ch[channel].cap_info.col))
2523                                 continue;
2524                         ret = data_training(dram, channel, params,
2525                                             PI_FULL_TRAINING);
2526                         if (ret)
2527                                 printf("%s: channel %d training failed!\n",
2528                                        __func__, channel);
2529                         else
2530                                 debug("%s: channel %d training pass\n",
2531                                       __func__, channel);
2532                 }
2533         }
2534
2535         return 0;
2536 }
2537
2538 static int lpddr4_set_rate(struct dram_info *dram,
2539                            struct rk3399_sdram_params *params)
2540 {
2541         u32 ctl_fn;
2542         u32 phy_fn;
2543
2544         for (ctl_fn = 0; ctl_fn < 2; ctl_fn++) {
2545                 phy_fn = lpddr4_get_phy_fn(params, ctl_fn);
2546
2547                 lpddr4_set_phy(dram, params, phy_fn, &dfs_cfgs_lpddr4[ctl_fn]);
2548                 lpddr4_set_ctl(dram, params, ctl_fn,
2549                                dfs_cfgs_lpddr4[ctl_fn].base.ddr_freq);
2550
2551                 printf("%s: change freq to %d mhz %d, %d\n", __func__,
2552                        dfs_cfgs_lpddr4[ctl_fn].base.ddr_freq, ctl_fn, phy_fn);
2553         }
2554
2555         return 0;
2556 }
2557 #endif /* CONFIG_RAM_RK3399_LPDDR4 */
2558
2559 /* CS0,n=1
2560  * CS1,n=2
2561  * CS0 & CS1, n=3
2562  * cs0_cap: MB unit
2563  */
2564 static void dram_set_cs(const struct chan_info *chan, u32 cs_map, u32 cs0_cap,
2565                         unsigned char dramtype)
2566 {
2567         u32 *denali_ctl = chan->pctl->denali_ctl;
2568         u32 *denali_pi = chan->pi->denali_pi;
2569         struct msch_regs *ddr_msch_regs = chan->msch;
2570
2571         clrsetbits_le32(&denali_ctl[196], 0x3, cs_map);
2572         writel((cs0_cap / 32) | (((4096 - cs0_cap) / 32) << 8),
2573                &ddr_msch_regs->ddrsize);
2574         if (dramtype == LPDDR4) {
2575                 if (cs_map == 1)
2576                         cs_map = 0x5;
2577                 else if (cs_map == 2)
2578                         cs_map = 0xa;
2579                 else
2580                         cs_map = 0xF;
2581         }
2582         /*PI_41 PI_CS_MAP:RW:24:4*/
2583         clrsetbits_le32(&denali_pi[41],
2584                         0xf << 24, cs_map << 24);
2585         if (cs_map == 1 && dramtype == DDR3)
2586                 writel(0x2EC7FFFF, &denali_pi[34]);
2587 }
2588
2589 static void dram_set_bw(const struct chan_info *chan, u32 bw)
2590 {
2591         u32 *denali_ctl = chan->pctl->denali_ctl;
2592
2593         if (bw == 2)
2594                 clrbits_le32(&denali_ctl[196], 1 << 16);
2595         else
2596                 setbits_le32(&denali_ctl[196], 1 << 16);
2597 }
2598
2599 static void dram_set_max_col(const struct chan_info *chan, u32 bw, u32 *pcol)
2600 {
2601         u32 *denali_ctl = chan->pctl->denali_ctl;
2602         struct msch_regs *ddr_msch_regs = chan->msch;
2603         u32 *denali_pi = chan->pi->denali_pi;
2604         u32 ddrconfig;
2605
2606         clrbits_le32(&denali_ctl[191], 0xf);
2607         clrsetbits_le32(&denali_ctl[190],
2608                         (7 << 24),
2609                         ((16 - ((bw == 2) ? 14 : 15)) << 24));
2610         /*PI_199 PI_COL_DIFF:RW:0:4*/
2611         clrbits_le32(&denali_pi[199], 0xf);
2612         /*PI_155 PI_ROW_DIFF:RW:24:3*/
2613         clrsetbits_le32(&denali_pi[155],
2614                         (7 << 24),
2615                         ((16 - 12) << 24));
2616         ddrconfig = (bw == 2) ? 3 : 2;
2617         writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf);
2618         /* set max cs0 size */
2619         writel((4096 / 32) | ((0 / 32) << 8),
2620                &ddr_msch_regs->ddrsize);
2621
2622         *pcol = 12;
2623 }
2624
2625 static void dram_set_max_bank(const struct chan_info *chan, u32 bw, u32 *pbank,
2626                               u32 *pcol)
2627 {
2628         u32 *denali_ctl = chan->pctl->denali_ctl;
2629         u32 *denali_pi = chan->pi->denali_pi;
2630
2631         clrbits_le32(&denali_ctl[191], 0xf);
2632         clrbits_le32(&denali_ctl[190], (3 << 16));
2633         /*PI_199 PI_COL_DIFF:RW:0:4*/
2634         clrbits_le32(&denali_pi[199], 0xf);
2635         /*PI_155 PI_BANK_DIFF:RW:16:2*/
2636         clrbits_le32(&denali_pi[155], (3 << 16));
2637
2638         *pbank = 3;
2639         *pcol = 12;
2640 }
2641
2642 static void dram_set_max_row(const struct chan_info *chan, u32 bw, u32 *prow,
2643                              u32 *pbank, u32 *pcol)
2644 {
2645         u32 *denali_ctl = chan->pctl->denali_ctl;
2646         u32 *denali_pi = chan->pi->denali_pi;
2647         struct msch_regs *ddr_msch_regs = chan->msch;
2648
2649         clrsetbits_le32(&denali_ctl[191], 0xf, 12 - 10);
2650         clrbits_le32(&denali_ctl[190],
2651                      (0x3 << 16) | (0x7 << 24));
2652         /*PI_199 PI_COL_DIFF:RW:0:4*/
2653         clrsetbits_le32(&denali_pi[199], 0xf, 12 - 10);
2654         /*PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2*/
2655         clrbits_le32(&denali_pi[155],
2656                      (0x3 << 16) | (0x7 << 24));
2657         writel(1 | (1 << 8), &ddr_msch_regs->ddrconf);
2658         /* set max cs0 size */
2659         writel((4096 / 32) | ((0 / 32) << 8),
2660                &ddr_msch_regs->ddrsize);
2661
2662         *prow = 16;
2663         *pbank = 3;
2664         *pcol = (bw == 2) ? 10 : 11;
2665 }
2666
2667 static u64 dram_detect_cap(struct dram_info *dram,
2668                            struct rk3399_sdram_params *params,
2669                            unsigned char channel)
2670 {
2671         const struct chan_info *chan = &dram->chan[channel];
2672         struct sdram_cap_info *cap_info = &params->ch[channel].cap_info;
2673         u32 bw;
2674         u32 col_tmp;
2675         u32 bk_tmp;
2676         u32 row_tmp;
2677         u32 cs0_cap;
2678         u32 training_flag;
2679         u32 ddrconfig;
2680
2681         /* detect bw */
2682         bw = 2;
2683         if (params->base.dramtype != LPDDR4) {
2684                 dram_set_bw(chan, bw);
2685                 cap_info->bw = bw;
2686                 if (data_training(dram, channel, params,
2687                                   PI_READ_GATE_TRAINING)) {
2688                         bw = 1;
2689                         dram_set_bw(chan, 1);
2690                         cap_info->bw = bw;
2691                         if (data_training(dram, channel, params,
2692                                           PI_READ_GATE_TRAINING)) {
2693                                 printf("16bit error!!!\n");
2694                                 goto error;
2695                         }
2696                 }
2697         }
2698         /*
2699          * LPDDR3 CA training msut be trigger before other training.
2700          * DDR3 is not have CA training.
2701          */
2702         if (params->base.dramtype == LPDDR3)
2703                 training_flag = PI_WRITE_LEVELING;
2704         else
2705                 training_flag = PI_FULL_TRAINING;
2706
2707         if (params->base.dramtype != LPDDR4) {
2708                 if (data_training(dram, channel, params, training_flag)) {
2709                         printf("full training error!!!\n");
2710                         goto error;
2711                 }
2712         }
2713
2714         /* detect col */
2715         dram_set_max_col(chan, bw, &col_tmp);
2716         if (sdram_detect_col(cap_info, col_tmp) != 0)
2717                 goto error;
2718
2719         /* detect bank */
2720         dram_set_max_bank(chan, bw, &bk_tmp, &col_tmp);
2721         sdram_detect_bank(cap_info, col_tmp, bk_tmp);
2722
2723         /* detect row */
2724         dram_set_max_row(chan, bw, &row_tmp, &bk_tmp, &col_tmp);
2725         if (sdram_detect_row(cap_info, col_tmp, bk_tmp, row_tmp) != 0)
2726                 goto error;
2727
2728         /* detect row_3_4 */
2729         sdram_detect_row_3_4(cap_info, col_tmp, bk_tmp);
2730
2731         /* set ddrconfig */
2732         cs0_cap = (1 << (cap_info->cs0_row + cap_info->col + cap_info->bk +
2733                          cap_info->bw - 20));
2734         if (cap_info->row_3_4)
2735                 cs0_cap = cs0_cap * 3 / 4;
2736
2737         cap_info->cs1_row = cap_info->cs0_row;
2738         set_memory_map(chan, channel, params);
2739         ddrconfig = calculate_ddrconfig(params, channel);
2740         if (-1 == ddrconfig)
2741                 goto error;
2742         set_ddrconfig(chan, params, channel,
2743                       cap_info->ddrconfig);
2744
2745         /* detect cs1 row */
2746         sdram_detect_cs1_row(cap_info, params->base.dramtype);
2747
2748         /* detect die bw */
2749         sdram_detect_dbw(cap_info, params->base.dramtype);
2750
2751         return 0;
2752 error:
2753         return (-1);
2754 }
2755
2756 static unsigned char calculate_stride(struct rk3399_sdram_params *params)
2757 {
2758         unsigned int gstride_type;
2759         unsigned int channel;
2760         unsigned int chinfo = 0;
2761         unsigned int cap = 0;
2762         unsigned int stride = -1;
2763         unsigned int ch_cap[2] = {0, 0};
2764
2765         gstride_type = STRIDE_256B;
2766
2767         for (channel = 0; channel < 2; channel++) {
2768                 unsigned int cs0_cap = 0;
2769                 unsigned int cs1_cap = 0;
2770                 struct sdram_cap_info *cap_info =
2771                         &params->ch[channel].cap_info;
2772
2773                 if (cap_info->col == 0)
2774                         continue;
2775
2776                 cs0_cap = (1 << (cap_info->cs0_row + cap_info->col +
2777                                  cap_info->bk + cap_info->bw - 20));
2778                 if (cap_info->rank > 1)
2779                         cs1_cap = cs0_cap >> (cap_info->cs0_row
2780                                               - cap_info->cs1_row);
2781                 if (cap_info->row_3_4) {
2782                         cs0_cap = cs0_cap * 3 / 4;
2783                         cs1_cap = cs1_cap * 3 / 4;
2784                 }
2785                 ch_cap[channel] = cs0_cap + cs1_cap;
2786                 chinfo |= 1 << channel;
2787         }
2788
2789         cap = ch_cap[0] + ch_cap[1];
2790         if (params->base.num_channels == 1) {
2791                 if (chinfo & 1) /* channel a only */
2792                         stride = 0x17;
2793                 else /* channel b only */
2794                         stride = 0x18;
2795         } else {/* 2 channel */
2796                 if (ch_cap[0] == ch_cap[1]) {
2797                         /* interleaved */
2798                         if (gstride_type == PART_STRIDE) {
2799                         /*
2800                          * first 64MB no interleaved other 256B interleaved
2801                          * if 786M+768M.useful space from 0-1280MB and
2802                          * 1536MB-1792MB
2803                          * if 1.5G+1.5G(continuous).useful space from 0-2560MB
2804                          * and 3072MB-3584MB
2805                          */
2806                                 stride = 0x1F;
2807                         } else {
2808                                 switch (cap) {
2809                                 /* 512MB */
2810                                 case 512:
2811                                         stride = 0;
2812                                         break;
2813                                 /* 1GB unstride or 256B stride*/
2814                                 case 1024:
2815                                         stride = (gstride_type == UN_STRIDE) ?
2816                                                 0x1 : 0x5;
2817                                         break;
2818                                 /*
2819                                  * 768MB + 768MB same as total 2GB memory
2820                                  * useful space: 0-768MB 1GB-1792MB
2821                                  */
2822                                 case 1536:
2823                                 /* 2GB unstride or 256B or 512B stride */
2824                                 case 2048:
2825                                         stride = (gstride_type == UN_STRIDE) ?
2826                                                 0x2 :
2827                                                 ((gstride_type == STRIDE_512B) ?
2828                                                  0xA : 0x9);
2829                                         break;
2830                                 /* 1536MB + 1536MB */
2831                                 case 3072:
2832                                         stride = (gstride_type == UN_STRIDE) ?
2833                                                 0x3 :
2834                                                 ((gstride_type == STRIDE_512B) ?
2835                                                  0x12 : 0x11);
2836                                         break;
2837                                 /* 4GB  unstride or 128B,256B,512B,4KB stride */
2838                                 case 4096:
2839                                         stride = (gstride_type == UN_STRIDE) ?
2840                                                 0x3 : (0xC + gstride_type);
2841                                         break;
2842                                 }
2843                         }
2844                 }
2845                 if (ch_cap[0] == 2048 && ch_cap[1] == 1024) {
2846                         /* 2GB + 1GB */
2847                         stride = (gstride_type == UN_STRIDE) ? 0x3 : 0x19;
2848                 }
2849                 /*
2850                  * remain two channel capability not equal OR capability
2851                  * power function of 2
2852                  */
2853                 if (stride == (-1)) {
2854                         switch ((ch_cap[0] > ch_cap[1]) ?
2855                                 ch_cap[0] : ch_cap[1]) {
2856                         case 256: /* 256MB + 128MB */
2857                                 stride = 0;
2858                                 break;
2859                         case 512: /* 512MB + 256MB */
2860                                 stride = 1;
2861                                 break;
2862                         case 1024:/* 1GB + 128MB/256MB/384MB/512MB/768MB */
2863                                 stride = 2;
2864                                 break;
2865                         case 2048: /* 2GB + 128MB/256MB/384MB/512MB/768MB/1GB */
2866                                 stride = 3;
2867                                 break;
2868                         default:
2869                                 break;
2870                         }
2871                 }
2872                 if (stride == (-1))
2873                         goto error;
2874         }
2875         switch (stride) {
2876         case 0xc:
2877                 printf("128B stride\n");
2878                 break;
2879         case 5:
2880         case 9:
2881         case 0xd:
2882         case 0x11:
2883         case 0x19:
2884                 printf("256B stride\n");
2885                 break;
2886         case 0xa:
2887         case 0xe:
2888         case 0x12:
2889                 printf("512B stride\n");
2890                 break;
2891         case 0xf:
2892                 printf("4K stride\n");
2893                 break;
2894         case 0x1f:
2895                 printf("32MB + 256B stride\n");
2896                 break;
2897         default:
2898                 printf("no stride\n");
2899         }
2900
2901         sdram_print_stride(stride);
2902
2903         return stride;
2904 error:
2905         printf("Cap not support!\n");
2906         return (-1);
2907 }
2908
2909 static void clear_channel_params(struct rk3399_sdram_params *params, u8 channel)
2910 {
2911         params->ch[channel].cap_info.rank = 0;
2912         params->ch[channel].cap_info.col = 0;
2913         params->ch[channel].cap_info.bk = 0;
2914         params->ch[channel].cap_info.bw = 32;
2915         params->ch[channel].cap_info.dbw = 32;
2916         params->ch[channel].cap_info.row_3_4 = 0;
2917         params->ch[channel].cap_info.cs0_row = 0;
2918         params->ch[channel].cap_info.cs1_row = 0;
2919         params->ch[channel].cap_info.ddrconfig = 0;
2920 }
2921
2922 static int sdram_init(struct dram_info *dram,
2923                       struct rk3399_sdram_params *params)
2924 {
2925         unsigned char dramtype = params->base.dramtype;
2926         unsigned int ddr_freq = params->base.ddr_freq;
2927         int channel, ch, rank;
2928         u32 tmp, ret;
2929
2930         debug("Starting SDRAM initialization...\n");
2931
2932         if ((dramtype == DDR3 && ddr_freq > 933) ||
2933             (dramtype == LPDDR3 && ddr_freq > 933) ||
2934             (dramtype == LPDDR4 && ddr_freq > 800)) {
2935                 debug("SDRAM frequency is to high!");
2936                 return -E2BIG;
2937         }
2938
2939         /* detect rank */
2940         for (ch = 0; ch < 2; ch++) {
2941                 params->ch[ch].cap_info.rank = 2;
2942                 for (rank = 2; rank != 0; rank--) {
2943                         for (channel = 0; channel < 2; channel++) {
2944                                 const struct chan_info *chan =
2945                                         &dram->chan[channel];
2946                                 struct rockchip_cru *cru = dram->cru;
2947                                 struct rk3399_ddr_publ_regs *publ = chan->publ;
2948
2949                                 phy_pctrl_reset(cru, channel);
2950                                 phy_dll_bypass_set(publ, ddr_freq);
2951                                 pctl_cfg(dram, chan, channel, params);
2952                         }
2953
2954                         /* start to trigger initialization */
2955                         pctl_start(dram, params, 3);
2956
2957                         /* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */
2958                         if (dramtype == LPDDR3)
2959                                 udelay(10);
2960
2961                         tmp = (rank == 2) ? 3 : 1;
2962                         dram_set_cs(&dram->chan[ch], tmp, 2048,
2963                                     params->base.dramtype);
2964                         params->ch[ch].cap_info.rank = rank;
2965
2966                         ret = dram->ops->data_training_first(dram, ch,
2967                                                              rank, params);
2968                         if (!ret) {
2969                                 debug("%s: data trained for rank %d, ch %d\n",
2970                                       __func__, rank, ch);
2971                                 break;
2972                         }
2973                 }
2974                 /* Computed rank with associated channel number */
2975                 params->ch[ch].cap_info.rank = rank;
2976         }
2977
2978         params->base.num_channels = 0;
2979         for (channel = 0; channel < 2; channel++) {
2980                 const struct chan_info *chan = &dram->chan[channel];
2981                 struct sdram_cap_info *cap_info =
2982                         &params->ch[channel].cap_info;
2983
2984                 if (cap_info->rank == 0) {
2985                         clear_channel_params(params, 1);
2986                         continue;
2987                 } else {
2988                         params->base.num_channels++;
2989                 }
2990
2991                 printf("Channel ");
2992                 printf(channel ? "1: " : "0: ");
2993
2994                 if (channel == 0)
2995                         set_ddr_stride(dram->pmusgrf, 0x17);
2996                 else
2997                         set_ddr_stride(dram->pmusgrf, 0x18);
2998
2999                 if (dram_detect_cap(dram, params, channel)) {
3000                         printf("Cap error!\n");
3001                         continue;
3002                 }
3003
3004                 sdram_print_ddr_info(cap_info, &params->base);
3005                 set_memory_map(chan, channel, params);
3006                 cap_info->ddrconfig =
3007                         calculate_ddrconfig(params, channel);
3008                 if (-1 == cap_info->ddrconfig) {
3009                         printf("no ddrconfig find, Cap not support!\n");
3010                         continue;
3011                 }
3012                 set_ddrconfig(chan, params, channel, cap_info->ddrconfig);
3013                 set_cap_relate_config(chan, params, channel);
3014         }
3015
3016         if (params->base.num_channels == 0) {
3017                 printf("%s: ", __func__);
3018                 sdram_print_dram_type(params->base.dramtype);
3019                 printf(" - %dMHz failed!\n", params->base.ddr_freq);
3020                 return -EINVAL;
3021         }
3022
3023         params->base.stride = calculate_stride(params);
3024         dram_all_config(dram, params);
3025
3026         dram->ops->set_rate_index(dram, params);
3027
3028         debug("Finish SDRAM initialization...\n");
3029         return 0;
3030 }
3031
3032 static int rk3399_dmc_ofdata_to_platdata(struct udevice *dev)
3033 {
3034 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
3035         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
3036         int ret;
3037
3038         ret = dev_read_u32_array(dev, "rockchip,sdram-params",
3039                                  (u32 *)&plat->sdram_params,
3040                                  sizeof(plat->sdram_params) / sizeof(u32));
3041         if (ret) {
3042                 printf("%s: Cannot read rockchip,sdram-params %d\n",
3043                        __func__, ret);
3044                 return ret;
3045         }
3046         ret = regmap_init_mem(dev_ofnode(dev), &plat->map);
3047         if (ret)
3048                 printf("%s: regmap failed %d\n", __func__, ret);
3049
3050 #endif
3051         return 0;
3052 }
3053
3054 #if CONFIG_IS_ENABLED(OF_PLATDATA)
3055 static int conv_of_platdata(struct udevice *dev)
3056 {
3057         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
3058         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
3059         int ret;
3060
3061         ret = regmap_init_mem_platdata(dev, dtplat->reg,
3062                                        ARRAY_SIZE(dtplat->reg) / 2,
3063                                        &plat->map);
3064         if (ret)
3065                 return ret;
3066
3067         return 0;
3068 }
3069 #endif
3070
3071 static const struct sdram_rk3399_ops rk3399_ops = {
3072 #if !defined(CONFIG_RAM_RK3399_LPDDR4)
3073         .data_training_first = data_training_first,
3074         .set_rate_index = switch_to_phy_index1,
3075         .modify_param = modify_param,
3076         .get_phy_index_params = get_phy_index_params,
3077 #else
3078         .data_training_first = lpddr4_mr_detect,
3079         .set_rate_index = lpddr4_set_rate,
3080         .modify_param = lpddr4_modify_param,
3081         .get_phy_index_params = lpddr4_get_phy_index_params,
3082 #endif
3083 };
3084
3085 static int rk3399_dmc_init(struct udevice *dev)
3086 {
3087         struct dram_info *priv = dev_get_priv(dev);
3088         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
3089         int ret;
3090 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
3091         struct rk3399_sdram_params *params = &plat->sdram_params;
3092 #else
3093         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
3094         struct rk3399_sdram_params *params =
3095                                         (void *)dtplat->rockchip_sdram_params;
3096
3097         ret = conv_of_platdata(dev);
3098         if (ret)
3099                 return ret;
3100 #endif
3101
3102         priv->ops = &rk3399_ops;
3103         priv->cic = syscon_get_first_range(ROCKCHIP_SYSCON_CIC);
3104         priv->grf = syscon_get_first_range(ROCKCHIP_SYSCON_GRF);
3105         priv->pmu = syscon_get_first_range(ROCKCHIP_SYSCON_PMU);
3106         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
3107         priv->pmusgrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUSGRF);
3108         priv->pmucru = rockchip_get_pmucru();
3109         priv->cru = rockchip_get_cru();
3110         priv->chan[0].pctl = regmap_get_range(plat->map, 0);
3111         priv->chan[0].pi = regmap_get_range(plat->map, 1);
3112         priv->chan[0].publ = regmap_get_range(plat->map, 2);
3113         priv->chan[0].msch = regmap_get_range(plat->map, 3);
3114         priv->chan[1].pctl = regmap_get_range(plat->map, 4);
3115         priv->chan[1].pi = regmap_get_range(plat->map, 5);
3116         priv->chan[1].publ = regmap_get_range(plat->map, 6);
3117         priv->chan[1].msch = regmap_get_range(plat->map, 7);
3118
3119         debug("con reg %p %p %p %p %p %p %p %p\n",
3120               priv->chan[0].pctl, priv->chan[0].pi,
3121               priv->chan[0].publ, priv->chan[0].msch,
3122               priv->chan[1].pctl, priv->chan[1].pi,
3123               priv->chan[1].publ, priv->chan[1].msch);
3124         debug("cru %p, cic %p, grf %p, sgrf %p, pmucru %p, pmu %p\n", priv->cru,
3125               priv->cic, priv->pmugrf, priv->pmusgrf, priv->pmucru, priv->pmu);
3126
3127 #if CONFIG_IS_ENABLED(OF_PLATDATA)
3128         ret = clk_get_by_index_platdata(dev, 0, dtplat->clocks, &priv->ddr_clk);
3129 #else
3130         ret = clk_get_by_index(dev, 0, &priv->ddr_clk);
3131 #endif
3132         if (ret) {
3133                 printf("%s clk get failed %d\n", __func__, ret);
3134                 return ret;
3135         }
3136
3137         ret = clk_set_rate(&priv->ddr_clk, params->base.ddr_freq * MHz);
3138         if (ret < 0) {
3139                 printf("%s clk set failed %d\n", __func__, ret);
3140                 return ret;
3141         }
3142
3143         ret = sdram_init(priv, params);
3144         if (ret < 0) {
3145                 printf("%s DRAM init failed %d\n", __func__, ret);
3146                 return ret;
3147         }
3148
3149         return 0;
3150 }
3151 #endif
3152
3153 static int rk3399_dmc_probe(struct udevice *dev)
3154 {
3155 #if defined(CONFIG_TPL_BUILD) || \
3156         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
3157         if (rk3399_dmc_init(dev))
3158                 return 0;
3159 #else
3160         struct dram_info *priv = dev_get_priv(dev);
3161
3162         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
3163         debug("%s: pmugrf = %p\n", __func__, priv->pmugrf);
3164         priv->info.base = CONFIG_SYS_SDRAM_BASE;
3165         priv->info.size =
3166                 rockchip_sdram_size((phys_addr_t)&priv->pmugrf->os_reg2);
3167 #endif
3168         return 0;
3169 }
3170
3171 static int rk3399_dmc_get_info(struct udevice *dev, struct ram_info *info)
3172 {
3173         struct dram_info *priv = dev_get_priv(dev);
3174
3175         *info = priv->info;
3176
3177         return 0;
3178 }
3179
3180 static struct ram_ops rk3399_dmc_ops = {
3181         .get_info = rk3399_dmc_get_info,
3182 };
3183
3184 static const struct udevice_id rk3399_dmc_ids[] = {
3185         { .compatible = "rockchip,rk3399-dmc" },
3186         { }
3187 };
3188
3189 U_BOOT_DRIVER(dmc_rk3399) = {
3190         .name = "rockchip_rk3399_dmc",
3191         .id = UCLASS_RAM,
3192         .of_match = rk3399_dmc_ids,
3193         .ops = &rk3399_dmc_ops,
3194 #if defined(CONFIG_TPL_BUILD) || \
3195         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
3196         .ofdata_to_platdata = rk3399_dmc_ofdata_to_platdata,
3197 #endif
3198         .probe = rk3399_dmc_probe,
3199         .priv_auto_alloc_size = sizeof(struct dram_info),
3200 #if defined(CONFIG_TPL_BUILD) || \
3201         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
3202         .platdata_auto_alloc_size = sizeof(struct rockchip_dmc_plat),
3203 #endif
3204 };