drivers/ram/rockchip/sdram_rk3399.c

   1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
   2 /*
   3  * (C) Copyright 2016-2017 Rockchip Inc.
   4  *
   5  * Adapted from coreboot.
   6  */
   7
   8 #include <common.h>
   9 #include <clk.h>
  10 #include <dm.h>
  11 #include <dt-structs.h>
  12 #include <ram.h>
  13 #include <regmap.h>
  14 #include <syscon.h>
  15 #include <asm/io.h>
  16 #include <asm/arch-rockchip/clock.h>
  17 #include <asm/arch-rockchip/cru_rk3399.h>
  18 #include <asm/arch-rockchip/grf_rk3399.h>
  19 #include <asm/arch-rockchip/hardware.h>
  20 #include <asm/arch-rockchip/sdram_common.h>
  21 #include <asm/arch-rockchip/sdram_rk3399.h>
  22 #include <linux/err.h>
  23 #include <time.h>
  24
  25 #define PRESET_SGRF_HOLD(n)     ((0x1 << (6 + 16)) | ((n) << 6))
  26 #define PRESET_GPIO0_HOLD(n)    ((0x1 << (7 + 16)) | ((n) << 7))
  27 #define PRESET_GPIO1_HOLD(n)    ((0x1 << (8 + 16)) | ((n) << 8))
  28
  29 #define PHY_DRV_ODT_HI_Z        0x0
  30 #define PHY_DRV_ODT_240         0x1
  31 #define PHY_DRV_ODT_120         0x8
  32 #define PHY_DRV_ODT_80          0x9
  33 #define PHY_DRV_ODT_60          0xc
  34 #define PHY_DRV_ODT_48          0xd
  35 #define PHY_DRV_ODT_40          0xe
  36 #define PHY_DRV_ODT_34_3        0xf
  37
  38 #define CRU_SFTRST_DDR_CTRL(ch, n)      ((0x1 << (8 + 16 + (ch) * 4)) | \
  39                                         ((n) << (8 + (ch) * 4)))
  40 #define CRU_SFTRST_DDR_PHY(ch, n)       ((0x1 << (9 + 16 + (ch) * 4)) | \
  41                                         ((n) << (9 + (ch) * 4)))
  42 struct chan_info {
  43         struct rk3399_ddr_pctl_regs *pctl;
  44         struct rk3399_ddr_pi_regs *pi;
  45         struct rk3399_ddr_publ_regs *publ;
  46         struct rk3399_msch_regs *msch;
  47 };
  48
  49 struct dram_info {
  50 #if defined(CONFIG_TPL_BUILD) || \
  51         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  52         u32 pwrup_srefresh_exit;
  53         struct chan_info chan[2];
  54         struct clk ddr_clk;
  55         struct rk3399_cru *cru;
  56         struct rk3399_pmucru *pmucru;
  57         struct rk3399_pmusgrf_regs *pmusgrf;
  58         struct rk3399_ddr_cic_regs *cic;
  59 #endif
  60         struct ram_info info;
  61         struct rk3399_pmugrf_regs *pmugrf;
  62 };
  63
  64 #if defined(CONFIG_TPL_BUILD) || \
  65         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
  66
  67 struct rockchip_dmc_plat {
  68 #if CONFIG_IS_ENABLED(OF_PLATDATA)
  69         struct dtd_rockchip_rk3399_dmc dtplat;
  70 #else
  71         struct rk3399_sdram_params sdram_params;
  72 #endif
  73         struct regmap *map;
  74 };
  75
  76 static void copy_to_reg(u32 *dest, const u32 *src, u32 n)
  77 {
  78         int i;
  79
  80         for (i = 0; i < n / sizeof(u32); i++) {
  81                 writel(*src, dest);
  82                 src++;
  83                 dest++;
  84         }
  85 }
  86
  87 static void rkclk_ddr_reset(struct rk3399_cru *cru, u32 channel, u32 ctl,
  88                             u32 phy)
  89 {
  90         channel &= 0x1;
  91         ctl &= 0x1;
  92         phy &= 0x1;
  93         writel(CRU_SFTRST_DDR_CTRL(channel, ctl) |
  94                                    CRU_SFTRST_DDR_PHY(channel, phy),
  95                                    &cru->softrst_con[4]);
  96 }
  97
  98 static void phy_pctrl_reset(struct rk3399_cru *cru,  u32 channel)
  99 {
 100         rkclk_ddr_reset(cru, channel, 1, 1);
 101         udelay(10);
 102
 103         rkclk_ddr_reset(cru, channel, 1, 0);
 104         udelay(10);
 105
 106         rkclk_ddr_reset(cru, channel, 0, 0);
 107         udelay(10);
 108 }
 109
 110 static void phy_dll_bypass_set(struct rk3399_ddr_publ_regs *ddr_publ_regs,
 111                                u32 freq)
 112 {
 113         u32 *denali_phy = ddr_publ_regs->denali_phy;
 114
 115         /* From IP spec, only freq small than 125 can enter dll bypass mode */
 116         if (freq <= 125) {
 117                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 118                 setbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 119                 setbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 120                 setbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 121                 setbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 122
 123                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 124                 setbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 125                 setbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 126                 setbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 127         } else {
 128                 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */
 129                 clrbits_le32(&denali_phy[86], (0x3 << 2) << 8);
 130                 clrbits_le32(&denali_phy[214], (0x3 << 2) << 8);
 131                 clrbits_le32(&denali_phy[342], (0x3 << 2) << 8);
 132                 clrbits_le32(&denali_phy[470], (0x3 << 2) << 8);
 133
 134                 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */
 135                 clrbits_le32(&denali_phy[547], (0x3 << 2) << 16);
 136                 clrbits_le32(&denali_phy[675], (0x3 << 2) << 16);
 137                 clrbits_le32(&denali_phy[803], (0x3 << 2) << 16);
 138         }
 139 }
 140
 141 static void set_memory_map(const struct chan_info *chan, u32 channel,
 142                            const struct rk3399_sdram_params *params)
 143 {
 144         const struct rk3399_sdram_channel *sdram_ch = &params->ch[channel];
 145         u32 *denali_ctl = chan->pctl->denali_ctl;
 146         u32 *denali_pi = chan->pi->denali_pi;
 147         u32 cs_map;
 148         u32 reduc;
 149         u32 row;
 150
 151         /* Get row number from ddrconfig setting */
 152         if (sdram_ch->cap_info.ddrconfig < 2 ||
 153             sdram_ch->cap_info.ddrconfig == 4)
 154                 row = 16;
 155         else if (sdram_ch->cap_info.ddrconfig == 3)
 156                 row = 14;
 157         else
 158                 row = 15;
 159
 160         cs_map = (sdram_ch->cap_info.rank > 1) ? 3 : 1;
 161         reduc = (sdram_ch->cap_info.bw == 2) ? 0 : 1;
 162
 163         /* Set the dram configuration to ctrl */
 164         clrsetbits_le32(&denali_ctl[191], 0xF, (12 - sdram_ch->cap_info.col));
 165         clrsetbits_le32(&denali_ctl[190], (0x3 << 16) | (0x7 << 24),
 166                         ((3 - sdram_ch->cap_info.bk) << 16) |
 167                         ((16 - row) << 24));
 168
 169         clrsetbits_le32(&denali_ctl[196], 0x3 | (1 << 16),
 170                         cs_map | (reduc << 16));
 171
 172         /* PI_199 PI_COL_DIFF:RW:0:4 */
 173         clrsetbits_le32(&denali_pi[199], 0xF, (12 - sdram_ch->cap_info.col));
 174
 175         /* PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2 */
 176         clrsetbits_le32(&denali_pi[155], (0x3 << 16) | (0x7 << 24),
 177                         ((3 - sdram_ch->cap_info.bk) << 16) |
 178                         ((16 - row) << 24));
 179         /* PI_41 PI_CS_MAP:RW:24:4 */
 180         clrsetbits_le32(&denali_pi[41], 0xf << 24, cs_map << 24);
 181         if (sdram_ch->cap_info.rank == 1 && params->base.dramtype == DDR3)
 182                 writel(0x2EC7FFFF, &denali_pi[34]);
 183 }
 184
 185 static void set_ds_odt(const struct chan_info *chan,
 186                        const struct rk3399_sdram_params *params)
 187 {
 188         u32 *denali_phy = chan->publ->denali_phy;
 189
 190         u32 tsel_idle_en, tsel_wr_en, tsel_rd_en;
 191         u32 tsel_idle_select_p, tsel_rd_select_p;
 192         u32 tsel_idle_select_n, tsel_rd_select_n;
 193         u32 tsel_wr_select_dq_p, tsel_wr_select_ca_p;
 194         u32 tsel_wr_select_dq_n, tsel_wr_select_ca_n;
 195         u32 reg_value;
 196
 197         if (params->base.dramtype == LPDDR4) {
 198                 tsel_rd_select_p = PHY_DRV_ODT_HI_Z;
 199                 tsel_rd_select_n = PHY_DRV_ODT_240;
 200
 201                 tsel_idle_select_p = PHY_DRV_ODT_HI_Z;
 202                 tsel_idle_select_n = PHY_DRV_ODT_240;
 203
 204                 tsel_wr_select_dq_p = PHY_DRV_ODT_40;
 205                 tsel_wr_select_dq_n = PHY_DRV_ODT_40;
 206
 207                 tsel_wr_select_ca_p = PHY_DRV_ODT_40;
 208                 tsel_wr_select_ca_n = PHY_DRV_ODT_40;
 209         } else if (params->base.dramtype == LPDDR3) {
 210                 tsel_rd_select_p = PHY_DRV_ODT_240;
 211                 tsel_rd_select_n = PHY_DRV_ODT_HI_Z;
 212
 213                 tsel_idle_select_p = PHY_DRV_ODT_240;
 214                 tsel_idle_select_n = PHY_DRV_ODT_HI_Z;
 215
 216                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 217                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 218
 219                 tsel_wr_select_ca_p = PHY_DRV_ODT_48;
 220                 tsel_wr_select_ca_n = PHY_DRV_ODT_48;
 221         } else {
 222                 tsel_rd_select_p = PHY_DRV_ODT_240;
 223                 tsel_rd_select_n = PHY_DRV_ODT_240;
 224
 225                 tsel_idle_select_p = PHY_DRV_ODT_240;
 226                 tsel_idle_select_n = PHY_DRV_ODT_240;
 227
 228                 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3;
 229                 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3;
 230
 231                 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3;
 232                 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3;
 233         }
 234
 235         if (params->base.odt == 1)
 236                 tsel_rd_en = 1;
 237         else
 238                 tsel_rd_en = 0;
 239
 240         tsel_wr_en = 0;
 241         tsel_idle_en = 0;
 242
 243         /*
 244          * phy_dq_tsel_select_X 24bits DENALI_PHY_6/134/262/390 offset_0
 245          * sets termination values for read/idle cycles and drive strength
 246          * for write cycles for DQ/DM
 247          */
 248         reg_value = tsel_rd_select_n | (tsel_rd_select_p << 0x4) |
 249                     (tsel_wr_select_dq_n << 8) | (tsel_wr_select_dq_p << 12) |
 250                     (tsel_idle_select_n << 16) | (tsel_idle_select_p << 20);
 251         clrsetbits_le32(&denali_phy[6], 0xffffff, reg_value);
 252         clrsetbits_le32(&denali_phy[134], 0xffffff, reg_value);
 253         clrsetbits_le32(&denali_phy[262], 0xffffff, reg_value);
 254         clrsetbits_le32(&denali_phy[390], 0xffffff, reg_value);
 255
 256         /*
 257          * phy_dqs_tsel_select_X 24bits DENALI_PHY_7/135/263/391 offset_0
 258          * sets termination values for read/idle cycles and drive strength
 259          * for write cycles for DQS
 260          */
 261         clrsetbits_le32(&denali_phy[7], 0xffffff, reg_value);
 262         clrsetbits_le32(&denali_phy[135], 0xffffff, reg_value);
 263         clrsetbits_le32(&denali_phy[263], 0xffffff, reg_value);
 264         clrsetbits_le32(&denali_phy[391], 0xffffff, reg_value);
 265
 266         /* phy_adr_tsel_select_ 8bits DENALI_PHY_544/672/800 offset_0 */
 267         reg_value = tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 0x4);
 268         clrsetbits_le32(&denali_phy[544], 0xff, reg_value);
 269         clrsetbits_le32(&denali_phy[672], 0xff, reg_value);
 270         clrsetbits_le32(&denali_phy[800], 0xff, reg_value);
 271
 272         /* phy_pad_addr_drive 8bits DENALI_PHY_928 offset_0 */
 273         clrsetbits_le32(&denali_phy[928], 0xff, reg_value);
 274
 275         /* phy_pad_rst_drive 8bits DENALI_PHY_937 offset_0 */
 276         clrsetbits_le32(&denali_phy[937], 0xff, reg_value);
 277
 278         /* phy_pad_cke_drive 8bits DENALI_PHY_935 offset_0 */
 279         clrsetbits_le32(&denali_phy[935], 0xff, reg_value);
 280
 281         /* phy_pad_cs_drive 8bits DENALI_PHY_939 offset_0 */
 282         clrsetbits_le32(&denali_phy[939], 0xff, reg_value);
 283
 284         /* phy_pad_clk_drive 8bits DENALI_PHY_929 offset_0 */
 285         clrsetbits_le32(&denali_phy[929], 0xff, reg_value);
 286
 287         /* phy_pad_fdbk_drive 23bit DENALI_PHY_924/925 */
 288         clrsetbits_le32(&denali_phy[924], 0xff,
 289                         tsel_wr_select_dq_n | (tsel_wr_select_dq_p << 4));
 290         clrsetbits_le32(&denali_phy[925], 0xff,
 291                         tsel_rd_select_n | (tsel_rd_select_p << 4));
 292
 293         /* phy_dq_tsel_enable_X 3bits DENALI_PHY_5/133/261/389 offset_16 */
 294         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 295                 << 16;
 296         clrsetbits_le32(&denali_phy[5], 0x7 << 16, reg_value);
 297         clrsetbits_le32(&denali_phy[133], 0x7 << 16, reg_value);
 298         clrsetbits_le32(&denali_phy[261], 0x7 << 16, reg_value);
 299         clrsetbits_le32(&denali_phy[389], 0x7 << 16, reg_value);
 300
 301         /* phy_dqs_tsel_enable_X 3bits DENALI_PHY_6/134/262/390 offset_24 */
 302         reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2))
 303                 << 24;
 304         clrsetbits_le32(&denali_phy[6], 0x7 << 24, reg_value);
 305         clrsetbits_le32(&denali_phy[134], 0x7 << 24, reg_value);
 306         clrsetbits_le32(&denali_phy[262], 0x7 << 24, reg_value);
 307         clrsetbits_le32(&denali_phy[390], 0x7 << 24, reg_value);
 308
 309         /* phy_adr_tsel_enable_ 1bit DENALI_PHY_518/646/774 offset_8 */
 310         reg_value = tsel_wr_en << 8;
 311         clrsetbits_le32(&denali_phy[518], 0x1 << 8, reg_value);
 312         clrsetbits_le32(&denali_phy[646], 0x1 << 8, reg_value);
 313         clrsetbits_le32(&denali_phy[774], 0x1 << 8, reg_value);
 314
 315         /* phy_pad_addr_term tsel 1bit DENALI_PHY_933 offset_17 */
 316         reg_value = tsel_wr_en << 17;
 317         clrsetbits_le32(&denali_phy[933], 0x1 << 17, reg_value);
 318         /*
 319          * pad_rst/cke/cs/clk_term tsel 1bits
 320          * DENALI_PHY_938/936/940/934 offset_17
 321          */
 322         clrsetbits_le32(&denali_phy[938], 0x1 << 17, reg_value);
 323         clrsetbits_le32(&denali_phy[936], 0x1 << 17, reg_value);
 324         clrsetbits_le32(&denali_phy[940], 0x1 << 17, reg_value);
 325         clrsetbits_le32(&denali_phy[934], 0x1 << 17, reg_value);
 326
 327         /* phy_pad_fdbk_term 1bit DENALI_PHY_930 offset_17 */
 328         clrsetbits_le32(&denali_phy[930], 0x1 << 17, reg_value);
 329 }
 330
 331 static int phy_io_config(const struct chan_info *chan,
 332                          const struct rk3399_sdram_params *params)
 333 {
 334         u32 *denali_phy = chan->publ->denali_phy;
 335         u32 vref_mode_dq, vref_value_dq, vref_mode_ac, vref_value_ac;
 336         u32 mode_sel;
 337         u32 reg_value;
 338         u32 drv_value, odt_value;
 339         u32 speed;
 340
 341         /* vref setting */
 342         if (params->base.dramtype == LPDDR4) {
 343                 /* LPDDR4 */
 344                 vref_mode_dq = 0x6;
 345                 vref_value_dq = 0x1f;
 346                 vref_mode_ac = 0x6;
 347                 vref_value_ac = 0x1f;
 348         } else if (params->base.dramtype == LPDDR3) {
 349                 if (params->base.odt == 1) {
 350                         vref_mode_dq = 0x5;  /* LPDDR3 ODT */
 351                         drv_value = (readl(&denali_phy[6]) >> 12) & 0xf;
 352                         odt_value = (readl(&denali_phy[6]) >> 4) & 0xf;
 353                         if (drv_value == PHY_DRV_ODT_48) {
 354                                 switch (odt_value) {
 355                                 case PHY_DRV_ODT_240:
 356                                         vref_value_dq = 0x16;
 357                                         break;
 358                                 case PHY_DRV_ODT_120:
 359                                         vref_value_dq = 0x26;
 360                                         break;
 361                                 case PHY_DRV_ODT_60:
 362                                         vref_value_dq = 0x36;
 363                                         break;
 364                                 default:
 365                                         debug("Invalid ODT value.\n");
 366                                         return -EINVAL;
 367                                 }
 368                         } else if (drv_value == PHY_DRV_ODT_40) {
 369                                 switch (odt_value) {
 370                                 case PHY_DRV_ODT_240:
 371                                         vref_value_dq = 0x19;
 372                                         break;
 373                                 case PHY_DRV_ODT_120:
 374                                         vref_value_dq = 0x23;
 375                                         break;
 376                                 case PHY_DRV_ODT_60:
 377                                         vref_value_dq = 0x31;
 378                                         break;
 379                                 default:
 380                                         debug("Invalid ODT value.\n");
 381                                         return -EINVAL;
 382                                 }
 383                         } else if (drv_value == PHY_DRV_ODT_34_3) {
 384                                 switch (odt_value) {
 385                                 case PHY_DRV_ODT_240:
 386                                         vref_value_dq = 0x17;
 387                                         break;
 388                                 case PHY_DRV_ODT_120:
 389                                         vref_value_dq = 0x20;
 390                                         break;
 391                                 case PHY_DRV_ODT_60:
 392                                         vref_value_dq = 0x2e;
 393                                         break;
 394                                 default:
 395                                         debug("Invalid ODT value.\n");
 396                                         return -EINVAL;
 397                                 }
 398                         } else {
 399                                 debug("Invalid DRV value.\n");
 400                                 return -EINVAL;
 401                         }
 402                 } else {
 403                         vref_mode_dq = 0x2;  /* LPDDR3 */
 404                         vref_value_dq = 0x1f;
 405                 }
 406                 vref_mode_ac = 0x2;
 407                 vref_value_ac = 0x1f;
 408         } else if (params->base.dramtype == DDR3) {
 409                 /* DDR3L */
 410                 vref_mode_dq = 0x1;
 411                 vref_value_dq = 0x1f;
 412                 vref_mode_ac = 0x1;
 413                 vref_value_ac = 0x1f;
 414         } else {
 415                 debug("Unknown DRAM type.\n");
 416                 return -EINVAL;
 417         }
 418
 419         reg_value = (vref_mode_dq << 9) | (0x1 << 8) | vref_value_dq;
 420
 421         /* PHY_913 PHY_PAD_VREF_CTRL_DQ_0 12bits offset_8 */
 422         clrsetbits_le32(&denali_phy[913], 0xfff << 8, reg_value << 8);
 423         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_1 12bits offset_0 */
 424         clrsetbits_le32(&denali_phy[914], 0xfff, reg_value);
 425         /* PHY_914 PHY_PAD_VREF_CTRL_DQ_2 12bits offset_16 */
 426         clrsetbits_le32(&denali_phy[914], 0xfff << 16, reg_value << 16);
 427         /* PHY_915 PHY_PAD_VREF_CTRL_DQ_3 12bits offset_0 */
 428         clrsetbits_le32(&denali_phy[915], 0xfff, reg_value);
 429
 430         reg_value = (vref_mode_ac << 9) | (0x1 << 8) | vref_value_ac;
 431
 432         /* PHY_915 PHY_PAD_VREF_CTRL_AC 12bits offset_16 */
 433         clrsetbits_le32(&denali_phy[915], 0xfff << 16, reg_value << 16);
 434
 435         if (params->base.dramtype == LPDDR4)
 436                 mode_sel = 0x6;
 437         else if (params->base.dramtype == LPDDR3)
 438                 mode_sel = 0x0;
 439         else if (params->base.dramtype == DDR3)
 440                 mode_sel = 0x1;
 441         else
 442                 return -EINVAL;
 443
 444         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 445         clrsetbits_le32(&denali_phy[924], 0x7 << 15, mode_sel << 15);
 446         /* PHY_926 PHY_PAD_DATA_DRIVE */
 447         clrsetbits_le32(&denali_phy[926], 0x7 << 6, mode_sel << 6);
 448         /* PHY_927 PHY_PAD_DQS_DRIVE */
 449         clrsetbits_le32(&denali_phy[927], 0x7 << 6, mode_sel << 6);
 450         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 451         clrsetbits_le32(&denali_phy[928], 0x7 << 14, mode_sel << 14);
 452         /* PHY_929 PHY_PAD_CLK_DRIVE */
 453         clrsetbits_le32(&denali_phy[929], 0x7 << 14, mode_sel << 14);
 454         /* PHY_935 PHY_PAD_CKE_DRIVE */
 455         clrsetbits_le32(&denali_phy[935], 0x7 << 14, mode_sel << 14);
 456         /* PHY_937 PHY_PAD_RST_DRIVE */
 457         clrsetbits_le32(&denali_phy[937], 0x7 << 14, mode_sel << 14);
 458         /* PHY_939 PHY_PAD_CS_DRIVE */
 459         clrsetbits_le32(&denali_phy[939], 0x7 << 14, mode_sel << 14);
 460
 461         /* speed setting */
 462         if (params->base.ddr_freq < 400)
 463                 speed = 0x0;
 464         else if (params->base.ddr_freq < 800)
 465                 speed = 0x1;
 466         else if (params->base.ddr_freq < 1200)
 467                 speed = 0x2;
 468         else
 469                 speed = 0x3;
 470
 471         /* PHY_924 PHY_PAD_FDBK_DRIVE */
 472         clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21);
 473         /* PHY_926 PHY_PAD_DATA_DRIVE */
 474         clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9);
 475         /* PHY_927 PHY_PAD_DQS_DRIVE */
 476         clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9);
 477         /* PHY_928 PHY_PAD_ADDR_DRIVE */
 478         clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17);
 479         /* PHY_929 PHY_PAD_CLK_DRIVE */
 480         clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17);
 481         /* PHY_935 PHY_PAD_CKE_DRIVE */
 482         clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17);
 483         /* PHY_937 PHY_PAD_RST_DRIVE */
 484         clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17);
 485         /* PHY_939 PHY_PAD_CS_DRIVE */
 486         clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17);
 487
 488         return 0;
 489 }
 490
 491 static int pctl_cfg(struct dram_info *dram, const struct chan_info *chan,
 492                     u32 channel, const struct rk3399_sdram_params *params)
 493 {
 494         u32 *denali_ctl = chan->pctl->denali_ctl;
 495         u32 *denali_pi = chan->pi->denali_pi;
 496         u32 *denali_phy = chan->publ->denali_phy;
 497         const u32 *params_ctl = params->pctl_regs.denali_ctl;
 498         const u32 *params_phy = params->phy_regs.denali_phy;
 499         u32 tmp, tmp1, tmp2;
 500         int ret;
 501         const ulong timeout_ms = 200;
 502
 503         /*
 504          * work around controller bug:
 505          * Do not program DRAM_CLASS until NO_PHY_IND_TRAIN_INT is programmed
 506          */
 507         copy_to_reg(&denali_ctl[1], &params_ctl[1],
 508                     sizeof(struct rk3399_ddr_pctl_regs) - 4);
 509         writel(params_ctl[0], &denali_ctl[0]);
 510
 511         copy_to_reg(denali_pi, &params->pi_regs.denali_pi[0],
 512                     sizeof(struct rk3399_ddr_pi_regs));
 513
 514         /* rank count need to set for init */
 515         set_memory_map(chan, channel, params);
 516
 517         writel(params->phy_regs.denali_phy[910], &denali_phy[910]);
 518         writel(params->phy_regs.denali_phy[911], &denali_phy[911]);
 519         writel(params->phy_regs.denali_phy[912], &denali_phy[912]);
 520
 521         dram->pwrup_srefresh_exit = readl(&denali_ctl[68]) &
 522                                     PWRUP_SREFRESH_EXIT;
 523         clrbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT);
 524
 525         /* PHY_DLL_RST_EN */
 526         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 1 << 24);
 527
 528         setbits_le32(&denali_pi[0], START);
 529         setbits_le32(&denali_ctl[0], START);
 530
 531         /* Waiting for phy DLL lock */
 532         while (1) {
 533                 tmp = readl(&denali_phy[920]);
 534                 tmp1 = readl(&denali_phy[921]);
 535                 tmp2 = readl(&denali_phy[922]);
 536                 if ((((tmp >> 16) & 0x1) == 0x1) &&
 537                     (((tmp1 >> 16) & 0x1) == 0x1) &&
 538                     (((tmp1 >> 0) & 0x1) == 0x1) &&
 539                     (((tmp2 >> 0) & 0x1) == 0x1))
 540                         break;
 541         }
 542
 543         copy_to_reg(&denali_phy[896], &params_phy[896], (958 - 895) * 4);
 544         copy_to_reg(&denali_phy[0], &params_phy[0], (90 - 0 + 1) * 4);
 545         copy_to_reg(&denali_phy[128], &params_phy[128], (218 - 128 + 1) * 4);
 546         copy_to_reg(&denali_phy[256], &params_phy[256], (346 - 256 + 1) * 4);
 547         copy_to_reg(&denali_phy[384], &params_phy[384], (474 - 384 + 1) * 4);
 548         copy_to_reg(&denali_phy[512], &params_phy[512], (549 - 512 + 1) * 4);
 549         copy_to_reg(&denali_phy[640], &params_phy[640], (677 - 640 + 1) * 4);
 550         copy_to_reg(&denali_phy[768], &params_phy[768], (805 - 768 + 1) * 4);
 551         set_ds_odt(chan, params);
 552
 553         /*
 554          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_84/212/340/468 offset_8
 555          * dqs_tsel_wr_end[7:4] add Half cycle
 556          */
 557         tmp = (readl(&denali_phy[84]) >> 8) & 0xff;
 558         clrsetbits_le32(&denali_phy[84], 0xff << 8, (tmp + 0x10) << 8);
 559         tmp = (readl(&denali_phy[212]) >> 8) & 0xff;
 560         clrsetbits_le32(&denali_phy[212], 0xff << 8, (tmp + 0x10) << 8);
 561         tmp = (readl(&denali_phy[340]) >> 8) & 0xff;
 562         clrsetbits_le32(&denali_phy[340], 0xff << 8, (tmp + 0x10) << 8);
 563         tmp = (readl(&denali_phy[468]) >> 8) & 0xff;
 564         clrsetbits_le32(&denali_phy[468], 0xff << 8, (tmp + 0x10) << 8);
 565
 566         /*
 567          * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_83/211/339/467 offset_8
 568          * dq_tsel_wr_end[7:4] add Half cycle
 569          */
 570         tmp = (readl(&denali_phy[83]) >> 16) & 0xff;
 571         clrsetbits_le32(&denali_phy[83], 0xff << 16, (tmp + 0x10) << 16);
 572         tmp = (readl(&denali_phy[211]) >> 16) & 0xff;
 573         clrsetbits_le32(&denali_phy[211], 0xff << 16, (tmp + 0x10) << 16);
 574         tmp = (readl(&denali_phy[339]) >> 16) & 0xff;
 575         clrsetbits_le32(&denali_phy[339], 0xff << 16, (tmp + 0x10) << 16);
 576         tmp = (readl(&denali_phy[467]) >> 16) & 0xff;
 577         clrsetbits_le32(&denali_phy[467], 0xff << 16, (tmp + 0x10) << 16);
 578
 579         ret = phy_io_config(chan, params);
 580         if (ret)
 581                 return ret;
 582
 583         /* PHY_DLL_RST_EN */
 584         clrsetbits_le32(&denali_phy[957], 0x3 << 24, 0x2 << 24);
 585
 586         /* Waiting for PHY and DRAM init complete */
 587         tmp = get_timer(0);
 588         do {
 589                 if (get_timer(tmp) > timeout_ms) {
 590                         pr_err("DRAM (%s): phy failed to lock within  %ld ms\n",
 591                                __func__, timeout_ms);
 592                         return -ETIME;
 593                 }
 594         } while (!(readl(&denali_ctl[203]) & (1 << 3)));
 595         debug("DRAM (%s): phy locked after %ld ms\n", __func__, get_timer(tmp));
 596
 597         clrsetbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT,
 598                         dram->pwrup_srefresh_exit);
 599         return 0;
 600 }
 601
 602 static void select_per_cs_training_index(const struct chan_info *chan,
 603                                          u32 rank)
 604 {
 605         u32 *denali_phy = chan->publ->denali_phy;
 606
 607         /* PHY_84 PHY_PER_CS_TRAINING_EN_0 1bit offset_16 */
 608         if ((readl(&denali_phy[84]) >> 16) & 1) {
 609                 /*
 610                  * PHY_8/136/264/392
 611                  * phy_per_cs_training_index_X 1bit offset_24
 612                  */
 613                 clrsetbits_le32(&denali_phy[8], 0x1 << 24, rank << 24);
 614                 clrsetbits_le32(&denali_phy[136], 0x1 << 24, rank << 24);
 615                 clrsetbits_le32(&denali_phy[264], 0x1 << 24, rank << 24);
 616                 clrsetbits_le32(&denali_phy[392], 0x1 << 24, rank << 24);
 617         }
 618 }
 619
 620 static void override_write_leveling_value(const struct chan_info *chan)
 621 {
 622         u32 *denali_ctl = chan->pctl->denali_ctl;
 623         u32 *denali_phy = chan->publ->denali_phy;
 624         u32 byte;
 625
 626         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
 627         setbits_le32(&denali_phy[896], 1);
 628
 629         /*
 630          * PHY_8/136/264/392
 631          * phy_per_cs_training_multicast_en_X 1bit offset_16
 632          */
 633         clrsetbits_le32(&denali_phy[8], 0x1 << 16, 1 << 16);
 634         clrsetbits_le32(&denali_phy[136], 0x1 << 16, 1 << 16);
 635         clrsetbits_le32(&denali_phy[264], 0x1 << 16, 1 << 16);
 636         clrsetbits_le32(&denali_phy[392], 0x1 << 16, 1 << 16);
 637
 638         for (byte = 0; byte < 4; byte++)
 639                 clrsetbits_le32(&denali_phy[63 + (128 * byte)], 0xffff << 16,
 640                                 0x200 << 16);
 641
 642         /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */
 643         clrbits_le32(&denali_phy[896], 1);
 644
 645         /* CTL_200 ctrlupd_req 1bit offset_8 */
 646         clrsetbits_le32(&denali_ctl[200], 0x1 << 8, 0x1 << 8);
 647 }
 648
 649 static int data_training_ca(const struct chan_info *chan, u32 channel,
 650                             const struct rk3399_sdram_params *params)
 651 {
 652         u32 *denali_pi = chan->pi->denali_pi;
 653         u32 *denali_phy = chan->publ->denali_phy;
 654         u32 i, tmp;
 655         u32 obs_0, obs_1, obs_2, obs_err = 0;
 656         u32 rank = params->ch[channel].cap_info.rank;
 657         u32 rank_mask;
 658
 659         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 660         writel(0x00003f7c, (&denali_pi[175]));
 661
 662         rank_mask = (rank == 1) ? 0x1 : 0x3;
 663
 664         for (i = 0; i < 4; i++) {
 665                 if (!(rank_mask & (1 << i)))
 666                         continue;
 667
 668                 select_per_cs_training_index(chan, i);
 669
 670                 /* PI_100 PI_CALVL_EN:RW:8:2 */
 671                 clrsetbits_le32(&denali_pi[100], 0x3 << 8, 0x2 << 8);
 672
 673                 /* PI_92 PI_CALVL_REQ:WR:16:1,PI_CALVL_CS:RW:24:2 */
 674                 clrsetbits_le32(&denali_pi[92],
 675                                 (0x1 << 16) | (0x3 << 24),
 676                                 (0x1 << 16) | (i << 24));
 677
 678                 /* Waiting for training complete */
 679                 while (1) {
 680                         /* PI_174 PI_INT_STATUS:RD:8:18 */
 681                         tmp = readl(&denali_pi[174]) >> 8;
 682                         /*
 683                          * check status obs
 684                          * PHY_532/660/789 phy_adr_calvl_obs1_:0:32
 685                          */
 686                         obs_0 = readl(&denali_phy[532]);
 687                         obs_1 = readl(&denali_phy[660]);
 688                         obs_2 = readl(&denali_phy[788]);
 689                         if (((obs_0 >> 30) & 0x3) ||
 690                             ((obs_1 >> 30) & 0x3) ||
 691                             ((obs_2 >> 30) & 0x3))
 692                                 obs_err = 1;
 693                         if ((((tmp >> 11) & 0x1) == 0x1) &&
 694                             (((tmp >> 13) & 0x1) == 0x1) &&
 695                             (((tmp >> 5) & 0x1) == 0x0) &&
 696                             obs_err == 0)
 697                                 break;
 698                         else if ((((tmp >> 5) & 0x1) == 0x1) ||
 699                                  (obs_err == 1))
 700                                 return -EIO;
 701                 }
 702
 703                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 704                 writel(0x00003f7c, (&denali_pi[175]));
 705         }
 706
 707         clrbits_le32(&denali_pi[100], 0x3 << 8);
 708
 709         return 0;
 710 }
 711
 712 static int data_training_wl(const struct chan_info *chan, u32 channel,
 713                             const struct rk3399_sdram_params *params)
 714 {
 715         u32 *denali_pi = chan->pi->denali_pi;
 716         u32 *denali_phy = chan->publ->denali_phy;
 717         u32 i, tmp;
 718         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
 719         u32 rank = params->ch[channel].cap_info.rank;
 720
 721         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 722         writel(0x00003f7c, (&denali_pi[175]));
 723
 724         for (i = 0; i < rank; i++) {
 725                 select_per_cs_training_index(chan, i);
 726
 727                 /* PI_60 PI_WRLVL_EN:RW:8:2 */
 728                 clrsetbits_le32(&denali_pi[60], 0x3 << 8, 0x2 << 8);
 729
 730                 /* PI_59 PI_WRLVL_REQ:WR:8:1,PI_WRLVL_CS:RW:16:2 */
 731                 clrsetbits_le32(&denali_pi[59],
 732                                 (0x1 << 8) | (0x3 << 16),
 733                                 (0x1 << 8) | (i << 16));
 734
 735                 /* Waiting for training complete */
 736                 while (1) {
 737                         /* PI_174 PI_INT_STATUS:RD:8:18 */
 738                         tmp = readl(&denali_pi[174]) >> 8;
 739
 740                         /*
 741                          * check status obs, if error maybe can not
 742                          * get leveling done PHY_40/168/296/424
 743                          * phy_wrlvl_status_obs_X:0:13
 744                          */
 745                         obs_0 = readl(&denali_phy[40]);
 746                         obs_1 = readl(&denali_phy[168]);
 747                         obs_2 = readl(&denali_phy[296]);
 748                         obs_3 = readl(&denali_phy[424]);
 749                         if (((obs_0 >> 12) & 0x1) ||
 750                             ((obs_1 >> 12) & 0x1) ||
 751                             ((obs_2 >> 12) & 0x1) ||
 752                             ((obs_3 >> 12) & 0x1))
 753                                 obs_err = 1;
 754                         if ((((tmp >> 10) & 0x1) == 0x1) &&
 755                             (((tmp >> 13) & 0x1) == 0x1) &&
 756                             (((tmp >> 4) & 0x1) == 0x0) &&
 757                             obs_err == 0)
 758                                 break;
 759                         else if ((((tmp >> 4) & 0x1) == 0x1) ||
 760                                  (obs_err == 1))
 761                                 return -EIO;
 762                 }
 763
 764                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 765                 writel(0x00003f7c, (&denali_pi[175]));
 766         }
 767
 768         override_write_leveling_value(chan);
 769         clrbits_le32(&denali_pi[60], 0x3 << 8);
 770
 771         return 0;
 772 }
 773
 774 static int data_training_rg(const struct chan_info *chan, u32 channel,
 775                             const struct rk3399_sdram_params *params)
 776 {
 777         u32 *denali_pi = chan->pi->denali_pi;
 778         u32 *denali_phy = chan->publ->denali_phy;
 779         u32 i, tmp;
 780         u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0;
 781         u32 rank = params->ch[channel].cap_info.rank;
 782
 783         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 784         writel(0x00003f7c, (&denali_pi[175]));
 785
 786         for (i = 0; i < rank; i++) {
 787                 select_per_cs_training_index(chan, i);
 788
 789                 /* PI_80 PI_RDLVL_GATE_EN:RW:24:2 */
 790                 clrsetbits_le32(&denali_pi[80], 0x3 << 24, 0x2 << 24);
 791
 792                 /*
 793                  * PI_74 PI_RDLVL_GATE_REQ:WR:16:1
 794                  * PI_RDLVL_CS:RW:24:2
 795                  */
 796                 clrsetbits_le32(&denali_pi[74],
 797                                 (0x1 << 16) | (0x3 << 24),
 798                                 (0x1 << 16) | (i << 24));
 799
 800                 /* Waiting for training complete */
 801                 while (1) {
 802                         /* PI_174 PI_INT_STATUS:RD:8:18 */
 803                         tmp = readl(&denali_pi[174]) >> 8;
 804
 805                         /*
 806                          * check status obs
 807                          * PHY_43/171/299/427
 808                          *     PHY_GTLVL_STATUS_OBS_x:16:8
 809                          */
 810                         obs_0 = readl(&denali_phy[43]);
 811                         obs_1 = readl(&denali_phy[171]);
 812                         obs_2 = readl(&denali_phy[299]);
 813                         obs_3 = readl(&denali_phy[427]);
 814                         if (((obs_0 >> (16 + 6)) & 0x3) ||
 815                             ((obs_1 >> (16 + 6)) & 0x3) ||
 816                             ((obs_2 >> (16 + 6)) & 0x3) ||
 817                             ((obs_3 >> (16 + 6)) & 0x3))
 818                                 obs_err = 1;
 819                         if ((((tmp >> 9) & 0x1) == 0x1) &&
 820                             (((tmp >> 13) & 0x1) == 0x1) &&
 821                             (((tmp >> 3) & 0x1) == 0x0) &&
 822                             obs_err == 0)
 823                                 break;
 824                         else if ((((tmp >> 3) & 0x1) == 0x1) ||
 825                                  (obs_err == 1))
 826                                 return -EIO;
 827                 }
 828
 829                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 830                 writel(0x00003f7c, (&denali_pi[175]));
 831         }
 832
 833         clrbits_le32(&denali_pi[80], 0x3 << 24);
 834
 835         return 0;
 836 }
 837
 838 static int data_training_rl(const struct chan_info *chan, u32 channel,
 839                             const struct rk3399_sdram_params *params)
 840 {
 841         u32 *denali_pi = chan->pi->denali_pi;
 842         u32 i, tmp;
 843         u32 rank = params->ch[channel].cap_info.rank;
 844
 845         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 846         writel(0x00003f7c, (&denali_pi[175]));
 847
 848         for (i = 0; i < rank; i++) {
 849                 select_per_cs_training_index(chan, i);
 850
 851                 /* PI_80 PI_RDLVL_EN:RW:16:2 */
 852                 clrsetbits_le32(&denali_pi[80], 0x3 << 16, 0x2 << 16);
 853
 854                 /* PI_74 PI_RDLVL_REQ:WR:8:1,PI_RDLVL_CS:RW:24:2 */
 855                 clrsetbits_le32(&denali_pi[74],
 856                                 (0x1 << 8) | (0x3 << 24),
 857                                 (0x1 << 8) | (i << 24));
 858
 859                 /* Waiting for training complete */
 860                 while (1) {
 861                         /* PI_174 PI_INT_STATUS:RD:8:18 */
 862                         tmp = readl(&denali_pi[174]) >> 8;
 863
 864                         /*
 865                          * make sure status obs not report error bit
 866                          * PHY_46/174/302/430
 867                          *     phy_rdlvl_status_obs_X:16:8
 868                          */
 869                         if ((((tmp >> 8) & 0x1) == 0x1) &&
 870                             (((tmp >> 13) & 0x1) == 0x1) &&
 871                             (((tmp >> 2) & 0x1) == 0x0))
 872                                 break;
 873                         else if (((tmp >> 2) & 0x1) == 0x1)
 874                                 return -EIO;
 875                 }
 876
 877                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 878                 writel(0x00003f7c, (&denali_pi[175]));
 879         }
 880
 881         clrbits_le32(&denali_pi[80], 0x3 << 16);
 882
 883         return 0;
 884 }
 885
 886 static int data_training_wdql(const struct chan_info *chan, u32 channel,
 887                               const struct rk3399_sdram_params *params)
 888 {
 889         u32 *denali_pi = chan->pi->denali_pi;
 890         u32 i, tmp;
 891         u32 rank = params->ch[channel].cap_info.rank;
 892         u32 rank_mask;
 893
 894         /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 895         writel(0x00003f7c, (&denali_pi[175]));
 896
 897         rank_mask = (rank == 1) ? 0x1 : 0x3;
 898
 899         for (i = 0; i < 4; i++) {
 900                 if (!(rank_mask & (1 << i)))
 901                         continue;
 902
 903                 select_per_cs_training_index(chan, i);
 904
 905                 /*
 906                  * disable PI_WDQLVL_VREF_EN before wdq leveling?
 907                  * PI_181 PI_WDQLVL_VREF_EN:RW:8:1
 908                  */
 909                 clrbits_le32(&denali_pi[181], 0x1 << 8);
 910
 911                 /* PI_124 PI_WDQLVL_EN:RW:16:2 */
 912                 clrsetbits_le32(&denali_pi[124], 0x3 << 16, 0x2 << 16);
 913
 914                 /* PI_121 PI_WDQLVL_REQ:WR:8:1,PI_WDQLVL_CS:RW:16:2 */
 915                 clrsetbits_le32(&denali_pi[121],
 916                                 (0x1 << 8) | (0x3 << 16),
 917                                 (0x1 << 8) | (i << 16));
 918
 919                 /* Waiting for training complete */
 920                 while (1) {
 921                         /* PI_174 PI_INT_STATUS:RD:8:18 */
 922                         tmp = readl(&denali_pi[174]) >> 8;
 923                         if ((((tmp >> 12) & 0x1) == 0x1) &&
 924                             (((tmp >> 13) & 0x1) == 0x1) &&
 925                             (((tmp >> 6) & 0x1) == 0x0))
 926                                 break;
 927                         else if (((tmp >> 6) & 0x1) == 0x1)
 928                                 return -EIO;
 929                 }
 930
 931                 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */
 932                 writel(0x00003f7c, (&denali_pi[175]));
 933         }
 934
 935         clrbits_le32(&denali_pi[124], 0x3 << 16);
 936
 937         return 0;
 938 }
 939
 940 static int data_training(const struct chan_info *chan, u32 channel,
 941                          const struct rk3399_sdram_params *params,
 942                          u32 training_flag)
 943 {
 944         u32 *denali_phy = chan->publ->denali_phy;
 945         int ret;
 946
 947         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
 948         setbits_le32(&denali_phy[927], (1 << 22));
 949
 950         if (training_flag == PI_FULL_TRAINING) {
 951                 if (params->base.dramtype == LPDDR4) {
 952                         training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING |
 953                                         PI_READ_GATE_TRAINING |
 954                                         PI_READ_LEVELING | PI_WDQ_LEVELING;
 955                 } else if (params->base.dramtype == LPDDR3) {
 956                         training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING |
 957                                         PI_READ_GATE_TRAINING;
 958                 } else if (params->base.dramtype == DDR3) {
 959                         training_flag = PI_WRITE_LEVELING |
 960                                         PI_READ_GATE_TRAINING |
 961                                         PI_READ_LEVELING;
 962                 }
 963         }
 964
 965         /* ca training(LPDDR4,LPDDR3 support) */
 966         if ((training_flag & PI_CA_TRAINING) == PI_CA_TRAINING) {
 967                 ret = data_training_ca(chan, channel, params);
 968                 if (ret < 0) {
 969                         debug("%s: data training ca failed\n", __func__);
 970                         return ret;
 971                 }
 972         }
 973
 974         /* write leveling(LPDDR4,LPDDR3,DDR3 support) */
 975         if ((training_flag & PI_WRITE_LEVELING) == PI_WRITE_LEVELING) {
 976                 ret = data_training_wl(chan, channel, params);
 977                 if (ret < 0) {
 978                         debug("%s: data training wl failed\n", __func__);
 979                         return ret;
 980                 }
 981         }
 982
 983         /* read gate training(LPDDR4,LPDDR3,DDR3 support) */
 984         if ((training_flag & PI_READ_GATE_TRAINING) == PI_READ_GATE_TRAINING) {
 985                 ret = data_training_rg(chan, channel, params);
 986                 if (ret < 0) {
 987                         debug("%s: data training rg failed\n", __func__);
 988                         return ret;
 989                 }
 990         }
 991
 992         /* read leveling(LPDDR4,LPDDR3,DDR3 support) */
 993         if ((training_flag & PI_READ_LEVELING) == PI_READ_LEVELING) {
 994                 ret = data_training_rl(chan, channel, params);
 995                 if (ret < 0) {
 996                         debug("%s: data training rl failed\n", __func__);
 997                         return ret;
 998                 }
 999         }
1000
1001         /* wdq leveling(LPDDR4 support) */
1002         if ((training_flag & PI_WDQ_LEVELING) == PI_WDQ_LEVELING) {
1003                 ret = data_training_wdql(chan, channel, params);
1004                 if (ret < 0) {
1005                         debug("%s: data training wdql failed\n", __func__);
1006                         return ret;
1007                 }
1008         }
1009
1010         /* PHY_927 PHY_PAD_DQS_DRIVE  RPULL offset_22 */
1011         clrbits_le32(&denali_phy[927], (1 << 22));
1012
1013         return 0;
1014 }
1015
1016 static void set_ddrconfig(const struct chan_info *chan,
1017                           const struct rk3399_sdram_params *params,
1018                           unsigned char channel, u32 ddrconfig)
1019 {
1020         /* only need to set ddrconfig */
1021         struct rk3399_msch_regs *ddr_msch_regs = chan->msch;
1022         unsigned int cs0_cap = 0;
1023         unsigned int cs1_cap = 0;
1024
1025         cs0_cap = (1 << (params->ch[channel].cap_info.cs0_row
1026                         + params->ch[channel].cap_info.col
1027                         + params->ch[channel].cap_info.bk
1028                         + params->ch[channel].cap_info.bw - 20));
1029         if (params->ch[channel].cap_info.rank > 1)
1030                 cs1_cap = cs0_cap >> (params->ch[channel].cap_info.cs0_row
1031                                 - params->ch[channel].cap_info.cs1_row);
1032         if (params->ch[channel].cap_info.row_3_4) {
1033                 cs0_cap = cs0_cap * 3 / 4;
1034                 cs1_cap = cs1_cap * 3 / 4;
1035         }
1036
1037         writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf);
1038         writel(((cs0_cap / 32) & 0xff) | (((cs1_cap / 32) & 0xff) << 8),
1039                &ddr_msch_regs->ddrsize);
1040 }
1041
1042 static void dram_all_config(struct dram_info *dram,
1043                             const struct rk3399_sdram_params *params)
1044 {
1045         u32 sys_reg = 0;
1046         unsigned int channel, idx;
1047
1048         sys_reg |= params->base.dramtype << SYS_REG_DDRTYPE_SHIFT;
1049         sys_reg |= (params->base.num_channels - 1) << SYS_REG_NUM_CH_SHIFT;
1050
1051         for (channel = 0, idx = 0;
1052              (idx < params->base.num_channels) && (channel < 2);
1053              channel++) {
1054                 const struct rk3399_sdram_channel *info = &params->ch[channel];
1055                 struct rk3399_msch_regs *ddr_msch_regs;
1056                 const struct rk3399_msch_timings *noc_timing;
1057
1058                 if (params->ch[channel].cap_info.col == 0)
1059                         continue;
1060                 idx++;
1061                 sys_reg |= info->cap_info.row_3_4 <<
1062                            SYS_REG_ROW_3_4_SHIFT(channel);
1063                 sys_reg |= 1 << SYS_REG_CHINFO_SHIFT(channel);
1064                 sys_reg |= (info->cap_info.rank - 1) <<
1065                            SYS_REG_RANK_SHIFT(channel);
1066                 sys_reg |= (info->cap_info.col - 9) <<
1067                            SYS_REG_COL_SHIFT(channel);
1068                 sys_reg |= info->cap_info.bk == 3 ? 0 : 1 <<
1069                            SYS_REG_BK_SHIFT(channel);
1070                 sys_reg |= (info->cap_info.cs0_row - 13) <<
1071                             SYS_REG_CS0_ROW_SHIFT(channel);
1072                 sys_reg |= (info->cap_info.cs1_row - 13) <<
1073                             SYS_REG_CS1_ROW_SHIFT(channel);
1074                 sys_reg |= (2 >> info->cap_info.bw) <<
1075                            SYS_REG_BW_SHIFT(channel);
1076                 sys_reg |= (2 >> info->cap_info.dbw) <<
1077                            SYS_REG_DBW_SHIFT(channel);
1078
1079                 ddr_msch_regs = dram->chan[channel].msch;
1080                 noc_timing = &params->ch[channel].noc_timings;
1081                 writel(noc_timing->ddrtiminga0,
1082                        &ddr_msch_regs->ddrtiminga0);
1083                 writel(noc_timing->ddrtimingb0,
1084                        &ddr_msch_regs->ddrtimingb0);
1085                 writel(noc_timing->ddrtimingc0,
1086                        &ddr_msch_regs->ddrtimingc0);
1087                 writel(noc_timing->devtodev0,
1088                        &ddr_msch_regs->devtodev0);
1089                 writel(noc_timing->ddrmode,
1090                        &ddr_msch_regs->ddrmode);
1091
1092                 /* rank 1 memory clock disable (dfi_dram_clk_disable = 1) */
1093                 if (params->ch[channel].cap_info.rank == 1)
1094                         setbits_le32(&dram->chan[channel].pctl->denali_ctl[276],
1095                                      1 << 17);
1096         }
1097
1098         writel(sys_reg, &dram->pmugrf->os_reg2);
1099         rk_clrsetreg(&dram->pmusgrf->soc_con4, 0x1f << 10,
1100                      params->base.stride << 10);
1101
1102         /* reboot hold register set */
1103         writel(PRESET_SGRF_HOLD(0) | PRESET_GPIO0_HOLD(1) |
1104                 PRESET_GPIO1_HOLD(1),
1105                 &dram->pmucru->pmucru_rstnhold_con[1]);
1106         clrsetbits_le32(&dram->cru->glb_rst_con, 0x3, 0x3);
1107 }
1108
1109 static int switch_to_phy_index1(struct dram_info *dram,
1110                                 const struct rk3399_sdram_params *params)
1111 {
1112         u32 channel;
1113         u32 *denali_phy;
1114         u32 ch_count = params->base.num_channels;
1115         int ret;
1116         int i = 0;
1117
1118         writel(RK_CLRSETBITS(0x03 << 4 | 1 << 2 | 1,
1119                              1 << 4 | 1 << 2 | 1),
1120                         &dram->cic->cic_ctrl0);
1121         while (!(readl(&dram->cic->cic_status0) & (1 << 2))) {
1122                 mdelay(10);
1123                 i++;
1124                 if (i > 10) {
1125                         debug("index1 frequency change overtime\n");
1126                         return -ETIME;
1127                 }
1128         }
1129
1130         i = 0;
1131         writel(RK_CLRSETBITS(1 << 1, 1 << 1), &dram->cic->cic_ctrl0);
1132         while (!(readl(&dram->cic->cic_status0) & (1 << 0))) {
1133                 mdelay(10);
1134                 i++;
1135                 if (i > 10) {
1136                         debug("index1 frequency done overtime\n");
1137                         return -ETIME;
1138                 }
1139         }
1140
1141         for (channel = 0; channel < ch_count; channel++) {
1142                 denali_phy = dram->chan[channel].publ->denali_phy;
1143                 clrsetbits_le32(&denali_phy[896], (0x3 << 8) | 1, 1 << 8);
1144                 ret = data_training(&dram->chan[channel], channel,
1145                                     params, PI_FULL_TRAINING);
1146                 if (ret < 0) {
1147                         debug("index1 training failed\n");
1148                         return ret;
1149                 }
1150         }
1151
1152         return 0;
1153 }
1154
1155 static int sdram_init(struct dram_info *dram,
1156                       const struct rk3399_sdram_params *params)
1157 {
1158         unsigned char dramtype = params->base.dramtype;
1159         unsigned int ddr_freq = params->base.ddr_freq;
1160         struct rk3399_cru *cru = dram->cru;
1161         int channel;
1162         int ret;
1163
1164         debug("Starting SDRAM initialization...\n");
1165
1166         if ((dramtype == DDR3 && ddr_freq > 933) ||
1167             (dramtype == LPDDR3 && ddr_freq > 933) ||
1168             (dramtype == LPDDR4 && ddr_freq > 800)) {
1169                 debug("SDRAM frequency is to high!");
1170                 return -E2BIG;
1171         }
1172
1173         for (channel = 0; channel < 2; channel++) {
1174                 const struct chan_info *chan = &dram->chan[channel];
1175                 struct rk3399_ddr_publ_regs *publ = chan->publ;
1176
1177                 phy_pctrl_reset(cru, channel);
1178                 phy_dll_bypass_set(publ, ddr_freq);
1179
1180                 if (channel >= params->base.num_channels)
1181                         continue;
1182
1183                 ret = pctl_cfg(dram, chan, channel, params);
1184                 if (ret < 0) {
1185                         printf("%s: pctl config failed\n", __func__);
1186                         return ret;
1187                 }
1188
1189                 /* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */
1190                 if (dramtype == LPDDR3)
1191                         udelay(10);
1192
1193                 if (data_training(chan, channel, params, PI_FULL_TRAINING)) {
1194                         printf("%s: data training failed\n", __func__);
1195                         return -EIO;
1196                 }
1197
1198                 set_ddrconfig(chan, params, channel,
1199                               params->ch[channel].cap_info.ddrconfig);
1200         }
1201         dram_all_config(dram, params);
1202         switch_to_phy_index1(dram, params);
1203
1204         debug("Finish SDRAM initialization...\n");
1205         return 0;
1206 }
1207
1208 static int rk3399_dmc_ofdata_to_platdata(struct udevice *dev)
1209 {
1210 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
1211         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
1212         int ret;
1213
1214         ret = dev_read_u32_array(dev, "rockchip,sdram-params",
1215                                  (u32 *)&plat->sdram_params,
1216                                  sizeof(plat->sdram_params) / sizeof(u32));
1217         if (ret) {
1218                 printf("%s: Cannot read rockchip,sdram-params %d\n",
1219                        __func__, ret);
1220                 return ret;
1221         }
1222         ret = regmap_init_mem(dev_ofnode(dev), &plat->map);
1223         if (ret)
1224                 printf("%s: regmap failed %d\n", __func__, ret);
1225
1226 #endif
1227         return 0;
1228 }
1229
1230 #if CONFIG_IS_ENABLED(OF_PLATDATA)
1231 static int conv_of_platdata(struct udevice *dev)
1232 {
1233         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
1234         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
1235         int ret;
1236
1237         ret = regmap_init_mem_platdata(dev, dtplat->reg,
1238                                        ARRAY_SIZE(dtplat->reg) / 2,
1239                                        &plat->map);
1240         if (ret)
1241                 return ret;
1242
1243         return 0;
1244 }
1245 #endif
1246
1247 static int rk3399_dmc_init(struct udevice *dev)
1248 {
1249         struct dram_info *priv = dev_get_priv(dev);
1250         struct rockchip_dmc_plat *plat = dev_get_platdata(dev);
1251         int ret;
1252 #if !CONFIG_IS_ENABLED(OF_PLATDATA)
1253         struct rk3399_sdram_params *params = &plat->sdram_params;
1254 #else
1255         struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat;
1256         struct rk3399_sdram_params *params =
1257                                         (void *)dtplat->rockchip_sdram_params;
1258
1259         ret = conv_of_platdata(dev);
1260         if (ret)
1261                 return ret;
1262 #endif
1263
1264         priv->cic = syscon_get_first_range(ROCKCHIP_SYSCON_CIC);
1265         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
1266         priv->pmusgrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUSGRF);
1267         priv->pmucru = rockchip_get_pmucru();
1268         priv->cru = rockchip_get_cru();
1269         priv->chan[0].pctl = regmap_get_range(plat->map, 0);
1270         priv->chan[0].pi = regmap_get_range(plat->map, 1);
1271         priv->chan[0].publ = regmap_get_range(plat->map, 2);
1272         priv->chan[0].msch = regmap_get_range(plat->map, 3);
1273         priv->chan[1].pctl = regmap_get_range(plat->map, 4);
1274         priv->chan[1].pi = regmap_get_range(plat->map, 5);
1275         priv->chan[1].publ = regmap_get_range(plat->map, 6);
1276         priv->chan[1].msch = regmap_get_range(plat->map, 7);
1277
1278         debug("con reg %p %p %p %p %p %p %p %p\n",
1279               priv->chan[0].pctl, priv->chan[0].pi,
1280               priv->chan[0].publ, priv->chan[0].msch,
1281               priv->chan[1].pctl, priv->chan[1].pi,
1282               priv->chan[1].publ, priv->chan[1].msch);
1283         debug("cru %p, cic %p, grf %p, sgrf %p, pmucru %p\n", priv->cru,
1284               priv->cic, priv->pmugrf, priv->pmusgrf, priv->pmucru);
1285
1286 #if CONFIG_IS_ENABLED(OF_PLATDATA)
1287         ret = clk_get_by_index_platdata(dev, 0, dtplat->clocks, &priv->ddr_clk);
1288 #else
1289         ret = clk_get_by_index(dev, 0, &priv->ddr_clk);
1290 #endif
1291         if (ret) {
1292                 printf("%s clk get failed %d\n", __func__, ret);
1293                 return ret;
1294         }
1295
1296         ret = clk_set_rate(&priv->ddr_clk, params->base.ddr_freq * MHz);
1297         if (ret < 0) {
1298                 printf("%s clk set failed %d\n", __func__, ret);
1299                 return ret;
1300         }
1301
1302         ret = sdram_init(priv, params);
1303         if (ret < 0) {
1304                 printf("%s DRAM init failed %d\n", __func__, ret);
1305                 return ret;
1306         }
1307
1308         return 0;
1309 }
1310 #endif
1311
1312 static int rk3399_dmc_probe(struct udevice *dev)
1313 {
1314 #if defined(CONFIG_TPL_BUILD) || \
1315         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
1316         if (rk3399_dmc_init(dev))
1317                 return 0;
1318 #else
1319         struct dram_info *priv = dev_get_priv(dev);
1320
1321         priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF);
1322         debug("%s: pmugrf = %p\n", __func__, priv->pmugrf);
1323         priv->info.base = CONFIG_SYS_SDRAM_BASE;
1324         priv->info.size =
1325                 rockchip_sdram_size((phys_addr_t)&priv->pmugrf->os_reg2);
1326 #endif
1327         return 0;
1328 }
1329
1330 static int rk3399_dmc_get_info(struct udevice *dev, struct ram_info *info)
1331 {
1332         struct dram_info *priv = dev_get_priv(dev);
1333
1334         *info = priv->info;
1335
1336         return 0;
1337 }
1338
1339 static struct ram_ops rk3399_dmc_ops = {
1340         .get_info = rk3399_dmc_get_info,
1341 };
1342
1343 static const struct udevice_id rk3399_dmc_ids[] = {
1344         { .compatible = "rockchip,rk3399-dmc" },
1345         { }
1346 };
1347
1348 U_BOOT_DRIVER(dmc_rk3399) = {
1349         .name = "rockchip_rk3399_dmc",
1350         .id = UCLASS_RAM,
1351         .of_match = rk3399_dmc_ids,
1352         .ops = &rk3399_dmc_ops,
1353 #if defined(CONFIG_TPL_BUILD) || \
1354         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
1355         .ofdata_to_platdata = rk3399_dmc_ofdata_to_platdata,
1356 #endif
1357         .probe = rk3399_dmc_probe,
1358         .priv_auto_alloc_size = sizeof(struct dram_info),
1359 #if defined(CONFIG_TPL_BUILD) || \
1360         (!defined(CONFIG_TPL) && defined(CONFIG_SPL_BUILD))
1361         .platdata_auto_alloc_size = sizeof(struct rockchip_dmc_plat),
1362 #endif
1363 };