drivers/ram/stm32mp1/stm32mp1_tuning.c

   1 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
   2 /*
   3  * Copyright (C) 2019, STMicroelectronics - All Rights Reserved
   4  */
   5 #include <common.h>
   6 #include <console.h>
   7 #include <clk.h>
   8 #include <log.h>
   9 #include <ram.h>
  10 #include <rand.h>
  11 #include <reset.h>
  12 #include <asm/io.h>
  13 #include <linux/bitops.h>
  14 #include <linux/iopoll.h>
  15
  16 #include "stm32mp1_ddr_regs.h"
  17 #include "stm32mp1_ddr.h"
  18 #include "stm32mp1_tests.h"
  19
  20 #define MAX_DQS_PHASE_IDX _144deg
  21 #define MAX_DQS_UNIT_IDX 7
  22 #define MAX_GSL_IDX 5
  23 #define MAX_GPS_IDX 3
  24
  25 /* Number of bytes used in this SW. ( min 1--> max 4). */
  26 #define NUM_BYTES 4
  27
  28 enum dqs_phase_enum {
  29         _36deg = 0,
  30         _54deg = 1,
  31         _72deg = 2,
  32         _90deg = 3,
  33         _108deg = 4,
  34         _126deg = 5,
  35         _144deg = 6
  36 };
  37
  38 /* BIST Result struct */
  39 struct BIST_result {
  40         /* Overall test result:
  41          * 0 Fail (any bit failed) ,
  42          * 1 Success (All bits success)
  43          */
  44         bool test_result;
  45         /* 1: true, all fail /  0: False, not all bits fail */
  46         bool all_bits_fail;
  47         bool bit_i_test_result[8];  /* 0 fail / 1 success */
  48 };
  49
  50 /* a struct that defines tuning parameters of a byte. */
  51 struct tuning_position {
  52         u8 phase; /* DQS phase */
  53         u8 unit; /* DQS unit delay */
  54         u32 bits_delay; /* Bits deskew in this byte */
  55 };
  56
  57 /* 36deg, 54deg, 72deg, 90deg, 108deg, 126deg, 144deg */
  58 const u8 dx_dll_phase[7] = {3, 2, 1, 0, 14, 13, 12};
  59
  60 static u8 BIST_error_max = 1;
  61 static u32 BIST_seed = 0x1234ABCD;
  62
  63 static u8 get_nb_bytes(struct stm32mp1_ddrctl *ctl)
  64 {
  65         u32 data_bus = readl(&ctl->mstr) & DDRCTRL_MSTR_DATA_BUS_WIDTH_MASK;
  66         u8 nb_bytes = NUM_BYTES;
  67
  68         switch (data_bus) {
  69         case DDRCTRL_MSTR_DATA_BUS_WIDTH_HALF:
  70                 nb_bytes /= 2;
  71                 break;
  72         case DDRCTRL_MSTR_DATA_BUS_WIDTH_QUARTER:
  73                 nb_bytes /= 4;
  74                 break;
  75         default:
  76                 break;
  77         }
  78
  79         return nb_bytes;
  80 }
  81
  82 static u8 get_nb_bank(struct stm32mp1_ddrctl *ctl)
  83 {
  84         /* Count bank address bits */
  85         u8 bits = 0;
  86         u32 reg, val;
  87
  88         reg = readl(&ctl->addrmap1);
  89         /* addrmap1.addrmap_bank_b1 */
  90         val = (reg & GENMASK(5, 0)) >> 0;
  91         if (val <= 31)
  92                 bits++;
  93         /* addrmap1.addrmap_bank_b2 */
  94         val = (reg & GENMASK(13, 8)) >> 8;
  95         if (val <= 31)
  96                 bits++;
  97         /* addrmap1.addrmap_bank_b3 */
  98         val = (reg & GENMASK(21, 16)) >> 16;
  99         if (val <= 31)
 100                 bits++;
 101
 102         return bits;
 103 }
 104
 105 static u8 get_nb_col(struct stm32mp1_ddrctl *ctl)
 106 {
 107         u8 bits;
 108         u32 reg, val;
 109
 110         /* Count column address bits, start at 2 for b0 and b1 (fixed) */
 111         bits = 2;
 112
 113         reg = readl(&ctl->addrmap2);
 114         /* addrmap2.addrmap_col_b2 */
 115         val = (reg & GENMASK(3, 0)) >> 0;
 116         if (val <= 7)
 117                 bits++;
 118         /* addrmap2.addrmap_col_b3 */
 119         val = (reg & GENMASK(11, 8)) >> 8;
 120         if (val <= 7)
 121                 bits++;
 122         /* addrmap2.addrmap_col_b4 */
 123         val = (reg & GENMASK(19, 16)) >> 16;
 124         if (val <= 7)
 125                 bits++;
 126         /* addrmap2.addrmap_col_b5 */
 127         val = (reg & GENMASK(27, 24)) >> 24;
 128         if (val <= 7)
 129                 bits++;
 130
 131         reg = readl(&ctl->addrmap3);
 132         /* addrmap3.addrmap_col_b6 */
 133         val = (reg & GENMASK(3, 0)) >> 0;
 134         if (val <= 7)
 135                 bits++;
 136         /* addrmap3.addrmap_col_b7 */
 137         val = (reg & GENMASK(11, 8)) >> 8;
 138         if (val <= 7)
 139                 bits++;
 140         /* addrmap3.addrmap_col_b8 */
 141         val = (reg & GENMASK(19, 16)) >> 16;
 142         if (val <= 7)
 143                 bits++;
 144         /* addrmap3.addrmap_col_b9 */
 145         val = (reg & GENMASK(27, 24)) >> 24;
 146         if (val <= 7)
 147                 bits++;
 148
 149         reg = readl(&ctl->addrmap4);
 150         /* addrmap4.addrmap_col_b10 */
 151         val = (reg & GENMASK(3, 0)) >> 0;
 152         if (val <= 7)
 153                 bits++;
 154         /* addrmap4.addrmap_col_b11 */
 155         val = (reg & GENMASK(11, 8)) >> 8;
 156         if (val <= 7)
 157                 bits++;
 158
 159         return bits;
 160 }
 161
 162 static u8 get_nb_row(struct stm32mp1_ddrctl *ctl)
 163 {
 164         /* Count row address bits */
 165         u8 bits = 0;
 166         u32 reg, val;
 167
 168         reg = readl(&ctl->addrmap5);
 169         /* addrmap5.addrmap_row_b0 */
 170         val = (reg & GENMASK(3, 0)) >> 0;
 171         if (val <= 11)
 172                 bits++;
 173         /* addrmap5.addrmap_row_b1 */
 174         val = (reg & GENMASK(11, 8)) >> 8;
 175         if (val <= 11)
 176                 bits++;
 177         /* addrmap5.addrmap_row_b2_10 */
 178         val = (reg & GENMASK(19, 16)) >> 16;
 179         if (val <= 11)
 180                 bits += 9;
 181         else
 182                 printf("warning: addrmap5.addrmap_row_b2_10 not supported\n");
 183         /* addrmap5.addrmap_row_b11 */
 184         val = (reg & GENMASK(27, 24)) >> 24;
 185         if (val <= 11)
 186                 bits++;
 187
 188         reg = readl(&ctl->addrmap6);
 189         /* addrmap6.addrmap_row_b12 */
 190         val = (reg & GENMASK(3, 0)) >> 0;
 191         if (val <= 7)
 192                 bits++;
 193         /* addrmap6.addrmap_row_b13 */
 194         val = (reg & GENMASK(11, 8)) >> 8;
 195         if (val <= 7)
 196                 bits++;
 197         /* addrmap6.addrmap_row_b14 */
 198         val = (reg & GENMASK(19, 16)) >> 16;
 199         if (val <= 7)
 200                 bits++;
 201         /* addrmap6.addrmap_row_b15 */
 202         val = (reg & GENMASK(27, 24)) >> 24;
 203         if (val <= 7)
 204                 bits++;
 205
 206         return bits;
 207 }
 208
 209 static void itm_soft_reset(struct stm32mp1_ddrphy *phy)
 210 {
 211         stm32mp1_ddrphy_init(phy, DDRPHYC_PIR_ITMSRST);
 212 }
 213
 214 /* Read DQ unit delay register and provides the retrieved value for DQS
 215  * We are assuming that we have the same delay when clocking
 216  * by DQS and when clocking by DQSN
 217  */
 218 static u8 DQ_unit_index(struct stm32mp1_ddrphy *phy, u8 byte, u8 bit)
 219 {
 220         u32 index;
 221         u32 addr = DXNDQTR(phy, byte);
 222
 223         /* We are assuming that we have the same delay when clocking by DQS
 224          * and when clocking by DQSN : use only the low bits
 225          */
 226         index = (readl(addr) >> DDRPHYC_DXNDQTR_DQDLY_SHIFT(bit))
 227                 & DDRPHYC_DXNDQTR_DQDLY_LOW_MASK;
 228
 229         pr_debug("%s: [%x]: %x => DQ unit index = %x\n",
 230                  __func__, addr, readl(addr), index);
 231
 232         return index;
 233 }
 234
 235 /* Sets the DQS phase delay for a byte lane.
 236  *phase delay is specified by giving the index of the desired delay
 237  * in the dx_dll_phase array.
 238  */
 239 static void DQS_phase_delay(struct stm32mp1_ddrphy *phy, u8 byte, u8 phase_idx)
 240 {
 241         u8 sdphase_val = 0;
 242
 243         /*      Write DXNDLLCR.SDPHASE = dx_dll_phase(phase_index); */
 244         sdphase_val = dx_dll_phase[phase_idx];
 245         clrsetbits_le32(DXNDLLCR(phy, byte),
 246                         DDRPHYC_DXNDLLCR_SDPHASE_MASK,
 247                         sdphase_val << DDRPHYC_DXNDLLCR_SDPHASE_SHIFT);
 248 }
 249
 250 /* Sets the DQS unit delay for a byte lane.
 251  * unit delay is specified by giving the index of the desired delay
 252  * for dgsdly and dqsndly (same value).
 253  */
 254 static void DQS_unit_delay(struct stm32mp1_ddrphy *phy,
 255                            u8 byte, u8 unit_dly_idx)
 256 {
 257         /* Write the same value in DXNDQSTR.DQSDLY and DXNDQSTR.DQSNDLY */
 258         clrsetbits_le32(DXNDQSTR(phy, byte),
 259                         DDRPHYC_DXNDQSTR_DQSDLY_MASK |
 260                         DDRPHYC_DXNDQSTR_DQSNDLY_MASK,
 261                         (unit_dly_idx << DDRPHYC_DXNDQSTR_DQSDLY_SHIFT) |
 262                         (unit_dly_idx << DDRPHYC_DXNDQSTR_DQSNDLY_SHIFT));
 263
 264         /* After changing this value, an ITM soft reset (PIR.ITMSRST=1,
 265          * plus PIR.INIT=1) must be issued.
 266          */
 267         stm32mp1_ddrphy_init(phy, DDRPHYC_PIR_ITMSRST);
 268 }
 269
 270 /* Sets the DQ unit delay for a bit line in particular byte lane.
 271  * unit delay is specified by giving the desired delay
 272  */
 273 static void set_DQ_unit_delay(struct stm32mp1_ddrphy *phy,
 274                               u8 byte, u8 bit,
 275                               u8 dq_delay_index)
 276 {
 277         u8 dq_bit_delay_val = dq_delay_index | (dq_delay_index << 2);
 278
 279         /* same value on delay for clock DQ an DQS_b */
 280         clrsetbits_le32(DXNDQTR(phy, byte),
 281                         DDRPHYC_DXNDQTR_DQDLY_MASK
 282                         << DDRPHYC_DXNDQTR_DQDLY_SHIFT(bit),
 283                         dq_bit_delay_val << DDRPHYC_DXNDQTR_DQDLY_SHIFT(bit));
 284 }
 285
 286 static void set_r0dgsl_delay(struct stm32mp1_ddrphy *phy,
 287                              u8 byte, u8 r0dgsl_idx)
 288 {
 289         clrsetbits_le32(DXNDQSTR(phy, byte),
 290                         DDRPHYC_DXNDQSTR_R0DGSL_MASK,
 291                         r0dgsl_idx << DDRPHYC_DXNDQSTR_R0DGSL_SHIFT);
 292 }
 293
 294 static void set_r0dgps_delay(struct stm32mp1_ddrphy *phy,
 295                              u8 byte, u8 r0dgps_idx)
 296 {
 297         clrsetbits_le32(DXNDQSTR(phy, byte),
 298                         DDRPHYC_DXNDQSTR_R0DGPS_MASK,
 299                         r0dgps_idx << DDRPHYC_DXNDQSTR_R0DGPS_SHIFT);
 300 }
 301
 302 /* Basic BIST configuration for data lane tests. */
 303 static void config_BIST(struct stm32mp1_ddrctl *ctl,
 304                         struct stm32mp1_ddrphy *phy)
 305 {
 306         u8 nb_bank = get_nb_bank(ctl);
 307         u8 nb_row = get_nb_row(ctl);
 308         u8 nb_col = get_nb_col(ctl);
 309
 310         /* Selects the SDRAM bank address to be used during BIST. */
 311         u32 bbank = 0;
 312         /* Selects the SDRAM row address to be used during BIST. */
 313         u32 brow = 0;
 314         /* Selects the SDRAM column address to be used during BIST. */
 315         u32 bcol = 0;
 316         /* Selects the value by which the SDRAM address is incremented
 317          * for each write/read access.
 318          */
 319         u32 bainc = 0x00000008;
 320         /* Specifies the maximum SDRAM rank to be used during BIST.
 321          * The default value is set to maximum ranks minus 1.
 322          * must be 0 with single rank
 323          */
 324         u32 bmrank = 0;
 325         /* Selects the SDRAM rank to be used during BIST.
 326          * must be 0 with single rank
 327          */
 328         u32 brank = 0;
 329
 330         /* Specifies the maximum SDRAM bank address to be used during
 331          * BIST before the address & increments to the next rank.
 332          */
 333         u32 bmbank = (1 << nb_bank) - 1;
 334         /* Specifies the maximum SDRAM row address to be used during
 335          * BIST before the address & increments to the next bank.
 336          */
 337         u32 bmrow = (1 << nb_row) - 1;
 338         /* Specifies the maximum SDRAM column address to be used during
 339          * BIST before the address & increments to the next row.
 340          */
 341         u32 bmcol = (1 << nb_col) - 1;
 342
 343         u32 bmode_conf = 0x00000001;  /* DRam mode */
 344         u32 bdxen_conf = 0x00000001;  /* BIST on Data byte */
 345         u32 bdpat_conf = 0x00000002;  /* Select LFSR pattern */
 346
 347         /*Setup BIST for DRAM mode,  and LFSR-random data pattern.*/
 348         /*Write BISTRR.BMODE = 1?b1;*/
 349         /*Write BISTRR.BDXEN = 1?b1;*/
 350         /*Write BISTRR.BDPAT = 2?b10;*/
 351
 352         /* reset BIST */
 353         writel(0x3, &phy->bistrr);
 354
 355         writel((bmode_conf << 3) | (bdxen_conf << 14) | (bdpat_conf << 17),
 356                &phy->bistrr);
 357
 358         /*Setup BIST Word Count*/
 359         /*Write BISTWCR.BWCNT = 16?b0008;*/
 360         writel(0x00000200, &phy->bistwcr); /* A multiple of BL/2 */
 361
 362         writel(bcol | (brow << 12) | (bbank << 28), &phy->bistar0);
 363         writel(brank | (bmrank << 2) | (bainc << 4), &phy->bistar1);
 364         writel(bmcol | (bmrow << 12) | (bmbank << 28), &phy->bistar2);
 365 }
 366
 367 /* Select the Byte lane to be tested by BIST. */
 368 static void BIST_datx8_sel(struct stm32mp1_ddrphy *phy, u8 datx8)
 369 {
 370         clrsetbits_le32(&phy->bistrr,
 371                         DDRPHYC_BISTRR_BDXSEL_MASK,
 372                         datx8 << DDRPHYC_BISTRR_BDXSEL_SHIFT);
 373
 374         /*(For example, selecting Byte Lane 3, BISTRR.BDXSEL = 4?b0011)*/
 375         /* Write BISTRR.BDXSEL = datx8; */
 376 }
 377
 378 /* Perform BIST Write_Read test on a byte lane and return test result. */
 379 static void BIST_test(struct stm32mp1_ddrphy *phy, u8 byte,
 380                       struct BIST_result *bist)
 381 {
 382         bool result = true; /* BIST_SUCCESS */
 383         u32 cnt = 0;
 384         u32 error = 0;
 385         u32 val;
 386         int ret;
 387
 388         bist->test_result = true;
 389
 390 run:
 391         itm_soft_reset(phy);
 392
 393         /*Perform BIST Reset*/
 394         /* Write BISTRR.BINST = 3?b011; */
 395         clrsetbits_le32(&phy->bistrr,
 396                         0x00000007,
 397                         0x00000003);
 398
 399         /*Re-seed LFSR*/
 400         /* Write BISTLSR.SEED = 32'h1234ABCD; */
 401         if (BIST_seed)
 402                 writel(BIST_seed, &phy->bistlsr);
 403         else
 404                 writel(rand(), &phy->bistlsr);
 405
 406         /* some delay to reset BIST */
 407         udelay(10);
 408
 409         /*Perform BIST Run*/
 410         clrsetbits_le32(&phy->bistrr,
 411                         0x00000007,
 412                         0x00000001);
 413         /* Write BISTRR.BINST = 3?b001; */
 414
 415         /* poll on BISTGSR.BDONE and wait max 1000 us */
 416         ret = readl_poll_timeout(&phy->bistgsr, val,
 417                                  val & DDRPHYC_BISTGSR_BDDONE, 1000);
 418
 419         if (ret < 0) {
 420                 printf("warning: BIST timeout\n");
 421                 result = false; /* BIST_FAIL; */
 422                 /*Perform BIST Stop */
 423                 clrsetbits_le32(&phy->bistrr, 0x00000007, 0x00000002);
 424         } else {
 425                 /*Check if received correct number of words*/
 426                 /* if (Read BISTWCSR.DXWCNT = Read BISTWCR.BWCNT) */
 427                 if (((readl(&phy->bistwcsr)) >> DDRPHYC_BISTWCSR_DXWCNT_SHIFT)
 428                     == readl(&phy->bistwcr)) {
 429                         /*Determine if there is a data comparison error*/
 430                         /* if (Read BISTGSR.BDXERR = 1?b0) */
 431                         if (readl(&phy->bistgsr) & DDRPHYC_BISTGSR_BDXERR)
 432                                 result = false; /* BIST_FAIL; */
 433                         else
 434                                 result = true; /* BIST_SUCCESS; */
 435                 } else {
 436                         result = false; /* BIST_FAIL; */
 437                 }
 438         }
 439
 440         /* loop while success */
 441         cnt++;
 442         if (result && cnt != 1000)
 443                 goto run;
 444
 445         if (!result)
 446                 error++;
 447
 448         if (error < BIST_error_max) {
 449                 if (cnt != 1000)
 450                         goto run;
 451                 bist->test_result = true;
 452         } else {
 453                 bist->test_result = false;
 454         }
 455 }
 456
 457 /* After running the deskew algo, this function applies the new DQ delays
 458  * by reading them from the array "deskew_delay"and writing in PHY registers.
 459  * The bits that are not deskewed parfectly (too much skew on them,
 460  * or data eye very wide) are marked in the array deskew_non_converge.
 461  */
 462 static void apply_deskew_results(struct stm32mp1_ddrphy *phy, u8 byte,
 463                                  u8 deskew_delay[NUM_BYTES][8],
 464                                  u8 deskew_non_converge[NUM_BYTES][8])
 465 {
 466         u8  bit_i;
 467         u8  index;
 468
 469         for (bit_i = 0; bit_i < 8; bit_i++) {
 470                 set_DQ_unit_delay(phy, byte, bit_i, deskew_delay[byte][bit_i]);
 471                 index = DQ_unit_index(phy, byte, bit_i);
 472                 pr_debug("Byte %d ; bit %d : The new DQ delay (%d) index=%d [delta=%d, 3 is the default]",
 473                          byte, bit_i, deskew_delay[byte][bit_i],
 474                          index, index - 3);
 475                 printf("Byte %d, bit %d, DQ delay = %d",
 476                        byte, bit_i, deskew_delay[byte][bit_i]);
 477                 if (deskew_non_converge[byte][bit_i] == 1)
 478                         pr_debug(" - not converged : still more skew");
 479                 printf("\n");
 480         }
 481 }
 482
 483 /* DQ Bit de-skew algorithm.
 484  * Deskews data lines as much as possible.
 485  * 1. Add delay to DQS line until finding the failure
 486  *    (normally a hold time violation)
 487  * 2. Reduce DQS line by small steps until finding the very first time
 488  *    we go back to "Pass" condition.
 489  * 3. For each DQ line, Reduce DQ delay until finding the very first failure
 490  *    (normally a hold time fail)
 491  * 4. When all bits are at their first failure delay, we can consider them
 492  *    aligned.
 493  * Handle conrer situation (Can't find Pass-fail, or fail-pass transitions
 494  * at any step)
 495  * TODO Provide a return Status. Improve doc
 496  */
 497 static enum test_result bit_deskew(struct stm32mp1_ddrctl *ctl,
 498                                    struct stm32mp1_ddrphy *phy, char *string)
 499 {
 500         /* New DQ delay value (index), set during Deskew algo */
 501         u8 deskew_delay[NUM_BYTES][8];
 502         /*If there is still skew on a bit, mark this bit. */
 503         u8 deskew_non_converge[NUM_BYTES][8];
 504         struct BIST_result result;
 505         s8 dqs_unit_delay_index = 0;
 506         u8 datx8 = 0;
 507         u8 bit_i = 0;
 508         s8 phase_idx = 0;
 509         s8 bit_i_delay_index = 0;
 510         u8 success = 0;
 511         struct tuning_position last_right_ok;
 512         u8 force_stop = 0;
 513         u8 fail_found;
 514         u8 error = 0;
 515         u8 nb_bytes = get_nb_bytes(ctl);
 516         /* u8 last_pass_dqs_unit = 0; */
 517
 518         memset(deskew_delay, 0, sizeof(deskew_delay));
 519         memset(deskew_non_converge, 0, sizeof(deskew_non_converge));
 520
 521         /*Disable DQS Drift Compensation*/
 522         clrbits_le32(&phy->pgcr, DDRPHYC_PGCR_DFTCMP);
 523         /*Disable all bytes*/
 524         /* Disable automatic power down of DLL and IOs when disabling
 525          * a byte (To avoid having to add programming and  delay
 526          * for a DLL re-lock when later re-enabling a disabled Byte Lane)
 527          */
 528         clrbits_le32(&phy->pgcr, DDRPHYC_PGCR_PDDISDX);
 529
 530         /* Disable all data bytes */
 531         clrbits_le32(&phy->dx0gcr, DDRPHYC_DXNGCR_DXEN);
 532         clrbits_le32(&phy->dx1gcr, DDRPHYC_DXNGCR_DXEN);
 533         clrbits_le32(&phy->dx2gcr, DDRPHYC_DXNGCR_DXEN);
 534         clrbits_le32(&phy->dx3gcr, DDRPHYC_DXNGCR_DXEN);
 535
 536         /* Config the BIST block */
 537         config_BIST(ctl, phy);
 538         pr_debug("BIST Config done.\n");
 539
 540         /* Train each byte */
 541         for (datx8 = 0; datx8 < nb_bytes; datx8++) {
 542                 if (ctrlc()) {
 543                         sprintf(string, "interrupted at byte %d/%d, error=%d",
 544                                 datx8 + 1, nb_bytes, error);
 545                         return TEST_FAILED;
 546                 }
 547                 pr_debug("\n======================\n");
 548                 pr_debug("Start deskew byte %d .\n", datx8);
 549                 pr_debug("======================\n");
 550                 /* Enable Byte (DXNGCR, bit DXEN) */
 551                 setbits_le32(DXNGCR(phy, datx8), DDRPHYC_DXNGCR_DXEN);
 552
 553                 /* Select the byte lane for comparison of read data */
 554                 BIST_datx8_sel(phy, datx8);
 555
 556                 /* Set all DQDLYn to maximum value. All bits within the byte
 557                  * will be delayed with DQSTR = 2 instead of max = 3
 558                  * to avoid inter bits fail influence
 559                  */
 560                 writel(0xAAAAAAAA, DXNDQTR(phy, datx8));
 561
 562                 /* Set the DQS phase delay to 90 DEG (default).
 563                  * What is defined here is the index of the desired config
 564                  * in the PHASE array.
 565                  */
 566                 phase_idx = _90deg;
 567
 568                 /* Set DQS unit delay to the max value. */
 569                 dqs_unit_delay_index = MAX_DQS_UNIT_IDX;
 570                 DQS_unit_delay(phy, datx8, dqs_unit_delay_index);
 571                 DQS_phase_delay(phy, datx8, phase_idx);
 572
 573                 /* Issue a DLL soft reset */
 574                 clrbits_le32(DXNDLLCR(phy, datx8), DDRPHYC_DXNDLLCR_DLLSRST);
 575                 setbits_le32(DXNDLLCR(phy, datx8), DDRPHYC_DXNDLLCR_DLLSRST);
 576
 577                 /* Test this typical init condition */
 578                 BIST_test(phy, datx8, &result);
 579                 success = result.test_result;
 580
 581                 /* If the test pass in this typical condition,
 582                  * start the algo with it.
 583                  * Else, look for Pass init condition
 584                  */
 585                 if (!success) {
 586                         pr_debug("Fail at init condtion. Let's look for a good init condition.\n");
 587                         success = 0; /* init */
 588                         /* Make sure we start with a PASS condition before
 589                          * looking for a fail condition.
 590                          * Find the first PASS PHASE condition
 591                          */
 592
 593                         /* escape if we find a PASS */
 594                         pr_debug("increase Phase idx\n");
 595                         while (!success && (phase_idx <= MAX_DQS_PHASE_IDX)) {
 596                                 DQS_phase_delay(phy, datx8, phase_idx);
 597                                 BIST_test(phy, datx8, &result);
 598                                 success = result.test_result;
 599                                 phase_idx++;
 600                         }
 601                         /* if ended with success
 602                          * ==>> Restore the fist success condition
 603                          */
 604                         if (success)
 605                                 phase_idx--; /* because it ended with ++ */
 606                 }
 607                 if (ctrlc()) {
 608                         sprintf(string, "interrupted at byte %d/%d, error=%d",
 609                                 datx8 + 1, nb_bytes, error);
 610                         return TEST_FAILED;
 611                 }
 612                 /* We couldn't find a successful condition, its seems
 613                  * we have hold violation, lets try reduce DQS_unit Delay
 614                  */
 615                 if (!success) {
 616                         /* We couldn't find a successful condition, its seems
 617                          * we have hold violation, lets try reduce DQS_unit
 618                          * Delay
 619                          */
 620                         pr_debug("Still fail. Try decrease DQS Unit delay\n");
 621
 622                         phase_idx = 0;
 623                         dqs_unit_delay_index = 0;
 624                         DQS_phase_delay(phy, datx8, phase_idx);
 625
 626                         /* escape if we find a PASS */
 627                         while (!success &&
 628                                (dqs_unit_delay_index <=
 629                                 MAX_DQS_UNIT_IDX)) {
 630                                 DQS_unit_delay(phy, datx8,
 631                                                dqs_unit_delay_index);
 632                                 BIST_test(phy, datx8, &result);
 633                                 success = result.test_result;
 634                                 dqs_unit_delay_index++;
 635                         }
 636                         if (success) {
 637                                 /* Restore the first success condition*/
 638                                 dqs_unit_delay_index--;
 639                                 /* last_pass_dqs_unit = dqs_unit_delay_index;*/
 640                                 DQS_unit_delay(phy, datx8,
 641                                                dqs_unit_delay_index);
 642                         } else {
 643                                 /* No need to continue,
 644                                  * there is no pass region.
 645                                  */
 646                                 force_stop = 1;
 647                         }
 648                 }
 649
 650                 /* There is an initial PASS condition
 651                  * Look for the first failing condition by PHASE stepping.
 652                  * This part of the algo can finish without converging.
 653                  */
 654                 if (force_stop) {
 655                         printf("Result: Failed ");
 656                         printf("[Cannot Deskew lines, ");
 657                         printf("there is no PASS region]\n");
 658                         error++;
 659                         continue;
 660                 }
 661                 if (ctrlc()) {
 662                         sprintf(string, "interrupted at byte %d/%d, error=%d",
 663                                 datx8 + 1, nb_bytes, error);
 664                         return TEST_FAILED;
 665                 }
 666
 667                 pr_debug("there is a pass region for phase idx %d\n",
 668                          phase_idx);
 669                 pr_debug("Step1: Find the first failing condition\n");
 670                 /* Look for the first failing condition by PHASE stepping.
 671                  * This part of the algo can finish without converging.
 672                  */
 673
 674                 /* escape if we find a fail (hold time violation)
 675                  * condition at any bit or if out of delay range.
 676                  */
 677                 while (success && (phase_idx <= MAX_DQS_PHASE_IDX)) {
 678                         DQS_phase_delay(phy, datx8, phase_idx);
 679                         BIST_test(phy, datx8, &result);
 680                         success = result.test_result;
 681                         phase_idx++;
 682                 }
 683                 if (ctrlc()) {
 684                         sprintf(string, "interrupted at byte %d/%d, error=%d",
 685                                 datx8 + 1, nb_bytes, error);
 686                         return TEST_FAILED;
 687                 }
 688
 689                 /* if the loop ended with a failing condition at any bit,
 690                  * lets look for the first previous success condition by unit
 691                  * stepping (minimal delay)
 692                  */
 693                 if (!success) {
 694                         pr_debug("Fail region (PHASE) found phase idx %d\n",
 695                                  phase_idx);
 696                         pr_debug("Let's look for first success by DQS Unit steps\n");
 697                         /* This part, the algo always converge */
 698                         phase_idx--;
 699
 700                         /* escape if we find a success condition
 701                          * or if out of delay range.
 702                          */
 703                         while (!success && dqs_unit_delay_index >= 0) {
 704                                 DQS_unit_delay(phy, datx8,
 705                                                dqs_unit_delay_index);
 706                                 BIST_test(phy, datx8, &result);
 707                                 success = result.test_result;
 708                                 dqs_unit_delay_index--;
 709                         }
 710                         /* if the loop ended with a success condition,
 711                          * the last delay Right OK (before hold violation)
 712                          *  condition is then defined as following:
 713                          */
 714                         if (success) {
 715                                 /* Hold the dely parameters of the the last
 716                                  * delay Right OK condition.
 717                                  * -1 to get back to current condition
 718                                  */
 719                                 last_right_ok.phase = phase_idx;
 720                                 /*+1 to get back to current condition */
 721                                 last_right_ok.unit = dqs_unit_delay_index + 1;
 722                                 last_right_ok.bits_delay = 0xFFFFFFFF;
 723                                 pr_debug("Found %d\n", dqs_unit_delay_index);
 724                         } else {
 725                                 /* the last OK condition is then with the
 726                                  * previous phase_idx.
 727                                  * -2 instead of -1 because at the last
 728                                  * iteration of the while(),
 729                                  * we incremented phase_idx
 730                                  */
 731                                 last_right_ok.phase = phase_idx - 1;
 732                                 /* Nominal+1. Because we want the previous
 733                                  * delay after reducing the phase delay.
 734                                  */
 735                                 last_right_ok.unit = 1;
 736                                 last_right_ok.bits_delay = 0xFFFFFFFF;
 737                                 pr_debug("Not Found : try previous phase %d\n",
 738                                          phase_idx - 1);
 739
 740                                 DQS_phase_delay(phy, datx8, phase_idx - 1);
 741                                 dqs_unit_delay_index = 0;
 742                                 success = true;
 743                                 while (success &&
 744                                        (dqs_unit_delay_index <
 745                                         MAX_DQS_UNIT_IDX)) {
 746                                         DQS_unit_delay(phy, datx8,
 747                                                        dqs_unit_delay_index);
 748                                         BIST_test(phy, datx8, &result);
 749                                         success = result.test_result;
 750                                         dqs_unit_delay_index++;
 751                                         pr_debug("dqs_unit_delay_index = %d, result = %d\n",
 752                                                  dqs_unit_delay_index, success);
 753                                 }
 754
 755                                 if (!success) {
 756                                         last_right_ok.unit =
 757                                                  dqs_unit_delay_index - 1;
 758                                 } else {
 759                                         last_right_ok.unit = 0;
 760                                         pr_debug("ERROR: failed region not FOUND");
 761                                 }
 762                         }
 763                 } else {
 764                         /* we can't find a failing  condition at all bits
 765                          * ==> Just hold the last test condition
 766                          * (the max DQS delay)
 767                          * which is the most likely,
 768                          * the closest to a hold violation
 769                          * If we can't find a Fail condition after
 770                          * the Pass region, stick at this position
 771                          * In order to have max chances to find a fail
 772                          * when reducing DQ delays.
 773                          */
 774                         last_right_ok.phase = MAX_DQS_PHASE_IDX;
 775                         last_right_ok.unit = MAX_DQS_UNIT_IDX;
 776                         last_right_ok.bits_delay = 0xFFFFFFFF;
 777                         pr_debug("Can't find the a fail condition\n");
 778                 }
 779
 780                 /* step 2:
 781                  * if we arrive at this stage, it means that we found the last
 782                  * Right OK condition (by tweeking the DQS delay). Or we simply
 783                  * pushed DQS delay to the max
 784                  * This means that by reducing the delay on some DQ bits,
 785                  * we should find a failing condition.
 786                  */
 787                 printf("Byte %d, DQS unit = %d, phase = %d\n",
 788                        datx8, last_right_ok.unit, last_right_ok.phase);
 789                 pr_debug("Step2, unit = %d, phase = %d, bits delay=%x\n",
 790                          last_right_ok.unit, last_right_ok.phase,
 791                          last_right_ok.bits_delay);
 792
 793                 /* Restore the last_right_ok condtion. */
 794                 DQS_unit_delay(phy, datx8, last_right_ok.unit);
 795                 DQS_phase_delay(phy, datx8, last_right_ok.phase);
 796                 writel(last_right_ok.bits_delay, DXNDQTR(phy, datx8));
 797
 798                 /* train each bit
 799                  * reduce delay on each bit, and perform a write/read test
 800                  * and stop at the very first time it fails.
 801                  * the goal is the find the first failing condition
 802                  * for each bit.
 803                  * When we achieve this condition<  for all the bits,
 804                  * we are sure they are aligned (+/- step resolution)
 805                  */
 806                 fail_found = 0;
 807                 for (bit_i = 0; bit_i < 8; bit_i++) {
 808                         if (ctrlc()) {
 809                                 sprintf(string,
 810                                         "interrupted at byte %d/%d, error=%d",
 811                                         datx8 + 1, nb_bytes, error);
 812                                 return error;
 813                         }
 814                         pr_debug("deskewing bit %d:\n", bit_i);
 815                         success = 1; /* init */
 816                         /* Set all DQDLYn to maximum value.
 817                          * Only bit_i will be down-delayed
 818                          * ==> if we have a fail, it will be definitely
 819                          *     from bit_i
 820                          */
 821                         writel(0xFFFFFFFF, DXNDQTR(phy, datx8));
 822                         /* Arriving at this stage,
 823                          * we have a success condition with delay = 3;
 824                          */
 825                         bit_i_delay_index = 3;
 826
 827                         /* escape if bit delay is out of range or
 828                          * if a fatil occurs
 829                          */
 830                         while ((bit_i_delay_index >= 0) && success) {
 831                                 set_DQ_unit_delay(phy, datx8,
 832                                                   bit_i,
 833                                                   bit_i_delay_index);
 834                                 BIST_test(phy, datx8, &result);
 835                                 success = result.test_result;
 836                                 bit_i_delay_index--;
 837                         }
 838
 839                         /* if escape with a fail condition
 840                          * ==> save this position for bit_i
 841                          */
 842                         if (!success) {
 843                                 /* save the delay position.
 844                                  * Add 1 because the while loop ended with a --,
 845                                  * and that we need to hold the last success
 846                                  *  delay
 847                                  */
 848                                 deskew_delay[datx8][bit_i] =
 849                                         bit_i_delay_index + 2;
 850                                 if (deskew_delay[datx8][bit_i] > 3)
 851                                         deskew_delay[datx8][bit_i] = 3;
 852
 853                                 /* A flag that states we found at least a fail
 854                                  * at one bit.
 855                                  */
 856                                 fail_found = 1;
 857                                 pr_debug("Fail found on bit %d, for delay = %d => deskew[%d][%d] = %d\n",
 858                                          bit_i, bit_i_delay_index + 1,
 859                                          datx8, bit_i,
 860                                          deskew_delay[datx8][bit_i]);
 861                         } else {
 862                                 /* if we can find a success condition by
 863                                  * back-delaying this bit, just set the delay
 864                                  * to 0 (the best deskew
 865                                  * possible) and mark the bit.
 866                                  */
 867                                 deskew_delay[datx8][bit_i] = 0;
 868                                 /* set a flag that will be used later
 869                                  * in the report.
 870                                  */
 871                                 deskew_non_converge[datx8][bit_i] = 1;
 872                                 pr_debug("Fail not found on bit %d => deskew[%d][%d] = %d\n",
 873                                          bit_i, datx8, bit_i,
 874                                          deskew_delay[datx8][bit_i]);
 875                         }
 876                 }
 877                 pr_debug("**********byte %d tuning complete************\n",
 878                          datx8);
 879                 /* If we can't find any failure by back delaying DQ lines,
 880                  * hold the default values
 881                  */
 882                 if (!fail_found) {
 883                         for (bit_i = 0; bit_i < 8; bit_i++)
 884                                 deskew_delay[datx8][bit_i] = 0;
 885                         pr_debug("The Deskew algorithm can't converge, there is too much margin in your design. Good job!\n");
 886                 }
 887
 888                 apply_deskew_results(phy, datx8, deskew_delay,
 889                                      deskew_non_converge);
 890                 /* Restore nominal value for DQS delay */
 891                 DQS_phase_delay(phy, datx8, 3);
 892                 DQS_unit_delay(phy, datx8, 3);
 893                 /* disable byte after byte bits deskew */
 894                 clrbits_le32(DXNGCR(phy, datx8), DDRPHYC_DXNGCR_DXEN);
 895         }  /* end of byte deskew */
 896
 897         /* re-enable all data bytes */
 898         setbits_le32(&phy->dx0gcr, DDRPHYC_DXNGCR_DXEN);
 899         setbits_le32(&phy->dx1gcr, DDRPHYC_DXNGCR_DXEN);
 900         setbits_le32(&phy->dx2gcr, DDRPHYC_DXNGCR_DXEN);
 901         setbits_le32(&phy->dx3gcr, DDRPHYC_DXNGCR_DXEN);
 902
 903         if (error) {
 904                 sprintf(string, "error = %d", error);
 905                 return TEST_FAILED;
 906         }
 907
 908         return TEST_PASSED;
 909 } /* end function */
 910
 911 /* Trim DQS timings and set it in the centre of data eye.
 912  * Look for a PPPPF region, then look for a FPPP region and finally select
 913  * the mid of the FPPPPPF region
 914  */
 915 static enum test_result eye_training(struct stm32mp1_ddrctl *ctl,
 916                                      struct stm32mp1_ddrphy *phy, char *string)
 917 {
 918         /*Stores the DQS trim values (PHASE index, unit index) */
 919         u8 eye_training_val[NUM_BYTES][2];
 920         u8 byte = 0;
 921         struct BIST_result result;
 922         s8 dqs_unit_delay_index = 0;
 923         s8 phase_idx = 0;
 924         s8 dqs_unit_delay_index_pass = 0;
 925         s8 phase_idx_pass = 0;
 926         u8 success = 0;
 927         u8 left_phase_bound_found, right_phase_bound_found;
 928         u8 left_unit_bound_found, right_unit_bound_found;
 929         u8 left_bound_found, right_bound_found;
 930         struct tuning_position left_bound, right_bound;
 931         u8 error = 0;
 932         u8 nb_bytes = get_nb_bytes(ctl);
 933
 934         /*Disable DQS Drift Compensation*/
 935         clrbits_le32(&phy->pgcr, DDRPHYC_PGCR_DFTCMP);
 936         /*Disable all bytes*/
 937         /* Disable automatic power down of DLL and IOs when disabling a byte
 938          * (To avoid having to add programming and  delay
 939          * for a DLL re-lock when later re-enabling a disabled Byte Lane)
 940          */
 941         clrbits_le32(&phy->pgcr, DDRPHYC_PGCR_PDDISDX);
 942
 943         /*Disable all data bytes */
 944         clrbits_le32(&phy->dx0gcr, DDRPHYC_DXNGCR_DXEN);
 945         clrbits_le32(&phy->dx1gcr, DDRPHYC_DXNGCR_DXEN);
 946         clrbits_le32(&phy->dx2gcr, DDRPHYC_DXNGCR_DXEN);
 947         clrbits_le32(&phy->dx3gcr, DDRPHYC_DXNGCR_DXEN);
 948
 949         /* Config the BIST block */
 950         config_BIST(ctl, phy);
 951
 952         for (byte = 0; byte < nb_bytes; byte++) {
 953                 if (ctrlc()) {
 954                         sprintf(string, "interrupted at byte %d/%d, error=%d",
 955                                 byte + 1, nb_bytes, error);
 956                         return TEST_FAILED;
 957                 }
 958                 right_bound.phase = 0;
 959                 right_bound.unit = 0;
 960
 961                 left_bound.phase = 0;
 962                 left_bound.unit = 0;
 963
 964                 left_phase_bound_found = 0;
 965                 right_phase_bound_found = 0;
 966
 967                 left_unit_bound_found = 0;
 968                 right_unit_bound_found = 0;
 969
 970                 left_bound_found = 0;
 971                 right_bound_found = 0;
 972
 973                 /* Enable Byte (DXNGCR, bit DXEN) */
 974                 setbits_le32(DXNGCR(phy, byte), DDRPHYC_DXNGCR_DXEN);
 975
 976                 /* Select the byte lane for comparison of read data */
 977                 BIST_datx8_sel(phy, byte);
 978
 979                 /* Set DQS phase delay to the nominal value. */
 980                 phase_idx = _90deg;
 981                 phase_idx_pass = phase_idx;
 982
 983                 /* Set DQS unit delay to the nominal value. */
 984                 dqs_unit_delay_index = 3;
 985                 dqs_unit_delay_index_pass = dqs_unit_delay_index;
 986                 success = 0;
 987
 988                 pr_debug("STEP0: Find Init delay\n");
 989                 /* STEP0: Find Init delay: a delay that put the system
 990                  * in a "Pass" condition then (TODO) update
 991                  * dqs_unit_delay_index_pass & phase_idx_pass
 992                  */
 993                 DQS_unit_delay(phy, byte, dqs_unit_delay_index);
 994                 DQS_phase_delay(phy, byte, phase_idx);
 995                 BIST_test(phy, byte, &result);
 996                 success = result.test_result;
 997                 /* If we have a fail in the nominal condition */
 998                 if (!success) {
 999                         /* Look at the left */
1000                         while (phase_idx >= 0 && !success) {
1001                                 phase_idx--;
1002                                 DQS_phase_delay(phy, byte, phase_idx);
1003                                 BIST_test(phy, byte, &result);
1004                                 success = result.test_result;
1005                         }
1006                 }
1007                 if (!success) {
1008                         /* if we can't find pass condition,
1009                          * then look at the right
1010                          */
1011                         phase_idx = _90deg;
1012                         while (phase_idx <= MAX_DQS_PHASE_IDX &&
1013                                !success) {
1014                                 phase_idx++;
1015                                 DQS_phase_delay(phy, byte,
1016                                                 phase_idx);
1017                                 BIST_test(phy, byte, &result);
1018                                 success = result.test_result;
1019                         }
1020                 }
1021                 /* save the pass condition */
1022                 if (success) {
1023                         phase_idx_pass = phase_idx;
1024                 } else {
1025                         printf("Result: Failed ");
1026                         printf("[Cannot DQS timings, ");
1027                         printf("there is no PASS region]\n");
1028                         error++;
1029                         continue;
1030                 }
1031
1032                 if (ctrlc()) {
1033                         sprintf(string, "interrupted at byte %d/%d, error=%d",
1034                                 byte + 1, nb_bytes, error);
1035                         return TEST_FAILED;
1036                 }
1037                 pr_debug("STEP1: Find LEFT PHASE DQS Bound\n");
1038                 /* STEP1: Find LEFT PHASE DQS Bound */
1039                 while ((phase_idx >= 0) &&
1040                        (phase_idx <= MAX_DQS_PHASE_IDX) &&
1041                        !left_phase_bound_found) {
1042                         DQS_unit_delay(phy, byte,
1043                                        dqs_unit_delay_index);
1044                         DQS_phase_delay(phy, byte,
1045                                         phase_idx);
1046                         BIST_test(phy, byte, &result);
1047                         success = result.test_result;
1048
1049                         /*TODO: Manage the case were at the beginning
1050                          * there is already a fail
1051                          */
1052                         if (!success) {
1053                                 /* the last pass condition */
1054                                 left_bound.phase = ++phase_idx;
1055                                 left_phase_bound_found = 1;
1056                         } else if (success) {
1057                                 phase_idx--;
1058                         }
1059                 }
1060                 if (!left_phase_bound_found) {
1061                         left_bound.phase = 0;
1062                         phase_idx = 0;
1063                 }
1064                 /* If not found, lets take 0 */
1065
1066                 if (ctrlc()) {
1067                         sprintf(string, "interrupted at byte %d/%d, error=%d",
1068                                 byte + 1, nb_bytes, error);
1069                         return TEST_FAILED;
1070                 }
1071                 pr_debug("STEP2: Find UNIT left bound\n");
1072                 /* STEP2: Find UNIT left bound */
1073                 while ((dqs_unit_delay_index >= 0) &&
1074                        !left_unit_bound_found) {
1075                         DQS_unit_delay(phy, byte,
1076                                        dqs_unit_delay_index);
1077                         DQS_phase_delay(phy, byte, phase_idx);
1078                         BIST_test(phy, byte, &result);
1079                         success = result.test_result;
1080                         if (!success) {
1081                                 left_bound.unit =
1082                                         ++dqs_unit_delay_index;
1083                                 left_unit_bound_found = 1;
1084                                 left_bound_found = 1;
1085                         } else if (success) {
1086                                 dqs_unit_delay_index--;
1087                         }
1088                 }
1089
1090                 /* If not found, lets take 0 */
1091                 if (!left_unit_bound_found)
1092                         left_bound.unit = 0;
1093
1094                 if (ctrlc()) {
1095                         sprintf(string, "interrupted at byte %d/%d, error=%d",
1096                                 byte + 1, nb_bytes, error);
1097                         return TEST_FAILED;
1098                 }
1099                 pr_debug("STEP3: Find PHase right bound\n");
1100                 /* STEP3: Find PHase right bound, start with "pass"
1101                  * condition
1102                  */
1103
1104                 /* Set DQS phase delay to the pass value. */
1105                 phase_idx = phase_idx_pass;
1106
1107                 /* Set DQS unit delay to the pass value. */
1108                 dqs_unit_delay_index = dqs_unit_delay_index_pass;
1109
1110                 while ((phase_idx <= MAX_DQS_PHASE_IDX) &&
1111                        !right_phase_bound_found) {
1112                         DQS_unit_delay(phy, byte,
1113                                        dqs_unit_delay_index);
1114                         DQS_phase_delay(phy, byte, phase_idx);
1115                         BIST_test(phy, byte, &result);
1116                         success = result.test_result;
1117                         if (!success) {
1118                                 /* the last pass condition */
1119                                 right_bound.phase = --phase_idx;
1120                                 right_phase_bound_found = 1;
1121                         } else if (success) {
1122                                 phase_idx++;
1123                         }
1124                 }
1125
1126                 /* If not found, lets take the max value */
1127                 if (!right_phase_bound_found) {
1128                         right_bound.phase = MAX_DQS_PHASE_IDX;
1129                         phase_idx = MAX_DQS_PHASE_IDX;
1130                 }
1131
1132                 if (ctrlc()) {
1133                         sprintf(string, "interrupted at byte %d/%d, error=%d",
1134                                 byte + 1, nb_bytes, error);
1135                         return TEST_FAILED;
1136                 }
1137                 pr_debug("STEP4: Find UNIT right bound\n");
1138                 /* STEP4: Find UNIT right bound */
1139                 while ((dqs_unit_delay_index <= MAX_DQS_UNIT_IDX) &&
1140                        !right_unit_bound_found) {
1141                         DQS_unit_delay(phy, byte,
1142                                        dqs_unit_delay_index);
1143                         DQS_phase_delay(phy, byte, phase_idx);
1144                         BIST_test(phy, byte, &result);
1145                         success = result.test_result;
1146                         if (!success) {
1147                                 right_bound.unit =
1148                                         --dqs_unit_delay_index;
1149                                 right_unit_bound_found = 1;
1150                                 right_bound_found = 1;
1151                         } else if (success) {
1152                                 dqs_unit_delay_index++;
1153                         }
1154                 }
1155                 /* If not found, lets take the max value */
1156                 if (!right_unit_bound_found)
1157                         right_bound.unit = MAX_DQS_UNIT_IDX;
1158
1159                 /* If we found a regular FAil Pass FAil pattern
1160                  * FFPPPPPPFF
1161                  * OR PPPPPFF  Or FFPPPPP
1162                  */
1163
1164                 if (left_bound_found || right_bound_found) {
1165                         eye_training_val[byte][0] = (right_bound.phase +
1166                                                  left_bound.phase) / 2;
1167                         eye_training_val[byte][1] = (right_bound.unit +
1168                                                  left_bound.unit) / 2;
1169
1170                         /* If we already lost 1/2PHASE Tuning,
1171                          * let's try to recover by ++ on unit
1172                          */
1173                         if (((right_bound.phase + left_bound.phase) % 2 == 1) &&
1174                             eye_training_val[byte][1] != MAX_DQS_UNIT_IDX)
1175                                 eye_training_val[byte][1]++;
1176                         pr_debug("** found phase : %d -  %d & unit %d - %d\n",
1177                                  right_bound.phase, left_bound.phase,
1178                                  right_bound.unit, left_bound.unit);
1179                         pr_debug("** calculating mid region: phase: %d  unit: %d (nominal is 3)\n",
1180                                  eye_training_val[byte][0],
1181                                  eye_training_val[byte][1]);
1182                 } else {
1183                         /* PPPPPPPPPP, we're already good.
1184                          * Set nominal values.
1185                          */
1186                         eye_training_val[byte][0] = 3;
1187                         eye_training_val[byte][1] = 3;
1188                 }
1189                 DQS_phase_delay(phy, byte, eye_training_val[byte][0]);
1190                 DQS_unit_delay(phy, byte, eye_training_val[byte][1]);
1191
1192                 printf("Byte %d, DQS unit = %d, phase = %d\n",
1193                        byte,
1194                        eye_training_val[byte][1],
1195                        eye_training_val[byte][0]);
1196         }
1197
1198         if (error) {
1199                 sprintf(string, "error = %d", error);
1200                 return TEST_FAILED;
1201         }
1202
1203         return TEST_PASSED;
1204 }
1205
1206 static void display_reg_results(struct stm32mp1_ddrphy *phy, u8 byte)
1207 {
1208         u8 i = 0;
1209
1210         printf("Byte %d Dekew result, bit0 delay, bit1 delay...bit8 delay\n  ",
1211                byte);
1212
1213         for (i = 0; i < 8; i++)
1214                 printf("%d ", DQ_unit_index(phy, byte, i));
1215         printf("\n");
1216
1217         printf("dxndllcr: [%08x] val:%08x\n",
1218                DXNDLLCR(phy, byte),
1219                readl(DXNDLLCR(phy, byte)));
1220         printf("dxnqdstr: [%08x] val:%08x\n",
1221                DXNDQSTR(phy, byte),
1222                readl(DXNDQSTR(phy, byte)));
1223         printf("dxndqtr: [%08x] val:%08x\n",
1224                DXNDQTR(phy, byte),
1225                readl(DXNDQTR(phy, byte)));
1226 }
1227
1228 /* analyse the dgs gating log table, and determine the midpoint.*/
1229 static u8 set_midpoint_read_dqs_gating(struct stm32mp1_ddrphy *phy, u8 byte,
1230                                        u8 dqs_gating[NUM_BYTES]
1231                                                     [MAX_GSL_IDX + 1]
1232                                                     [MAX_GPS_IDX + 1])
1233 {
1234         /* stores the dqs gate values (gsl index, gps index) */
1235         u8 dqs_gate_values[NUM_BYTES][2];
1236         u8 gsl_idx, gps_idx = 0;
1237         u8 left_bound_idx[2] = {0, 0};
1238         u8 right_bound_idx[2] = {0, 0};
1239         u8 left_bound_found = 0;
1240         u8 right_bound_found = 0;
1241         u8 intermittent = 0;
1242         u8 value;
1243
1244         for (gsl_idx = 0; gsl_idx <= MAX_GSL_IDX; gsl_idx++) {
1245                 for (gps_idx = 0; gps_idx <= MAX_GPS_IDX; gps_idx++) {
1246                         value = dqs_gating[byte][gsl_idx][gps_idx];
1247                         if (value == 1 && left_bound_found == 0) {
1248                                 left_bound_idx[0] = gsl_idx;
1249                                 left_bound_idx[1] = gps_idx;
1250                                 left_bound_found = 1;
1251                         } else if (value == 0 &&
1252                                    left_bound_found == 1 &&
1253                                    !right_bound_found) {
1254                                 if (gps_idx == 0) {
1255                                         right_bound_idx[0] = gsl_idx - 1;
1256                                         right_bound_idx[1] = MAX_GPS_IDX;
1257                                 } else {
1258                                         right_bound_idx[0] = gsl_idx;
1259                                         right_bound_idx[1] = gps_idx - 1;
1260                                 }
1261                                 right_bound_found = 1;
1262                         } else if (value == 1 &&
1263                                    right_bound_found == 1) {
1264                                 intermittent = 1;
1265                         }
1266                 }
1267         }
1268
1269         /* if only ppppppp is found, there is no mid region. */
1270         if (left_bound_idx[0] == 0 && left_bound_idx[1] == 0 &&
1271             right_bound_idx[0] == 0 && right_bound_idx[1] == 0)
1272                 intermittent = 1;
1273
1274         /*if we found a regular fail pass fail pattern ffppppppff
1275          * or pppppff  or ffppppp
1276          */
1277         if (!intermittent) {
1278                 /*if we found a regular fail pass fail pattern ffppppppff
1279                  * or pppppff  or ffppppp
1280                  */
1281                 if (left_bound_found || right_bound_found) {
1282                         pr_debug("idx0(%d): %d %d      idx1(%d) : %d %d\n",
1283                                  left_bound_found,
1284                                  right_bound_idx[0], left_bound_idx[0],
1285                                  right_bound_found,
1286                                  right_bound_idx[1], left_bound_idx[1]);
1287                         dqs_gate_values[byte][0] =
1288                                 (right_bound_idx[0] + left_bound_idx[0]) / 2;
1289                         dqs_gate_values[byte][1] =
1290                                 (right_bound_idx[1] + left_bound_idx[1]) / 2;
1291                         /* if we already lost 1/2gsl tuning,
1292                          * let's try to recover by ++ on gps
1293                          */
1294                         if (((right_bound_idx[0] +
1295                               left_bound_idx[0]) % 2 == 1) &&
1296                             dqs_gate_values[byte][1] != MAX_GPS_IDX)
1297                                 dqs_gate_values[byte][1]++;
1298                         /* if we already lost 1/2gsl tuning and gps is on max*/
1299                         else if (((right_bound_idx[0] +
1300                                    left_bound_idx[0]) % 2 == 1) &&
1301                                  dqs_gate_values[byte][1] == MAX_GPS_IDX) {
1302                                 dqs_gate_values[byte][1] = 0;
1303                                 dqs_gate_values[byte][0]++;
1304                         }
1305                         /* if we have gsl left and write limit too close
1306                          * (difference=1)
1307                          */
1308                         if (((right_bound_idx[0] - left_bound_idx[0]) == 1)) {
1309                                 dqs_gate_values[byte][1] = (left_bound_idx[1] +
1310                                                             right_bound_idx[1] +
1311                                                             4) / 2;
1312                                 if (dqs_gate_values[byte][1] >= 4) {
1313                                         dqs_gate_values[byte][0] =
1314                                                 right_bound_idx[0];
1315                                         dqs_gate_values[byte][1] -= 4;
1316                                 } else {
1317                                         dqs_gate_values[byte][0] =
1318                                                 left_bound_idx[0];
1319                                 }
1320                         }
1321                         pr_debug("*******calculating mid region: system latency: %d  phase: %d********\n",
1322                                  dqs_gate_values[byte][0],
1323                                  dqs_gate_values[byte][1]);
1324                         pr_debug("*******the nominal values were system latency: 0  phase: 2*******\n");
1325                 }
1326         } else {
1327                 /* if intermitant, restore defaut values */
1328                 pr_debug("dqs gating:no regular fail/pass/fail found. defaults values restored.\n");
1329                 dqs_gate_values[byte][0] = 0;
1330                 dqs_gate_values[byte][1] = 2;
1331         }
1332         set_r0dgsl_delay(phy, byte, dqs_gate_values[byte][0]);
1333         set_r0dgps_delay(phy, byte, dqs_gate_values[byte][1]);
1334         printf("Byte %d, R0DGSL = %d, R0DGPS = %d\n",
1335                byte, dqs_gate_values[byte][0], dqs_gate_values[byte][1]);
1336
1337         /* return 0 if intermittent or if both left_bound
1338          * and right_bound are not found
1339          */
1340         return !(intermittent || (left_bound_found && right_bound_found));
1341 }
1342
1343 static enum test_result read_dqs_gating(struct stm32mp1_ddrctl *ctl,
1344                                         struct stm32mp1_ddrphy *phy,
1345                                         char *string)
1346 {
1347         /* stores the log of pass/fail */
1348         u8 dqs_gating[NUM_BYTES][MAX_GSL_IDX + 1][MAX_GPS_IDX + 1];
1349         u8 byte, gsl_idx, gps_idx = 0;
1350         struct BIST_result result;
1351         u8 success = 0;
1352         u8 nb_bytes = get_nb_bytes(ctl);
1353
1354         memset(dqs_gating, 0x0, sizeof(dqs_gating));
1355
1356         /*disable dqs drift compensation*/
1357         clrbits_le32(&phy->pgcr, DDRPHYC_PGCR_DFTCMP);
1358         /*disable all bytes*/
1359         /* disable automatic power down of dll and ios when disabling a byte
1360          * (to avoid having to add programming and  delay
1361          * for a dll re-lock when later re-enabling a disabled byte lane)
1362          */
1363         clrbits_le32(&phy->pgcr, DDRPHYC_PGCR_PDDISDX);
1364
1365         /* disable all data bytes */
1366         clrbits_le32(&phy->dx0gcr, DDRPHYC_DXNGCR_DXEN);
1367         clrbits_le32(&phy->dx1gcr, DDRPHYC_DXNGCR_DXEN);
1368         clrbits_le32(&phy->dx2gcr, DDRPHYC_DXNGCR_DXEN);
1369         clrbits_le32(&phy->dx3gcr, DDRPHYC_DXNGCR_DXEN);
1370
1371         /* config the bist block */
1372         config_BIST(ctl, phy);
1373
1374         for (byte = 0; byte < nb_bytes; byte++) {
1375                 if (ctrlc()) {
1376                         sprintf(string, "interrupted at byte %d/%d",
1377                                 byte + 1, nb_bytes);
1378                         return TEST_FAILED;
1379                 }
1380                 /* enable byte x (dxngcr, bit dxen) */
1381                 setbits_le32(DXNGCR(phy, byte), DDRPHYC_DXNGCR_DXEN);
1382
1383                 /* select the byte lane for comparison of read data */
1384                 BIST_datx8_sel(phy, byte);
1385                 for (gsl_idx = 0; gsl_idx <= MAX_GSL_IDX; gsl_idx++) {
1386                         for (gps_idx = 0; gps_idx <= MAX_GPS_IDX; gps_idx++) {
1387                                 if (ctrlc()) {
1388                                         sprintf(string,
1389                                                 "interrupted at byte %d/%d",
1390                                                 byte + 1, nb_bytes);
1391                                         return TEST_FAILED;
1392                                 }
1393                                 /* write cfg to dxndqstr */
1394                                 set_r0dgsl_delay(phy, byte, gsl_idx);
1395                                 set_r0dgps_delay(phy, byte, gps_idx);
1396
1397                                 BIST_test(phy, byte, &result);
1398                                 success = result.test_result;
1399                                 if (success)
1400                                         dqs_gating[byte][gsl_idx][gps_idx] = 1;
1401                                 itm_soft_reset(phy);
1402                         }
1403                 }
1404                 set_midpoint_read_dqs_gating(phy, byte, dqs_gating);
1405                 /* dummy reads */
1406                 readl(0xc0000000);
1407                 readl(0xc0000000);
1408         }
1409
1410         /* re-enable drift compensation */
1411         /* setbits_le32(&phy->pgcr, DDRPHYC_PGCR_DFTCMP); */
1412         return TEST_PASSED;
1413 }
1414
1415 /****************************************************************
1416  * TEST
1417  ****************************************************************
1418  */
1419 static enum test_result do_read_dqs_gating(struct stm32mp1_ddrctl *ctl,
1420                                            struct stm32mp1_ddrphy *phy,
1421                                            char *string, int argc,
1422                                            char *argv[])
1423 {
1424         u32 rfshctl3 = readl(&ctl->rfshctl3);
1425         u32 pwrctl = readl(&ctl->pwrctl);
1426         u32 derateen = readl(&ctl->derateen);
1427         enum test_result res;
1428
1429         writel(0x0, &ctl->derateen);
1430         stm32mp1_refresh_disable(ctl);
1431
1432         res = read_dqs_gating(ctl, phy, string);
1433
1434         stm32mp1_refresh_restore(ctl, rfshctl3, pwrctl);
1435         writel(derateen, &ctl->derateen);
1436
1437         return res;
1438 }
1439
1440 static enum test_result do_bit_deskew(struct stm32mp1_ddrctl *ctl,
1441                                       struct stm32mp1_ddrphy *phy,
1442                                       char *string, int argc, char *argv[])
1443 {
1444         u32 rfshctl3 = readl(&ctl->rfshctl3);
1445         u32 pwrctl = readl(&ctl->pwrctl);
1446         u32 derateen = readl(&ctl->derateen);
1447         enum test_result res;
1448
1449         writel(0x0, &ctl->derateen);
1450         stm32mp1_refresh_disable(ctl);
1451
1452         res = bit_deskew(ctl, phy, string);
1453
1454         stm32mp1_refresh_restore(ctl, rfshctl3, pwrctl);
1455         writel(derateen, &ctl->derateen);
1456
1457         return res;
1458 }
1459
1460 static enum test_result do_eye_training(struct stm32mp1_ddrctl *ctl,
1461                                         struct stm32mp1_ddrphy *phy,
1462                                         char *string, int argc, char *argv[])
1463 {
1464         u32 rfshctl3 = readl(&ctl->rfshctl3);
1465         u32 pwrctl = readl(&ctl->pwrctl);
1466         u32 derateen = readl(&ctl->derateen);
1467         enum test_result res;
1468
1469         writel(0x0, &ctl->derateen);
1470         stm32mp1_refresh_disable(ctl);
1471
1472         res = eye_training(ctl, phy, string);
1473
1474         stm32mp1_refresh_restore(ctl, rfshctl3, pwrctl);
1475         writel(derateen, &ctl->derateen);
1476
1477         return res;
1478 }
1479
1480 static enum test_result do_display(struct stm32mp1_ddrctl *ctl,
1481                                    struct stm32mp1_ddrphy *phy,
1482                                    char *string, int argc, char *argv[])
1483 {
1484         int byte;
1485         u8 nb_bytes = get_nb_bytes(ctl);
1486
1487         for (byte = 0; byte < nb_bytes; byte++)
1488                 display_reg_results(phy, byte);
1489
1490         return TEST_PASSED;
1491 }
1492
1493 static enum test_result do_bist_config(struct stm32mp1_ddrctl *ctl,
1494                                        struct stm32mp1_ddrphy *phy,
1495                                        char *string, int argc, char *argv[])
1496 {
1497         unsigned long value;
1498
1499         if (argc > 0) {
1500                 if (strict_strtoul(argv[0], 0, &value) < 0) {
1501                         sprintf(string, "invalid nbErr %s", argv[0]);
1502                         return TEST_FAILED;
1503                 }
1504                 BIST_error_max = value;
1505         }
1506         if (argc > 1) {
1507                 if (strict_strtoul(argv[1], 0, &value) < 0) {
1508                         sprintf(string, "invalid Seed %s", argv[1]);
1509                         return TEST_FAILED;
1510                 }
1511                 BIST_seed = value;
1512         }
1513         printf("Bist.nbErr = %d\n", BIST_error_max);
1514         if (BIST_seed)
1515                 printf("Bist.Seed = 0x%x\n", BIST_seed);
1516         else
1517                 printf("Bist.Seed = random\n");
1518
1519         return TEST_PASSED;
1520 }
1521
1522 /****************************************************************
1523  * TEST Description
1524  ****************************************************************
1525  */
1526
1527 const struct test_desc tuning[] = {
1528         {do_read_dqs_gating, "Read DQS gating",
1529                 "software read DQS Gating", "", 0 },
1530         {do_bit_deskew, "Bit de-skew", "", "", 0 },
1531         {do_eye_training, "Eye Training", "or DQS training", "", 0 },
1532         {do_display, "Display registers", "", "", 0 },
1533         {do_bist_config, "Bist config", "[nbErr] [seed]",
1534          "configure Bist test", 2},
1535 };
1536
1537 const int tuning_nb = ARRAY_SIZE(tuning);