drivers/mtd/nand/raw/brcmnand/brcmnand.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright © 2010-2015 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License for more details.
  13  */
  14
  15 #include <common.h>
  16 #include <asm/io.h>
  17 #include <memalign.h>
  18 #include <nand.h>
  19 #include <clk.h>
  20 #include <dm/devres.h>
  21 #include <linux/err.h>
  22 #include <linux/ioport.h>
  23 #include <linux/completion.h>
  24 #include <linux/errno.h>
  25 #include <linux/log2.h>
  26 #include <asm/processor.h>
  27 #include <dm.h>
  28
  29 #include "brcmnand.h"
  30 #include "brcmnand_compat.h"
  31
  32 /*
  33  * This flag controls if WP stays on between erase/write commands to mitigate
  34  * flash corruption due to power glitches. Values:
  35  * 0: NAND_WP is not used or not available
  36  * 1: NAND_WP is set by default, cleared for erase/write operations
  37  * 2: NAND_WP is always cleared
  38  */
  39 static int wp_on = 1;
  40 module_param(wp_on, int, 0444);
  41
  42 /***********************************************************************
  43  * Definitions
  44  ***********************************************************************/
  45
  46 #define DRV_NAME                        "brcmnand"
  47
  48 #define CMD_NULL                        0x00
  49 #define CMD_PAGE_READ                   0x01
  50 #define CMD_SPARE_AREA_READ             0x02
  51 #define CMD_STATUS_READ                 0x03
  52 #define CMD_PROGRAM_PAGE                0x04
  53 #define CMD_PROGRAM_SPARE_AREA          0x05
  54 #define CMD_COPY_BACK                   0x06
  55 #define CMD_DEVICE_ID_READ              0x07
  56 #define CMD_BLOCK_ERASE                 0x08
  57 #define CMD_FLASH_RESET                 0x09
  58 #define CMD_BLOCKS_LOCK                 0x0a
  59 #define CMD_BLOCKS_LOCK_DOWN            0x0b
  60 #define CMD_BLOCKS_UNLOCK               0x0c
  61 #define CMD_READ_BLOCKS_LOCK_STATUS     0x0d
  62 #define CMD_PARAMETER_READ              0x0e
  63 #define CMD_PARAMETER_CHANGE_COL        0x0f
  64 #define CMD_LOW_LEVEL_OP                0x10
  65
  66 struct brcm_nand_dma_desc {
  67         u32 next_desc;
  68         u32 next_desc_ext;
  69         u32 cmd_irq;
  70         u32 dram_addr;
  71         u32 dram_addr_ext;
  72         u32 tfr_len;
  73         u32 total_len;
  74         u32 flash_addr;
  75         u32 flash_addr_ext;
  76         u32 cs;
  77         u32 pad2[5];
  78         u32 status_valid;
  79 } __packed;
  80
  81 /* Bitfields for brcm_nand_dma_desc::status_valid */
  82 #define FLASH_DMA_ECC_ERROR     (1 << 8)
  83 #define FLASH_DMA_CORR_ERROR    (1 << 9)
  84
  85 /* 512B flash cache in the NAND controller HW */
  86 #define FC_SHIFT                9U
  87 #define FC_BYTES                512U
  88 #define FC_WORDS                (FC_BYTES >> 2)
  89
  90 #define BRCMNAND_MIN_PAGESIZE   512
  91 #define BRCMNAND_MIN_BLOCKSIZE  (8 * 1024)
  92 #define BRCMNAND_MIN_DEVSIZE    (4ULL * 1024 * 1024)
  93
  94 #define NAND_CTRL_RDY                   (INTFC_CTLR_READY | INTFC_FLASH_READY)
  95 #define NAND_POLL_STATUS_TIMEOUT_MS     100
  96
  97 /* Controller feature flags */
  98 enum {
  99         BRCMNAND_HAS_1K_SECTORS                 = BIT(0),
 100         BRCMNAND_HAS_PREFETCH                   = BIT(1),
 101         BRCMNAND_HAS_CACHE_MODE                 = BIT(2),
 102         BRCMNAND_HAS_WP                         = BIT(3),
 103 };
 104
 105 struct brcmnand_controller {
 106 #ifndef __UBOOT__
 107         struct device           *dev;
 108 #else
 109         struct udevice          *dev;
 110 #endif /* __UBOOT__ */
 111         struct nand_hw_control  controller;
 112         void __iomem            *nand_base;
 113         void __iomem            *nand_fc; /* flash cache */
 114         void __iomem            *flash_dma_base;
 115         unsigned int            irq;
 116         unsigned int            dma_irq;
 117         int                     nand_version;
 118         int                     parameter_page_big_endian;
 119
 120         /* Some SoCs provide custom interrupt status register(s) */
 121         struct brcmnand_soc     *soc;
 122
 123         /* Some SoCs have a gateable clock for the controller */
 124         struct clk              *clk;
 125
 126         int                     cmd_pending;
 127         bool                    dma_pending;
 128         struct completion       done;
 129         struct completion       dma_done;
 130
 131         /* List of NAND hosts (one for each chip-select) */
 132         struct list_head host_list;
 133
 134         struct brcm_nand_dma_desc *dma_desc;
 135         dma_addr_t              dma_pa;
 136
 137         /* in-memory cache of the FLASH_CACHE, used only for some commands */
 138         u8                      flash_cache[FC_BYTES];
 139
 140         /* Controller revision details */
 141         const u16               *reg_offsets;
 142         unsigned int            reg_spacing; /* between CS1, CS2, ... regs */
 143         const u8                *cs_offsets; /* within each chip-select */
 144         const u8                *cs0_offsets; /* within CS0, if different */
 145         unsigned int            max_block_size;
 146         const unsigned int      *block_sizes;
 147         unsigned int            max_page_size;
 148         const unsigned int      *page_sizes;
 149         unsigned int            max_oob;
 150         u32                     features;
 151
 152         /* for low-power standby/resume only */
 153         u32                     nand_cs_nand_select;
 154         u32                     nand_cs_nand_xor;
 155         u32                     corr_stat_threshold;
 156         u32                     flash_dma_mode;
 157 };
 158
 159 struct brcmnand_cfg {
 160         u64                     device_size;
 161         unsigned int            block_size;
 162         unsigned int            page_size;
 163         unsigned int            spare_area_size;
 164         unsigned int            device_width;
 165         unsigned int            col_adr_bytes;
 166         unsigned int            blk_adr_bytes;
 167         unsigned int            ful_adr_bytes;
 168         unsigned int            sector_size_1k;
 169         unsigned int            ecc_level;
 170         /* use for low-power standby/resume only */
 171         u32                     acc_control;
 172         u32                     config;
 173         u32                     config_ext;
 174         u32                     timing_1;
 175         u32                     timing_2;
 176 };
 177
 178 struct brcmnand_host {
 179         struct list_head        node;
 180
 181         struct nand_chip        chip;
 182 #ifndef __UBOOT__
 183         struct platform_device  *pdev;
 184 #else
 185         struct udevice  *pdev;
 186 #endif /* __UBOOT__ */
 187         int                     cs;
 188
 189         unsigned int            last_cmd;
 190         unsigned int            last_byte;
 191         u64                     last_addr;
 192         struct brcmnand_cfg     hwcfg;
 193         struct brcmnand_controller *ctrl;
 194 };
 195
 196 enum brcmnand_reg {
 197         BRCMNAND_CMD_START = 0,
 198         BRCMNAND_CMD_EXT_ADDRESS,
 199         BRCMNAND_CMD_ADDRESS,
 200         BRCMNAND_INTFC_STATUS,
 201         BRCMNAND_CS_SELECT,
 202         BRCMNAND_CS_XOR,
 203         BRCMNAND_LL_OP,
 204         BRCMNAND_CS0_BASE,
 205         BRCMNAND_CS1_BASE,              /* CS1 regs, if non-contiguous */
 206         BRCMNAND_CORR_THRESHOLD,
 207         BRCMNAND_CORR_THRESHOLD_EXT,
 208         BRCMNAND_UNCORR_COUNT,
 209         BRCMNAND_CORR_COUNT,
 210         BRCMNAND_CORR_EXT_ADDR,
 211         BRCMNAND_CORR_ADDR,
 212         BRCMNAND_UNCORR_EXT_ADDR,
 213         BRCMNAND_UNCORR_ADDR,
 214         BRCMNAND_SEMAPHORE,
 215         BRCMNAND_ID,
 216         BRCMNAND_ID_EXT,
 217         BRCMNAND_LL_RDATA,
 218         BRCMNAND_OOB_READ_BASE,
 219         BRCMNAND_OOB_READ_10_BASE,      /* offset 0x10, if non-contiguous */
 220         BRCMNAND_OOB_WRITE_BASE,
 221         BRCMNAND_OOB_WRITE_10_BASE,     /* offset 0x10, if non-contiguous */
 222         BRCMNAND_FC_BASE,
 223 };
 224
 225 /* BRCMNAND v4.0 */
 226 static const u16 brcmnand_regs_v40[] = {
 227         [BRCMNAND_CMD_START]            =  0x04,
 228         [BRCMNAND_CMD_EXT_ADDRESS]      =  0x08,
 229         [BRCMNAND_CMD_ADDRESS]          =  0x0c,
 230         [BRCMNAND_INTFC_STATUS]         =  0x6c,
 231         [BRCMNAND_CS_SELECT]            =  0x14,
 232         [BRCMNAND_CS_XOR]               =  0x18,
 233         [BRCMNAND_LL_OP]                = 0x178,
 234         [BRCMNAND_CS0_BASE]             =  0x40,
 235         [BRCMNAND_CS1_BASE]             =  0xd0,
 236         [BRCMNAND_CORR_THRESHOLD]       =  0x84,
 237         [BRCMNAND_CORR_THRESHOLD_EXT]   =     0,
 238         [BRCMNAND_UNCORR_COUNT]         =     0,
 239         [BRCMNAND_CORR_COUNT]           =     0,
 240         [BRCMNAND_CORR_EXT_ADDR]        =  0x70,
 241         [BRCMNAND_CORR_ADDR]            =  0x74,
 242         [BRCMNAND_UNCORR_EXT_ADDR]      =  0x78,
 243         [BRCMNAND_UNCORR_ADDR]          =  0x7c,
 244         [BRCMNAND_SEMAPHORE]            =  0x58,
 245         [BRCMNAND_ID]                   =  0x60,
 246         [BRCMNAND_ID_EXT]               =  0x64,
 247         [BRCMNAND_LL_RDATA]             = 0x17c,
 248         [BRCMNAND_OOB_READ_BASE]        =  0x20,
 249         [BRCMNAND_OOB_READ_10_BASE]     = 0x130,
 250         [BRCMNAND_OOB_WRITE_BASE]       =  0x30,
 251         [BRCMNAND_OOB_WRITE_10_BASE]    =     0,
 252         [BRCMNAND_FC_BASE]              = 0x200,
 253 };
 254
 255 /* BRCMNAND v5.0 */
 256 static const u16 brcmnand_regs_v50[] = {
 257         [BRCMNAND_CMD_START]            =  0x04,
 258         [BRCMNAND_CMD_EXT_ADDRESS]      =  0x08,
 259         [BRCMNAND_CMD_ADDRESS]          =  0x0c,
 260         [BRCMNAND_INTFC_STATUS]         =  0x6c,
 261         [BRCMNAND_CS_SELECT]            =  0x14,
 262         [BRCMNAND_CS_XOR]               =  0x18,
 263         [BRCMNAND_LL_OP]                = 0x178,
 264         [BRCMNAND_CS0_BASE]             =  0x40,
 265         [BRCMNAND_CS1_BASE]             =  0xd0,
 266         [BRCMNAND_CORR_THRESHOLD]       =  0x84,
 267         [BRCMNAND_CORR_THRESHOLD_EXT]   =     0,
 268         [BRCMNAND_UNCORR_COUNT]         =     0,
 269         [BRCMNAND_CORR_COUNT]           =     0,
 270         [BRCMNAND_CORR_EXT_ADDR]        =  0x70,
 271         [BRCMNAND_CORR_ADDR]            =  0x74,
 272         [BRCMNAND_UNCORR_EXT_ADDR]      =  0x78,
 273         [BRCMNAND_UNCORR_ADDR]          =  0x7c,
 274         [BRCMNAND_SEMAPHORE]            =  0x58,
 275         [BRCMNAND_ID]                   =  0x60,
 276         [BRCMNAND_ID_EXT]               =  0x64,
 277         [BRCMNAND_LL_RDATA]             = 0x17c,
 278         [BRCMNAND_OOB_READ_BASE]        =  0x20,
 279         [BRCMNAND_OOB_READ_10_BASE]     = 0x130,
 280         [BRCMNAND_OOB_WRITE_BASE]       =  0x30,
 281         [BRCMNAND_OOB_WRITE_10_BASE]    = 0x140,
 282         [BRCMNAND_FC_BASE]              = 0x200,
 283 };
 284
 285 /* BRCMNAND v6.0 - v7.1 */
 286 static const u16 brcmnand_regs_v60[] = {
 287         [BRCMNAND_CMD_START]            =  0x04,
 288         [BRCMNAND_CMD_EXT_ADDRESS]      =  0x08,
 289         [BRCMNAND_CMD_ADDRESS]          =  0x0c,
 290         [BRCMNAND_INTFC_STATUS]         =  0x14,
 291         [BRCMNAND_CS_SELECT]            =  0x18,
 292         [BRCMNAND_CS_XOR]               =  0x1c,
 293         [BRCMNAND_LL_OP]                =  0x20,
 294         [BRCMNAND_CS0_BASE]             =  0x50,
 295         [BRCMNAND_CS1_BASE]             =     0,
 296         [BRCMNAND_CORR_THRESHOLD]       =  0xc0,
 297         [BRCMNAND_CORR_THRESHOLD_EXT]   =  0xc4,
 298         [BRCMNAND_UNCORR_COUNT]         =  0xfc,
 299         [BRCMNAND_CORR_COUNT]           = 0x100,
 300         [BRCMNAND_CORR_EXT_ADDR]        = 0x10c,
 301         [BRCMNAND_CORR_ADDR]            = 0x110,
 302         [BRCMNAND_UNCORR_EXT_ADDR]      = 0x114,
 303         [BRCMNAND_UNCORR_ADDR]          = 0x118,
 304         [BRCMNAND_SEMAPHORE]            = 0x150,
 305         [BRCMNAND_ID]                   = 0x194,
 306         [BRCMNAND_ID_EXT]               = 0x198,
 307         [BRCMNAND_LL_RDATA]             = 0x19c,
 308         [BRCMNAND_OOB_READ_BASE]        = 0x200,
 309         [BRCMNAND_OOB_READ_10_BASE]     =     0,
 310         [BRCMNAND_OOB_WRITE_BASE]       = 0x280,
 311         [BRCMNAND_OOB_WRITE_10_BASE]    =     0,
 312         [BRCMNAND_FC_BASE]              = 0x400,
 313 };
 314
 315 /* BRCMNAND v7.1 */
 316 static const u16 brcmnand_regs_v71[] = {
 317         [BRCMNAND_CMD_START]            =  0x04,
 318         [BRCMNAND_CMD_EXT_ADDRESS]      =  0x08,
 319         [BRCMNAND_CMD_ADDRESS]          =  0x0c,
 320         [BRCMNAND_INTFC_STATUS]         =  0x14,
 321         [BRCMNAND_CS_SELECT]            =  0x18,
 322         [BRCMNAND_CS_XOR]               =  0x1c,
 323         [BRCMNAND_LL_OP]                =  0x20,
 324         [BRCMNAND_CS0_BASE]             =  0x50,
 325         [BRCMNAND_CS1_BASE]             =     0,
 326         [BRCMNAND_CORR_THRESHOLD]       =  0xdc,
 327         [BRCMNAND_CORR_THRESHOLD_EXT]   =  0xe0,
 328         [BRCMNAND_UNCORR_COUNT]         =  0xfc,
 329         [BRCMNAND_CORR_COUNT]           = 0x100,
 330         [BRCMNAND_CORR_EXT_ADDR]        = 0x10c,
 331         [BRCMNAND_CORR_ADDR]            = 0x110,
 332         [BRCMNAND_UNCORR_EXT_ADDR]      = 0x114,
 333         [BRCMNAND_UNCORR_ADDR]          = 0x118,
 334         [BRCMNAND_SEMAPHORE]            = 0x150,
 335         [BRCMNAND_ID]                   = 0x194,
 336         [BRCMNAND_ID_EXT]               = 0x198,
 337         [BRCMNAND_LL_RDATA]             = 0x19c,
 338         [BRCMNAND_OOB_READ_BASE]        = 0x200,
 339         [BRCMNAND_OOB_READ_10_BASE]     =     0,
 340         [BRCMNAND_OOB_WRITE_BASE]       = 0x280,
 341         [BRCMNAND_OOB_WRITE_10_BASE]    =     0,
 342         [BRCMNAND_FC_BASE]              = 0x400,
 343 };
 344
 345 /* BRCMNAND v7.2 */
 346 static const u16 brcmnand_regs_v72[] = {
 347         [BRCMNAND_CMD_START]            =  0x04,
 348         [BRCMNAND_CMD_EXT_ADDRESS]      =  0x08,
 349         [BRCMNAND_CMD_ADDRESS]          =  0x0c,
 350         [BRCMNAND_INTFC_STATUS]         =  0x14,
 351         [BRCMNAND_CS_SELECT]            =  0x18,
 352         [BRCMNAND_CS_XOR]               =  0x1c,
 353         [BRCMNAND_LL_OP]                =  0x20,
 354         [BRCMNAND_CS0_BASE]             =  0x50,
 355         [BRCMNAND_CS1_BASE]             =     0,
 356         [BRCMNAND_CORR_THRESHOLD]       =  0xdc,
 357         [BRCMNAND_CORR_THRESHOLD_EXT]   =  0xe0,
 358         [BRCMNAND_UNCORR_COUNT]         =  0xfc,
 359         [BRCMNAND_CORR_COUNT]           = 0x100,
 360         [BRCMNAND_CORR_EXT_ADDR]        = 0x10c,
 361         [BRCMNAND_CORR_ADDR]            = 0x110,
 362         [BRCMNAND_UNCORR_EXT_ADDR]      = 0x114,
 363         [BRCMNAND_UNCORR_ADDR]          = 0x118,
 364         [BRCMNAND_SEMAPHORE]            = 0x150,
 365         [BRCMNAND_ID]                   = 0x194,
 366         [BRCMNAND_ID_EXT]               = 0x198,
 367         [BRCMNAND_LL_RDATA]             = 0x19c,
 368         [BRCMNAND_OOB_READ_BASE]        = 0x200,
 369         [BRCMNAND_OOB_READ_10_BASE]     =     0,
 370         [BRCMNAND_OOB_WRITE_BASE]       = 0x400,
 371         [BRCMNAND_OOB_WRITE_10_BASE]    =     0,
 372         [BRCMNAND_FC_BASE]              = 0x600,
 373 };
 374
 375 enum brcmnand_cs_reg {
 376         BRCMNAND_CS_CFG_EXT = 0,
 377         BRCMNAND_CS_CFG,
 378         BRCMNAND_CS_ACC_CONTROL,
 379         BRCMNAND_CS_TIMING1,
 380         BRCMNAND_CS_TIMING2,
 381 };
 382
 383 /* Per chip-select offsets for v7.1 */
 384 static const u8 brcmnand_cs_offsets_v71[] = {
 385         [BRCMNAND_CS_ACC_CONTROL]       = 0x00,
 386         [BRCMNAND_CS_CFG_EXT]           = 0x04,
 387         [BRCMNAND_CS_CFG]               = 0x08,
 388         [BRCMNAND_CS_TIMING1]           = 0x0c,
 389         [BRCMNAND_CS_TIMING2]           = 0x10,
 390 };
 391
 392 /* Per chip-select offsets for pre v7.1, except CS0 on <= v5.0 */
 393 static const u8 brcmnand_cs_offsets[] = {
 394         [BRCMNAND_CS_ACC_CONTROL]       = 0x00,
 395         [BRCMNAND_CS_CFG_EXT]           = 0x04,
 396         [BRCMNAND_CS_CFG]               = 0x04,
 397         [BRCMNAND_CS_TIMING1]           = 0x08,
 398         [BRCMNAND_CS_TIMING2]           = 0x0c,
 399 };
 400
 401 /* Per chip-select offset for <= v5.0 on CS0 only */
 402 static const u8 brcmnand_cs_offsets_cs0[] = {
 403         [BRCMNAND_CS_ACC_CONTROL]       = 0x00,
 404         [BRCMNAND_CS_CFG_EXT]           = 0x08,
 405         [BRCMNAND_CS_CFG]               = 0x08,
 406         [BRCMNAND_CS_TIMING1]           = 0x10,
 407         [BRCMNAND_CS_TIMING2]           = 0x14,
 408 };
 409
 410 /*
 411  * Bitfields for the CFG and CFG_EXT registers. Pre-v7.1 controllers only had
 412  * one config register, but once the bitfields overflowed, newer controllers
 413  * (v7.1 and newer) added a CFG_EXT register and shuffled a few fields around.
 414  */
 415 enum {
 416         CFG_BLK_ADR_BYTES_SHIFT         = 8,
 417         CFG_COL_ADR_BYTES_SHIFT         = 12,
 418         CFG_FUL_ADR_BYTES_SHIFT         = 16,
 419         CFG_BUS_WIDTH_SHIFT             = 23,
 420         CFG_BUS_WIDTH                   = BIT(CFG_BUS_WIDTH_SHIFT),
 421         CFG_DEVICE_SIZE_SHIFT           = 24,
 422
 423         /* Only for pre-v7.1 (with no CFG_EXT register) */
 424         CFG_PAGE_SIZE_SHIFT             = 20,
 425         CFG_BLK_SIZE_SHIFT              = 28,
 426
 427         /* Only for v7.1+ (with CFG_EXT register) */
 428         CFG_EXT_PAGE_SIZE_SHIFT         = 0,
 429         CFG_EXT_BLK_SIZE_SHIFT          = 4,
 430 };
 431
 432 /* BRCMNAND_INTFC_STATUS */
 433 enum {
 434         INTFC_FLASH_STATUS              = GENMASK(7, 0),
 435
 436         INTFC_ERASED                    = BIT(27),
 437         INTFC_OOB_VALID                 = BIT(28),
 438         INTFC_CACHE_VALID               = BIT(29),
 439         INTFC_FLASH_READY               = BIT(30),
 440         INTFC_CTLR_READY                = BIT(31),
 441 };
 442
 443 static inline u32 nand_readreg(struct brcmnand_controller *ctrl, u32 offs)
 444 {
 445         return brcmnand_readl(ctrl->nand_base + offs);
 446 }
 447
 448 static inline void nand_writereg(struct brcmnand_controller *ctrl, u32 offs,
 449                                  u32 val)
 450 {
 451         brcmnand_writel(val, ctrl->nand_base + offs);
 452 }
 453
 454 static int brcmnand_revision_init(struct brcmnand_controller *ctrl)
 455 {
 456         static const unsigned int block_sizes_v6[] = { 8, 16, 128, 256, 512, 1024, 2048, 0 };
 457         static const unsigned int block_sizes_v4[] = { 16, 128, 8, 512, 256, 1024, 2048, 0 };
 458         static const unsigned int page_sizes[] = { 512, 2048, 4096, 8192, 0 };
 459
 460         ctrl->nand_version = nand_readreg(ctrl, 0) & 0xffff;
 461
 462         /* Only support v4.0+? */
 463         if (ctrl->nand_version < 0x0400) {
 464                 dev_err(ctrl->dev, "version %#x not supported\n",
 465                         ctrl->nand_version);
 466                 return -ENODEV;
 467         }
 468
 469         /* Register offsets */
 470         if (ctrl->nand_version >= 0x0702)
 471                 ctrl->reg_offsets = brcmnand_regs_v72;
 472         else if (ctrl->nand_version >= 0x0701)
 473                 ctrl->reg_offsets = brcmnand_regs_v71;
 474         else if (ctrl->nand_version >= 0x0600)
 475                 ctrl->reg_offsets = brcmnand_regs_v60;
 476         else if (ctrl->nand_version >= 0x0500)
 477                 ctrl->reg_offsets = brcmnand_regs_v50;
 478         else if (ctrl->nand_version >= 0x0400)
 479                 ctrl->reg_offsets = brcmnand_regs_v40;
 480
 481         /* Chip-select stride */
 482         if (ctrl->nand_version >= 0x0701)
 483                 ctrl->reg_spacing = 0x14;
 484         else
 485                 ctrl->reg_spacing = 0x10;
 486
 487         /* Per chip-select registers */
 488         if (ctrl->nand_version >= 0x0701) {
 489                 ctrl->cs_offsets = brcmnand_cs_offsets_v71;
 490         } else {
 491                 ctrl->cs_offsets = brcmnand_cs_offsets;
 492
 493                 /* v5.0 and earlier has a different CS0 offset layout */
 494                 if (ctrl->nand_version <= 0x0500)
 495                         ctrl->cs0_offsets = brcmnand_cs_offsets_cs0;
 496         }
 497
 498         /* Page / block sizes */
 499         if (ctrl->nand_version >= 0x0701) {
 500                 /* >= v7.1 use nice power-of-2 values! */
 501                 ctrl->max_page_size = 16 * 1024;
 502                 ctrl->max_block_size = 2 * 1024 * 1024;
 503         } else {
 504                 ctrl->page_sizes = page_sizes;
 505                 if (ctrl->nand_version >= 0x0600)
 506                         ctrl->block_sizes = block_sizes_v6;
 507                 else
 508                         ctrl->block_sizes = block_sizes_v4;
 509
 510                 if (ctrl->nand_version < 0x0400) {
 511                         ctrl->max_page_size = 4096;
 512                         ctrl->max_block_size = 512 * 1024;
 513                 }
 514         }
 515
 516         /* Maximum spare area sector size (per 512B) */
 517         if (ctrl->nand_version >= 0x0702)
 518                 ctrl->max_oob = 128;
 519         else if (ctrl->nand_version >= 0x0600)
 520                 ctrl->max_oob = 64;
 521         else if (ctrl->nand_version >= 0x0500)
 522                 ctrl->max_oob = 32;
 523         else
 524                 ctrl->max_oob = 16;
 525
 526         /* v6.0 and newer (except v6.1) have prefetch support */
 527         if (ctrl->nand_version >= 0x0600 && ctrl->nand_version != 0x0601)
 528                 ctrl->features |= BRCMNAND_HAS_PREFETCH;
 529
 530         /*
 531          * v6.x has cache mode, but it's implemented differently. Ignore it for
 532          * now.
 533          */
 534         if (ctrl->nand_version >= 0x0700)
 535                 ctrl->features |= BRCMNAND_HAS_CACHE_MODE;
 536
 537         if (ctrl->nand_version >= 0x0500)
 538                 ctrl->features |= BRCMNAND_HAS_1K_SECTORS;
 539
 540         if (ctrl->nand_version >= 0x0700)
 541                 ctrl->features |= BRCMNAND_HAS_WP;
 542 #ifndef __UBOOT__
 543         else if (of_property_read_bool(ctrl->dev->of_node, "brcm,nand-has-wp"))
 544 #else
 545         else if (dev_read_bool(ctrl->dev, "brcm,nand-has-wp"))
 546 #endif /* __UBOOT__ */
 547                 ctrl->features |= BRCMNAND_HAS_WP;
 548
 549         return 0;
 550 }
 551
 552 static inline u32 brcmnand_read_reg(struct brcmnand_controller *ctrl,
 553                 enum brcmnand_reg reg)
 554 {
 555         u16 offs = ctrl->reg_offsets[reg];
 556
 557         if (offs)
 558                 return nand_readreg(ctrl, offs);
 559         else
 560                 return 0;
 561 }
 562
 563 static inline void brcmnand_write_reg(struct brcmnand_controller *ctrl,
 564                                       enum brcmnand_reg reg, u32 val)
 565 {
 566         u16 offs = ctrl->reg_offsets[reg];
 567
 568         if (offs)
 569                 nand_writereg(ctrl, offs, val);
 570 }
 571
 572 static inline void brcmnand_rmw_reg(struct brcmnand_controller *ctrl,
 573                                     enum brcmnand_reg reg, u32 mask, unsigned
 574                                     int shift, u32 val)
 575 {
 576         u32 tmp = brcmnand_read_reg(ctrl, reg);
 577
 578         tmp &= ~mask;
 579         tmp |= val << shift;
 580         brcmnand_write_reg(ctrl, reg, tmp);
 581 }
 582
 583 static inline u32 brcmnand_read_fc(struct brcmnand_controller *ctrl, int word)
 584 {
 585         return __raw_readl(ctrl->nand_fc + word * 4);
 586 }
 587
 588 static inline void brcmnand_write_fc(struct brcmnand_controller *ctrl,
 589                                      int word, u32 val)
 590 {
 591         __raw_writel(val, ctrl->nand_fc + word * 4);
 592 }
 593
 594 static inline u16 brcmnand_cs_offset(struct brcmnand_controller *ctrl, int cs,
 595                                      enum brcmnand_cs_reg reg)
 596 {
 597         u16 offs_cs0 = ctrl->reg_offsets[BRCMNAND_CS0_BASE];
 598         u16 offs_cs1 = ctrl->reg_offsets[BRCMNAND_CS1_BASE];
 599         u8 cs_offs;
 600
 601         if (cs == 0 && ctrl->cs0_offsets)
 602                 cs_offs = ctrl->cs0_offsets[reg];
 603         else
 604                 cs_offs = ctrl->cs_offsets[reg];
 605
 606         if (cs && offs_cs1)
 607                 return offs_cs1 + (cs - 1) * ctrl->reg_spacing + cs_offs;
 608
 609         return offs_cs0 + cs * ctrl->reg_spacing + cs_offs;
 610 }
 611
 612 static inline u32 brcmnand_count_corrected(struct brcmnand_controller *ctrl)
 613 {
 614         if (ctrl->nand_version < 0x0600)
 615                 return 1;
 616         return brcmnand_read_reg(ctrl, BRCMNAND_CORR_COUNT);
 617 }
 618
 619 static void brcmnand_wr_corr_thresh(struct brcmnand_host *host, u8 val)
 620 {
 621         struct brcmnand_controller *ctrl = host->ctrl;
 622         unsigned int shift = 0, bits;
 623         enum brcmnand_reg reg = BRCMNAND_CORR_THRESHOLD;
 624         int cs = host->cs;
 625
 626         if (ctrl->nand_version >= 0x0702)
 627                 bits = 7;
 628         else if (ctrl->nand_version >= 0x0600)
 629                 bits = 6;
 630         else if (ctrl->nand_version >= 0x0500)
 631                 bits = 5;
 632         else
 633                 bits = 4;
 634
 635         if (ctrl->nand_version >= 0x0702) {
 636                 if (cs >= 4)
 637                         reg = BRCMNAND_CORR_THRESHOLD_EXT;
 638                 shift = (cs % 4) * bits;
 639         } else if (ctrl->nand_version >= 0x0600) {
 640                 if (cs >= 5)
 641                         reg = BRCMNAND_CORR_THRESHOLD_EXT;
 642                 shift = (cs % 5) * bits;
 643         }
 644         brcmnand_rmw_reg(ctrl, reg, (bits - 1) << shift, shift, val);
 645 }
 646
 647 static inline int brcmnand_cmd_shift(struct brcmnand_controller *ctrl)
 648 {
 649         if (ctrl->nand_version < 0x0602)
 650                 return 24;
 651         return 0;
 652 }
 653
 654 /***********************************************************************
 655  * NAND ACC CONTROL bitfield
 656  *
 657  * Some bits have remained constant throughout hardware revision, while
 658  * others have shifted around.
 659  ***********************************************************************/
 660
 661 /* Constant for all versions (where supported) */
 662 enum {
 663         /* See BRCMNAND_HAS_CACHE_MODE */
 664         ACC_CONTROL_CACHE_MODE                          = BIT(22),
 665
 666         /* See BRCMNAND_HAS_PREFETCH */
 667         ACC_CONTROL_PREFETCH                            = BIT(23),
 668
 669         ACC_CONTROL_PAGE_HIT                            = BIT(24),
 670         ACC_CONTROL_WR_PREEMPT                          = BIT(25),
 671         ACC_CONTROL_PARTIAL_PAGE                        = BIT(26),
 672         ACC_CONTROL_RD_ERASED                           = BIT(27),
 673         ACC_CONTROL_FAST_PGM_RDIN                       = BIT(28),
 674         ACC_CONTROL_WR_ECC                              = BIT(30),
 675         ACC_CONTROL_RD_ECC                              = BIT(31),
 676 };
 677
 678 static inline u32 brcmnand_spare_area_mask(struct brcmnand_controller *ctrl)
 679 {
 680         if (ctrl->nand_version >= 0x0702)
 681                 return GENMASK(7, 0);
 682         else if (ctrl->nand_version >= 0x0600)
 683                 return GENMASK(6, 0);
 684         else
 685                 return GENMASK(5, 0);
 686 }
 687
 688 #define NAND_ACC_CONTROL_ECC_SHIFT      16
 689 #define NAND_ACC_CONTROL_ECC_EXT_SHIFT  13
 690
 691 static inline u32 brcmnand_ecc_level_mask(struct brcmnand_controller *ctrl)
 692 {
 693         u32 mask = (ctrl->nand_version >= 0x0600) ? 0x1f : 0x0f;
 694
 695         mask <<= NAND_ACC_CONTROL_ECC_SHIFT;
 696
 697         /* v7.2 includes additional ECC levels */
 698         if (ctrl->nand_version >= 0x0702)
 699                 mask |= 0x7 << NAND_ACC_CONTROL_ECC_EXT_SHIFT;
 700
 701         return mask;
 702 }
 703
 704 static void brcmnand_set_ecc_enabled(struct brcmnand_host *host, int en)
 705 {
 706         struct brcmnand_controller *ctrl = host->ctrl;
 707         u16 offs = brcmnand_cs_offset(ctrl, host->cs, BRCMNAND_CS_ACC_CONTROL);
 708         u32 acc_control = nand_readreg(ctrl, offs);
 709         u32 ecc_flags = ACC_CONTROL_WR_ECC | ACC_CONTROL_RD_ECC;
 710
 711         if (en) {
 712                 acc_control |= ecc_flags; /* enable RD/WR ECC */
 713                 acc_control |= host->hwcfg.ecc_level
 714                                << NAND_ACC_CONTROL_ECC_SHIFT;
 715         } else {
 716                 acc_control &= ~ecc_flags; /* disable RD/WR ECC */
 717                 acc_control &= ~brcmnand_ecc_level_mask(ctrl);
 718         }
 719
 720         nand_writereg(ctrl, offs, acc_control);
 721 }
 722
 723 static inline int brcmnand_sector_1k_shift(struct brcmnand_controller *ctrl)
 724 {
 725         if (ctrl->nand_version >= 0x0702)
 726                 return 9;
 727         else if (ctrl->nand_version >= 0x0600)
 728                 return 7;
 729         else if (ctrl->nand_version >= 0x0500)
 730                 return 6;
 731         else
 732                 return -1;
 733 }
 734
 735 static int brcmnand_get_sector_size_1k(struct brcmnand_host *host)
 736 {
 737         struct brcmnand_controller *ctrl = host->ctrl;
 738         int shift = brcmnand_sector_1k_shift(ctrl);
 739         u16 acc_control_offs = brcmnand_cs_offset(ctrl, host->cs,
 740                                                   BRCMNAND_CS_ACC_CONTROL);
 741
 742         if (shift < 0)
 743                 return 0;
 744
 745         return (nand_readreg(ctrl, acc_control_offs) >> shift) & 0x1;
 746 }
 747
 748 static void brcmnand_set_sector_size_1k(struct brcmnand_host *host, int val)
 749 {
 750         struct brcmnand_controller *ctrl = host->ctrl;
 751         int shift = brcmnand_sector_1k_shift(ctrl);
 752         u16 acc_control_offs = brcmnand_cs_offset(ctrl, host->cs,
 753                                                   BRCMNAND_CS_ACC_CONTROL);
 754         u32 tmp;
 755
 756         if (shift < 0)
 757                 return;
 758
 759         tmp = nand_readreg(ctrl, acc_control_offs);
 760         tmp &= ~(1 << shift);
 761         tmp |= (!!val) << shift;
 762         nand_writereg(ctrl, acc_control_offs, tmp);
 763 }
 764
 765 /***********************************************************************
 766  * CS_NAND_SELECT
 767  ***********************************************************************/
 768
 769 enum {
 770         CS_SELECT_NAND_WP                       = BIT(29),
 771         CS_SELECT_AUTO_DEVICE_ID_CFG            = BIT(30),
 772 };
 773
 774 static int bcmnand_ctrl_poll_status(struct brcmnand_controller *ctrl,
 775                                     u32 mask, u32 expected_val,
 776                                     unsigned long timeout_ms)
 777 {
 778 #ifndef __UBOOT__
 779         unsigned long limit;
 780         u32 val;
 781
 782         if (!timeout_ms)
 783                 timeout_ms = NAND_POLL_STATUS_TIMEOUT_MS;
 784
 785         limit = jiffies + msecs_to_jiffies(timeout_ms);
 786         do {
 787                 val = brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS);
 788                 if ((val & mask) == expected_val)
 789                         return 0;
 790
 791                 cpu_relax();
 792         } while (time_after(limit, jiffies));
 793 #else
 794         unsigned long base, limit;
 795         u32 val;
 796
 797         if (!timeout_ms)
 798                 timeout_ms = NAND_POLL_STATUS_TIMEOUT_MS;
 799
 800         base = get_timer(0);
 801         limit = CONFIG_SYS_HZ * timeout_ms / 1000;
 802         do {
 803                 val = brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS);
 804                 if ((val & mask) == expected_val)
 805                         return 0;
 806
 807                 cpu_relax();
 808         } while (get_timer(base) < limit);
 809 #endif /* __UBOOT__ */
 810
 811         dev_warn(ctrl->dev, "timeout on status poll (expected %x got %x)\n",
 812                  expected_val, val & mask);
 813
 814         return -ETIMEDOUT;
 815 }
 816
 817 static inline void brcmnand_set_wp(struct brcmnand_controller *ctrl, bool en)
 818 {
 819         u32 val = en ? CS_SELECT_NAND_WP : 0;
 820
 821         brcmnand_rmw_reg(ctrl, BRCMNAND_CS_SELECT, CS_SELECT_NAND_WP, 0, val);
 822 }
 823
 824 /***********************************************************************
 825  * Flash DMA
 826  ***********************************************************************/
 827
 828 enum flash_dma_reg {
 829         FLASH_DMA_REVISION              = 0x00,
 830         FLASH_DMA_FIRST_DESC            = 0x04,
 831         FLASH_DMA_FIRST_DESC_EXT        = 0x08,
 832         FLASH_DMA_CTRL                  = 0x0c,
 833         FLASH_DMA_MODE                  = 0x10,
 834         FLASH_DMA_STATUS                = 0x14,
 835         FLASH_DMA_INTERRUPT_DESC        = 0x18,
 836         FLASH_DMA_INTERRUPT_DESC_EXT    = 0x1c,
 837         FLASH_DMA_ERROR_STATUS          = 0x20,
 838         FLASH_DMA_CURRENT_DESC          = 0x24,
 839         FLASH_DMA_CURRENT_DESC_EXT      = 0x28,
 840 };
 841
 842 static inline bool has_flash_dma(struct brcmnand_controller *ctrl)
 843 {
 844         return ctrl->flash_dma_base;
 845 }
 846
 847 static inline bool flash_dma_buf_ok(const void *buf)
 848 {
 849 #ifndef __UBOOT__
 850         return buf && !is_vmalloc_addr(buf) &&
 851                 likely(IS_ALIGNED((uintptr_t)buf, 4));
 852 #else
 853         return buf && likely(IS_ALIGNED((uintptr_t)buf, 4));
 854 #endif /* __UBOOT__ */
 855 }
 856
 857 static inline void flash_dma_writel(struct brcmnand_controller *ctrl, u8 offs,
 858                                     u32 val)
 859 {
 860         brcmnand_writel(val, ctrl->flash_dma_base + offs);
 861 }
 862
 863 static inline u32 flash_dma_readl(struct brcmnand_controller *ctrl, u8 offs)
 864 {
 865         return brcmnand_readl(ctrl->flash_dma_base + offs);
 866 }
 867
 868 /* Low-level operation types: command, address, write, or read */
 869 enum brcmnand_llop_type {
 870         LL_OP_CMD,
 871         LL_OP_ADDR,
 872         LL_OP_WR,
 873         LL_OP_RD,
 874 };
 875
 876 /***********************************************************************
 877  * Internal support functions
 878  ***********************************************************************/
 879
 880 static inline bool is_hamming_ecc(struct brcmnand_controller *ctrl,
 881                                   struct brcmnand_cfg *cfg)
 882 {
 883         if (ctrl->nand_version <= 0x0701)
 884                 return cfg->sector_size_1k == 0 && cfg->spare_area_size == 16 &&
 885                         cfg->ecc_level == 15;
 886         else
 887                 return cfg->sector_size_1k == 0 && ((cfg->spare_area_size == 16 &&
 888                         cfg->ecc_level == 15) ||
 889                         (cfg->spare_area_size == 28 && cfg->ecc_level == 16));
 890 }
 891
 892 /*
 893  * Returns a nand_ecclayout strucutre for the given layout/configuration.
 894  * Returns NULL on failure.
 895  */
 896 static struct nand_ecclayout *brcmnand_create_layout(int ecc_level,
 897                                                      struct brcmnand_host *host)
 898 {
 899         struct brcmnand_cfg *cfg = &host->hwcfg;
 900         int i, j;
 901         struct nand_ecclayout *layout;
 902         int req;
 903         int sectors;
 904         int sas;
 905         int idx1, idx2;
 906
 907 #ifndef __UBOOT__
 908         layout = devm_kzalloc(&host->pdev->dev, sizeof(*layout), GFP_KERNEL);
 909 #else
 910         layout = devm_kzalloc(host->pdev, sizeof(*layout), GFP_KERNEL);
 911 #endif
 912         if (!layout)
 913                 return NULL;
 914
 915         sectors = cfg->page_size / (512 << cfg->sector_size_1k);
 916         sas = cfg->spare_area_size << cfg->sector_size_1k;
 917
 918         /* Hamming */
 919         if (is_hamming_ecc(host->ctrl, cfg)) {
 920                 for (i = 0, idx1 = 0, idx2 = 0; i < sectors; i++) {
 921                         /* First sector of each page may have BBI */
 922                         if (i == 0) {
 923                                 layout->oobfree[idx2].offset = i * sas + 1;
 924                                 /* Small-page NAND use byte 6 for BBI */
 925                                 if (cfg->page_size == 512)
 926                                         layout->oobfree[idx2].offset--;
 927                                 layout->oobfree[idx2].length = 5;
 928                         } else {
 929                                 layout->oobfree[idx2].offset = i * sas;
 930                                 layout->oobfree[idx2].length = 6;
 931                         }
 932                         idx2++;
 933                         layout->eccpos[idx1++] = i * sas + 6;
 934                         layout->eccpos[idx1++] = i * sas + 7;
 935                         layout->eccpos[idx1++] = i * sas + 8;
 936                         layout->oobfree[idx2].offset = i * sas + 9;
 937                         layout->oobfree[idx2].length = 7;
 938                         idx2++;
 939                         /* Leave zero-terminated entry for OOBFREE */
 940                         if (idx1 >= MTD_MAX_ECCPOS_ENTRIES_LARGE ||
 941                             idx2 >= MTD_MAX_OOBFREE_ENTRIES_LARGE - 1)
 942                                 break;
 943                 }
 944
 945                 return layout;
 946         }
 947
 948         /*
 949          * CONTROLLER_VERSION:
 950          *   < v5.0: ECC_REQ = ceil(BCH_T * 13/8)
 951          *  >= v5.0: ECC_REQ = ceil(BCH_T * 14/8)
 952          * But we will just be conservative.
 953          */
 954         req = DIV_ROUND_UP(ecc_level * 14, 8);
 955         if (req >= sas) {
 956                 dev_err(&host->pdev->dev,
 957                         "error: ECC too large for OOB (ECC bytes %d, spare sector %d)\n",
 958                         req, sas);
 959                 return NULL;
 960         }
 961
 962         layout->eccbytes = req * sectors;
 963         for (i = 0, idx1 = 0, idx2 = 0; i < sectors; i++) {
 964                 for (j = sas - req; j < sas && idx1 <
 965                                 MTD_MAX_ECCPOS_ENTRIES_LARGE; j++, idx1++)
 966                         layout->eccpos[idx1] = i * sas + j;
 967
 968                 /* First sector of each page may have BBI */
 969                 if (i == 0) {
 970                         if (cfg->page_size == 512 && (sas - req >= 6)) {
 971                                 /* Small-page NAND use byte 6 for BBI */
 972                                 layout->oobfree[idx2].offset = 0;
 973                                 layout->oobfree[idx2].length = 5;
 974                                 idx2++;
 975                                 if (sas - req > 6) {
 976                                         layout->oobfree[idx2].offset = 6;
 977                                         layout->oobfree[idx2].length =
 978                                                 sas - req - 6;
 979                                         idx2++;
 980                                 }
 981                         } else if (sas > req + 1) {
 982                                 layout->oobfree[idx2].offset = i * sas + 1;
 983                                 layout->oobfree[idx2].length = sas - req - 1;
 984                                 idx2++;
 985                         }
 986                 } else if (sas > req) {
 987                         layout->oobfree[idx2].offset = i * sas;
 988                         layout->oobfree[idx2].length = sas - req;
 989                         idx2++;
 990                 }
 991                 /* Leave zero-terminated entry for OOBFREE */
 992                 if (idx1 >= MTD_MAX_ECCPOS_ENTRIES_LARGE ||
 993                     idx2 >= MTD_MAX_OOBFREE_ENTRIES_LARGE - 1)
 994                         break;
 995         }
 996
 997         return layout;
 998 }
 999
1000 static struct nand_ecclayout *brcmstb_choose_ecc_layout(
1001                 struct brcmnand_host *host)
1002 {
1003         struct nand_ecclayout *layout;
1004         struct brcmnand_cfg *p = &host->hwcfg;
1005         unsigned int ecc_level = p->ecc_level;
1006
1007         if (p->sector_size_1k)
1008                 ecc_level <<= 1;
1009
1010         layout = brcmnand_create_layout(ecc_level, host);
1011         if (!layout) {
1012                 dev_err(&host->pdev->dev,
1013                                 "no proper ecc_layout for this NAND cfg\n");
1014                 return NULL;
1015         }
1016
1017         return layout;
1018 }
1019
1020 static void brcmnand_wp(struct mtd_info *mtd, int wp)
1021 {
1022         struct nand_chip *chip = mtd_to_nand(mtd);
1023         struct brcmnand_host *host = nand_get_controller_data(chip);
1024         struct brcmnand_controller *ctrl = host->ctrl;
1025
1026         if ((ctrl->features & BRCMNAND_HAS_WP) && wp_on == 1) {
1027                 static int old_wp = -1;
1028                 int ret;
1029
1030                 if (old_wp != wp) {
1031                         dev_dbg(ctrl->dev, "WP %s\n", wp ? "on" : "off");
1032                         old_wp = wp;
1033                 }
1034
1035                 /*
1036                  * make sure ctrl/flash ready before and after
1037                  * changing state of #WP pin
1038                  */
1039                 ret = bcmnand_ctrl_poll_status(ctrl, NAND_CTRL_RDY |
1040                                                NAND_STATUS_READY,
1041                                                NAND_CTRL_RDY |
1042                                                NAND_STATUS_READY, 0);
1043                 if (ret)
1044                         return;
1045
1046                 brcmnand_set_wp(ctrl, wp);
1047                 nand_status_op(chip, NULL);
1048                 /* NAND_STATUS_WP 0x00 = protected, 0x80 = not protected */
1049                 ret = bcmnand_ctrl_poll_status(ctrl,
1050                                                NAND_CTRL_RDY |
1051                                                NAND_STATUS_READY |
1052                                                NAND_STATUS_WP,
1053                                                NAND_CTRL_RDY |
1054                                                NAND_STATUS_READY |
1055                                                (wp ? 0 : NAND_STATUS_WP), 0);
1056 #ifndef __UBOOT__
1057                 if (ret)
1058                         dev_err_ratelimited(&host->pdev->dev,
1059                                             "nand #WP expected %s\n",
1060                                             wp ? "on" : "off");
1061 #else
1062                 if (ret)
1063                         dev_err(&host->pdev->dev,
1064                                             "nand #WP expected %s\n",
1065                                             wp ? "on" : "off");
1066 #endif /* __UBOOT__ */
1067         }
1068 }
1069
1070 /* Helper functions for reading and writing OOB registers */
1071 static inline u8 oob_reg_read(struct brcmnand_controller *ctrl, u32 offs)
1072 {
1073         u16 offset0, offset10, reg_offs;
1074
1075         offset0 = ctrl->reg_offsets[BRCMNAND_OOB_READ_BASE];
1076         offset10 = ctrl->reg_offsets[BRCMNAND_OOB_READ_10_BASE];
1077
1078         if (offs >= ctrl->max_oob)
1079                 return 0x77;
1080
1081         if (offs >= 16 && offset10)
1082                 reg_offs = offset10 + ((offs - 0x10) & ~0x03);
1083         else
1084                 reg_offs = offset0 + (offs & ~0x03);
1085
1086         return nand_readreg(ctrl, reg_offs) >> (24 - ((offs & 0x03) << 3));
1087 }
1088
1089 static inline void oob_reg_write(struct brcmnand_controller *ctrl, u32 offs,
1090                                  u32 data)
1091 {
1092         u16 offset0, offset10, reg_offs;
1093
1094         offset0 = ctrl->reg_offsets[BRCMNAND_OOB_WRITE_BASE];
1095         offset10 = ctrl->reg_offsets[BRCMNAND_OOB_WRITE_10_BASE];
1096
1097         if (offs >= ctrl->max_oob)
1098                 return;
1099
1100         if (offs >= 16 && offset10)
1101                 reg_offs = offset10 + ((offs - 0x10) & ~0x03);
1102         else
1103                 reg_offs = offset0 + (offs & ~0x03);
1104
1105         nand_writereg(ctrl, reg_offs, data);
1106 }
1107
1108 /*
1109  * read_oob_from_regs - read data from OOB registers
1110  * @ctrl: NAND controller
1111  * @i: sub-page sector index
1112  * @oob: buffer to read to
1113  * @sas: spare area sector size (i.e., OOB size per FLASH_CACHE)
1114  * @sector_1k: 1 for 1KiB sectors, 0 for 512B, other values are illegal
1115  */
1116 static int read_oob_from_regs(struct brcmnand_controller *ctrl, int i, u8 *oob,
1117                               int sas, int sector_1k)
1118 {
1119         int tbytes = sas << sector_1k;
1120         int j;
1121
1122         /* Adjust OOB values for 1K sector size */
1123         if (sector_1k && (i & 0x01))
1124                 tbytes = max(0, tbytes - (int)ctrl->max_oob);
1125         tbytes = min_t(int, tbytes, ctrl->max_oob);
1126
1127         for (j = 0; j < tbytes; j++)
1128                 oob[j] = oob_reg_read(ctrl, j);
1129         return tbytes;
1130 }
1131
1132 /*
1133  * write_oob_to_regs - write data to OOB registers
1134  * @i: sub-page sector index
1135  * @oob: buffer to write from
1136  * @sas: spare area sector size (i.e., OOB size per FLASH_CACHE)
1137  * @sector_1k: 1 for 1KiB sectors, 0 for 512B, other values are illegal
1138  */
1139 static int write_oob_to_regs(struct brcmnand_controller *ctrl, int i,
1140                              const u8 *oob, int sas, int sector_1k)
1141 {
1142         int tbytes = sas << sector_1k;
1143         int j;
1144
1145         /* Adjust OOB values for 1K sector size */
1146         if (sector_1k && (i & 0x01))
1147                 tbytes = max(0, tbytes - (int)ctrl->max_oob);
1148         tbytes = min_t(int, tbytes, ctrl->max_oob);
1149
1150         for (j = 0; j < tbytes; j += 4)
1151                 oob_reg_write(ctrl, j,
1152                                 (oob[j + 0] << 24) |
1153                                 (oob[j + 1] << 16) |
1154                                 (oob[j + 2] <<  8) |
1155                                 (oob[j + 3] <<  0));
1156         return tbytes;
1157 }
1158
1159 #ifndef __UBOOT__
1160 static irqreturn_t brcmnand_ctlrdy_irq(int irq, void *data)
1161 {
1162         struct brcmnand_controller *ctrl = data;
1163
1164         /* Discard all NAND_CTLRDY interrupts during DMA */
1165         if (ctrl->dma_pending)
1166                 return IRQ_HANDLED;
1167
1168         complete(&ctrl->done);
1169         return IRQ_HANDLED;
1170 }
1171
1172 /* Handle SoC-specific interrupt hardware */
1173 static irqreturn_t brcmnand_irq(int irq, void *data)
1174 {
1175         struct brcmnand_controller *ctrl = data;
1176
1177         if (ctrl->soc->ctlrdy_ack(ctrl->soc))
1178                 return brcmnand_ctlrdy_irq(irq, data);
1179
1180         return IRQ_NONE;
1181 }
1182
1183 static irqreturn_t brcmnand_dma_irq(int irq, void *data)
1184 {
1185         struct brcmnand_controller *ctrl = data;
1186
1187         complete(&ctrl->dma_done);
1188
1189         return IRQ_HANDLED;
1190 }
1191 #endif /* __UBOOT__ */
1192
1193 static void brcmnand_send_cmd(struct brcmnand_host *host, int cmd)
1194 {
1195         struct brcmnand_controller *ctrl = host->ctrl;
1196         int ret;
1197
1198         dev_dbg(ctrl->dev, "send native cmd %d addr_lo 0x%x\n", cmd,
1199                 brcmnand_read_reg(ctrl, BRCMNAND_CMD_ADDRESS));
1200         BUG_ON(ctrl->cmd_pending != 0);
1201         ctrl->cmd_pending = cmd;
1202
1203         ret = bcmnand_ctrl_poll_status(ctrl, NAND_CTRL_RDY, NAND_CTRL_RDY, 0);
1204         WARN_ON(ret);
1205
1206         mb(); /* flush previous writes */
1207         brcmnand_write_reg(ctrl, BRCMNAND_CMD_START,
1208                            cmd << brcmnand_cmd_shift(ctrl));
1209 }
1210
1211 /***********************************************************************
1212  * NAND MTD API: read/program/erase
1213  ***********************************************************************/
1214
1215 static void brcmnand_cmd_ctrl(struct mtd_info *mtd, int dat,
1216         unsigned int ctrl)
1217 {
1218         /* intentionally left blank */
1219 }
1220
1221 static int brcmnand_waitfunc(struct mtd_info *mtd, struct nand_chip *this)
1222 {
1223         struct nand_chip *chip = mtd_to_nand(mtd);
1224         struct brcmnand_host *host = nand_get_controller_data(chip);
1225         struct brcmnand_controller *ctrl = host->ctrl;
1226
1227 #ifndef __UBOOT__
1228         unsigned long timeo = msecs_to_jiffies(100);
1229
1230         dev_dbg(ctrl->dev, "wait on native cmd %d\n", ctrl->cmd_pending);
1231         if (ctrl->cmd_pending &&
1232                         wait_for_completion_timeout(&ctrl->done, timeo) <= 0) {
1233                 u32 cmd = brcmnand_read_reg(ctrl, BRCMNAND_CMD_START)
1234                                         >> brcmnand_cmd_shift(ctrl);
1235
1236                 dev_err_ratelimited(ctrl->dev,
1237                         "timeout waiting for command %#02x\n", cmd);
1238                 dev_err_ratelimited(ctrl->dev, "intfc status %08x\n",
1239                         brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS));
1240         }
1241 #else
1242         unsigned long timeo = 100; /* 100 msec */
1243         int ret;
1244
1245         dev_dbg(ctrl->dev, "wait on native cmd %d\n", ctrl->cmd_pending);
1246
1247         ret = bcmnand_ctrl_poll_status(ctrl, NAND_CTRL_RDY, NAND_CTRL_RDY, timeo);
1248         WARN_ON(ret);
1249 #endif /* __UBOOT__ */
1250
1251         ctrl->cmd_pending = 0;
1252         return brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS) &
1253                                  INTFC_FLASH_STATUS;
1254 }
1255
1256 enum {
1257         LLOP_RE                         = BIT(16),
1258         LLOP_WE                         = BIT(17),
1259         LLOP_ALE                        = BIT(18),
1260         LLOP_CLE                        = BIT(19),
1261         LLOP_RETURN_IDLE                = BIT(31),
1262
1263         LLOP_DATA_MASK                  = GENMASK(15, 0),
1264 };
1265
1266 static int brcmnand_low_level_op(struct brcmnand_host *host,
1267                                  enum brcmnand_llop_type type, u32 data,
1268                                  bool last_op)
1269 {
1270         struct mtd_info *mtd = nand_to_mtd(&host->chip);
1271         struct nand_chip *chip = &host->chip;
1272         struct brcmnand_controller *ctrl = host->ctrl;
1273         u32 tmp;
1274
1275         tmp = data & LLOP_DATA_MASK;
1276         switch (type) {
1277         case LL_OP_CMD:
1278                 tmp |= LLOP_WE | LLOP_CLE;
1279                 break;
1280         case LL_OP_ADDR:
1281                 /* WE | ALE */
1282                 tmp |= LLOP_WE | LLOP_ALE;
1283                 break;
1284         case LL_OP_WR:
1285                 /* WE */
1286                 tmp |= LLOP_WE;
1287                 break;
1288         case LL_OP_RD:
1289                 /* RE */
1290                 tmp |= LLOP_RE;
1291                 break;
1292         }
1293         if (last_op)
1294                 /* RETURN_IDLE */
1295                 tmp |= LLOP_RETURN_IDLE;
1296
1297         dev_dbg(ctrl->dev, "ll_op cmd %#x\n", tmp);
1298
1299         brcmnand_write_reg(ctrl, BRCMNAND_LL_OP, tmp);
1300         (void)brcmnand_read_reg(ctrl, BRCMNAND_LL_OP);
1301
1302         brcmnand_send_cmd(host, CMD_LOW_LEVEL_OP);
1303         return brcmnand_waitfunc(mtd, chip);
1304 }
1305
1306 static void brcmnand_cmdfunc(struct mtd_info *mtd, unsigned command,
1307                              int column, int page_addr)
1308 {
1309         struct nand_chip *chip = mtd_to_nand(mtd);
1310         struct brcmnand_host *host = nand_get_controller_data(chip);
1311         struct brcmnand_controller *ctrl = host->ctrl;
1312         u64 addr = (u64)page_addr << chip->page_shift;
1313         int native_cmd = 0;
1314
1315         if (command == NAND_CMD_READID || command == NAND_CMD_PARAM ||
1316                         command == NAND_CMD_RNDOUT)
1317                 addr = (u64)column;
1318         /* Avoid propagating a negative, don't-care address */
1319         else if (page_addr < 0)
1320                 addr = 0;
1321
1322         dev_dbg(ctrl->dev, "cmd 0x%x addr 0x%llx\n", command,
1323                 (unsigned long long)addr);
1324
1325         host->last_cmd = command;
1326         host->last_byte = 0;
1327         host->last_addr = addr;
1328
1329         switch (command) {
1330         case NAND_CMD_RESET:
1331                 native_cmd = CMD_FLASH_RESET;
1332                 break;
1333         case NAND_CMD_STATUS:
1334                 native_cmd = CMD_STATUS_READ;
1335                 break;
1336         case NAND_CMD_READID:
1337                 native_cmd = CMD_DEVICE_ID_READ;
1338                 break;
1339         case NAND_CMD_READOOB:
1340                 native_cmd = CMD_SPARE_AREA_READ;
1341                 break;
1342         case NAND_CMD_ERASE1:
1343                 native_cmd = CMD_BLOCK_ERASE;
1344                 brcmnand_wp(mtd, 0);
1345                 break;
1346         case NAND_CMD_PARAM:
1347                 native_cmd = CMD_PARAMETER_READ;
1348                 break;
1349         case NAND_CMD_SET_FEATURES:
1350         case NAND_CMD_GET_FEATURES:
1351                 brcmnand_low_level_op(host, LL_OP_CMD, command, false);
1352                 brcmnand_low_level_op(host, LL_OP_ADDR, column, false);
1353                 break;
1354         case NAND_CMD_RNDOUT:
1355                 native_cmd = CMD_PARAMETER_CHANGE_COL;
1356                 addr &= ~((u64)(FC_BYTES - 1));
1357                 /*
1358                  * HW quirk: PARAMETER_CHANGE_COL requires SECTOR_SIZE_1K=0
1359                  * NB: hwcfg.sector_size_1k may not be initialized yet
1360                  */
1361                 if (brcmnand_get_sector_size_1k(host)) {
1362                         host->hwcfg.sector_size_1k =
1363                                 brcmnand_get_sector_size_1k(host);
1364                         brcmnand_set_sector_size_1k(host, 0);
1365                 }
1366                 break;
1367         }
1368
1369         if (!native_cmd)
1370                 return;
1371
1372         brcmnand_write_reg(ctrl, BRCMNAND_CMD_EXT_ADDRESS,
1373                 (host->cs << 16) | ((addr >> 32) & 0xffff));
1374         (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_EXT_ADDRESS);
1375         brcmnand_write_reg(ctrl, BRCMNAND_CMD_ADDRESS, lower_32_bits(addr));
1376         (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_ADDRESS);
1377
1378         brcmnand_send_cmd(host, native_cmd);
1379         brcmnand_waitfunc(mtd, chip);
1380
1381         if (native_cmd == CMD_PARAMETER_READ ||
1382                         native_cmd == CMD_PARAMETER_CHANGE_COL) {
1383                 /* Copy flash cache word-wise */
1384                 u32 *flash_cache = (u32 *)ctrl->flash_cache;
1385                 int i;
1386
1387                 brcmnand_soc_data_bus_prepare(ctrl->soc, true);
1388
1389                 /*
1390                  * Must cache the FLASH_CACHE now, since changes in
1391                  * SECTOR_SIZE_1K may invalidate it
1392                  */
1393                 for (i = 0; i < FC_WORDS; i++) {
1394                         u32 fc;
1395
1396                         fc = brcmnand_read_fc(ctrl, i);
1397
1398                         /*
1399                          * Flash cache is big endian for parameter pages, at
1400                          * least on STB SoCs
1401                          */
1402                         if (ctrl->parameter_page_big_endian)
1403                                 flash_cache[i] = be32_to_cpu(fc);
1404                         else
1405                                 flash_cache[i] = le32_to_cpu(fc);
1406                 }
1407
1408                 brcmnand_soc_data_bus_unprepare(ctrl->soc, true);
1409
1410                 /* Cleanup from HW quirk: restore SECTOR_SIZE_1K */
1411                 if (host->hwcfg.sector_size_1k)
1412                         brcmnand_set_sector_size_1k(host,
1413                                                     host->hwcfg.sector_size_1k);
1414         }
1415
1416         /* Re-enable protection is necessary only after erase */
1417         if (command == NAND_CMD_ERASE1)
1418                 brcmnand_wp(mtd, 1);
1419 }
1420
1421 static uint8_t brcmnand_read_byte(struct mtd_info *mtd)
1422 {
1423         struct nand_chip *chip = mtd_to_nand(mtd);
1424         struct brcmnand_host *host = nand_get_controller_data(chip);
1425         struct brcmnand_controller *ctrl = host->ctrl;
1426         uint8_t ret = 0;
1427         int addr, offs;
1428
1429         switch (host->last_cmd) {
1430         case NAND_CMD_READID:
1431                 if (host->last_byte < 4)
1432                         ret = brcmnand_read_reg(ctrl, BRCMNAND_ID) >>
1433                                 (24 - (host->last_byte << 3));
1434                 else if (host->last_byte < 8)
1435                         ret = brcmnand_read_reg(ctrl, BRCMNAND_ID_EXT) >>
1436                                 (56 - (host->last_byte << 3));
1437                 break;
1438
1439         case NAND_CMD_READOOB:
1440                 ret = oob_reg_read(ctrl, host->last_byte);
1441                 break;
1442
1443         case NAND_CMD_STATUS:
1444                 ret = brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS) &
1445                                         INTFC_FLASH_STATUS;
1446                 if (wp_on) /* hide WP status */
1447                         ret |= NAND_STATUS_WP;
1448                 break;
1449
1450         case NAND_CMD_PARAM:
1451         case NAND_CMD_RNDOUT:
1452                 addr = host->last_addr + host->last_byte;
1453                 offs = addr & (FC_BYTES - 1);
1454
1455                 /* At FC_BYTES boundary, switch to next column */
1456                 if (host->last_byte > 0 && offs == 0)
1457                         nand_change_read_column_op(chip, addr, NULL, 0, false);
1458
1459                 ret = ctrl->flash_cache[offs];
1460                 break;
1461         case NAND_CMD_GET_FEATURES:
1462                 if (host->last_byte >= ONFI_SUBFEATURE_PARAM_LEN) {
1463                         ret = 0;
1464                 } else {
1465                         bool last = host->last_byte ==
1466                                 ONFI_SUBFEATURE_PARAM_LEN - 1;
1467                         brcmnand_low_level_op(host, LL_OP_RD, 0, last);
1468                         ret = brcmnand_read_reg(ctrl, BRCMNAND_LL_RDATA) & 0xff;
1469                 }
1470         }
1471
1472         dev_dbg(ctrl->dev, "read byte = 0x%02x\n", ret);
1473         host->last_byte++;
1474
1475         return ret;
1476 }
1477
1478 static void brcmnand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
1479 {
1480         int i;
1481
1482         for (i = 0; i < len; i++, buf++)
1483                 *buf = brcmnand_read_byte(mtd);
1484 }
1485
1486 static void brcmnand_write_buf(struct mtd_info *mtd, const uint8_t *buf,
1487                                    int len)
1488 {
1489         int i;
1490         struct nand_chip *chip = mtd_to_nand(mtd);
1491         struct brcmnand_host *host = nand_get_controller_data(chip);
1492
1493         switch (host->last_cmd) {
1494         case NAND_CMD_SET_FEATURES:
1495                 for (i = 0; i < len; i++)
1496                         brcmnand_low_level_op(host, LL_OP_WR, buf[i],
1497                                                   (i + 1) == len);
1498                 break;
1499         default:
1500                 BUG();
1501                 break;
1502         }
1503 }
1504
1505 /**
1506  * Construct a FLASH_DMA descriptor as part of a linked list. You must know the
1507  * following ahead of time:
1508  *  - Is this descriptor the beginning or end of a linked list?
1509  *  - What is the (DMA) address of the next descriptor in the linked list?
1510  */
1511 #ifndef __UBOOT__
1512 static int brcmnand_fill_dma_desc(struct brcmnand_host *host,
1513                                   struct brcm_nand_dma_desc *desc, u64 addr,
1514                                   dma_addr_t buf, u32 len, u8 dma_cmd,
1515                                   bool begin, bool end,
1516                                   dma_addr_t next_desc)
1517 {
1518         memset(desc, 0, sizeof(*desc));
1519         /* Descriptors are written in native byte order (wordwise) */
1520         desc->next_desc = lower_32_bits(next_desc);
1521         desc->next_desc_ext = upper_32_bits(next_desc);
1522         desc->cmd_irq = (dma_cmd << 24) |
1523                 (end ? (0x03 << 8) : 0) | /* IRQ | STOP */
1524                 (!!begin) | ((!!end) << 1); /* head, tail */
1525 #ifdef CONFIG_CPU_BIG_ENDIAN
1526         desc->cmd_irq |= 0x01 << 12;
1527 #endif
1528         desc->dram_addr = lower_32_bits(buf);
1529         desc->dram_addr_ext = upper_32_bits(buf);
1530         desc->tfr_len = len;
1531         desc->total_len = len;
1532         desc->flash_addr = lower_32_bits(addr);
1533         desc->flash_addr_ext = upper_32_bits(addr);
1534         desc->cs = host->cs;
1535         desc->status_valid = 0x01;
1536         return 0;
1537 }
1538
1539 /**
1540  * Kick the FLASH_DMA engine, with a given DMA descriptor
1541  */
1542 static void brcmnand_dma_run(struct brcmnand_host *host, dma_addr_t desc)
1543 {
1544         struct brcmnand_controller *ctrl = host->ctrl;
1545         unsigned long timeo = msecs_to_jiffies(100);
1546
1547         flash_dma_writel(ctrl, FLASH_DMA_FIRST_DESC, lower_32_bits(desc));
1548         (void)flash_dma_readl(ctrl, FLASH_DMA_FIRST_DESC);
1549         flash_dma_writel(ctrl, FLASH_DMA_FIRST_DESC_EXT, upper_32_bits(desc));
1550         (void)flash_dma_readl(ctrl, FLASH_DMA_FIRST_DESC_EXT);
1551
1552         /* Start FLASH_DMA engine */
1553         ctrl->dma_pending = true;
1554         mb(); /* flush previous writes */
1555         flash_dma_writel(ctrl, FLASH_DMA_CTRL, 0x03); /* wake | run */
1556
1557         if (wait_for_completion_timeout(&ctrl->dma_done, timeo) <= 0) {
1558                 dev_err(ctrl->dev,
1559                                 "timeout waiting for DMA; status %#x, error status %#x\n",
1560                                 flash_dma_readl(ctrl, FLASH_DMA_STATUS),
1561                                 flash_dma_readl(ctrl, FLASH_DMA_ERROR_STATUS));
1562         }
1563         ctrl->dma_pending = false;
1564         flash_dma_writel(ctrl, FLASH_DMA_CTRL, 0); /* force stop */
1565 }
1566
1567 static int brcmnand_dma_trans(struct brcmnand_host *host, u64 addr, u32 *buf,
1568                               u32 len, u8 dma_cmd)
1569 {
1570         struct brcmnand_controller *ctrl = host->ctrl;
1571         dma_addr_t buf_pa;
1572         int dir = dma_cmd == CMD_PAGE_READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1573
1574         buf_pa = dma_map_single(ctrl->dev, buf, len, dir);
1575         if (dma_mapping_error(ctrl->dev, buf_pa)) {
1576                 dev_err(ctrl->dev, "unable to map buffer for DMA\n");
1577                 return -ENOMEM;
1578         }
1579
1580         brcmnand_fill_dma_desc(host, ctrl->dma_desc, addr, buf_pa, len,
1581                                    dma_cmd, true, true, 0);
1582
1583         brcmnand_dma_run(host, ctrl->dma_pa);
1584
1585         dma_unmap_single(ctrl->dev, buf_pa, len, dir);
1586
1587         if (ctrl->dma_desc->status_valid & FLASH_DMA_ECC_ERROR)
1588                 return -EBADMSG;
1589         else if (ctrl->dma_desc->status_valid & FLASH_DMA_CORR_ERROR)
1590                 return -EUCLEAN;
1591
1592         return 0;
1593 }
1594 #endif /* __UBOOT__ */
1595
1596 /*
1597  * Assumes proper CS is already set
1598  */
1599 static int brcmnand_read_by_pio(struct mtd_info *mtd, struct nand_chip *chip,
1600                                 u64 addr, unsigned int trans, u32 *buf,
1601                                 u8 *oob, u64 *err_addr)
1602 {
1603         struct brcmnand_host *host = nand_get_controller_data(chip);
1604         struct brcmnand_controller *ctrl = host->ctrl;
1605         int i, j, ret = 0;
1606
1607         /* Clear error addresses */
1608         brcmnand_write_reg(ctrl, BRCMNAND_UNCORR_ADDR, 0);
1609         brcmnand_write_reg(ctrl, BRCMNAND_CORR_ADDR, 0);
1610         brcmnand_write_reg(ctrl, BRCMNAND_UNCORR_EXT_ADDR, 0);
1611         brcmnand_write_reg(ctrl, BRCMNAND_CORR_EXT_ADDR, 0);
1612
1613         brcmnand_write_reg(ctrl, BRCMNAND_CMD_EXT_ADDRESS,
1614                         (host->cs << 16) | ((addr >> 32) & 0xffff));
1615         (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_EXT_ADDRESS);
1616
1617         for (i = 0; i < trans; i++, addr += FC_BYTES) {
1618                 brcmnand_write_reg(ctrl, BRCMNAND_CMD_ADDRESS,
1619                                    lower_32_bits(addr));
1620                 (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_ADDRESS);
1621                 /* SPARE_AREA_READ does not use ECC, so just use PAGE_READ */
1622                 brcmnand_send_cmd(host, CMD_PAGE_READ);
1623                 brcmnand_waitfunc(mtd, chip);
1624
1625                 if (likely(buf)) {
1626                         brcmnand_soc_data_bus_prepare(ctrl->soc, false);
1627
1628                         for (j = 0; j < FC_WORDS; j++, buf++)
1629                                 *buf = brcmnand_read_fc(ctrl, j);
1630
1631                         brcmnand_soc_data_bus_unprepare(ctrl->soc, false);
1632                 }
1633
1634                 if (oob)
1635                         oob += read_oob_from_regs(ctrl, i, oob,
1636                                         mtd->oobsize / trans,
1637                                         host->hwcfg.sector_size_1k);
1638
1639                 if (!ret) {
1640                         *err_addr = brcmnand_read_reg(ctrl,
1641                                         BRCMNAND_UNCORR_ADDR) |
1642                                 ((u64)(brcmnand_read_reg(ctrl,
1643                                                 BRCMNAND_UNCORR_EXT_ADDR)
1644                                         & 0xffff) << 32);
1645                         if (*err_addr)
1646                                 ret = -EBADMSG;
1647                 }
1648
1649                 if (!ret) {
1650                         *err_addr = brcmnand_read_reg(ctrl,
1651                                         BRCMNAND_CORR_ADDR) |
1652                                 ((u64)(brcmnand_read_reg(ctrl,
1653                                                 BRCMNAND_CORR_EXT_ADDR)
1654                                         & 0xffff) << 32);
1655                         if (*err_addr)
1656                                 ret = -EUCLEAN;
1657                 }
1658         }
1659
1660         return ret;
1661 }
1662
1663 /*
1664  * Check a page to see if it is erased (w/ bitflips) after an uncorrectable ECC
1665  * error
1666  *
1667  * Because the HW ECC signals an ECC error if an erase paged has even a single
1668  * bitflip, we must check each ECC error to see if it is actually an erased
1669  * page with bitflips, not a truly corrupted page.
1670  *
1671  * On a real error, return a negative error code (-EBADMSG for ECC error), and
1672  * buf will contain raw data.
1673  * Otherwise, buf gets filled with 0xffs and return the maximum number of
1674  * bitflips-per-ECC-sector to the caller.
1675  *
1676  */
1677 static int brcmstb_nand_verify_erased_page(struct mtd_info *mtd,
1678                   struct nand_chip *chip, void *buf, u64 addr)
1679 {
1680         int i, sas;
1681         void *oob = chip->oob_poi;
1682         int bitflips = 0;
1683         int page = addr >> chip->page_shift;
1684         int ret;
1685
1686         if (!buf) {
1687 #ifndef __UBOOT__
1688                 buf = chip->data_buf;
1689 #else
1690                 buf = chip->buffers->databuf;
1691 #endif
1692                 /* Invalidate page cache */
1693                 chip->pagebuf = -1;
1694         }
1695
1696         sas = mtd->oobsize / chip->ecc.steps;
1697
1698         /* read without ecc for verification */
1699         ret = chip->ecc.read_page_raw(mtd, chip, buf, true, page);
1700         if (ret)
1701                 return ret;
1702
1703         for (i = 0; i < chip->ecc.steps; i++, oob += sas) {
1704                 ret = nand_check_erased_ecc_chunk(buf, chip->ecc.size,
1705                                                   oob, sas, NULL, 0,
1706                                                   chip->ecc.strength);
1707                 if (ret < 0)
1708                         return ret;
1709
1710                 bitflips = max(bitflips, ret);
1711         }
1712
1713         return bitflips;
1714 }
1715
1716 static int brcmnand_read(struct mtd_info *mtd, struct nand_chip *chip,
1717                          u64 addr, unsigned int trans, u32 *buf, u8 *oob)
1718 {
1719         struct brcmnand_host *host = nand_get_controller_data(chip);
1720         struct brcmnand_controller *ctrl = host->ctrl;
1721         u64 err_addr = 0;
1722         int err;
1723         bool retry = true;
1724
1725         dev_dbg(ctrl->dev, "read %llx -> %p\n", (unsigned long long)addr, buf);
1726
1727 try_dmaread:
1728         brcmnand_write_reg(ctrl, BRCMNAND_UNCORR_COUNT, 0);
1729
1730 #ifndef __UBOOT__
1731         if (has_flash_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) {
1732                 err = brcmnand_dma_trans(host, addr, buf, trans * FC_BYTES,
1733                                              CMD_PAGE_READ);
1734                 if (err) {
1735                         if (mtd_is_bitflip_or_eccerr(err))
1736                                 err_addr = addr;
1737                         else
1738                                 return -EIO;
1739                 }
1740         } else {
1741                 if (oob)
1742                         memset(oob, 0x99, mtd->oobsize);
1743
1744                 err = brcmnand_read_by_pio(mtd, chip, addr, trans, buf,
1745                                                oob, &err_addr);
1746         }
1747 #else
1748         if (oob)
1749                 memset(oob, 0x99, mtd->oobsize);
1750
1751         err = brcmnand_read_by_pio(mtd, chip, addr, trans, buf,
1752                                                            oob, &err_addr);
1753 #endif /* __UBOOT__ */
1754
1755         if (mtd_is_eccerr(err)) {
1756                 /*
1757                  * On controller version and 7.0, 7.1 , DMA read after a
1758                  * prior PIO read that reported uncorrectable error,
1759                  * the DMA engine captures this error following DMA read
1760                  * cleared only on subsequent DMA read, so just retry once
1761                  * to clear a possible false error reported for current DMA
1762                  * read
1763                  */
1764                 if ((ctrl->nand_version == 0x0700) ||
1765                     (ctrl->nand_version == 0x0701)) {
1766                         if (retry) {
1767                                 retry = false;
1768                                 goto try_dmaread;
1769                         }
1770                 }
1771
1772                 /*
1773                  * Controller version 7.2 has hw encoder to detect erased page
1774                  * bitflips, apply sw verification for older controllers only
1775                  */
1776                 if (ctrl->nand_version < 0x0702) {
1777                         err = brcmstb_nand_verify_erased_page(mtd, chip, buf,
1778                                                               addr);
1779                         /* erased page bitflips corrected */
1780                         if (err >= 0)
1781                                 return err;
1782                 }
1783
1784                 dev_dbg(ctrl->dev, "uncorrectable error at 0x%llx\n",
1785                         (unsigned long long)err_addr);
1786                 mtd->ecc_stats.failed++;
1787                 /* NAND layer expects zero on ECC errors */
1788                 return 0;
1789         }
1790
1791         if (mtd_is_bitflip(err)) {
1792                 unsigned int corrected = brcmnand_count_corrected(ctrl);
1793
1794                 dev_dbg(ctrl->dev, "corrected error at 0x%llx\n",
1795                         (unsigned long long)err_addr);
1796                 mtd->ecc_stats.corrected += corrected;
1797                 /* Always exceed the software-imposed threshold */
1798                 return max(mtd->bitflip_threshold, corrected);
1799         }
1800
1801         return 0;
1802 }
1803
1804 static int brcmnand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1805                               uint8_t *buf, int oob_required, int page)
1806 {
1807         struct brcmnand_host *host = nand_get_controller_data(chip);
1808         u8 *oob = oob_required ? (u8 *)chip->oob_poi : NULL;
1809
1810         nand_read_page_op(chip, page, 0, NULL, 0);
1811
1812         return brcmnand_read(mtd, chip, host->last_addr,
1813                         mtd->writesize >> FC_SHIFT, (u32 *)buf, oob);
1814 }
1815
1816 static int brcmnand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
1817                                   uint8_t *buf, int oob_required, int page)
1818 {
1819         struct brcmnand_host *host = nand_get_controller_data(chip);
1820         u8 *oob = oob_required ? (u8 *)chip->oob_poi : NULL;
1821         int ret;
1822
1823         nand_read_page_op(chip, page, 0, NULL, 0);
1824
1825         brcmnand_set_ecc_enabled(host, 0);
1826         ret = brcmnand_read(mtd, chip, host->last_addr,
1827                         mtd->writesize >> FC_SHIFT, (u32 *)buf, oob);
1828         brcmnand_set_ecc_enabled(host, 1);
1829         return ret;
1830 }
1831
1832 static int brcmnand_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1833                              int page)
1834 {
1835         return brcmnand_read(mtd, chip, (u64)page << chip->page_shift,
1836                         mtd->writesize >> FC_SHIFT,
1837                         NULL, (u8 *)chip->oob_poi);
1838 }
1839
1840 static int brcmnand_read_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
1841                                  int page)
1842 {
1843         struct brcmnand_host *host = nand_get_controller_data(chip);
1844
1845         brcmnand_set_ecc_enabled(host, 0);
1846         brcmnand_read(mtd, chip, (u64)page << chip->page_shift,
1847                 mtd->writesize >> FC_SHIFT,
1848                 NULL, (u8 *)chip->oob_poi);
1849         brcmnand_set_ecc_enabled(host, 1);
1850         return 0;
1851 }
1852
1853 static int brcmnand_write(struct mtd_info *mtd, struct nand_chip *chip,
1854                           u64 addr, const u32 *buf, u8 *oob)
1855 {
1856         struct brcmnand_host *host = nand_get_controller_data(chip);
1857         struct brcmnand_controller *ctrl = host->ctrl;
1858         unsigned int i, j, trans = mtd->writesize >> FC_SHIFT;
1859         int status, ret = 0;
1860
1861         dev_dbg(ctrl->dev, "write %llx <- %p\n", (unsigned long long)addr, buf);
1862
1863         if (unlikely((unsigned long)buf & 0x03)) {
1864                 dev_warn(ctrl->dev, "unaligned buffer: %p\n", buf);
1865                 buf = (u32 *)((unsigned long)buf & ~0x03);
1866         }
1867
1868         brcmnand_wp(mtd, 0);
1869
1870         for (i = 0; i < ctrl->max_oob; i += 4)
1871                 oob_reg_write(ctrl, i, 0xffffffff);
1872
1873 #ifndef __UBOOT__
1874         if (has_flash_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) {
1875                 if (brcmnand_dma_trans(host, addr, (u32 *)buf,
1876                                         mtd->writesize, CMD_PROGRAM_PAGE))
1877                         ret = -EIO;
1878                 goto out;
1879         }
1880 #endif /* __UBOOT__ */
1881
1882         brcmnand_write_reg(ctrl, BRCMNAND_CMD_EXT_ADDRESS,
1883                         (host->cs << 16) | ((addr >> 32) & 0xffff));
1884         (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_EXT_ADDRESS);
1885
1886         for (i = 0; i < trans; i++, addr += FC_BYTES) {
1887                 /* full address MUST be set before populating FC */
1888                 brcmnand_write_reg(ctrl, BRCMNAND_CMD_ADDRESS,
1889                                    lower_32_bits(addr));
1890                 (void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_ADDRESS);
1891
1892                 if (buf) {
1893                         brcmnand_soc_data_bus_prepare(ctrl->soc, false);
1894
1895                         for (j = 0; j < FC_WORDS; j++, buf++)
1896                                 brcmnand_write_fc(ctrl, j, *buf);
1897
1898                         brcmnand_soc_data_bus_unprepare(ctrl->soc, false);
1899                 } else if (oob) {
1900                         for (j = 0; j < FC_WORDS; j++)
1901                                 brcmnand_write_fc(ctrl, j, 0xffffffff);
1902                 }
1903
1904                 if (oob) {
1905                         oob += write_oob_to_regs(ctrl, i, oob,
1906                                         mtd->oobsize / trans,
1907                                         host->hwcfg.sector_size_1k);
1908                 }
1909
1910                 /* we cannot use SPARE_AREA_PROGRAM when PARTIAL_PAGE_EN=0 */
1911                 brcmnand_send_cmd(host, CMD_PROGRAM_PAGE);
1912                 status = brcmnand_waitfunc(mtd, chip);
1913
1914                 if (status & NAND_STATUS_FAIL) {
1915                         dev_info(ctrl->dev, "program failed at %llx\n",
1916                                 (unsigned long long)addr);
1917                         ret = -EIO;
1918                         goto out;
1919                 }
1920         }
1921 out:
1922         brcmnand_wp(mtd, 1);
1923         return ret;
1924 }
1925
1926 static int brcmnand_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1927                                const uint8_t *buf, int oob_required, int page)
1928 {
1929         struct brcmnand_host *host = nand_get_controller_data(chip);
1930         void *oob = oob_required ? chip->oob_poi : NULL;
1931
1932         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1933         brcmnand_write(mtd, chip, host->last_addr, (const u32 *)buf, oob);
1934
1935         return nand_prog_page_end_op(chip);
1936 }
1937
1938 static int brcmnand_write_page_raw(struct mtd_info *mtd,
1939                                    struct nand_chip *chip, const uint8_t *buf,
1940                                    int oob_required, int page)
1941 {
1942         struct brcmnand_host *host = nand_get_controller_data(chip);
1943         void *oob = oob_required ? chip->oob_poi : NULL;
1944
1945         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1946         brcmnand_set_ecc_enabled(host, 0);
1947         brcmnand_write(mtd, chip, host->last_addr, (const u32 *)buf, oob);
1948         brcmnand_set_ecc_enabled(host, 1);
1949
1950         return nand_prog_page_end_op(chip);
1951 }
1952
1953 static int brcmnand_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1954                                   int page)
1955 {
1956         return brcmnand_write(mtd, chip, (u64)page << chip->page_shift,
1957                                   NULL, chip->oob_poi);
1958 }
1959
1960 static int brcmnand_write_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
1961                                   int page)
1962 {
1963         struct brcmnand_host *host = nand_get_controller_data(chip);
1964         int ret;
1965
1966         brcmnand_set_ecc_enabled(host, 0);
1967         ret = brcmnand_write(mtd, chip, (u64)page << chip->page_shift, NULL,
1968                                  (u8 *)chip->oob_poi);
1969         brcmnand_set_ecc_enabled(host, 1);
1970
1971         return ret;
1972 }
1973
1974 /***********************************************************************
1975  * Per-CS setup (1 NAND device)
1976  ***********************************************************************/
1977
1978 static int brcmnand_set_cfg(struct brcmnand_host *host,
1979                             struct brcmnand_cfg *cfg)
1980 {
1981         struct brcmnand_controller *ctrl = host->ctrl;
1982         struct nand_chip *chip = &host->chip;
1983         u16 cfg_offs = brcmnand_cs_offset(ctrl, host->cs, BRCMNAND_CS_CFG);
1984         u16 cfg_ext_offs = brcmnand_cs_offset(ctrl, host->cs,
1985                         BRCMNAND_CS_CFG_EXT);
1986         u16 acc_control_offs = brcmnand_cs_offset(ctrl, host->cs,
1987                         BRCMNAND_CS_ACC_CONTROL);
1988         u8 block_size = 0, page_size = 0, device_size = 0;
1989         u32 tmp;
1990
1991         if (ctrl->block_sizes) {
1992                 int i, found;
1993
1994                 for (i = 0, found = 0; ctrl->block_sizes[i]; i++)
1995                         if (ctrl->block_sizes[i] * 1024 == cfg->block_size) {
1996                                 block_size = i;
1997                                 found = 1;
1998                         }
1999                 if (!found) {
2000                         dev_warn(ctrl->dev, "invalid block size %u\n",
2001                                         cfg->block_size);
2002                         return -EINVAL;
2003                 }
2004         } else {
2005                 block_size = ffs(cfg->block_size) - ffs(BRCMNAND_MIN_BLOCKSIZE);
2006         }
2007
2008         if (cfg->block_size < BRCMNAND_MIN_BLOCKSIZE || (ctrl->max_block_size &&
2009                                 cfg->block_size > ctrl->max_block_size)) {
2010                 dev_warn(ctrl->dev, "invalid block size %u\n",
2011                                 cfg->block_size);
2012                 block_size = 0;
2013         }
2014
2015         if (ctrl->page_sizes) {
2016                 int i, found;
2017
2018                 for (i = 0, found = 0; ctrl->page_sizes[i]; i++)
2019                         if (ctrl->page_sizes[i] == cfg->page_size) {
2020                                 page_size = i;
2021                                 found = 1;
2022                         }
2023                 if (!found) {
2024                         dev_warn(ctrl->dev, "invalid page size %u\n",
2025                                         cfg->page_size);
2026                         return -EINVAL;
2027                 }
2028         } else {
2029                 page_size = ffs(cfg->page_size) - ffs(BRCMNAND_MIN_PAGESIZE);
2030         }
2031
2032         if (cfg->page_size < BRCMNAND_MIN_PAGESIZE || (ctrl->max_page_size &&
2033                                 cfg->page_size > ctrl->max_page_size)) {
2034                 dev_warn(ctrl->dev, "invalid page size %u\n", cfg->page_size);
2035                 return -EINVAL;
2036         }
2037
2038         if (fls64(cfg->device_size) < fls64(BRCMNAND_MIN_DEVSIZE)) {
2039                 dev_warn(ctrl->dev, "invalid device size 0x%llx\n",
2040                         (unsigned long long)cfg->device_size);
2041                 return -EINVAL;
2042         }
2043         device_size = fls64(cfg->device_size) - fls64(BRCMNAND_MIN_DEVSIZE);
2044
2045         tmp = (cfg->blk_adr_bytes << CFG_BLK_ADR_BYTES_SHIFT) |
2046                 (cfg->col_adr_bytes << CFG_COL_ADR_BYTES_SHIFT) |
2047                 (cfg->ful_adr_bytes << CFG_FUL_ADR_BYTES_SHIFT) |
2048                 (!!(cfg->device_width == 16) << CFG_BUS_WIDTH_SHIFT) |
2049                 (device_size << CFG_DEVICE_SIZE_SHIFT);
2050         if (cfg_offs == cfg_ext_offs) {
2051                 tmp |= (page_size << CFG_PAGE_SIZE_SHIFT) |
2052                        (block_size << CFG_BLK_SIZE_SHIFT);
2053                 nand_writereg(ctrl, cfg_offs, tmp);
2054         } else {
2055                 nand_writereg(ctrl, cfg_offs, tmp);
2056                 tmp = (page_size << CFG_EXT_PAGE_SIZE_SHIFT) |
2057                       (block_size << CFG_EXT_BLK_SIZE_SHIFT);
2058                 nand_writereg(ctrl, cfg_ext_offs, tmp);
2059         }
2060
2061         tmp = nand_readreg(ctrl, acc_control_offs);
2062         tmp &= ~brcmnand_ecc_level_mask(ctrl);
2063         tmp |= cfg->ecc_level << NAND_ACC_CONTROL_ECC_SHIFT;
2064         tmp &= ~brcmnand_spare_area_mask(ctrl);
2065         tmp |= cfg->spare_area_size;
2066         nand_writereg(ctrl, acc_control_offs, tmp);
2067
2068         brcmnand_set_sector_size_1k(host, cfg->sector_size_1k);
2069
2070         /* threshold = ceil(BCH-level * 0.75) */
2071         brcmnand_wr_corr_thresh(host, DIV_ROUND_UP(chip->ecc.strength * 3, 4));
2072
2073         return 0;
2074 }
2075
2076 static void brcmnand_print_cfg(struct brcmnand_host *host,
2077                                char *buf, struct brcmnand_cfg *cfg)
2078 {
2079         buf += sprintf(buf,
2080                 "%lluMiB total, %uKiB blocks, %u%s pages, %uB OOB, %u-bit",
2081                 (unsigned long long)cfg->device_size >> 20,
2082                 cfg->block_size >> 10,
2083                 cfg->page_size >= 1024 ? cfg->page_size >> 10 : cfg->page_size,
2084                 cfg->page_size >= 1024 ? "KiB" : "B",
2085                 cfg->spare_area_size, cfg->device_width);
2086
2087         /* Account for Hamming ECC and for BCH 512B vs 1KiB sectors */
2088         if (is_hamming_ecc(host->ctrl, cfg))
2089                 sprintf(buf, ", Hamming ECC");
2090         else if (cfg->sector_size_1k)
2091                 sprintf(buf, ", BCH-%u (1KiB sector)", cfg->ecc_level << 1);
2092         else
2093                 sprintf(buf, ", BCH-%u", cfg->ecc_level);
2094 }
2095
2096 /*
2097  * Minimum number of bytes to address a page. Calculated as:
2098  *     roundup(log2(size / page-size) / 8)
2099  *
2100  * NB: the following does not "round up" for non-power-of-2 'size'; but this is
2101  *     OK because many other things will break if 'size' is irregular...
2102  */
2103 static inline int get_blk_adr_bytes(u64 size, u32 writesize)
2104 {
2105         return ALIGN(ilog2(size) - ilog2(writesize), 8) >> 3;
2106 }
2107
2108 static int brcmnand_setup_dev(struct brcmnand_host *host)
2109 {
2110         struct mtd_info *mtd = nand_to_mtd(&host->chip);
2111         struct nand_chip *chip = &host->chip;
2112         struct brcmnand_controller *ctrl = host->ctrl;
2113         struct brcmnand_cfg *cfg = &host->hwcfg;
2114         char msg[128];
2115         u32 offs, tmp, oob_sector;
2116         int ret;
2117
2118         memset(cfg, 0, sizeof(*cfg));
2119
2120 #ifndef __UBOOT__
2121         ret = of_property_read_u32(nand_get_flash_node(chip),
2122                                    "brcm,nand-oob-sector-size",
2123                                    &oob_sector);
2124 #else
2125         ret = ofnode_read_u32(nand_get_flash_node(chip),
2126                               "brcm,nand-oob-sector-size",
2127                               &oob_sector);
2128 #endif /* __UBOOT__ */
2129         if (ret) {
2130                 /* Use detected size */
2131                 cfg->spare_area_size = mtd->oobsize /
2132                                         (mtd->writesize >> FC_SHIFT);
2133         } else {
2134                 cfg->spare_area_size = oob_sector;
2135         }
2136         if (cfg->spare_area_size > ctrl->max_oob)
2137                 cfg->spare_area_size = ctrl->max_oob;
2138         /*
2139          * Set oobsize to be consistent with controller's spare_area_size, as
2140          * the rest is inaccessible.
2141          */
2142         mtd->oobsize = cfg->spare_area_size * (mtd->writesize >> FC_SHIFT);
2143
2144         cfg->device_size = mtd->size;
2145         cfg->block_size = mtd->erasesize;
2146         cfg->page_size = mtd->writesize;
2147         cfg->device_width = (chip->options & NAND_BUSWIDTH_16) ? 16 : 8;
2148         cfg->col_adr_bytes = 2;
2149         cfg->blk_adr_bytes = get_blk_adr_bytes(mtd->size, mtd->writesize);
2150
2151         if (chip->ecc.mode != NAND_ECC_HW) {
2152                 dev_err(ctrl->dev, "only HW ECC supported; selected: %d\n",
2153                         chip->ecc.mode);
2154                 return -EINVAL;
2155         }
2156
2157         if (chip->ecc.algo == NAND_ECC_UNKNOWN) {
2158                 if (chip->ecc.strength == 1 && chip->ecc.size == 512)
2159                         /* Default to Hamming for 1-bit ECC, if unspecified */
2160                         chip->ecc.algo = NAND_ECC_HAMMING;
2161                 else
2162                         /* Otherwise, BCH */
2163                         chip->ecc.algo = NAND_ECC_BCH;
2164         }
2165
2166         if (chip->ecc.algo == NAND_ECC_HAMMING && (chip->ecc.strength != 1 ||
2167                                                    chip->ecc.size != 512)) {
2168                 dev_err(ctrl->dev, "invalid Hamming params: %d bits per %d bytes\n",
2169                         chip->ecc.strength, chip->ecc.size);
2170                 return -EINVAL;
2171         }
2172
2173         switch (chip->ecc.size) {
2174         case 512:
2175                 if (chip->ecc.algo == NAND_ECC_HAMMING)
2176                         cfg->ecc_level = 15;
2177                 else
2178                         cfg->ecc_level = chip->ecc.strength;
2179                 cfg->sector_size_1k = 0;
2180                 break;
2181         case 1024:
2182                 if (!(ctrl->features & BRCMNAND_HAS_1K_SECTORS)) {
2183                         dev_err(ctrl->dev, "1KB sectors not supported\n");
2184                         return -EINVAL;
2185                 }
2186                 if (chip->ecc.strength & 0x1) {
2187                         dev_err(ctrl->dev,
2188                                 "odd ECC not supported with 1KB sectors\n");
2189                         return -EINVAL;
2190                 }
2191
2192                 cfg->ecc_level = chip->ecc.strength >> 1;
2193                 cfg->sector_size_1k = 1;
2194                 break;
2195         default:
2196                 dev_err(ctrl->dev, "unsupported ECC size: %d\n",
2197                         chip->ecc.size);
2198                 return -EINVAL;
2199         }
2200
2201         cfg->ful_adr_bytes = cfg->blk_adr_bytes;
2202         if (mtd->writesize > 512)
2203                 cfg->ful_adr_bytes += cfg->col_adr_bytes;
2204         else
2205                 cfg->ful_adr_bytes += 1;
2206
2207         ret = brcmnand_set_cfg(host, cfg);
2208         if (ret)
2209                 return ret;
2210
2211         brcmnand_set_ecc_enabled(host, 1);
2212
2213         brcmnand_print_cfg(host, msg, cfg);
2214         dev_info(ctrl->dev, "detected %s\n", msg);
2215
2216         /* Configure ACC_CONTROL */
2217         offs = brcmnand_cs_offset(ctrl, host->cs, BRCMNAND_CS_ACC_CONTROL);
2218         tmp = nand_readreg(ctrl, offs);
2219         tmp &= ~ACC_CONTROL_PARTIAL_PAGE;
2220         tmp &= ~ACC_CONTROL_RD_ERASED;
2221
2222         /* We need to turn on Read from erased paged protected by ECC */
2223         if (ctrl->nand_version >= 0x0702)
2224                 tmp |= ACC_CONTROL_RD_ERASED;
2225         tmp &= ~ACC_CONTROL_FAST_PGM_RDIN;
2226         if (ctrl->features & BRCMNAND_HAS_PREFETCH)
2227                 tmp &= ~ACC_CONTROL_PREFETCH;
2228
2229         nand_writereg(ctrl, offs, tmp);
2230
2231         return 0;
2232 }
2233
2234 #ifndef __UBOOT__
2235 static int brcmnand_init_cs(struct brcmnand_host *host, struct device_node *dn)
2236 #else
2237 static int brcmnand_init_cs(struct brcmnand_host *host, ofnode dn)
2238 #endif
2239 {
2240         struct brcmnand_controller *ctrl = host->ctrl;
2241 #ifndef __UBOOT__
2242         struct platform_device *pdev = host->pdev;
2243 #else
2244         struct udevice *pdev = host->pdev;
2245 #endif /* __UBOOT__ */
2246         struct mtd_info *mtd;
2247         struct nand_chip *chip;
2248         int ret;
2249         u16 cfg_offs;
2250
2251 #ifndef __UBOOT__
2252         ret = of_property_read_u32(dn, "reg", &host->cs);
2253 #else
2254         ret = ofnode_read_s32(dn, "reg", &host->cs);
2255 #endif
2256         if (ret) {
2257                 dev_err(&pdev->dev, "can't get chip-select\n");
2258                 return -ENXIO;
2259         }
2260
2261         mtd = nand_to_mtd(&host->chip);
2262         chip = &host->chip;
2263
2264         nand_set_flash_node(chip, dn);
2265         nand_set_controller_data(chip, host);
2266 #ifndef __UBOOT__
2267         mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "brcmnand.%d",
2268                                    host->cs);
2269 #else
2270         mtd->name = devm_kasprintf(pdev, GFP_KERNEL, "brcmnand.%d",
2271                                    host->cs);
2272 #endif /* __UBOOT__ */
2273         if (!mtd->name)
2274                 return -ENOMEM;
2275
2276         mtd->owner = THIS_MODULE;
2277 #ifndef __UBOOT__
2278         mtd->dev.parent = &pdev->dev;
2279 #else
2280         mtd->dev->parent = pdev;
2281 #endif /* __UBOOT__ */
2282
2283         chip->IO_ADDR_R = (void __iomem *)0xdeadbeef;
2284         chip->IO_ADDR_W = (void __iomem *)0xdeadbeef;
2285
2286         chip->cmd_ctrl = brcmnand_cmd_ctrl;
2287         chip->cmdfunc = brcmnand_cmdfunc;
2288         chip->waitfunc = brcmnand_waitfunc;
2289         chip->read_byte = brcmnand_read_byte;
2290         chip->read_buf = brcmnand_read_buf;
2291         chip->write_buf = brcmnand_write_buf;
2292
2293         chip->ecc.mode = NAND_ECC_HW;
2294         chip->ecc.read_page = brcmnand_read_page;
2295         chip->ecc.write_page = brcmnand_write_page;
2296         chip->ecc.read_page_raw = brcmnand_read_page_raw;
2297         chip->ecc.write_page_raw = brcmnand_write_page_raw;
2298         chip->ecc.write_oob_raw = brcmnand_write_oob_raw;
2299         chip->ecc.read_oob_raw = brcmnand_read_oob_raw;
2300         chip->ecc.read_oob = brcmnand_read_oob;
2301         chip->ecc.write_oob = brcmnand_write_oob;
2302
2303         chip->controller = &ctrl->controller;
2304
2305         /*
2306          * The bootloader might have configured 16bit mode but
2307          * NAND READID command only works in 8bit mode. We force
2308          * 8bit mode here to ensure that NAND READID commands works.
2309          */
2310         cfg_offs = brcmnand_cs_offset(ctrl, host->cs, BRCMNAND_CS_CFG);
2311         nand_writereg(ctrl, cfg_offs,
2312                       nand_readreg(ctrl, cfg_offs) & ~CFG_BUS_WIDTH);
2313
2314         ret = nand_scan_ident(mtd, 1, NULL);
2315         if (ret)
2316                 return ret;
2317
2318         chip->options |= NAND_NO_SUBPAGE_WRITE;
2319         /*
2320          * Avoid (for instance) kmap()'d buffers from JFFS2, which we can't DMA
2321          * to/from, and have nand_base pass us a bounce buffer instead, as
2322          * needed.
2323          */
2324         chip->options |= NAND_USE_BOUNCE_BUFFER;
2325
2326         if (chip->bbt_options & NAND_BBT_USE_FLASH)
2327                 chip->bbt_options |= NAND_BBT_NO_OOB;
2328
2329         if (brcmnand_setup_dev(host))
2330                 return -ENXIO;
2331
2332         chip->ecc.size = host->hwcfg.sector_size_1k ? 1024 : 512;
2333         /* only use our internal HW threshold */
2334         mtd->bitflip_threshold = 1;
2335
2336         chip->ecc.layout = brcmstb_choose_ecc_layout(host);
2337         if (!chip->ecc.layout)
2338                 return -ENXIO;
2339
2340         ret = nand_scan_tail(mtd);
2341         if (ret)
2342                 return ret;
2343
2344 #ifndef __UBOOT__
2345         ret = mtd_device_register(mtd, NULL, 0);
2346         if (ret)
2347                 nand_cleanup(chip);
2348 #else
2349         ret = nand_register(0, mtd);
2350 #endif /* __UBOOT__ */
2351
2352         return ret;
2353 }
2354
2355 #ifndef __UBOOT__
2356 static void brcmnand_save_restore_cs_config(struct brcmnand_host *host,
2357                                             int restore)
2358 {
2359         struct brcmnand_controller *ctrl = host->ctrl;
2360         u16 cfg_offs = brcmnand_cs_offset(ctrl, host->cs, BRCMNAND_CS_CFG);
2361         u16 cfg_ext_offs = brcmnand_cs_offset(ctrl, host->cs,
2362                         BRCMNAND_CS_CFG_EXT);
2363         u16 acc_control_offs = brcmnand_cs_offset(ctrl, host->cs,
2364                         BRCMNAND_CS_ACC_CONTROL);
2365         u16 t1_offs = brcmnand_cs_offset(ctrl, host->cs, BRCMNAND_CS_TIMING1);
2366         u16 t2_offs = brcmnand_cs_offset(ctrl, host->cs, BRCMNAND_CS_TIMING2);
2367
2368         if (restore) {
2369                 nand_writereg(ctrl, cfg_offs, host->hwcfg.config);
2370                 if (cfg_offs != cfg_ext_offs)
2371                         nand_writereg(ctrl, cfg_ext_offs,
2372                                       host->hwcfg.config_ext);
2373                 nand_writereg(ctrl, acc_control_offs, host->hwcfg.acc_control);
2374                 nand_writereg(ctrl, t1_offs, host->hwcfg.timing_1);
2375                 nand_writereg(ctrl, t2_offs, host->hwcfg.timing_2);
2376         } else {
2377                 host->hwcfg.config = nand_readreg(ctrl, cfg_offs);
2378                 if (cfg_offs != cfg_ext_offs)
2379                         host->hwcfg.config_ext =
2380                                 nand_readreg(ctrl, cfg_ext_offs);
2381                 host->hwcfg.acc_control = nand_readreg(ctrl, acc_control_offs);
2382                 host->hwcfg.timing_1 = nand_readreg(ctrl, t1_offs);
2383                 host->hwcfg.timing_2 = nand_readreg(ctrl, t2_offs);
2384         }
2385 }
2386
2387 static int brcmnand_suspend(struct device *dev)
2388 {
2389         struct brcmnand_controller *ctrl = dev_get_drvdata(dev);
2390         struct brcmnand_host *host;
2391
2392         list_for_each_entry(host, &ctrl->host_list, node)
2393                 brcmnand_save_restore_cs_config(host, 0);
2394
2395         ctrl->nand_cs_nand_select = brcmnand_read_reg(ctrl, BRCMNAND_CS_SELECT);
2396         ctrl->nand_cs_nand_xor = brcmnand_read_reg(ctrl, BRCMNAND_CS_XOR);
2397         ctrl->corr_stat_threshold =
2398                 brcmnand_read_reg(ctrl, BRCMNAND_CORR_THRESHOLD);
2399
2400         if (has_flash_dma(ctrl))
2401                 ctrl->flash_dma_mode = flash_dma_readl(ctrl, FLASH_DMA_MODE);
2402
2403         return 0;
2404 }
2405
2406 static int brcmnand_resume(struct device *dev)
2407 {
2408         struct brcmnand_controller *ctrl = dev_get_drvdata(dev);
2409         struct brcmnand_host *host;
2410
2411         if (has_flash_dma(ctrl)) {
2412                 flash_dma_writel(ctrl, FLASH_DMA_MODE, ctrl->flash_dma_mode);
2413                 flash_dma_writel(ctrl, FLASH_DMA_ERROR_STATUS, 0);
2414         }
2415
2416         brcmnand_write_reg(ctrl, BRCMNAND_CS_SELECT, ctrl->nand_cs_nand_select);
2417         brcmnand_write_reg(ctrl, BRCMNAND_CS_XOR, ctrl->nand_cs_nand_xor);
2418         brcmnand_write_reg(ctrl, BRCMNAND_CORR_THRESHOLD,
2419                         ctrl->corr_stat_threshold);
2420         if (ctrl->soc) {
2421                 /* Clear/re-enable interrupt */
2422                 ctrl->soc->ctlrdy_ack(ctrl->soc);
2423                 ctrl->soc->ctlrdy_set_enabled(ctrl->soc, true);
2424         }
2425
2426         list_for_each_entry(host, &ctrl->host_list, node) {
2427                 struct nand_chip *chip = &host->chip;
2428
2429                 brcmnand_save_restore_cs_config(host, 1);
2430
2431                 /* Reset the chip, required by some chips after power-up */
2432                 nand_reset_op(chip);
2433         }
2434
2435         return 0;
2436 }
2437
2438 const struct dev_pm_ops brcmnand_pm_ops = {
2439         .suspend                = brcmnand_suspend,
2440         .resume                 = brcmnand_resume,
2441 };
2442 EXPORT_SYMBOL_GPL(brcmnand_pm_ops);
2443
2444 static const struct of_device_id brcmnand_of_match[] = {
2445         { .compatible = "brcm,brcmnand-v4.0" },
2446         { .compatible = "brcm,brcmnand-v5.0" },
2447         { .compatible = "brcm,brcmnand-v6.0" },
2448         { .compatible = "brcm,brcmnand-v6.1" },
2449         { .compatible = "brcm,brcmnand-v6.2" },
2450         { .compatible = "brcm,brcmnand-v7.0" },
2451         { .compatible = "brcm,brcmnand-v7.1" },
2452         { .compatible = "brcm,brcmnand-v7.2" },
2453         {},
2454 };
2455 MODULE_DEVICE_TABLE(of, brcmnand_of_match);
2456 #endif  /* __UBOOT__ */
2457
2458 /***********************************************************************
2459  * Platform driver setup (per controller)
2460  ***********************************************************************/
2461
2462 #ifndef __UBOOT__
2463 int brcmnand_probe(struct platform_device *pdev, struct brcmnand_soc *soc)
2464 #else
2465 int brcmnand_probe(struct udevice *dev, struct brcmnand_soc *soc)
2466 #endif /* __UBOOT__ */
2467 {
2468 #ifndef __UBOOT__
2469         struct device *dev = &pdev->dev;
2470         struct device_node *dn = dev->of_node, *child;
2471 #else
2472         ofnode child;
2473         struct udevice *pdev = dev;
2474 #endif /* __UBOOT__ */
2475         struct brcmnand_controller *ctrl;
2476 #ifndef __UBOOT__
2477         struct resource *res;
2478 #else
2479         struct resource res;
2480 #endif /* __UBOOT__ */
2481         int ret;
2482
2483 #ifndef __UBOOT__
2484         /* We only support device-tree instantiation */
2485         if (!dn)
2486                 return -ENODEV;
2487
2488         if (!of_match_node(brcmnand_of_match, dn))
2489                 return -ENODEV;
2490 #endif /* __UBOOT__ */
2491
2492         ctrl = devm_kzalloc(dev, sizeof(*ctrl), GFP_KERNEL);
2493         if (!ctrl)
2494                 return -ENOMEM;
2495
2496 #ifndef __UBOOT__
2497         dev_set_drvdata(dev, ctrl);
2498 #else
2499         /*
2500          * in u-boot, the data for the driver is allocated before probing
2501          * so to keep the reference to ctrl, we store it in the variable soc
2502          */
2503         soc->ctrl = ctrl;
2504 #endif /* __UBOOT__ */
2505         ctrl->dev = dev;
2506
2507         init_completion(&ctrl->done);
2508         init_completion(&ctrl->dma_done);
2509         nand_hw_control_init(&ctrl->controller);
2510         INIT_LIST_HEAD(&ctrl->host_list);
2511
2512         /* Is parameter page in big endian ? */
2513         ctrl->parameter_page_big_endian =
2514             dev_read_u32_default(dev, "parameter-page-big-endian", 1);
2515
2516         /* NAND register range */
2517 #ifndef __UBOOT__
2518         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2519         ctrl->nand_base = devm_ioremap_resource(dev, res);
2520 #else
2521         dev_read_resource(pdev, 0, &res);
2522         ctrl->nand_base = devm_ioremap(pdev, res.start, resource_size(&res));
2523 #endif
2524         if (IS_ERR(ctrl->nand_base))
2525                 return PTR_ERR(ctrl->nand_base);
2526
2527         /* Enable clock before using NAND registers */
2528         ctrl->clk = devm_clk_get(dev, "nand");
2529         if (!IS_ERR(ctrl->clk)) {
2530                 ret = clk_prepare_enable(ctrl->clk);
2531                 if (ret)
2532                         return ret;
2533         } else {
2534                 ret = PTR_ERR(ctrl->clk);
2535                 if (ret == -EPROBE_DEFER)
2536                         return ret;
2537
2538                 ctrl->clk = NULL;
2539         }
2540
2541         /* Initialize NAND revision */
2542         ret = brcmnand_revision_init(ctrl);
2543         if (ret)
2544                 goto err;
2545
2546         /*
2547          * Most chips have this cache at a fixed offset within 'nand' block.
2548          * Some must specify this region separately.
2549          */
2550 #ifndef __UBOOT__
2551         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nand-cache");
2552         if (res) {
2553                 ctrl->nand_fc = devm_ioremap_resource(dev, res);
2554                 if (IS_ERR(ctrl->nand_fc)) {
2555                         ret = PTR_ERR(ctrl->nand_fc);
2556                         goto err;
2557                 }
2558         } else {
2559                 ctrl->nand_fc = ctrl->nand_base +
2560                                 ctrl->reg_offsets[BRCMNAND_FC_BASE];
2561         }
2562 #else
2563         if (!dev_read_resource_byname(pdev, "nand-cache", &res)) {
2564                 ctrl->nand_fc = devm_ioremap(dev, res.start,
2565                                              resource_size(&res));
2566                 if (IS_ERR(ctrl->nand_fc)) {
2567                         ret = PTR_ERR(ctrl->nand_fc);
2568                         goto err;
2569                 }
2570         } else {
2571                 ctrl->nand_fc = ctrl->nand_base +
2572                                 ctrl->reg_offsets[BRCMNAND_FC_BASE];
2573         }
2574 #endif
2575
2576 #ifndef __UBOOT__
2577         /* FLASH_DMA */
2578         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "flash-dma");
2579         if (res) {
2580                 ctrl->flash_dma_base = devm_ioremap_resource(dev, res);
2581                 if (IS_ERR(ctrl->flash_dma_base)) {
2582                         ret = PTR_ERR(ctrl->flash_dma_base);
2583                         goto err;
2584                 }
2585
2586                 flash_dma_writel(ctrl, FLASH_DMA_MODE, 1); /* linked-list */
2587                 flash_dma_writel(ctrl, FLASH_DMA_ERROR_STATUS, 0);
2588
2589                 /* Allocate descriptor(s) */
2590                 ctrl->dma_desc = dmam_alloc_coherent(dev,
2591                                                      sizeof(*ctrl->dma_desc),
2592                                                      &ctrl->dma_pa, GFP_KERNEL);
2593                 if (!ctrl->dma_desc) {
2594                         ret = -ENOMEM;
2595                         goto err;
2596                 }
2597
2598                 ctrl->dma_irq = platform_get_irq(pdev, 1);
2599                 if ((int)ctrl->dma_irq < 0) {
2600                         dev_err(dev, "missing FLASH_DMA IRQ\n");
2601                         ret = -ENODEV;
2602                         goto err;
2603                 }
2604
2605                 ret = devm_request_irq(dev, ctrl->dma_irq,
2606                                 brcmnand_dma_irq, 0, DRV_NAME,
2607                                 ctrl);
2608                 if (ret < 0) {
2609                         dev_err(dev, "can't allocate IRQ %d: error %d\n",
2610                                         ctrl->dma_irq, ret);
2611                         goto err;
2612                 }
2613
2614                 dev_info(dev, "enabling FLASH_DMA\n");
2615         }
2616 #endif /* __UBOOT__ */
2617
2618         /* Disable automatic device ID config, direct addressing */
2619         brcmnand_rmw_reg(ctrl, BRCMNAND_CS_SELECT,
2620                          CS_SELECT_AUTO_DEVICE_ID_CFG | 0xff, 0, 0);
2621         /* Disable XOR addressing */
2622         brcmnand_rmw_reg(ctrl, BRCMNAND_CS_XOR, 0xff, 0, 0);
2623
2624         /* Read the write-protect configuration in the device tree */
2625         wp_on = dev_read_u32_default(dev, "write-protect", wp_on);
2626
2627         if (ctrl->features & BRCMNAND_HAS_WP) {
2628                 /* Permanently disable write protection */
2629                 if (wp_on == 2)
2630                         brcmnand_set_wp(ctrl, false);
2631         } else {
2632                 wp_on = 0;
2633         }
2634
2635 #ifndef __UBOOT__
2636         /* IRQ */
2637         ctrl->irq = platform_get_irq(pdev, 0);
2638         if ((int)ctrl->irq < 0) {
2639                 dev_err(dev, "no IRQ defined\n");
2640                 ret = -ENODEV;
2641                 goto err;
2642         }
2643
2644         /*
2645          * Some SoCs integrate this controller (e.g., its interrupt bits) in
2646          * interesting ways
2647          */
2648         if (soc) {
2649                 ctrl->soc = soc;
2650
2651                 ret = devm_request_irq(dev, ctrl->irq, brcmnand_irq, 0,
2652                                        DRV_NAME, ctrl);
2653
2654                 /* Enable interrupt */
2655                 ctrl->soc->ctlrdy_ack(ctrl->soc);
2656                 ctrl->soc->ctlrdy_set_enabled(ctrl->soc, true);
2657         } else {
2658                 /* Use standard interrupt infrastructure */
2659                 ret = devm_request_irq(dev, ctrl->irq, brcmnand_ctlrdy_irq, 0,
2660                                        DRV_NAME, ctrl);
2661         }
2662         if (ret < 0) {
2663                 dev_err(dev, "can't allocate IRQ %d: error %d\n",
2664                         ctrl->irq, ret);
2665                 goto err;
2666         }
2667 #endif /* __UBOOT__ */
2668
2669 #ifndef __UBOOT__
2670         for_each_available_child_of_node(dn, child) {
2671                 if (of_device_is_compatible(child, "brcm,nandcs")) {
2672                         struct brcmnand_host *host;
2673
2674                         host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2675                         if (!host) {
2676                                 of_node_put(child);
2677                                 ret = -ENOMEM;
2678                                 goto err;
2679                         }
2680                         host->pdev = pdev;
2681                         host->ctrl = ctrl;
2682
2683                         ret = brcmnand_init_cs(host, child);
2684                         if (ret) {
2685                                 devm_kfree(dev, host);
2686                                 continue; /* Try all chip-selects */
2687                         }
2688
2689                         list_add_tail(&host->node, &ctrl->host_list);
2690                 }
2691         }
2692 #else
2693         ofnode_for_each_subnode(child, dev_ofnode(dev)) {
2694                 if (ofnode_device_is_compatible(child, "brcm,nandcs")) {
2695                         struct brcmnand_host *host;
2696
2697                         host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2698                         if (!host) {
2699                                 ret = -ENOMEM;
2700                                 goto err;
2701                         }
2702                         host->pdev = pdev;
2703                         host->ctrl = ctrl;
2704
2705                         ret = brcmnand_init_cs(host, child);
2706                         if (ret) {
2707                                 devm_kfree(dev, host);
2708                                 continue; /* Try all chip-selects */
2709                         }
2710
2711                         list_add_tail(&host->node, &ctrl->host_list);
2712                 }
2713         }
2714 #endif /* __UBOOT__ */
2715
2716 err:
2717 #ifndef __UBOOT__
2718         clk_disable_unprepare(ctrl->clk);
2719 #else
2720         if (ctrl->clk)
2721                 clk_disable(ctrl->clk);
2722 #endif /* __UBOOT__ */
2723         return ret;
2724
2725 }
2726 EXPORT_SYMBOL_GPL(brcmnand_probe);
2727
2728 #ifndef __UBOOT__
2729 int brcmnand_remove(struct platform_device *pdev)
2730 {
2731         struct brcmnand_controller *ctrl = dev_get_drvdata(&pdev->dev);
2732         struct brcmnand_host *host;
2733
2734         list_for_each_entry(host, &ctrl->host_list, node)
2735                 nand_release(nand_to_mtd(&host->chip));
2736
2737         clk_disable_unprepare(ctrl->clk);
2738
2739         dev_set_drvdata(&pdev->dev, NULL);
2740
2741         return 0;
2742 }
2743 #else
2744 int brcmnand_remove(struct udevice *pdev)
2745 {
2746         return 0;
2747 }
2748 #endif /* __UBOOT__ */
2749 EXPORT_SYMBOL_GPL(brcmnand_remove);
2750
2751 MODULE_LICENSE("GPL v2");
2752 MODULE_AUTHOR("Kevin Cernekee");
2753 MODULE_AUTHOR("Brian Norris");
2754 MODULE_DESCRIPTION("NAND driver for Broadcom chips");
2755 MODULE_ALIAS("platform:brcmnand");