drivers/mmc/host/bcm2835.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * bcm2835 sdhost driver.
   4  *
   5  * The 2835 has two SD controllers: The Arasan sdhci controller
   6  * (supported by the iproc driver) and a custom sdhost controller
   7  * (supported by this driver).
   8  *
   9  * The sdhci controller supports both sdcard and sdio.  The sdhost
  10  * controller supports the sdcard only, but has better performance.
  11  * Also note that the rpi3 has sdio wifi, so driving the sdcard with
  12  * the sdhost controller allows to use the sdhci controller for wifi
  13  * support.
  14  *
  15  * The configuration is done by devicetree via pin muxing.  Both
  16  * SD controller are available on the same pins (2 pin groups = pin 22
  17  * to 27 + pin 48 to 53).  So it's possible to use both SD controllers
  18  * at the same time with different pin groups.
  19  *
  20  * Author:      Phil Elwell <phil@raspberrypi.org>
  21  *              Copyright (C) 2015-2016 Raspberry Pi (Trading) Ltd.
  22  *
  23  * Based on
  24  *  mmc-bcm2835.c by Gellert Weisz
  25  * which is, in turn, based on
  26  *  sdhci-bcm2708.c by Broadcom
  27  *  sdhci-bcm2835.c by Stephen Warren and Oleksandr Tymoshenko
  28  *  sdhci.c and sdhci-pci.c by Pierre Ossman
  29  */
  30 #include <linux/clk.h>
  31 #include <linux/delay.h>
  32 #include <linux/device.h>
  33 #include <linux/dmaengine.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/highmem.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/io.h>
  39 #include <linux/iopoll.h>
  40 #include <linux/module.h>
  41 #include <linux/of_address.h>
  42 #include <linux/of_irq.h>
  43 #include <linux/platform_device.h>
  44 #include <linux/scatterlist.h>
  45 #include <linux/time.h>
  46 #include <linux/workqueue.h>
  47
  48 #include <linux/mmc/host.h>
  49 #include <linux/mmc/mmc.h>
  50 #include <linux/mmc/sd.h>
  51
  52 #define SDCMD  0x00 /* Command to SD card              - 16 R/W */
  53 #define SDARG  0x04 /* Argument to SD card             - 32 R/W */
  54 #define SDTOUT 0x08 /* Start value for timeout counter - 32 R/W */
  55 #define SDCDIV 0x0c /* Start value for clock divider   - 11 R/W */
  56 #define SDRSP0 0x10 /* SD card response (31:0)         - 32 R   */
  57 #define SDRSP1 0x14 /* SD card response (63:32)        - 32 R   */
  58 #define SDRSP2 0x18 /* SD card response (95:64)        - 32 R   */
  59 #define SDRSP3 0x1c /* SD card response (127:96)       - 32 R   */
  60 #define SDHSTS 0x20 /* SD host status                  - 11 R/W */
  61 #define SDVDD  0x30 /* SD card power control           -  1 R/W */
  62 #define SDEDM  0x34 /* Emergency Debug Mode            - 13 R/W */
  63 #define SDHCFG 0x38 /* Host configuration              -  2 R/W */
  64 #define SDHBCT 0x3c /* Host byte count (debug)         - 32 R/W */
  65 #define SDDATA 0x40 /* Data to/from SD card            - 32 R/W */
  66 #define SDHBLC 0x50 /* Host block count (SDIO/SDHC)    -  9 R/W */
  67
  68 #define SDCMD_NEW_FLAG                  0x8000
  69 #define SDCMD_FAIL_FLAG                 0x4000
  70 #define SDCMD_BUSYWAIT                  0x800
  71 #define SDCMD_NO_RESPONSE               0x400
  72 #define SDCMD_LONG_RESPONSE             0x200
  73 #define SDCMD_WRITE_CMD                 0x80
  74 #define SDCMD_READ_CMD                  0x40
  75 #define SDCMD_CMD_MASK                  0x3f
  76
  77 #define SDCDIV_MAX_CDIV                 0x7ff
  78
  79 #define SDHSTS_BUSY_IRPT                0x400
  80 #define SDHSTS_BLOCK_IRPT               0x200
  81 #define SDHSTS_SDIO_IRPT                0x100
  82 #define SDHSTS_REW_TIME_OUT             0x80
  83 #define SDHSTS_CMD_TIME_OUT             0x40
  84 #define SDHSTS_CRC16_ERROR              0x20
  85 #define SDHSTS_CRC7_ERROR               0x10
  86 #define SDHSTS_FIFO_ERROR               0x08
  87 /* Reserved */
  88 /* Reserved */
  89 #define SDHSTS_DATA_FLAG                0x01
  90
  91 #define SDHSTS_TRANSFER_ERROR_MASK      (SDHSTS_CRC7_ERROR | \
  92                                          SDHSTS_CRC16_ERROR | \
  93                                          SDHSTS_REW_TIME_OUT | \
  94                                          SDHSTS_FIFO_ERROR)
  95
  96 #define SDHSTS_ERROR_MASK               (SDHSTS_CMD_TIME_OUT | \
  97                                          SDHSTS_TRANSFER_ERROR_MASK)
  98
  99 #define SDHCFG_BUSY_IRPT_EN     BIT(10)
 100 #define SDHCFG_BLOCK_IRPT_EN    BIT(8)
 101 #define SDHCFG_SDIO_IRPT_EN     BIT(5)
 102 #define SDHCFG_DATA_IRPT_EN     BIT(4)
 103 #define SDHCFG_SLOW_CARD        BIT(3)
 104 #define SDHCFG_WIDE_EXT_BUS     BIT(2)
 105 #define SDHCFG_WIDE_INT_BUS     BIT(1)
 106 #define SDHCFG_REL_CMD_LINE     BIT(0)
 107
 108 #define SDVDD_POWER_OFF         0
 109 #define SDVDD_POWER_ON          1
 110
 111 #define SDEDM_FORCE_DATA_MODE   BIT(19)
 112 #define SDEDM_CLOCK_PULSE       BIT(20)
 113 #define SDEDM_BYPASS            BIT(21)
 114
 115 #define SDEDM_WRITE_THRESHOLD_SHIFT     9
 116 #define SDEDM_READ_THRESHOLD_SHIFT      14
 117 #define SDEDM_THRESHOLD_MASK            0x1f
 118
 119 #define SDEDM_FSM_MASK          0xf
 120 #define SDEDM_FSM_IDENTMODE     0x0
 121 #define SDEDM_FSM_DATAMODE      0x1
 122 #define SDEDM_FSM_READDATA      0x2
 123 #define SDEDM_FSM_WRITEDATA     0x3
 124 #define SDEDM_FSM_READWAIT      0x4
 125 #define SDEDM_FSM_READCRC       0x5
 126 #define SDEDM_FSM_WRITECRC      0x6
 127 #define SDEDM_FSM_WRITEWAIT1    0x7
 128 #define SDEDM_FSM_POWERDOWN     0x8
 129 #define SDEDM_FSM_POWERUP       0x9
 130 #define SDEDM_FSM_WRITESTART1   0xa
 131 #define SDEDM_FSM_WRITESTART2   0xb
 132 #define SDEDM_FSM_GENPULSES     0xc
 133 #define SDEDM_FSM_WRITEWAIT2    0xd
 134 #define SDEDM_FSM_STARTPOWDOWN  0xf
 135
 136 #define SDDATA_FIFO_WORDS       16
 137
 138 #define FIFO_READ_THRESHOLD     4
 139 #define FIFO_WRITE_THRESHOLD    4
 140 #define SDDATA_FIFO_PIO_BURST   8
 141
 142 #define PIO_THRESHOLD   1  /* Maximum block count for PIO (0 = always DMA) */
 143
 144 struct bcm2835_host {
 145         spinlock_t              lock;
 146         struct mutex            mutex;
 147
 148         void __iomem            *ioaddr;
 149         u32                     phys_addr;
 150
 151         struct platform_device  *pdev;
 152
 153         int                     clock;          /* Current clock speed */
 154         unsigned int            max_clk;        /* Max possible freq */
 155         struct work_struct      dma_work;
 156         struct delayed_work     timeout_work;   /* Timer for timeouts */
 157         struct sg_mapping_iter  sg_miter;       /* SG state for PIO */
 158         unsigned int            blocks;         /* remaining PIO blocks */
 159         int                     irq;            /* Device IRQ */
 160
 161         u32                     ns_per_fifo_word;
 162
 163         /* cached registers */
 164         u32                     hcfg;
 165         u32                     cdiv;
 166
 167         struct mmc_request      *mrq;           /* Current request */
 168         struct mmc_command      *cmd;           /* Current command */
 169         struct mmc_data         *data;          /* Current data request */
 170         bool                    data_complete:1;/* Data finished before cmd */
 171         bool                    use_busy:1;     /* Wait for busy interrupt */
 172         bool                    use_sbc:1;      /* Send CMD23 */
 173
 174         /* for threaded irq handler */
 175         bool                    irq_block;
 176         bool                    irq_busy;
 177         bool                    irq_data;
 178
 179         /* DMA part */
 180         struct dma_chan         *dma_chan_rxtx;
 181         struct dma_chan         *dma_chan;
 182         struct dma_slave_config dma_cfg_rx;
 183         struct dma_slave_config dma_cfg_tx;
 184         struct dma_async_tx_descriptor  *dma_desc;
 185         u32                     dma_dir;
 186         u32                     drain_words;
 187         struct page             *drain_page;
 188         u32                     drain_offset;
 189         bool                    use_dma;
 190 };
 191
 192 static void bcm2835_dumpcmd(struct bcm2835_host *host, struct mmc_command *cmd,
 193                             const char *label)
 194 {
 195         struct device *dev = &host->pdev->dev;
 196
 197         if (!cmd)
 198                 return;
 199
 200         dev_dbg(dev, "%c%s op %d arg 0x%x flags 0x%x - resp %08x %08x %08x %08x, err %d\n",
 201                 (cmd == host->cmd) ? '>' : ' ',
 202                 label, cmd->opcode, cmd->arg, cmd->flags,
 203                 cmd->resp[0], cmd->resp[1], cmd->resp[2], cmd->resp[3],
 204                 cmd->error);
 205 }
 206
 207 static void bcm2835_dumpregs(struct bcm2835_host *host)
 208 {
 209         struct mmc_request *mrq = host->mrq;
 210         struct device *dev = &host->pdev->dev;
 211
 212         if (mrq) {
 213                 bcm2835_dumpcmd(host, mrq->sbc, "sbc");
 214                 bcm2835_dumpcmd(host, mrq->cmd, "cmd");
 215                 if (mrq->data) {
 216                         dev_dbg(dev, "data blocks %x blksz %x - err %d\n",
 217                                 mrq->data->blocks,
 218                                 mrq->data->blksz,
 219                                 mrq->data->error);
 220                 }
 221                 bcm2835_dumpcmd(host, mrq->stop, "stop");
 222         }
 223
 224         dev_dbg(dev, "=========== REGISTER DUMP ===========\n");
 225         dev_dbg(dev, "SDCMD  0x%08x\n", readl(host->ioaddr + SDCMD));
 226         dev_dbg(dev, "SDARG  0x%08x\n", readl(host->ioaddr + SDARG));
 227         dev_dbg(dev, "SDTOUT 0x%08x\n", readl(host->ioaddr + SDTOUT));
 228         dev_dbg(dev, "SDCDIV 0x%08x\n", readl(host->ioaddr + SDCDIV));
 229         dev_dbg(dev, "SDRSP0 0x%08x\n", readl(host->ioaddr + SDRSP0));
 230         dev_dbg(dev, "SDRSP1 0x%08x\n", readl(host->ioaddr + SDRSP1));
 231         dev_dbg(dev, "SDRSP2 0x%08x\n", readl(host->ioaddr + SDRSP2));
 232         dev_dbg(dev, "SDRSP3 0x%08x\n", readl(host->ioaddr + SDRSP3));
 233         dev_dbg(dev, "SDHSTS 0x%08x\n", readl(host->ioaddr + SDHSTS));
 234         dev_dbg(dev, "SDVDD  0x%08x\n", readl(host->ioaddr + SDVDD));
 235         dev_dbg(dev, "SDEDM  0x%08x\n", readl(host->ioaddr + SDEDM));
 236         dev_dbg(dev, "SDHCFG 0x%08x\n", readl(host->ioaddr + SDHCFG));
 237         dev_dbg(dev, "SDHBCT 0x%08x\n", readl(host->ioaddr + SDHBCT));
 238         dev_dbg(dev, "SDHBLC 0x%08x\n", readl(host->ioaddr + SDHBLC));
 239         dev_dbg(dev, "===========================================\n");
 240 }
 241
 242 static void bcm2835_reset_internal(struct bcm2835_host *host)
 243 {
 244         u32 temp;
 245
 246         writel(SDVDD_POWER_OFF, host->ioaddr + SDVDD);
 247         writel(0, host->ioaddr + SDCMD);
 248         writel(0, host->ioaddr + SDARG);
 249         writel(0xf00000, host->ioaddr + SDTOUT);
 250         writel(0, host->ioaddr + SDCDIV);
 251         writel(0x7f8, host->ioaddr + SDHSTS); /* Write 1s to clear */
 252         writel(0, host->ioaddr + SDHCFG);
 253         writel(0, host->ioaddr + SDHBCT);
 254         writel(0, host->ioaddr + SDHBLC);
 255
 256         /* Limit fifo usage due to silicon bug */
 257         temp = readl(host->ioaddr + SDEDM);
 258         temp &= ~((SDEDM_THRESHOLD_MASK << SDEDM_READ_THRESHOLD_SHIFT) |
 259                   (SDEDM_THRESHOLD_MASK << SDEDM_WRITE_THRESHOLD_SHIFT));
 260         temp |= (FIFO_READ_THRESHOLD << SDEDM_READ_THRESHOLD_SHIFT) |
 261                 (FIFO_WRITE_THRESHOLD << SDEDM_WRITE_THRESHOLD_SHIFT);
 262         writel(temp, host->ioaddr + SDEDM);
 263         msleep(20);
 264         writel(SDVDD_POWER_ON, host->ioaddr + SDVDD);
 265         msleep(20);
 266         host->clock = 0;
 267         writel(host->hcfg, host->ioaddr + SDHCFG);
 268         writel(host->cdiv, host->ioaddr + SDCDIV);
 269 }
 270
 271 static void bcm2835_reset(struct mmc_host *mmc)
 272 {
 273         struct bcm2835_host *host = mmc_priv(mmc);
 274
 275         if (host->dma_chan)
 276                 dmaengine_terminate_sync(host->dma_chan);
 277         host->dma_chan = NULL;
 278         bcm2835_reset_internal(host);
 279 }
 280
 281 static void bcm2835_finish_command(struct bcm2835_host *host);
 282
 283 static void bcm2835_wait_transfer_complete(struct bcm2835_host *host)
 284 {
 285         int timediff;
 286         u32 alternate_idle;
 287
 288         alternate_idle = (host->mrq->data->flags & MMC_DATA_READ) ?
 289                 SDEDM_FSM_READWAIT : SDEDM_FSM_WRITESTART1;
 290
 291         timediff = 0;
 292
 293         while (1) {
 294                 u32 edm, fsm;
 295
 296                 edm = readl(host->ioaddr + SDEDM);
 297                 fsm = edm & SDEDM_FSM_MASK;
 298
 299                 if ((fsm == SDEDM_FSM_IDENTMODE) ||
 300                     (fsm == SDEDM_FSM_DATAMODE))
 301                         break;
 302                 if (fsm == alternate_idle) {
 303                         writel(edm | SDEDM_FORCE_DATA_MODE,
 304                                host->ioaddr + SDEDM);
 305                         break;
 306                 }
 307
 308                 timediff++;
 309                 if (timediff == 100000) {
 310                         dev_err(&host->pdev->dev,
 311                                 "wait_transfer_complete - still waiting after %d retries\n",
 312                                 timediff);
 313                         bcm2835_dumpregs(host);
 314                         host->mrq->data->error = -ETIMEDOUT;
 315                         return;
 316                 }
 317                 cpu_relax();
 318         }
 319 }
 320
 321 static void bcm2835_dma_complete(void *param)
 322 {
 323         struct bcm2835_host *host = param;
 324
 325         schedule_work(&host->dma_work);
 326 }
 327
 328 static void bcm2835_transfer_block_pio(struct bcm2835_host *host, bool is_read)
 329 {
 330         unsigned long flags;
 331         size_t blksize;
 332         unsigned long wait_max;
 333
 334         blksize = host->data->blksz;
 335
 336         wait_max = jiffies + msecs_to_jiffies(500);
 337
 338         local_irq_save(flags);
 339
 340         while (blksize) {
 341                 int copy_words;
 342                 u32 hsts = 0;
 343                 size_t len;
 344                 u32 *buf;
 345
 346                 if (!sg_miter_next(&host->sg_miter)) {
 347                         host->data->error = -EINVAL;
 348                         break;
 349                 }
 350
 351                 len = min(host->sg_miter.length, blksize);
 352                 if (len % 4) {
 353                         host->data->error = -EINVAL;
 354                         break;
 355                 }
 356
 357                 blksize -= len;
 358                 host->sg_miter.consumed = len;
 359
 360                 buf = (u32 *)host->sg_miter.addr;
 361
 362                 copy_words = len / 4;
 363
 364                 while (copy_words) {
 365                         int burst_words, words;
 366                         u32 edm;
 367
 368                         burst_words = min(SDDATA_FIFO_PIO_BURST, copy_words);
 369                         edm = readl(host->ioaddr + SDEDM);
 370                         if (is_read)
 371                                 words = ((edm >> 4) & 0x1f);
 372                         else
 373                                 words = SDDATA_FIFO_WORDS - ((edm >> 4) & 0x1f);
 374
 375                         if (words < burst_words) {
 376                                 int fsm_state = (edm & SDEDM_FSM_MASK);
 377                                 struct device *dev = &host->pdev->dev;
 378
 379                                 if ((is_read &&
 380                                      (fsm_state != SDEDM_FSM_READDATA &&
 381                                       fsm_state != SDEDM_FSM_READWAIT &&
 382                                       fsm_state != SDEDM_FSM_READCRC)) ||
 383                                     (!is_read &&
 384                                      (fsm_state != SDEDM_FSM_WRITEDATA &&
 385                                       fsm_state != SDEDM_FSM_WRITESTART1 &&
 386                                       fsm_state != SDEDM_FSM_WRITESTART2))) {
 387                                         hsts = readl(host->ioaddr + SDHSTS);
 388                                         dev_err(dev, "fsm %x, hsts %08x\n",
 389                                                 fsm_state, hsts);
 390                                         if (hsts & SDHSTS_ERROR_MASK)
 391                                                 break;
 392                                 }
 393
 394                                 if (time_after(jiffies, wait_max)) {
 395                                         dev_err(dev, "PIO %s timeout - EDM %08x\n",
 396                                                 is_read ? "read" : "write",
 397                                                 edm);
 398                                         hsts = SDHSTS_REW_TIME_OUT;
 399                                         break;
 400                                 }
 401                                 ndelay((burst_words - words) *
 402                                        host->ns_per_fifo_word);
 403                                 continue;
 404                         } else if (words > copy_words) {
 405                                 words = copy_words;
 406                         }
 407
 408                         copy_words -= words;
 409
 410                         while (words) {
 411                                 if (is_read)
 412                                         *(buf++) = readl(host->ioaddr + SDDATA);
 413                                 else
 414                                         writel(*(buf++), host->ioaddr + SDDATA);
 415                                 words--;
 416                         }
 417                 }
 418
 419                 if (hsts & SDHSTS_ERROR_MASK)
 420                         break;
 421         }
 422
 423         sg_miter_stop(&host->sg_miter);
 424
 425         local_irq_restore(flags);
 426 }
 427
 428 static void bcm2835_transfer_pio(struct bcm2835_host *host)
 429 {
 430         struct device *dev = &host->pdev->dev;
 431         u32 sdhsts;
 432         bool is_read;
 433
 434         is_read = (host->data->flags & MMC_DATA_READ) != 0;
 435         bcm2835_transfer_block_pio(host, is_read);
 436
 437         sdhsts = readl(host->ioaddr + SDHSTS);
 438         if (sdhsts & (SDHSTS_CRC16_ERROR |
 439                       SDHSTS_CRC7_ERROR |
 440                       SDHSTS_FIFO_ERROR)) {
 441                 dev_err(dev, "%s transfer error - HSTS %08x\n",
 442                         is_read ? "read" : "write", sdhsts);
 443                 host->data->error = -EILSEQ;
 444         } else if ((sdhsts & (SDHSTS_CMD_TIME_OUT |
 445                               SDHSTS_REW_TIME_OUT))) {
 446                 dev_err(dev, "%s timeout error - HSTS %08x\n",
 447                         is_read ? "read" : "write", sdhsts);
 448                 host->data->error = -ETIMEDOUT;
 449         }
 450 }
 451
 452 static
 453 void bcm2835_prepare_dma(struct bcm2835_host *host, struct mmc_data *data)
 454 {
 455         int sg_len, dir_data, dir_slave;
 456         struct dma_async_tx_descriptor *desc = NULL;
 457         struct dma_chan *dma_chan;
 458
 459         dma_chan = host->dma_chan_rxtx;
 460         if (data->flags & MMC_DATA_READ) {
 461                 dir_data = DMA_FROM_DEVICE;
 462                 dir_slave = DMA_DEV_TO_MEM;
 463         } else {
 464                 dir_data = DMA_TO_DEVICE;
 465                 dir_slave = DMA_MEM_TO_DEV;
 466         }
 467
 468         /* The block doesn't manage the FIFO DREQs properly for
 469          * multi-block transfers, so don't attempt to DMA the final
 470          * few words.  Unfortunately this requires the final sg entry
 471          * to be trimmed.  N.B. This code demands that the overspill
 472          * is contained in a single sg entry.
 473          */
 474
 475         host->drain_words = 0;
 476         if ((data->blocks > 1) && (dir_data == DMA_FROM_DEVICE)) {
 477                 struct scatterlist *sg;
 478                 u32 len;
 479                 int i;
 480
 481                 len = min((u32)(FIFO_READ_THRESHOLD - 1) * 4,
 482                           (u32)data->blocks * data->blksz);
 483
 484                 for_each_sg(data->sg, sg, data->sg_len, i) {
 485                         if (sg_is_last(sg)) {
 486                                 WARN_ON(sg->length < len);
 487                                 sg->length -= len;
 488                                 host->drain_page = sg_page(sg);
 489                                 host->drain_offset = sg->offset + sg->length;
 490                         }
 491                 }
 492                 host->drain_words = len / 4;
 493         }
 494
 495         /* The parameters have already been validated, so this will not fail */
 496         (void)dmaengine_slave_config(dma_chan,
 497                                      (dir_data == DMA_FROM_DEVICE) ?
 498                                      &host->dma_cfg_rx :
 499                                      &host->dma_cfg_tx);
 500
 501         sg_len = dma_map_sg(dma_chan->device->dev, data->sg, data->sg_len,
 502                             dir_data);
 503         if (!sg_len)
 504                 return;
 505
 506         desc = dmaengine_prep_slave_sg(dma_chan, data->sg, sg_len, dir_slave,
 507                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 508
 509         if (!desc) {
 510                 dma_unmap_sg(dma_chan->device->dev, data->sg, sg_len, dir_data);
 511                 return;
 512         }
 513
 514         desc->callback = bcm2835_dma_complete;
 515         desc->callback_param = host;
 516         host->dma_desc = desc;
 517         host->dma_chan = dma_chan;
 518         host->dma_dir = dir_data;
 519 }
 520
 521 static void bcm2835_start_dma(struct bcm2835_host *host)
 522 {
 523         dmaengine_submit(host->dma_desc);
 524         dma_async_issue_pending(host->dma_chan);
 525 }
 526
 527 static void bcm2835_set_transfer_irqs(struct bcm2835_host *host)
 528 {
 529         u32 all_irqs = SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN |
 530                 SDHCFG_BUSY_IRPT_EN;
 531
 532         if (host->dma_desc) {
 533                 host->hcfg = (host->hcfg & ~all_irqs) |
 534                         SDHCFG_BUSY_IRPT_EN;
 535         } else {
 536                 host->hcfg = (host->hcfg & ~all_irqs) |
 537                         SDHCFG_DATA_IRPT_EN |
 538                         SDHCFG_BUSY_IRPT_EN;
 539         }
 540
 541         writel(host->hcfg, host->ioaddr + SDHCFG);
 542 }
 543
 544 static
 545 void bcm2835_prepare_data(struct bcm2835_host *host, struct mmc_command *cmd)
 546 {
 547         struct mmc_data *data = cmd->data;
 548
 549         WARN_ON(host->data);
 550
 551         host->data = data;
 552         if (!data)
 553                 return;
 554
 555         host->data_complete = false;
 556         host->data->bytes_xfered = 0;
 557
 558         if (!host->dma_desc) {
 559                 /* Use PIO */
 560                 int flags = SG_MITER_ATOMIC;
 561
 562                 if (data->flags & MMC_DATA_READ)
 563                         flags |= SG_MITER_TO_SG;
 564                 else
 565                         flags |= SG_MITER_FROM_SG;
 566                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 567                 host->blocks = data->blocks;
 568         }
 569
 570         bcm2835_set_transfer_irqs(host);
 571
 572         writel(data->blksz, host->ioaddr + SDHBCT);
 573         writel(data->blocks, host->ioaddr + SDHBLC);
 574 }
 575
 576 static u32 bcm2835_read_wait_sdcmd(struct bcm2835_host *host, u32 max_ms)
 577 {
 578         struct device *dev = &host->pdev->dev;
 579         u32 value;
 580         int ret;
 581
 582         ret = readl_poll_timeout(host->ioaddr + SDCMD, value,
 583                                  !(value & SDCMD_NEW_FLAG), 1, 10);
 584         if (ret == -ETIMEDOUT)
 585                 /* if it takes a while make poll interval bigger */
 586                 ret = readl_poll_timeout(host->ioaddr + SDCMD, value,
 587                                          !(value & SDCMD_NEW_FLAG),
 588                                          10, max_ms * 1000);
 589         if (ret == -ETIMEDOUT)
 590                 dev_err(dev, "%s: timeout (%d ms)\n", __func__, max_ms);
 591
 592         return value;
 593 }
 594
 595 static void bcm2835_finish_request(struct bcm2835_host *host)
 596 {
 597         struct dma_chan *terminate_chan = NULL;
 598         struct mmc_request *mrq;
 599
 600         cancel_delayed_work(&host->timeout_work);
 601
 602         mrq = host->mrq;
 603
 604         host->mrq = NULL;
 605         host->cmd = NULL;
 606         host->data = NULL;
 607
 608         host->dma_desc = NULL;
 609         terminate_chan = host->dma_chan;
 610         host->dma_chan = NULL;
 611
 612         if (terminate_chan) {
 613                 int err = dmaengine_terminate_all(terminate_chan);
 614
 615                 if (err)
 616                         dev_err(&host->pdev->dev,
 617                                 "failed to terminate DMA (%d)\n", err);
 618         }
 619
 620         mmc_request_done(mmc_from_priv(host), mrq);
 621 }
 622
 623 static
 624 bool bcm2835_send_command(struct bcm2835_host *host, struct mmc_command *cmd)
 625 {
 626         struct device *dev = &host->pdev->dev;
 627         u32 sdcmd, sdhsts;
 628         unsigned long timeout;
 629
 630         WARN_ON(host->cmd);
 631
 632         sdcmd = bcm2835_read_wait_sdcmd(host, 100);
 633         if (sdcmd & SDCMD_NEW_FLAG) {
 634                 dev_err(dev, "previous command never completed.\n");
 635                 bcm2835_dumpregs(host);
 636                 cmd->error = -EILSEQ;
 637                 bcm2835_finish_request(host);
 638                 return false;
 639         }
 640
 641         if (!cmd->data && cmd->busy_timeout > 9000)
 642                 timeout = DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
 643         else
 644                 timeout = 10 * HZ;
 645         schedule_delayed_work(&host->timeout_work, timeout);
 646
 647         host->cmd = cmd;
 648
 649         /* Clear any error flags */
 650         sdhsts = readl(host->ioaddr + SDHSTS);
 651         if (sdhsts & SDHSTS_ERROR_MASK)
 652                 writel(sdhsts, host->ioaddr + SDHSTS);
 653
 654         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
 655                 dev_err(dev, "unsupported response type!\n");
 656                 cmd->error = -EINVAL;
 657                 bcm2835_finish_request(host);
 658                 return false;
 659         }
 660
 661         bcm2835_prepare_data(host, cmd);
 662
 663         writel(cmd->arg, host->ioaddr + SDARG);
 664
 665         sdcmd = cmd->opcode & SDCMD_CMD_MASK;
 666
 667         host->use_busy = false;
 668         if (!(cmd->flags & MMC_RSP_PRESENT)) {
 669                 sdcmd |= SDCMD_NO_RESPONSE;
 670         } else {
 671                 if (cmd->flags & MMC_RSP_136)
 672                         sdcmd |= SDCMD_LONG_RESPONSE;
 673                 if (cmd->flags & MMC_RSP_BUSY) {
 674                         sdcmd |= SDCMD_BUSYWAIT;
 675                         host->use_busy = true;
 676                 }
 677         }
 678
 679         if (cmd->data) {
 680                 if (cmd->data->flags & MMC_DATA_WRITE)
 681                         sdcmd |= SDCMD_WRITE_CMD;
 682                 if (cmd->data->flags & MMC_DATA_READ)
 683                         sdcmd |= SDCMD_READ_CMD;
 684         }
 685
 686         writel(sdcmd | SDCMD_NEW_FLAG, host->ioaddr + SDCMD);
 687
 688         return true;
 689 }
 690
 691 static void bcm2835_transfer_complete(struct bcm2835_host *host)
 692 {
 693         struct mmc_data *data;
 694
 695         WARN_ON(!host->data_complete);
 696
 697         data = host->data;
 698         host->data = NULL;
 699
 700         /* Need to send CMD12 if -
 701          * a) open-ended multiblock transfer (no CMD23)
 702          * b) error in multiblock transfer
 703          */
 704         if (host->mrq->stop && (data->error || !host->use_sbc)) {
 705                 if (bcm2835_send_command(host, host->mrq->stop)) {
 706                         /* No busy, so poll for completion */
 707                         if (!host->use_busy)
 708                                 bcm2835_finish_command(host);
 709                 }
 710         } else {
 711                 bcm2835_wait_transfer_complete(host);
 712                 bcm2835_finish_request(host);
 713         }
 714 }
 715
 716 static void bcm2835_finish_data(struct bcm2835_host *host)
 717 {
 718         struct device *dev = &host->pdev->dev;
 719         struct mmc_data *data;
 720
 721         data = host->data;
 722
 723         host->hcfg &= ~(SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN);
 724         writel(host->hcfg, host->ioaddr + SDHCFG);
 725
 726         data->bytes_xfered = data->error ? 0 : (data->blksz * data->blocks);
 727
 728         host->data_complete = true;
 729
 730         if (host->cmd) {
 731                 /* Data managed to finish before the
 732                  * command completed. Make sure we do
 733                  * things in the proper order.
 734                  */
 735                 dev_dbg(dev, "Finished early - HSTS %08x\n",
 736                         readl(host->ioaddr + SDHSTS));
 737         } else {
 738                 bcm2835_transfer_complete(host);
 739         }
 740 }
 741
 742 static void bcm2835_finish_command(struct bcm2835_host *host)
 743 {
 744         struct device *dev = &host->pdev->dev;
 745         struct mmc_command *cmd = host->cmd;
 746         u32 sdcmd;
 747
 748         sdcmd = bcm2835_read_wait_sdcmd(host, 100);
 749
 750         /* Check for errors */
 751         if (sdcmd & SDCMD_NEW_FLAG) {
 752                 dev_err(dev, "command never completed.\n");
 753                 bcm2835_dumpregs(host);
 754                 host->cmd->error = -EIO;
 755                 bcm2835_finish_request(host);
 756                 return;
 757         } else if (sdcmd & SDCMD_FAIL_FLAG) {
 758                 u32 sdhsts = readl(host->ioaddr + SDHSTS);
 759
 760                 /* Clear the errors */
 761                 writel(SDHSTS_ERROR_MASK, host->ioaddr + SDHSTS);
 762
 763                 if (!(sdhsts & SDHSTS_CRC7_ERROR) ||
 764                     (host->cmd->opcode != MMC_SEND_OP_COND)) {
 765                         u32 edm, fsm;
 766
 767                         if (sdhsts & SDHSTS_CMD_TIME_OUT) {
 768                                 host->cmd->error = -ETIMEDOUT;
 769                         } else {
 770                                 dev_err(dev, "unexpected command %d error\n",
 771                                         host->cmd->opcode);
 772                                 bcm2835_dumpregs(host);
 773                                 host->cmd->error = -EILSEQ;
 774                         }
 775                         edm = readl(host->ioaddr + SDEDM);
 776                         fsm = edm & SDEDM_FSM_MASK;
 777                         if (fsm == SDEDM_FSM_READWAIT ||
 778                             fsm == SDEDM_FSM_WRITESTART1)
 779                                 /* Kick the FSM out of its wait */
 780                                 writel(edm | SDEDM_FORCE_DATA_MODE,
 781                                        host->ioaddr + SDEDM);
 782                         bcm2835_finish_request(host);
 783                         return;
 784                 }
 785         }
 786
 787         if (cmd->flags & MMC_RSP_PRESENT) {
 788                 if (cmd->flags & MMC_RSP_136) {
 789                         int i;
 790
 791                         for (i = 0; i < 4; i++) {
 792                                 cmd->resp[3 - i] =
 793                                         readl(host->ioaddr + SDRSP0 + i * 4);
 794                         }
 795                 } else {
 796                         cmd->resp[0] = readl(host->ioaddr + SDRSP0);
 797                 }
 798         }
 799
 800         if (cmd == host->mrq->sbc) {
 801                 /* Finished CMD23, now send actual command. */
 802                 host->cmd = NULL;
 803                 if (bcm2835_send_command(host, host->mrq->cmd)) {
 804                         if (host->data && host->dma_desc)
 805                                 /* DMA transfer starts now, PIO starts
 806                                  * after irq
 807                                  */
 808                                 bcm2835_start_dma(host);
 809
 810                         if (!host->use_busy)
 811                                 bcm2835_finish_command(host);
 812                 }
 813         } else if (cmd == host->mrq->stop) {
 814                 /* Finished CMD12 */
 815                 bcm2835_finish_request(host);
 816         } else {
 817                 /* Processed actual command. */
 818                 host->cmd = NULL;
 819                 if (!host->data)
 820                         bcm2835_finish_request(host);
 821                 else if (host->data_complete)
 822                         bcm2835_transfer_complete(host);
 823         }
 824 }
 825
 826 static void bcm2835_timeout(struct work_struct *work)
 827 {
 828         struct delayed_work *d = to_delayed_work(work);
 829         struct bcm2835_host *host =
 830                 container_of(d, struct bcm2835_host, timeout_work);
 831         struct device *dev = &host->pdev->dev;
 832
 833         mutex_lock(&host->mutex);
 834
 835         if (host->mrq) {
 836                 dev_err(dev, "timeout waiting for hardware interrupt.\n");
 837                 bcm2835_dumpregs(host);
 838
 839                 bcm2835_reset(mmc_from_priv(host));
 840
 841                 if (host->data) {
 842                         host->data->error = -ETIMEDOUT;
 843                         bcm2835_finish_data(host);
 844                 } else {
 845                         if (host->cmd)
 846                                 host->cmd->error = -ETIMEDOUT;
 847                         else
 848                                 host->mrq->cmd->error = -ETIMEDOUT;
 849
 850                         bcm2835_finish_request(host);
 851                 }
 852         }
 853
 854         mutex_unlock(&host->mutex);
 855 }
 856
 857 static bool bcm2835_check_cmd_error(struct bcm2835_host *host, u32 intmask)
 858 {
 859         struct device *dev = &host->pdev->dev;
 860
 861         if (!(intmask & SDHSTS_ERROR_MASK))
 862                 return false;
 863
 864         if (!host->cmd)
 865                 return true;
 866
 867         dev_err(dev, "sdhost_busy_irq: intmask %08x\n", intmask);
 868         if (intmask & SDHSTS_CRC7_ERROR) {
 869                 host->cmd->error = -EILSEQ;
 870         } else if (intmask & (SDHSTS_CRC16_ERROR |
 871                               SDHSTS_FIFO_ERROR)) {
 872                 if (host->mrq->data)
 873                         host->mrq->data->error = -EILSEQ;
 874                 else
 875                         host->cmd->error = -EILSEQ;
 876         } else if (intmask & SDHSTS_REW_TIME_OUT) {
 877                 if (host->mrq->data)
 878                         host->mrq->data->error = -ETIMEDOUT;
 879                 else
 880                         host->cmd->error = -ETIMEDOUT;
 881         } else if (intmask & SDHSTS_CMD_TIME_OUT) {
 882                 host->cmd->error = -ETIMEDOUT;
 883         }
 884         bcm2835_dumpregs(host);
 885         return true;
 886 }
 887
 888 static void bcm2835_check_data_error(struct bcm2835_host *host, u32 intmask)
 889 {
 890         if (!host->data)
 891                 return;
 892         if (intmask & (SDHSTS_CRC16_ERROR | SDHSTS_FIFO_ERROR))
 893                 host->data->error = -EILSEQ;
 894         if (intmask & SDHSTS_REW_TIME_OUT)
 895                 host->data->error = -ETIMEDOUT;
 896 }
 897
 898 static void bcm2835_busy_irq(struct bcm2835_host *host)
 899 {
 900         if (WARN_ON(!host->cmd)) {
 901                 bcm2835_dumpregs(host);
 902                 return;
 903         }
 904
 905         if (WARN_ON(!host->use_busy)) {
 906                 bcm2835_dumpregs(host);
 907                 return;
 908         }
 909         host->use_busy = false;
 910
 911         bcm2835_finish_command(host);
 912 }
 913
 914 static void bcm2835_data_irq(struct bcm2835_host *host, u32 intmask)
 915 {
 916         /* There are no dedicated data/space available interrupt
 917          * status bits, so it is necessary to use the single shared
 918          * data/space available FIFO status bits. It is therefore not
 919          * an error to get here when there is no data transfer in
 920          * progress.
 921          */
 922         if (!host->data)
 923                 return;
 924
 925         bcm2835_check_data_error(host, intmask);
 926         if (host->data->error)
 927                 goto finished;
 928
 929         if (host->data->flags & MMC_DATA_WRITE) {
 930                 /* Use the block interrupt for writes after the first block */
 931                 host->hcfg &= ~(SDHCFG_DATA_IRPT_EN);
 932                 host->hcfg |= SDHCFG_BLOCK_IRPT_EN;
 933                 writel(host->hcfg, host->ioaddr + SDHCFG);
 934                 bcm2835_transfer_pio(host);
 935         } else {
 936                 bcm2835_transfer_pio(host);
 937                 host->blocks--;
 938                 if ((host->blocks == 0) || host->data->error)
 939                         goto finished;
 940         }
 941         return;
 942
 943 finished:
 944         host->hcfg &= ~(SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN);
 945         writel(host->hcfg, host->ioaddr + SDHCFG);
 946 }
 947
 948 static void bcm2835_data_threaded_irq(struct bcm2835_host *host)
 949 {
 950         if (!host->data)
 951                 return;
 952         if ((host->blocks == 0) || host->data->error)
 953                 bcm2835_finish_data(host);
 954 }
 955
 956 static void bcm2835_block_irq(struct bcm2835_host *host)
 957 {
 958         if (WARN_ON(!host->data)) {
 959                 bcm2835_dumpregs(host);
 960                 return;
 961         }
 962
 963         if (!host->dma_desc) {
 964                 WARN_ON(!host->blocks);
 965                 if (host->data->error || (--host->blocks == 0))
 966                         bcm2835_finish_data(host);
 967                 else
 968                         bcm2835_transfer_pio(host);
 969         } else if (host->data->flags & MMC_DATA_WRITE) {
 970                 bcm2835_finish_data(host);
 971         }
 972 }
 973
 974 static irqreturn_t bcm2835_irq(int irq, void *dev_id)
 975 {
 976         irqreturn_t result = IRQ_NONE;
 977         struct bcm2835_host *host = dev_id;
 978         u32 intmask;
 979
 980         spin_lock(&host->lock);
 981
 982         intmask = readl(host->ioaddr + SDHSTS);
 983
 984         writel(SDHSTS_BUSY_IRPT |
 985                SDHSTS_BLOCK_IRPT |
 986                SDHSTS_SDIO_IRPT |
 987                SDHSTS_DATA_FLAG,
 988                host->ioaddr + SDHSTS);
 989
 990         if (intmask & SDHSTS_BLOCK_IRPT) {
 991                 bcm2835_check_data_error(host, intmask);
 992                 host->irq_block = true;
 993                 result = IRQ_WAKE_THREAD;
 994         }
 995
 996         if (intmask & SDHSTS_BUSY_IRPT) {
 997                 if (!bcm2835_check_cmd_error(host, intmask)) {
 998                         host->irq_busy = true;
 999                         result = IRQ_WAKE_THREAD;
1000                 } else {
1001                         result = IRQ_HANDLED;
1002                 }
1003         }
1004
1005         /* There is no true data interrupt status bit, so it is
1006          * necessary to qualify the data flag with the interrupt
1007          * enable bit.
1008          */
1009         if ((intmask & SDHSTS_DATA_FLAG) &&
1010             (host->hcfg & SDHCFG_DATA_IRPT_EN)) {
1011                 bcm2835_data_irq(host, intmask);
1012                 host->irq_data = true;
1013                 result = IRQ_WAKE_THREAD;
1014         }
1015
1016         spin_unlock(&host->lock);
1017
1018         return result;
1019 }
1020
1021 static irqreturn_t bcm2835_threaded_irq(int irq, void *dev_id)
1022 {
1023         struct bcm2835_host *host = dev_id;
1024         unsigned long flags;
1025         bool block, busy, data;
1026
1027         spin_lock_irqsave(&host->lock, flags);
1028
1029         block = host->irq_block;
1030         busy  = host->irq_busy;
1031         data  = host->irq_data;
1032         host->irq_block = false;
1033         host->irq_busy  = false;
1034         host->irq_data  = false;
1035
1036         spin_unlock_irqrestore(&host->lock, flags);
1037
1038         mutex_lock(&host->mutex);
1039
1040         if (block)
1041                 bcm2835_block_irq(host);
1042         if (busy)
1043                 bcm2835_busy_irq(host);
1044         if (data)
1045                 bcm2835_data_threaded_irq(host);
1046
1047         mutex_unlock(&host->mutex);
1048
1049         return IRQ_HANDLED;
1050 }
1051
1052 static void bcm2835_dma_complete_work(struct work_struct *work)
1053 {
1054         struct bcm2835_host *host =
1055                 container_of(work, struct bcm2835_host, dma_work);
1056         struct mmc_data *data;
1057
1058         mutex_lock(&host->mutex);
1059
1060         data = host->data;
1061
1062         if (host->dma_chan) {
1063                 dma_unmap_sg(host->dma_chan->device->dev,
1064                              data->sg, data->sg_len,
1065                              host->dma_dir);
1066
1067                 host->dma_chan = NULL;
1068         }
1069
1070         if (host->drain_words) {
1071                 unsigned long flags;
1072                 void *page;
1073                 u32 *buf;
1074
1075                 if (host->drain_offset & PAGE_MASK) {
1076                         host->drain_page += host->drain_offset >> PAGE_SHIFT;
1077                         host->drain_offset &= ~PAGE_MASK;
1078                 }
1079                 local_irq_save(flags);
1080                 page = kmap_atomic(host->drain_page);
1081                 buf = page + host->drain_offset;
1082
1083                 while (host->drain_words) {
1084                         u32 edm = readl(host->ioaddr + SDEDM);
1085
1086                         if ((edm >> 4) & 0x1f)
1087                                 *(buf++) = readl(host->ioaddr + SDDATA);
1088                         host->drain_words--;
1089                 }
1090
1091                 kunmap_atomic(page);
1092                 local_irq_restore(flags);
1093         }
1094
1095         bcm2835_finish_data(host);
1096
1097         mutex_unlock(&host->mutex);
1098 }
1099
1100 static void bcm2835_set_clock(struct bcm2835_host *host, unsigned int clock)
1101 {
1102         struct mmc_host *mmc = mmc_from_priv(host);
1103         int div;
1104
1105         /* The SDCDIV register has 11 bits, and holds (div - 2).  But
1106          * in data mode the max is 50MHz wihout a minimum, and only
1107          * the bottom 3 bits are used. Since the switch over is
1108          * automatic (unless we have marked the card as slow...),
1109          * chosen values have to make sense in both modes.  Ident mode
1110          * must be 100-400KHz, so can range check the requested
1111          * clock. CMD15 must be used to return to data mode, so this
1112          * can be monitored.
1113          *
1114          * clock 250MHz -> 0->125MHz, 1->83.3MHz, 2->62.5MHz, 3->50.0MHz
1115          *                 4->41.7MHz, 5->35.7MHz, 6->31.3MHz, 7->27.8MHz
1116          *
1117          *               623->400KHz/27.8MHz
1118          *               reset value (507)->491159/50MHz
1119          *
1120          * BUT, the 3-bit clock divisor in data mode is too small if
1121          * the core clock is higher than 250MHz, so instead use the
1122          * SLOW_CARD configuration bit to force the use of the ident
1123          * clock divisor at all times.
1124          */
1125
1126         if (clock < 100000) {
1127                 /* Can't stop the clock, but make it as slow as possible
1128                  * to show willing
1129                  */
1130                 host->cdiv = SDCDIV_MAX_CDIV;
1131                 writel(host->cdiv, host->ioaddr + SDCDIV);
1132                 return;
1133         }
1134
1135         div = host->max_clk / clock;
1136         if (div < 2)
1137                 div = 2;
1138         if ((host->max_clk / div) > clock)
1139                 div++;
1140         div -= 2;
1141
1142         if (div > SDCDIV_MAX_CDIV)
1143                 div = SDCDIV_MAX_CDIV;
1144
1145         clock = host->max_clk / (div + 2);
1146         mmc->actual_clock = clock;
1147
1148         /* Calibrate some delays */
1149
1150         host->ns_per_fifo_word = (1000000000 / clock) *
1151                 ((mmc->caps & MMC_CAP_4_BIT_DATA) ? 8 : 32);
1152
1153         host->cdiv = div;
1154         writel(host->cdiv, host->ioaddr + SDCDIV);
1155
1156         /* Set the timeout to 500ms */
1157         writel(mmc->actual_clock / 2, host->ioaddr + SDTOUT);
1158 }
1159
1160 static void bcm2835_request(struct mmc_host *mmc, struct mmc_request *mrq)
1161 {
1162         struct bcm2835_host *host = mmc_priv(mmc);
1163         struct device *dev = &host->pdev->dev;
1164         u32 edm, fsm;
1165
1166         /* Reset the error statuses in case this is a retry */
1167         if (mrq->sbc)
1168                 mrq->sbc->error = 0;
1169         if (mrq->cmd)
1170                 mrq->cmd->error = 0;
1171         if (mrq->data)
1172                 mrq->data->error = 0;
1173         if (mrq->stop)
1174                 mrq->stop->error = 0;
1175
1176         if (mrq->data && !is_power_of_2(mrq->data->blksz)) {
1177                 dev_err(dev, "unsupported block size (%d bytes)\n",
1178                         mrq->data->blksz);
1179
1180                 if (mrq->cmd)
1181                         mrq->cmd->error = -EINVAL;
1182
1183                 mmc_request_done(mmc, mrq);
1184                 return;
1185         }
1186
1187         mutex_lock(&host->mutex);
1188
1189         WARN_ON(host->mrq);
1190         host->mrq = mrq;
1191
1192         edm = readl(host->ioaddr + SDEDM);
1193         fsm = edm & SDEDM_FSM_MASK;
1194
1195         if ((fsm != SDEDM_FSM_IDENTMODE) &&
1196             (fsm != SDEDM_FSM_DATAMODE)) {
1197                 dev_err(dev, "previous command (%d) not complete (EDM %08x)\n",
1198                         readl(host->ioaddr + SDCMD) & SDCMD_CMD_MASK,
1199                         edm);
1200                 bcm2835_dumpregs(host);
1201
1202                 if (mrq->cmd)
1203                         mrq->cmd->error = -EILSEQ;
1204
1205                 bcm2835_finish_request(host);
1206                 mutex_unlock(&host->mutex);
1207                 return;
1208         }
1209
1210         if (host->use_dma && mrq->data && (mrq->data->blocks > PIO_THRESHOLD))
1211                 bcm2835_prepare_dma(host, mrq->data);
1212
1213         host->use_sbc = !!mrq->sbc && host->mrq->data &&
1214                         (host->mrq->data->flags & MMC_DATA_READ);
1215         if (host->use_sbc) {
1216                 if (bcm2835_send_command(host, mrq->sbc)) {
1217                         if (!host->use_busy)
1218                                 bcm2835_finish_command(host);
1219                 }
1220         } else if (mrq->cmd && bcm2835_send_command(host, mrq->cmd)) {
1221                 if (host->data && host->dma_desc) {
1222                         /* DMA transfer starts now, PIO starts after irq */
1223                         bcm2835_start_dma(host);
1224                 }
1225
1226                 if (!host->use_busy)
1227                         bcm2835_finish_command(host);
1228         }
1229
1230         mutex_unlock(&host->mutex);
1231 }
1232
1233 static void bcm2835_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1234 {
1235         struct bcm2835_host *host = mmc_priv(mmc);
1236
1237         mutex_lock(&host->mutex);
1238
1239         if (!ios->clock || ios->clock != host->clock) {
1240                 bcm2835_set_clock(host, ios->clock);
1241                 host->clock = ios->clock;
1242         }
1243
1244         /* set bus width */
1245         host->hcfg &= ~SDHCFG_WIDE_EXT_BUS;
1246         if (ios->bus_width == MMC_BUS_WIDTH_4)
1247                 host->hcfg |= SDHCFG_WIDE_EXT_BUS;
1248
1249         host->hcfg |= SDHCFG_WIDE_INT_BUS;
1250
1251         /* Disable clever clock switching, to cope with fast core clocks */
1252         host->hcfg |= SDHCFG_SLOW_CARD;
1253
1254         writel(host->hcfg, host->ioaddr + SDHCFG);
1255
1256         mutex_unlock(&host->mutex);
1257 }
1258
1259 static const struct mmc_host_ops bcm2835_ops = {
1260         .request = bcm2835_request,
1261         .set_ios = bcm2835_set_ios,
1262         .hw_reset = bcm2835_reset,
1263 };
1264
1265 static int bcm2835_add_host(struct bcm2835_host *host)
1266 {
1267         struct mmc_host *mmc = mmc_from_priv(host);
1268         struct device *dev = &host->pdev->dev;
1269         char pio_limit_string[20];
1270         int ret;
1271
1272         if (!mmc->f_max || mmc->f_max > host->max_clk)
1273                 mmc->f_max = host->max_clk;
1274         mmc->f_min = host->max_clk / SDCDIV_MAX_CDIV;
1275
1276         mmc->max_busy_timeout = ~0 / (mmc->f_max / 1000);
1277
1278         dev_dbg(dev, "f_max %d, f_min %d, max_busy_timeout %d\n",
1279                 mmc->f_max, mmc->f_min, mmc->max_busy_timeout);
1280
1281         /* host controller capabilities */
1282         mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED |
1283                      MMC_CAP_NEEDS_POLL | MMC_CAP_HW_RESET | MMC_CAP_ERASE |
1284                      MMC_CAP_CMD23;
1285
1286         spin_lock_init(&host->lock);
1287         mutex_init(&host->mutex);
1288
1289         if (!host->dma_chan_rxtx) {
1290                 dev_warn(dev, "unable to initialise DMA channel. Falling back to PIO\n");
1291                 host->use_dma = false;
1292         } else {
1293                 host->use_dma = true;
1294
1295                 host->dma_cfg_tx.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1296                 host->dma_cfg_tx.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1297                 host->dma_cfg_tx.slave_id = 13;         /* DREQ channel */
1298                 host->dma_cfg_tx.direction = DMA_MEM_TO_DEV;
1299                 host->dma_cfg_tx.src_addr = 0;
1300                 host->dma_cfg_tx.dst_addr = host->phys_addr + SDDATA;
1301
1302                 host->dma_cfg_rx.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1303                 host->dma_cfg_rx.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1304                 host->dma_cfg_rx.slave_id = 13;         /* DREQ channel */
1305                 host->dma_cfg_rx.direction = DMA_DEV_TO_MEM;
1306                 host->dma_cfg_rx.src_addr = host->phys_addr + SDDATA;
1307                 host->dma_cfg_rx.dst_addr = 0;
1308
1309                 if (dmaengine_slave_config(host->dma_chan_rxtx,
1310                                            &host->dma_cfg_tx) != 0 ||
1311                     dmaengine_slave_config(host->dma_chan_rxtx,
1312                                            &host->dma_cfg_rx) != 0)
1313                         host->use_dma = false;
1314         }
1315
1316         mmc->max_segs = 128;
1317         mmc->max_req_size = min_t(size_t, 524288, dma_max_mapping_size(dev));
1318         mmc->max_seg_size = mmc->max_req_size;
1319         mmc->max_blk_size = 1024;
1320         mmc->max_blk_count =  65535;
1321
1322         /* report supported voltage ranges */
1323         mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
1324
1325         INIT_WORK(&host->dma_work, bcm2835_dma_complete_work);
1326         INIT_DELAYED_WORK(&host->timeout_work, bcm2835_timeout);
1327
1328         /* Set interrupt enables */
1329         host->hcfg = SDHCFG_BUSY_IRPT_EN;
1330
1331         bcm2835_reset_internal(host);
1332
1333         ret = request_threaded_irq(host->irq, bcm2835_irq,
1334                                    bcm2835_threaded_irq,
1335                                    0, mmc_hostname(mmc), host);
1336         if (ret) {
1337                 dev_err(dev, "failed to request IRQ %d: %d\n", host->irq, ret);
1338                 return ret;
1339         }
1340
1341         ret = mmc_add_host(mmc);
1342         if (ret) {
1343                 free_irq(host->irq, host);
1344                 return ret;
1345         }
1346
1347         pio_limit_string[0] = '\0';
1348         if (host->use_dma && (PIO_THRESHOLD > 0))
1349                 sprintf(pio_limit_string, " (>%d)", PIO_THRESHOLD);
1350         dev_info(dev, "loaded - DMA %s%s\n",
1351                  host->use_dma ? "enabled" : "disabled", pio_limit_string);
1352
1353         return 0;
1354 }
1355
1356 static int bcm2835_probe(struct platform_device *pdev)
1357 {
1358         struct device *dev = &pdev->dev;
1359         struct clk *clk;
1360         struct resource *iomem;
1361         struct bcm2835_host *host;
1362         struct mmc_host *mmc;
1363         const __be32 *regaddr_p;
1364         int ret;
1365
1366         dev_dbg(dev, "%s\n", __func__);
1367         mmc = mmc_alloc_host(sizeof(*host), dev);
1368         if (!mmc)
1369                 return -ENOMEM;
1370
1371         mmc->ops = &bcm2835_ops;
1372         host = mmc_priv(mmc);
1373         host->pdev = pdev;
1374         spin_lock_init(&host->lock);
1375
1376         iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1377         host->ioaddr = devm_ioremap_resource(dev, iomem);
1378         if (IS_ERR(host->ioaddr)) {
1379                 ret = PTR_ERR(host->ioaddr);
1380                 goto err;
1381         }
1382
1383         /* Parse OF address directly to get the physical address for
1384          * DMA to our registers.
1385          */
1386         regaddr_p = of_get_address(pdev->dev.of_node, 0, NULL, NULL);
1387         if (!regaddr_p) {
1388                 dev_err(dev, "Can't get phys address\n");
1389                 ret = -EINVAL;
1390                 goto err;
1391         }
1392
1393         host->phys_addr = be32_to_cpup(regaddr_p);
1394
1395         host->dma_chan = NULL;
1396         host->dma_desc = NULL;
1397
1398         host->dma_chan_rxtx = dma_request_slave_channel(dev, "rx-tx");
1399
1400         clk = devm_clk_get(dev, NULL);
1401         if (IS_ERR(clk)) {
1402                 ret = PTR_ERR(clk);
1403                 if (ret != -EPROBE_DEFER)
1404                         dev_err(dev, "could not get clk: %d\n", ret);
1405                 goto err;
1406         }
1407
1408         host->max_clk = clk_get_rate(clk);
1409
1410         host->irq = platform_get_irq(pdev, 0);
1411         if (host->irq <= 0) {
1412                 ret = -EINVAL;
1413                 goto err;
1414         }
1415
1416         ret = mmc_of_parse(mmc);
1417         if (ret)
1418                 goto err;
1419
1420         ret = bcm2835_add_host(host);
1421         if (ret)
1422                 goto err;
1423
1424         platform_set_drvdata(pdev, host);
1425
1426         dev_dbg(dev, "%s -> OK\n", __func__);
1427
1428         return 0;
1429
1430 err:
1431         dev_dbg(dev, "%s -> err %d\n", __func__, ret);
1432         if (host->dma_chan_rxtx)
1433                 dma_release_channel(host->dma_chan_rxtx);
1434         mmc_free_host(mmc);
1435
1436         return ret;
1437 }
1438
1439 static int bcm2835_remove(struct platform_device *pdev)
1440 {
1441         struct bcm2835_host *host = platform_get_drvdata(pdev);
1442         struct mmc_host *mmc = mmc_from_priv(host);
1443
1444         mmc_remove_host(mmc);
1445
1446         writel(SDVDD_POWER_OFF, host->ioaddr + SDVDD);
1447
1448         free_irq(host->irq, host);
1449
1450         cancel_work_sync(&host->dma_work);
1451         cancel_delayed_work_sync(&host->timeout_work);
1452
1453         if (host->dma_chan_rxtx)
1454                 dma_release_channel(host->dma_chan_rxtx);
1455
1456         mmc_free_host(mmc);
1457
1458         return 0;
1459 }
1460
1461 static const struct of_device_id bcm2835_match[] = {
1462         { .compatible = "brcm,bcm2835-sdhost" },
1463         { }
1464 };
1465 MODULE_DEVICE_TABLE(of, bcm2835_match);
1466
1467 static struct platform_driver bcm2835_driver = {
1468         .probe      = bcm2835_probe,
1469         .remove     = bcm2835_remove,
1470         .driver     = {
1471                 .name           = "sdhost-bcm2835",
1472                 .of_match_table = bcm2835_match,
1473         },
1474 };
1475 module_platform_driver(bcm2835_driver);
1476
1477 MODULE_ALIAS("platform:sdhost-bcm2835");
1478 MODULE_DESCRIPTION("BCM2835 SDHost driver");
1479 MODULE_LICENSE("GPL v2");
1480 MODULE_AUTHOR("Phil Elwell");