drivers/crypto/omap-aes.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for OMAP AES HW acceleration.
   5  *
   6  * Copyright (c) 2010 Nokia Corporation
   7  * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
   8  * Copyright (c) 2011 Texas Instruments Incorporated
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as published
  12  * by the Free Software Foundation.
  13  *
  14  */
  15
  16 #define pr_fmt(fmt) "%20s: " fmt, __func__
  17 #define prn(num) pr_debug(#num "=%d\n", num)
  18 #define prx(num) pr_debug(#num "=%x\n", num)
  19
  20 #include <linux/err.h>
  21 #include <linux/module.h>
  22 #include <linux/init.h>
  23 #include <linux/errno.h>
  24 #include <linux/kernel.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/scatterlist.h>
  27 #include <linux/dma-mapping.h>
  28 #include <linux/dmaengine.h>
  29 #include <linux/pm_runtime.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_address.h>
  33 #include <linux/io.h>
  34 #include <linux/crypto.h>
  35 #include <linux/interrupt.h>
  36 #include <crypto/scatterwalk.h>
  37 #include <crypto/aes.h>
  38 #include <crypto/engine.h>
  39 #include <crypto/internal/skcipher.h>
  40 #include <crypto/internal/aead.h>
  41
  42 #include "omap-crypto.h"
  43 #include "omap-aes.h"
  44
  45 /* keep registered devices data here */
  46 static LIST_HEAD(dev_list);
  47 static DEFINE_SPINLOCK(list_lock);
  48
  49 #ifdef DEBUG
  50 #define omap_aes_read(dd, offset)                               \
  51 ({                                                              \
  52         int _read_ret;                                          \
  53         _read_ret = __raw_readl(dd->io_base + offset);          \
  54         pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n",       \
  55                  offset, _read_ret);                            \
  56         _read_ret;                                              \
  57 })
  58 #else
  59 inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
  60 {
  61         return __raw_readl(dd->io_base + offset);
  62 }
  63 #endif
  64
  65 #ifdef DEBUG
  66 #define omap_aes_write(dd, offset, value)                               \
  67         do {                                                            \
  68                 pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \
  69                          offset, value);                                \
  70                 __raw_writel(value, dd->io_base + offset);              \
  71         } while (0)
  72 #else
  73 inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
  74                                   u32 value)
  75 {
  76         __raw_writel(value, dd->io_base + offset);
  77 }
  78 #endif
  79
  80 static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
  81                                         u32 value, u32 mask)
  82 {
  83         u32 val;
  84
  85         val = omap_aes_read(dd, offset);
  86         val &= ~mask;
  87         val |= value;
  88         omap_aes_write(dd, offset, val);
  89 }
  90
  91 static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
  92                                         u32 *value, int count)
  93 {
  94         for (; count--; value++, offset += 4)
  95                 omap_aes_write(dd, offset, *value);
  96 }
  97
  98 static int omap_aes_hw_init(struct omap_aes_dev *dd)
  99 {
 100         int err;
 101
 102         if (!(dd->flags & FLAGS_INIT)) {
 103                 dd->flags |= FLAGS_INIT;
 104                 dd->err = 0;
 105         }
 106
 107         err = pm_runtime_get_sync(dd->dev);
 108         if (err < 0) {
 109                 dev_err(dd->dev, "failed to get sync: %d\n", err);
 110                 return err;
 111         }
 112
 113         return 0;
 114 }
 115
 116 void omap_aes_clear_copy_flags(struct omap_aes_dev *dd)
 117 {
 118         dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_IN_DATA_ST_SHIFT);
 119         dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_OUT_DATA_ST_SHIFT);
 120         dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_ASSOC_DATA_ST_SHIFT);
 121 }
 122
 123 int omap_aes_write_ctrl(struct omap_aes_dev *dd)
 124 {
 125         struct omap_aes_reqctx *rctx;
 126         unsigned int key32;
 127         int i, err;
 128         u32 val;
 129
 130         err = omap_aes_hw_init(dd);
 131         if (err)
 132                 return err;
 133
 134         key32 = dd->ctx->keylen / sizeof(u32);
 135
 136         /* RESET the key as previous HASH keys should not get affected*/
 137         if (dd->flags & FLAGS_GCM)
 138                 for (i = 0; i < 0x40; i = i + 4)
 139                         omap_aes_write(dd, i, 0x0);
 140
 141         for (i = 0; i < key32; i++) {
 142                 omap_aes_write(dd, AES_REG_KEY(dd, i),
 143                         __le32_to_cpu(dd->ctx->key[i]));
 144         }
 145
 146         if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->info)
 147                 omap_aes_write_n(dd, AES_REG_IV(dd, 0), dd->req->info, 4);
 148
 149         if ((dd->flags & (FLAGS_GCM)) && dd->aead_req->iv) {
 150                 rctx = aead_request_ctx(dd->aead_req);
 151                 omap_aes_write_n(dd, AES_REG_IV(dd, 0), (u32 *)rctx->iv, 4);
 152         }
 153
 154         val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
 155         if (dd->flags & FLAGS_CBC)
 156                 val |= AES_REG_CTRL_CBC;
 157
 158         if (dd->flags & (FLAGS_CTR | FLAGS_GCM))
 159                 val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128;
 160
 161         if (dd->flags & FLAGS_GCM)
 162                 val |= AES_REG_CTRL_GCM;
 163
 164         if (dd->flags & FLAGS_ENCRYPT)
 165                 val |= AES_REG_CTRL_DIRECTION;
 166
 167         omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK);
 168
 169         return 0;
 170 }
 171
 172 static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length)
 173 {
 174         u32 mask, val;
 175
 176         val = dd->pdata->dma_start;
 177
 178         if (dd->dma_lch_out != NULL)
 179                 val |= dd->pdata->dma_enable_out;
 180         if (dd->dma_lch_in != NULL)
 181                 val |= dd->pdata->dma_enable_in;
 182
 183         mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
 184                dd->pdata->dma_start;
 185
 186         omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask);
 187
 188 }
 189
 190 static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length)
 191 {
 192         omap_aes_write(dd, AES_REG_LENGTH_N(0), length);
 193         omap_aes_write(dd, AES_REG_LENGTH_N(1), 0);
 194         if (dd->flags & FLAGS_GCM)
 195                 omap_aes_write(dd, AES_REG_A_LEN, dd->assoc_len);
 196
 197         omap_aes_dma_trigger_omap2(dd, length);
 198 }
 199
 200 static void omap_aes_dma_stop(struct omap_aes_dev *dd)
 201 {
 202         u32 mask;
 203
 204         mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
 205                dd->pdata->dma_start;
 206
 207         omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask);
 208 }
 209
 210 struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx)
 211 {
 212         struct omap_aes_dev *dd;
 213
 214         spin_lock_bh(&list_lock);
 215         dd = list_first_entry(&dev_list, struct omap_aes_dev, list);
 216         list_move_tail(&dd->list, &dev_list);
 217         rctx->dd = dd;
 218         spin_unlock_bh(&list_lock);
 219
 220         return dd;
 221 }
 222
 223 static void omap_aes_dma_out_callback(void *data)
 224 {
 225         struct omap_aes_dev *dd = data;
 226
 227         /* dma_lch_out - completed */
 228         tasklet_schedule(&dd->done_task);
 229 }
 230
 231 static int omap_aes_dma_init(struct omap_aes_dev *dd)
 232 {
 233         int err;
 234
 235         dd->dma_lch_out = NULL;
 236         dd->dma_lch_in = NULL;
 237
 238         dd->dma_lch_in = dma_request_chan(dd->dev, "rx");
 239         if (IS_ERR(dd->dma_lch_in)) {
 240                 dev_err(dd->dev, "Unable to request in DMA channel\n");
 241                 return PTR_ERR(dd->dma_lch_in);
 242         }
 243
 244         dd->dma_lch_out = dma_request_chan(dd->dev, "tx");
 245         if (IS_ERR(dd->dma_lch_out)) {
 246                 dev_err(dd->dev, "Unable to request out DMA channel\n");
 247                 err = PTR_ERR(dd->dma_lch_out);
 248                 goto err_dma_out;
 249         }
 250
 251         return 0;
 252
 253 err_dma_out:
 254         dma_release_channel(dd->dma_lch_in);
 255
 256         return err;
 257 }
 258
 259 static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
 260 {
 261         if (dd->pio_only)
 262                 return;
 263
 264         dma_release_channel(dd->dma_lch_out);
 265         dma_release_channel(dd->dma_lch_in);
 266 }
 267
 268 static int omap_aes_crypt_dma(struct omap_aes_dev *dd,
 269                               struct scatterlist *in_sg,
 270                               struct scatterlist *out_sg,
 271                               int in_sg_len, int out_sg_len)
 272 {
 273         struct dma_async_tx_descriptor *tx_in, *tx_out;
 274         struct dma_slave_config cfg;
 275         int ret;
 276
 277         if (dd->pio_only) {
 278                 scatterwalk_start(&dd->in_walk, dd->in_sg);
 279                 scatterwalk_start(&dd->out_walk, dd->out_sg);
 280
 281                 /* Enable DATAIN interrupt and let it take
 282                    care of the rest */
 283                 omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
 284                 return 0;
 285         }
 286
 287         dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
 288
 289         memset(&cfg, 0, sizeof(cfg));
 290
 291         cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
 292         cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
 293         cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 294         cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 295         cfg.src_maxburst = DST_MAXBURST;
 296         cfg.dst_maxburst = DST_MAXBURST;
 297
 298         /* IN */
 299         ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
 300         if (ret) {
 301                 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
 302                         ret);
 303                 return ret;
 304         }
 305
 306         tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
 307                                         DMA_MEM_TO_DEV,
 308                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 309         if (!tx_in) {
 310                 dev_err(dd->dev, "IN prep_slave_sg() failed\n");
 311                 return -EINVAL;
 312         }
 313
 314         /* No callback necessary */
 315         tx_in->callback_param = dd;
 316
 317         /* OUT */
 318         ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
 319         if (ret) {
 320                 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
 321                         ret);
 322                 return ret;
 323         }
 324
 325         tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,
 326                                         DMA_DEV_TO_MEM,
 327                                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 328         if (!tx_out) {
 329                 dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
 330                 return -EINVAL;
 331         }
 332
 333         if (dd->flags & FLAGS_GCM)
 334                 tx_out->callback = omap_aes_gcm_dma_out_callback;
 335         else
 336                 tx_out->callback = omap_aes_dma_out_callback;
 337         tx_out->callback_param = dd;
 338
 339         dmaengine_submit(tx_in);
 340         dmaengine_submit(tx_out);
 341
 342         dma_async_issue_pending(dd->dma_lch_in);
 343         dma_async_issue_pending(dd->dma_lch_out);
 344
 345         /* start DMA */
 346         dd->pdata->trigger(dd, dd->total);
 347
 348         return 0;
 349 }
 350
 351 int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
 352 {
 353         int err;
 354
 355         pr_debug("total: %d\n", dd->total);
 356
 357         if (!dd->pio_only) {
 358                 err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
 359                                  DMA_TO_DEVICE);
 360                 if (!err) {
 361                         dev_err(dd->dev, "dma_map_sg() error\n");
 362                         return -EINVAL;
 363                 }
 364
 365                 err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 366                                  DMA_FROM_DEVICE);
 367                 if (!err) {
 368                         dev_err(dd->dev, "dma_map_sg() error\n");
 369                         return -EINVAL;
 370                 }
 371         }
 372
 373         err = omap_aes_crypt_dma(dd, dd->in_sg, dd->out_sg, dd->in_sg_len,
 374                                  dd->out_sg_len);
 375         if (err && !dd->pio_only) {
 376                 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
 377                 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 378                              DMA_FROM_DEVICE);
 379         }
 380
 381         return err;
 382 }
 383
 384 static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
 385 {
 386         struct ablkcipher_request *req = dd->req;
 387
 388         pr_debug("err: %d\n", err);
 389
 390         crypto_finalize_cipher_request(dd->engine, req, err);
 391
 392         pm_runtime_mark_last_busy(dd->dev);
 393         pm_runtime_put_autosuspend(dd->dev);
 394 }
 395
 396 int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
 397 {
 398         pr_debug("total: %d\n", dd->total);
 399
 400         omap_aes_dma_stop(dd);
 401
 402
 403         return 0;
 404 }
 405
 406 static int omap_aes_handle_queue(struct omap_aes_dev *dd,
 407                                  struct ablkcipher_request *req)
 408 {
 409         if (req)
 410                 return crypto_transfer_cipher_request_to_engine(dd->engine, req);
 411
 412         return 0;
 413 }
 414
 415 static int omap_aes_prepare_req(struct crypto_engine *engine,
 416                                 struct ablkcipher_request *req)
 417 {
 418         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 419                         crypto_ablkcipher_reqtfm(req));
 420         struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 421         struct omap_aes_dev *dd = rctx->dd;
 422         int ret;
 423         u16 flags;
 424
 425         if (!dd)
 426                 return -ENODEV;
 427
 428         /* assign new request to device */
 429         dd->req = req;
 430         dd->total = req->nbytes;
 431         dd->total_save = req->nbytes;
 432         dd->in_sg = req->src;
 433         dd->out_sg = req->dst;
 434         dd->orig_out = req->dst;
 435
 436         flags = OMAP_CRYPTO_COPY_DATA;
 437         if (req->src == req->dst)
 438                 flags |= OMAP_CRYPTO_FORCE_COPY;
 439
 440         ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE,
 441                                    dd->in_sgl, flags,
 442                                    FLAGS_IN_DATA_ST_SHIFT, &dd->flags);
 443         if (ret)
 444                 return ret;
 445
 446         ret = omap_crypto_align_sg(&dd->out_sg, dd->total, AES_BLOCK_SIZE,
 447                                    &dd->out_sgl, 0,
 448                                    FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
 449         if (ret)
 450                 return ret;
 451
 452         dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
 453         if (dd->in_sg_len < 0)
 454                 return dd->in_sg_len;
 455
 456         dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);
 457         if (dd->out_sg_len < 0)
 458                 return dd->out_sg_len;
 459
 460         rctx->mode &= FLAGS_MODE_MASK;
 461         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 462
 463         dd->ctx = ctx;
 464         rctx->dd = dd;
 465
 466         return omap_aes_write_ctrl(dd);
 467 }
 468
 469 static int omap_aes_crypt_req(struct crypto_engine *engine,
 470                               struct ablkcipher_request *req)
 471 {
 472         struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 473         struct omap_aes_dev *dd = rctx->dd;
 474
 475         if (!dd)
 476                 return -ENODEV;
 477
 478         return omap_aes_crypt_dma_start(dd);
 479 }
 480
 481 static void omap_aes_done_task(unsigned long data)
 482 {
 483         struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 484
 485         pr_debug("enter done_task\n");
 486
 487         if (!dd->pio_only) {
 488                 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
 489                                        DMA_FROM_DEVICE);
 490                 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
 491                 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
 492                              DMA_FROM_DEVICE);
 493                 omap_aes_crypt_dma_stop(dd);
 494         }
 495
 496         omap_crypto_cleanup(dd->in_sgl, NULL, 0, dd->total_save,
 497                             FLAGS_IN_DATA_ST_SHIFT, dd->flags);
 498
 499         omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save,
 500                             FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
 501
 502         omap_aes_finish_req(dd, 0);
 503
 504         pr_debug("exit\n");
 505 }
 506
 507 static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 508 {
 509         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 510                         crypto_ablkcipher_reqtfm(req));
 511         struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 512         struct omap_aes_dev *dd;
 513         int ret;
 514
 515         pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 516                   !!(mode & FLAGS_ENCRYPT),
 517                   !!(mode & FLAGS_CBC));
 518
 519         if (req->nbytes < 200) {
 520                 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
 521
 522                 skcipher_request_set_tfm(subreq, ctx->fallback);
 523                 skcipher_request_set_callback(subreq, req->base.flags, NULL,
 524                                               NULL);
 525                 skcipher_request_set_crypt(subreq, req->src, req->dst,
 526                                            req->nbytes, req->info);
 527
 528                 if (mode & FLAGS_ENCRYPT)
 529                         ret = crypto_skcipher_encrypt(subreq);
 530                 else
 531                         ret = crypto_skcipher_decrypt(subreq);
 532
 533                 skcipher_request_zero(subreq);
 534                 return ret;
 535         }
 536         dd = omap_aes_find_dev(rctx);
 537         if (!dd)
 538                 return -ENODEV;
 539
 540         rctx->mode = mode;
 541
 542         return omap_aes_handle_queue(dd, req);
 543 }
 544
 545 /* ********************** ALG API ************************************ */
 546
 547 static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 548                            unsigned int keylen)
 549 {
 550         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 551         int ret;
 552
 553         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 554                    keylen != AES_KEYSIZE_256)
 555                 return -EINVAL;
 556
 557         pr_debug("enter, keylen: %d\n", keylen);
 558
 559         memcpy(ctx->key, key, keylen);
 560         ctx->keylen = keylen;
 561
 562         crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
 563         crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
 564                                                  CRYPTO_TFM_REQ_MASK);
 565
 566         ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
 567         if (!ret)
 568                 return 0;
 569
 570         return 0;
 571 }
 572
 573 static int omap_aes_ecb_encrypt(struct ablkcipher_request *req)
 574 {
 575         return omap_aes_crypt(req, FLAGS_ENCRYPT);
 576 }
 577
 578 static int omap_aes_ecb_decrypt(struct ablkcipher_request *req)
 579 {
 580         return omap_aes_crypt(req, 0);
 581 }
 582
 583 static int omap_aes_cbc_encrypt(struct ablkcipher_request *req)
 584 {
 585         return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 586 }
 587
 588 static int omap_aes_cbc_decrypt(struct ablkcipher_request *req)
 589 {
 590         return omap_aes_crypt(req, FLAGS_CBC);
 591 }
 592
 593 static int omap_aes_ctr_encrypt(struct ablkcipher_request *req)
 594 {
 595         return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CTR);
 596 }
 597
 598 static int omap_aes_ctr_decrypt(struct ablkcipher_request *req)
 599 {
 600         return omap_aes_crypt(req, FLAGS_CTR);
 601 }
 602
 603 static int omap_aes_cra_init(struct crypto_tfm *tfm)
 604 {
 605         const char *name = crypto_tfm_alg_name(tfm);
 606         const u32 flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
 607         struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 608         struct crypto_skcipher *blk;
 609
 610         blk = crypto_alloc_skcipher(name, 0, flags);
 611         if (IS_ERR(blk))
 612                 return PTR_ERR(blk);
 613
 614         ctx->fallback = blk;
 615
 616         tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx);
 617
 618         return 0;
 619 }
 620
 621 static int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
 622 {
 623         struct omap_aes_dev *dd = NULL;
 624         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 625         int err;
 626
 627         /* Find AES device, currently picks the first device */
 628         spin_lock_bh(&list_lock);
 629         list_for_each_entry(dd, &dev_list, list) {
 630                 break;
 631         }
 632         spin_unlock_bh(&list_lock);
 633
 634         err = pm_runtime_get_sync(dd->dev);
 635         if (err < 0) {
 636                 dev_err(dd->dev, "%s: failed to get_sync(%d)\n",
 637                         __func__, err);
 638                 return err;
 639         }
 640
 641         tfm->reqsize = sizeof(struct omap_aes_reqctx);
 642         ctx->ctr = crypto_alloc_skcipher("ecb(aes)", 0, 0);
 643         if (IS_ERR(ctx->ctr)) {
 644                 pr_warn("could not load aes driver for encrypting IV\n");
 645                 return PTR_ERR(ctx->ctr);
 646         }
 647
 648         return 0;
 649 }
 650
 651 static void omap_aes_cra_exit(struct crypto_tfm *tfm)
 652 {
 653         struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 654
 655         if (ctx->fallback)
 656                 crypto_free_skcipher(ctx->fallback);
 657
 658         ctx->fallback = NULL;
 659 }
 660
 661 static void omap_aes_gcm_cra_exit(struct crypto_aead *tfm)
 662 {
 663         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 664
 665         omap_aes_cra_exit(crypto_aead_tfm(tfm));
 666
 667         if (ctx->ctr)
 668                 crypto_free_skcipher(ctx->ctr);
 669 }
 670
 671 /* ********************** ALGS ************************************ */
 672
 673 static struct crypto_alg algs_ecb_cbc[] = {
 674 {
 675         .cra_name               = "ecb(aes)",
 676         .cra_driver_name        = "ecb-aes-omap",
 677         .cra_priority           = 300,
 678         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 679                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 680                                   CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 681         .cra_blocksize          = AES_BLOCK_SIZE,
 682         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 683         .cra_alignmask          = 0,
 684         .cra_type               = &crypto_ablkcipher_type,
 685         .cra_module             = THIS_MODULE,
 686         .cra_init               = omap_aes_cra_init,
 687         .cra_exit               = omap_aes_cra_exit,
 688         .cra_u.ablkcipher = {
 689                 .min_keysize    = AES_MIN_KEY_SIZE,
 690                 .max_keysize    = AES_MAX_KEY_SIZE,
 691                 .setkey         = omap_aes_setkey,
 692                 .encrypt        = omap_aes_ecb_encrypt,
 693                 .decrypt        = omap_aes_ecb_decrypt,
 694         }
 695 },
 696 {
 697         .cra_name               = "cbc(aes)",
 698         .cra_driver_name        = "cbc-aes-omap",
 699         .cra_priority           = 300,
 700         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 701                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 702                                   CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 703         .cra_blocksize          = AES_BLOCK_SIZE,
 704         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 705         .cra_alignmask          = 0,
 706         .cra_type               = &crypto_ablkcipher_type,
 707         .cra_module             = THIS_MODULE,
 708         .cra_init               = omap_aes_cra_init,
 709         .cra_exit               = omap_aes_cra_exit,
 710         .cra_u.ablkcipher = {
 711                 .min_keysize    = AES_MIN_KEY_SIZE,
 712                 .max_keysize    = AES_MAX_KEY_SIZE,
 713                 .ivsize         = AES_BLOCK_SIZE,
 714                 .setkey         = omap_aes_setkey,
 715                 .encrypt        = omap_aes_cbc_encrypt,
 716                 .decrypt        = omap_aes_cbc_decrypt,
 717         }
 718 }
 719 };
 720
 721 static struct crypto_alg algs_ctr[] = {
 722 {
 723         .cra_name               = "ctr(aes)",
 724         .cra_driver_name        = "ctr-aes-omap",
 725         .cra_priority           = 300,
 726         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 727                                   CRYPTO_ALG_KERN_DRIVER_ONLY |
 728                                   CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 729         .cra_blocksize          = AES_BLOCK_SIZE,
 730         .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 731         .cra_alignmask          = 0,
 732         .cra_type               = &crypto_ablkcipher_type,
 733         .cra_module             = THIS_MODULE,
 734         .cra_init               = omap_aes_cra_init,
 735         .cra_exit               = omap_aes_cra_exit,
 736         .cra_u.ablkcipher = {
 737                 .min_keysize    = AES_MIN_KEY_SIZE,
 738                 .max_keysize    = AES_MAX_KEY_SIZE,
 739                 .geniv          = "eseqiv",
 740                 .ivsize         = AES_BLOCK_SIZE,
 741                 .setkey         = omap_aes_setkey,
 742                 .encrypt        = omap_aes_ctr_encrypt,
 743                 .decrypt        = omap_aes_ctr_decrypt,
 744         }
 745 } ,
 746 };
 747
 748 static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = {
 749         {
 750                 .algs_list      = algs_ecb_cbc,
 751                 .size           = ARRAY_SIZE(algs_ecb_cbc),
 752         },
 753 };
 754
 755 static struct aead_alg algs_aead_gcm[] = {
 756 {
 757         .base = {
 758                 .cra_name               = "gcm(aes)",
 759                 .cra_driver_name        = "gcm-aes-omap",
 760                 .cra_priority           = 300,
 761                 .cra_flags              = CRYPTO_ALG_ASYNC |
 762                                           CRYPTO_ALG_KERN_DRIVER_ONLY,
 763                 .cra_blocksize          = 1,
 764                 .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 765                 .cra_alignmask          = 0xf,
 766                 .cra_module             = THIS_MODULE,
 767         },
 768         .init           = omap_aes_gcm_cra_init,
 769         .exit           = omap_aes_gcm_cra_exit,
 770         .ivsize         = 12,
 771         .maxauthsize    = AES_BLOCK_SIZE,
 772         .setkey         = omap_aes_gcm_setkey,
 773         .encrypt        = omap_aes_gcm_encrypt,
 774         .decrypt        = omap_aes_gcm_decrypt,
 775 },
 776 {
 777         .base = {
 778                 .cra_name               = "rfc4106(gcm(aes))",
 779                 .cra_driver_name        = "rfc4106-gcm-aes-omap",
 780                 .cra_priority           = 300,
 781                 .cra_flags              = CRYPTO_ALG_ASYNC |
 782                                           CRYPTO_ALG_KERN_DRIVER_ONLY,
 783                 .cra_blocksize          = 1,
 784                 .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 785                 .cra_alignmask          = 0xf,
 786                 .cra_module             = THIS_MODULE,
 787         },
 788         .init           = omap_aes_gcm_cra_init,
 789         .exit           = omap_aes_gcm_cra_exit,
 790         .maxauthsize    = AES_BLOCK_SIZE,
 791         .ivsize         = 8,
 792         .setkey         = omap_aes_4106gcm_setkey,
 793         .encrypt        = omap_aes_4106gcm_encrypt,
 794         .decrypt        = omap_aes_4106gcm_decrypt,
 795 },
 796 };
 797
 798 static struct omap_aes_aead_algs omap_aes_aead_info = {
 799         .algs_list      =       algs_aead_gcm,
 800         .size           =       ARRAY_SIZE(algs_aead_gcm),
 801 };
 802
 803 static const struct omap_aes_pdata omap_aes_pdata_omap2 = {
 804         .algs_info      = omap_aes_algs_info_ecb_cbc,
 805         .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc),
 806         .trigger        = omap_aes_dma_trigger_omap2,
 807         .key_ofs        = 0x1c,
 808         .iv_ofs         = 0x20,
 809         .ctrl_ofs       = 0x30,
 810         .data_ofs       = 0x34,
 811         .rev_ofs        = 0x44,
 812         .mask_ofs       = 0x48,
 813         .dma_enable_in  = BIT(2),
 814         .dma_enable_out = BIT(3),
 815         .dma_start      = BIT(5),
 816         .major_mask     = 0xf0,
 817         .major_shift    = 4,
 818         .minor_mask     = 0x0f,
 819         .minor_shift    = 0,
 820 };
 821
 822 #ifdef CONFIG_OF
 823 static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc_ctr[] = {
 824         {
 825                 .algs_list      = algs_ecb_cbc,
 826                 .size           = ARRAY_SIZE(algs_ecb_cbc),
 827         },
 828         {
 829                 .algs_list      = algs_ctr,
 830                 .size           = ARRAY_SIZE(algs_ctr),
 831         },
 832 };
 833
 834 static const struct omap_aes_pdata omap_aes_pdata_omap3 = {
 835         .algs_info      = omap_aes_algs_info_ecb_cbc_ctr,
 836         .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
 837         .trigger        = omap_aes_dma_trigger_omap2,
 838         .key_ofs        = 0x1c,
 839         .iv_ofs         = 0x20,
 840         .ctrl_ofs       = 0x30,
 841         .data_ofs       = 0x34,
 842         .rev_ofs        = 0x44,
 843         .mask_ofs       = 0x48,
 844         .dma_enable_in  = BIT(2),
 845         .dma_enable_out = BIT(3),
 846         .dma_start      = BIT(5),
 847         .major_mask     = 0xf0,
 848         .major_shift    = 4,
 849         .minor_mask     = 0x0f,
 850         .minor_shift    = 0,
 851 };
 852
 853 static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
 854         .algs_info      = omap_aes_algs_info_ecb_cbc_ctr,
 855         .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
 856         .aead_algs_info = &omap_aes_aead_info,
 857         .trigger        = omap_aes_dma_trigger_omap4,
 858         .key_ofs        = 0x3c,
 859         .iv_ofs         = 0x40,
 860         .ctrl_ofs       = 0x50,
 861         .data_ofs       = 0x60,
 862         .rev_ofs        = 0x80,
 863         .mask_ofs       = 0x84,
 864         .irq_status_ofs = 0x8c,
 865         .irq_enable_ofs = 0x90,
 866         .dma_enable_in  = BIT(5),
 867         .dma_enable_out = BIT(6),
 868         .major_mask     = 0x0700,
 869         .major_shift    = 8,
 870         .minor_mask     = 0x003f,
 871         .minor_shift    = 0,
 872 };
 873
 874 static irqreturn_t omap_aes_irq(int irq, void *dev_id)
 875 {
 876         struct omap_aes_dev *dd = dev_id;
 877         u32 status, i;
 878         u32 *src, *dst;
 879
 880         status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd));
 881         if (status & AES_REG_IRQ_DATA_IN) {
 882                 omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
 883
 884                 BUG_ON(!dd->in_sg);
 885
 886                 BUG_ON(_calc_walked(in) > dd->in_sg->length);
 887
 888                 src = sg_virt(dd->in_sg) + _calc_walked(in);
 889
 890                 for (i = 0; i < AES_BLOCK_WORDS; i++) {
 891                         omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src);
 892
 893                         scatterwalk_advance(&dd->in_walk, 4);
 894                         if (dd->in_sg->length == _calc_walked(in)) {
 895                                 dd->in_sg = sg_next(dd->in_sg);
 896                                 if (dd->in_sg) {
 897                                         scatterwalk_start(&dd->in_walk,
 898                                                           dd->in_sg);
 899                                         src = sg_virt(dd->in_sg) +
 900                                               _calc_walked(in);
 901                                 }
 902                         } else {
 903                                 src++;
 904                         }
 905                 }
 906
 907                 /* Clear IRQ status */
 908                 status &= ~AES_REG_IRQ_DATA_IN;
 909                 omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
 910
 911                 /* Enable DATA_OUT interrupt */
 912                 omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4);
 913
 914         } else if (status & AES_REG_IRQ_DATA_OUT) {
 915                 omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
 916
 917                 BUG_ON(!dd->out_sg);
 918
 919                 BUG_ON(_calc_walked(out) > dd->out_sg->length);
 920
 921                 dst = sg_virt(dd->out_sg) + _calc_walked(out);
 922
 923                 for (i = 0; i < AES_BLOCK_WORDS; i++) {
 924                         *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
 925                         scatterwalk_advance(&dd->out_walk, 4);
 926                         if (dd->out_sg->length == _calc_walked(out)) {
 927                                 dd->out_sg = sg_next(dd->out_sg);
 928                                 if (dd->out_sg) {
 929                                         scatterwalk_start(&dd->out_walk,
 930                                                           dd->out_sg);
 931                                         dst = sg_virt(dd->out_sg) +
 932                                               _calc_walked(out);
 933                                 }
 934                         } else {
 935                                 dst++;
 936                         }
 937                 }
 938
 939                 dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total);
 940
 941                 /* Clear IRQ status */
 942                 status &= ~AES_REG_IRQ_DATA_OUT;
 943                 omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
 944
 945                 if (!dd->total)
 946                         /* All bytes read! */
 947                         tasklet_schedule(&dd->done_task);
 948                 else
 949                         /* Enable DATA_IN interrupt for next block */
 950                         omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
 951         }
 952
 953         return IRQ_HANDLED;
 954 }
 955
 956 static const struct of_device_id omap_aes_of_match[] = {
 957         {
 958                 .compatible     = "ti,omap2-aes",
 959                 .data           = &omap_aes_pdata_omap2,
 960         },
 961         {
 962                 .compatible     = "ti,omap3-aes",
 963                 .data           = &omap_aes_pdata_omap3,
 964         },
 965         {
 966                 .compatible     = "ti,omap4-aes",
 967                 .data           = &omap_aes_pdata_omap4,
 968         },
 969         {},
 970 };
 971 MODULE_DEVICE_TABLE(of, omap_aes_of_match);
 972
 973 static int omap_aes_get_res_of(struct omap_aes_dev *dd,
 974                 struct device *dev, struct resource *res)
 975 {
 976         struct device_node *node = dev->of_node;
 977         const struct of_device_id *match;
 978         int err = 0;
 979
 980         match = of_match_device(of_match_ptr(omap_aes_of_match), dev);
 981         if (!match) {
 982                 dev_err(dev, "no compatible OF match\n");
 983                 err = -EINVAL;
 984                 goto err;
 985         }
 986
 987         err = of_address_to_resource(node, 0, res);
 988         if (err < 0) {
 989                 dev_err(dev, "can't translate OF node address\n");
 990                 err = -EINVAL;
 991                 goto err;
 992         }
 993
 994         dd->pdata = match->data;
 995
 996 err:
 997         return err;
 998 }
 999 #else
1000 static const struct of_device_id omap_aes_of_match[] = {
1001         {},
1002 };
1003
1004 static int omap_aes_get_res_of(struct omap_aes_dev *dd,
1005                 struct device *dev, struct resource *res)
1006 {
1007         return -EINVAL;
1008 }
1009 #endif
1010
1011 static int omap_aes_get_res_pdev(struct omap_aes_dev *dd,
1012                 struct platform_device *pdev, struct resource *res)
1013 {
1014         struct device *dev = &pdev->dev;
1015         struct resource *r;
1016         int err = 0;
1017
1018         /* Get the base address */
1019         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1020         if (!r) {
1021                 dev_err(dev, "no MEM resource info\n");
1022                 err = -ENODEV;
1023                 goto err;
1024         }
1025         memcpy(res, r, sizeof(*res));
1026
1027         /* Only OMAP2/3 can be non-DT */
1028         dd->pdata = &omap_aes_pdata_omap2;
1029
1030 err:
1031         return err;
1032 }
1033
1034 static int omap_aes_probe(struct platform_device *pdev)
1035 {
1036         struct device *dev = &pdev->dev;
1037         struct omap_aes_dev *dd;
1038         struct crypto_alg *algp;
1039         struct aead_alg *aalg;
1040         struct resource res;
1041         int err = -ENOMEM, i, j, irq = -1;
1042         u32 reg;
1043
1044         dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL);
1045         if (dd == NULL) {
1046                 dev_err(dev, "unable to alloc data struct.\n");
1047                 goto err_data;
1048         }
1049         dd->dev = dev;
1050         platform_set_drvdata(pdev, dd);
1051
1052         aead_init_queue(&dd->aead_queue, OMAP_AES_QUEUE_LENGTH);
1053
1054         err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) :
1055                                omap_aes_get_res_pdev(dd, pdev, &res);
1056         if (err)
1057                 goto err_res;
1058
1059         dd->io_base = devm_ioremap_resource(dev, &res);
1060         if (IS_ERR(dd->io_base)) {
1061                 err = PTR_ERR(dd->io_base);
1062                 goto err_res;
1063         }
1064         dd->phys_base = res.start;
1065
1066         pm_runtime_use_autosuspend(dev);
1067         pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
1068
1069         pm_runtime_enable(dev);
1070         err = pm_runtime_get_sync(dev);
1071         if (err < 0) {
1072                 dev_err(dev, "%s: failed to get_sync(%d)\n",
1073                         __func__, err);
1074                 goto err_res;
1075         }
1076
1077         omap_aes_dma_stop(dd);
1078
1079         reg = omap_aes_read(dd, AES_REG_REV(dd));
1080
1081         pm_runtime_put_sync(dev);
1082
1083         dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
1084                  (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
1085                  (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
1086
1087         tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
1088
1089         err = omap_aes_dma_init(dd);
1090         if (err == -EPROBE_DEFER) {
1091                 goto err_irq;
1092         } else if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) {
1093                 dd->pio_only = 1;
1094
1095                 irq = platform_get_irq(pdev, 0);
1096                 if (irq < 0) {
1097                         dev_err(dev, "can't get IRQ resource\n");
1098                         err = irq;
1099                         goto err_irq;
1100                 }
1101
1102                 err = devm_request_irq(dev, irq, omap_aes_irq, 0,
1103                                 dev_name(dev), dd);
1104                 if (err) {
1105                         dev_err(dev, "Unable to grab omap-aes IRQ\n");
1106                         goto err_irq;
1107                 }
1108         }
1109
1110         spin_lock_init(&dd->lock);
1111
1112         INIT_LIST_HEAD(&dd->list);
1113         spin_lock(&list_lock);
1114         list_add_tail(&dd->list, &dev_list);
1115         spin_unlock(&list_lock);
1116
1117         /* Initialize crypto engine */
1118         dd->engine = crypto_engine_alloc_init(dev, 1);
1119         if (!dd->engine) {
1120                 err = -ENOMEM;
1121                 goto err_engine;
1122         }
1123
1124         dd->engine->prepare_cipher_request = omap_aes_prepare_req;
1125         dd->engine->cipher_one_request = omap_aes_crypt_req;
1126         err = crypto_engine_start(dd->engine);
1127         if (err)
1128                 goto err_engine;
1129
1130         for (i = 0; i < dd->pdata->algs_info_size; i++) {
1131                 if (!dd->pdata->algs_info[i].registered) {
1132                         for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1133                                 algp = &dd->pdata->algs_info[i].algs_list[j];
1134
1135                                 pr_debug("reg alg: %s\n", algp->cra_name);
1136                                 INIT_LIST_HEAD(&algp->cra_list);
1137
1138                                 err = crypto_register_alg(algp);
1139                                 if (err)
1140                                         goto err_algs;
1141
1142                                 dd->pdata->algs_info[i].registered++;
1143                         }
1144                 }
1145         }
1146
1147         if (dd->pdata->aead_algs_info &&
1148             !dd->pdata->aead_algs_info->registered) {
1149                 for (i = 0; i < dd->pdata->aead_algs_info->size; i++) {
1150                         aalg = &dd->pdata->aead_algs_info->algs_list[i];
1151                         algp = &aalg->base;
1152
1153                         pr_debug("reg alg: %s\n", algp->cra_name);
1154                         INIT_LIST_HEAD(&algp->cra_list);
1155
1156                         err = crypto_register_aead(aalg);
1157                         if (err)
1158                                 goto err_aead_algs;
1159
1160                         dd->pdata->aead_algs_info->registered++;
1161                 }
1162         }
1163
1164         return 0;
1165 err_aead_algs:
1166         for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
1167                 aalg = &dd->pdata->aead_algs_info->algs_list[i];
1168                 crypto_unregister_aead(aalg);
1169         }
1170 err_algs:
1171         for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1172                 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1173                         crypto_unregister_alg(
1174                                         &dd->pdata->algs_info[i].algs_list[j]);
1175
1176 err_engine:
1177         if (dd->engine)
1178                 crypto_engine_exit(dd->engine);
1179
1180         omap_aes_dma_cleanup(dd);
1181 err_irq:
1182         tasklet_kill(&dd->done_task);
1183         pm_runtime_disable(dev);
1184 err_res:
1185         dd = NULL;
1186 err_data:
1187         dev_err(dev, "initialization failed.\n");
1188         return err;
1189 }
1190
1191 static int omap_aes_remove(struct platform_device *pdev)
1192 {
1193         struct omap_aes_dev *dd = platform_get_drvdata(pdev);
1194         struct aead_alg *aalg;
1195         int i, j;
1196
1197         if (!dd)
1198                 return -ENODEV;
1199
1200         spin_lock(&list_lock);
1201         list_del(&dd->list);
1202         spin_unlock(&list_lock);
1203
1204         for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1205                 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1206                         crypto_unregister_alg(
1207                                         &dd->pdata->algs_info[i].algs_list[j]);
1208
1209         for (i = dd->pdata->aead_algs_info->size - 1; i >= 0; i--) {
1210                 aalg = &dd->pdata->aead_algs_info->algs_list[i];
1211                 crypto_unregister_aead(aalg);
1212         }
1213
1214         crypto_engine_exit(dd->engine);
1215
1216         tasklet_kill(&dd->done_task);
1217         omap_aes_dma_cleanup(dd);
1218         pm_runtime_disable(dd->dev);
1219         dd = NULL;
1220
1221         return 0;
1222 }
1223
1224 #ifdef CONFIG_PM_SLEEP
1225 static int omap_aes_suspend(struct device *dev)
1226 {
1227         pm_runtime_put_sync(dev);
1228         return 0;
1229 }
1230
1231 static int omap_aes_resume(struct device *dev)
1232 {
1233         pm_runtime_get_sync(dev);
1234         return 0;
1235 }
1236 #endif
1237
1238 static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops, omap_aes_suspend, omap_aes_resume);
1239
1240 static struct platform_driver omap_aes_driver = {
1241         .probe  = omap_aes_probe,
1242         .remove = omap_aes_remove,
1243         .driver = {
1244                 .name   = "omap-aes",
1245                 .pm     = &omap_aes_pm_ops,
1246                 .of_match_table = omap_aes_of_match,
1247         },
1248 };
1249
1250 module_platform_driver(omap_aes_driver);
1251
1252 MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
1253 MODULE_LICENSE("GPL v2");
1254 MODULE_AUTHOR("Dmitry Kasatkin");
1255