drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/pci.h>
  17 #include <linux/interrupt.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <crypto/des.h>
  20 #include <linux/ccp.h>
  21
  22 #include "ccp-dev.h"
  23
  24 /* SHA initial context values */
  25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  26         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  27         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  28         cpu_to_be32(SHA1_H4),
  29 };
  30
  31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  32         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  33         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  34         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  35         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  36 };
  37
  38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  39         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  40         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  41         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  42         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  43 };
  44
  45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  46         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  47         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  48         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  49         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  50 };
  51
  52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  53         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  54         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  55         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  56         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  57 };
  58
  59 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  60                                         ccp_gen_jobid(ccp) : 0)
  61
  62 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  63 {
  64         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  65 }
  66
  67 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  68 {
  69         if (wa->dma_count)
  70                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
  71
  72         wa->dma_count = 0;
  73 }
  74
  75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  76                                 struct scatterlist *sg, u64 len,
  77                                 enum dma_data_direction dma_dir)
  78 {
  79         memset(wa, 0, sizeof(*wa));
  80
  81         wa->sg = sg;
  82         if (!sg)
  83                 return 0;
  84
  85         wa->nents = sg_nents_for_len(sg, len);
  86         if (wa->nents < 0)
  87                 return wa->nents;
  88
  89         wa->bytes_left = len;
  90         wa->sg_used = 0;
  91
  92         if (len == 0)
  93                 return 0;
  94
  95         if (dma_dir == DMA_NONE)
  96                 return 0;
  97
  98         wa->dma_sg = sg;
  99         wa->dma_dev = dev;
 100         wa->dma_dir = dma_dir;
 101         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 102         if (!wa->dma_count)
 103                 return -ENOMEM;
 104
 105         return 0;
 106 }
 107
 108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 109 {
 110         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 111
 112         if (!wa->sg)
 113                 return;
 114
 115         wa->sg_used += nbytes;
 116         wa->bytes_left -= nbytes;
 117         if (wa->sg_used == wa->sg->length) {
 118                 wa->sg = sg_next(wa->sg);
 119                 wa->sg_used = 0;
 120         }
 121 }
 122
 123 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 124 {
 125         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 126                 if (wa->address)
 127                         dma_pool_free(wa->dma_pool, wa->address,
 128                                       wa->dma.address);
 129         } else {
 130                 if (wa->dma.address)
 131                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 132                                          wa->dma.dir);
 133                 kfree(wa->address);
 134         }
 135
 136         wa->address = NULL;
 137         wa->dma.address = 0;
 138 }
 139
 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 141                                 struct ccp_cmd_queue *cmd_q,
 142                                 unsigned int len,
 143                                 enum dma_data_direction dir)
 144 {
 145         memset(wa, 0, sizeof(*wa));
 146
 147         if (!len)
 148                 return 0;
 149
 150         wa->dev = cmd_q->ccp->dev;
 151         wa->length = len;
 152
 153         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 154                 wa->dma_pool = cmd_q->dma_pool;
 155
 156                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 157                                              &wa->dma.address);
 158                 if (!wa->address)
 159                         return -ENOMEM;
 160
 161                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 162
 163                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 164         } else {
 165                 wa->address = kzalloc(len, GFP_KERNEL);
 166                 if (!wa->address)
 167                         return -ENOMEM;
 168
 169                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 170                                                  dir);
 171                 if (dma_mapping_error(wa->dev, wa->dma.address))
 172                         return -ENOMEM;
 173
 174                 wa->dma.length = len;
 175         }
 176         wa->dma.dir = dir;
 177
 178         return 0;
 179 }
 180
 181 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 182                            struct scatterlist *sg, unsigned int sg_offset,
 183                            unsigned int len)
 184 {
 185         WARN_ON(!wa->address);
 186
 187         if (len > (wa->length - wa_offset))
 188                 return -EINVAL;
 189
 190         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 191                                  0);
 192         return 0;
 193 }
 194
 195 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 196                             struct scatterlist *sg, unsigned int sg_offset,
 197                             unsigned int len)
 198 {
 199         WARN_ON(!wa->address);
 200
 201         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 202                                  1);
 203 }
 204
 205 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 206                                    unsigned int wa_offset,
 207                                    struct scatterlist *sg,
 208                                    unsigned int sg_offset,
 209                                    unsigned int len)
 210 {
 211         u8 *p, *q;
 212         int     rc;
 213
 214         rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 215         if (rc)
 216                 return rc;
 217
 218         p = wa->address + wa_offset;
 219         q = p + len - 1;
 220         while (p < q) {
 221                 *p = *p ^ *q;
 222                 *q = *p ^ *q;
 223                 *p = *p ^ *q;
 224                 p++;
 225                 q--;
 226         }
 227         return 0;
 228 }
 229
 230 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 231                                     unsigned int wa_offset,
 232                                     struct scatterlist *sg,
 233                                     unsigned int sg_offset,
 234                                     unsigned int len)
 235 {
 236         u8 *p, *q;
 237
 238         p = wa->address + wa_offset;
 239         q = p + len - 1;
 240         while (p < q) {
 241                 *p = *p ^ *q;
 242                 *q = *p ^ *q;
 243                 *p = *p ^ *q;
 244                 p++;
 245                 q--;
 246         }
 247
 248         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 249 }
 250
 251 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 252 {
 253         ccp_dm_free(&data->dm_wa);
 254         ccp_sg_free(&data->sg_wa);
 255 }
 256
 257 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 258                          struct scatterlist *sg, u64 sg_len,
 259                          unsigned int dm_len,
 260                          enum dma_data_direction dir)
 261 {
 262         int ret;
 263
 264         memset(data, 0, sizeof(*data));
 265
 266         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 267                                    dir);
 268         if (ret)
 269                 goto e_err;
 270
 271         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 272         if (ret)
 273                 goto e_err;
 274
 275         return 0;
 276
 277 e_err:
 278         ccp_free_data(data, cmd_q);
 279
 280         return ret;
 281 }
 282
 283 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 284 {
 285         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 286         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 287         unsigned int buf_count, nbytes;
 288
 289         /* Clear the buffer if setting it */
 290         if (!from)
 291                 memset(dm_wa->address, 0, dm_wa->length);
 292
 293         if (!sg_wa->sg)
 294                 return 0;
 295
 296         /* Perform the copy operation
 297          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 298          *   an unsigned int
 299          */
 300         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 301         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 302                                  nbytes, from);
 303
 304         /* Update the structures and generate the count */
 305         buf_count = 0;
 306         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 307                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 308                              dm_wa->length - buf_count);
 309                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 310
 311                 buf_count += nbytes;
 312                 ccp_update_sg_workarea(sg_wa, nbytes);
 313         }
 314
 315         return buf_count;
 316 }
 317
 318 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 319 {
 320         return ccp_queue_buf(data, 0);
 321 }
 322
 323 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 324 {
 325         return ccp_queue_buf(data, 1);
 326 }
 327
 328 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 329                              struct ccp_op *op, unsigned int block_size,
 330                              bool blocksize_op)
 331 {
 332         unsigned int sg_src_len, sg_dst_len, op_len;
 333
 334         /* The CCP can only DMA from/to one address each per operation. This
 335          * requires that we find the smallest DMA area between the source
 336          * and destination. The resulting len values will always be <= UINT_MAX
 337          * because the dma length is an unsigned int.
 338          */
 339         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 340         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 341
 342         if (dst) {
 343                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 344                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 345                 op_len = min(sg_src_len, sg_dst_len);
 346         } else {
 347                 op_len = sg_src_len;
 348         }
 349
 350         /* The data operation length will be at least block_size in length
 351          * or the smaller of available sg room remaining for the source or
 352          * the destination
 353          */
 354         op_len = max(op_len, block_size);
 355
 356         /* Unless we have to buffer data, there's no reason to wait */
 357         op->soc = 0;
 358
 359         if (sg_src_len < block_size) {
 360                 /* Not enough data in the sg element, so it
 361                  * needs to be buffered into a blocksize chunk
 362                  */
 363                 int cp_len = ccp_fill_queue_buf(src);
 364
 365                 op->soc = 1;
 366                 op->src.u.dma.address = src->dm_wa.dma.address;
 367                 op->src.u.dma.offset = 0;
 368                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 369         } else {
 370                 /* Enough data in the sg element, but we need to
 371                  * adjust for any previously copied data
 372                  */
 373                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 374                 op->src.u.dma.offset = src->sg_wa.sg_used;
 375                 op->src.u.dma.length = op_len & ~(block_size - 1);
 376
 377                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 378         }
 379
 380         if (dst) {
 381                 if (sg_dst_len < block_size) {
 382                         /* Not enough room in the sg element or we're on the
 383                          * last piece of data (when using padding), so the
 384                          * output needs to be buffered into a blocksize chunk
 385                          */
 386                         op->soc = 1;
 387                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 388                         op->dst.u.dma.offset = 0;
 389                         op->dst.u.dma.length = op->src.u.dma.length;
 390                 } else {
 391                         /* Enough room in the sg element, but we need to
 392                          * adjust for any previously used area
 393                          */
 394                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 395                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 396                         op->dst.u.dma.length = op->src.u.dma.length;
 397                 }
 398         }
 399 }
 400
 401 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 402                              struct ccp_op *op)
 403 {
 404         op->init = 0;
 405
 406         if (dst) {
 407                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 408                         ccp_empty_queue_buf(dst);
 409                 else
 410                         ccp_update_sg_workarea(&dst->sg_wa,
 411                                                op->dst.u.dma.length);
 412         }
 413 }
 414
 415 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 416                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 417                                u32 byte_swap, bool from)
 418 {
 419         struct ccp_op op;
 420
 421         memset(&op, 0, sizeof(op));
 422
 423         op.cmd_q = cmd_q;
 424         op.jobid = jobid;
 425         op.eom = 1;
 426
 427         if (from) {
 428                 op.soc = 1;
 429                 op.src.type = CCP_MEMTYPE_SB;
 430                 op.src.u.sb = sb;
 431                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 432                 op.dst.u.dma.address = wa->dma.address;
 433                 op.dst.u.dma.length = wa->length;
 434         } else {
 435                 op.src.type = CCP_MEMTYPE_SYSTEM;
 436                 op.src.u.dma.address = wa->dma.address;
 437                 op.src.u.dma.length = wa->length;
 438                 op.dst.type = CCP_MEMTYPE_SB;
 439                 op.dst.u.sb = sb;
 440         }
 441
 442         op.u.passthru.byte_swap = byte_swap;
 443
 444         return cmd_q->ccp->vdata->perform->passthru(&op);
 445 }
 446
 447 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 448                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 449                           u32 byte_swap)
 450 {
 451         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 452 }
 453
 454 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 455                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 456                             u32 byte_swap)
 457 {
 458         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 459 }
 460
 461 static noinline_for_stack int
 462 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 463 {
 464         struct ccp_aes_engine *aes = &cmd->u.aes;
 465         struct ccp_dm_workarea key, ctx;
 466         struct ccp_data src;
 467         struct ccp_op op;
 468         unsigned int dm_offset;
 469         int ret;
 470
 471         if (!((aes->key_len == AES_KEYSIZE_128) ||
 472               (aes->key_len == AES_KEYSIZE_192) ||
 473               (aes->key_len == AES_KEYSIZE_256)))
 474                 return -EINVAL;
 475
 476         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 477                 return -EINVAL;
 478
 479         if (aes->iv_len != AES_BLOCK_SIZE)
 480                 return -EINVAL;
 481
 482         if (!aes->key || !aes->iv || !aes->src)
 483                 return -EINVAL;
 484
 485         if (aes->cmac_final) {
 486                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 487                         return -EINVAL;
 488
 489                 if (!aes->cmac_key)
 490                         return -EINVAL;
 491         }
 492
 493         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 494         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 495
 496         ret = -EIO;
 497         memset(&op, 0, sizeof(op));
 498         op.cmd_q = cmd_q;
 499         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 500         op.sb_key = cmd_q->sb_key;
 501         op.sb_ctx = cmd_q->sb_ctx;
 502         op.init = 1;
 503         op.u.aes.type = aes->type;
 504         op.u.aes.mode = aes->mode;
 505         op.u.aes.action = aes->action;
 506
 507         /* All supported key sizes fit in a single (32-byte) SB entry
 508          * and must be in little endian format. Use the 256-bit byte
 509          * swap passthru option to convert from big endian to little
 510          * endian.
 511          */
 512         ret = ccp_init_dm_workarea(&key, cmd_q,
 513                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 514                                    DMA_TO_DEVICE);
 515         if (ret)
 516                 return ret;
 517
 518         dm_offset = CCP_SB_BYTES - aes->key_len;
 519         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 520         if (ret)
 521                 goto e_key;
 522         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 523                              CCP_PASSTHRU_BYTESWAP_256BIT);
 524         if (ret) {
 525                 cmd->engine_error = cmd_q->cmd_error;
 526                 goto e_key;
 527         }
 528
 529         /* The AES context fits in a single (32-byte) SB entry and
 530          * must be in little endian format. Use the 256-bit byte swap
 531          * passthru option to convert from big endian to little endian.
 532          */
 533         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 534                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 535                                    DMA_BIDIRECTIONAL);
 536         if (ret)
 537                 goto e_key;
 538
 539         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 540         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 541         if (ret)
 542                 goto e_ctx;
 543         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 544                              CCP_PASSTHRU_BYTESWAP_256BIT);
 545         if (ret) {
 546                 cmd->engine_error = cmd_q->cmd_error;
 547                 goto e_ctx;
 548         }
 549
 550         /* Send data to the CCP AES engine */
 551         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 552                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 553         if (ret)
 554                 goto e_ctx;
 555
 556         while (src.sg_wa.bytes_left) {
 557                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 558                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 559                         op.eom = 1;
 560
 561                         /* Push the K1/K2 key to the CCP now */
 562                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 563                                                op.sb_ctx,
 564                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 565                         if (ret) {
 566                                 cmd->engine_error = cmd_q->cmd_error;
 567                                 goto e_src;
 568                         }
 569
 570                         ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 571                                               aes->cmac_key_len);
 572                         if (ret)
 573                                 goto e_src;
 574                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 575                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 576                         if (ret) {
 577                                 cmd->engine_error = cmd_q->cmd_error;
 578                                 goto e_src;
 579                         }
 580                 }
 581
 582                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 583                 if (ret) {
 584                         cmd->engine_error = cmd_q->cmd_error;
 585                         goto e_src;
 586                 }
 587
 588                 ccp_process_data(&src, NULL, &op);
 589         }
 590
 591         /* Retrieve the AES context - convert from LE to BE using
 592          * 32-byte (256-bit) byteswapping
 593          */
 594         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 595                                CCP_PASSTHRU_BYTESWAP_256BIT);
 596         if (ret) {
 597                 cmd->engine_error = cmd_q->cmd_error;
 598                 goto e_src;
 599         }
 600
 601         /* ...but we only need AES_BLOCK_SIZE bytes */
 602         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 603         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 604
 605 e_src:
 606         ccp_free_data(&src, cmd_q);
 607
 608 e_ctx:
 609         ccp_dm_free(&ctx);
 610
 611 e_key:
 612         ccp_dm_free(&key);
 613
 614         return ret;
 615 }
 616
 617 static noinline_for_stack int
 618 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 619 {
 620         struct ccp_aes_engine *aes = &cmd->u.aes;
 621         struct ccp_dm_workarea key, ctx, final_wa, tag;
 622         struct ccp_data src, dst;
 623         struct ccp_data aad;
 624         struct ccp_op op;
 625
 626         unsigned long long *final;
 627         unsigned int dm_offset;
 628         unsigned int authsize;
 629         unsigned int jobid;
 630         unsigned int ilen;
 631         bool in_place = true; /* Default value */
 632         int ret;
 633
 634         struct scatterlist *p_inp, sg_inp[2];
 635         struct scatterlist *p_tag, sg_tag[2];
 636         struct scatterlist *p_outp, sg_outp[2];
 637         struct scatterlist *p_aad;
 638
 639         if (!aes->iv)
 640                 return -EINVAL;
 641
 642         if (!((aes->key_len == AES_KEYSIZE_128) ||
 643                 (aes->key_len == AES_KEYSIZE_192) ||
 644                 (aes->key_len == AES_KEYSIZE_256)))
 645                 return -EINVAL;
 646
 647         if (!aes->key) /* Gotta have a key SGL */
 648                 return -EINVAL;
 649
 650         /* Zero defaults to 16 bytes, the maximum size */
 651         authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
 652         switch (authsize) {
 653         case 16:
 654         case 15:
 655         case 14:
 656         case 13:
 657         case 12:
 658         case 8:
 659         case 4:
 660                 break;
 661         default:
 662                 return -EINVAL;
 663         }
 664
 665         /* First, decompose the source buffer into AAD & PT,
 666          * and the destination buffer into AAD, CT & tag, or
 667          * the input into CT & tag.
 668          * It is expected that the input and output SGs will
 669          * be valid, even if the AAD and input lengths are 0.
 670          */
 671         p_aad = aes->src;
 672         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 673         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 674         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 675                 ilen = aes->src_len;
 676                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 677         } else {
 678                 /* Input length for decryption includes tag */
 679                 ilen = aes->src_len - authsize;
 680                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 681         }
 682
 683         jobid = CCP_NEW_JOBID(cmd_q->ccp);
 684
 685         memset(&op, 0, sizeof(op));
 686         op.cmd_q = cmd_q;
 687         op.jobid = jobid;
 688         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 689         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 690         op.init = 1;
 691         op.u.aes.type = aes->type;
 692
 693         /* Copy the key to the LSB */
 694         ret = ccp_init_dm_workarea(&key, cmd_q,
 695                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 696                                    DMA_TO_DEVICE);
 697         if (ret)
 698                 return ret;
 699
 700         dm_offset = CCP_SB_BYTES - aes->key_len;
 701         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 702         if (ret)
 703                 goto e_key;
 704         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 705                              CCP_PASSTHRU_BYTESWAP_256BIT);
 706         if (ret) {
 707                 cmd->engine_error = cmd_q->cmd_error;
 708                 goto e_key;
 709         }
 710
 711         /* Copy the context (IV) to the LSB.
 712          * There is an assumption here that the IV is 96 bits in length, plus
 713          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 714          */
 715         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 716                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 717                                    DMA_BIDIRECTIONAL);
 718         if (ret)
 719                 goto e_key;
 720
 721         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 722         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 723         if (ret)
 724                 goto e_ctx;
 725
 726         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 727                              CCP_PASSTHRU_BYTESWAP_256BIT);
 728         if (ret) {
 729                 cmd->engine_error = cmd_q->cmd_error;
 730                 goto e_ctx;
 731         }
 732
 733         op.init = 1;
 734         if (aes->aad_len > 0) {
 735                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 736                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 737                                     AES_BLOCK_SIZE,
 738                                     DMA_TO_DEVICE);
 739                 if (ret)
 740                         goto e_ctx;
 741
 742                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 743                 op.u.aes.action = CCP_AES_GHASHAAD;
 744
 745                 while (aad.sg_wa.bytes_left) {
 746                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 747
 748                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 749                         if (ret) {
 750                                 cmd->engine_error = cmd_q->cmd_error;
 751                                 goto e_aad;
 752                         }
 753
 754                         ccp_process_data(&aad, NULL, &op);
 755                         op.init = 0;
 756                 }
 757         }
 758
 759         op.u.aes.mode = CCP_AES_MODE_GCTR;
 760         op.u.aes.action = aes->action;
 761
 762         if (ilen > 0) {
 763                 /* Step 2: Run a GCTR over the plaintext */
 764                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 765
 766                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 767                                     AES_BLOCK_SIZE,
 768                                     in_place ? DMA_BIDIRECTIONAL
 769                                              : DMA_TO_DEVICE);
 770                 if (ret)
 771                         goto e_ctx;
 772
 773                 if (in_place) {
 774                         dst = src;
 775                 } else {
 776                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 777                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 778                         if (ret)
 779                                 goto e_src;
 780                 }
 781
 782                 op.soc = 0;
 783                 op.eom = 0;
 784                 op.init = 1;
 785                 while (src.sg_wa.bytes_left) {
 786                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 787                         if (!src.sg_wa.bytes_left) {
 788                                 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
 789
 790                                 if (nbytes) {
 791                                         op.eom = 1;
 792                                         op.u.aes.size = (nbytes * 8) - 1;
 793                                 }
 794                         }
 795
 796                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 797                         if (ret) {
 798                                 cmd->engine_error = cmd_q->cmd_error;
 799                                 goto e_dst;
 800                         }
 801
 802                         ccp_process_data(&src, &dst, &op);
 803                         op.init = 0;
 804                 }
 805         }
 806
 807         /* Step 3: Update the IV portion of the context with the original IV */
 808         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 809                                CCP_PASSTHRU_BYTESWAP_256BIT);
 810         if (ret) {
 811                 cmd->engine_error = cmd_q->cmd_error;
 812                 goto e_dst;
 813         }
 814
 815         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 816         if (ret)
 817                 goto e_dst;
 818
 819         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 820                              CCP_PASSTHRU_BYTESWAP_256BIT);
 821         if (ret) {
 822                 cmd->engine_error = cmd_q->cmd_error;
 823                 goto e_dst;
 824         }
 825
 826         /* Step 4: Concatenate the lengths of the AAD and source, and
 827          * hash that 16 byte buffer.
 828          */
 829         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 830                                    DMA_BIDIRECTIONAL);
 831         if (ret)
 832                 goto e_dst;
 833         final = (unsigned long long *) final_wa.address;
 834         final[0] = cpu_to_be64(aes->aad_len * 8);
 835         final[1] = cpu_to_be64(ilen * 8);
 836
 837         memset(&op, 0, sizeof(op));
 838         op.cmd_q = cmd_q;
 839         op.jobid = jobid;
 840         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 841         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 842         op.init = 1;
 843         op.u.aes.type = aes->type;
 844         op.u.aes.mode = CCP_AES_MODE_GHASH;
 845         op.u.aes.action = CCP_AES_GHASHFINAL;
 846         op.src.type = CCP_MEMTYPE_SYSTEM;
 847         op.src.u.dma.address = final_wa.dma.address;
 848         op.src.u.dma.length = AES_BLOCK_SIZE;
 849         op.dst.type = CCP_MEMTYPE_SYSTEM;
 850         op.dst.u.dma.address = final_wa.dma.address;
 851         op.dst.u.dma.length = AES_BLOCK_SIZE;
 852         op.eom = 1;
 853         op.u.aes.size = 0;
 854         ret = cmd_q->ccp->vdata->perform->aes(&op);
 855         if (ret)
 856                 goto e_dst;
 857
 858         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 859                 /* Put the ciphered tag after the ciphertext. */
 860                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
 861         } else {
 862                 /* Does this ciphered tag match the input? */
 863                 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
 864                                            DMA_BIDIRECTIONAL);
 865                 if (ret)
 866                         goto e_tag;
 867                 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
 868                 if (ret)
 869                         goto e_tag;
 870
 871                 ret = crypto_memneq(tag.address, final_wa.address,
 872                                     authsize) ? -EBADMSG : 0;
 873                 ccp_dm_free(&tag);
 874         }
 875
 876 e_tag:
 877         ccp_dm_free(&final_wa);
 878
 879 e_dst:
 880         if (ilen > 0 && !in_place)
 881                 ccp_free_data(&dst, cmd_q);
 882
 883 e_src:
 884         if (ilen > 0)
 885                 ccp_free_data(&src, cmd_q);
 886
 887 e_aad:
 888         if (aes->aad_len)
 889                 ccp_free_data(&aad, cmd_q);
 890
 891 e_ctx:
 892         ccp_dm_free(&ctx);
 893
 894 e_key:
 895         ccp_dm_free(&key);
 896
 897         return ret;
 898 }
 899
 900 static noinline_for_stack int
 901 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 902 {
 903         struct ccp_aes_engine *aes = &cmd->u.aes;
 904         struct ccp_dm_workarea key, ctx;
 905         struct ccp_data src, dst;
 906         struct ccp_op op;
 907         unsigned int dm_offset;
 908         bool in_place = false;
 909         int ret;
 910
 911         if (!((aes->key_len == AES_KEYSIZE_128) ||
 912               (aes->key_len == AES_KEYSIZE_192) ||
 913               (aes->key_len == AES_KEYSIZE_256)))
 914                 return -EINVAL;
 915
 916         if (((aes->mode == CCP_AES_MODE_ECB) ||
 917              (aes->mode == CCP_AES_MODE_CBC) ||
 918              (aes->mode == CCP_AES_MODE_CFB)) &&
 919             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 920                 return -EINVAL;
 921
 922         if (!aes->key || !aes->src || !aes->dst)
 923                 return -EINVAL;
 924
 925         if (aes->mode != CCP_AES_MODE_ECB) {
 926                 if (aes->iv_len != AES_BLOCK_SIZE)
 927                         return -EINVAL;
 928
 929                 if (!aes->iv)
 930                         return -EINVAL;
 931         }
 932
 933         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 934         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 935
 936         ret = -EIO;
 937         memset(&op, 0, sizeof(op));
 938         op.cmd_q = cmd_q;
 939         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 940         op.sb_key = cmd_q->sb_key;
 941         op.sb_ctx = cmd_q->sb_ctx;
 942         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 943         op.u.aes.type = aes->type;
 944         op.u.aes.mode = aes->mode;
 945         op.u.aes.action = aes->action;
 946
 947         /* All supported key sizes fit in a single (32-byte) SB entry
 948          * and must be in little endian format. Use the 256-bit byte
 949          * swap passthru option to convert from big endian to little
 950          * endian.
 951          */
 952         ret = ccp_init_dm_workarea(&key, cmd_q,
 953                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 954                                    DMA_TO_DEVICE);
 955         if (ret)
 956                 return ret;
 957
 958         dm_offset = CCP_SB_BYTES - aes->key_len;
 959         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 960         if (ret)
 961                 goto e_key;
 962         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 963                              CCP_PASSTHRU_BYTESWAP_256BIT);
 964         if (ret) {
 965                 cmd->engine_error = cmd_q->cmd_error;
 966                 goto e_key;
 967         }
 968
 969         /* The AES context fits in a single (32-byte) SB entry and
 970          * must be in little endian format. Use the 256-bit byte swap
 971          * passthru option to convert from big endian to little endian.
 972          */
 973         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 974                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 975                                    DMA_BIDIRECTIONAL);
 976         if (ret)
 977                 goto e_key;
 978
 979         if (aes->mode != CCP_AES_MODE_ECB) {
 980                 /* Load the AES context - convert to LE */
 981                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 982                 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 983                 if (ret)
 984                         goto e_ctx;
 985                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 986                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 987                 if (ret) {
 988                         cmd->engine_error = cmd_q->cmd_error;
 989                         goto e_ctx;
 990                 }
 991         }
 992         switch (aes->mode) {
 993         case CCP_AES_MODE_CFB: /* CFB128 only */
 994         case CCP_AES_MODE_CTR:
 995                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
 996                 break;
 997         default:
 998                 op.u.aes.size = 0;
 999         }
1000
1001         /* Prepare the input and output data workareas. For in-place
1002          * operations we need to set the dma direction to BIDIRECTIONAL
1003          * and copy the src workarea to the dst workarea.
1004          */
1005         if (sg_virt(aes->src) == sg_virt(aes->dst))
1006                 in_place = true;
1007
1008         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1009                             AES_BLOCK_SIZE,
1010                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1011         if (ret)
1012                 goto e_ctx;
1013
1014         if (in_place) {
1015                 dst = src;
1016         } else {
1017                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1018                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1019                 if (ret)
1020                         goto e_src;
1021         }
1022
1023         /* Send data to the CCP AES engine */
1024         while (src.sg_wa.bytes_left) {
1025                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1026                 if (!src.sg_wa.bytes_left) {
1027                         op.eom = 1;
1028
1029                         /* Since we don't retrieve the AES context in ECB
1030                          * mode we have to wait for the operation to complete
1031                          * on the last piece of data
1032                          */
1033                         if (aes->mode == CCP_AES_MODE_ECB)
1034                                 op.soc = 1;
1035                 }
1036
1037                 ret = cmd_q->ccp->vdata->perform->aes(&op);
1038                 if (ret) {
1039                         cmd->engine_error = cmd_q->cmd_error;
1040                         goto e_dst;
1041                 }
1042
1043                 ccp_process_data(&src, &dst, &op);
1044         }
1045
1046         if (aes->mode != CCP_AES_MODE_ECB) {
1047                 /* Retrieve the AES context - convert from LE to BE using
1048                  * 32-byte (256-bit) byteswapping
1049                  */
1050                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1051                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1052                 if (ret) {
1053                         cmd->engine_error = cmd_q->cmd_error;
1054                         goto e_dst;
1055                 }
1056
1057                 /* ...but we only need AES_BLOCK_SIZE bytes */
1058                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1059                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1060         }
1061
1062 e_dst:
1063         if (!in_place)
1064                 ccp_free_data(&dst, cmd_q);
1065
1066 e_src:
1067         ccp_free_data(&src, cmd_q);
1068
1069 e_ctx:
1070         ccp_dm_free(&ctx);
1071
1072 e_key:
1073         ccp_dm_free(&key);
1074
1075         return ret;
1076 }
1077
1078 static noinline_for_stack int
1079 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1080 {
1081         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1082         struct ccp_dm_workarea key, ctx;
1083         struct ccp_data src, dst;
1084         struct ccp_op op;
1085         unsigned int unit_size, dm_offset;
1086         bool in_place = false;
1087         unsigned int sb_count;
1088         enum ccp_aes_type aestype;
1089         int ret;
1090
1091         switch (xts->unit_size) {
1092         case CCP_XTS_AES_UNIT_SIZE_16:
1093                 unit_size = 16;
1094                 break;
1095         case CCP_XTS_AES_UNIT_SIZE_512:
1096                 unit_size = 512;
1097                 break;
1098         case CCP_XTS_AES_UNIT_SIZE_1024:
1099                 unit_size = 1024;
1100                 break;
1101         case CCP_XTS_AES_UNIT_SIZE_2048:
1102                 unit_size = 2048;
1103                 break;
1104         case CCP_XTS_AES_UNIT_SIZE_4096:
1105                 unit_size = 4096;
1106                 break;
1107
1108         default:
1109                 return -EINVAL;
1110         }
1111
1112         if (xts->key_len == AES_KEYSIZE_128)
1113                 aestype = CCP_AES_TYPE_128;
1114         else if (xts->key_len == AES_KEYSIZE_256)
1115                 aestype = CCP_AES_TYPE_256;
1116         else
1117                 return -EINVAL;
1118
1119         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1120                 return -EINVAL;
1121
1122         if (xts->iv_len != AES_BLOCK_SIZE)
1123                 return -EINVAL;
1124
1125         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1126                 return -EINVAL;
1127
1128         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1129         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1130
1131         ret = -EIO;
1132         memset(&op, 0, sizeof(op));
1133         op.cmd_q = cmd_q;
1134         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1135         op.sb_key = cmd_q->sb_key;
1136         op.sb_ctx = cmd_q->sb_ctx;
1137         op.init = 1;
1138         op.u.xts.type = aestype;
1139         op.u.xts.action = xts->action;
1140         op.u.xts.unit_size = xts->unit_size;
1141
1142         /* A version 3 device only supports 128-bit keys, which fits into a
1143          * single SB entry. A version 5 device uses a 512-bit vector, so two
1144          * SB entries.
1145          */
1146         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1147                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1148         else
1149                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1150         ret = ccp_init_dm_workarea(&key, cmd_q,
1151                                    sb_count * CCP_SB_BYTES,
1152                                    DMA_TO_DEVICE);
1153         if (ret)
1154                 return ret;
1155
1156         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1157                 /* All supported key sizes must be in little endian format.
1158                  * Use the 256-bit byte swap passthru option to convert from
1159                  * big endian to little endian.
1160                  */
1161                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1162                 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1163                 if (ret)
1164                         goto e_key;
1165                 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1166                 if (ret)
1167                         goto e_key;
1168         } else {
1169                 /* Version 5 CCPs use a 512-bit space for the key: each portion
1170                  * occupies 256 bits, or one entire slot, and is zero-padded.
1171                  */
1172                 unsigned int pad;
1173
1174                 dm_offset = CCP_SB_BYTES;
1175                 pad = dm_offset - xts->key_len;
1176                 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1177                 if (ret)
1178                         goto e_key;
1179                 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1180                                       xts->key_len, xts->key_len);
1181                 if (ret)
1182                         goto e_key;
1183         }
1184         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1185                              CCP_PASSTHRU_BYTESWAP_256BIT);
1186         if (ret) {
1187                 cmd->engine_error = cmd_q->cmd_error;
1188                 goto e_key;
1189         }
1190
1191         /* The AES context fits in a single (32-byte) SB entry and
1192          * for XTS is already in little endian format so no byte swapping
1193          * is needed.
1194          */
1195         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1196                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1197                                    DMA_BIDIRECTIONAL);
1198         if (ret)
1199                 goto e_key;
1200
1201         ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1202         if (ret)
1203                 goto e_ctx;
1204         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1205                              CCP_PASSTHRU_BYTESWAP_NOOP);
1206         if (ret) {
1207                 cmd->engine_error = cmd_q->cmd_error;
1208                 goto e_ctx;
1209         }
1210
1211         /* Prepare the input and output data workareas. For in-place
1212          * operations we need to set the dma direction to BIDIRECTIONAL
1213          * and copy the src workarea to the dst workarea.
1214          */
1215         if (sg_virt(xts->src) == sg_virt(xts->dst))
1216                 in_place = true;
1217
1218         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1219                             unit_size,
1220                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1221         if (ret)
1222                 goto e_ctx;
1223
1224         if (in_place) {
1225                 dst = src;
1226         } else {
1227                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1228                                     unit_size, DMA_FROM_DEVICE);
1229                 if (ret)
1230                         goto e_src;
1231         }
1232
1233         /* Send data to the CCP AES engine */
1234         while (src.sg_wa.bytes_left) {
1235                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1236                 if (!src.sg_wa.bytes_left)
1237                         op.eom = 1;
1238
1239                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1240                 if (ret) {
1241                         cmd->engine_error = cmd_q->cmd_error;
1242                         goto e_dst;
1243                 }
1244
1245                 ccp_process_data(&src, &dst, &op);
1246         }
1247
1248         /* Retrieve the AES context - convert from LE to BE using
1249          * 32-byte (256-bit) byteswapping
1250          */
1251         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1252                                CCP_PASSTHRU_BYTESWAP_256BIT);
1253         if (ret) {
1254                 cmd->engine_error = cmd_q->cmd_error;
1255                 goto e_dst;
1256         }
1257
1258         /* ...but we only need AES_BLOCK_SIZE bytes */
1259         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1260         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1261
1262 e_dst:
1263         if (!in_place)
1264                 ccp_free_data(&dst, cmd_q);
1265
1266 e_src:
1267         ccp_free_data(&src, cmd_q);
1268
1269 e_ctx:
1270         ccp_dm_free(&ctx);
1271
1272 e_key:
1273         ccp_dm_free(&key);
1274
1275         return ret;
1276 }
1277
1278 static noinline_for_stack int
1279 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1280 {
1281         struct ccp_des3_engine *des3 = &cmd->u.des3;
1282
1283         struct ccp_dm_workarea key, ctx;
1284         struct ccp_data src, dst;
1285         struct ccp_op op;
1286         unsigned int dm_offset;
1287         unsigned int len_singlekey;
1288         bool in_place = false;
1289         int ret;
1290
1291         /* Error checks */
1292         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1293                 return -EINVAL;
1294
1295         if (!cmd_q->ccp->vdata->perform->des3)
1296                 return -EINVAL;
1297
1298         if (des3->key_len != DES3_EDE_KEY_SIZE)
1299                 return -EINVAL;
1300
1301         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1302                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1303                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1304                 return -EINVAL;
1305
1306         if (!des3->key || !des3->src || !des3->dst)
1307                 return -EINVAL;
1308
1309         if (des3->mode != CCP_DES3_MODE_ECB) {
1310                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1311                         return -EINVAL;
1312
1313                 if (!des3->iv)
1314                         return -EINVAL;
1315         }
1316
1317         ret = -EIO;
1318         /* Zero out all the fields of the command desc */
1319         memset(&op, 0, sizeof(op));
1320
1321         /* Set up the Function field */
1322         op.cmd_q = cmd_q;
1323         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1324         op.sb_key = cmd_q->sb_key;
1325
1326         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1327         op.u.des3.type = des3->type;
1328         op.u.des3.mode = des3->mode;
1329         op.u.des3.action = des3->action;
1330
1331         /*
1332          * All supported key sizes fit in a single (32-byte) KSB entry and
1333          * (like AES) must be in little endian format. Use the 256-bit byte
1334          * swap passthru option to convert from big endian to little endian.
1335          */
1336         ret = ccp_init_dm_workarea(&key, cmd_q,
1337                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1338                                    DMA_TO_DEVICE);
1339         if (ret)
1340                 return ret;
1341
1342         /*
1343          * The contents of the key triplet are in the reverse order of what
1344          * is required by the engine. Copy the 3 pieces individually to put
1345          * them where they belong.
1346          */
1347         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1348
1349         len_singlekey = des3->key_len / 3;
1350         ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1351                               des3->key, 0, len_singlekey);
1352         if (ret)
1353                 goto e_key;
1354         ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1355                               des3->key, len_singlekey, len_singlekey);
1356         if (ret)
1357                 goto e_key;
1358         ret = ccp_set_dm_area(&key, dm_offset,
1359                               des3->key, 2 * len_singlekey, len_singlekey);
1360         if (ret)
1361                 goto e_key;
1362
1363         /* Copy the key to the SB */
1364         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1365                              CCP_PASSTHRU_BYTESWAP_256BIT);
1366         if (ret) {
1367                 cmd->engine_error = cmd_q->cmd_error;
1368                 goto e_key;
1369         }
1370
1371         /*
1372          * The DES3 context fits in a single (32-byte) KSB entry and
1373          * must be in little endian format. Use the 256-bit byte swap
1374          * passthru option to convert from big endian to little endian.
1375          */
1376         if (des3->mode != CCP_DES3_MODE_ECB) {
1377                 op.sb_ctx = cmd_q->sb_ctx;
1378
1379                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1380                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1381                                            DMA_BIDIRECTIONAL);
1382                 if (ret)
1383                         goto e_key;
1384
1385                 /* Load the context into the LSB */
1386                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1387                 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1388                                       des3->iv_len);
1389                 if (ret)
1390                         goto e_ctx;
1391
1392                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1393                                      CCP_PASSTHRU_BYTESWAP_256BIT);
1394                 if (ret) {
1395                         cmd->engine_error = cmd_q->cmd_error;
1396                         goto e_ctx;
1397                 }
1398         }
1399
1400         /*
1401          * Prepare the input and output data workareas. For in-place
1402          * operations we need to set the dma direction to BIDIRECTIONAL
1403          * and copy the src workarea to the dst workarea.
1404          */
1405         if (sg_virt(des3->src) == sg_virt(des3->dst))
1406                 in_place = true;
1407
1408         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1409                         DES3_EDE_BLOCK_SIZE,
1410                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1411         if (ret)
1412                 goto e_ctx;
1413
1414         if (in_place)
1415                 dst = src;
1416         else {
1417                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1418                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1419                 if (ret)
1420                         goto e_src;
1421         }
1422
1423         /* Send data to the CCP DES3 engine */
1424         while (src.sg_wa.bytes_left) {
1425                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1426                 if (!src.sg_wa.bytes_left) {
1427                         op.eom = 1;
1428
1429                         /* Since we don't retrieve the context in ECB mode
1430                          * we have to wait for the operation to complete
1431                          * on the last piece of data
1432                          */
1433                         op.soc = 0;
1434                 }
1435
1436                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1437                 if (ret) {
1438                         cmd->engine_error = cmd_q->cmd_error;
1439                         goto e_dst;
1440                 }
1441
1442                 ccp_process_data(&src, &dst, &op);
1443         }
1444
1445         if (des3->mode != CCP_DES3_MODE_ECB) {
1446                 /* Retrieve the context and make BE */
1447                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1448                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1449                 if (ret) {
1450                         cmd->engine_error = cmd_q->cmd_error;
1451                         goto e_dst;
1452                 }
1453
1454                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1455                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1456                                 DES3_EDE_BLOCK_SIZE);
1457         }
1458 e_dst:
1459         if (!in_place)
1460                 ccp_free_data(&dst, cmd_q);
1461
1462 e_src:
1463         ccp_free_data(&src, cmd_q);
1464
1465 e_ctx:
1466         if (des3->mode != CCP_DES3_MODE_ECB)
1467                 ccp_dm_free(&ctx);
1468
1469 e_key:
1470         ccp_dm_free(&key);
1471
1472         return ret;
1473 }
1474
1475 static noinline_for_stack int
1476 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1477 {
1478         struct ccp_sha_engine *sha = &cmd->u.sha;
1479         struct ccp_dm_workarea ctx;
1480         struct ccp_data src;
1481         struct ccp_op op;
1482         unsigned int ioffset, ooffset;
1483         unsigned int digest_size;
1484         int sb_count;
1485         const void *init;
1486         u64 block_size;
1487         int ctx_size;
1488         int ret;
1489
1490         switch (sha->type) {
1491         case CCP_SHA_TYPE_1:
1492                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1493                         return -EINVAL;
1494                 block_size = SHA1_BLOCK_SIZE;
1495                 break;
1496         case CCP_SHA_TYPE_224:
1497                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1498                         return -EINVAL;
1499                 block_size = SHA224_BLOCK_SIZE;
1500                 break;
1501         case CCP_SHA_TYPE_256:
1502                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1503                         return -EINVAL;
1504                 block_size = SHA256_BLOCK_SIZE;
1505                 break;
1506         case CCP_SHA_TYPE_384:
1507                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1508                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1509                         return -EINVAL;
1510                 block_size = SHA384_BLOCK_SIZE;
1511                 break;
1512         case CCP_SHA_TYPE_512:
1513                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1514                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1515                         return -EINVAL;
1516                 block_size = SHA512_BLOCK_SIZE;
1517                 break;
1518         default:
1519                 return -EINVAL;
1520         }
1521
1522         if (!sha->ctx)
1523                 return -EINVAL;
1524
1525         if (!sha->final && (sha->src_len & (block_size - 1)))
1526                 return -EINVAL;
1527
1528         /* The version 3 device can't handle zero-length input */
1529         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1530
1531                 if (!sha->src_len) {
1532                         unsigned int digest_len;
1533                         const u8 *sha_zero;
1534
1535                         /* Not final, just return */
1536                         if (!sha->final)
1537                                 return 0;
1538
1539                         /* CCP can't do a zero length sha operation so the
1540                          * caller must buffer the data.
1541                          */
1542                         if (sha->msg_bits)
1543                                 return -EINVAL;
1544
1545                         /* The CCP cannot perform zero-length sha operations
1546                          * so the caller is required to buffer data for the
1547                          * final operation. However, a sha operation for a
1548                          * message with a total length of zero is valid so
1549                          * known values are required to supply the result.
1550                          */
1551                         switch (sha->type) {
1552                         case CCP_SHA_TYPE_1:
1553                                 sha_zero = sha1_zero_message_hash;
1554                                 digest_len = SHA1_DIGEST_SIZE;
1555                                 break;
1556                         case CCP_SHA_TYPE_224:
1557                                 sha_zero = sha224_zero_message_hash;
1558                                 digest_len = SHA224_DIGEST_SIZE;
1559                                 break;
1560                         case CCP_SHA_TYPE_256:
1561                                 sha_zero = sha256_zero_message_hash;
1562                                 digest_len = SHA256_DIGEST_SIZE;
1563                                 break;
1564                         default:
1565                                 return -EINVAL;
1566                         }
1567
1568                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1569                                                  digest_len, 1);
1570
1571                         return 0;
1572                 }
1573         }
1574
1575         /* Set variables used throughout */
1576         switch (sha->type) {
1577         case CCP_SHA_TYPE_1:
1578                 digest_size = SHA1_DIGEST_SIZE;
1579                 init = (void *) ccp_sha1_init;
1580                 ctx_size = SHA1_DIGEST_SIZE;
1581                 sb_count = 1;
1582                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1583                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1584                 else
1585                         ooffset = ioffset = 0;
1586                 break;
1587         case CCP_SHA_TYPE_224:
1588                 digest_size = SHA224_DIGEST_SIZE;
1589                 init = (void *) ccp_sha224_init;
1590                 ctx_size = SHA256_DIGEST_SIZE;
1591                 sb_count = 1;
1592                 ioffset = 0;
1593                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1594                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1595                 else
1596                         ooffset = 0;
1597                 break;
1598         case CCP_SHA_TYPE_256:
1599                 digest_size = SHA256_DIGEST_SIZE;
1600                 init = (void *) ccp_sha256_init;
1601                 ctx_size = SHA256_DIGEST_SIZE;
1602                 sb_count = 1;
1603                 ooffset = ioffset = 0;
1604                 break;
1605         case CCP_SHA_TYPE_384:
1606                 digest_size = SHA384_DIGEST_SIZE;
1607                 init = (void *) ccp_sha384_init;
1608                 ctx_size = SHA512_DIGEST_SIZE;
1609                 sb_count = 2;
1610                 ioffset = 0;
1611                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1612                 break;
1613         case CCP_SHA_TYPE_512:
1614                 digest_size = SHA512_DIGEST_SIZE;
1615                 init = (void *) ccp_sha512_init;
1616                 ctx_size = SHA512_DIGEST_SIZE;
1617                 sb_count = 2;
1618                 ooffset = ioffset = 0;
1619                 break;
1620         default:
1621                 ret = -EINVAL;
1622                 goto e_data;
1623         }
1624
1625         /* For zero-length plaintext the src pointer is ignored;
1626          * otherwise both parts must be valid
1627          */
1628         if (sha->src_len && !sha->src)
1629                 return -EINVAL;
1630
1631         memset(&op, 0, sizeof(op));
1632         op.cmd_q = cmd_q;
1633         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1634         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1635         op.u.sha.type = sha->type;
1636         op.u.sha.msg_bits = sha->msg_bits;
1637
1638         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1639          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1640          * first slot, and the left half in the second. Each portion must then
1641          * be in little endian format: use the 256-bit byte swap option.
1642          */
1643         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1644                                    DMA_BIDIRECTIONAL);
1645         if (ret)
1646                 return ret;
1647         if (sha->first) {
1648                 switch (sha->type) {
1649                 case CCP_SHA_TYPE_1:
1650                 case CCP_SHA_TYPE_224:
1651                 case CCP_SHA_TYPE_256:
1652                         memcpy(ctx.address + ioffset, init, ctx_size);
1653                         break;
1654                 case CCP_SHA_TYPE_384:
1655                 case CCP_SHA_TYPE_512:
1656                         memcpy(ctx.address + ctx_size / 2, init,
1657                                ctx_size / 2);
1658                         memcpy(ctx.address, init + ctx_size / 2,
1659                                ctx_size / 2);
1660                         break;
1661                 default:
1662                         ret = -EINVAL;
1663                         goto e_ctx;
1664                 }
1665         } else {
1666                 /* Restore the context */
1667                 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1668                                       sb_count * CCP_SB_BYTES);
1669                 if (ret)
1670                         goto e_ctx;
1671         }
1672
1673         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1674                              CCP_PASSTHRU_BYTESWAP_256BIT);
1675         if (ret) {
1676                 cmd->engine_error = cmd_q->cmd_error;
1677                 goto e_ctx;
1678         }
1679
1680         if (sha->src) {
1681                 /* Send data to the CCP SHA engine; block_size is set above */
1682                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1683                                     block_size, DMA_TO_DEVICE);
1684                 if (ret)
1685                         goto e_ctx;
1686
1687                 while (src.sg_wa.bytes_left) {
1688                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1689                         if (sha->final && !src.sg_wa.bytes_left)
1690                                 op.eom = 1;
1691
1692                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1693                         if (ret) {
1694                                 cmd->engine_error = cmd_q->cmd_error;
1695                                 goto e_data;
1696                         }
1697
1698                         ccp_process_data(&src, NULL, &op);
1699                 }
1700         } else {
1701                 op.eom = 1;
1702                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1703                 if (ret) {
1704                         cmd->engine_error = cmd_q->cmd_error;
1705                         goto e_data;
1706                 }
1707         }
1708
1709         /* Retrieve the SHA context - convert from LE to BE using
1710          * 32-byte (256-bit) byteswapping to BE
1711          */
1712         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1713                                CCP_PASSTHRU_BYTESWAP_256BIT);
1714         if (ret) {
1715                 cmd->engine_error = cmd_q->cmd_error;
1716                 goto e_data;
1717         }
1718
1719         if (sha->final) {
1720                 /* Finishing up, so get the digest */
1721                 switch (sha->type) {
1722                 case CCP_SHA_TYPE_1:
1723                 case CCP_SHA_TYPE_224:
1724                 case CCP_SHA_TYPE_256:
1725                         ccp_get_dm_area(&ctx, ooffset,
1726                                         sha->ctx, 0,
1727                                         digest_size);
1728                         break;
1729                 case CCP_SHA_TYPE_384:
1730                 case CCP_SHA_TYPE_512:
1731                         ccp_get_dm_area(&ctx, 0,
1732                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1733                                         LSB_ITEM_SIZE);
1734                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1735                                         sha->ctx, 0,
1736                                         LSB_ITEM_SIZE - ooffset);
1737                         break;
1738                 default:
1739                         ret = -EINVAL;
1740                         goto e_ctx;
1741                 }
1742         } else {
1743                 /* Stash the context */
1744                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1745                                 sb_count * CCP_SB_BYTES);
1746         }
1747
1748         if (sha->final && sha->opad) {
1749                 /* HMAC operation, recursively perform final SHA */
1750                 struct ccp_cmd hmac_cmd;
1751                 struct scatterlist sg;
1752                 u8 *hmac_buf;
1753
1754                 if (sha->opad_len != block_size) {
1755                         ret = -EINVAL;
1756                         goto e_data;
1757                 }
1758
1759                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1760                 if (!hmac_buf) {
1761                         ret = -ENOMEM;
1762                         goto e_data;
1763                 }
1764                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1765
1766                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1767                 switch (sha->type) {
1768                 case CCP_SHA_TYPE_1:
1769                 case CCP_SHA_TYPE_224:
1770                 case CCP_SHA_TYPE_256:
1771                         memcpy(hmac_buf + block_size,
1772                                ctx.address + ooffset,
1773                                digest_size);
1774                         break;
1775                 case CCP_SHA_TYPE_384:
1776                 case CCP_SHA_TYPE_512:
1777                         memcpy(hmac_buf + block_size,
1778                                ctx.address + LSB_ITEM_SIZE + ooffset,
1779                                LSB_ITEM_SIZE);
1780                         memcpy(hmac_buf + block_size +
1781                                (LSB_ITEM_SIZE - ooffset),
1782                                ctx.address,
1783                                LSB_ITEM_SIZE);
1784                         break;
1785                 default:
1786                         ret = -EINVAL;
1787                         goto e_ctx;
1788                 }
1789
1790                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1791                 hmac_cmd.engine = CCP_ENGINE_SHA;
1792                 hmac_cmd.u.sha.type = sha->type;
1793                 hmac_cmd.u.sha.ctx = sha->ctx;
1794                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1795                 hmac_cmd.u.sha.src = &sg;
1796                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1797                 hmac_cmd.u.sha.opad = NULL;
1798                 hmac_cmd.u.sha.opad_len = 0;
1799                 hmac_cmd.u.sha.first = 1;
1800                 hmac_cmd.u.sha.final = 1;
1801                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1802
1803                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1804                 if (ret)
1805                         cmd->engine_error = hmac_cmd.engine_error;
1806
1807                 kfree(hmac_buf);
1808         }
1809
1810 e_data:
1811         if (sha->src)
1812                 ccp_free_data(&src, cmd_q);
1813
1814 e_ctx:
1815         ccp_dm_free(&ctx);
1816
1817         return ret;
1818 }
1819
1820 static noinline_for_stack int
1821 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1822 {
1823         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1824         struct ccp_dm_workarea exp, src, dst;
1825         struct ccp_op op;
1826         unsigned int sb_count, i_len, o_len;
1827         int ret;
1828
1829         /* Check against the maximum allowable size, in bits */
1830         if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1831                 return -EINVAL;
1832
1833         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1834                 return -EINVAL;
1835
1836         memset(&op, 0, sizeof(op));
1837         op.cmd_q = cmd_q;
1838         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1839
1840         /* The RSA modulus must precede the message being acted upon, so
1841          * it must be copied to a DMA area where the message and the
1842          * modulus can be concatenated.  Therefore the input buffer
1843          * length required is twice the output buffer length (which
1844          * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1845          * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1846          * required.
1847          */
1848         o_len = 32 * ((rsa->key_size + 255) / 256);
1849         i_len = o_len * 2;
1850
1851         sb_count = 0;
1852         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1853                 /* sb_count is the number of storage block slots required
1854                  * for the modulus.
1855                  */
1856                 sb_count = o_len / CCP_SB_BYTES;
1857                 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1858                                                                 sb_count);
1859                 if (!op.sb_key)
1860                         return -EIO;
1861         } else {
1862                 /* A version 5 device allows a modulus size that will not fit
1863                  * in the LSB, so the command will transfer it from memory.
1864                  * Set the sb key to the default, even though it's not used.
1865                  */
1866                 op.sb_key = cmd_q->sb_key;
1867         }
1868
1869         /* The RSA exponent must be in little endian format. Reverse its
1870          * byte order.
1871          */
1872         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1873         if (ret)
1874                 goto e_sb;
1875
1876         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1877         if (ret)
1878                 goto e_exp;
1879
1880         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1881                 /* Copy the exponent to the local storage block, using
1882                  * as many 32-byte blocks as were allocated above. It's
1883                  * already little endian, so no further change is required.
1884                  */
1885                 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1886                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1887                 if (ret) {
1888                         cmd->engine_error = cmd_q->cmd_error;
1889                         goto e_exp;
1890                 }
1891         } else {
1892                 /* The exponent can be retrieved from memory via DMA. */
1893                 op.exp.u.dma.address = exp.dma.address;
1894                 op.exp.u.dma.offset = 0;
1895         }
1896
1897         /* Concatenate the modulus and the message. Both the modulus and
1898          * the operands must be in little endian format.  Since the input
1899          * is in big endian format it must be converted.
1900          */
1901         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1902         if (ret)
1903                 goto e_exp;
1904
1905         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1906         if (ret)
1907                 goto e_src;
1908         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1909         if (ret)
1910                 goto e_src;
1911
1912         /* Prepare the output area for the operation */
1913         ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1914         if (ret)
1915                 goto e_src;
1916
1917         op.soc = 1;
1918         op.src.u.dma.address = src.dma.address;
1919         op.src.u.dma.offset = 0;
1920         op.src.u.dma.length = i_len;
1921         op.dst.u.dma.address = dst.dma.address;
1922         op.dst.u.dma.offset = 0;
1923         op.dst.u.dma.length = o_len;
1924
1925         op.u.rsa.mod_size = rsa->key_size;
1926         op.u.rsa.input_len = i_len;
1927
1928         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1929         if (ret) {
1930                 cmd->engine_error = cmd_q->cmd_error;
1931                 goto e_dst;
1932         }
1933
1934         ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1935
1936 e_dst:
1937         ccp_dm_free(&dst);
1938
1939 e_src:
1940         ccp_dm_free(&src);
1941
1942 e_exp:
1943         ccp_dm_free(&exp);
1944
1945 e_sb:
1946         if (sb_count)
1947                 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1948
1949         return ret;
1950 }
1951
1952 static noinline_for_stack int
1953 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1954 {
1955         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1956         struct ccp_dm_workarea mask;
1957         struct ccp_data src, dst;
1958         struct ccp_op op;
1959         bool in_place = false;
1960         unsigned int i;
1961         int ret = 0;
1962
1963         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1964                 return -EINVAL;
1965
1966         if (!pt->src || !pt->dst)
1967                 return -EINVAL;
1968
1969         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1970                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1971                         return -EINVAL;
1972                 if (!pt->mask)
1973                         return -EINVAL;
1974         }
1975
1976         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1977
1978         memset(&op, 0, sizeof(op));
1979         op.cmd_q = cmd_q;
1980         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1981
1982         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1983                 /* Load the mask */
1984                 op.sb_key = cmd_q->sb_key;
1985
1986                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1987                                            CCP_PASSTHRU_SB_COUNT *
1988                                            CCP_SB_BYTES,
1989                                            DMA_TO_DEVICE);
1990                 if (ret)
1991                         return ret;
1992
1993                 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1994                 if (ret)
1995                         goto e_mask;
1996                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1997                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1998                 if (ret) {
1999                         cmd->engine_error = cmd_q->cmd_error;
2000                         goto e_mask;
2001                 }
2002         }
2003
2004         /* Prepare the input and output data workareas. For in-place
2005          * operations we need to set the dma direction to BIDIRECTIONAL
2006          * and copy the src workarea to the dst workarea.
2007          */
2008         if (sg_virt(pt->src) == sg_virt(pt->dst))
2009                 in_place = true;
2010
2011         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2012                             CCP_PASSTHRU_MASKSIZE,
2013                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2014         if (ret)
2015                 goto e_mask;
2016
2017         if (in_place) {
2018                 dst = src;
2019         } else {
2020                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2021                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2022                 if (ret)
2023                         goto e_src;
2024         }
2025
2026         /* Send data to the CCP Passthru engine
2027          *   Because the CCP engine works on a single source and destination
2028          *   dma address at a time, each entry in the source scatterlist
2029          *   (after the dma_map_sg call) must be less than or equal to the
2030          *   (remaining) length in the destination scatterlist entry and the
2031          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2032          */
2033         dst.sg_wa.sg_used = 0;
2034         for (i = 1; i <= src.sg_wa.dma_count; i++) {
2035                 if (!dst.sg_wa.sg ||
2036                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
2037                         ret = -EINVAL;
2038                         goto e_dst;
2039                 }
2040
2041                 if (i == src.sg_wa.dma_count) {
2042                         op.eom = 1;
2043                         op.soc = 1;
2044                 }
2045
2046                 op.src.type = CCP_MEMTYPE_SYSTEM;
2047                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2048                 op.src.u.dma.offset = 0;
2049                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2050
2051                 op.dst.type = CCP_MEMTYPE_SYSTEM;
2052                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2053                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2054                 op.dst.u.dma.length = op.src.u.dma.length;
2055
2056                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2057                 if (ret) {
2058                         cmd->engine_error = cmd_q->cmd_error;
2059                         goto e_dst;
2060                 }
2061
2062                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
2063                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
2064                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2065                         dst.sg_wa.sg_used = 0;
2066                 }
2067                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2068         }
2069
2070 e_dst:
2071         if (!in_place)
2072                 ccp_free_data(&dst, cmd_q);
2073
2074 e_src:
2075         ccp_free_data(&src, cmd_q);
2076
2077 e_mask:
2078         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2079                 ccp_dm_free(&mask);
2080
2081         return ret;
2082 }
2083
2084 static noinline_for_stack int
2085 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2086                                       struct ccp_cmd *cmd)
2087 {
2088         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2089         struct ccp_dm_workarea mask;
2090         struct ccp_op op;
2091         int ret;
2092
2093         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2094                 return -EINVAL;
2095
2096         if (!pt->src_dma || !pt->dst_dma)
2097                 return -EINVAL;
2098
2099         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2100                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2101                         return -EINVAL;
2102                 if (!pt->mask)
2103                         return -EINVAL;
2104         }
2105
2106         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2107
2108         memset(&op, 0, sizeof(op));
2109         op.cmd_q = cmd_q;
2110         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2111
2112         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2113                 /* Load the mask */
2114                 op.sb_key = cmd_q->sb_key;
2115
2116                 mask.length = pt->mask_len;
2117                 mask.dma.address = pt->mask;
2118                 mask.dma.length = pt->mask_len;
2119
2120                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2121                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2122                 if (ret) {
2123                         cmd->engine_error = cmd_q->cmd_error;
2124                         return ret;
2125                 }
2126         }
2127
2128         /* Send data to the CCP Passthru engine */
2129         op.eom = 1;
2130         op.soc = 1;
2131
2132         op.src.type = CCP_MEMTYPE_SYSTEM;
2133         op.src.u.dma.address = pt->src_dma;
2134         op.src.u.dma.offset = 0;
2135         op.src.u.dma.length = pt->src_len;
2136
2137         op.dst.type = CCP_MEMTYPE_SYSTEM;
2138         op.dst.u.dma.address = pt->dst_dma;
2139         op.dst.u.dma.offset = 0;
2140         op.dst.u.dma.length = pt->src_len;
2141
2142         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2143         if (ret)
2144                 cmd->engine_error = cmd_q->cmd_error;
2145
2146         return ret;
2147 }
2148
2149 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2150 {
2151         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2152         struct ccp_dm_workarea src, dst;
2153         struct ccp_op op;
2154         int ret;
2155         u8 *save;
2156
2157         if (!ecc->u.mm.operand_1 ||
2158             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2159                 return -EINVAL;
2160
2161         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2162                 if (!ecc->u.mm.operand_2 ||
2163                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2164                         return -EINVAL;
2165
2166         if (!ecc->u.mm.result ||
2167             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2168                 return -EINVAL;
2169
2170         memset(&op, 0, sizeof(op));
2171         op.cmd_q = cmd_q;
2172         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2173
2174         /* Concatenate the modulus and the operands. Both the modulus and
2175          * the operands must be in little endian format.  Since the input
2176          * is in big endian format it must be converted and placed in a
2177          * fixed length buffer.
2178          */
2179         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2180                                    DMA_TO_DEVICE);
2181         if (ret)
2182                 return ret;
2183
2184         /* Save the workarea address since it is updated in order to perform
2185          * the concatenation
2186          */
2187         save = src.address;
2188
2189         /* Copy the ECC modulus */
2190         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2191         if (ret)
2192                 goto e_src;
2193         src.address += CCP_ECC_OPERAND_SIZE;
2194
2195         /* Copy the first operand */
2196         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2197                                       ecc->u.mm.operand_1_len);
2198         if (ret)
2199                 goto e_src;
2200         src.address += CCP_ECC_OPERAND_SIZE;
2201
2202         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2203                 /* Copy the second operand */
2204                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2205                                               ecc->u.mm.operand_2_len);
2206                 if (ret)
2207                         goto e_src;
2208                 src.address += CCP_ECC_OPERAND_SIZE;
2209         }
2210
2211         /* Restore the workarea address */
2212         src.address = save;
2213
2214         /* Prepare the output area for the operation */
2215         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2216                                    DMA_FROM_DEVICE);
2217         if (ret)
2218                 goto e_src;
2219
2220         op.soc = 1;
2221         op.src.u.dma.address = src.dma.address;
2222         op.src.u.dma.offset = 0;
2223         op.src.u.dma.length = src.length;
2224         op.dst.u.dma.address = dst.dma.address;
2225         op.dst.u.dma.offset = 0;
2226         op.dst.u.dma.length = dst.length;
2227
2228         op.u.ecc.function = cmd->u.ecc.function;
2229
2230         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2231         if (ret) {
2232                 cmd->engine_error = cmd_q->cmd_error;
2233                 goto e_dst;
2234         }
2235
2236         ecc->ecc_result = le16_to_cpup(
2237                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2238         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2239                 ret = -EIO;
2240                 goto e_dst;
2241         }
2242
2243         /* Save the ECC result */
2244         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2245                                 CCP_ECC_MODULUS_BYTES);
2246
2247 e_dst:
2248         ccp_dm_free(&dst);
2249
2250 e_src:
2251         ccp_dm_free(&src);
2252
2253         return ret;
2254 }
2255
2256 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2257 {
2258         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2259         struct ccp_dm_workarea src, dst;
2260         struct ccp_op op;
2261         int ret;
2262         u8 *save;
2263
2264         if (!ecc->u.pm.point_1.x ||
2265             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2266             !ecc->u.pm.point_1.y ||
2267             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2268                 return -EINVAL;
2269
2270         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2271                 if (!ecc->u.pm.point_2.x ||
2272                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2273                     !ecc->u.pm.point_2.y ||
2274                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2275                         return -EINVAL;
2276         } else {
2277                 if (!ecc->u.pm.domain_a ||
2278                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2279                         return -EINVAL;
2280
2281                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2282                         if (!ecc->u.pm.scalar ||
2283                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2284                                 return -EINVAL;
2285         }
2286
2287         if (!ecc->u.pm.result.x ||
2288             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2289             !ecc->u.pm.result.y ||
2290             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2291                 return -EINVAL;
2292
2293         memset(&op, 0, sizeof(op));
2294         op.cmd_q = cmd_q;
2295         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2296
2297         /* Concatenate the modulus and the operands. Both the modulus and
2298          * the operands must be in little endian format.  Since the input
2299          * is in big endian format it must be converted and placed in a
2300          * fixed length buffer.
2301          */
2302         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2303                                    DMA_TO_DEVICE);
2304         if (ret)
2305                 return ret;
2306
2307         /* Save the workarea address since it is updated in order to perform
2308          * the concatenation
2309          */
2310         save = src.address;
2311
2312         /* Copy the ECC modulus */
2313         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2314         if (ret)
2315                 goto e_src;
2316         src.address += CCP_ECC_OPERAND_SIZE;
2317
2318         /* Copy the first point X and Y coordinate */
2319         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2320                                       ecc->u.pm.point_1.x_len);
2321         if (ret)
2322                 goto e_src;
2323         src.address += CCP_ECC_OPERAND_SIZE;
2324         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2325                                       ecc->u.pm.point_1.y_len);
2326         if (ret)
2327                 goto e_src;
2328         src.address += CCP_ECC_OPERAND_SIZE;
2329
2330         /* Set the first point Z coordinate to 1 */
2331         *src.address = 0x01;
2332         src.address += CCP_ECC_OPERAND_SIZE;
2333
2334         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2335                 /* Copy the second point X and Y coordinate */
2336                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2337                                               ecc->u.pm.point_2.x_len);
2338                 if (ret)
2339                         goto e_src;
2340                 src.address += CCP_ECC_OPERAND_SIZE;
2341                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2342                                               ecc->u.pm.point_2.y_len);
2343                 if (ret)
2344                         goto e_src;
2345                 src.address += CCP_ECC_OPERAND_SIZE;
2346
2347                 /* Set the second point Z coordinate to 1 */
2348                 *src.address = 0x01;
2349                 src.address += CCP_ECC_OPERAND_SIZE;
2350         } else {
2351                 /* Copy the Domain "a" parameter */
2352                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2353                                               ecc->u.pm.domain_a_len);
2354                 if (ret)
2355                         goto e_src;
2356                 src.address += CCP_ECC_OPERAND_SIZE;
2357
2358                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2359                         /* Copy the scalar value */
2360                         ret = ccp_reverse_set_dm_area(&src, 0,
2361                                                       ecc->u.pm.scalar, 0,
2362                                                       ecc->u.pm.scalar_len);
2363                         if (ret)
2364                                 goto e_src;
2365                         src.address += CCP_ECC_OPERAND_SIZE;
2366                 }
2367         }
2368
2369         /* Restore the workarea address */
2370         src.address = save;
2371
2372         /* Prepare the output area for the operation */
2373         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2374                                    DMA_FROM_DEVICE);
2375         if (ret)
2376                 goto e_src;
2377
2378         op.soc = 1;
2379         op.src.u.dma.address = src.dma.address;
2380         op.src.u.dma.offset = 0;
2381         op.src.u.dma.length = src.length;
2382         op.dst.u.dma.address = dst.dma.address;
2383         op.dst.u.dma.offset = 0;
2384         op.dst.u.dma.length = dst.length;
2385
2386         op.u.ecc.function = cmd->u.ecc.function;
2387
2388         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2389         if (ret) {
2390                 cmd->engine_error = cmd_q->cmd_error;
2391                 goto e_dst;
2392         }
2393
2394         ecc->ecc_result = le16_to_cpup(
2395                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2396         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2397                 ret = -EIO;
2398                 goto e_dst;
2399         }
2400
2401         /* Save the workarea address since it is updated as we walk through
2402          * to copy the point math result
2403          */
2404         save = dst.address;
2405
2406         /* Save the ECC result X and Y coordinates */
2407         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2408                                 CCP_ECC_MODULUS_BYTES);
2409         dst.address += CCP_ECC_OUTPUT_SIZE;
2410         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2411                                 CCP_ECC_MODULUS_BYTES);
2412         dst.address += CCP_ECC_OUTPUT_SIZE;
2413
2414         /* Restore the workarea address */
2415         dst.address = save;
2416
2417 e_dst:
2418         ccp_dm_free(&dst);
2419
2420 e_src:
2421         ccp_dm_free(&src);
2422
2423         return ret;
2424 }
2425
2426 static noinline_for_stack int
2427 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2428 {
2429         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2430
2431         ecc->ecc_result = 0;
2432
2433         if (!ecc->mod ||
2434             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2435                 return -EINVAL;
2436
2437         switch (ecc->function) {
2438         case CCP_ECC_FUNCTION_MMUL_384BIT:
2439         case CCP_ECC_FUNCTION_MADD_384BIT:
2440         case CCP_ECC_FUNCTION_MINV_384BIT:
2441                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2442
2443         case CCP_ECC_FUNCTION_PADD_384BIT:
2444         case CCP_ECC_FUNCTION_PMUL_384BIT:
2445         case CCP_ECC_FUNCTION_PDBL_384BIT:
2446                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2447
2448         default:
2449                 return -EINVAL;
2450         }
2451 }
2452
2453 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2454 {
2455         int ret;
2456
2457         cmd->engine_error = 0;
2458         cmd_q->cmd_error = 0;
2459         cmd_q->int_rcvd = 0;
2460         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2461
2462         switch (cmd->engine) {
2463         case CCP_ENGINE_AES:
2464                 switch (cmd->u.aes.mode) {
2465                 case CCP_AES_MODE_CMAC:
2466                         ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2467                         break;
2468                 case CCP_AES_MODE_GCM:
2469                         ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2470                         break;
2471                 default:
2472                         ret = ccp_run_aes_cmd(cmd_q, cmd);
2473                         break;
2474                 }
2475                 break;
2476         case CCP_ENGINE_XTS_AES_128:
2477                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2478                 break;
2479         case CCP_ENGINE_DES3:
2480                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2481                 break;
2482         case CCP_ENGINE_SHA:
2483                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2484                 break;
2485         case CCP_ENGINE_RSA:
2486                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2487                 break;
2488         case CCP_ENGINE_PASSTHRU:
2489                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2490                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2491                 else
2492                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2493                 break;
2494         case CCP_ENGINE_ECC:
2495                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2496                 break;
2497         default:
2498                 ret = -EINVAL;
2499         }
2500
2501         return ret;
2502 }