arch/s390/crypto/aes_s390.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * s390 implementation of the AES Cipher Algorithm.
   5  *
   6  * s390 Version:
   7  *   Copyright IBM Corp. 2005, 2007
   8  *   Author(s): Jan Glauber (jang@de.ibm.com)
   9  *              Sebastian Siewior (sebastian@breakpoint.cc> SW-Fallback
  10  *
  11  * Derived from "crypto/aes_generic.c"
  12  *
  13  * This program is free software; you can redistribute it and/or modify it
  14  * under the terms of the GNU General Public License as published by the Free
  15  * Software Foundation; either version 2 of the License, or (at your option)
  16  * any later version.
  17  *
  18  */
  19
  20 #define KMSG_COMPONENT "aes_s390"
  21 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  22
  23 #include <crypto/aes.h>
  24 #include <crypto/algapi.h>
  25 #include <linux/err.h>
  26 #include <linux/module.h>
  27 #include <linux/cpufeature.h>
  28 #include <linux/init.h>
  29 #include <linux/spinlock.h>
  30 #include "crypt_s390.h"
  31
  32 #define AES_KEYLEN_128          1
  33 #define AES_KEYLEN_192          2
  34 #define AES_KEYLEN_256          4
  35
  36 static u8 *ctrblk;
  37 static DEFINE_SPINLOCK(ctrblk_lock);
  38 static char keylen_flag;
  39
  40 struct s390_aes_ctx {
  41         u8 key[AES_MAX_KEY_SIZE];
  42         long enc;
  43         long dec;
  44         int key_len;
  45         union {
  46                 struct crypto_blkcipher *blk;
  47                 struct crypto_cipher *cip;
  48         } fallback;
  49 };
  50
  51 struct pcc_param {
  52         u8 key[32];
  53         u8 tweak[16];
  54         u8 block[16];
  55         u8 bit[16];
  56         u8 xts[16];
  57 };
  58
  59 struct s390_xts_ctx {
  60         u8 key[32];
  61         u8 pcc_key[32];
  62         long enc;
  63         long dec;
  64         int key_len;
  65         struct crypto_blkcipher *fallback;
  66 };
  67
  68 /*
  69  * Check if the key_len is supported by the HW.
  70  * Returns 0 if it is, a positive number if it is not and software fallback is
  71  * required or a negative number in case the key size is not valid
  72  */
  73 static int need_fallback(unsigned int key_len)
  74 {
  75         switch (key_len) {
  76         case 16:
  77                 if (!(keylen_flag & AES_KEYLEN_128))
  78                         return 1;
  79                 break;
  80         case 24:
  81                 if (!(keylen_flag & AES_KEYLEN_192))
  82                         return 1;
  83                 break;
  84         case 32:
  85                 if (!(keylen_flag & AES_KEYLEN_256))
  86                         return 1;
  87                 break;
  88         default:
  89                 return -1;
  90                 break;
  91         }
  92         return 0;
  93 }
  94
  95 static int setkey_fallback_cip(struct crypto_tfm *tfm, const u8 *in_key,
  96                 unsigned int key_len)
  97 {
  98         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
  99         int ret;
 100
 101         sctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 102         sctx->fallback.cip->base.crt_flags |= (tfm->crt_flags &
 103                         CRYPTO_TFM_REQ_MASK);
 104
 105         ret = crypto_cipher_setkey(sctx->fallback.cip, in_key, key_len);
 106         if (ret) {
 107                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 108                 tfm->crt_flags |= (sctx->fallback.cip->base.crt_flags &
 109                                 CRYPTO_TFM_RES_MASK);
 110         }
 111         return ret;
 112 }
 113
 114 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 115                        unsigned int key_len)
 116 {
 117         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 118         u32 *flags = &tfm->crt_flags;
 119         int ret;
 120
 121         ret = need_fallback(key_len);
 122         if (ret < 0) {
 123                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 124                 return -EINVAL;
 125         }
 126
 127         sctx->key_len = key_len;
 128         if (!ret) {
 129                 memcpy(sctx->key, in_key, key_len);
 130                 return 0;
 131         }
 132
 133         return setkey_fallback_cip(tfm, in_key, key_len);
 134 }
 135
 136 static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 137 {
 138         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 139
 140         if (unlikely(need_fallback(sctx->key_len))) {
 141                 crypto_cipher_encrypt_one(sctx->fallback.cip, out, in);
 142                 return;
 143         }
 144
 145         switch (sctx->key_len) {
 146         case 16:
 147                 crypt_s390_km(KM_AES_128_ENCRYPT, &sctx->key, out, in,
 148                               AES_BLOCK_SIZE);
 149                 break;
 150         case 24:
 151                 crypt_s390_km(KM_AES_192_ENCRYPT, &sctx->key, out, in,
 152                               AES_BLOCK_SIZE);
 153                 break;
 154         case 32:
 155                 crypt_s390_km(KM_AES_256_ENCRYPT, &sctx->key, out, in,
 156                               AES_BLOCK_SIZE);
 157                 break;
 158         }
 159 }
 160
 161 static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 162 {
 163         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 164
 165         if (unlikely(need_fallback(sctx->key_len))) {
 166                 crypto_cipher_decrypt_one(sctx->fallback.cip, out, in);
 167                 return;
 168         }
 169
 170         switch (sctx->key_len) {
 171         case 16:
 172                 crypt_s390_km(KM_AES_128_DECRYPT, &sctx->key, out, in,
 173                               AES_BLOCK_SIZE);
 174                 break;
 175         case 24:
 176                 crypt_s390_km(KM_AES_192_DECRYPT, &sctx->key, out, in,
 177                               AES_BLOCK_SIZE);
 178                 break;
 179         case 32:
 180                 crypt_s390_km(KM_AES_256_DECRYPT, &sctx->key, out, in,
 181                               AES_BLOCK_SIZE);
 182                 break;
 183         }
 184 }
 185
 186 static int fallback_init_cip(struct crypto_tfm *tfm)
 187 {
 188         const char *name = tfm->__crt_alg->cra_name;
 189         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 190
 191         sctx->fallback.cip = crypto_alloc_cipher(name, 0,
 192                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 193
 194         if (IS_ERR(sctx->fallback.cip)) {
 195                 pr_err("Allocating AES fallback algorithm %s failed\n",
 196                        name);
 197                 return PTR_ERR(sctx->fallback.cip);
 198         }
 199
 200         return 0;
 201 }
 202
 203 static void fallback_exit_cip(struct crypto_tfm *tfm)
 204 {
 205         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 206
 207         crypto_free_cipher(sctx->fallback.cip);
 208         sctx->fallback.cip = NULL;
 209 }
 210
 211 static struct crypto_alg aes_alg = {
 212         .cra_name               =       "aes",
 213         .cra_driver_name        =       "aes-s390",
 214         .cra_priority           =       CRYPT_S390_PRIORITY,
 215         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER |
 216                                         CRYPTO_ALG_NEED_FALLBACK,
 217         .cra_blocksize          =       AES_BLOCK_SIZE,
 218         .cra_ctxsize            =       sizeof(struct s390_aes_ctx),
 219         .cra_module             =       THIS_MODULE,
 220         .cra_init               =       fallback_init_cip,
 221         .cra_exit               =       fallback_exit_cip,
 222         .cra_u                  =       {
 223                 .cipher = {
 224                         .cia_min_keysize        =       AES_MIN_KEY_SIZE,
 225                         .cia_max_keysize        =       AES_MAX_KEY_SIZE,
 226                         .cia_setkey             =       aes_set_key,
 227                         .cia_encrypt            =       aes_encrypt,
 228                         .cia_decrypt            =       aes_decrypt,
 229                 }
 230         }
 231 };
 232
 233 static int setkey_fallback_blk(struct crypto_tfm *tfm, const u8 *key,
 234                 unsigned int len)
 235 {
 236         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 237         unsigned int ret;
 238
 239         sctx->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 240         sctx->fallback.blk->base.crt_flags |= (tfm->crt_flags &
 241                         CRYPTO_TFM_REQ_MASK);
 242
 243         ret = crypto_blkcipher_setkey(sctx->fallback.blk, key, len);
 244         if (ret) {
 245                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 246                 tfm->crt_flags |= (sctx->fallback.blk->base.crt_flags &
 247                                 CRYPTO_TFM_RES_MASK);
 248         }
 249         return ret;
 250 }
 251
 252 static int fallback_blk_dec(struct blkcipher_desc *desc,
 253                 struct scatterlist *dst, struct scatterlist *src,
 254                 unsigned int nbytes)
 255 {
 256         unsigned int ret;
 257         struct crypto_blkcipher *tfm;
 258         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 259
 260         tfm = desc->tfm;
 261         desc->tfm = sctx->fallback.blk;
 262
 263         ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
 264
 265         desc->tfm = tfm;
 266         return ret;
 267 }
 268
 269 static int fallback_blk_enc(struct blkcipher_desc *desc,
 270                 struct scatterlist *dst, struct scatterlist *src,
 271                 unsigned int nbytes)
 272 {
 273         unsigned int ret;
 274         struct crypto_blkcipher *tfm;
 275         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 276
 277         tfm = desc->tfm;
 278         desc->tfm = sctx->fallback.blk;
 279
 280         ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
 281
 282         desc->tfm = tfm;
 283         return ret;
 284 }
 285
 286 static int ecb_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 287                            unsigned int key_len)
 288 {
 289         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 290         int ret;
 291
 292         ret = need_fallback(key_len);
 293         if (ret > 0) {
 294                 sctx->key_len = key_len;
 295                 return setkey_fallback_blk(tfm, in_key, key_len);
 296         }
 297
 298         switch (key_len) {
 299         case 16:
 300                 sctx->enc = KM_AES_128_ENCRYPT;
 301                 sctx->dec = KM_AES_128_DECRYPT;
 302                 break;
 303         case 24:
 304                 sctx->enc = KM_AES_192_ENCRYPT;
 305                 sctx->dec = KM_AES_192_DECRYPT;
 306                 break;
 307         case 32:
 308                 sctx->enc = KM_AES_256_ENCRYPT;
 309                 sctx->dec = KM_AES_256_DECRYPT;
 310                 break;
 311         }
 312
 313         return aes_set_key(tfm, in_key, key_len);
 314 }
 315
 316 static int ecb_aes_crypt(struct blkcipher_desc *desc, long func, void *param,
 317                          struct blkcipher_walk *walk)
 318 {
 319         int ret = blkcipher_walk_virt(desc, walk);
 320         unsigned int nbytes;
 321
 322         while ((nbytes = walk->nbytes)) {
 323                 /* only use complete blocks */
 324                 unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
 325                 u8 *out = walk->dst.virt.addr;
 326                 u8 *in = walk->src.virt.addr;
 327
 328                 ret = crypt_s390_km(func, param, out, in, n);
 329                 if (ret < 0 || ret != n)
 330                         return -EIO;
 331
 332                 nbytes &= AES_BLOCK_SIZE - 1;
 333                 ret = blkcipher_walk_done(desc, walk, nbytes);
 334         }
 335
 336         return ret;
 337 }
 338
 339 static int ecb_aes_encrypt(struct blkcipher_desc *desc,
 340                            struct scatterlist *dst, struct scatterlist *src,
 341                            unsigned int nbytes)
 342 {
 343         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 344         struct blkcipher_walk walk;
 345
 346         if (unlikely(need_fallback(sctx->key_len)))
 347                 return fallback_blk_enc(desc, dst, src, nbytes);
 348
 349         blkcipher_walk_init(&walk, dst, src, nbytes);
 350         return ecb_aes_crypt(desc, sctx->enc, sctx->key, &walk);
 351 }
 352
 353 static int ecb_aes_decrypt(struct blkcipher_desc *desc,
 354                            struct scatterlist *dst, struct scatterlist *src,
 355                            unsigned int nbytes)
 356 {
 357         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 358         struct blkcipher_walk walk;
 359
 360         if (unlikely(need_fallback(sctx->key_len)))
 361                 return fallback_blk_dec(desc, dst, src, nbytes);
 362
 363         blkcipher_walk_init(&walk, dst, src, nbytes);
 364         return ecb_aes_crypt(desc, sctx->dec, sctx->key, &walk);
 365 }
 366
 367 static int fallback_init_blk(struct crypto_tfm *tfm)
 368 {
 369         const char *name = tfm->__crt_alg->cra_name;
 370         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 371
 372         sctx->fallback.blk = crypto_alloc_blkcipher(name, 0,
 373                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 374
 375         if (IS_ERR(sctx->fallback.blk)) {
 376                 pr_err("Allocating AES fallback algorithm %s failed\n",
 377                        name);
 378                 return PTR_ERR(sctx->fallback.blk);
 379         }
 380
 381         return 0;
 382 }
 383
 384 static void fallback_exit_blk(struct crypto_tfm *tfm)
 385 {
 386         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 387
 388         crypto_free_blkcipher(sctx->fallback.blk);
 389         sctx->fallback.blk = NULL;
 390 }
 391
 392 static struct crypto_alg ecb_aes_alg = {
 393         .cra_name               =       "ecb(aes)",
 394         .cra_driver_name        =       "ecb-aes-s390",
 395         .cra_priority           =       CRYPT_S390_COMPOSITE_PRIORITY,
 396         .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER |
 397                                         CRYPTO_ALG_NEED_FALLBACK,
 398         .cra_blocksize          =       AES_BLOCK_SIZE,
 399         .cra_ctxsize            =       sizeof(struct s390_aes_ctx),
 400         .cra_type               =       &crypto_blkcipher_type,
 401         .cra_module             =       THIS_MODULE,
 402         .cra_init               =       fallback_init_blk,
 403         .cra_exit               =       fallback_exit_blk,
 404         .cra_u                  =       {
 405                 .blkcipher = {
 406                         .min_keysize            =       AES_MIN_KEY_SIZE,
 407                         .max_keysize            =       AES_MAX_KEY_SIZE,
 408                         .setkey                 =       ecb_aes_set_key,
 409                         .encrypt                =       ecb_aes_encrypt,
 410                         .decrypt                =       ecb_aes_decrypt,
 411                 }
 412         }
 413 };
 414
 415 static int cbc_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 416                            unsigned int key_len)
 417 {
 418         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 419         int ret;
 420
 421         ret = need_fallback(key_len);
 422         if (ret > 0) {
 423                 sctx->key_len = key_len;
 424                 return setkey_fallback_blk(tfm, in_key, key_len);
 425         }
 426
 427         switch (key_len) {
 428         case 16:
 429                 sctx->enc = KMC_AES_128_ENCRYPT;
 430                 sctx->dec = KMC_AES_128_DECRYPT;
 431                 break;
 432         case 24:
 433                 sctx->enc = KMC_AES_192_ENCRYPT;
 434                 sctx->dec = KMC_AES_192_DECRYPT;
 435                 break;
 436         case 32:
 437                 sctx->enc = KMC_AES_256_ENCRYPT;
 438                 sctx->dec = KMC_AES_256_DECRYPT;
 439                 break;
 440         }
 441
 442         return aes_set_key(tfm, in_key, key_len);
 443 }
 444
 445 static int cbc_aes_crypt(struct blkcipher_desc *desc, long func,
 446                          struct blkcipher_walk *walk)
 447 {
 448         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 449         int ret = blkcipher_walk_virt(desc, walk);
 450         unsigned int nbytes = walk->nbytes;
 451         struct {
 452                 u8 iv[AES_BLOCK_SIZE];
 453                 u8 key[AES_MAX_KEY_SIZE];
 454         } param;
 455
 456         if (!nbytes)
 457                 goto out;
 458
 459         memcpy(param.iv, walk->iv, AES_BLOCK_SIZE);
 460         memcpy(param.key, sctx->key, sctx->key_len);
 461         do {
 462                 /* only use complete blocks */
 463                 unsigned int n = nbytes & ~(AES_BLOCK_SIZE - 1);
 464                 u8 *out = walk->dst.virt.addr;
 465                 u8 *in = walk->src.virt.addr;
 466
 467                 ret = crypt_s390_kmc(func, &param, out, in, n);
 468                 if (ret < 0 || ret != n)
 469                         return -EIO;
 470
 471                 nbytes &= AES_BLOCK_SIZE - 1;
 472                 ret = blkcipher_walk_done(desc, walk, nbytes);
 473         } while ((nbytes = walk->nbytes));
 474         memcpy(walk->iv, param.iv, AES_BLOCK_SIZE);
 475
 476 out:
 477         return ret;
 478 }
 479
 480 static int cbc_aes_encrypt(struct blkcipher_desc *desc,
 481                            struct scatterlist *dst, struct scatterlist *src,
 482                            unsigned int nbytes)
 483 {
 484         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 485         struct blkcipher_walk walk;
 486
 487         if (unlikely(need_fallback(sctx->key_len)))
 488                 return fallback_blk_enc(desc, dst, src, nbytes);
 489
 490         blkcipher_walk_init(&walk, dst, src, nbytes);
 491         return cbc_aes_crypt(desc, sctx->enc, &walk);
 492 }
 493
 494 static int cbc_aes_decrypt(struct blkcipher_desc *desc,
 495                            struct scatterlist *dst, struct scatterlist *src,
 496                            unsigned int nbytes)
 497 {
 498         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 499         struct blkcipher_walk walk;
 500
 501         if (unlikely(need_fallback(sctx->key_len)))
 502                 return fallback_blk_dec(desc, dst, src, nbytes);
 503
 504         blkcipher_walk_init(&walk, dst, src, nbytes);
 505         return cbc_aes_crypt(desc, sctx->dec, &walk);
 506 }
 507
 508 static struct crypto_alg cbc_aes_alg = {
 509         .cra_name               =       "cbc(aes)",
 510         .cra_driver_name        =       "cbc-aes-s390",
 511         .cra_priority           =       CRYPT_S390_COMPOSITE_PRIORITY,
 512         .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER |
 513                                         CRYPTO_ALG_NEED_FALLBACK,
 514         .cra_blocksize          =       AES_BLOCK_SIZE,
 515         .cra_ctxsize            =       sizeof(struct s390_aes_ctx),
 516         .cra_type               =       &crypto_blkcipher_type,
 517         .cra_module             =       THIS_MODULE,
 518         .cra_init               =       fallback_init_blk,
 519         .cra_exit               =       fallback_exit_blk,
 520         .cra_u                  =       {
 521                 .blkcipher = {
 522                         .min_keysize            =       AES_MIN_KEY_SIZE,
 523                         .max_keysize            =       AES_MAX_KEY_SIZE,
 524                         .ivsize                 =       AES_BLOCK_SIZE,
 525                         .setkey                 =       cbc_aes_set_key,
 526                         .encrypt                =       cbc_aes_encrypt,
 527                         .decrypt                =       cbc_aes_decrypt,
 528                 }
 529         }
 530 };
 531
 532 static int xts_fallback_setkey(struct crypto_tfm *tfm, const u8 *key,
 533                                    unsigned int len)
 534 {
 535         struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
 536         unsigned int ret;
 537
 538         xts_ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 539         xts_ctx->fallback->base.crt_flags |= (tfm->crt_flags &
 540                         CRYPTO_TFM_REQ_MASK);
 541
 542         ret = crypto_blkcipher_setkey(xts_ctx->fallback, key, len);
 543         if (ret) {
 544                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 545                 tfm->crt_flags |= (xts_ctx->fallback->base.crt_flags &
 546                                 CRYPTO_TFM_RES_MASK);
 547         }
 548         return ret;
 549 }
 550
 551 static int xts_fallback_decrypt(struct blkcipher_desc *desc,
 552                 struct scatterlist *dst, struct scatterlist *src,
 553                 unsigned int nbytes)
 554 {
 555         struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
 556         struct crypto_blkcipher *tfm;
 557         unsigned int ret;
 558
 559         tfm = desc->tfm;
 560         desc->tfm = xts_ctx->fallback;
 561
 562         ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
 563
 564         desc->tfm = tfm;
 565         return ret;
 566 }
 567
 568 static int xts_fallback_encrypt(struct blkcipher_desc *desc,
 569                 struct scatterlist *dst, struct scatterlist *src,
 570                 unsigned int nbytes)
 571 {
 572         struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
 573         struct crypto_blkcipher *tfm;
 574         unsigned int ret;
 575
 576         tfm = desc->tfm;
 577         desc->tfm = xts_ctx->fallback;
 578
 579         ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
 580
 581         desc->tfm = tfm;
 582         return ret;
 583 }
 584
 585 static int xts_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 586                            unsigned int key_len)
 587 {
 588         struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
 589         u32 *flags = &tfm->crt_flags;
 590
 591         switch (key_len) {
 592         case 32:
 593                 xts_ctx->enc = KM_XTS_128_ENCRYPT;
 594                 xts_ctx->dec = KM_XTS_128_DECRYPT;
 595                 memcpy(xts_ctx->key + 16, in_key, 16);
 596                 memcpy(xts_ctx->pcc_key + 16, in_key + 16, 16);
 597                 break;
 598         case 48:
 599                 xts_ctx->enc = 0;
 600                 xts_ctx->dec = 0;
 601                 xts_fallback_setkey(tfm, in_key, key_len);
 602                 break;
 603         case 64:
 604                 xts_ctx->enc = KM_XTS_256_ENCRYPT;
 605                 xts_ctx->dec = KM_XTS_256_DECRYPT;
 606                 memcpy(xts_ctx->key, in_key, 32);
 607                 memcpy(xts_ctx->pcc_key, in_key + 32, 32);
 608                 break;
 609         default:
 610                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 611                 return -EINVAL;
 612         }
 613         xts_ctx->key_len = key_len;
 614         return 0;
 615 }
 616
 617 static int xts_aes_crypt(struct blkcipher_desc *desc, long func,
 618                          struct s390_xts_ctx *xts_ctx,
 619                          struct blkcipher_walk *walk)
 620 {
 621         unsigned int offset = (xts_ctx->key_len >> 1) & 0x10;
 622         int ret = blkcipher_walk_virt(desc, walk);
 623         unsigned int nbytes = walk->nbytes;
 624         unsigned int n;
 625         u8 *in, *out;
 626         struct pcc_param pcc_param;
 627         struct {
 628                 u8 key[32];
 629                 u8 init[16];
 630         } xts_param;
 631
 632         if (!nbytes)
 633                 goto out;
 634
 635         memset(pcc_param.block, 0, sizeof(pcc_param.block));
 636         memset(pcc_param.bit, 0, sizeof(pcc_param.bit));
 637         memset(pcc_param.xts, 0, sizeof(pcc_param.xts));
 638         memcpy(pcc_param.tweak, walk->iv, sizeof(pcc_param.tweak));
 639         memcpy(pcc_param.key, xts_ctx->pcc_key, 32);
 640         ret = crypt_s390_pcc(func, &pcc_param.key[offset]);
 641         if (ret < 0)
 642                 return -EIO;
 643
 644         memcpy(xts_param.key, xts_ctx->key, 32);
 645         memcpy(xts_param.init, pcc_param.xts, 16);
 646         do {
 647                 /* only use complete blocks */
 648                 n = nbytes & ~(AES_BLOCK_SIZE - 1);
 649                 out = walk->dst.virt.addr;
 650                 in = walk->src.virt.addr;
 651
 652                 ret = crypt_s390_km(func, &xts_param.key[offset], out, in, n);
 653                 if (ret < 0 || ret != n)
 654                         return -EIO;
 655
 656                 nbytes &= AES_BLOCK_SIZE - 1;
 657                 ret = blkcipher_walk_done(desc, walk, nbytes);
 658         } while ((nbytes = walk->nbytes));
 659 out:
 660         return ret;
 661 }
 662
 663 static int xts_aes_encrypt(struct blkcipher_desc *desc,
 664                            struct scatterlist *dst, struct scatterlist *src,
 665                            unsigned int nbytes)
 666 {
 667         struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
 668         struct blkcipher_walk walk;
 669
 670         if (unlikely(xts_ctx->key_len == 48))
 671                 return xts_fallback_encrypt(desc, dst, src, nbytes);
 672
 673         blkcipher_walk_init(&walk, dst, src, nbytes);
 674         return xts_aes_crypt(desc, xts_ctx->enc, xts_ctx, &walk);
 675 }
 676
 677 static int xts_aes_decrypt(struct blkcipher_desc *desc,
 678                            struct scatterlist *dst, struct scatterlist *src,
 679                            unsigned int nbytes)
 680 {
 681         struct s390_xts_ctx *xts_ctx = crypto_blkcipher_ctx(desc->tfm);
 682         struct blkcipher_walk walk;
 683
 684         if (unlikely(xts_ctx->key_len == 48))
 685                 return xts_fallback_decrypt(desc, dst, src, nbytes);
 686
 687         blkcipher_walk_init(&walk, dst, src, nbytes);
 688         return xts_aes_crypt(desc, xts_ctx->dec, xts_ctx, &walk);
 689 }
 690
 691 static int xts_fallback_init(struct crypto_tfm *tfm)
 692 {
 693         const char *name = tfm->__crt_alg->cra_name;
 694         struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
 695
 696         xts_ctx->fallback = crypto_alloc_blkcipher(name, 0,
 697                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 698
 699         if (IS_ERR(xts_ctx->fallback)) {
 700                 pr_err("Allocating XTS fallback algorithm %s failed\n",
 701                        name);
 702                 return PTR_ERR(xts_ctx->fallback);
 703         }
 704         return 0;
 705 }
 706
 707 static void xts_fallback_exit(struct crypto_tfm *tfm)
 708 {
 709         struct s390_xts_ctx *xts_ctx = crypto_tfm_ctx(tfm);
 710
 711         crypto_free_blkcipher(xts_ctx->fallback);
 712         xts_ctx->fallback = NULL;
 713 }
 714
 715 static struct crypto_alg xts_aes_alg = {
 716         .cra_name               =       "xts(aes)",
 717         .cra_driver_name        =       "xts-aes-s390",
 718         .cra_priority           =       CRYPT_S390_COMPOSITE_PRIORITY,
 719         .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER |
 720                                         CRYPTO_ALG_NEED_FALLBACK,
 721         .cra_blocksize          =       AES_BLOCK_SIZE,
 722         .cra_ctxsize            =       sizeof(struct s390_xts_ctx),
 723         .cra_type               =       &crypto_blkcipher_type,
 724         .cra_module             =       THIS_MODULE,
 725         .cra_init               =       xts_fallback_init,
 726         .cra_exit               =       xts_fallback_exit,
 727         .cra_u                  =       {
 728                 .blkcipher = {
 729                         .min_keysize            =       2 * AES_MIN_KEY_SIZE,
 730                         .max_keysize            =       2 * AES_MAX_KEY_SIZE,
 731                         .ivsize                 =       AES_BLOCK_SIZE,
 732                         .setkey                 =       xts_aes_set_key,
 733                         .encrypt                =       xts_aes_encrypt,
 734                         .decrypt                =       xts_aes_decrypt,
 735                 }
 736         }
 737 };
 738
 739 static int xts_aes_alg_reg;
 740
 741 static int ctr_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 742                            unsigned int key_len)
 743 {
 744         struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
 745
 746         switch (key_len) {
 747         case 16:
 748                 sctx->enc = KMCTR_AES_128_ENCRYPT;
 749                 sctx->dec = KMCTR_AES_128_DECRYPT;
 750                 break;
 751         case 24:
 752                 sctx->enc = KMCTR_AES_192_ENCRYPT;
 753                 sctx->dec = KMCTR_AES_192_DECRYPT;
 754                 break;
 755         case 32:
 756                 sctx->enc = KMCTR_AES_256_ENCRYPT;
 757                 sctx->dec = KMCTR_AES_256_DECRYPT;
 758                 break;
 759         }
 760
 761         return aes_set_key(tfm, in_key, key_len);
 762 }
 763
 764 static unsigned int __ctrblk_init(u8 *ctrptr, unsigned int nbytes)
 765 {
 766         unsigned int i, n;
 767
 768         /* only use complete blocks, max. PAGE_SIZE */
 769         n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1);
 770         for (i = AES_BLOCK_SIZE; i < n; i += AES_BLOCK_SIZE) {
 771                 memcpy(ctrptr + i, ctrptr + i - AES_BLOCK_SIZE,
 772                        AES_BLOCK_SIZE);
 773                 crypto_inc(ctrptr + i, AES_BLOCK_SIZE);
 774         }
 775         return n;
 776 }
 777
 778 static int ctr_aes_crypt(struct blkcipher_desc *desc, long func,
 779                          struct s390_aes_ctx *sctx, struct blkcipher_walk *walk)
 780 {
 781         int ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE);
 782         unsigned int n, nbytes;
 783         u8 buf[AES_BLOCK_SIZE], ctrbuf[AES_BLOCK_SIZE];
 784         u8 *out, *in, *ctrptr = ctrbuf;
 785
 786         if (!walk->nbytes)
 787                 return ret;
 788
 789         if (spin_trylock(&ctrblk_lock))
 790                 ctrptr = ctrblk;
 791
 792         memcpy(ctrptr, walk->iv, AES_BLOCK_SIZE);
 793         while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) {
 794                 out = walk->dst.virt.addr;
 795                 in = walk->src.virt.addr;
 796                 while (nbytes >= AES_BLOCK_SIZE) {
 797                         if (ctrptr == ctrblk)
 798                                 n = __ctrblk_init(ctrptr, nbytes);
 799                         else
 800                                 n = AES_BLOCK_SIZE;
 801                         ret = crypt_s390_kmctr(func, sctx->key, out, in,
 802                                                n, ctrptr);
 803                         if (ret < 0 || ret != n) {
 804                                 if (ctrptr == ctrblk)
 805                                         spin_unlock(&ctrblk_lock);
 806                                 return -EIO;
 807                         }
 808                         if (n > AES_BLOCK_SIZE)
 809                                 memcpy(ctrptr, ctrptr + n - AES_BLOCK_SIZE,
 810                                        AES_BLOCK_SIZE);
 811                         crypto_inc(ctrptr, AES_BLOCK_SIZE);
 812                         out += n;
 813                         in += n;
 814                         nbytes -= n;
 815                 }
 816                 ret = blkcipher_walk_done(desc, walk, nbytes);
 817         }
 818         if (ctrptr == ctrblk) {
 819                 if (nbytes)
 820                         memcpy(ctrbuf, ctrptr, AES_BLOCK_SIZE);
 821                 else
 822                         memcpy(walk->iv, ctrptr, AES_BLOCK_SIZE);
 823                 spin_unlock(&ctrblk_lock);
 824         } else {
 825                 if (!nbytes)
 826                         memcpy(walk->iv, ctrptr, AES_BLOCK_SIZE);
 827         }
 828         /*
 829          * final block may be < AES_BLOCK_SIZE, copy only nbytes
 830          */
 831         if (nbytes) {
 832                 out = walk->dst.virt.addr;
 833                 in = walk->src.virt.addr;
 834                 ret = crypt_s390_kmctr(func, sctx->key, buf, in,
 835                                        AES_BLOCK_SIZE, ctrbuf);
 836                 if (ret < 0 || ret != AES_BLOCK_SIZE)
 837                         return -EIO;
 838                 memcpy(out, buf, nbytes);
 839                 crypto_inc(ctrbuf, AES_BLOCK_SIZE);
 840                 ret = blkcipher_walk_done(desc, walk, 0);
 841                 memcpy(walk->iv, ctrbuf, AES_BLOCK_SIZE);
 842         }
 843
 844         return ret;
 845 }
 846
 847 static int ctr_aes_encrypt(struct blkcipher_desc *desc,
 848                            struct scatterlist *dst, struct scatterlist *src,
 849                            unsigned int nbytes)
 850 {
 851         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 852         struct blkcipher_walk walk;
 853
 854         blkcipher_walk_init(&walk, dst, src, nbytes);
 855         return ctr_aes_crypt(desc, sctx->enc, sctx, &walk);
 856 }
 857
 858 static int ctr_aes_decrypt(struct blkcipher_desc *desc,
 859                            struct scatterlist *dst, struct scatterlist *src,
 860                            unsigned int nbytes)
 861 {
 862         struct s390_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm);
 863         struct blkcipher_walk walk;
 864
 865         blkcipher_walk_init(&walk, dst, src, nbytes);
 866         return ctr_aes_crypt(desc, sctx->dec, sctx, &walk);
 867 }
 868
 869 static struct crypto_alg ctr_aes_alg = {
 870         .cra_name               =       "ctr(aes)",
 871         .cra_driver_name        =       "ctr-aes-s390",
 872         .cra_priority           =       CRYPT_S390_COMPOSITE_PRIORITY,
 873         .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER,
 874         .cra_blocksize          =       1,
 875         .cra_ctxsize            =       sizeof(struct s390_aes_ctx),
 876         .cra_type               =       &crypto_blkcipher_type,
 877         .cra_module             =       THIS_MODULE,
 878         .cra_u                  =       {
 879                 .blkcipher = {
 880                         .min_keysize            =       AES_MIN_KEY_SIZE,
 881                         .max_keysize            =       AES_MAX_KEY_SIZE,
 882                         .ivsize                 =       AES_BLOCK_SIZE,
 883                         .setkey                 =       ctr_aes_set_key,
 884                         .encrypt                =       ctr_aes_encrypt,
 885                         .decrypt                =       ctr_aes_decrypt,
 886                 }
 887         }
 888 };
 889
 890 static int ctr_aes_alg_reg;
 891
 892 static int __init aes_s390_init(void)
 893 {
 894         int ret;
 895
 896         if (crypt_s390_func_available(KM_AES_128_ENCRYPT, CRYPT_S390_MSA))
 897                 keylen_flag |= AES_KEYLEN_128;
 898         if (crypt_s390_func_available(KM_AES_192_ENCRYPT, CRYPT_S390_MSA))
 899                 keylen_flag |= AES_KEYLEN_192;
 900         if (crypt_s390_func_available(KM_AES_256_ENCRYPT, CRYPT_S390_MSA))
 901                 keylen_flag |= AES_KEYLEN_256;
 902
 903         if (!keylen_flag)
 904                 return -EOPNOTSUPP;
 905
 906         /* z9 109 and z9 BC/EC only support 128 bit key length */
 907         if (keylen_flag == AES_KEYLEN_128)
 908                 pr_info("AES hardware acceleration is only available for"
 909                         " 128-bit keys\n");
 910
 911         ret = crypto_register_alg(&aes_alg);
 912         if (ret)
 913                 goto aes_err;
 914
 915         ret = crypto_register_alg(&ecb_aes_alg);
 916         if (ret)
 917                 goto ecb_aes_err;
 918
 919         ret = crypto_register_alg(&cbc_aes_alg);
 920         if (ret)
 921                 goto cbc_aes_err;
 922
 923         if (crypt_s390_func_available(KM_XTS_128_ENCRYPT,
 924                         CRYPT_S390_MSA | CRYPT_S390_MSA4) &&
 925             crypt_s390_func_available(KM_XTS_256_ENCRYPT,
 926                         CRYPT_S390_MSA | CRYPT_S390_MSA4)) {
 927                 ret = crypto_register_alg(&xts_aes_alg);
 928                 if (ret)
 929                         goto xts_aes_err;
 930                 xts_aes_alg_reg = 1;
 931         }
 932
 933         if (crypt_s390_func_available(KMCTR_AES_128_ENCRYPT,
 934                                 CRYPT_S390_MSA | CRYPT_S390_MSA4) &&
 935             crypt_s390_func_available(KMCTR_AES_192_ENCRYPT,
 936                                 CRYPT_S390_MSA | CRYPT_S390_MSA4) &&
 937             crypt_s390_func_available(KMCTR_AES_256_ENCRYPT,
 938                                 CRYPT_S390_MSA | CRYPT_S390_MSA4)) {
 939                 ctrblk = (u8 *) __get_free_page(GFP_KERNEL);
 940                 if (!ctrblk) {
 941                         ret = -ENOMEM;
 942                         goto ctr_aes_err;
 943                 }
 944                 ret = crypto_register_alg(&ctr_aes_alg);
 945                 if (ret) {
 946                         free_page((unsigned long) ctrblk);
 947                         goto ctr_aes_err;
 948                 }
 949                 ctr_aes_alg_reg = 1;
 950         }
 951
 952 out:
 953         return ret;
 954
 955 ctr_aes_err:
 956         crypto_unregister_alg(&xts_aes_alg);
 957 xts_aes_err:
 958         crypto_unregister_alg(&cbc_aes_alg);
 959 cbc_aes_err:
 960         crypto_unregister_alg(&ecb_aes_alg);
 961 ecb_aes_err:
 962         crypto_unregister_alg(&aes_alg);
 963 aes_err:
 964         goto out;
 965 }
 966
 967 static void __exit aes_s390_fini(void)
 968 {
 969         if (ctr_aes_alg_reg) {
 970                 crypto_unregister_alg(&ctr_aes_alg);
 971                 free_page((unsigned long) ctrblk);
 972         }
 973         if (xts_aes_alg_reg)
 974                 crypto_unregister_alg(&xts_aes_alg);
 975         crypto_unregister_alg(&cbc_aes_alg);
 976         crypto_unregister_alg(&ecb_aes_alg);
 977         crypto_unregister_alg(&aes_alg);
 978 }
 979
 980 module_cpu_feature_match(MSA, aes_s390_init);
 981 module_exit(aes_s390_fini);
 982
 983 MODULE_ALIAS_CRYPTO("aes-all");
 984
 985 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
 986 MODULE_LICENSE("GPL");