arch/arm/crypto/chacha-neon-glue.c

   1 /*
   2  * ARM NEON accelerated ChaCha and XChaCha stream ciphers,
   3  * including ChaCha20 (RFC7539)
   4  *
   5  * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * Based on:
  12  * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
  13  *
  14  * Copyright (C) 2015 Martin Willi
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <crypto/algapi.h>
  23 #include <crypto/chacha.h>
  24 #include <crypto/internal/simd.h>
  25 #include <crypto/internal/skcipher.h>
  26 #include <linux/kernel.h>
  27 #include <linux/module.h>
  28
  29 #include <asm/hwcap.h>
  30 #include <asm/neon.h>
  31 #include <asm/simd.h>
  32
  33 asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
  34                                       int nrounds);
  35 asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
  36                                        int nrounds);
  37 asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
  38
  39 static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
  40                           unsigned int bytes, int nrounds)
  41 {
  42         u8 buf[CHACHA_BLOCK_SIZE];
  43
  44         while (bytes >= CHACHA_BLOCK_SIZE * 4) {
  45                 chacha_4block_xor_neon(state, dst, src, nrounds);
  46                 bytes -= CHACHA_BLOCK_SIZE * 4;
  47                 src += CHACHA_BLOCK_SIZE * 4;
  48                 dst += CHACHA_BLOCK_SIZE * 4;
  49                 state[12] += 4;
  50         }
  51         while (bytes >= CHACHA_BLOCK_SIZE) {
  52                 chacha_block_xor_neon(state, dst, src, nrounds);
  53                 bytes -= CHACHA_BLOCK_SIZE;
  54                 src += CHACHA_BLOCK_SIZE;
  55                 dst += CHACHA_BLOCK_SIZE;
  56                 state[12]++;
  57         }
  58         if (bytes) {
  59                 memcpy(buf, src, bytes);
  60                 chacha_block_xor_neon(state, buf, buf, nrounds);
  61                 memcpy(dst, buf, bytes);
  62         }
  63 }
  64
  65 static int chacha_neon_stream_xor(struct skcipher_request *req,
  66                                   const struct chacha_ctx *ctx, const u8 *iv)
  67 {
  68         struct skcipher_walk walk;
  69         u32 state[16];
  70         int err;
  71
  72         err = skcipher_walk_virt(&walk, req, false);
  73
  74         crypto_chacha_init(state, ctx, iv);
  75
  76         while (walk.nbytes > 0) {
  77                 unsigned int nbytes = walk.nbytes;
  78
  79                 if (nbytes < walk.total)
  80                         nbytes = round_down(nbytes, walk.stride);
  81
  82                 kernel_neon_begin();
  83                 chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
  84                               nbytes, ctx->nrounds);
  85                 kernel_neon_end();
  86                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
  87         }
  88
  89         return err;
  90 }
  91
  92 static int chacha_neon(struct skcipher_request *req)
  93 {
  94         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  95         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
  96
  97         if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
  98                 return crypto_chacha_crypt(req);
  99
 100         return chacha_neon_stream_xor(req, ctx, req->iv);
 101 }
 102
 103 static int xchacha_neon(struct skcipher_request *req)
 104 {
 105         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 106         struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 107         struct chacha_ctx subctx;
 108         u32 state[16];
 109         u8 real_iv[16];
 110
 111         if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
 112                 return crypto_xchacha_crypt(req);
 113
 114         crypto_chacha_init(state, ctx, req->iv);
 115
 116         kernel_neon_begin();
 117         hchacha_block_neon(state, subctx.key, ctx->nrounds);
 118         kernel_neon_end();
 119         subctx.nrounds = ctx->nrounds;
 120
 121         memcpy(&real_iv[0], req->iv + 24, 8);
 122         memcpy(&real_iv[8], req->iv + 16, 8);
 123         return chacha_neon_stream_xor(req, &subctx, real_iv);
 124 }
 125
 126 static struct skcipher_alg algs[] = {
 127         {
 128                 .base.cra_name          = "chacha20",
 129                 .base.cra_driver_name   = "chacha20-neon",
 130                 .base.cra_priority      = 300,
 131                 .base.cra_blocksize     = 1,
 132                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 133                 .base.cra_module        = THIS_MODULE,
 134
 135                 .min_keysize            = CHACHA_KEY_SIZE,
 136                 .max_keysize            = CHACHA_KEY_SIZE,
 137                 .ivsize                 = CHACHA_IV_SIZE,
 138                 .chunksize              = CHACHA_BLOCK_SIZE,
 139                 .walksize               = 4 * CHACHA_BLOCK_SIZE,
 140                 .setkey                 = crypto_chacha20_setkey,
 141                 .encrypt                = chacha_neon,
 142                 .decrypt                = chacha_neon,
 143         }, {
 144                 .base.cra_name          = "xchacha20",
 145                 .base.cra_driver_name   = "xchacha20-neon",
 146                 .base.cra_priority      = 300,
 147                 .base.cra_blocksize     = 1,
 148                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 149                 .base.cra_module        = THIS_MODULE,
 150
 151                 .min_keysize            = CHACHA_KEY_SIZE,
 152                 .max_keysize            = CHACHA_KEY_SIZE,
 153                 .ivsize                 = XCHACHA_IV_SIZE,
 154                 .chunksize              = CHACHA_BLOCK_SIZE,
 155                 .walksize               = 4 * CHACHA_BLOCK_SIZE,
 156                 .setkey                 = crypto_chacha20_setkey,
 157                 .encrypt                = xchacha_neon,
 158                 .decrypt                = xchacha_neon,
 159         }, {
 160                 .base.cra_name          = "xchacha12",
 161                 .base.cra_driver_name   = "xchacha12-neon",
 162                 .base.cra_priority      = 300,
 163                 .base.cra_blocksize     = 1,
 164                 .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 165                 .base.cra_module        = THIS_MODULE,
 166
 167                 .min_keysize            = CHACHA_KEY_SIZE,
 168                 .max_keysize            = CHACHA_KEY_SIZE,
 169                 .ivsize                 = XCHACHA_IV_SIZE,
 170                 .chunksize              = CHACHA_BLOCK_SIZE,
 171                 .walksize               = 4 * CHACHA_BLOCK_SIZE,
 172                 .setkey                 = crypto_chacha12_setkey,
 173                 .encrypt                = xchacha_neon,
 174                 .decrypt                = xchacha_neon,
 175         }
 176 };
 177
 178 static int __init chacha_simd_mod_init(void)
 179 {
 180         if (!(elf_hwcap & HWCAP_NEON))
 181                 return -ENODEV;
 182
 183         return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
 184 }
 185
 186 static void __exit chacha_simd_mod_fini(void)
 187 {
 188         crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
 189 }
 190
 191 module_init(chacha_simd_mod_init);
 192 module_exit(chacha_simd_mod_fini);
 193
 194 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)");
 195 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 196 MODULE_LICENSE("GPL v2");
 197 MODULE_ALIAS_CRYPTO("chacha20");
 198 MODULE_ALIAS_CRYPTO("chacha20-neon");
 199 MODULE_ALIAS_CRYPTO("xchacha20");
 200 MODULE_ALIAS_CRYPTO("xchacha20-neon");
 201 MODULE_ALIAS_CRYPTO("xchacha12");
 202 MODULE_ALIAS_CRYPTO("xchacha12-neon");