arch/x86/crypto/serpent_avx2_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
   4  *
   5  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/types.h>
  10 #include <linux/crypto.h>
  11 #include <linux/err.h>
  12 #include <crypto/algapi.h>
  13 #include <crypto/internal/simd.h>
  14 #include <crypto/serpent.h>
  15 #include <crypto/xts.h>
  16 #include <asm/crypto/glue_helper.h>
  17 #include <asm/crypto/serpent-avx.h>
  18
  19 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
  20
  21 /* 16-way AVX2 parallel cipher functions */
  22 asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
  23                                       const u8 *src);
  24 asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
  25                                       const u8 *src);
  26 asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);
  27
  28 asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
  29                                   le128 *iv);
  30 asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
  31                                       const u8 *src, le128 *iv);
  32 asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
  33                                       const u8 *src, le128 *iv);
  34
  35 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
  36                                    const u8 *key, unsigned int keylen)
  37 {
  38         return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
  39 }
  40
  41 static const struct common_glue_ctx serpent_enc = {
  42         .num_funcs = 3,
  43         .fpu_blocks_limit = 8,
  44
  45         .funcs = { {
  46                 .num_blocks = 16,
  47                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
  48         }, {
  49                 .num_blocks = 8,
  50                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
  51         }, {
  52                 .num_blocks = 1,
  53                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
  54         } }
  55 };
  56
  57 static const struct common_glue_ctx serpent_ctr = {
  58         .num_funcs = 3,
  59         .fpu_blocks_limit = 8,
  60
  61         .funcs = { {
  62                 .num_blocks = 16,
  63                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
  64         },  {
  65                 .num_blocks = 8,
  66                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
  67         }, {
  68                 .num_blocks = 1,
  69                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
  70         } }
  71 };
  72
  73 static const struct common_glue_ctx serpent_enc_xts = {
  74         .num_funcs = 3,
  75         .fpu_blocks_limit = 8,
  76
  77         .funcs = { {
  78                 .num_blocks = 16,
  79                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
  80         }, {
  81                 .num_blocks = 8,
  82                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
  83         }, {
  84                 .num_blocks = 1,
  85                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
  86         } }
  87 };
  88
  89 static const struct common_glue_ctx serpent_dec = {
  90         .num_funcs = 3,
  91         .fpu_blocks_limit = 8,
  92
  93         .funcs = { {
  94                 .num_blocks = 16,
  95                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
  96         }, {
  97                 .num_blocks = 8,
  98                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
  99         }, {
 100                 .num_blocks = 1,
 101                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
 102         } }
 103 };
 104
 105 static const struct common_glue_ctx serpent_dec_cbc = {
 106         .num_funcs = 3,
 107         .fpu_blocks_limit = 8,
 108
 109         .funcs = { {
 110                 .num_blocks = 16,
 111                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
 112         }, {
 113                 .num_blocks = 8,
 114                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
 115         }, {
 116                 .num_blocks = 1,
 117                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
 118         } }
 119 };
 120
 121 static const struct common_glue_ctx serpent_dec_xts = {
 122         .num_funcs = 3,
 123         .fpu_blocks_limit = 8,
 124
 125         .funcs = { {
 126                 .num_blocks = 16,
 127                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
 128         }, {
 129                 .num_blocks = 8,
 130                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
 131         }, {
 132                 .num_blocks = 1,
 133                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
 134         } }
 135 };
 136
 137 static int ecb_encrypt(struct skcipher_request *req)
 138 {
 139         return glue_ecb_req_128bit(&serpent_enc, req);
 140 }
 141
 142 static int ecb_decrypt(struct skcipher_request *req)
 143 {
 144         return glue_ecb_req_128bit(&serpent_dec, req);
 145 }
 146
 147 static int cbc_encrypt(struct skcipher_request *req)
 148 {
 149         return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
 150                                            req);
 151 }
 152
 153 static int cbc_decrypt(struct skcipher_request *req)
 154 {
 155         return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
 156 }
 157
 158 static int ctr_crypt(struct skcipher_request *req)
 159 {
 160         return glue_ctr_req_128bit(&serpent_ctr, req);
 161 }
 162
 163 static int xts_encrypt(struct skcipher_request *req)
 164 {
 165         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 166         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 167
 168         return glue_xts_req_128bit(&serpent_enc_xts, req,
 169                                    XTS_TWEAK_CAST(__serpent_encrypt),
 170                                    &ctx->tweak_ctx, &ctx->crypt_ctx, false);
 171 }
 172
 173 static int xts_decrypt(struct skcipher_request *req)
 174 {
 175         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 176         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 177
 178         return glue_xts_req_128bit(&serpent_dec_xts, req,
 179                                    XTS_TWEAK_CAST(__serpent_encrypt),
 180                                    &ctx->tweak_ctx, &ctx->crypt_ctx, true);
 181 }
 182
 183 static struct skcipher_alg serpent_algs[] = {
 184         {
 185                 .base.cra_name          = "__ecb(serpent)",
 186                 .base.cra_driver_name   = "__ecb-serpent-avx2",
 187                 .base.cra_priority      = 600,
 188                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 189                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 190                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 191                 .base.cra_module        = THIS_MODULE,
 192                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 193                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 194                 .setkey                 = serpent_setkey_skcipher,
 195                 .encrypt                = ecb_encrypt,
 196                 .decrypt                = ecb_decrypt,
 197         }, {
 198                 .base.cra_name          = "__cbc(serpent)",
 199                 .base.cra_driver_name   = "__cbc-serpent-avx2",
 200                 .base.cra_priority      = 600,
 201                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 202                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 203                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 204                 .base.cra_module        = THIS_MODULE,
 205                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 206                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 207                 .ivsize                 = SERPENT_BLOCK_SIZE,
 208                 .setkey                 = serpent_setkey_skcipher,
 209                 .encrypt                = cbc_encrypt,
 210                 .decrypt                = cbc_decrypt,
 211         }, {
 212                 .base.cra_name          = "__ctr(serpent)",
 213                 .base.cra_driver_name   = "__ctr-serpent-avx2",
 214                 .base.cra_priority      = 600,
 215                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 216                 .base.cra_blocksize     = 1,
 217                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 218                 .base.cra_module        = THIS_MODULE,
 219                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 220                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 221                 .ivsize                 = SERPENT_BLOCK_SIZE,
 222                 .chunksize              = SERPENT_BLOCK_SIZE,
 223                 .setkey                 = serpent_setkey_skcipher,
 224                 .encrypt                = ctr_crypt,
 225                 .decrypt                = ctr_crypt,
 226         }, {
 227                 .base.cra_name          = "__xts(serpent)",
 228                 .base.cra_driver_name   = "__xts-serpent-avx2",
 229                 .base.cra_priority      = 600,
 230                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 231                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 232                 .base.cra_ctxsize       = sizeof(struct serpent_xts_ctx),
 233                 .base.cra_module        = THIS_MODULE,
 234                 .min_keysize            = 2 * SERPENT_MIN_KEY_SIZE,
 235                 .max_keysize            = 2 * SERPENT_MAX_KEY_SIZE,
 236                 .ivsize                 = SERPENT_BLOCK_SIZE,
 237                 .setkey                 = xts_serpent_setkey,
 238                 .encrypt                = xts_encrypt,
 239                 .decrypt                = xts_decrypt,
 240         },
 241 };
 242
 243 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
 244
 245 static int __init init(void)
 246 {
 247         const char *feature_name;
 248
 249         if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 250                 pr_info("AVX2 instructions are not detected.\n");
 251                 return -ENODEV;
 252         }
 253         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 254                                 &feature_name)) {
 255                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 256                 return -ENODEV;
 257         }
 258
 259         return simd_register_skciphers_compat(serpent_algs,
 260                                               ARRAY_SIZE(serpent_algs),
 261                                               serpent_simd_algs);
 262 }
 263
 264 static void __exit fini(void)
 265 {
 266         simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
 267                                   serpent_simd_algs);
 268 }
 269
 270 module_init(init);
 271 module_exit(fini);
 272
 273 MODULE_LICENSE("GPL");
 274 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
 275 MODULE_ALIAS_CRYPTO("serpent");
 276 MODULE_ALIAS_CRYPTO("serpent-asm");