1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/printk.h>
4 #include <linux/crypto.h>
5 #include <linux/rtnetlink.h>
7 #include <crypto/aead.h>
8 #include <crypto/authenc.h>
9 #include <crypto/des.h>
10 #include <crypto/sha.h>
11 #include <crypto/internal/aead.h>
12 #include <crypto/scatterwalk.h>
13 #include <crypto/gcm.h>
15 #include "nitrox_dev.h"
16 #include "nitrox_common.h"
17 #include "nitrox_req.h"
19 #define GCM_AES_SALT_SIZE 4
23 #ifdef __BIG_ENDIAN_BITFIELD
34 static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
38 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
39 struct flexi_crypto_context *fctx;
40 union fc_ctx_flags flags;
42 aes_keylen = flexi_aes_keylen(keylen);
44 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
48 /* fill crypto context */
50 flags.f = be64_to_cpu(fctx->flags.f);
51 flags.w0.aes_keylen = aes_keylen;
52 fctx->flags.f = cpu_to_be64(flags.f);
54 /* copy enc key to context */
55 memset(&fctx->crypto, 0, sizeof(fctx->crypto));
56 memcpy(fctx->crypto.u.key, key, keylen);
61 static int nitrox_aead_setauthsize(struct crypto_aead *aead,
62 unsigned int authsize)
64 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
65 struct flexi_crypto_context *fctx = nctx->u.fctx;
66 union fc_ctx_flags flags;
68 flags.f = be64_to_cpu(fctx->flags.f);
69 flags.w0.mac_len = authsize;
70 fctx->flags.f = cpu_to_be64(flags.f);
72 aead->authsize = authsize;
77 static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
78 struct scatterlist *src, char *iv, int ivsize,
81 int nents = sg_nents_for_len(src, buflen);
89 /* Allocate buffer to hold IV and input scatterlist array */
90 ret = alloc_src_req_buf(nkreq, nents, ivsize);
94 nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
95 nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
100 static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
101 struct scatterlist *dst, int ivsize, int buflen)
103 int nents = sg_nents_for_len(dst, buflen);
109 /* IV, ORH, COMPLETION entries */
111 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist
114 ret = alloc_dst_req_buf(nkreq, nents);
118 nitrox_creq_set_orh(nkreq);
119 nitrox_creq_set_comp(nkreq);
120 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
125 static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
130 static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
135 static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
137 struct se_crypto_request *creq = &rctx->nkreq.creq;
141 creq->flags = rctx->flags;
142 creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
145 creq->ctrl.value = 0;
146 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
147 creq->ctrl.s.arg = rctx->ctrl_arg;
149 creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
150 creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
151 creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
152 param3.iv_offset = 0;
153 param3.auth_offset = rctx->ivsize;
154 creq->gph.param3 = cpu_to_be16(param3.param);
156 creq->ctx_handle = rctx->ctx_handle;
157 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
159 ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
164 ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
167 free_src_sglist(&rctx->nkreq);
174 static void nitrox_aead_callback(void *arg, int err)
176 struct aead_request *areq = arg;
177 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
179 free_src_sglist(&rctx->nkreq);
180 free_dst_sglist(&rctx->nkreq);
182 pr_err_ratelimited("request failed status 0x%0x\n", err);
186 areq->base.complete(&areq->base, err);
189 static int nitrox_aes_gcm_enc(struct aead_request *areq)
191 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
192 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
193 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
194 struct se_crypto_request *creq = &rctx->nkreq.creq;
195 struct flexi_crypto_context *fctx = nctx->u.fctx;
198 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
200 rctx->cryptlen = areq->cryptlen;
201 rctx->assoclen = areq->assoclen;
202 rctx->srclen = areq->assoclen + areq->cryptlen;
203 rctx->dstlen = rctx->srclen + aead->authsize;
204 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
205 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
206 rctx->flags = areq->base.flags;
207 rctx->ctx_handle = nctx->u.ctx_handle;
208 rctx->src = areq->src;
209 rctx->dst = areq->dst;
210 rctx->ctrl_arg = ENCRYPT;
211 ret = nitrox_set_creq(rctx);
215 /* send the crypto request */
216 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
220 static int nitrox_aes_gcm_dec(struct aead_request *areq)
222 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
223 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
224 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
225 struct se_crypto_request *creq = &rctx->nkreq.creq;
226 struct flexi_crypto_context *fctx = nctx->u.fctx;
229 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
231 rctx->cryptlen = areq->cryptlen - aead->authsize;
232 rctx->assoclen = areq->assoclen;
233 rctx->srclen = areq->cryptlen + areq->assoclen;
234 rctx->dstlen = rctx->srclen - aead->authsize;
235 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
236 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
237 rctx->flags = areq->base.flags;
238 rctx->ctx_handle = nctx->u.ctx_handle;
239 rctx->src = areq->src;
240 rctx->dst = areq->dst;
241 rctx->ctrl_arg = DECRYPT;
242 ret = nitrox_set_creq(rctx);
246 /* send the crypto request */
247 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
251 static int nitrox_aead_init(struct crypto_aead *aead)
253 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
254 struct crypto_ctx_hdr *chdr;
256 /* get the first device */
257 nctx->ndev = nitrox_get_first_device();
261 /* allocate nitrox crypto context */
262 chdr = crypto_alloc_context(nctx->ndev);
264 nitrox_put_device(nctx->ndev);
268 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
269 sizeof(struct ctx_hdr));
270 nctx->u.fctx->flags.f = 0;
275 static int nitrox_gcm_common_init(struct crypto_aead *aead)
278 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
279 union fc_ctx_flags *flags;
281 ret = nitrox_aead_init(aead);
285 flags = &nctx->u.fctx->flags;
286 flags->w0.cipher_type = CIPHER_AES_GCM;
287 flags->w0.hash_type = AUTH_NULL;
288 flags->w0.iv_source = IV_FROM_DPTR;
289 /* ask microcode to calculate ipad/opad */
290 flags->w0.auth_input_type = 1;
291 flags->f = be64_to_cpu(flags->f);
296 static int nitrox_aes_gcm_init(struct crypto_aead *aead)
300 ret = nitrox_gcm_common_init(aead);
304 crypto_aead_set_reqsize(aead,
305 sizeof(struct aead_request) +
306 sizeof(struct nitrox_aead_rctx));
311 static void nitrox_aead_exit(struct crypto_aead *aead)
313 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
315 /* free the nitrox crypto context */
316 if (nctx->u.ctx_handle) {
317 struct flexi_crypto_context *fctx = nctx->u.fctx;
319 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
320 memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
321 crypto_free_context((void *)nctx->chdr);
323 nitrox_put_device(nctx->ndev);
325 nctx->u.ctx_handle = 0;
329 static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
332 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
333 struct flexi_crypto_context *fctx = nctx->u.fctx;
336 if (keylen < GCM_AES_SALT_SIZE)
339 keylen -= GCM_AES_SALT_SIZE;
340 ret = nitrox_aes_gcm_setkey(aead, key, keylen);
344 memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
348 static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
349 unsigned int authsize)
360 return nitrox_aead_setauthsize(aead, authsize);
363 static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
365 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
366 struct nitrox_aead_rctx *aead_rctx = &rctx->base;
367 unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
368 struct scatterlist *sg;
370 if (areq->assoclen != 16 && areq->assoclen != 20)
373 scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
374 sg_init_table(rctx->src, 3);
375 sg_set_buf(rctx->src, rctx->assoc, assoclen);
376 sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
377 if (sg != rctx->src + 1)
378 sg_chain(rctx->src, 2, sg);
380 if (areq->src != areq->dst) {
381 sg_init_table(rctx->dst, 3);
382 sg_set_buf(rctx->dst, rctx->assoc, assoclen);
383 sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
384 if (sg != rctx->dst + 1)
385 sg_chain(rctx->dst, 2, sg);
388 aead_rctx->src = rctx->src;
389 aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
394 static void nitrox_rfc4106_callback(void *arg, int err)
396 struct aead_request *areq = arg;
397 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
398 struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
400 free_src_sglist(nkreq);
401 free_dst_sglist(nkreq);
403 pr_err_ratelimited("request failed status 0x%0x\n", err);
407 areq->base.complete(&areq->base, err);
410 static int nitrox_rfc4106_enc(struct aead_request *areq)
412 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
413 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
414 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
415 struct nitrox_aead_rctx *aead_rctx = &rctx->base;
416 struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
419 aead_rctx->cryptlen = areq->cryptlen;
420 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
421 aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
422 aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
423 aead_rctx->iv = areq->iv;
424 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
425 aead_rctx->flags = areq->base.flags;
426 aead_rctx->ctx_handle = nctx->u.ctx_handle;
427 aead_rctx->ctrl_arg = ENCRYPT;
429 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
433 ret = nitrox_set_creq(aead_rctx);
437 /* send the crypto request */
438 return nitrox_process_se_request(nctx->ndev, creq,
439 nitrox_rfc4106_callback, areq);
442 static int nitrox_rfc4106_dec(struct aead_request *areq)
444 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
445 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
446 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
447 struct nitrox_aead_rctx *aead_rctx = &rctx->base;
448 struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
451 aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
452 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
454 areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
455 aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
456 aead_rctx->iv = areq->iv;
457 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
458 aead_rctx->flags = areq->base.flags;
459 aead_rctx->ctx_handle = nctx->u.ctx_handle;
460 aead_rctx->ctrl_arg = DECRYPT;
462 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
466 ret = nitrox_set_creq(aead_rctx);
470 /* send the crypto request */
471 return nitrox_process_se_request(nctx->ndev, creq,
472 nitrox_rfc4106_callback, areq);
475 static int nitrox_rfc4106_init(struct crypto_aead *aead)
479 ret = nitrox_gcm_common_init(aead);
483 crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
484 sizeof(struct nitrox_rfc4106_rctx));
489 static struct aead_alg nitrox_aeads[] = { {
491 .cra_name = "gcm(aes)",
492 .cra_driver_name = "n5_aes_gcm",
493 .cra_priority = PRIO,
494 .cra_flags = CRYPTO_ALG_ASYNC,
495 .cra_blocksize = AES_BLOCK_SIZE,
496 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
498 .cra_module = THIS_MODULE,
500 .setkey = nitrox_aes_gcm_setkey,
501 .setauthsize = nitrox_aead_setauthsize,
502 .encrypt = nitrox_aes_gcm_enc,
503 .decrypt = nitrox_aes_gcm_dec,
504 .init = nitrox_aes_gcm_init,
505 .exit = nitrox_aead_exit,
506 .ivsize = GCM_AES_IV_SIZE,
507 .maxauthsize = AES_BLOCK_SIZE,
510 .cra_name = "rfc4106(gcm(aes))",
511 .cra_driver_name = "n5_rfc4106",
512 .cra_priority = PRIO,
513 .cra_flags = CRYPTO_ALG_ASYNC,
514 .cra_blocksize = AES_BLOCK_SIZE,
515 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
517 .cra_module = THIS_MODULE,
519 .setkey = nitrox_rfc4106_setkey,
520 .setauthsize = nitrox_rfc4106_setauthsize,
521 .encrypt = nitrox_rfc4106_enc,
522 .decrypt = nitrox_rfc4106_dec,
523 .init = nitrox_rfc4106_init,
524 .exit = nitrox_aead_exit,
525 .ivsize = GCM_RFC4106_IV_SIZE,
526 .maxauthsize = AES_BLOCK_SIZE,
529 int nitrox_register_aeads(void)
531 return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
534 void nitrox_unregister_aeads(void)
536 crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));