2 * Crypto acceleration support for Rockchip RK3288
4 * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
6 * Author: Zain Wang <zain.wang@rock-chips.com>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
12 * Some ideas are from marvell-cesa.c and s5p-sss.c driver.
14 #include "rk3288_crypto.h"
16 #define RK_CRYPTO_DEC BIT(0)
18 static void rk_crypto_complete(struct crypto_async_request *base, int err)
21 base->complete(base, err);
24 static int rk_handle_req(struct rk_crypto_info *dev,
25 struct ablkcipher_request *req)
27 if (!IS_ALIGNED(req->nbytes, dev->align_size))
30 return dev->enqueue(dev, &req->base);
33 static int rk_aes_setkey(struct crypto_ablkcipher *cipher,
34 const u8 *key, unsigned int keylen)
36 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
37 struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
39 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
40 keylen != AES_KEYSIZE_256) {
41 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
45 memcpy_toio(ctx->dev->reg + RK_CRYPTO_AES_KEY_0, key, keylen);
49 static int rk_tdes_setkey(struct crypto_ablkcipher *cipher,
50 const u8 *key, unsigned int keylen)
52 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
53 struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
54 u32 tmp[DES_EXPKEY_WORDS];
56 if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) {
57 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
61 if (keylen == DES_KEY_SIZE) {
62 if (!des_ekey(tmp, key) &&
63 (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
64 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
70 memcpy_toio(ctx->dev->reg + RK_CRYPTO_TDES_KEY1_0, key, keylen);
74 static int rk_aes_ecb_encrypt(struct ablkcipher_request *req)
76 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
77 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
78 struct rk_crypto_info *dev = ctx->dev;
80 ctx->mode = RK_CRYPTO_AES_ECB_MODE;
81 return rk_handle_req(dev, req);
84 static int rk_aes_ecb_decrypt(struct ablkcipher_request *req)
86 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
87 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
88 struct rk_crypto_info *dev = ctx->dev;
90 ctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;
91 return rk_handle_req(dev, req);
94 static int rk_aes_cbc_encrypt(struct ablkcipher_request *req)
96 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
97 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
98 struct rk_crypto_info *dev = ctx->dev;
100 ctx->mode = RK_CRYPTO_AES_CBC_MODE;
101 return rk_handle_req(dev, req);
104 static int rk_aes_cbc_decrypt(struct ablkcipher_request *req)
106 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
107 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
108 struct rk_crypto_info *dev = ctx->dev;
110 ctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;
111 return rk_handle_req(dev, req);
114 static int rk_des_ecb_encrypt(struct ablkcipher_request *req)
116 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
117 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
118 struct rk_crypto_info *dev = ctx->dev;
121 return rk_handle_req(dev, req);
124 static int rk_des_ecb_decrypt(struct ablkcipher_request *req)
126 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
127 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
128 struct rk_crypto_info *dev = ctx->dev;
130 ctx->mode = RK_CRYPTO_DEC;
131 return rk_handle_req(dev, req);
134 static int rk_des_cbc_encrypt(struct ablkcipher_request *req)
136 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
137 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
138 struct rk_crypto_info *dev = ctx->dev;
140 ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;
141 return rk_handle_req(dev, req);
144 static int rk_des_cbc_decrypt(struct ablkcipher_request *req)
146 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
147 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
148 struct rk_crypto_info *dev = ctx->dev;
150 ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;
151 return rk_handle_req(dev, req);
154 static int rk_des3_ede_ecb_encrypt(struct ablkcipher_request *req)
156 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
157 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
158 struct rk_crypto_info *dev = ctx->dev;
160 ctx->mode = RK_CRYPTO_TDES_SELECT;
161 return rk_handle_req(dev, req);
164 static int rk_des3_ede_ecb_decrypt(struct ablkcipher_request *req)
166 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
167 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
168 struct rk_crypto_info *dev = ctx->dev;
170 ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;
171 return rk_handle_req(dev, req);
174 static int rk_des3_ede_cbc_encrypt(struct ablkcipher_request *req)
176 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
177 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
178 struct rk_crypto_info *dev = ctx->dev;
180 ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;
181 return rk_handle_req(dev, req);
184 static int rk_des3_ede_cbc_decrypt(struct ablkcipher_request *req)
186 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
187 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
188 struct rk_crypto_info *dev = ctx->dev;
190 ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |
192 return rk_handle_req(dev, req);
195 static void rk_ablk_hw_init(struct rk_crypto_info *dev)
197 struct ablkcipher_request *req =
198 ablkcipher_request_cast(dev->async_req);
199 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
200 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
201 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher);
202 u32 ivsize, block, conf_reg = 0;
204 block = crypto_tfm_alg_blocksize(tfm);
205 ivsize = crypto_ablkcipher_ivsize(cipher);
207 if (block == DES_BLOCK_SIZE) {
208 ctx->mode |= RK_CRYPTO_TDES_FIFO_MODE |
209 RK_CRYPTO_TDES_BYTESWAP_KEY |
210 RK_CRYPTO_TDES_BYTESWAP_IV;
211 CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, ctx->mode);
212 memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, req->info, ivsize);
213 conf_reg = RK_CRYPTO_DESSEL;
215 ctx->mode |= RK_CRYPTO_AES_FIFO_MODE |
216 RK_CRYPTO_AES_KEY_CHANGE |
217 RK_CRYPTO_AES_BYTESWAP_KEY |
218 RK_CRYPTO_AES_BYTESWAP_IV;
219 if (ctx->keylen == AES_KEYSIZE_192)
220 ctx->mode |= RK_CRYPTO_AES_192BIT_key;
221 else if (ctx->keylen == AES_KEYSIZE_256)
222 ctx->mode |= RK_CRYPTO_AES_256BIT_key;
223 CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, ctx->mode);
224 memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, req->info, ivsize);
226 conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO |
227 RK_CRYPTO_BYTESWAP_BRFIFO;
228 CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg);
229 CRYPTO_WRITE(dev, RK_CRYPTO_INTENA,
230 RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA);
233 static void crypto_dma_start(struct rk_crypto_info *dev)
235 CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, dev->addr_in);
236 CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, dev->count / 4);
237 CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, dev->addr_out);
238 CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START |
239 _SBF(RK_CRYPTO_BLOCK_START, 16));
242 static int rk_set_data_start(struct rk_crypto_info *dev)
245 struct ablkcipher_request *req =
246 ablkcipher_request_cast(dev->async_req);
247 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
248 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
249 u32 ivsize = crypto_ablkcipher_ivsize(tfm);
250 u8 *src_last_blk = page_address(sg_page(dev->sg_src)) +
251 dev->sg_src->offset + dev->sg_src->length - ivsize;
253 /* store the iv that need to be updated in chain mode */
254 if (ctx->mode & RK_CRYPTO_DEC)
255 memcpy(ctx->iv, src_last_blk, ivsize);
257 err = dev->load_data(dev, dev->sg_src, dev->sg_dst);
259 crypto_dma_start(dev);
263 static int rk_ablk_start(struct rk_crypto_info *dev)
265 struct ablkcipher_request *req =
266 ablkcipher_request_cast(dev->async_req);
270 dev->left_bytes = req->nbytes;
271 dev->total = req->nbytes;
272 dev->sg_src = req->src;
273 dev->first = req->src;
274 dev->src_nents = sg_nents(req->src);
275 dev->sg_dst = req->dst;
276 dev->dst_nents = sg_nents(req->dst);
279 spin_lock_irqsave(&dev->lock, flags);
280 rk_ablk_hw_init(dev);
281 err = rk_set_data_start(dev);
282 spin_unlock_irqrestore(&dev->lock, flags);
286 static void rk_iv_copyback(struct rk_crypto_info *dev)
288 struct ablkcipher_request *req =
289 ablkcipher_request_cast(dev->async_req);
290 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
291 u32 ivsize = crypto_ablkcipher_ivsize(tfm);
293 if (ivsize == DES_BLOCK_SIZE)
294 memcpy_fromio(req->info, dev->reg + RK_CRYPTO_TDES_IV_0,
296 else if (ivsize == AES_BLOCK_SIZE)
297 memcpy_fromio(req->info, dev->reg + RK_CRYPTO_AES_IV_0, ivsize);
300 static void rk_update_iv(struct rk_crypto_info *dev)
302 struct ablkcipher_request *req =
303 ablkcipher_request_cast(dev->async_req);
304 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
305 struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
306 u32 ivsize = crypto_ablkcipher_ivsize(tfm);
309 if (ctx->mode & RK_CRYPTO_DEC) {
312 new_iv = page_address(sg_page(dev->sg_dst)) +
313 dev->sg_dst->offset + dev->sg_dst->length - ivsize;
316 if (ivsize == DES_BLOCK_SIZE)
317 memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, new_iv, ivsize);
318 else if (ivsize == AES_BLOCK_SIZE)
319 memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, new_iv, ivsize);
323 * true some err was occurred
324 * fault no err, continue
326 static int rk_ablk_rx(struct rk_crypto_info *dev)
329 struct ablkcipher_request *req =
330 ablkcipher_request_cast(dev->async_req);
332 dev->unload_data(dev);
334 if (!sg_pcopy_from_buffer(req->dst, dev->dst_nents,
335 dev->addr_vir, dev->count,
336 dev->total - dev->left_bytes -
342 if (dev->left_bytes) {
345 if (sg_is_last(dev->sg_src)) {
346 dev_err(dev->dev, "[%s:%d] Lack of data\n",
351 dev->sg_src = sg_next(dev->sg_src);
352 dev->sg_dst = sg_next(dev->sg_dst);
354 err = rk_set_data_start(dev);
357 /* here show the calculation is over without any err */
358 dev->complete(dev->async_req, 0);
359 tasklet_schedule(&dev->queue_task);
365 static int rk_ablk_cra_init(struct crypto_tfm *tfm)
367 struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
368 struct crypto_alg *alg = tfm->__crt_alg;
369 struct rk_crypto_tmp *algt;
371 algt = container_of(alg, struct rk_crypto_tmp, alg.crypto);
373 ctx->dev = algt->dev;
374 ctx->dev->align_size = crypto_tfm_alg_alignmask(tfm) + 1;
375 ctx->dev->start = rk_ablk_start;
376 ctx->dev->update = rk_ablk_rx;
377 ctx->dev->complete = rk_crypto_complete;
378 ctx->dev->addr_vir = (char *)__get_free_page(GFP_KERNEL);
380 return ctx->dev->addr_vir ? ctx->dev->enable_clk(ctx->dev) : -ENOMEM;
383 static void rk_ablk_cra_exit(struct crypto_tfm *tfm)
385 struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
387 free_page((unsigned long)ctx->dev->addr_vir);
388 ctx->dev->disable_clk(ctx->dev);
391 struct rk_crypto_tmp rk_ecb_aes_alg = {
392 .type = ALG_TYPE_CIPHER,
394 .cra_name = "ecb(aes)",
395 .cra_driver_name = "ecb-aes-rk",
397 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
399 .cra_blocksize = AES_BLOCK_SIZE,
400 .cra_ctxsize = sizeof(struct rk_cipher_ctx),
401 .cra_alignmask = 0x0f,
402 .cra_type = &crypto_ablkcipher_type,
403 .cra_module = THIS_MODULE,
404 .cra_init = rk_ablk_cra_init,
405 .cra_exit = rk_ablk_cra_exit,
406 .cra_u.ablkcipher = {
407 .min_keysize = AES_MIN_KEY_SIZE,
408 .max_keysize = AES_MAX_KEY_SIZE,
409 .setkey = rk_aes_setkey,
410 .encrypt = rk_aes_ecb_encrypt,
411 .decrypt = rk_aes_ecb_decrypt,
416 struct rk_crypto_tmp rk_cbc_aes_alg = {
417 .type = ALG_TYPE_CIPHER,
419 .cra_name = "cbc(aes)",
420 .cra_driver_name = "cbc-aes-rk",
422 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
424 .cra_blocksize = AES_BLOCK_SIZE,
425 .cra_ctxsize = sizeof(struct rk_cipher_ctx),
426 .cra_alignmask = 0x0f,
427 .cra_type = &crypto_ablkcipher_type,
428 .cra_module = THIS_MODULE,
429 .cra_init = rk_ablk_cra_init,
430 .cra_exit = rk_ablk_cra_exit,
431 .cra_u.ablkcipher = {
432 .min_keysize = AES_MIN_KEY_SIZE,
433 .max_keysize = AES_MAX_KEY_SIZE,
434 .ivsize = AES_BLOCK_SIZE,
435 .setkey = rk_aes_setkey,
436 .encrypt = rk_aes_cbc_encrypt,
437 .decrypt = rk_aes_cbc_decrypt,
442 struct rk_crypto_tmp rk_ecb_des_alg = {
443 .type = ALG_TYPE_CIPHER,
445 .cra_name = "ecb(des)",
446 .cra_driver_name = "ecb-des-rk",
448 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
450 .cra_blocksize = DES_BLOCK_SIZE,
451 .cra_ctxsize = sizeof(struct rk_cipher_ctx),
452 .cra_alignmask = 0x07,
453 .cra_type = &crypto_ablkcipher_type,
454 .cra_module = THIS_MODULE,
455 .cra_init = rk_ablk_cra_init,
456 .cra_exit = rk_ablk_cra_exit,
457 .cra_u.ablkcipher = {
458 .min_keysize = DES_KEY_SIZE,
459 .max_keysize = DES_KEY_SIZE,
460 .setkey = rk_tdes_setkey,
461 .encrypt = rk_des_ecb_encrypt,
462 .decrypt = rk_des_ecb_decrypt,
467 struct rk_crypto_tmp rk_cbc_des_alg = {
468 .type = ALG_TYPE_CIPHER,
470 .cra_name = "cbc(des)",
471 .cra_driver_name = "cbc-des-rk",
473 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
475 .cra_blocksize = DES_BLOCK_SIZE,
476 .cra_ctxsize = sizeof(struct rk_cipher_ctx),
477 .cra_alignmask = 0x07,
478 .cra_type = &crypto_ablkcipher_type,
479 .cra_module = THIS_MODULE,
480 .cra_init = rk_ablk_cra_init,
481 .cra_exit = rk_ablk_cra_exit,
482 .cra_u.ablkcipher = {
483 .min_keysize = DES_KEY_SIZE,
484 .max_keysize = DES_KEY_SIZE,
485 .ivsize = DES_BLOCK_SIZE,
486 .setkey = rk_tdes_setkey,
487 .encrypt = rk_des_cbc_encrypt,
488 .decrypt = rk_des_cbc_decrypt,
493 struct rk_crypto_tmp rk_ecb_des3_ede_alg = {
494 .type = ALG_TYPE_CIPHER,
496 .cra_name = "ecb(des3_ede)",
497 .cra_driver_name = "ecb-des3-ede-rk",
499 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
501 .cra_blocksize = DES_BLOCK_SIZE,
502 .cra_ctxsize = sizeof(struct rk_cipher_ctx),
503 .cra_alignmask = 0x07,
504 .cra_type = &crypto_ablkcipher_type,
505 .cra_module = THIS_MODULE,
506 .cra_init = rk_ablk_cra_init,
507 .cra_exit = rk_ablk_cra_exit,
508 .cra_u.ablkcipher = {
509 .min_keysize = DES3_EDE_KEY_SIZE,
510 .max_keysize = DES3_EDE_KEY_SIZE,
511 .ivsize = DES_BLOCK_SIZE,
512 .setkey = rk_tdes_setkey,
513 .encrypt = rk_des3_ede_ecb_encrypt,
514 .decrypt = rk_des3_ede_ecb_decrypt,
519 struct rk_crypto_tmp rk_cbc_des3_ede_alg = {
520 .type = ALG_TYPE_CIPHER,
522 .cra_name = "cbc(des3_ede)",
523 .cra_driver_name = "cbc-des3-ede-rk",
525 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
527 .cra_blocksize = DES_BLOCK_SIZE,
528 .cra_ctxsize = sizeof(struct rk_cipher_ctx),
529 .cra_alignmask = 0x07,
530 .cra_type = &crypto_ablkcipher_type,
531 .cra_module = THIS_MODULE,
532 .cra_init = rk_ablk_cra_init,
533 .cra_exit = rk_ablk_cra_exit,
534 .cra_u.ablkcipher = {
535 .min_keysize = DES3_EDE_KEY_SIZE,
536 .max_keysize = DES3_EDE_KEY_SIZE,
537 .ivsize = DES_BLOCK_SIZE,
538 .setkey = rk_tdes_setkey,
539 .encrypt = rk_des3_ede_cbc_encrypt,
540 .decrypt = rk_des3_ede_cbc_decrypt,