drivers/crypto/caam/caamalg_desc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Shared descriptors for aead, skcipher algorithms
   4  *
   5  * Copyright 2016-2019 NXP
   6  */
   7
   8 #include "compat.h"
   9 #include "desc_constr.h"
  10 #include "caamalg_desc.h"
  11
  12 /*
  13  * For aead functions, read payload and write payload,
  14  * both of which are specified in req->src and req->dst
  15  */
  16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
  17 {
  18         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
  19         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
  20                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
  21 }
  22
  23 /* Set DK bit in class 1 operation if shared */
  24 static inline void append_dec_op1(u32 *desc, u32 type)
  25 {
  26         u32 *jump_cmd, *uncond_jump_cmd;
  27
  28         /* DK bit is valid only for AES */
  29         if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
  30                 append_operation(desc, type | OP_ALG_AS_INITFINAL |
  31                                  OP_ALG_DECRYPT);
  32                 return;
  33         }
  34
  35         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
  36         append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
  37         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
  38         set_jump_tgt_here(desc, jump_cmd);
  39         append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
  40                          OP_ALG_AAI_DK);
  41         set_jump_tgt_here(desc, uncond_jump_cmd);
  42 }
  43
  44 /**
  45  * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
  46  *                               (non-protocol) with no (null) encryption.
  47  * @desc: pointer to buffer used for descriptor construction
  48  * @adata: pointer to authentication transform definitions.
  49  *         A split key is required for SEC Era < 6; the size of the split key
  50  *         is specified in this case. Valid algorithm values - one of
  51  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
  52  *         with OP_ALG_AAI_HMAC_PRECOMP.
  53  * @icvsize: integrity check value (ICV) size (truncated or full)
  54  * @era: SEC Era
  55  */
  56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
  57                                  unsigned int icvsize, int era)
  58 {
  59         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
  60
  61         init_sh_desc(desc, HDR_SHARE_SERIAL);
  62
  63         /* Skip if already shared */
  64         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
  65                                    JUMP_COND_SHRD);
  66         if (era < 6) {
  67                 if (adata->key_inline)
  68                         append_key_as_imm(desc, adata->key_virt,
  69                                           adata->keylen_pad, adata->keylen,
  70                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
  71                                           KEY_ENC);
  72                 else
  73                         append_key(desc, adata->key_dma, adata->keylen,
  74                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
  75         } else {
  76                 append_proto_dkp(desc, adata);
  77         }
  78         set_jump_tgt_here(desc, key_jump_cmd);
  79
  80         /* assoclen + cryptlen = seqinlen */
  81         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
  82
  83         /* Prepare to read and write cryptlen + assoclen bytes */
  84         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
  85         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
  86
  87         /*
  88          * MOVE_LEN opcode is not available in all SEC HW revisions,
  89          * thus need to do some magic, i.e. self-patch the descriptor
  90          * buffer.
  91          */
  92         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
  93                                     MOVE_DEST_MATH3 |
  94                                     (0x6 << MOVE_LEN_SHIFT));
  95         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
  96                                      MOVE_DEST_DESCBUF |
  97                                      MOVE_WAITCOMP |
  98                                      (0x8 << MOVE_LEN_SHIFT));
  99
 100         /* Class 2 operation */
 101         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 102                          OP_ALG_ENCRYPT);
 103
 104         /* Read and write cryptlen bytes */
 105         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 106
 107         set_move_tgt_here(desc, read_move_cmd);
 108         set_move_tgt_here(desc, write_move_cmd);
 109         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 110         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 111                     MOVE_AUX_LS);
 112
 113         /* Write ICV */
 114         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 115                          LDST_SRCDST_BYTE_CONTEXT);
 116
 117         print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
 118                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 119                              1);
 120 }
 121 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
 122
 123 /**
 124  * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
 125  *                               (non-protocol) with no (null) decryption.
 126  * @desc: pointer to buffer used for descriptor construction
 127  * @adata: pointer to authentication transform definitions.
 128  *         A split key is required for SEC Era < 6; the size of the split key
 129  *         is specified in this case. Valid algorithm values - one of
 130  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 131  *         with OP_ALG_AAI_HMAC_PRECOMP.
 132  * @icvsize: integrity check value (ICV) size (truncated or full)
 133  * @era: SEC Era
 134  */
 135 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
 136                                  unsigned int icvsize, int era)
 137 {
 138         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
 139
 140         init_sh_desc(desc, HDR_SHARE_SERIAL);
 141
 142         /* Skip if already shared */
 143         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 144                                    JUMP_COND_SHRD);
 145         if (era < 6) {
 146                 if (adata->key_inline)
 147                         append_key_as_imm(desc, adata->key_virt,
 148                                           adata->keylen_pad, adata->keylen,
 149                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
 150                                           KEY_ENC);
 151                 else
 152                         append_key(desc, adata->key_dma, adata->keylen,
 153                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 154         } else {
 155                 append_proto_dkp(desc, adata);
 156         }
 157         set_jump_tgt_here(desc, key_jump_cmd);
 158
 159         /* Class 2 operation */
 160         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 161                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 162
 163         /* assoclen + cryptlen = seqoutlen */
 164         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 165
 166         /* Prepare to read and write cryptlen + assoclen bytes */
 167         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 168         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 169
 170         /*
 171          * MOVE_LEN opcode is not available in all SEC HW revisions,
 172          * thus need to do some magic, i.e. self-patch the descriptor
 173          * buffer.
 174          */
 175         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 176                                     MOVE_DEST_MATH2 |
 177                                     (0x6 << MOVE_LEN_SHIFT));
 178         write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
 179                                      MOVE_DEST_DESCBUF |
 180                                      MOVE_WAITCOMP |
 181                                      (0x8 << MOVE_LEN_SHIFT));
 182
 183         /* Read and write cryptlen bytes */
 184         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 185
 186         /*
 187          * Insert a NOP here, since we need at least 4 instructions between
 188          * code patching the descriptor buffer and the location being patched.
 189          */
 190         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 191         set_jump_tgt_here(desc, jump_cmd);
 192
 193         set_move_tgt_here(desc, read_move_cmd);
 194         set_move_tgt_here(desc, write_move_cmd);
 195         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 196         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 197                     MOVE_AUX_LS);
 198         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 199
 200         /* Load ICV */
 201         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
 202                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 203
 204         print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
 205                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 206                              1);
 207 }
 208 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
 209
 210 static void init_sh_desc_key_aead(u32 * const desc,
 211                                   struct alginfo * const cdata,
 212                                   struct alginfo * const adata,
 213                                   const bool is_rfc3686, u32 *nonce, int era)
 214 {
 215         u32 *key_jump_cmd;
 216         unsigned int enckeylen = cdata->keylen;
 217
 218         /* Note: Context registers are saved. */
 219         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
 220
 221         /* Skip if already shared */
 222         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 223                                    JUMP_COND_SHRD);
 224
 225         /*
 226          * RFC3686 specific:
 227          *      | key = {AUTH_KEY, ENC_KEY, NONCE}
 228          *      | enckeylen = encryption key size + nonce size
 229          */
 230         if (is_rfc3686)
 231                 enckeylen -= CTR_RFC3686_NONCE_SIZE;
 232
 233         if (era < 6) {
 234                 if (adata->key_inline)
 235                         append_key_as_imm(desc, adata->key_virt,
 236                                           adata->keylen_pad, adata->keylen,
 237                                           CLASS_2 | KEY_DEST_MDHA_SPLIT |
 238                                           KEY_ENC);
 239                 else
 240                         append_key(desc, adata->key_dma, adata->keylen,
 241                                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 242         } else {
 243                 append_proto_dkp(desc, adata);
 244         }
 245
 246         if (cdata->key_inline)
 247                 append_key_as_imm(desc, cdata->key_virt, enckeylen,
 248                                   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 249         else
 250                 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
 251                            KEY_DEST_CLASS_REG);
 252
 253         /* Load Counter into CONTEXT1 reg */
 254         if (is_rfc3686) {
 255                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
 256                                    LDST_CLASS_IND_CCB |
 257                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 258                 append_move(desc,
 259                             MOVE_SRC_OUTFIFO |
 260                             MOVE_DEST_CLASS1CTX |
 261                             (16 << MOVE_OFFSET_SHIFT) |
 262                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
 263         }
 264
 265         set_jump_tgt_here(desc, key_jump_cmd);
 266 }
 267
 268 /**
 269  * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
 270  *                          (non-protocol).
 271  * @desc: pointer to buffer used for descriptor construction
 272  * @cdata: pointer to block cipher transform definitions
 273  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 274  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 275  * @adata: pointer to authentication transform definitions.
 276  *         A split key is required for SEC Era < 6; the size of the split key
 277  *         is specified in this case. Valid algorithm values - one of
 278  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 279  *         with OP_ALG_AAI_HMAC_PRECOMP.
 280  * @ivsize: initialization vector size
 281  * @icvsize: integrity check value (ICV) size (truncated or full)
 282  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 283  * @nonce: pointer to rfc3686 nonce
 284  * @ctx1_iv_off: IV offset in CONTEXT1 register
 285  * @is_qi: true when called from caam/qi
 286  * @era: SEC Era
 287  */
 288 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
 289                             struct alginfo *adata, unsigned int ivsize,
 290                             unsigned int icvsize, const bool is_rfc3686,
 291                             u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
 292                             int era)
 293 {
 294         /* Note: Context registers are saved. */
 295         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 296
 297         /* Class 2 operation */
 298         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 299                          OP_ALG_ENCRYPT);
 300
 301         if (is_qi) {
 302                 u32 *wait_load_cmd;
 303
 304                 /* REG3 = assoclen */
 305                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 306                                 LDST_SRCDST_WORD_DECO_MATH3 |
 307                                 (4 << LDST_OFFSET_SHIFT));
 308
 309                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 310                                             JUMP_COND_CALM | JUMP_COND_NCP |
 311                                             JUMP_COND_NOP | JUMP_COND_NIP |
 312                                             JUMP_COND_NIFP);
 313                 set_jump_tgt_here(desc, wait_load_cmd);
 314
 315                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 316                                 LDST_SRCDST_BYTE_CONTEXT |
 317                                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
 318         }
 319
 320         /* Read and write assoclen bytes */
 321         if (is_qi || era < 3) {
 322                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 323                 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 324         } else {
 325                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 326                 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 327         }
 328
 329         /* Skip assoc data */
 330         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 331
 332         /* read assoc before reading payload */
 333         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 334                                       FIFOLDST_VLF);
 335
 336         /* Load Counter into CONTEXT1 reg */
 337         if (is_rfc3686)
 338                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 339                                      LDST_SRCDST_BYTE_CONTEXT |
 340                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 341                                       LDST_OFFSET_SHIFT));
 342
 343         /* Class 1 operation */
 344         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 345                          OP_ALG_ENCRYPT);
 346
 347         /* Read and write cryptlen bytes */
 348         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 349         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 350         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 351
 352         /* Write ICV */
 353         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 354                          LDST_SRCDST_BYTE_CONTEXT);
 355
 356         print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
 357                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 358                              1);
 359 }
 360 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
 361
 362 /**
 363  * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
 364  *                          (non-protocol).
 365  * @desc: pointer to buffer used for descriptor construction
 366  * @cdata: pointer to block cipher transform definitions
 367  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 368  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 369  * @adata: pointer to authentication transform definitions.
 370  *         A split key is required for SEC Era < 6; the size of the split key
 371  *         is specified in this case. Valid algorithm values - one of
 372  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 373  *         with OP_ALG_AAI_HMAC_PRECOMP.
 374  * @ivsize: initialization vector size
 375  * @icvsize: integrity check value (ICV) size (truncated or full)
 376  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 377  * @nonce: pointer to rfc3686 nonce
 378  * @ctx1_iv_off: IV offset in CONTEXT1 register
 379  * @is_qi: true when called from caam/qi
 380  * @era: SEC Era
 381  */
 382 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
 383                             struct alginfo *adata, unsigned int ivsize,
 384                             unsigned int icvsize, const bool geniv,
 385                             const bool is_rfc3686, u32 *nonce,
 386                             const u32 ctx1_iv_off, const bool is_qi, int era)
 387 {
 388         /* Note: Context registers are saved. */
 389         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 390
 391         /* Class 2 operation */
 392         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 393                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 394
 395         if (is_qi) {
 396                 u32 *wait_load_cmd;
 397
 398                 /* REG3 = assoclen */
 399                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 400                                 LDST_SRCDST_WORD_DECO_MATH3 |
 401                                 (4 << LDST_OFFSET_SHIFT));
 402
 403                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 404                                             JUMP_COND_CALM | JUMP_COND_NCP |
 405                                             JUMP_COND_NOP | JUMP_COND_NIP |
 406                                             JUMP_COND_NIFP);
 407                 set_jump_tgt_here(desc, wait_load_cmd);
 408
 409                 if (!geniv)
 410                         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 411                                         LDST_SRCDST_BYTE_CONTEXT |
 412                                         (ctx1_iv_off << LDST_OFFSET_SHIFT));
 413         }
 414
 415         /* Read and write assoclen bytes */
 416         if (is_qi || era < 3) {
 417                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 418                 if (geniv)
 419                         append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
 420                                                 ivsize);
 421                 else
 422                         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
 423                                         CAAM_CMD_SZ);
 424         } else {
 425                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 426                 if (geniv)
 427                         append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
 428                                                 ivsize);
 429                 else
 430                         append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
 431                                         CAAM_CMD_SZ);
 432         }
 433
 434         /* Skip assoc data */
 435         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 436
 437         /* read assoc before reading payload */
 438         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 439                              KEY_VLF);
 440
 441         if (geniv) {
 442                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 443                                 LDST_SRCDST_BYTE_CONTEXT |
 444                                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
 445                 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
 446                             (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
 447         }
 448
 449         /* Load Counter into CONTEXT1 reg */
 450         if (is_rfc3686)
 451                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 452                                      LDST_SRCDST_BYTE_CONTEXT |
 453                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 454                                       LDST_OFFSET_SHIFT));
 455
 456         /* Choose operation */
 457         if (ctx1_iv_off)
 458                 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 459                                  OP_ALG_DECRYPT);
 460         else
 461                 append_dec_op1(desc, cdata->algtype);
 462
 463         /* Read and write cryptlen bytes */
 464         append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 465         append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 466         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 467
 468         /* Load ICV */
 469         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
 470                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 471
 472         print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
 473                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 474                              1);
 475 }
 476 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
 477
 478 /**
 479  * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
 480  *                             (non-protocol) with HW-generated initialization
 481  *                             vector.
 482  * @desc: pointer to buffer used for descriptor construction
 483  * @cdata: pointer to block cipher transform definitions
 484  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 485  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 486  * @adata: pointer to authentication transform definitions.
 487  *         A split key is required for SEC Era < 6; the size of the split key
 488  *         is specified in this case. Valid algorithm values - one of
 489  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 490  *         with OP_ALG_AAI_HMAC_PRECOMP.
 491  * @ivsize: initialization vector size
 492  * @icvsize: integrity check value (ICV) size (truncated or full)
 493  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 494  * @nonce: pointer to rfc3686 nonce
 495  * @ctx1_iv_off: IV offset in CONTEXT1 register
 496  * @is_qi: true when called from caam/qi
 497  * @era: SEC Era
 498  */
 499 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
 500                                struct alginfo *adata, unsigned int ivsize,
 501                                unsigned int icvsize, const bool is_rfc3686,
 502                                u32 *nonce, const u32 ctx1_iv_off,
 503                                const bool is_qi, int era)
 504 {
 505         u32 geniv, moveiv;
 506         u32 *wait_cmd;
 507
 508         /* Note: Context registers are saved. */
 509         init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 510
 511         if (is_qi) {
 512                 u32 *wait_load_cmd;
 513
 514                 /* REG3 = assoclen */
 515                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 516                                 LDST_SRCDST_WORD_DECO_MATH3 |
 517                                 (4 << LDST_OFFSET_SHIFT));
 518
 519                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 520                                             JUMP_COND_CALM | JUMP_COND_NCP |
 521                                             JUMP_COND_NOP | JUMP_COND_NIP |
 522                                             JUMP_COND_NIFP);
 523                 set_jump_tgt_here(desc, wait_load_cmd);
 524         }
 525
 526         if (is_rfc3686) {
 527                 if (is_qi)
 528                         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 529                                         LDST_SRCDST_BYTE_CONTEXT |
 530                                         (ctx1_iv_off << LDST_OFFSET_SHIFT));
 531
 532                 goto copy_iv;
 533         }
 534
 535         /* Generate IV */
 536         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 537                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 538                 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 539         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 540                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 541         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 542         append_move(desc, MOVE_WAITCOMP |
 543                     MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
 544                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 545                     (ivsize << MOVE_LEN_SHIFT));
 546         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 547
 548 copy_iv:
 549         /* Copy IV to class 1 context */
 550         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
 551                     (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 552                     (ivsize << MOVE_LEN_SHIFT));
 553
 554         /* Return to encryption */
 555         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 556                          OP_ALG_ENCRYPT);
 557
 558         /* Read and write assoclen bytes */
 559         if (is_qi || era < 3) {
 560                 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 561                 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 562         } else {
 563                 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 564                 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 565         }
 566
 567         /* Skip assoc data */
 568         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 569
 570         /* read assoc before reading payload */
 571         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 572                              KEY_VLF);
 573
 574         /* Copy iv from outfifo to class 2 fifo */
 575         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 576                  NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 577         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 578                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 579         append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
 580                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 581
 582         /* Load Counter into CONTEXT1 reg */
 583         if (is_rfc3686)
 584                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 585                                      LDST_SRCDST_BYTE_CONTEXT |
 586                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 587                                       LDST_OFFSET_SHIFT));
 588
 589         /* Class 1 operation */
 590         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 591                          OP_ALG_ENCRYPT);
 592
 593         /* Will write ivsize + cryptlen */
 594         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 595
 596         /* Not need to reload iv */
 597         append_seq_fifo_load(desc, ivsize,
 598                              FIFOLD_CLASS_SKIP);
 599
 600         /* Will read cryptlen */
 601         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 602
 603         /*
 604          * Wait for IV transfer (ofifo -> class2) to finish before starting
 605          * ciphertext transfer (ofifo -> external memory).
 606          */
 607         wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
 608         set_jump_tgt_here(desc, wait_cmd);
 609
 610         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
 611                              FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
 612         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 613
 614         /* Write ICV */
 615         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 616                          LDST_SRCDST_BYTE_CONTEXT);
 617
 618         print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
 619                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 620                              1);
 621 }
 622 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
 623
 624 /**
 625  * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
 626  * @desc: pointer to buffer used for descriptor construction
 627  * @cdata: pointer to block cipher transform definitions
 628  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 629  * @ivsize: initialization vector size
 630  * @icvsize: integrity check value (ICV) size (truncated or full)
 631  * @is_qi: true when called from caam/qi
 632  */
 633 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
 634                            unsigned int ivsize, unsigned int icvsize,
 635                            const bool is_qi)
 636 {
 637         u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
 638             *zero_assoc_jump_cmd2;
 639
 640         init_sh_desc(desc, HDR_SHARE_SERIAL);
 641
 642         /* skip key loading if they are loaded due to sharing */
 643         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 644                                    JUMP_COND_SHRD);
 645         if (cdata->key_inline)
 646                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 647                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 648         else
 649                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 650                            KEY_DEST_CLASS_REG);
 651         set_jump_tgt_here(desc, key_jump_cmd);
 652
 653         /* class 1 operation */
 654         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 655                          OP_ALG_ENCRYPT);
 656
 657         if (is_qi) {
 658                 u32 *wait_load_cmd;
 659
 660                 /* REG3 = assoclen */
 661                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 662                                 LDST_SRCDST_WORD_DECO_MATH3 |
 663                                 (4 << LDST_OFFSET_SHIFT));
 664
 665                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 666                                             JUMP_COND_CALM | JUMP_COND_NCP |
 667                                             JUMP_COND_NOP | JUMP_COND_NIP |
 668                                             JUMP_COND_NIFP);
 669                 set_jump_tgt_here(desc, wait_load_cmd);
 670
 671                 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
 672                                         ivsize);
 673         } else {
 674                 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
 675                                 CAAM_CMD_SZ);
 676         }
 677
 678         /* if assoclen + cryptlen is ZERO, skip to ICV write */
 679         zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
 680                                                  JUMP_COND_MATH_Z);
 681
 682         if (is_qi)
 683                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 684                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 685
 686         /* if assoclen is ZERO, skip reading the assoc data */
 687         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 688         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 689                                            JUMP_COND_MATH_Z);
 690
 691         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 692
 693         /* skip assoc data */
 694         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 695
 696         /* cryptlen = seqinlen - assoclen */
 697         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 698
 699         /* if cryptlen is ZERO jump to zero-payload commands */
 700         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 701                                             JUMP_COND_MATH_Z);
 702
 703         /* read assoc data */
 704         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 705                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 706         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 707
 708         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 709
 710         /* write encrypted data */
 711         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 712
 713         /* read payload data */
 714         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 715                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 716
 717         /* jump to ICV writing */
 718         if (is_qi)
 719                 append_jump(desc, JUMP_TEST_ALL | 4);
 720         else
 721                 append_jump(desc, JUMP_TEST_ALL | 2);
 722
 723         /* zero-payload commands */
 724         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 725
 726         /* read assoc data */
 727         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 728                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
 729         if (is_qi)
 730                 /* jump to ICV writing */
 731                 append_jump(desc, JUMP_TEST_ALL | 2);
 732
 733         /* There is no input data */
 734         set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
 735
 736         if (is_qi)
 737                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 738                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
 739                                      FIFOLD_TYPE_LAST1);
 740
 741         /* write ICV */
 742         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
 743                          LDST_SRCDST_BYTE_CONTEXT);
 744
 745         print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
 746                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 747                              1);
 748 }
 749 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
 750
 751 /**
 752  * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
 753  * @desc: pointer to buffer used for descriptor construction
 754  * @cdata: pointer to block cipher transform definitions
 755  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 756  * @ivsize: initialization vector size
 757  * @icvsize: integrity check value (ICV) size (truncated or full)
 758  * @is_qi: true when called from caam/qi
 759  */
 760 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
 761                            unsigned int ivsize, unsigned int icvsize,
 762                            const bool is_qi)
 763 {
 764         u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
 765
 766         init_sh_desc(desc, HDR_SHARE_SERIAL);
 767
 768         /* skip key loading if they are loaded due to sharing */
 769         key_jump_cmd = append_jump(desc, JUMP_JSL |
 770                                    JUMP_TEST_ALL | JUMP_COND_SHRD);
 771         if (cdata->key_inline)
 772                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 773                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 774         else
 775                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 776                            KEY_DEST_CLASS_REG);
 777         set_jump_tgt_here(desc, key_jump_cmd);
 778
 779         /* class 1 operation */
 780         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 781                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 782
 783         if (is_qi) {
 784                 u32 *wait_load_cmd;
 785
 786                 /* REG3 = assoclen */
 787                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 788                                 LDST_SRCDST_WORD_DECO_MATH3 |
 789                                 (4 << LDST_OFFSET_SHIFT));
 790
 791                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 792                                             JUMP_COND_CALM | JUMP_COND_NCP |
 793                                             JUMP_COND_NOP | JUMP_COND_NIP |
 794                                             JUMP_COND_NIFP);
 795                 set_jump_tgt_here(desc, wait_load_cmd);
 796
 797                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 798                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 799         }
 800
 801         /* if assoclen is ZERO, skip reading the assoc data */
 802         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 803         zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 804                                                  JUMP_COND_MATH_Z);
 805
 806         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 807
 808         /* skip assoc data */
 809         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 810
 811         /* read assoc data */
 812         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 813                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 814
 815         set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 816
 817         /* cryptlen = seqoutlen - assoclen */
 818         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 819
 820         /* jump to zero-payload command if cryptlen is zero */
 821         zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 822                                             JUMP_COND_MATH_Z);
 823
 824         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 825
 826         /* store encrypted data */
 827         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 828
 829         /* read payload data */
 830         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 831                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 832
 833         /* zero-payload command */
 834         set_jump_tgt_here(desc, zero_payload_jump_cmd);
 835
 836         /* read ICV */
 837         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
 838                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 839
 840         print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
 841                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 842                              1);
 843 }
 844 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
 845
 846 /**
 847  * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
 848  *                             (non-protocol).
 849  * @desc: pointer to buffer used for descriptor construction
 850  * @cdata: pointer to block cipher transform definitions
 851  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 852  * @ivsize: initialization vector size
 853  * @icvsize: integrity check value (ICV) size (truncated or full)
 854  * @is_qi: true when called from caam/qi
 855  */
 856 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
 857                                unsigned int ivsize, unsigned int icvsize,
 858                                const bool is_qi)
 859 {
 860         u32 *key_jump_cmd;
 861
 862         init_sh_desc(desc, HDR_SHARE_SERIAL);
 863
 864         /* Skip key loading if it is loaded due to sharing */
 865         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 866                                    JUMP_COND_SHRD);
 867         if (cdata->key_inline)
 868                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 869                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 870         else
 871                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 872                            KEY_DEST_CLASS_REG);
 873         set_jump_tgt_here(desc, key_jump_cmd);
 874
 875         /* Class 1 operation */
 876         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 877                          OP_ALG_ENCRYPT);
 878
 879         if (is_qi) {
 880                 u32 *wait_load_cmd;
 881
 882                 /* REG3 = assoclen */
 883                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 884                                 LDST_SRCDST_WORD_DECO_MATH3 |
 885                                 (4 << LDST_OFFSET_SHIFT));
 886
 887                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 888                                             JUMP_COND_CALM | JUMP_COND_NCP |
 889                                             JUMP_COND_NOP | JUMP_COND_NIP |
 890                                             JUMP_COND_NIFP);
 891                 set_jump_tgt_here(desc, wait_load_cmd);
 892
 893                 /* Read salt and IV */
 894                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
 895                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
 896                                         FIFOLD_TYPE_IV);
 897                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 898                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 899         }
 900
 901         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
 902         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 903
 904         /* Read assoc data */
 905         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 906                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 907
 908         /* Skip IV */
 909         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
 910
 911         /* Will read cryptlen bytes */
 912         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 913
 914         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
 915         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
 916
 917         /* Skip assoc data */
 918         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 919
 920         /* cryptlen = seqoutlen - assoclen */
 921         append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
 922
 923         /* Write encrypted data */
 924         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 925
 926         /* Read payload data */
 927         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 928                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 929
 930         /* Write ICV */
 931         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
 932                          LDST_SRCDST_BYTE_CONTEXT);
 933
 934         print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
 935                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 936                              1);
 937 }
 938 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
 939
 940 /**
 941  * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
 942  *                             (non-protocol).
 943  * @desc: pointer to buffer used for descriptor construction
 944  * @cdata: pointer to block cipher transform definitions
 945  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 946  * @ivsize: initialization vector size
 947  * @icvsize: integrity check value (ICV) size (truncated or full)
 948  * @is_qi: true when called from caam/qi
 949  */
 950 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
 951                                unsigned int ivsize, unsigned int icvsize,
 952                                const bool is_qi)
 953 {
 954         u32 *key_jump_cmd;
 955
 956         init_sh_desc(desc, HDR_SHARE_SERIAL);
 957
 958         /* Skip key loading if it is loaded due to sharing */
 959         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 960                                    JUMP_COND_SHRD);
 961         if (cdata->key_inline)
 962                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 963                                   cdata->keylen, CLASS_1 |
 964                                   KEY_DEST_CLASS_REG);
 965         else
 966                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 967                            KEY_DEST_CLASS_REG);
 968         set_jump_tgt_here(desc, key_jump_cmd);
 969
 970         /* Class 1 operation */
 971         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 972                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 973
 974         if (is_qi) {
 975                 u32 *wait_load_cmd;
 976
 977                 /* REG3 = assoclen */
 978                 append_seq_load(desc, 4, LDST_CLASS_DECO |
 979                                 LDST_SRCDST_WORD_DECO_MATH3 |
 980                                 (4 << LDST_OFFSET_SHIFT));
 981
 982                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 983                                             JUMP_COND_CALM | JUMP_COND_NCP |
 984                                             JUMP_COND_NOP | JUMP_COND_NIP |
 985                                             JUMP_COND_NIFP);
 986                 set_jump_tgt_here(desc, wait_load_cmd);
 987
 988                 /* Read salt and IV */
 989                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
 990                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
 991                                         FIFOLD_TYPE_IV);
 992                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 993                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 994         }
 995
 996         append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
 997         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 998
 999         /* Read assoc data */
1000         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1001                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1002
1003         /* Skip IV */
1004         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1005
1006         /* Will read cryptlen bytes */
1007         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1008
1009         /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1010         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1011
1012         /* Skip assoc data */
1013         append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1014
1015         /* Will write cryptlen bytes */
1016         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1017
1018         /* Store payload data */
1019         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1020
1021         /* Read encrypted data */
1022         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1023                              FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1024
1025         /* Read ICV */
1026         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1027                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1028
1029         print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1030                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1031                              1);
1032 }
1033 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1034
1035 /**
1036  * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1037  *                             (non-protocol).
1038  * @desc: pointer to buffer used for descriptor construction
1039  * @cdata: pointer to block cipher transform definitions
1040  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1041  * @ivsize: initialization vector size
1042  * @icvsize: integrity check value (ICV) size (truncated or full)
1043  * @is_qi: true when called from caam/qi
1044  */
1045 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1046                                unsigned int ivsize, unsigned int icvsize,
1047                                const bool is_qi)
1048 {
1049         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1050
1051         init_sh_desc(desc, HDR_SHARE_SERIAL);
1052
1053         /* Skip key loading if it is loaded due to sharing */
1054         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1055                                    JUMP_COND_SHRD);
1056         if (cdata->key_inline)
1057                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1058                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1059         else
1060                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1061                            KEY_DEST_CLASS_REG);
1062         set_jump_tgt_here(desc, key_jump_cmd);
1063
1064         /* Class 1 operation */
1065         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1066                          OP_ALG_ENCRYPT);
1067
1068         if (is_qi) {
1069                 /* assoclen is not needed, skip it */
1070                 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1071
1072                 /* Read salt and IV */
1073                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1074                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1075                                         FIFOLD_TYPE_IV);
1076                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1077                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1078         }
1079
1080         /* assoclen + cryptlen = seqinlen */
1081         append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1082
1083         /*
1084          * MOVE_LEN opcode is not available in all SEC HW revisions,
1085          * thus need to do some magic, i.e. self-patch the descriptor
1086          * buffer.
1087          */
1088         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1089                                     (0x6 << MOVE_LEN_SHIFT));
1090         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1091                                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1092
1093         /* Will read assoclen + cryptlen bytes */
1094         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1095
1096         /* Will write assoclen + cryptlen bytes */
1097         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1098
1099         /* Read and write assoclen + cryptlen bytes */
1100         aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1101
1102         set_move_tgt_here(desc, read_move_cmd);
1103         set_move_tgt_here(desc, write_move_cmd);
1104         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1105         /* Move payload data to OFIFO */
1106         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1107
1108         /* Write ICV */
1109         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1110                          LDST_SRCDST_BYTE_CONTEXT);
1111
1112         print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1113                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1114                              1);
1115 }
1116 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1117
1118 /**
1119  * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1120  *                             (non-protocol).
1121  * @desc: pointer to buffer used for descriptor construction
1122  * @cdata: pointer to block cipher transform definitions
1123  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1124  * @ivsize: initialization vector size
1125  * @icvsize: integrity check value (ICV) size (truncated or full)
1126  * @is_qi: true when called from caam/qi
1127  */
1128 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1129                                unsigned int ivsize, unsigned int icvsize,
1130                                const bool is_qi)
1131 {
1132         u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1133
1134         init_sh_desc(desc, HDR_SHARE_SERIAL);
1135
1136         /* Skip key loading if it is loaded due to sharing */
1137         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1138                                    JUMP_COND_SHRD);
1139         if (cdata->key_inline)
1140                 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1141                                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1142         else
1143                 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1144                            KEY_DEST_CLASS_REG);
1145         set_jump_tgt_here(desc, key_jump_cmd);
1146
1147         /* Class 1 operation */
1148         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1149                          OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1150
1151         if (is_qi) {
1152                 /* assoclen is not needed, skip it */
1153                 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1154
1155                 /* Read salt and IV */
1156                 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1157                                         cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1158                                         FIFOLD_TYPE_IV);
1159                 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1160                                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1161         }
1162
1163         /* assoclen + cryptlen = seqoutlen */
1164         append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1165
1166         /*
1167          * MOVE_LEN opcode is not available in all SEC HW revisions,
1168          * thus need to do some magic, i.e. self-patch the descriptor
1169          * buffer.
1170          */
1171         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1172                                     (0x6 << MOVE_LEN_SHIFT));
1173         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1174                                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1175
1176         /* Will read assoclen + cryptlen bytes */
1177         append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1178
1179         /* Will write assoclen + cryptlen bytes */
1180         append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1181
1182         /* Store payload data */
1183         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1184
1185         /* In-snoop assoclen + cryptlen data */
1186         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1187                              FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1188
1189         set_move_tgt_here(desc, read_move_cmd);
1190         set_move_tgt_here(desc, write_move_cmd);
1191         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1192         /* Move payload data to OFIFO */
1193         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1194         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1195
1196         /* Read ICV */
1197         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1198                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1199
1200         print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1201                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1202                              1);
1203 }
1204 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1205
1206 /**
1207  * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1208  *                          IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1209  *                          descriptor (non-protocol).
1210  * @desc: pointer to buffer used for descriptor construction
1211  * @cdata: pointer to block cipher transform definitions
1212  *         Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1213  *         OP_ALG_AAI_AEAD.
1214  * @adata: pointer to authentication transform definitions
1215  *         Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1216  *         OP_ALG_AAI_AEAD.
1217  * @ivsize: initialization vector size
1218  * @icvsize: integrity check value (ICV) size (truncated or full)
1219  * @encap: true if encapsulation, false if decapsulation
1220  * @is_qi: true when called from caam/qi
1221  */
1222 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1223                             struct alginfo *adata, unsigned int ivsize,
1224                             unsigned int icvsize, const bool encap,
1225                             const bool is_qi)
1226 {
1227         u32 *key_jump_cmd, *wait_cmd;
1228         u32 nfifo;
1229         const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1230
1231         /* Note: Context registers are saved. */
1232         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1233
1234         /* skip key loading if they are loaded due to sharing */
1235         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1236                                    JUMP_COND_SHRD);
1237
1238         append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1239                           CLASS_1 | KEY_DEST_CLASS_REG);
1240
1241         /* For IPsec load the salt from keymat in the context register */
1242         if (is_ipsec)
1243                 append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1244                                    LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1245                                    4 << LDST_OFFSET_SHIFT);
1246
1247         set_jump_tgt_here(desc, key_jump_cmd);
1248
1249         /* Class 2 and 1 operations: Poly & ChaCha */
1250         if (encap) {
1251                 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1252                                  OP_ALG_ENCRYPT);
1253                 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1254                                  OP_ALG_ENCRYPT);
1255         } else {
1256                 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1257                                  OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1258                 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1259                                  OP_ALG_DECRYPT);
1260         }
1261
1262         if (is_qi) {
1263                 u32 *wait_load_cmd;
1264                 u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1265
1266                 /* REG3 = assoclen */
1267                 append_seq_load(desc, 4, LDST_CLASS_DECO |
1268                                 LDST_SRCDST_WORD_DECO_MATH3 |
1269                                 4 << LDST_OFFSET_SHIFT);
1270
1271                 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1272                                             JUMP_COND_CALM | JUMP_COND_NCP |
1273                                             JUMP_COND_NOP | JUMP_COND_NIP |
1274                                             JUMP_COND_NIFP);
1275                 set_jump_tgt_here(desc, wait_load_cmd);
1276
1277                 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1278                                 LDST_SRCDST_BYTE_CONTEXT |
1279                                 ctx1_iv_off << LDST_OFFSET_SHIFT);
1280         }
1281
1282         /*
1283          * MAGIC with NFIFO
1284          * Read associated data from the input and send them to class1 and
1285          * class2 alignment blocks. From class1 send data to output fifo and
1286          * then write it to memory since we don't need to encrypt AD.
1287          */
1288         nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1289                 NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1290         append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1291                             LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1292
1293         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1294         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1295         append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1296                              FIFOLD_CLASS_CLASS1 | LDST_VLF);
1297         append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1298                         MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1299         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1300
1301         /* IPsec - copy IV at the output */
1302         if (is_ipsec)
1303                 append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1304                                       0x2 << 25);
1305
1306         wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1307                                JUMP_COND_NOP | JUMP_TEST_ALL);
1308         set_jump_tgt_here(desc, wait_cmd);
1309
1310         if (encap) {
1311                 /* Read and write cryptlen bytes */
1312                 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1313                 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1314                                 CAAM_CMD_SZ);
1315                 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1316
1317                 /* Write ICV */
1318                 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1319                                  LDST_SRCDST_BYTE_CONTEXT);
1320         } else {
1321                 /* Read and write cryptlen bytes */
1322                 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1323                                 CAAM_CMD_SZ);
1324                 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1325                                 CAAM_CMD_SZ);
1326                 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1327
1328                 /* Load ICV for verification */
1329                 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1330                                      FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1331         }
1332
1333         print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1334                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1335                              1);
1336 }
1337 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1338
1339 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1340 static inline void skcipher_append_src_dst(u32 *desc)
1341 {
1342         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1343         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1344         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1345                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1346         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1347 }
1348
1349 /**
1350  * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1351  * @desc: pointer to buffer used for descriptor construction
1352  * @cdata: pointer to block cipher transform definitions
1353  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1354  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1355  *                                - OP_ALG_ALGSEL_CHACHA20
1356  * @ivsize: initialization vector size
1357  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1358  * @ctx1_iv_off: IV offset in CONTEXT1 register
1359  */
1360 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1361                                 unsigned int ivsize, const bool is_rfc3686,
1362                                 const u32 ctx1_iv_off)
1363 {
1364         u32 *key_jump_cmd;
1365
1366         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1367         /* Skip if already shared */
1368         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1369                                    JUMP_COND_SHRD);
1370
1371         /* Load class1 key only */
1372         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1373                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1374
1375         /* Load nonce into CONTEXT1 reg */
1376         if (is_rfc3686) {
1377                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1378
1379                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1380                                    LDST_CLASS_IND_CCB |
1381                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1382                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1383                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1384                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1385         }
1386
1387         set_jump_tgt_here(desc, key_jump_cmd);
1388
1389         /* Load IV, if there is one */
1390         if (ivsize)
1391                 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1392                                 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1393                                 LDST_OFFSET_SHIFT));
1394
1395         /* Load counter into CONTEXT1 reg */
1396         if (is_rfc3686)
1397                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1398                                      LDST_SRCDST_BYTE_CONTEXT |
1399                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1400                                       LDST_OFFSET_SHIFT));
1401
1402         /* Load operation */
1403         append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1404                          OP_ALG_ENCRYPT);
1405
1406         /* Perform operation */
1407         skcipher_append_src_dst(desc);
1408
1409         /* Store IV */
1410         if (ivsize)
1411                 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1412                                  LDST_CLASS_1_CCB | (ctx1_iv_off <<
1413                                  LDST_OFFSET_SHIFT));
1414
1415         print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1416                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1417                              1);
1418 }
1419 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1420
1421 /**
1422  * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1423  * @desc: pointer to buffer used for descriptor construction
1424  * @cdata: pointer to block cipher transform definitions
1425  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1426  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1427  *                                - OP_ALG_ALGSEL_CHACHA20
1428  * @ivsize: initialization vector size
1429  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1430  * @ctx1_iv_off: IV offset in CONTEXT1 register
1431  */
1432 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1433                                 unsigned int ivsize, const bool is_rfc3686,
1434                                 const u32 ctx1_iv_off)
1435 {
1436         u32 *key_jump_cmd;
1437
1438         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1439         /* Skip if already shared */
1440         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1441                                    JUMP_COND_SHRD);
1442
1443         /* Load class1 key only */
1444         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1445                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1446
1447         /* Load nonce into CONTEXT1 reg */
1448         if (is_rfc3686) {
1449                 const u8 *nonce = cdata->key_virt + cdata->keylen;
1450
1451                 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1452                                    LDST_CLASS_IND_CCB |
1453                                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1454                 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1455                             MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1456                             (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1457         }
1458
1459         set_jump_tgt_here(desc, key_jump_cmd);
1460
1461         /* Load IV, if there is one */
1462         if (ivsize)
1463                 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1464                                 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1465                                 LDST_OFFSET_SHIFT));
1466
1467         /* Load counter into CONTEXT1 reg */
1468         if (is_rfc3686)
1469                 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1470                                      LDST_SRCDST_BYTE_CONTEXT |
1471                                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1472                                       LDST_OFFSET_SHIFT));
1473
1474         /* Choose operation */
1475         if (ctx1_iv_off)
1476                 append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1477                                  OP_ALG_DECRYPT);
1478         else
1479                 append_dec_op1(desc, cdata->algtype);
1480
1481         /* Perform operation */
1482         skcipher_append_src_dst(desc);
1483
1484         /* Store IV */
1485         if (ivsize)
1486                 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1487                                  LDST_CLASS_1_CCB | (ctx1_iv_off <<
1488                                  LDST_OFFSET_SHIFT));
1489
1490         print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1491                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1492                              1);
1493 }
1494 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1495
1496 /**
1497  * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1498  * @desc: pointer to buffer used for descriptor construction
1499  * @cdata: pointer to block cipher transform definitions
1500  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1501  */
1502 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1503 {
1504         __be64 sector_size = cpu_to_be64(512);
1505         u32 *key_jump_cmd;
1506
1507         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1508         /* Skip if already shared */
1509         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1510                                    JUMP_COND_SHRD);
1511
1512         /* Load class1 keys only */
1513         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1514                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1515
1516         /* Load sector size with index 40 bytes (0x28) */
1517         append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1518                            LDST_SRCDST_BYTE_CONTEXT |
1519                            (0x28 << LDST_OFFSET_SHIFT));
1520
1521         set_jump_tgt_here(desc, key_jump_cmd);
1522
1523         /*
1524          * create sequence for loading the sector index
1525          * Upper 8B of IV - will be used as sector index
1526          * Lower 8B of IV - will be discarded
1527          */
1528         append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1529                         (0x20 << LDST_OFFSET_SHIFT));
1530         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1531
1532         /* Load operation */
1533         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1534                          OP_ALG_ENCRYPT);
1535
1536         /* Perform operation */
1537         skcipher_append_src_dst(desc);
1538
1539         /* Store upper 8B of IV */
1540         append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1541                          (0x20 << LDST_OFFSET_SHIFT));
1542
1543         print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1544                              ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1545                              desc, desc_bytes(desc), 1);
1546 }
1547 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1548
1549 /**
1550  * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1551  * @desc: pointer to buffer used for descriptor construction
1552  * @cdata: pointer to block cipher transform definitions
1553  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1554  */
1555 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1556 {
1557         __be64 sector_size = cpu_to_be64(512);
1558         u32 *key_jump_cmd;
1559
1560         init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1561         /* Skip if already shared */
1562         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1563                                    JUMP_COND_SHRD);
1564
1565         /* Load class1 key only */
1566         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1567                           cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1568
1569         /* Load sector size with index 40 bytes (0x28) */
1570         append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1571                            LDST_SRCDST_BYTE_CONTEXT |
1572                            (0x28 << LDST_OFFSET_SHIFT));
1573
1574         set_jump_tgt_here(desc, key_jump_cmd);
1575
1576         /*
1577          * create sequence for loading the sector index
1578          * Upper 8B of IV - will be used as sector index
1579          * Lower 8B of IV - will be discarded
1580          */
1581         append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1582                         (0x20 << LDST_OFFSET_SHIFT));
1583         append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1584
1585         /* Load operation */
1586         append_dec_op1(desc, cdata->algtype);
1587
1588         /* Perform operation */
1589         skcipher_append_src_dst(desc);
1590
1591         /* Store upper 8B of IV */
1592         append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1593                          (0x20 << LDST_OFFSET_SHIFT));
1594
1595         print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1596                              ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1597                              desc_bytes(desc), 1);
1598 }
1599 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1600
1601 MODULE_LICENSE("GPL");
1602 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1603 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");