crypto/rsa/rsa_pmeth.c

   1 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
   2  * project 2006.
   3  */
   4 /* ====================================================================
   5  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  *
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  *
  19  * 3. All advertising materials mentioning features or use of this
  20  *    software must display the following acknowledgment:
  21  *    "This product includes software developed by the OpenSSL Project
  22  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  23  *
  24  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  25  *    endorse or promote products derived from this software without
  26  *    prior written permission. For written permission, please contact
  27  *    licensing@OpenSSL.org.
  28  *
  29  * 5. Products derived from this software may not be called "OpenSSL"
  30  *    nor may "OpenSSL" appear in their names without prior written
  31  *    permission of the OpenSSL Project.
  32  *
  33  * 6. Redistributions of any form whatsoever must retain the following
  34  *    acknowledgment:
  35  *    "This product includes software developed by the OpenSSL Project
  36  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  37  *
  38  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  39  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  40  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  41  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  42  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  43  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  44  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  45  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  46  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  47  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  48  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  49  * OF THE POSSIBILITY OF SUCH DAMAGE.
  50  * ====================================================================
  51  *
  52  * This product includes cryptographic software written by Eric Young
  53  * (eay@cryptsoft.com).  This product includes software written by Tim
  54  * Hudson (tjh@cryptsoft.com).
  55  *
  56  */
  57
  58 #include <stdio.h>
  59 #include "cryptlib.h"
  60 #include <openssl/asn1t.h>
  61 #include <openssl/x509.h>
  62 #include <openssl/rsa.h>
  63 #include <openssl/evp.h>
  64 #include "evp_locl.h"
  65
  66 extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
  67                 unsigned char *rm, unsigned int *prm_len,
  68                 const unsigned char *sigbuf, unsigned int siglen,
  69                 RSA *rsa);
  70
  71 /* RSA pkey context structure */
  72
  73 typedef struct
  74         {
  75         /* Key gen parameters */
  76         int nbits;
  77         BIGNUM *pub_exp;
  78         /* Keygen callback info */
  79         int gentmp[2];
  80         /* RSA padding mode */
  81         int pad_mode;
  82         /* message digest */
  83         const EVP_MD *md;
  84         /* PSS/OAEP salt length */
  85         int saltlen;
  86         /* Temp buffer */
  87         unsigned char *tbuf;
  88         } RSA_PKEY_CTX;
  89
  90 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
  91         {
  92         RSA_PKEY_CTX *rctx;
  93         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
  94         if (!rctx)
  95                 return 0;
  96         rctx->nbits = 1024;
  97         rctx->pub_exp = NULL;
  98         rctx->pad_mode = RSA_PKCS1_PADDING;
  99         rctx->md = NULL;
 100         rctx->tbuf = NULL;
 101
 102         rctx->saltlen = -2;
 103
 104         ctx->data = rctx;
 105         ctx->keygen_info = rctx->gentmp;
 106         ctx->keygen_info_count = 2;
 107
 108         return 1;
 109         }
 110
 111 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
 112         {
 113         if (ctx->tbuf)
 114                 return 1;
 115         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
 116         if (!ctx->tbuf)
 117                 return 0;
 118         return 1;
 119         }
 120
 121 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 122         {
 123         RSA_PKEY_CTX *rctx = ctx->data;
 124         if (rctx)
 125                 {
 126                 if (rctx->pub_exp)
 127                         BN_free(rctx->pub_exp);
 128                 if (rctx->tbuf)
 129                         OPENSSL_free(rctx->tbuf);
 130                 OPENSSL_free(rctx);
 131                 }
 132         }
 133
 134 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 135                                         const unsigned char *tbs, int tbslen)
 136         {
 137         int ret;
 138         RSA_PKEY_CTX *rctx = ctx->data;
 139         RSA *rsa = ctx->pkey->pkey.rsa;
 140
 141         if (rctx->md)
 142                 {
 143                 if (tbslen != EVP_MD_size(rctx->md))
 144                         {
 145                         RSAerr(RSA_F_PKEY_RSA_SIGN,
 146                                         RSA_R_INVALID_DIGEST_LENGTH);
 147                         return -1;
 148                         }
 149                 if (rctx->pad_mode == RSA_X931_PADDING)
 150                         {
 151                         if (!setup_tbuf(rctx, ctx))
 152                                 return -1;
 153                         memcpy(rctx->tbuf, tbs, tbslen);
 154                         rctx->tbuf[tbslen] =
 155                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
 156                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
 157                                                 sig, rsa, RSA_X931_PADDING);
 158                         }
 159                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 160                         {
 161                         unsigned int sltmp;
 162                         ret = RSA_sign(EVP_MD_type(rctx->md),
 163                                                 tbs, tbslen, sig, &sltmp, rsa);
 164                         if (ret <= 0)
 165                                 return ret;
 166                         ret = sltmp;
 167                         }
 168                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 169                         {
 170                         if (!setup_tbuf(rctx, ctx))
 171                                 return -1;
 172                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
 173                                                 rctx->md, rctx->saltlen))
 174                                 return -1;
 175                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
 176                                                 sig, rsa, RSA_NO_PADDING);
 177                         }
 178                 else
 179                         return -1;
 180                 }
 181         else
 182                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 183                                                         rctx->pad_mode);
 184         if (ret < 0)
 185                 return ret;
 186         *siglen = ret;
 187         return 1;
 188         }
 189
 190
 191 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 192                                         unsigned char *rout, int *routlen,
 193                                         const unsigned char *sig, int siglen)
 194         {
 195         int ret;
 196         RSA_PKEY_CTX *rctx = ctx->data;
 197
 198         if (rctx->md)
 199                 {
 200                 if (rctx->pad_mode == RSA_X931_PADDING)
 201                         {
 202                         if (!setup_tbuf(rctx, ctx))
 203                                 return -1;
 204                         ret = RSA_public_decrypt(siglen, sig,
 205                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
 206                                                 RSA_X931_PADDING);
 207                         if (ret < 1)
 208                                 return 0;
 209                         ret--;
 210                         if (rctx->tbuf[ret] !=
 211                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
 212                                 {
 213                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 214                                                 RSA_R_ALGORITHM_MISMATCH);
 215                                 return 0;
 216                                 }
 217                         if (ret != EVP_MD_size(rctx->md))
 218                                 {
 219                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 220                                         RSA_R_INVALID_DIGEST_LENGTH);
 221                                 return 0;
 222                                 }
 223                         if (rout)
 224                                 memcpy(rout, rctx->tbuf, ret);
 225                         }
 226                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 227                         {
 228                         unsigned int sltmp;
 229                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
 230                                                 NULL, 0, rout, &sltmp,
 231                                         sig, siglen, ctx->pkey->pkey.rsa);
 232                         ret = sltmp;
 233                         }
 234                 else
 235                         return -1;
 236                 }
 237         else
 238                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
 239                                                         rctx->pad_mode);
 240         if (ret < 0)
 241                 return ret;
 242         *routlen = ret;
 243         return 1;
 244         }
 245
 246 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
 247                                         const unsigned char *sig, int siglen,
 248                                         const unsigned char *tbs, int tbslen)
 249         {
 250         RSA_PKEY_CTX *rctx = ctx->data;
 251         RSA *rsa = ctx->pkey->pkey.rsa;
 252         int rslen;
 253         if (rctx->md)
 254                 {
 255                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
 256                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
 257                                         sig, siglen, rsa);
 258                 if (rctx->pad_mode == RSA_X931_PADDING)
 259                         {
 260                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
 261                                         sig, siglen) <= 0)
 262                                 return 0;
 263                         }
 264                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 265                         {
 266                         int ret;
 267                         if (!setup_tbuf(rctx, ctx))
 268                                 return -1;
 269                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 270                                                         rsa, RSA_NO_PADDING);
 271                         if (ret <= 0)
 272                                 return 0;
 273                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
 274                                                 rctx->tbuf, rctx->saltlen);
 275                         if (ret <= 0)
 276                                 return 0;
 277                         return 1;
 278                         }
 279                 else
 280                         return -1;
 281                 }
 282         else
 283                 {
 284                 if (!setup_tbuf(rctx, ctx))
 285                         return -1;
 286                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 287                                                 rsa, rctx->pad_mode);
 288                 if (rslen <= 0)
 289                         return 0;
 290                 }
 291
 292         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
 293                 return 0;
 294
 295         return 1;
 296
 297         }
 298
 299
 300 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 301                                         const unsigned char *in, int inlen)
 302         {
 303         int ret;
 304         RSA_PKEY_CTX *rctx = ctx->data;
 305         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 306                                                         rctx->pad_mode);
 307         if (ret < 0)
 308                 return ret;
 309         *outlen = ret;
 310         return 1;
 311         }
 312
 313 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 314                                         const unsigned char *in, int inlen)
 315         {
 316         int ret;
 317         RSA_PKEY_CTX *rctx = ctx->data;
 318         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 319                                                         rctx->pad_mode);
 320         if (ret < 0)
 321                 return ret;
 322         *outlen = ret;
 323         return 1;
 324         }
 325
 326 static int check_padding_md(const EVP_MD *md, int padding)
 327         {
 328         if (!md)
 329                 return 1;
 330
 331         if (padding == RSA_NO_PADDING)
 332                 {
 333                 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
 334                 return 0;
 335                 }
 336
 337         if (padding == RSA_X931_PADDING)
 338                 {
 339                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
 340                         {
 341                         RSAerr(RSA_F_CHECK_PADDING_MD,
 342                                                 RSA_R_INVALID_X931_DIGEST);
 343                         return 0;
 344                         }
 345                 return 1;
 346                 }
 347
 348         return 1;
 349         }
 350
 351
 352 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 353         {
 354         RSA_PKEY_CTX *rctx = ctx->data;
 355         switch (type)
 356                 {
 357                 case EVP_PKEY_CTRL_RSA_PADDING:
 358                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
 359                         {
 360                         if (!check_padding_md(rctx->md, p1))
 361                                 return 0;
 362                         if (p1 == RSA_PKCS1_PSS_PADDING)
 363                                 {
 364                                 if (ctx->operation == EVP_PKEY_OP_VERIFYRECOVER)
 365                                         return -2;
 366                                 if (!rctx->md)
 367                                         rctx->md = EVP_sha1();
 368                                 }
 369                         if (p1 == RSA_PKCS1_OAEP_PADDING)
 370                                 {
 371                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
 372                                         return -2;
 373                                 if (!rctx->md)
 374                                         rctx->md = EVP_sha1();
 375                                 }
 376                         rctx->pad_mode = p1;
 377                         return 1;
 378                         }
 379                 return -2;
 380
 381                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
 382                 if (p1 < -2)
 383                         return -2;
 384                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
 385                         return -2;
 386                 rctx->saltlen = p1;
 387                 return 1;
 388
 389                 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
 390                 if (p1 < 256)
 391                         return -2;
 392                 rctx->nbits = p1;
 393                 return 1;
 394
 395                 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
 396                 if (!p2)
 397                         return -2;
 398                 rctx->pub_exp = p2;
 399                 return 1;
 400
 401                 case EVP_PKEY_CTRL_MD:
 402                 if (!check_padding_md(p2, rctx->pad_mode))
 403                         return 0;
 404                 rctx->md = p2;
 405                 return 1;
 406
 407                 default:
 408                 return -2;
 409
 410                 }
 411         }
 412
 413 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 414                         const char *type, const char *value)
 415         {
 416         if (!strcmp(type, "rsa_padding_mode"))
 417                 {
 418                 int pm;
 419                 if (!value)
 420                         return 0;
 421                 if (!strcmp(value, "pkcs1"))
 422                         pm = RSA_PKCS1_PADDING;
 423                 else if (!strcmp(value, "sslv23"))
 424                         pm = RSA_SSLV23_PADDING;
 425                 else if (!strcmp(value, "none"))
 426                         pm = RSA_NO_PADDING;
 427                 else if (!strcmp(value, "oeap"))
 428                         pm = RSA_PKCS1_OAEP_PADDING;
 429                 else if (!strcmp(value, "x931"))
 430                         pm = RSA_X931_PADDING;
 431                 else if (!strcmp(value, "pss"))
 432                         pm = RSA_PKCS1_PSS_PADDING;
 433                 else
 434                         return -2;
 435                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
 436                 }
 437
 438         if (!strcmp(type, "rsa_pss_saltlen"))
 439                 {
 440                 int saltlen;
 441                 saltlen = atoi(value);
 442                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
 443                 }
 444
 445         if (!strcmp(type, "rsa_keygen_bits"))
 446                 {
 447                 int nbits;
 448                 nbits = atoi(value);
 449                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
 450                 }
 451
 452         if (!strcmp(type, "rsa_keygen_pubexp"))
 453                 {
 454                 int ret;
 455                 BIGNUM *pubexp = NULL;
 456                 if (!BN_asc2bn(&pubexp, value))
 457                         return 0;
 458                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
 459                 if (ret <= 0)
 460                         BN_free(pubexp);
 461                 return ret;
 462                 }
 463
 464         return -2;
 465         }
 466
 467 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 468         {
 469         RSA *rsa = NULL;
 470         RSA_PKEY_CTX *rctx = ctx->data;
 471         BN_GENCB *pcb, cb;
 472         int ret;
 473         if (!rctx->pub_exp)
 474                 {
 475                 rctx->pub_exp = BN_new();
 476                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
 477                         return 0;
 478                 }
 479         rsa = RSA_new();
 480         if (!rsa)
 481                 return 0;
 482         if (ctx->pkey_gencb)
 483                 {
 484                 pcb = &cb;
 485                 evp_pkey_set_cb_translate(pcb, ctx);
 486                 }
 487         else
 488                 pcb = NULL;
 489         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
 490         if (ret > 0)
 491                 EVP_PKEY_assign_RSA(pkey, rsa);
 492         else
 493                 RSA_free(rsa);
 494         return ret;
 495         }
 496
 497 const EVP_PKEY_METHOD rsa_pkey_meth =
 498         {
 499         EVP_PKEY_RSA,
 500         0,
 501         pkey_rsa_init,
 502         pkey_rsa_cleanup,
 503
 504         0,0,
 505
 506         0,
 507         pkey_rsa_keygen,
 508
 509         0,
 510         pkey_rsa_sign,
 511
 512         0,
 513         pkey_rsa_verify,
 514
 515         0,
 516         pkey_rsa_verifyrecover,
 517
 518
 519         0,0,0,0,
 520
 521         0,
 522         pkey_rsa_encrypt,
 523
 524         0,
 525         pkey_rsa_decrypt,
 526
 527         0,0,
 528
 529         pkey_rsa_ctrl,
 530         pkey_rsa_ctrl_str
 531
 532
 533         };