crypto/rsa/rsa_pmeth.c

   1 /* crypto/rsa/rsa_pmeth.c */
   2 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
   3  * project 2006.
   4  */
   5 /* ====================================================================
   6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  *
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  *
  20  * 3. All advertising materials mentioning features or use of this
  21  *    software must display the following acknowledgment:
  22  *    "This product includes software developed by the OpenSSL Project
  23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  24  *
  25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  26  *    endorse or promote products derived from this software without
  27  *    prior written permission. For written permission, please contact
  28  *    licensing@OpenSSL.org.
  29  *
  30  * 5. Products derived from this software may not be called "OpenSSL"
  31  *    nor may "OpenSSL" appear in their names without prior written
  32  *    permission of the OpenSSL Project.
  33  *
  34  * 6. Redistributions of any form whatsoever must retain the following
  35  *    acknowledgment:
  36  *    "This product includes software developed by the OpenSSL Project
  37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  38  *
  39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  50  * OF THE POSSIBILITY OF SUCH DAMAGE.
  51  * ====================================================================
  52  *
  53  * This product includes cryptographic software written by Eric Young
  54  * (eay@cryptsoft.com).  This product includes software written by Tim
  55  * Hudson (tjh@cryptsoft.com).
  56  *
  57  */
  58
  59 #include <stdio.h>
  60 #include "cryptlib.h"
  61 #include <openssl/asn1t.h>
  62 #include <openssl/x509.h>
  63 #include <openssl/rsa.h>
  64 #include <openssl/evp.h>
  65 #include "evp_locl.h"
  66 #include "rsa_locl.h"
  67
  68 /* RSA pkey context structure */
  69
  70 typedef struct
  71         {
  72         /* Key gen parameters */
  73         int nbits;
  74         BIGNUM *pub_exp;
  75         /* Keygen callback info */
  76         int gentmp[2];
  77         /* RSA padding mode */
  78         int pad_mode;
  79         /* message digest */
  80         const EVP_MD *md;
  81         /* PSS/OAEP salt length */
  82         int saltlen;
  83         /* Temp buffer */
  84         unsigned char *tbuf;
  85         } RSA_PKEY_CTX;
  86
  87 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
  88         {
  89         RSA_PKEY_CTX *rctx;
  90         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
  91         if (!rctx)
  92                 return 0;
  93         rctx->nbits = 1024;
  94         rctx->pub_exp = NULL;
  95         rctx->pad_mode = RSA_PKCS1_PADDING;
  96         rctx->md = NULL;
  97         rctx->tbuf = NULL;
  98
  99         rctx->saltlen = -2;
 100
 101         ctx->data = rctx;
 102         ctx->keygen_info = rctx->gentmp;
 103         ctx->keygen_info_count = 2;
 104
 105         return 1;
 106         }
 107
 108 static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
 109         {
 110         RSA_PKEY_CTX *dctx, *sctx;
 111         if (!pkey_rsa_init(dst))
 112                 return 0;
 113         sctx = src->data;
 114         dctx = dst->data;
 115         dctx->nbits = sctx->nbits;
 116         if (sctx->pub_exp)
 117                 {
 118                 dctx->pub_exp = BN_dup(sctx->pub_exp);
 119                 if (!dctx->pub_exp)
 120                         return 0;
 121                 }
 122         dctx->pad_mode = sctx->pad_mode;
 123         dctx->md = sctx->md;
 124         return 1;
 125         }
 126
 127 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
 128         {
 129         if (ctx->tbuf)
 130                 return 1;
 131         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
 132         if (!ctx->tbuf)
 133                 return 0;
 134         return 1;
 135         }
 136
 137 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 138         {
 139         RSA_PKEY_CTX *rctx = ctx->data;
 140         if (rctx)
 141                 {
 142                 if (rctx->pub_exp)
 143                         BN_free(rctx->pub_exp);
 144                 if (rctx->tbuf)
 145                         OPENSSL_free(rctx->tbuf);
 146                 OPENSSL_free(rctx);
 147                 }
 148         }
 149
 150 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
 151                                         const unsigned char *tbs, size_t tbslen)
 152         {
 153         int ret;
 154         RSA_PKEY_CTX *rctx = ctx->data;
 155         RSA *rsa = ctx->pkey->pkey.rsa;
 156
 157         if (rctx->md)
 158                 {
 159                 if (tbslen != (size_t)EVP_MD_size(rctx->md))
 160                         {
 161                         RSAerr(RSA_F_PKEY_RSA_SIGN,
 162                                         RSA_R_INVALID_DIGEST_LENGTH);
 163                         return -1;
 164                         }
 165                 if (rctx->pad_mode == RSA_X931_PADDING)
 166                         {
 167                         if (!setup_tbuf(rctx, ctx))
 168                                 return -1;
 169                         memcpy(rctx->tbuf, tbs, tbslen);
 170                         rctx->tbuf[tbslen] =
 171                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
 172                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
 173                                                 sig, rsa, RSA_X931_PADDING);
 174                         }
 175                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 176                         {
 177                         unsigned int sltmp;
 178                         ret = RSA_sign(EVP_MD_type(rctx->md),
 179                                                 tbs, tbslen, sig, &sltmp, rsa);
 180                         if (ret <= 0)
 181                                 return ret;
 182                         ret = sltmp;
 183                         }
 184                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 185                         {
 186                         if (!setup_tbuf(rctx, ctx))
 187                                 return -1;
 188                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
 189                                                 rctx->md, rctx->saltlen))
 190                                 return -1;
 191                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
 192                                                 sig, rsa, RSA_NO_PADDING);
 193                         }
 194                 else
 195                         return -1;
 196                 }
 197         else
 198                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 199                                                         rctx->pad_mode);
 200         if (ret < 0)
 201                 return ret;
 202         *siglen = ret;
 203         return 1;
 204         }
 205
 206
 207 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 208                                         unsigned char *rout, size_t *routlen,
 209                                         const unsigned char *sig, size_t siglen)
 210         {
 211         int ret;
 212         RSA_PKEY_CTX *rctx = ctx->data;
 213
 214         if (rctx->md)
 215                 {
 216                 if (rctx->pad_mode == RSA_X931_PADDING)
 217                         {
 218                         if (!setup_tbuf(rctx, ctx))
 219                                 return -1;
 220                         ret = RSA_public_decrypt(siglen, sig,
 221                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
 222                                                 RSA_X931_PADDING);
 223                         if (ret < 1)
 224                                 return 0;
 225                         ret--;
 226                         if (rctx->tbuf[ret] !=
 227                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
 228                                 {
 229                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 230                                                 RSA_R_ALGORITHM_MISMATCH);
 231                                 return 0;
 232                                 }
 233                         if (ret != EVP_MD_size(rctx->md))
 234                                 {
 235                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 236                                         RSA_R_INVALID_DIGEST_LENGTH);
 237                                 return 0;
 238                                 }
 239                         if (rout)
 240                                 memcpy(rout, rctx->tbuf, ret);
 241                         }
 242                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 243                         {
 244                         size_t sltmp;
 245                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
 246                                                 NULL, 0, rout, &sltmp,
 247                                         sig, siglen, ctx->pkey->pkey.rsa);
 248                         ret = sltmp;
 249                         }
 250                 else
 251                         return -1;
 252                 }
 253         else
 254                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
 255                                                         rctx->pad_mode);
 256         if (ret < 0)
 257                 return ret;
 258         *routlen = ret;
 259         return 1;
 260         }
 261
 262 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
 263                                         const unsigned char *sig, size_t siglen,
 264                                         const unsigned char *tbs, size_t tbslen)
 265         {
 266         RSA_PKEY_CTX *rctx = ctx->data;
 267         RSA *rsa = ctx->pkey->pkey.rsa;
 268         size_t rslen;
 269         if (rctx->md)
 270                 {
 271                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
 272                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
 273                                         sig, siglen, rsa);
 274                 if (rctx->pad_mode == RSA_X931_PADDING)
 275                         {
 276                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
 277                                         sig, siglen) <= 0)
 278                                 return 0;
 279                         }
 280                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 281                         {
 282                         int ret;
 283                         if (!setup_tbuf(rctx, ctx))
 284                                 return -1;
 285                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 286                                                         rsa, RSA_NO_PADDING);
 287                         if (ret <= 0)
 288                                 return 0;
 289                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
 290                                                 rctx->tbuf, rctx->saltlen);
 291                         if (ret <= 0)
 292                                 return 0;
 293                         return 1;
 294                         }
 295                 else
 296                         return -1;
 297                 }
 298         else
 299                 {
 300                 if (!setup_tbuf(rctx, ctx))
 301                         return -1;
 302                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 303                                                 rsa, rctx->pad_mode);
 304                 if (rslen == 0)
 305                         return 0;
 306                 }
 307
 308         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
 309                 return 0;
 310
 311         return 1;
 312
 313         }
 314
 315
 316 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
 317                                         unsigned char *out, size_t *outlen,
 318                                         const unsigned char *in, size_t inlen)
 319         {
 320         int ret;
 321         RSA_PKEY_CTX *rctx = ctx->data;
 322         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 323                                                         rctx->pad_mode);
 324         if (ret < 0)
 325                 return ret;
 326         *outlen = ret;
 327         return 1;
 328         }
 329
 330 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
 331                                         unsigned char *out, size_t *outlen,
 332                                         const unsigned char *in, size_t inlen)
 333         {
 334         int ret;
 335         RSA_PKEY_CTX *rctx = ctx->data;
 336         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 337                                                         rctx->pad_mode);
 338         if (ret < 0)
 339                 return ret;
 340         *outlen = ret;
 341         return 1;
 342         }
 343
 344 static int check_padding_md(const EVP_MD *md, int padding)
 345         {
 346         if (!md)
 347                 return 1;
 348
 349         if (padding == RSA_NO_PADDING)
 350                 {
 351                 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
 352                 return 0;
 353                 }
 354
 355         if (padding == RSA_X931_PADDING)
 356                 {
 357                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
 358                         {
 359                         RSAerr(RSA_F_CHECK_PADDING_MD,
 360                                                 RSA_R_INVALID_X931_DIGEST);
 361                         return 0;
 362                         }
 363                 return 1;
 364                 }
 365
 366         return 1;
 367         }
 368
 369
 370 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 371         {
 372         RSA_PKEY_CTX *rctx = ctx->data;
 373         switch (type)
 374                 {
 375                 case EVP_PKEY_CTRL_RSA_PADDING:
 376                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
 377                         {
 378                         if (!check_padding_md(rctx->md, p1))
 379                                 return 0;
 380                         if (p1 == RSA_PKCS1_PSS_PADDING)
 381                                 {
 382                                 if (!(ctx->operation &
 383                                      (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
 384                                         goto bad_pad;
 385                                 if (!rctx->md)
 386                                         rctx->md = EVP_sha1();
 387                                 }
 388                         if (p1 == RSA_PKCS1_OAEP_PADDING)
 389                                 {
 390                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
 391                                         goto bad_pad;
 392                                 if (!rctx->md)
 393                                         rctx->md = EVP_sha1();
 394                                 }
 395                         rctx->pad_mode = p1;
 396                         return 1;
 397                         }
 398                 bad_pad:
 399                 RSAerr(RSA_F_PKEY_RSA_CTRL,
 400                                 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
 401                 return -2;
 402
 403                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
 404                 if (p1 < -2)
 405                         return -2;
 406                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
 407                         {
 408                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
 409                         return -2;
 410                         }
 411                 rctx->saltlen = p1;
 412                 return 1;
 413
 414                 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
 415                 if (p1 < 256)
 416                         {
 417                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
 418                         return -2;
 419                         }
 420                 rctx->nbits = p1;
 421                 return 1;
 422
 423                 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
 424                 if (!p2)
 425                         return -2;
 426                 rctx->pub_exp = p2;
 427                 return 1;
 428
 429                 case EVP_PKEY_CTRL_MD:
 430                 if (!check_padding_md(p2, rctx->pad_mode))
 431                         return 0;
 432                 rctx->md = p2;
 433                 return 1;
 434
 435                 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
 436                 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
 437                 case EVP_PKEY_CTRL_PKCS7_SIGN:
 438                 return 1;
 439
 440                 default:
 441                 return -2;
 442
 443                 }
 444         }
 445
 446 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 447                         const char *type, const char *value)
 448         {
 449         if (!value)
 450                 {
 451                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
 452                 return 0;
 453                 }
 454         if (!strcmp(type, "rsa_padding_mode"))
 455                 {
 456                 int pm;
 457                 if (!strcmp(value, "pkcs1"))
 458                         pm = RSA_PKCS1_PADDING;
 459                 else if (!strcmp(value, "sslv23"))
 460                         pm = RSA_SSLV23_PADDING;
 461                 else if (!strcmp(value, "none"))
 462                         pm = RSA_NO_PADDING;
 463                 else if (!strcmp(value, "oeap"))
 464                         pm = RSA_PKCS1_OAEP_PADDING;
 465                 else if (!strcmp(value, "x931"))
 466                         pm = RSA_X931_PADDING;
 467                 else if (!strcmp(value, "pss"))
 468                         pm = RSA_PKCS1_PSS_PADDING;
 469                 else
 470                         {
 471                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
 472                                                 RSA_R_UNKNOWN_PADDING_TYPE);
 473                         return -2;
 474                         }
 475                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
 476                 }
 477
 478         if (!strcmp(type, "rsa_pss_saltlen"))
 479                 {
 480                 int saltlen;
 481                 saltlen = atoi(value);
 482                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
 483                 }
 484
 485         if (!strcmp(type, "rsa_keygen_bits"))
 486                 {
 487                 int nbits;
 488                 nbits = atoi(value);
 489                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
 490                 }
 491
 492         if (!strcmp(type, "rsa_keygen_pubexp"))
 493                 {
 494                 int ret;
 495                 BIGNUM *pubexp = NULL;
 496                 if (!BN_asc2bn(&pubexp, value))
 497                         return 0;
 498                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
 499                 if (ret <= 0)
 500                         BN_free(pubexp);
 501                 return ret;
 502                 }
 503
 504         return -2;
 505         }
 506
 507 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 508         {
 509         RSA *rsa = NULL;
 510         RSA_PKEY_CTX *rctx = ctx->data;
 511         BN_GENCB *pcb, cb;
 512         int ret;
 513         if (!rctx->pub_exp)
 514                 {
 515                 rctx->pub_exp = BN_new();
 516                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
 517                         return 0;
 518                 }
 519         rsa = RSA_new();
 520         if (!rsa)
 521                 return 0;
 522         if (ctx->pkey_gencb)
 523                 {
 524                 pcb = &cb;
 525                 evp_pkey_set_cb_translate(pcb, ctx);
 526                 }
 527         else
 528                 pcb = NULL;
 529         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
 530         if (ret > 0)
 531                 EVP_PKEY_assign_RSA(pkey, rsa);
 532         else
 533                 RSA_free(rsa);
 534         return ret;
 535         }
 536
 537 const EVP_PKEY_METHOD rsa_pkey_meth =
 538         {
 539         EVP_PKEY_RSA,
 540         EVP_PKEY_FLAG_AUTOARGLEN,
 541         pkey_rsa_init,
 542         pkey_rsa_copy,
 543         pkey_rsa_cleanup,
 544
 545         0,0,
 546
 547         0,
 548         pkey_rsa_keygen,
 549
 550         0,
 551         pkey_rsa_sign,
 552
 553         0,
 554         pkey_rsa_verify,
 555
 556         0,
 557         pkey_rsa_verifyrecover,
 558
 559
 560         0,0,0,0,
 561
 562         0,
 563         pkey_rsa_encrypt,
 564
 565         0,
 566         pkey_rsa_decrypt,
 567
 568         0,0,
 569
 570         pkey_rsa_ctrl,
 571         pkey_rsa_ctrl_str
 572
 573
 574         };