Use size_t for new crypto size parameters.
[oweals/openssl.git] / crypto / rsa / rsa_pmeth.c
1 /* crypto/rsa/rsa_pmeth.c */
2 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer. 
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58
59 #include <stdio.h>
60 #include "cryptlib.h"
61 #include <openssl/asn1t.h>
62 #include <openssl/x509.h>
63 #include <openssl/rsa.h>
64 #include <openssl/evp.h>
65 #include "evp_locl.h"
66
67 extern int int_rsa_verify(int dtype, const unsigned char *m, size_t m_len,
68                 unsigned char *rm, size_t *prm_len,
69                 const unsigned char *sigbuf, size_t siglen,
70                 RSA *rsa);
71
72 /* RSA pkey context structure */
73
74 typedef struct
75         {
76         /* Key gen parameters */
77         int nbits;
78         BIGNUM *pub_exp;
79         /* Keygen callback info */
80         int gentmp[2];
81         /* RSA padding mode */
82         int pad_mode;
83         /* message digest */
84         const EVP_MD *md;
85         /* PSS/OAEP salt length */
86         int saltlen;
87         /* Temp buffer */
88         unsigned char *tbuf;
89         } RSA_PKEY_CTX;
90
91 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
92         {
93         RSA_PKEY_CTX *rctx;
94         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
95         if (!rctx)
96                 return 0;
97         rctx->nbits = 1024;
98         rctx->pub_exp = NULL;
99         rctx->pad_mode = RSA_PKCS1_PADDING;
100         rctx->md = NULL;
101         rctx->tbuf = NULL;
102
103         rctx->saltlen = -2;
104
105         ctx->data = rctx;
106         ctx->keygen_info = rctx->gentmp;
107         ctx->keygen_info_count = 2;
108         
109         return 1;
110         }
111
112 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
113         {
114         if (ctx->tbuf)
115                 return 1;
116         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
117         if (!ctx->tbuf)
118                 return 0;
119         return 1;
120         }
121
122 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
123         {
124         RSA_PKEY_CTX *rctx = ctx->data;
125         if (rctx)
126                 {
127                 if (rctx->pub_exp)
128                         BN_free(rctx->pub_exp);
129                 if (rctx->tbuf)
130                         OPENSSL_free(rctx->tbuf);
131                 OPENSSL_free(rctx);
132                 }
133         }
134
135 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
136                                         const unsigned char *tbs, size_t tbslen)
137         {
138         int ret;
139         RSA_PKEY_CTX *rctx = ctx->data;
140         RSA *rsa = ctx->pkey->pkey.rsa;
141
142         if (rctx->md)
143                 {
144                 if (tbslen != (size_t)EVP_MD_size(rctx->md))
145                         {
146                         RSAerr(RSA_F_PKEY_RSA_SIGN,
147                                         RSA_R_INVALID_DIGEST_LENGTH);
148                         return -1;
149                         }
150                 if (rctx->pad_mode == RSA_X931_PADDING)
151                         {
152                         if (!setup_tbuf(rctx, ctx))
153                                 return -1;
154                         memcpy(rctx->tbuf, tbs, tbslen);
155                         rctx->tbuf[tbslen] =
156                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
157                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
158                                                 sig, rsa, RSA_X931_PADDING);
159                         }
160                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
161                         {
162                         unsigned int sltmp;
163                         ret = RSA_sign(EVP_MD_type(rctx->md),
164                                                 tbs, tbslen, sig, &sltmp, rsa);
165                         if (ret <= 0)
166                                 return ret;
167                         ret = sltmp;
168                         }
169                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
170                         {
171                         if (!setup_tbuf(rctx, ctx))
172                                 return -1;
173                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
174                                                 rctx->md, rctx->saltlen))
175                                 return -1;
176                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
177                                                 sig, rsa, RSA_NO_PADDING);
178                         }
179                 else
180                         return -1;
181                 }
182         else
183                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
184                                                         rctx->pad_mode);
185         if (ret < 0)
186                 return ret;
187         *siglen = ret;
188         return 1;
189         }
190
191
192 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
193                                         unsigned char *rout, size_t *routlen,
194                                         const unsigned char *sig, size_t siglen)
195         {
196         int ret;
197         RSA_PKEY_CTX *rctx = ctx->data;
198
199         if (rctx->md)
200                 {
201                 if (rctx->pad_mode == RSA_X931_PADDING)
202                         {
203                         if (!setup_tbuf(rctx, ctx))
204                                 return -1;
205                         ret = RSA_public_decrypt(siglen, sig,
206                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
207                                                 RSA_X931_PADDING);
208                         if (ret < 1)
209                                 return 0;
210                         ret--;
211                         if (rctx->tbuf[ret] !=
212                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
213                                 {
214                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
215                                                 RSA_R_ALGORITHM_MISMATCH);
216                                 return 0;
217                                 }
218                         if (ret != EVP_MD_size(rctx->md))
219                                 {
220                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
221                                         RSA_R_INVALID_DIGEST_LENGTH);
222                                 return 0;
223                                 }
224                         if (rout)
225                                 memcpy(rout, rctx->tbuf, ret);
226                         }
227                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
228                         {
229                         unsigned int sltmp;
230                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
231                                                 NULL, 0, rout, &sltmp,
232                                         sig, siglen, ctx->pkey->pkey.rsa);
233                         ret = sltmp;
234                         }
235                 else
236                         return -1;
237                 }
238         else
239                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
240                                                         rctx->pad_mode);
241         if (ret < 0)
242                 return ret;
243         *routlen = ret;
244         return 1;
245         }
246
247 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
248                                         const unsigned char *sig, size_t siglen,
249                                         const unsigned char *tbs, size_t tbslen)
250         {
251         RSA_PKEY_CTX *rctx = ctx->data;
252         RSA *rsa = ctx->pkey->pkey.rsa;
253         size_t rslen;
254         if (rctx->md)
255                 {
256                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
257                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
258                                         sig, siglen, rsa);
259                 if (rctx->pad_mode == RSA_X931_PADDING)
260                         {
261                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
262                                         sig, siglen) <= 0)
263                                 return 0;
264                         }
265                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
266                         {
267                         int ret;
268                         if (!setup_tbuf(rctx, ctx))
269                                 return -1;
270                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
271                                                         rsa, RSA_NO_PADDING);
272                         if (ret <= 0)
273                                 return 0;
274                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
275                                                 rctx->tbuf, rctx->saltlen);
276                         if (ret <= 0)
277                                 return 0;
278                         return 1;
279                         }
280                 else
281                         return -1;
282                 }
283         else
284                 {
285                 if (!setup_tbuf(rctx, ctx))
286                         return -1;
287                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
288                                                 rsa, rctx->pad_mode);
289                 if (rslen <= 0)
290                         return 0;
291                 }
292
293         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
294                 return 0;
295
296         return 1;
297                         
298         }
299         
300
301 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
302                                         unsigned char *out, size_t *outlen,
303                                         const unsigned char *in, size_t inlen)
304         {
305         int ret;
306         RSA_PKEY_CTX *rctx = ctx->data;
307         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
308                                                         rctx->pad_mode);
309         if (ret < 0)
310                 return ret;
311         *outlen = ret;
312         return 1;
313         }
314
315 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
316                                         unsigned char *out, size_t *outlen,
317                                         const unsigned char *in, size_t inlen)
318         {
319         int ret;
320         RSA_PKEY_CTX *rctx = ctx->data;
321         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
322                                                         rctx->pad_mode);
323         if (ret < 0)
324                 return ret;
325         *outlen = ret;
326         return 1;
327         }
328
329 static int check_padding_md(const EVP_MD *md, int padding)
330         {
331         if (!md)
332                 return 1;
333
334         if (padding == RSA_NO_PADDING)
335                 {
336                 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
337                 return 0;
338                 }
339
340         if (padding == RSA_X931_PADDING)
341                 {
342                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
343                         {
344                         RSAerr(RSA_F_CHECK_PADDING_MD,
345                                                 RSA_R_INVALID_X931_DIGEST);
346                         return 0;
347                         }
348                 return 1;
349                 }
350
351         return 1;
352         }
353                         
354
355 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
356         {
357         RSA_PKEY_CTX *rctx = ctx->data;
358         switch (type)
359                 {
360                 case EVP_PKEY_CTRL_RSA_PADDING:
361                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
362                         {
363                         if (!check_padding_md(rctx->md, p1))
364                                 return 0;
365                         if (p1 == RSA_PKCS1_PSS_PADDING) 
366                                 {
367                                 if (!(ctx->operation &
368                                      (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
369                                         goto bad_pad;
370                                 if (!rctx->md)
371                                         rctx->md = EVP_sha1();
372                                 }
373                         if (p1 == RSA_PKCS1_OAEP_PADDING) 
374                                 {
375                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
376                                         goto bad_pad;
377                                 if (!rctx->md)
378                                         rctx->md = EVP_sha1();
379                                 }
380                         rctx->pad_mode = p1;
381                         return 1;
382                         }
383                 bad_pad:
384                 RSAerr(RSA_F_PKEY_RSA_CTRL,
385                                 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
386                 return -2;
387
388                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
389                 if (p1 < -2)
390                         return -2;
391                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
392                         {
393                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
394                         return -2;
395                         }
396                 rctx->saltlen = p1;
397                 return 1;
398
399                 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
400                 if (p1 < 256)
401                         {
402                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
403                         return -2;
404                         }
405                 rctx->nbits = p1;
406                 return 1;
407
408                 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
409                 if (!p2)
410                         return -2;
411                 rctx->pub_exp = p2;
412                 return 1;
413
414                 case EVP_PKEY_CTRL_MD:
415                 if (!check_padding_md(p2, rctx->pad_mode))
416                         return 0;
417                 rctx->md = p2;
418                 return 1;
419
420                 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
421                 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
422                 return 1;
423
424                 default:
425                 return -2;
426
427                 }
428         }
429                         
430 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
431                         const char *type, const char *value)
432         {
433         if (!value)
434                 {
435                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
436                 return 0;
437                 }
438         if (!strcmp(type, "rsa_padding_mode"))
439                 {
440                 int pm;
441                 if (!strcmp(value, "pkcs1"))
442                         pm = RSA_PKCS1_PADDING;
443                 else if (!strcmp(value, "sslv23"))
444                         pm = RSA_SSLV23_PADDING;
445                 else if (!strcmp(value, "none"))
446                         pm = RSA_NO_PADDING;
447                 else if (!strcmp(value, "oeap"))
448                         pm = RSA_PKCS1_OAEP_PADDING;
449                 else if (!strcmp(value, "x931"))
450                         pm = RSA_X931_PADDING;
451                 else if (!strcmp(value, "pss"))
452                         pm = RSA_PKCS1_PSS_PADDING;
453                 else
454                         {
455                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
456                                                 RSA_R_UNKNOWN_PADDING_TYPE);
457                         return -2;
458                         }
459                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
460                 }
461
462         if (!strcmp(type, "rsa_pss_saltlen"))
463                 {
464                 int saltlen;
465                 saltlen = atoi(value);
466                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
467                 }
468
469         if (!strcmp(type, "rsa_keygen_bits"))
470                 {
471                 int nbits;
472                 nbits = atoi(value);
473                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
474                 }
475
476         if (!strcmp(type, "rsa_keygen_pubexp"))
477                 {
478                 int ret;
479                 BIGNUM *pubexp = NULL;
480                 if (!BN_asc2bn(&pubexp, value))
481                         return 0;
482                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
483                 if (ret <= 0)
484                         BN_free(pubexp);
485                 return ret;
486                 }
487
488         return -2;
489         }
490
491 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
492         {
493         RSA *rsa = NULL;
494         RSA_PKEY_CTX *rctx = ctx->data;
495         BN_GENCB *pcb, cb;
496         int ret;
497         if (!rctx->pub_exp)
498                 {
499                 rctx->pub_exp = BN_new();
500                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
501                         return 0;
502                 }
503         rsa = RSA_new();
504         if (!rsa)
505                 return 0;
506         if (ctx->pkey_gencb)
507                 {
508                 pcb = &cb;
509                 evp_pkey_set_cb_translate(pcb, ctx);
510                 }
511         else
512                 pcb = NULL;
513         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
514         if (ret > 0)
515                 EVP_PKEY_assign_RSA(pkey, rsa);
516         else
517                 RSA_free(rsa);
518         return ret;
519         }
520
521 const EVP_PKEY_METHOD rsa_pkey_meth = 
522         {
523         EVP_PKEY_RSA,
524         EVP_PKEY_FLAG_AUTOARGLEN,
525         pkey_rsa_init,
526         pkey_rsa_cleanup,
527
528         0,0,
529
530         0,
531         pkey_rsa_keygen,
532
533         0,
534         pkey_rsa_sign,
535
536         0,
537         pkey_rsa_verify,
538
539         0,
540         pkey_rsa_verifyrecover,
541
542
543         0,0,0,0,
544
545         0,
546         pkey_rsa_encrypt,
547
548         0,
549         pkey_rsa_decrypt,
550
551         0,0,
552
553         pkey_rsa_ctrl,
554         pkey_rsa_ctrl_str
555
556
557         };