networking/tls_rsa.c

   1 /*
   2  * Copyright (C) 2017 Denys Vlasenko
   3  *
   4  * Licensed under GPLv2, see file LICENSE in this source tree.
   5  */
   6 #include "tls.h"
   7
   8 /* The code below is taken from parts of
   9  *  matrixssl-3-7-2b-open/crypto/pubkey/pkcs.c
  10  *  matrixssl-3-7-2b-open/crypto/pubkey/rsa.c
  11  * and (so far) almost not modified. Changes are flagged with //bbox
  12  */
  13
  14 #define pkcs1Pad(in, inlen, out, outlen, cryptType, userPtr) \
  15         pkcs1Pad(in, inlen, out, outlen, cryptType)
  16 static //bbox
  17 int32 pkcs1Pad(unsigned char *in, uint32 inlen, unsigned char *out,
  18                                            uint32 outlen, int32 cryptType, void *userPtr)
  19 {
  20         unsigned char   *c;
  21         int32           randomLen;
  22
  23         randomLen = outlen - 3 - inlen;
  24         if (randomLen < 8) {
  25                 psTraceCrypto("pkcs1Pad failure\n");
  26                 return PS_LIMIT_FAIL;
  27         }
  28         c = out;
  29         *c = 0x00;
  30         c++;
  31         *c = (unsigned char)cryptType;
  32         c++;
  33         if (cryptType == PUBKEY_TYPE) {
  34                 while (randomLen-- > 0) {
  35                         *c++ = 0xFF;
  36                 }
  37         } else {
  38                 if (matrixCryptoGetPrngData(c, (uint32)randomLen, userPtr) < 0) {
  39                         return PS_PLATFORM_FAIL;
  40                 }
  41 /*
  42                 SECURITY:  Read through the random data and change all 0x0 to 0x01.
  43                 This is per spec that no random bytes should be 0
  44 */
  45                 while (randomLen-- > 0) {
  46                         if (*c == 0x0) {
  47                                 *c = 0x01;
  48                         }
  49                         c++;
  50                 }
  51         }
  52         *c = 0x00;
  53         c++;
  54         memcpy(c, in, inlen);
  55
  56         return outlen;
  57 }
  58
  59 #define psRsaCrypt(pool, in, inlen, out, outlen, key, type, data) \
  60         psRsaCrypt(      in, inlen, out, outlen, key, type)
  61 static //bbox
  62 int32 psRsaCrypt(psPool_t *pool, const unsigned char *in, uint32 inlen,
  63                         unsigned char *out, uint32 *outlen,     psRsaKey_t *key, int32 type,
  64                         void *data)
  65 {
  66         pstm_int                tmp, tmpa, tmpb;
  67         int32                   res;
  68         uint32                  x;
  69
  70 //bbox
  71 //      if (in == NULL || out == NULL || outlen == NULL || key == NULL) {
  72 //              psTraceCrypto("NULL parameter error in psRsaCrypt\n");
  73 //              return PS_ARG_FAIL;
  74 //      }
  75
  76         tmp.dp = tmpa.dp = tmpb.dp = NULL;
  77
  78         /* Init and copy into tmp */
  79         if (pstm_init_for_read_unsigned_bin(pool, &tmp, inlen + sizeof(pstm_digit))
  80                         != PS_SUCCESS) {
  81                 return PS_FAILURE;
  82         }
  83         if (pstm_read_unsigned_bin(&tmp, (unsigned char *)in, inlen) != PS_SUCCESS){
  84                 pstm_clear(&tmp);
  85                 return PS_FAILURE;
  86         }
  87         /* Sanity check on the input */
  88         if (pstm_cmp(&key->N, &tmp) == PSTM_LT) {
  89                 res = PS_LIMIT_FAIL;
  90                 goto done;
  91         }
  92         if (type == PRIVKEY_TYPE) {
  93                 if (key->optimized) {
  94                         if (pstm_init_size(pool, &tmpa, key->p.alloc) != PS_SUCCESS) {
  95                                 res = PS_FAILURE;
  96                                 goto done;
  97                         }
  98                         if (pstm_init_size(pool, &tmpb, key->q.alloc) != PS_SUCCESS) {
  99                                 pstm_clear(&tmpa);
 100                                 res = PS_FAILURE;
 101                                 goto done;
 102                         }
 103                         if (pstm_exptmod(pool, &tmp, &key->dP, &key->p, &tmpa) !=
 104                                         PS_SUCCESS) {
 105                                 psTraceCrypto("decrypt error: pstm_exptmod dP, p\n");
 106                                 goto error;
 107                         }
 108                         if (pstm_exptmod(pool, &tmp, &key->dQ, &key->q, &tmpb) !=
 109                                         PS_SUCCESS) {
 110                                 psTraceCrypto("decrypt error: pstm_exptmod dQ, q\n");
 111                                 goto error;
 112                         }
 113                         if (pstm_sub(&tmpa, &tmpb, &tmp) != PS_SUCCESS) {
 114                                 psTraceCrypto("decrypt error: sub tmpb, tmp\n");
 115                                 goto error;
 116                         }
 117                         if (pstm_mulmod(pool, &tmp, &key->qP, &key->p, &tmp) != PS_SUCCESS) {
 118                                 psTraceCrypto("decrypt error: pstm_mulmod qP, p\n");
 119                                 goto error;
 120                         }
 121                         if (pstm_mul_comba(pool, &tmp, &key->q, &tmp, NULL, 0)
 122                                         != PS_SUCCESS){
 123                                 psTraceCrypto("decrypt error: pstm_mul q \n");
 124                                 goto error;
 125                         }
 126                         if (pstm_add(&tmp, &tmpb, &tmp) != PS_SUCCESS) {
 127                                 psTraceCrypto("decrypt error: pstm_add tmp \n");
 128                                 goto error;
 129                         }
 130                 } else {
 131                         if (pstm_exptmod(pool, &tmp, &key->d, &key->N, &tmp) !=
 132                                         PS_SUCCESS) {
 133                                 psTraceCrypto("psRsaCrypt error: pstm_exptmod\n");
 134                                 goto error;
 135                         }
 136                 }
 137         } else if (type == PUBKEY_TYPE) {
 138                 if (pstm_exptmod(pool, &tmp, &key->e, &key->N, &tmp) != PS_SUCCESS) {
 139                         psTraceCrypto("psRsaCrypt error: pstm_exptmod\n");
 140                         goto error;
 141                 }
 142         } else {
 143                 psTraceCrypto("psRsaCrypt error: invalid type param\n");
 144                 goto error;
 145         }
 146         /* Read it back */
 147         x = pstm_unsigned_bin_size(&key->N);
 148
 149         if ((uint32)x > *outlen) {
 150                 res = -1;
 151                 psTraceCrypto("psRsaCrypt error: pstm_unsigned_bin_size\n");
 152                 goto done;
 153         }
 154         /* We want the encrypted value to always be the key size.  Pad with 0x0 */
 155         while ((uint32)x < (unsigned long)key->size) {
 156                 *out++ = 0x0;
 157                 x++;
 158         }
 159
 160         *outlen = x;
 161         /* Convert it */
 162         memset(out, 0x0, x);
 163
 164         if (pstm_to_unsigned_bin(pool, &tmp, out+(x-pstm_unsigned_bin_size(&tmp)))
 165                         != PS_SUCCESS) {
 166                 psTraceCrypto("psRsaCrypt error: pstm_to_unsigned_bin\n");
 167                 goto error;
 168         }
 169         /* Clean up and return */
 170         res = PS_SUCCESS;
 171         goto done;
 172 error:
 173         res = PS_FAILURE;
 174 done:
 175         if (type == PRIVKEY_TYPE && key->optimized) {
 176                 pstm_clear_multi(&tmpa, &tmpb, NULL, NULL, NULL, NULL, NULL, NULL);
 177         }
 178         pstm_clear(&tmp);
 179         return res;
 180 }
 181
 182 int32 psRsaEncryptPub(psPool_t *pool, psRsaKey_t *key,
 183                                                 unsigned char *in, uint32 inlen,
 184                                                 unsigned char *out, uint32 outlen, void *data)
 185 {
 186         int32   err;
 187         uint32  size;
 188
 189         size = key->size;
 190         if (outlen < size) {
 191 //bbox          psTraceCrypto("Error on bad outlen parameter to psRsaEncryptPub\n");
 192                 bb_error_msg_and_die("RSA crypt outlen:%d < size:%d", outlen, size);
 193                 return PS_ARG_FAIL;
 194         }
 195
 196         if ((err = pkcs1Pad(in, inlen, out, size, PRIVKEY_TYPE, data))
 197                         < PS_SUCCESS) {
 198                 psTraceCrypto("Error padding psRsaEncryptPub. Likely data too long\n");
 199                 return err;
 200         }
 201         if ((err = psRsaCrypt(pool, out, size, out, (uint32*)&outlen, key,
 202                         PUBKEY_TYPE, data)) < PS_SUCCESS) {
 203                 psTraceCrypto("Error performing psRsaEncryptPub\n");
 204                 return err;
 205         }
 206         if (outlen != size) {
 207                 psTraceCrypto("Encrypted size error in psRsaEncryptPub\n");
 208                 return PS_FAILURE;
 209         }
 210         return size;
 211 }