networking/tls_rsa.c

   1 /*
   2  * Copyright (C) 2017 Denys Vlasenko
   3  *
   4  * Licensed under GPLv2, see file LICENSE in this source tree.
   5  */
   6 #include "tls.h"
   7
   8 /* The code below is taken from parts of
   9  *  matrixssl-3-7-2b-open/crypto/pubkey/pkcs.c
  10  *  matrixssl-3-7-2b-open/crypto/pubkey/rsa.c
  11  * and (so far) almost not modified. Changes are flagged with ///bbox
  12  */
  13
  14 #define pkcs1Pad(in, inlen, out, outlen, cryptType, userPtr) \
  15         pkcs1Pad(in, inlen, out, outlen, cryptType)
  16 static ///bbox
  17 int32 pkcs1Pad(unsigned char *in, uint32 inlen, unsigned char *out,
  18                                            uint32 outlen, int32 cryptType, void *userPtr)
  19 {
  20         unsigned char   *c;
  21         int32           randomLen;
  22
  23         randomLen = outlen - 3 - inlen;
  24         if (randomLen < 8) {
  25                 psTraceCrypto("pkcs1Pad failure\n");
  26                 return PS_LIMIT_FAIL;
  27         }
  28         c = out;
  29         *c = 0x00;
  30         c++;
  31         *c = (unsigned char)cryptType;
  32         c++;
  33         if (cryptType == PUBKEY_TYPE) {
  34                 while (randomLen-- > 0) {
  35                         *c++ = 0xFF;
  36                 }
  37         } else {
  38                 if (matrixCryptoGetPrngData(c, (uint32)randomLen, userPtr) < 0) {
  39                         return PS_PLATFORM_FAIL;
  40                 }
  41 /*
  42                 SECURITY:  Read through the random data and change all 0x0 to 0x01.
  43                 This is per spec that no random bytes should be 0
  44 */
  45                 while (randomLen-- > 0) {
  46                         if (*c == 0x0) {
  47                                 *c = 0x01;
  48                         }
  49                         c++;
  50                 }
  51         }
  52         *c = 0x00;
  53         c++;
  54         memcpy(c, in, inlen);
  55
  56         return outlen;
  57 }
  58
  59 #define psRsaCrypt(pool, in, inlen, out, outlen, key, type, data) \
  60         psRsaCrypt(pool, in, inlen, out, outlen, key, type)
  61 static ///bbox
  62 int32 psRsaCrypt(psPool_t *pool, const unsigned char *in, uint32 inlen,
  63                         unsigned char *out, uint32 *outlen,     psRsaKey_t *key, int32 type,
  64                         void *data)
  65 {
  66         pstm_int                tmp, tmpa, tmpb;
  67         int32                   res;
  68         uint32                  x;
  69
  70         if (in == NULL || out == NULL || outlen == NULL || key == NULL) {
  71                 psTraceCrypto("NULL parameter error in psRsaCrypt\n");
  72                 return PS_ARG_FAIL;
  73         }
  74
  75         tmp.dp = tmpa.dp = tmpb.dp = NULL;
  76
  77         /* Init and copy into tmp */
  78         if (pstm_init_for_read_unsigned_bin(pool, &tmp, inlen + sizeof(pstm_digit))
  79                         != PS_SUCCESS) {
  80                 return PS_FAILURE;
  81         }
  82         if (pstm_read_unsigned_bin(&tmp, (unsigned char *)in, inlen) != PS_SUCCESS){
  83                 pstm_clear(&tmp);
  84                 return PS_FAILURE;
  85         }
  86         /* Sanity check on the input */
  87         if (pstm_cmp(&key->N, &tmp) == PSTM_LT) {
  88                 res = PS_LIMIT_FAIL;
  89                 goto done;
  90         }
  91         if (type == PRIVKEY_TYPE) {
  92                 if (key->optimized) {
  93                         if (pstm_init_size(pool, &tmpa, key->p.alloc) != PS_SUCCESS) {
  94                                 res = PS_FAILURE;
  95                                 goto done;
  96                         }
  97                         if (pstm_init_size(pool, &tmpb, key->q.alloc) != PS_SUCCESS) {
  98                                 pstm_clear(&tmpa);
  99                                 res = PS_FAILURE;
 100                                 goto done;
 101                         }
 102                         if (pstm_exptmod(pool, &tmp, &key->dP, &key->p, &tmpa) !=
 103                                         PS_SUCCESS) {
 104                                 psTraceCrypto("decrypt error: pstm_exptmod dP, p\n");
 105                                 goto error;
 106                         }
 107                         if (pstm_exptmod(pool, &tmp, &key->dQ, &key->q, &tmpb) !=
 108                                         PS_SUCCESS) {
 109                                 psTraceCrypto("decrypt error: pstm_exptmod dQ, q\n");
 110                                 goto error;
 111                         }
 112                         if (pstm_sub(&tmpa, &tmpb, &tmp) != PS_SUCCESS) {
 113                                 psTraceCrypto("decrypt error: sub tmpb, tmp\n");
 114                                 goto error;
 115                         }
 116                         if (pstm_mulmod(pool, &tmp, &key->qP, &key->p, &tmp) != PS_SUCCESS) {
 117                                 psTraceCrypto("decrypt error: pstm_mulmod qP, p\n");
 118                                 goto error;
 119                         }
 120                         if (pstm_mul_comba(pool, &tmp, &key->q, &tmp, NULL, 0)
 121                                         != PS_SUCCESS){
 122                                 psTraceCrypto("decrypt error: pstm_mul q \n");
 123                                 goto error;
 124                         }
 125                         if (pstm_add(&tmp, &tmpb, &tmp) != PS_SUCCESS) {
 126                                 psTraceCrypto("decrypt error: pstm_add tmp \n");
 127                                 goto error;
 128                         }
 129                 } else {
 130                         if (pstm_exptmod(pool, &tmp, &key->d, &key->N, &tmp) !=
 131                                         PS_SUCCESS) {
 132                                 psTraceCrypto("psRsaCrypt error: pstm_exptmod\n");
 133                                 goto error;
 134                         }
 135                 }
 136         } else if (type == PUBKEY_TYPE) {
 137                 if (pstm_exptmod(pool, &tmp, &key->e, &key->N, &tmp) != PS_SUCCESS) {
 138                         psTraceCrypto("psRsaCrypt error: pstm_exptmod\n");
 139                         goto error;
 140                 }
 141         } else {
 142                 psTraceCrypto("psRsaCrypt error: invalid type param\n");
 143                 goto error;
 144         }
 145         /* Read it back */
 146         x = pstm_unsigned_bin_size(&key->N);
 147
 148         if ((uint32)x > *outlen) {
 149                 res = -1;
 150                 psTraceCrypto("psRsaCrypt error: pstm_unsigned_bin_size\n");
 151                 goto done;
 152         }
 153         /* We want the encrypted value to always be the key size.  Pad with 0x0 */
 154         while ((uint32)x < (unsigned long)key->size) {
 155                 *out++ = 0x0;
 156                 x++;
 157         }
 158
 159         *outlen = x;
 160         /* Convert it */
 161         memset(out, 0x0, x);
 162
 163         if (pstm_to_unsigned_bin(pool, &tmp, out+(x-pstm_unsigned_bin_size(&tmp)))
 164                         != PS_SUCCESS) {
 165                 psTraceCrypto("psRsaCrypt error: pstm_to_unsigned_bin\n");
 166                 goto error;
 167         }
 168         /* Clean up and return */
 169         res = PS_SUCCESS;
 170         goto done;
 171 error:
 172         res = PS_FAILURE;
 173 done:
 174         if (type == PRIVKEY_TYPE && key->optimized) {
 175                 pstm_clear_multi(&tmpa, &tmpb, NULL, NULL, NULL, NULL, NULL, NULL);
 176         }
 177         pstm_clear(&tmp);
 178         return res;
 179 }
 180
 181 int32 psRsaEncryptPub(psPool_t *pool, psRsaKey_t *key,
 182                                                 unsigned char *in, uint32 inlen,
 183                                                 unsigned char *out, uint32 outlen, void *data)
 184 {
 185         int32   err;
 186         uint32  size;
 187
 188         size = key->size;
 189         if (outlen < size) {
 190                 psTraceCrypto("Error on bad outlen parameter to psRsaEncryptPub: outlen:%d < size:%d", outlen, size);
 191                 return PS_ARG_FAIL;
 192         }
 193
 194         if ((err = pkcs1Pad(in, inlen, out, size, PRIVKEY_TYPE, data))
 195                         < PS_SUCCESS) {
 196                 psTraceCrypto("Error padding psRsaEncryptPub. Likely data too long\n");
 197                 return err;
 198         }
 199         if ((err = psRsaCrypt(pool, out, size, out, (uint32*)&outlen, key,
 200                         PUBKEY_TYPE, data)) < PS_SUCCESS) {
 201                 psTraceCrypto("Error performing psRsaEncryptPub\n");
 202                 return err;
 203         }
 204         if (outlen != size) {
 205                 psTraceCrypto("Encrypted size error in psRsaEncryptPub\n");
 206                 return PS_FAILURE;
 207         }
 208         return size;
 209 }