networking/tls_rsa.c

   1 /*
   2  * Copyright (C) 2017 Denys Vlasenko
   3  *
   4  * Licensed under GPLv2, see file LICENSE in this source tree.
   5  */
   6 #include "tls.h"
   7
   8 #define pkcs1Pad(in, inlen, out, outlen, cryptType, userPtr) \
   9         pkcs1Pad(in, inlen, out, outlen, cryptType)
  10 static ///bbox
  11 int32 pkcs1Pad(unsigned char *in, uint32 inlen, unsigned char *out,
  12                                            uint32 outlen, int32 cryptType, void *userPtr)
  13 {
  14         unsigned char   *c;
  15         int32           randomLen;
  16
  17         randomLen = outlen - 3 - inlen;
  18         if (randomLen < 8) {
  19                 psTraceCrypto("pkcs1Pad failure\n");
  20                 return PS_LIMIT_FAIL;
  21         }
  22         c = out;
  23         *c = 0x00;
  24         c++;
  25         *c = (unsigned char)cryptType;
  26         c++;
  27         if (cryptType == PUBKEY_TYPE) {
  28                 while (randomLen-- > 0) {
  29                         *c++ = 0xFF;
  30                 }
  31         } else {
  32                 if (matrixCryptoGetPrngData(c, (uint32)randomLen, userPtr) < 0) {
  33                         return PS_PLATFORM_FAIL;
  34                 }
  35 /*
  36                 SECURITY:  Read through the random data and change all 0x0 to 0x01.
  37                 This is per spec that no random bytes should be 0
  38 */
  39                 while (randomLen-- > 0) {
  40                         if (*c == 0x0) {
  41                                 *c = 0x01;
  42                         }
  43                         c++;
  44                 }
  45         }
  46         *c = 0x00;
  47         c++;
  48         memcpy(c, in, inlen);
  49
  50         return outlen;
  51 }
  52
  53 #define psRsaCrypt(pool, in, inlen, out, outlen, key, type, data) \
  54         psRsaCrypt(pool, in, inlen, out, outlen, key, type)
  55 static ///bbox
  56 int32 psRsaCrypt(psPool_t *pool, const unsigned char *in, uint32 inlen,
  57                         unsigned char *out, uint32 *outlen,     psRsaKey_t *key, int32 type,
  58                         void *data)
  59 {
  60         pstm_int                tmp, tmpa, tmpb;
  61         int32                   res;
  62         uint32                  x;
  63
  64         if (in == NULL || out == NULL || outlen == NULL || key == NULL) {
  65                 psTraceCrypto("NULL parameter error in psRsaCrypt\n");
  66                 return PS_ARG_FAIL;
  67         }
  68
  69         tmp.dp = tmpa.dp = tmpb.dp = NULL;
  70
  71         /* Init and copy into tmp */
  72         if (pstm_init_for_read_unsigned_bin(pool, &tmp, inlen + sizeof(pstm_digit))
  73                         != PS_SUCCESS) {
  74                 return PS_FAILURE;
  75         }
  76         if (pstm_read_unsigned_bin(&tmp, (unsigned char *)in, inlen) != PS_SUCCESS){
  77                 pstm_clear(&tmp);
  78                 return PS_FAILURE;
  79         }
  80         /* Sanity check on the input */
  81         if (pstm_cmp(&key->N, &tmp) == PSTM_LT) {
  82                 res = PS_LIMIT_FAIL;
  83                 goto done;
  84         }
  85         if (type == PRIVKEY_TYPE) {
  86                 if (key->optimized) {
  87                         if (pstm_init_size(pool, &tmpa, key->p.alloc) != PS_SUCCESS) {
  88                                 res = PS_FAILURE;
  89                                 goto done;
  90                         }
  91                         if (pstm_init_size(pool, &tmpb, key->q.alloc) != PS_SUCCESS) {
  92                                 pstm_clear(&tmpa);
  93                                 res = PS_FAILURE;
  94                                 goto done;
  95                         }
  96                         if (pstm_exptmod(pool, &tmp, &key->dP, &key->p, &tmpa) !=
  97                                         PS_SUCCESS) {
  98                                 psTraceCrypto("decrypt error: pstm_exptmod dP, p\n");
  99                                 goto error;
 100                         }
 101                         if (pstm_exptmod(pool, &tmp, &key->dQ, &key->q, &tmpb) !=
 102                                         PS_SUCCESS) {
 103                                 psTraceCrypto("decrypt error: pstm_exptmod dQ, q\n");
 104                                 goto error;
 105                         }
 106                         if (pstm_sub(&tmpa, &tmpb, &tmp) != PS_SUCCESS) {
 107                                 psTraceCrypto("decrypt error: sub tmpb, tmp\n");
 108                                 goto error;
 109                         }
 110                         if (pstm_mulmod(pool, &tmp, &key->qP, &key->p, &tmp) != PS_SUCCESS) {
 111                                 psTraceCrypto("decrypt error: pstm_mulmod qP, p\n");
 112                                 goto error;
 113                         }
 114                         if (pstm_mul_comba(pool, &tmp, &key->q, &tmp, NULL, 0)
 115                                         != PS_SUCCESS){
 116                                 psTraceCrypto("decrypt error: pstm_mul q \n");
 117                                 goto error;
 118                         }
 119                         if (pstm_add(&tmp, &tmpb, &tmp) != PS_SUCCESS) {
 120                                 psTraceCrypto("decrypt error: pstm_add tmp \n");
 121                                 goto error;
 122                         }
 123                 } else {
 124                         if (pstm_exptmod(pool, &tmp, &key->d, &key->N, &tmp) !=
 125                                         PS_SUCCESS) {
 126                                 psTraceCrypto("psRsaCrypt error: pstm_exptmod\n");
 127                                 goto error;
 128                         }
 129                 }
 130         } else if (type == PUBKEY_TYPE) {
 131                 if (pstm_exptmod(pool, &tmp, &key->e, &key->N, &tmp) != PS_SUCCESS) {
 132                         psTraceCrypto("psRsaCrypt error: pstm_exptmod\n");
 133                         goto error;
 134                 }
 135         } else {
 136                 psTraceCrypto("psRsaCrypt error: invalid type param\n");
 137                 goto error;
 138         }
 139         /* Read it back */
 140         x = pstm_unsigned_bin_size(&key->N);
 141
 142         if ((uint32)x > *outlen) {
 143                 res = -1;
 144                 psTraceCrypto("psRsaCrypt error: pstm_unsigned_bin_size\n");
 145                 goto done;
 146         }
 147         /* We want the encrypted value to always be the key size.  Pad with 0x0 */
 148         while ((uint32)x < (unsigned long)key->size) {
 149                 *out++ = 0x0;
 150                 x++;
 151         }
 152
 153         *outlen = x;
 154         /* Convert it */
 155         memset(out, 0x0, x);
 156
 157         if (pstm_to_unsigned_bin(pool, &tmp, out+(x-pstm_unsigned_bin_size(&tmp)))
 158                         != PS_SUCCESS) {
 159                 psTraceCrypto("psRsaCrypt error: pstm_to_unsigned_bin\n");
 160                 goto error;
 161         }
 162         /* Clean up and return */
 163         res = PS_SUCCESS;
 164         goto done;
 165 error:
 166         res = PS_FAILURE;
 167 done:
 168         if (type == PRIVKEY_TYPE && key->optimized) {
 169                 pstm_clear_multi(&tmpa, &tmpb, NULL, NULL, NULL, NULL, NULL, NULL);
 170         }
 171         pstm_clear(&tmp);
 172         return res;
 173 }
 174
 175 int32 psRsaEncryptPub(psPool_t *pool, psRsaKey_t *key,
 176                                                 unsigned char *in, uint32 inlen,
 177                                                 unsigned char *out, uint32 outlen, void *data)
 178 {
 179         int32   err;
 180         uint32  size;
 181
 182         size = key->size;
 183         if (outlen < size) {
 184                 psTraceCrypto("Error on bad outlen parameter to psRsaEncryptPub\n");
 185                 return PS_ARG_FAIL;
 186         }
 187
 188         if ((err = pkcs1Pad(in, inlen, out, size, PRIVKEY_TYPE, data))
 189                         < PS_SUCCESS) {
 190                 psTraceCrypto("Error padding psRsaEncryptPub. Likely data too long\n");
 191                 return err;
 192         }
 193         if ((err = psRsaCrypt(pool, out, size, out, (uint32*)&outlen, key,
 194                         PUBKEY_TYPE, data)) < PS_SUCCESS) {
 195                 psTraceCrypto("Error performing psRsaEncryptPub\n");
 196                 return err;
 197         }
 198         if (outlen != size) {
 199                 psTraceCrypto("Encrypted size error in psRsaEncryptPub\n");
 200                 return PS_FAILURE;
 201         }
 202         return size;
 203 }