From f4be878edae21755fe83ef26f3f3821a2fe58a97 Mon Sep 17 00:00:00 2001 From: Matt Caswell Date: Fri, 27 Apr 2018 17:36:11 +0100 Subject: [PATCH] Return an error from BN_mod_inverse if n is 1 (or -1) Calculating BN_mod_inverse where n is 1 (or -1) doesn't make sense. We should return an error in that case. Instead we were returning a valid result with value 0. Fixes #6004 Reviewed-by: Rich Salz (Merged from https://github.com/openssl/openssl/pull/6119) (cherry picked from commit b1860d6c71733314417d053a72af66ae72e8268e) --- crypto/bn/bn_gcd.c | 9 ++++++++- crypto/bn/bn_mont.c | 8 ++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/crypto/bn/bn_gcd.c b/crypto/bn/bn_gcd.c index 067642644e..fd1c7a2947 100644 --- a/crypto/bn/bn_gcd.c +++ b/crypto/bn/bn_gcd.c @@ -140,7 +140,14 @@ BIGNUM *int_bn_mod_inverse(BIGNUM *in, BIGNUM *ret = NULL; int sign; - if (pnoinv) + /* This is invalid input so we don't worry about constant time here */ + if (BN_abs_is_word(n, 1) || BN_is_zero(n)) { + if (pnoinv != NULL) + *pnoinv = 1; + return NULL; + } + + if (pnoinv != NULL) *pnoinv = 0; if ((BN_get_flags(a, BN_FLG_CONSTTIME) != 0) diff --git a/crypto/bn/bn_mont.c b/crypto/bn/bn_mont.c index faef581571..c0c174650c 100644 --- a/crypto/bn/bn_mont.c +++ b/crypto/bn/bn_mont.c @@ -278,7 +278,9 @@ int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx) if ((buf[1] = mod->top > 1 ? mod->d[1] : 0)) tmod.top = 2; - if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL) + if (BN_is_one(&tmod)) + BN_zero(Ri); + else if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL) goto err; if (!BN_lshift(Ri, Ri, 2 * BN_BITS2)) goto err; /* R*Ri */ @@ -311,7 +313,9 @@ int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx) buf[1] = 0; tmod.top = buf[0] != 0 ? 1 : 0; /* Ri = R^-1 mod N */ - if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL) + if (BN_is_one(&tmod)) + BN_zero(Ri); + else if ((BN_mod_inverse(Ri, R, &tmod, ctx)) == NULL) goto err; if (!BN_lshift(Ri, Ri, BN_BITS2)) goto err; /* R*Ri */ -- 2.25.1