Fix the error handling in EC_POINTs_mul

This was pointed out by a false-positive
-fsanitizer warning ;-)

However from the cryptographical POV the
code is wrong:
A point R^0 on the wrong curve
is infinity on the wrong curve.

[extended tests]

Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/11475)

diff --git a/crypto/ec/ec_lib.c b/crypto/ec/ec_lib.c
index 5540ec1bc22c1bd1a42abf9194714e37dfdae280..f90d8339141f1a1275791c089d24ef6503e222c4 100644 (file)
--- a/crypto/ec/ec_lib.c
+++ b/crypto/ec/ec_lib.c
@@ -1051,14 +1051,14 @@ int EC_POINTs_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar,
     BN_CTX *new_ctx = NULL;
 #endif
 
-    if ((scalar == NULL) && (num == 0)) {
-        return EC_POINT_set_to_infinity(group, r);
-    }
-
     if (!ec_point_is_compat(r, group)) {
         ECerr(EC_F_EC_POINTS_MUL, EC_R_INCOMPATIBLE_OBJECTS);
         return 0;
     }
+
+    if (scalar == NULL && num == 0)
+        return EC_POINT_set_to_infinity(group, r);
+
     for (i = 0; i < num; i++) {
         if (!ec_point_is_compat(points[i], group)) {
             ECerr(EC_F_EC_POINTS_MUL, EC_R_INCOMPATIBLE_OBJECTS);