Fix calling convention bug in ecp_nistz256_ord_sqr_mont

The rep parameter takes an int in C, but the assembly implementation
looks at the upper bits. While it's unlikely to happen here, where all
calls pass a constant, in other scenarios x86_64 compilers will leave
arbitrary values in the upper half.

Fix this by making the C prototype match the assembly. (This aspect of
the calling convention implies smaller-than-word arguments in assembly
functions should be avoided. There are far fewer things to test if
everything consistently takes word-sized arguments.)

This was found as part of ABI testing work in BoringSSL.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/8108)

diff --git a/crypto/ec/asm/ecp_nistz256-armv8.pl b/crypto/ec/asm/ecp_nistz256-armv8.pl
index 8914f1a619dafe060f8fbd6df21b9ba738f68324..4daa8cc026b78f9f6f469ed5b226b00cdffc10d6 100644 (file)
--- a/crypto/ec/asm/ecp_nistz256-armv8.pl
+++ b/crypto/ec/asm/ecp_nistz256-armv8.pl
@@ -1488,7 +1488,7 @@ $code.=<<___;
 
 ////////////////////////////////////////////////////////////////////////
 // void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
-//                                int rep);
+//                                uint64_t rep);
 .globl ecp_nistz256_ord_sqr_mont
 .type  ecp_nistz256_ord_sqr_mont,%function
 .align 4

diff --git a/crypto/ec/asm/ecp_nistz256-ppc64.pl b/crypto/ec/asm/ecp_nistz256-ppc64.pl
index b1cd190c151461a00ba3466ade1a705c5f594dc1..c06a7c0d02f9929c1c6b67282bceb734ca8c1b21 100755 (executable)
--- a/crypto/ec/asm/ecp_nistz256-ppc64.pl
+++ b/crypto/ec/asm/ecp_nistz256-ppc64.pl
@@ -1919,7 +1919,7 @@ $code.=<<___;
 
 ################################################################################
 # void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
-#                                int rep);
+#                                uint64_t rep);
 .globl ecp_nistz256_ord_sqr_mont
 .align 5
 ecp_nistz256_ord_sqr_mont:

diff --git a/crypto/ec/asm/ecp_nistz256-x86_64.pl b/crypto/ec/asm/ecp_nistz256-x86_64.pl
index a28ee8e94770439f39d55a1c0de5f64c7e5de39b..e1e23ca90a05cb372517c2069c5b536464bd8b6f 100755 (executable)
--- a/crypto/ec/asm/ecp_nistz256-x86_64.pl
+++ b/crypto/ec/asm/ecp_nistz256-x86_64.pl
@@ -826,7 +826,7 @@ $code.=<<___;
 # void ecp_nistz256_ord_sqr_mont(
 #   uint64_t res[4],
 #   uint64_t a[4],
-#   int rep);
+#   uint64_t rep);
 
 .globl ecp_nistz256_ord_sqr_mont
 .type  ecp_nistz256_ord_sqr_mont,\@function,3

diff --git a/crypto/ec/ecp_nistz256.c b/crypto/ec/ecp_nistz256.c
index 6a64bc4f2a25882a48991aae374e1a7a2e8eafbb..66bf4ecb1e3332761787bcf2a391acc8ed13a416 100644 (file)
--- a/crypto/ec/ecp_nistz256.c
+++ b/crypto/ec/ecp_nistz256.c
@@ -1467,7 +1467,7 @@ void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS],
                                const BN_ULONG b[P256_LIMBS]);
 void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS],
                                const BN_ULONG a[P256_LIMBS],
-                               int rep);
+                               BN_ULONG rep);
 
 static int ecp_nistz256_inv_mod_ord(const EC_GROUP *group, BIGNUM *r,
                                     const BIGNUM *x, BN_CTX *ctx)