From 87954638a691b3188ae4bd07f2612b534055da78 Mon Sep 17 00:00:00 2001
From: Andy Polyakov <appro@openssl.org>
Date: Thu, 3 Oct 2013 00:30:12 +0200
Subject: [PATCH] rsaz-x86_64.pl: add MULX/ADCX/ADOX code path.

---
 crypto/bn/asm/rsaz-x86_64.pl | 803 ++++++++++++++++++++++++++---------
 1 file changed, 611 insertions(+), 192 deletions(-)

diff --git a/crypto/bn/asm/rsaz-x86_64.pl b/crypto/bn/asm/rsaz-x86_64.pl
index f6adb30b7b..dfcc7df4ee 100755
--- a/crypto/bn/asm/rsaz-x86_64.pl
+++ b/crypto/bn/asm/rsaz-x86_64.pl
@@ -70,8 +70,7 @@
 #
 # (*)	rsax engine and fips numbers are presented for reference
 #	purposes;
-# (**)	you might notice MULX code below, strangely enough gain is
-#	marginal, which is why code remains disabled;
+# (**)	MULX was attempted, but found to give only marginal improvement;
 
 $flavour = shift;
 $output  = shift;
@@ -87,6 +86,21 @@ die "can't locate x86_64-xlate.pl";
 open OUT,"| $^X $xlate $flavour $output";
 *STDOUT=*OUT;
 
+if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
+		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
+	$addx = ($1>=2.22);
+}
+
+if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
+	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
+	$addx = ($1>=2.10);
+}
+
+if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
+	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {
+	$addx = ($1>=11);
+}
+
 ($out, $inp, $mod) = ("%rdi", "%rsi", "%rbp");	# common internal API
 {
 my ($out,$inp,$mod,$n0,$times) = ("%rdi","%rsi","%rdx","%rcx","%r8d");
@@ -94,6 +108,8 @@ my ($out,$inp,$mod,$n0,$times) = ("%rdi","%rsi","%rdx","%rcx","%r8d");
 $code.=<<___;
 .text
 
+.extern	OPENSSL_ia32cap_P
+
 .globl	rsaz_512_sqr
 .type	rsaz_512_sqr,\@function,4
 .align	32
@@ -111,14 +127,19 @@ rsaz_512_sqr:				# 25-29% faster than rsaz_512_mul
 	movq	($inp), %rdx
 	movq	8($inp), %rax
 	movq	$n0, 128(%rsp)
+___
+$code.=<<___ if ($addx);
+	movl	\$0x80100,%r11d
+	andl	OPENSSL_ia32cap_P+8(%rip),%r11d
+	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX
+	je	.Loop_sqrx
+___
+$code.=<<___;
 	jmp	.Loop_sqr
 
 .align	32
 .Loop_sqr:
 	movl	$times,128+8(%rsp)
-___
-if (1) {
-$code.=<<___;
 #first iteration
 	movq	%rdx, %rbx
 	mulq	%rdx
@@ -446,242 +467,278 @@ $code.=<<___;
 
 	movq	%r13, 112(%rsp)
 	movq	%r14, 120(%rsp)
+
+	movq	(%rsp), %r8
+	movq	8(%rsp), %r9
+	movq	16(%rsp), %r10
+	movq	24(%rsp), %r11
+	movq	32(%rsp), %r12
+	movq	40(%rsp), %r13
+	movq	48(%rsp), %r14
+	movq	56(%rsp), %r15
+
+	call	__rsaz_512_reduce
+
+	addq	64(%rsp), %r8
+	adcq	72(%rsp), %r9
+	adcq	80(%rsp), %r10
+	adcq	88(%rsp), %r11
+	adcq	96(%rsp), %r12
+	adcq	104(%rsp), %r13
+	adcq	112(%rsp), %r14
+	adcq	120(%rsp), %r15
+	sbbq	%rcx, %rcx
+
+	call	__rsaz_512_subtract
+
+	movq	%r8, %rdx
+	movq	%r9, %rax
+	movl	128+8(%rsp), $times
+	movq	$out, $inp
+
+	decl	$times
+	jnz	.Loop_sqr
 ___
-} else {
+if ($addx) {
 $code.=<<___;
+	jmp	.Lsqr_tail
+
+.align	32
+.Loop_sqrx:
+	movl	$times,128+8(%rsp)
 	movq	$out, %xmm0		# off-load
+	movq	%rbp, %xmm1		# off-load
 #first iteration	
 	mulx	%rax, %r8, %r9
 
 	mulx	16($inp), %rcx, %r10
+	xor	%rbp, %rbp		# cf=0, of=0
 
 	mulx	24($inp), %rax, %r11
-	add	%rcx, %r9
+	adcx	%rcx, %r9
 
 	mulx	32($inp), %rcx, %r12
-	adc	%rax, %r10
+	adcx	%rax, %r10
 
 	mulx	40($inp), %rax, %r13
-	adc	%rcx, %r11
+	adcx	%rcx, %r11
 
-	mulx	48($inp), %rcx, %r14
-	adc	%rax, %r12
+	.byte	0xc4,0x62,0xf3,0xf6,0xb6,0x30,0x00,0x00,0x00	# mulx	48($inp), %rcx, %r14
+	adcx	%rax, %r12
+	adcx	%rcx, %r13
 
-	mulx	56($inp), %rax, %r15
-	adc	%rcx, %r13
-	mov	%r9, %rcx
-	adc	%rax, %r14
-	adc	\$0, %r15
+	.byte	0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00	# mulx	56($inp), %rax, %r15
+	adcx	%rax, %r14
+	adcx	%rbp, %r15		# %rbp is 0
 
+	mov	%r9, %rcx
 	shld	\$1, %r8, %r9
 	shl	\$1, %r8
 
+	xor	%ebp, %ebp
 	mulx	%rdx, %rax, %rdx
-	add	%rdx, %r8
-	adc	\$0, %r9
+	adcx	%rdx, %r8
+	 mov	8($inp), %rdx
+	adcx	%rbp, %r9
 
 	mov	%rax, (%rsp)
 	mov	%r8, 8(%rsp)
 
 #second iteration	
-	mov	8($inp), %rdx
 	mulx	16($inp), %rax, %rbx
+	adox	%rax, %r10
+	adcx	%rbx, %r11
 
-	mulx	24($inp), $out, %r8
-	add	%rax, %r10
-	adc	%rbx, %r11
-	adc	\$0, %r8
+	.byte	0xc4,0x62,0xc3,0xf6,0x86,0x18,0x00,0x00,0x00	# mulx	24($inp), $out, %r8
+	adox	$out, %r11
+	adcx	%r8, %r12
 
 	mulx	32($inp), %rax, %rbx
-	add	$out, %r11
-	adc	%r8, %r12
-	adc	\$0, %rbx
+	adox	%rax, %r12
+	adcx	%rbx, %r13
 
 	mulx	40($inp), $out, %r8
-	add	%rax, %r12
-	adc	%rbx, %r13
-	adc	\$0, %r8
+	adox	$out, %r13
+	adcx	%r8, %r14
 
-	mulx	48($inp), %rax, %rbx
-	add	$out, %r13
-	adc	%r8, %r14
-	adc	\$0, %rbx
+	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rbx
+	adox	%rax, %r14
+	adcx	%rbx, %r15
 
-	mulx	56($inp), $out, %r8
-	add	%rax, %r14
-	adc	%rbx, %r15
-	mov	%r11, %rbx
-	adc	\$0, %r8
-	add	$out, %r15
-	adc	\$0, %r8
+	.byte	0xc4,0x62,0xc3,0xf6,0x86,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r8
+	adox	$out, %r15
+	adcx	%rbp, %r8
+	adox	%rbp, %r8
 
+	mov	%r11, %rbx
 	shld	\$1, %r10, %r11
 	shld	\$1, %rcx, %r10
 
+	xor	%ebp,%ebp
 	mulx	%rdx, %rax, %rcx
-	add	%rax, %r9
-	adc	%rcx, %r10
-	adc	\$0, %r11
+	 mov	16($inp), %rdx
+	adcx	%rax, %r9
+	adcx	%rcx, %r10
+	adcx	%rbp, %r11
 
 	mov	%r9, 16(%rsp)
-	mov	%r10, 24(%rsp)
+	.byte	0x4c,0x89,0x94,0x24,0x18,0x00,0x00,0x00		# mov	%r10, 24(%rsp)
 	
 #third iteration	
-	mov	16($inp), %rdx
-	mulx	24($inp), $out, %r9
+	.byte	0xc4,0x62,0xc3,0xf6,0x8e,0x18,0x00,0x00,0x00	# mulx	24($inp), $out, %r9
+	adox	$out, %r12
+	adcx	%r9, %r13
 
 	mulx	32($inp), %rax, %rcx
-	add	$out, %r12
-	adc	%r9, %r13
-	adc	\$0, %rcx
+	adox	%rax, %r13
+	adcx	%rcx, %r14
 
 	mulx	40($inp), $out, %r9
-	add	%rax, %r13
-	adc	%rcx, %r14
-	adc	\$0, %r9
+	adox	$out, %r14
+	adcx	%r9, %r15
 
-	mulx	48($inp), %rax, %rcx
-	add	$out, %r14
-	adc	%r9, %r15
-	adc	\$0, %rcx
+	.byte	0xc4,0xe2,0xfb,0xf6,0x8e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rcx
+	adox	%rax, %r15
+	adcx	%rcx, %r8
 
-	mulx	56($inp), $out, %r9
-	add	%rax, %r15
-	adc	%rcx, %r8
-	mov	%r13, %rcx
-	adc	\$0, %r9
-	add	$out, %r8
-	adc	\$0, %r9
+	.byte	0xc4,0x62,0xc3,0xf6,0x8e,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r9
+	adox	$out, %r8
+	adcx	%rbp, %r9
+	adox	%rbp, %r9
 
+	mov	%r13, %rcx
 	shld	\$1, %r12, %r13
 	shld	\$1, %rbx, %r12
 
+	xor	%ebp, %ebp
 	mulx	%rdx, %rax, %rdx
-	add	%rax, %r11
-	adc	%rdx, %r12
-	adc	\$0, %r13
+	adcx	%rax, %r11
+	adcx	%rdx, %r12
+	 mov	24($inp), %rdx
+	adcx	%rbp, %r13
 
 	mov	%r11, 32(%rsp)
-	mov	%r12, 40(%rsp)
+	.byte	0x4c,0x89,0xa4,0x24,0x28,0x00,0x00,0x00		# mov	%r12, 40(%rsp)
 	
 #fourth iteration	
-	mov	24($inp), %rdx
-	mulx	32($inp), %rax, %rbx
+	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x20,0x00,0x00,0x00	# mulx	32($inp), %rax, %rbx
+	adox	%rax, %r14
+	adcx	%rbx, %r15
 
 	mulx	40($inp), $out, %r10
-	add	%rax, %r14
-	adc	%rbx, %r15
-	adc	\$0, %r10
+	adox	$out, %r15
+	adcx	%r10, %r8
 
 	mulx	48($inp), %rax, %rbx
-	add	$out, %r15
-	adc	%r10, %r8
-	adc	\$0, %rbx
+	adox	%rax, %r8
+	adcx	%rbx, %r9
 
 	mulx	56($inp), $out, %r10
-	add	%rax, %r8
-	adc	\$0, %rbx
-	add	$out, %r9
-	adc	\$0, %r10
-	add	%rbx, %r9
-	mov	%r15, %rbx
-	adc	\$0, %r10
+	adox	$out, %r9
+	adcx	%rbp, %r10
+	adox	%rbp, %r10
 
+	.byte	0x66
+	mov	%r15, %rbx
 	shld	\$1, %r14, %r15
 	shld	\$1, %rcx, %r14
 
+	xor	%ebp, %ebp
 	mulx	%rdx, %rax, %rdx
-	add	%rax, %r13
-	adc	%rdx, %r14
-	adc	\$0, %r15
+	adcx	%rax, %r13
+	adcx	%rdx, %r14
+	 mov	32($inp), %rdx
+	adcx	%rbp, %r15
 
 	mov	%r13, 48(%rsp)
 	mov	%r14, 56(%rsp)
 	
 #fifth iteration	
-	mov	32($inp), %rdx
-	mulx	40($inp), $out, %r11
+	.byte	0xc4,0x62,0xc3,0xf6,0x9e,0x28,0x00,0x00,0x00	# mulx	40($inp), $out, %r11
+	adox	$out, %r8
+	adcx	%r11, %r9
 
 	mulx	48($inp), %rax, %rcx
-	add	$out, %r8
-	adc	%r11, %r9
-	adc	\$0, %rcx
+	adox	%rax, %r9
+	adcx	%rcx, %r10
 
 	mulx	56($inp), $out, %r11
-	add	%rax, %r9
-	adc	%rcx, %r10
-	adc	\$0, %r11
-	add	$out, %r10
-	adc	\$0, %r11
+	adox	$out, %r10
+	adcx	%rbp, %r11
+	adox	%rbp, %r11
 
 	mov	%r9, %rcx
 	shld	\$1, %r8, %r9
 	shld	\$1, %rbx, %r8
 
+	xor	%ebp, %ebp
 	mulx	%rdx, %rax, %rdx
-	add	%rax, %r15
-	adc	%rdx, %r8
-	adc	\$0, %r9
+	adcx	%rax, %r15
+	adcx	%rdx, %r8
+	 mov	40($inp), %rdx
+	adcx	%rbp, %r9
 
 	mov	%r15, 64(%rsp)
 	mov	%r8, 72(%rsp)
 	
 #sixth iteration	
-	mov	40($inp), %rdx
-	mulx	48($inp), %rax, %rbx
+	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rbx
+	adox	%rax, %r10
+	adcx	%rbx, %r11
 
-	mulx	56($inp), $out, %r12
-	add	%rax, %r10
-	adc	%rbx, %r11
-	adc	\$0, %r12
-	add	$out, %r11
-	adc	\$0, %r12
+	.byte	0xc4,0x62,0xc3,0xf6,0xa6,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r12
+	adox	$out, %r11
+	adcx	%rbp, %r12
+	adox	%rbp, %r12
 
 	mov	%r11, %rbx
 	shld	\$1, %r10, %r11
 	shld	\$1, %rcx, %r10
 
+	xor	%ebp, %ebp
 	mulx	%rdx, %rax, %rdx
-	add	%rax, %r9
-	adc	%rdx, %r10
-	adc	\$0, %r11
+	adcx	%rax, %r9
+	adcx	%rdx, %r10
+	 mov	48($inp), %rdx
+	adcx	%rbp, %r11
 
 	mov	%r9, 80(%rsp)
 	mov	%r10, 88(%rsp)
 
 #seventh iteration
-	mov	48($inp), %rdx
-	mulx	56($inp), %rax, %r13
-	add	%rax, %r12
-	adc	\$0, %r13
+	.byte	0xc4,0x62,0xfb,0xf6,0xae,0x38,0x00,0x00,0x00	# mulx	56($inp), %rax, %r13
+	adox	%rax, %r12
+	adox	%rbp, %r13
 
 	xor	%r14, %r14
 	shld	\$1, %r13, %r14
 	shld	\$1, %r12, %r13
 	shld	\$1, %rbx, %r12
 
+	xor	%ebp, %ebp
 	mulx	%rdx, %rax, %rdx
-	add	%rax, %r11
-	adc	%rdx, %r12
-	adc	\$0, %r13
+	adcx	%rax, %r11
+	adcx	%rdx, %r12
+	 mov	56($inp), %rdx
+	adcx	%rbp, %r13
 
-	mov	%r11, 96(%rsp)
-	mov	%r12, 104(%rsp)
+	.byte	0x4c,0x89,0x9c,0x24,0x60,0x00,0x00,0x00		# mov	%r11, 96(%rsp)
+	.byte	0x4c,0x89,0xa4,0x24,0x68,0x00,0x00,0x00		# mov	%r12, 104(%rsp)
 
 #eighth iteration
-	mov	56($inp), %rdx
 	mulx	%rdx, %rax, %rdx
-	add	%rax, %r13
-	adc	\$0, %rdx
-	
+	adox	%rax, %r13
+	adox	%rbp, %rdx
+
+	.byte	0x66
 	add	%rdx, %r14
 
 	movq	%r13, 112(%rsp)
 	movq	%r14, 120(%rsp)
 	movq	%xmm0, $out
-___
-}
-$code.=<<___;
+	movq	%xmm1, %rbp
+
+	movq	128(%rsp), %rdx		# pull $n0
 	movq	(%rsp), %r8
 	movq	8(%rsp), %r9
 	movq	16(%rsp), %r10
@@ -691,7 +748,7 @@ $code.=<<___;
 	movq	48(%rsp), %r14
 	movq	56(%rsp), %r15
 
-	call	_rsaz_512_reduce
+	call	__rsaz_512_reducex
 
 	addq	64(%rsp), %r8
 	adcq	72(%rsp), %r9
@@ -703,7 +760,7 @@ $code.=<<___;
 	adcq	120(%rsp), %r15
 	sbbq	%rcx, %rcx
 
-	call	_rsaz_512_subtract
+	call	__rsaz_512_subtract
 
 	movq	%r8, %rdx
 	movq	%r9, %rax
@@ -711,7 +768,12 @@ $code.=<<___;
 	movq	$out, $inp
 
 	decl	$times
-	jnz	.Loop_sqr
+	jnz	.Loop_sqrx
+
+.Lsqr_tail:
+___
+}
+$code.=<<___;
 
 	leaq	128+24+48(%rsp), %rax
 	movq	-48(%rax), %r15
@@ -745,7 +807,15 @@ rsaz_512_mul:
 	movq	$out, %xmm0		# off-load arguments
 	movq	$mod, %xmm1
 	movq	$n0, 128(%rsp)
-
+___
+$code.=<<___ if ($addx);
+	movl	\$0x80100,%r11d
+	andl	OPENSSL_ia32cap_P+8(%rip),%r11d
+	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX
+	je	.Lmulx
+___
+$code.=<<___;
+	movq	($bp), %rbx		# pass b[0]
 	movq	$bp, %rbp		# pass argument
 	call	__rsaz_512_mul
 
@@ -761,8 +831,34 @@ rsaz_512_mul:
 	movq	48(%rsp), %r14
 	movq	56(%rsp), %r15
 
-	call	_rsaz_512_reduce
+	call	__rsaz_512_reduce
+___
+$code.=<<___ if ($addx);
+	jmp	.Lmul_tail
 
+.align	32
+.Lmulx:
+	movq	$bp, %rbp		# pass argument
+	movq	($bp), %rdx		# pass b[0]
+	call	__rsaz_512_mulx
+
+	movq	%xmm0, $out
+	movq	%xmm1, %rbp
+
+	movq	128(%rsp), %rdx		# pull $n0
+	movq	(%rsp), %r8
+	movq	8(%rsp), %r9
+	movq	16(%rsp), %r10
+	movq	24(%rsp), %r11
+	movq	32(%rsp), %r12
+	movq	40(%rsp), %r13
+	movq	48(%rsp), %r14
+	movq	56(%rsp), %r15
+
+	call	__rsaz_512_reducex
+.Lmul_tail:
+___
+$code.=<<___;
 	addq	64(%rsp), %r8
 	adcq	72(%rsp), %r9
 	adcq	80(%rsp), %r10
@@ -773,7 +869,7 @@ rsaz_512_mul:
 	adcq	120(%rsp), %r15
 	sbbq	%rcx, %rcx
 
-	call	_rsaz_512_subtract
+	call	__rsaz_512_subtract
 
 	leaq	128+24+48(%rsp), %rax
 	movq	-48(%rax), %r15
@@ -804,6 +900,14 @@ rsaz_512_mul_gather4:
 
 	subq	\$128+24, %rsp
 .Lmul_gather4_body:
+___
+$code.=<<___ if ($addx);
+	movl	\$0x80100,%r11d
+	andl	OPENSSL_ia32cap_P+8(%rip),%r11d
+	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX
+	je	.Lmulx_gather
+___
+$code.=<<___;
 	movl	64($bp,$pwr,4), %eax
 	movq	$out, %xmm0		# off-load arguments
 	movl	($bp,$pwr,4), %ebx
@@ -969,8 +1073,131 @@ rsaz_512_mul_gather4:
 	movq	48(%rsp), %r14
 	movq	56(%rsp), %r15
 
-	call	_rsaz_512_reduce
+	call	__rsaz_512_reduce
+___
+$code.=<<___ if ($addx);
+	jmp	.Lmul_gather_tail
+
+.align	32
+.Lmulx_gather:
+	mov	64($bp,$pwr,4), %eax
+	movq	$out, %xmm0		# off-load arguments
+	lea	128($bp,$pwr,4), %rbp
+	mov	($bp,$pwr,4), %edx
+	movq	$mod, %xmm1
+	mov	$n0, 128(%rsp)
+
+	shl	\$32, %rax
+	or	%rax, %rdx
+	mulx	($ap), %rbx, %r8	# 0 iteration
+	mov	%rbx, (%rsp)
+	xor	%edi, %edi		# cf=0, of=0
+
+	mulx	8($ap), %rax, %r9
+	 movd	(%rbp), %xmm4
+
+	mulx	16($ap), %rbx, %r10
+	 movd	64(%rbp), %xmm5
+	adcx	%rax, %r8
+
+	mulx	24($ap), %rax, %r11
+	 pslldq	\$4, %xmm5
+	adcx	%rbx, %r9
+
+	mulx	32($ap), %rbx, %r12
+	 por	%xmm5, %xmm4
+	adcx	%rax, %r10
+
+	mulx	40($ap), %rax, %r13
+	adcx	%rbx, %r11
+
+	mulx	48($ap), %rbx, %r14
+	 lea	128(%rbp), %rbp
+	adcx	%rax, %r12
+	
+	mulx	56($ap), %rax, %r15
+	 movq	%xmm4, %rdx
+	adcx	%rbx, %r13
+	adcx	%rax, %r14
+	mov	%r8, %rbx
+	adcx	%rdi, %r15		# %rdi is 0
+
+	mov	\$-7, %rcx
+	jmp	.Loop_mulx_gather
+
+.align	32
+.Loop_mulx_gather:
+	mulx	($ap), %rax, %r8
+	adcx	%rax, %rbx
+	adox	%r9, %r8
+
+	mulx	8($ap), %rax, %r9
+	.byte	0x66,0x0f,0x6e,0xa5,0x00,0x00,0x00,0x00		# movd	(%rbp), %xmm4
+	adcx	%rax, %r8
+	adox	%r10, %r9
 
+	mulx	16($ap), %rax, %r10
+	 movd	64(%rbp), %xmm5
+	 lea	128(%rbp), %rbp
+	adcx	%rax, %r9
+	adox	%r11, %r10
+
+	.byte	0xc4,0x62,0xfb,0xf6,0x9e,0x18,0x00,0x00,0x00	# mulx	24($ap), %rax, %r11
+	 pslldq	\$4, %xmm5
+	 por	%xmm5, %xmm4
+	adcx	%rax, %r10
+	adox	%r12, %r11
+
+	mulx	32($ap), %rax, %r12
+	adcx	%rax, %r11
+	adox	%r13, %r12
+
+	mulx	40($ap), %rax, %r13
+	adcx	%rax, %r12
+	adox	%r14, %r13
+
+	.byte	0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00	# mulx	48($ap), %rax, %r14
+	adcx	%rax, %r13
+	adox	%r15, %r14
+
+	mulx	56($ap), %rax, %r15
+	 movq	%xmm4, %rdx
+	 mov	%rbx, 64(%rsp,%rcx,8)
+	adcx	%rax, %r14
+	adox	%rdi, %r15
+	mov	%r8, %rbx
+	adcx	%rdi, %r15		# cf=0
+
+	inc	%rcx			# of=0
+	jnz	.Loop_mulx_gather
+
+	mov	%r8, 64(%rsp)
+	mov	%r9, 64+8(%rsp)
+	mov	%r10, 64+16(%rsp)
+	mov	%r11, 64+24(%rsp)
+	mov	%r12, 64+32(%rsp)
+	mov	%r13, 64+40(%rsp)
+	mov	%r14, 64+48(%rsp)
+	mov	%r15, 64+56(%rsp)
+
+	movq	%xmm0, $out
+	movq	%xmm1, %rbp
+
+	mov	128(%rsp), %rdx		# pull $n0
+	mov	(%rsp), %r8
+	mov	8(%rsp), %r9
+	mov	16(%rsp), %r10
+	mov	24(%rsp), %r11
+	mov	32(%rsp), %r12
+	mov	40(%rsp), %r13
+	mov	48(%rsp), %r14
+	mov	56(%rsp), %r15
+
+	call	__rsaz_512_reducex
+
+.Lmul_gather_tail:
+___
+$code.=<<___;
 	addq	64(%rsp), %r8
 	adcq	72(%rsp), %r9
 	adcq	80(%rsp), %r10
@@ -981,7 +1208,7 @@ rsaz_512_mul_gather4:
 	adcq	120(%rsp), %r15
 	sbbq	%rcx, %rcx
 
-	call	_rsaz_512_subtract
+	call	__rsaz_512_subtract
 
 	leaq	128+24+48(%rsp), %rax
 	movq	-48(%rax), %r15
@@ -1019,6 +1246,15 @@ rsaz_512_mul_scatter4:
 	movq	$n0, 128(%rsp)
 
 	movq	$out, %rbp
+___
+$code.=<<___ if ($addx);
+	movl	\$0x80100,%r11d
+	andl	OPENSSL_ia32cap_P+8(%rip),%r11d
+	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX
+	je	.Lmulx_scatter
+___
+$code.=<<___;
+	movq	($out),%rbx		# pass b[0]
 	call	__rsaz_512_mul
 
 	movq	%xmm0, $out
@@ -1033,8 +1269,34 @@ rsaz_512_mul_scatter4:
 	movq	48(%rsp), %r14
 	movq	56(%rsp), %r15
 
-	call	_rsaz_512_reduce
+	call	__rsaz_512_reduce
+___
+$code.=<<___ if ($addx);
+	jmp	.Lmul_scatter_tail
+	
+.align	32
+.Lmulx_scatter:
+	movq	($out), %rdx		# pass b[0]
+	call	__rsaz_512_mulx
+
+	movq	%xmm0, $out
+	movq	%xmm1, %rbp
+
+	movq	128(%rsp), %rdx		# pull $n0
+	movq	(%rsp), %r8
+	movq	8(%rsp), %r9
+	movq	16(%rsp), %r10
+	movq	24(%rsp), %r11
+	movq	32(%rsp), %r12
+	movq	40(%rsp), %r13
+	movq	48(%rsp), %r14
+	movq	56(%rsp), %r15
+
+	call	__rsaz_512_reducex
 
+.Lmul_scatter_tail:
+___
+$code.=<<___;
 	addq	64(%rsp), %r8
 	adcq	72(%rsp), %r9
 	adcq	80(%rsp), %r10
@@ -1046,7 +1308,7 @@ rsaz_512_mul_scatter4:
 	movq	%xmm2, $inp
 	sbbq	%rcx, %rcx
 
-	call	_rsaz_512_subtract
+	call	__rsaz_512_subtract
 
 	movl	%r8d, 64*0($inp)	# scatter
 	shrq	\$32, %r8
@@ -1102,6 +1364,11 @@ rsaz_512_mul_by_one:
 
 	subq	\$128+24, %rsp
 .Lmul_by_one_body:
+___
+$code.=<<___ if ($addx);
+	movl	OPENSSL_ia32cap_P+8(%rip),%eax
+___
+$code.=<<___;
 	movq	$mod, %rbp	# reassign argument
 	movq	$n0, 128(%rsp)
 
@@ -1122,9 +1389,24 @@ rsaz_512_mul_by_one:
 	movdqa	%xmm0, 64(%rsp)
 	movdqa	%xmm0, 80(%rsp)
 	movdqa	%xmm0, 96(%rsp)
-
-	call	_rsaz_512_reduce
-
+___
+$code.=<<___ if ($addx);
+	andl	\$0x80100,%eax
+	cmpl	\$0x80100,%eax		# check for MULX and ADO/CX
+	je	.Lby_one_callx
+___
+$code.=<<___;
+	call	__rsaz_512_reduce
+___
+$code.=<<___ if ($addx);
+	jmp	.Lby_one_tail
+.align	32
+.Lby_one_callx:
+	movq	128(%rsp), %rdx		# pull $n0
+	call	__rsaz_512_reducex
+.Lby_one_tail:
+___
+$code.=<<___;
 	movq	%r8, ($out)
 	movq	%r9, 8($out)
 	movq	%r10, 16($out)
@@ -1147,18 +1429,15 @@ rsaz_512_mul_by_one:
 .size	rsaz_512_mul_by_one,.-rsaz_512_mul_by_one
 ___
 }
-{	# _rsaz_512_reduce
+{	# __rsaz_512_reduce
 	#
 	# input:	%r8-%r15, %rbp - mod, 128(%rsp) - n0
 	# output:	%r8-%r15
 	# clobbers:	everything except %rbp and %rdi
 $code.=<<___;
-.type	_rsaz_512_reduce,\@abi-omnipotent
+.type	__rsaz_512_reduce,\@abi-omnipotent
 .align	32
-_rsaz_512_reduce:
-___
-if (1) {
-$code.=<<___;
+__rsaz_512_reduce:
 	movq	%r8, %rbx
 	imulq	128+8(%rsp), %rbx
 	movq	0(%rbp), %rax
@@ -1195,8 +1474,10 @@ $code.=<<___;
 	adcq	\$0, %rdx
 	addq	%r11, %r10
 	 movq	128+8(%rsp), %rsi
+	#movq	%rdx, %r11
+	#adcq	\$0, %r11
+	adcq	\$0, %rdx
 	movq	%rdx, %r11
-	adcq	\$0, %r11
 
 	mulq	%rbx
 	addq	%rax, %r12
@@ -1234,75 +1515,84 @@ $code.=<<___;
 
 	decl	%ecx
 	jne	.Lreduction_loop
+
+	ret
+.size	__rsaz_512_reduce,.-__rsaz_512_reduce
 ___
-} else {
+}
+if ($addx) {
+	# __rsaz_512_reducex
+	#
+	# input:	%r8-%r15, %rbp - mod, 128(%rsp) - n0
+	# output:	%r8-%r15
+	# clobbers:	everything except %rbp and %rdi
 $code.=<<___;
-	movq	128+8(%rsp), %rdx		# pull $n0
+.type	__rsaz_512_reducex,\@abi-omnipotent
+.align	32
+__rsaz_512_reducex:
+	#movq	128+8(%rsp), %rdx		# pull $n0
 	imulq	%r8, %rdx
+	xorq	%rsi, %rsi			# cf=0,of=0
 	movl	\$8, %ecx
-	jmp	.Lreduction_loop
+	jmp	.Lreduction_loopx
 
 .align	32
-.Lreduction_loop:
-	neg	%r8
+.Lreduction_loopx:
+	mov	%r8, %rbx
 	mulx	0(%rbp), %rax, %r8
-	adc	%r9, %r8
+	adcx	%rbx, %rax
+	adox	%r9, %r8
 
 	mulx	8(%rbp), %rax, %r9
-	adc	\$0, %r9
-	add	%rax, %r8
-	adc	%r10, %r9
-
-	mulx	16(%rbp), %rax, %r10
-	adc	\$0, %r10
-	 mov	128+8(%rsp), %rbx		# pull $n0
-	 imul	%r8, %rbx
-	add	%rax, %r9
-	adc	%r11, %r10
-
-	mulx	24(%rbp), %rax, %r11
-	adc	\$0, %r11
-	add	%rax, %r10
-	adc	%r12, %r11
-
-	mulx	32(%rbp), %rax, %r12
-	adc	\$0, %r12
-	add	%rax, %r11
-	adc	%r13, %r12
+	adcx	%rax, %r8
+	adox	%r10, %r9
+
+	mulx	16(%rbp), %rbx, %r10
+	adcx	%rbx, %r9
+	adox	%r11, %r10
+
+	mulx	24(%rbp), %rbx, %r11
+	adcx	%rbx, %r10
+	adox	%r12, %r11
+
+	.byte	0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00	# mulx	32(%rbp), %rbx, %r12
+	 mov	%rdx, %rax
+	 mov	%r8, %rdx
+	adcx	%rbx, %r11
+	adox	%r13, %r12
+
+	 mulx	128+8(%rsp), %rbx, %rdx
+	 mov	%rax, %rdx
 
 	mulx	40(%rbp), %rax, %r13
-	adc	\$0, %r13
-	add	%rax, %r12
-	adc	%r14, %r13
+	adcx	%rax, %r12
+	adox	%r14, %r13
 
-	mulx	48(%rbp), %rax, %r14
-	adc	\$0, %r14
-	add	%rax, %r13
-	adc	%r15, %r14
+	.byte	0xc4,0x62,0xfb,0xf6,0xb5,0x30,0x00,0x00,0x00	# mulx	48(%rbp), %rax, %r14
+	adcx	%rax, %r13
+	adox	%r15, %r14
 
 	mulx	56(%rbp), %rax, %r15
 	 mov	%rbx, %rdx
-	adc	\$0, %r15
-	add	%rax, %r14
-	adc	\$0, %r15
+	adcx	%rax, %r14
+	adox	%rsi, %r15			# %rsi is 0
+	adcx	%rsi, %r15			# cf=0
+
+	decl	%ecx				# of=0
+	jne	.Lreduction_loopx
 
-	dec	%ecx
-	jne	.Lreduction_loop
-___
-}
-$code.=<<___;
 	ret
-.size	_rsaz_512_reduce,.-_rsaz_512_reduce
+.size	__rsaz_512_reducex,.-__rsaz_512_reducex
 ___
 }
-{	# _rsaz_512_subtract
+{	# __rsaz_512_subtract
 	# input: %r8-%r15, %rdi - $out, %rbp - $mod, %rcx - mask
 	# output:
 	# clobbers: everything but %rdi, %rsi and %rbp
 $code.=<<___;
-.type	_rsaz_512_subtract,\@abi-omnipotent
+.type	__rsaz_512_subtract,\@abi-omnipotent
 .align	32
-_rsaz_512_subtract:
+__rsaz_512_subtract:
 	movq	%r8, ($out)
 	movq	%r9, 8($out)
 	movq	%r10, 16($out)
@@ -1356,7 +1646,7 @@ _rsaz_512_subtract:
 	movq	%r15, 56($out)
 
 	ret
-.size	_rsaz_512_subtract,.-_rsaz_512_subtract
+.size	__rsaz_512_subtract,.-__rsaz_512_subtract
 ___
 }
 {	# __rsaz_512_mul
@@ -1371,7 +1661,6 @@ $code.=<<___;
 __rsaz_512_mul:
 	leaq	8(%rsp), %rdi
 
-	movq	($bp), %rbx
 	movq	($ap), %rax
 	mulq	%rbx
 	movq	%rax, (%rdi)
@@ -1511,6 +1800,136 @@ __rsaz_512_mul:
 .size	__rsaz_512_mul,.-__rsaz_512_mul
 ___
 }
+if ($addx) {
+	# __rsaz_512_mulx
+	#
+	# input: %rsi - ap, %rbp - bp
+	# ouput:
+	# clobbers: everything
+my ($ap,$bp,$zero) = ("%rsi","%rbp","%rdi");
+$code.=<<___;
+.type	__rsaz_512_mulx,\@abi-omnipotent
+.align	32
+__rsaz_512_mulx:
+	mulx	($ap), %rbx, %r8	# initial %rdx preloaded by caller
+	xor	$zero, $zero		# cf=0,of=0
+
+	mulx	8($ap), %rax, %r9
+	movq	%rbx, 8(%rsp)
+
+	mulx	16($ap), %rbx, %r10
+	adcx	%rax, %r8
+
+	mulx	24($ap), %rax, %r11
+	adcx	%rbx, %r9
+
+	.byte	0xc4,0x62,0xe3,0xf6,0xa6,0x20,0x00,0x00,0x00	# mulx	32($ap), %rbx, %r12
+	adcx	%rax, %r10
+
+	mulx	40($ap), %rax, %r13
+	adcx	%rbx, %r11
+
+	mulx	48($ap), %rbx, %r14
+	adcx	%rax, %r12
+
+	mulx	56($ap), %rax, %r15
+	 mov	8($bp), %rdx
+	adcx	%rbx, %r13
+	adcx	%rax, %r14
+	adcx	$zero, %r15		# cf=0
+
+	mov	\$-6, %rcx
+	jmp	.Loop_mulx
+
+.align	32
+.Loop_mulx:
+	movq	%r8, %rbx
+	mulx	($ap), %rax, %r8
+	adcx	%rax, %rbx
+	adox	%r9, %r8
+
+	mulx	8($ap), %rax, %r9
+	adcx	%rax, %r8
+	adox	%r10, %r9
+
+	mulx	16($ap), %rax, %r10
+	adcx	%rax, %r9
+	adox	%r11, %r10
+
+	mulx	24($ap), %rax, %r11
+	adcx	%rax, %r10
+	adox	%r12, %r11
+
+	.byte	0x3e,0xc4,0x62,0xfb,0xf6,0xa6,0x20,0x00,0x00,0x00	# mulx	32($ap), %rax, %r12
+	adcx	%rax, %r11
+	adox	%r13, %r12
+
+	mulx	40($ap), %rax, %r13
+	adcx	%rax, %r12
+	adox	%r14, %r13
+
+	mulx	48($ap), %rax, %r14
+	adcx	%rax, %r13
+	adox	%r15, %r14
+
+	mulx	56($ap), %rax, %r15
+	 movq	64($bp,%rcx,8), %rdx
+	 movq	%rbx, 8+64-8(%rsp,%rcx,8)
+	adcx	%rax, %r14
+	adox	$zero, %r15
+	adcx	$zero, %r15		# cf=0
+
+	inc	%rcx			# of=0
+	jnz	.Loop_mulx
+
+	movq	%r8, %rbx
+	mulx	($ap), %rax, %r8
+	adcx	%rax, %rbx
+	adox	%r9, %r8
+
+	.byte	0xc4,0x62,0xfb,0xf6,0x8e,0x08,0x00,0x00,0x00	# mulx	8($ap), %rax, %r9
+	adcx	%rax, %r8
+	adox	%r10, %r9
+
+	.byte	0xc4,0x62,0xfb,0xf6,0x96,0x10,0x00,0x00,0x00	# mulx	16($ap), %rax, %r10
+	adcx	%rax, %r9
+	adox	%r11, %r10
+
+	mulx	24($ap), %rax, %r11
+	adcx	%rax, %r10
+	adox	%r12, %r11
+
+	mulx	32($ap), %rax, %r12
+	adcx	%rax, %r11
+	adox	%r13, %r12
+
+	mulx	40($ap), %rax, %r13
+	adcx	%rax, %r12
+	adox	%r14, %r13
+
+	.byte	0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00	# mulx	48($ap), %rax, %r14
+	adcx	%rax, %r13
+	adox	%r15, %r14
+
+	.byte	0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00	# mulx	56($ap), %rax, %r15
+	adcx	%rax, %r14
+	adox	$zero, %r15
+	adcx	$zero, %r15
+
+	mov	%rbx, 8+64-8(%rsp)
+	mov	%r8, 8+64(%rsp)
+	mov	%r9, 8+64+8(%rsp)
+	mov	%r10, 8+64+16(%rsp)
+	mov	%r11, 8+64+24(%rsp)
+	mov	%r12, 8+64+32(%rsp)
+	mov	%r13, 8+64+40(%rsp)
+	mov	%r14, 8+64+48(%rsp)
+	mov	%r15, 8+64+56(%rsp)
+
+	ret
+.size	__rsaz_512_mulx,.-__rsaz_512_mulx
+___
+}
 {
 my ($out,$inp,$power)= $win64 ? ("%rcx","%rdx","%r8d") : ("%rdi","%rsi","%edx");
 $code.=<<___;
-- 
2.25.1