crypto/bn/asm/armv4-mont.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # January 2007.
  11
  12 # Montgomery multiplication for ARMv4.
  13 #
  14 # Performance improvement naturally varies among CPU implementations
  15 # and compilers. The code was observed to provide +65-35% improvement
  16 # [depending on key length, less for longer keys] on ARM920T, and
  17 # +115-80% on Intel IXP425. This is compared to pre-bn_mul_mont code
  18 # base and compiler generated code with in-lined umull and even umlal
  19 # instructions. The latter means that this code didn't really have an
  20 # "advantage" of utilizing some "secret" instruction.
  21 #
  22 # The code is interoperable with Thumb ISA and is rather compact, less
  23 # than 1/2KB. Windows CE port would be trivial, as it's exclusively
  24 # about decorations, ABI and instruction syntax are identical.
  25
  26 # November 2013
  27 #
  28 # Add NEON code path, which handles lengths divisible by 8. RSA/DSA
  29 # performance improvement on Cortex-A8 is ~45-100% depending on key
  30 # length, more for longer keys. On Cortex-A15 the span is ~10-105%.
  31 # On Snapdragon S4 improvement was measured to vary from ~70% to
  32 # incredible ~380%, yes, 4.8x faster, for RSA4096 sign. But this is
  33 # rather because original integer-only code seems to perform
  34 # suboptimally on S4. Situation on Cortex-A9 is unfortunately
  35 # different. It's being looked into, but the trouble is that
  36 # performance for vectors longer than 256 bits is actually couple
  37 # of percent worse than for integer-only code. The code is chosen
  38 # for execution on all NEON-capable processors, because gain on
  39 # others outweighs the marginal loss on Cortex-A9.
  40
  41 while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
  42 open STDOUT,">$output";
  43
  44 $num="r0";      # starts as num argument, but holds &tp[num-1]
  45 $ap="r1";
  46 $bp="r2"; $bi="r2"; $rp="r2";
  47 $np="r3";
  48 $tp="r4";
  49 $aj="r5";
  50 $nj="r6";
  51 $tj="r7";
  52 $n0="r8";
  53 ###########     # r9 is reserved by ELF as platform specific, e.g. TLS pointer
  54 $alo="r10";     # sl, gcc uses it to keep @GOT
  55 $ahi="r11";     # fp
  56 $nlo="r12";     # ip
  57 ###########     # r13 is stack pointer
  58 $nhi="r14";     # lr
  59 ###########     # r15 is program counter
  60
  61 #### argument block layout relative to &tp[num-1], a.k.a. $num
  62 $_rp="$num,#12*4";
  63 # ap permanently resides in r1
  64 $_bp="$num,#13*4";
  65 # np permanently resides in r3
  66 $_n0="$num,#14*4";
  67 $_num="$num,#15*4";     $_bpend=$_num;
  68
  69 $code=<<___;
  70 #include "arm_arch.h"
  71
  72 .text
  73 .code   32
  74
  75 #if __ARM_ARCH__>=7
  76 .align  5
  77 .LOPENSSL_armcap:
  78 .word   OPENSSL_armcap_P-bn_mul_mont
  79 #endif
  80
  81 .global bn_mul_mont
  82 .type   bn_mul_mont,%function
  83
  84 .align  5
  85 bn_mul_mont:
  86         ldr     ip,[sp,#4]              @ load num
  87         stmdb   sp!,{r0,r2}             @ sp points at argument block
  88 #if __ARM_ARCH__>=7
  89         tst     ip,#7
  90         bne     .Lialu
  91         adr     r0,bn_mul_mont
  92         ldr     r2,.LOPENSSL_armcap
  93         ldr     r0,[r0,r2]
  94         tst     r0,#1                   @ NEON available?
  95         ldmia   sp, {r0,r2}
  96         beq     .Lialu
  97         add     sp,sp,#8
  98         b       bn_mul8x_mont_neon
  99 .align  4
 100 .Lialu:
 101 #endif
 102         cmp     ip,#2
 103         mov     $num,ip                 @ load num
 104         movlt   r0,#0
 105         addlt   sp,sp,#2*4
 106         blt     .Labrt
 107
 108         stmdb   sp!,{r4-r12,lr}         @ save 10 registers
 109
 110         mov     $num,$num,lsl#2         @ rescale $num for byte count
 111         sub     sp,sp,$num              @ alloca(4*num)
 112         sub     sp,sp,#4                @ +extra dword
 113         sub     $num,$num,#4            @ "num=num-1"
 114         add     $tp,$bp,$num            @ &bp[num-1]
 115
 116         add     $num,sp,$num            @ $num to point at &tp[num-1]
 117         ldr     $n0,[$_n0]              @ &n0
 118         ldr     $bi,[$bp]               @ bp[0]
 119         ldr     $aj,[$ap],#4            @ ap[0],ap++
 120         ldr     $nj,[$np],#4            @ np[0],np++
 121         ldr     $n0,[$n0]               @ *n0
 122         str     $tp,[$_bpend]           @ save &bp[num]
 123
 124         umull   $alo,$ahi,$aj,$bi       @ ap[0]*bp[0]
 125         str     $n0,[$_n0]              @ save n0 value
 126         mul     $n0,$alo,$n0            @ "tp[0]"*n0
 127         mov     $nlo,#0
 128         umlal   $alo,$nlo,$nj,$n0       @ np[0]*n0+"t[0]"
 129         mov     $tp,sp
 130
 131 .L1st:
 132         ldr     $aj,[$ap],#4            @ ap[j],ap++
 133         mov     $alo,$ahi
 134         ldr     $nj,[$np],#4            @ np[j],np++
 135         mov     $ahi,#0
 136         umlal   $alo,$ahi,$aj,$bi       @ ap[j]*bp[0]
 137         mov     $nhi,#0
 138         umlal   $nlo,$nhi,$nj,$n0       @ np[j]*n0
 139         adds    $nlo,$nlo,$alo
 140         str     $nlo,[$tp],#4           @ tp[j-1]=,tp++
 141         adc     $nlo,$nhi,#0
 142         cmp     $tp,$num
 143         bne     .L1st
 144
 145         adds    $nlo,$nlo,$ahi
 146         ldr     $tp,[$_bp]              @ restore bp
 147         mov     $nhi,#0
 148         ldr     $n0,[$_n0]              @ restore n0
 149         adc     $nhi,$nhi,#0
 150         str     $nlo,[$num]             @ tp[num-1]=
 151         str     $nhi,[$num,#4]          @ tp[num]=
 152 \f
 153 .Louter:
 154         sub     $tj,$num,sp             @ "original" $num-1 value
 155         sub     $ap,$ap,$tj             @ "rewind" ap to &ap[1]
 156         ldr     $bi,[$tp,#4]!           @ *(++bp)
 157         sub     $np,$np,$tj             @ "rewind" np to &np[1]
 158         ldr     $aj,[$ap,#-4]           @ ap[0]
 159         ldr     $alo,[sp]               @ tp[0]
 160         ldr     $nj,[$np,#-4]           @ np[0]
 161         ldr     $tj,[sp,#4]             @ tp[1]
 162
 163         mov     $ahi,#0
 164         umlal   $alo,$ahi,$aj,$bi       @ ap[0]*bp[i]+tp[0]
 165         str     $tp,[$_bp]              @ save bp
 166         mul     $n0,$alo,$n0
 167         mov     $nlo,#0
 168         umlal   $alo,$nlo,$nj,$n0       @ np[0]*n0+"tp[0]"
 169         mov     $tp,sp
 170
 171 .Linner:
 172         ldr     $aj,[$ap],#4            @ ap[j],ap++
 173         adds    $alo,$ahi,$tj           @ +=tp[j]
 174         ldr     $nj,[$np],#4            @ np[j],np++
 175         mov     $ahi,#0
 176         umlal   $alo,$ahi,$aj,$bi       @ ap[j]*bp[i]
 177         mov     $nhi,#0
 178         umlal   $nlo,$nhi,$nj,$n0       @ np[j]*n0
 179         adc     $ahi,$ahi,#0
 180         ldr     $tj,[$tp,#8]            @ tp[j+1]
 181         adds    $nlo,$nlo,$alo
 182         str     $nlo,[$tp],#4           @ tp[j-1]=,tp++
 183         adc     $nlo,$nhi,#0
 184         cmp     $tp,$num
 185         bne     .Linner
 186
 187         adds    $nlo,$nlo,$ahi
 188         mov     $nhi,#0
 189         ldr     $tp,[$_bp]              @ restore bp
 190         adc     $nhi,$nhi,#0
 191         ldr     $n0,[$_n0]              @ restore n0
 192         adds    $nlo,$nlo,$tj
 193         ldr     $tj,[$_bpend]           @ restore &bp[num]
 194         adc     $nhi,$nhi,#0
 195         str     $nlo,[$num]             @ tp[num-1]=
 196         str     $nhi,[$num,#4]          @ tp[num]=
 197
 198         cmp     $tp,$tj
 199         bne     .Louter
 200 \f
 201         ldr     $rp,[$_rp]              @ pull rp
 202         add     $num,$num,#4            @ $num to point at &tp[num]
 203         sub     $aj,$num,sp             @ "original" num value
 204         mov     $tp,sp                  @ "rewind" $tp
 205         mov     $ap,$tp                 @ "borrow" $ap
 206         sub     $np,$np,$aj             @ "rewind" $np to &np[0]
 207
 208         subs    $tj,$tj,$tj             @ "clear" carry flag
 209 .Lsub:  ldr     $tj,[$tp],#4
 210         ldr     $nj,[$np],#4
 211         sbcs    $tj,$tj,$nj             @ tp[j]-np[j]
 212         str     $tj,[$rp],#4            @ rp[j]=
 213         teq     $tp,$num                @ preserve carry
 214         bne     .Lsub
 215         sbcs    $nhi,$nhi,#0            @ upmost carry
 216         mov     $tp,sp                  @ "rewind" $tp
 217         sub     $rp,$rp,$aj             @ "rewind" $rp
 218
 219         and     $ap,$tp,$nhi
 220         bic     $np,$rp,$nhi
 221         orr     $ap,$ap,$np             @ ap=borrow?tp:rp
 222
 223 .Lcopy: ldr     $tj,[$ap],#4            @ copy or in-place refresh
 224         str     sp,[$tp],#4             @ zap tp
 225         str     $tj,[$rp],#4
 226         cmp     $tp,$num
 227         bne     .Lcopy
 228
 229         add     sp,$num,#4              @ skip over tp[num+1]
 230         ldmia   sp!,{r4-r12,lr}         @ restore registers
 231         add     sp,sp,#2*4              @ skip over {r0,r2}
 232         mov     r0,#1
 233 .Labrt: tst     lr,#1
 234         moveq   pc,lr                   @ be binary compatible with V4, yet
 235         bx      lr                      @ interoperable with Thumb ISA:-)
 236 .size   bn_mul_mont,.-bn_mul_mont
 237 ___
 238 {
 239 sub Dlo()   { shift=~m|q([1]?[0-9])|?"d".($1*2):"";     }
 240 sub Dhi()   { shift=~m|q([1]?[0-9])|?"d".($1*2+1):"";   }
 241
 242 my ($A0,$A1,$A2,$A3)=map("d$_",(0..3));
 243 my ($N0,$N1,$N2,$N3)=map("d$_",(4..7));
 244 my ($Z,$Temp)=("q4","q5");
 245 my ($A0xB,$A1xB,$A2xB,$A3xB,$A4xB,$A5xB,$A6xB,$A7xB)=map("q$_",(6..13));
 246 my ($Bi,$Ni,$M0)=map("d$_",(28..31));
 247 my $zero=&Dlo($Z);
 248 my $temp=&Dlo($Temp);
 249
 250 my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("r$_",(0..5));
 251 my ($tinptr,$toutptr,$inner,$outer)=map("r$_",(6..9));
 252
 253 $code.=<<___;
 254 #if __ARM_ARCH__>=7
 255 .fpu    neon
 256
 257 .type   bn_mul8x_mont_neon,%function
 258 .align  5
 259 bn_mul8x_mont_neon:
 260         mov     ip,sp
 261         stmdb   sp!,{r4-r11}
 262         vstmdb  sp!,{d8-d15}            @ ABI specification says so
 263         ldmia   ip,{r4-r5}              @ load rest of parameter block
 264
 265         sub             $toutptr,sp,#16
 266         vld1.32         {${Bi}[0]}, [$bptr,:32]!
 267         sub             $toutptr,$toutptr,$num,lsl#4
 268         vld1.32         {$A0-$A3},  [$aptr]!            @ can't specify :32 :-(
 269         and             $toutptr,$toutptr,#-64
 270         vld1.32         {${M0}[0]}, [$n0,:32]
 271         mov             sp,$toutptr                     @ alloca
 272         veor            $zero,$zero,$zero
 273         subs            $inner,$num,#8
 274         vzip.16         $Bi,$zero
 275
 276         vmull.u32       $A0xB,$Bi,${A0}[0]
 277         vmull.u32       $A1xB,$Bi,${A0}[1]
 278         vmull.u32       $A2xB,$Bi,${A1}[0]
 279         vshl.i64        $temp,`&Dhi("$A0xB")`,#16
 280         vmull.u32       $A3xB,$Bi,${A1}[1]
 281
 282         vadd.u64        $temp,$temp,`&Dlo("$A0xB")`
 283         veor            $zero,$zero,$zero
 284         vmul.u32        $Ni,$temp,$M0
 285
 286         vmull.u32       $A4xB,$Bi,${A2}[0]
 287          vld1.32        {$N0-$N3}, [$nptr]!
 288         vmull.u32       $A5xB,$Bi,${A2}[1]
 289         vmull.u32       $A6xB,$Bi,${A3}[0]
 290         vzip.16         $Ni,$zero
 291         vmull.u32       $A7xB,$Bi,${A3}[1]
 292
 293         bne     .LNEON_1st
 294
 295         @ special case for num=8, everything is in register bank...
 296
 297         vmlal.u32       $A0xB,$Ni,${N0}[0]
 298         sub             $outer,$num,#1
 299         vmlal.u32       $A1xB,$Ni,${N0}[1]
 300         vmlal.u32       $A2xB,$Ni,${N1}[0]
 301         vmlal.u32       $A3xB,$Ni,${N1}[1]
 302
 303         vmlal.u32       $A4xB,$Ni,${N2}[0]
 304         vmov            $Temp,$A0xB
 305         vmlal.u32       $A5xB,$Ni,${N2}[1]
 306         vmov            $A0xB,$A1xB
 307         vmlal.u32       $A6xB,$Ni,${N3}[0]
 308         vmov            $A1xB,$A2xB
 309         vmlal.u32       $A7xB,$Ni,${N3}[1]
 310         vmov            $A2xB,$A3xB
 311         vmov            $A3xB,$A4xB
 312         vshr.u64        $temp,$temp,#16
 313         vmov            $A4xB,$A5xB
 314         vmov            $A5xB,$A6xB
 315         vadd.u64        $temp,$temp,`&Dhi("$Temp")`
 316         vmov            $A6xB,$A7xB
 317         veor            $A7xB,$A7xB
 318         vshr.u64        $temp,$temp,#16
 319
 320         b       .LNEON_outer8
 321
 322 .align  4
 323 .LNEON_outer8:
 324         vld1.32         {${Bi}[0]}, [$bptr,:32]!
 325         veor            $zero,$zero,$zero
 326         vzip.16         $Bi,$zero
 327         vadd.u64        `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
 328
 329         vmlal.u32       $A0xB,$Bi,${A0}[0]
 330         vmlal.u32       $A1xB,$Bi,${A0}[1]
 331         vmlal.u32       $A2xB,$Bi,${A1}[0]
 332         vshl.i64        $temp,`&Dhi("$A0xB")`,#16
 333         vmlal.u32       $A3xB,$Bi,${A1}[1]
 334
 335         vadd.u64        $temp,$temp,`&Dlo("$A0xB")`
 336         veor            $zero,$zero,$zero
 337         subs            $outer,$outer,#1
 338         vmul.u32        $Ni,$temp,$M0
 339
 340         vmlal.u32       $A4xB,$Bi,${A2}[0]
 341         vmlal.u32       $A5xB,$Bi,${A2}[1]
 342         vmlal.u32       $A6xB,$Bi,${A3}[0]
 343         vzip.16         $Ni,$zero
 344         vmlal.u32       $A7xB,$Bi,${A3}[1]
 345
 346         vmlal.u32       $A0xB,$Ni,${N0}[0]
 347         vmlal.u32       $A1xB,$Ni,${N0}[1]
 348         vmlal.u32       $A2xB,$Ni,${N1}[0]
 349         vmlal.u32       $A3xB,$Ni,${N1}[1]
 350
 351         vmlal.u32       $A4xB,$Ni,${N2}[0]
 352         vmov            $Temp,$A0xB
 353         vmlal.u32       $A5xB,$Ni,${N2}[1]
 354         vmov            $A0xB,$A1xB
 355         vmlal.u32       $A6xB,$Ni,${N3}[0]
 356         vmov            $A1xB,$A2xB
 357         vmlal.u32       $A7xB,$Ni,${N3}[1]
 358         vmov            $A2xB,$A3xB
 359         vmov            $A3xB,$A4xB
 360         vshr.u64        $temp,$temp,#16
 361         vmov            $A4xB,$A5xB
 362         vmov            $A5xB,$A6xB
 363         vadd.u64        $temp,$temp,`&Dhi("$Temp")`
 364         vmov            $A6xB,$A7xB
 365         veor            $A7xB,$A7xB
 366         vshr.u64        $temp,$temp,#16
 367
 368         bne     .LNEON_outer8
 369
 370         vadd.u64        `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
 371         mov             $toutptr,sp
 372         vshr.u64        $temp,`&Dlo("$A0xB")`,#16
 373         mov             $inner,$num
 374         vadd.u64        `&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp
 375         add             $tinptr,sp,#16
 376         vshr.u64        $temp,`&Dhi("$A0xB")`,#16
 377         vzip.16         `&Dlo("$A0xB")`,`&Dhi("$A0xB")`
 378
 379         b       .LNEON_tail2
 380
 381 .align  4
 382 .LNEON_1st:
 383         vmlal.u32       $A0xB,$Ni,${N0}[0]
 384          vld1.32        {$A0-$A3}, [$aptr]!
 385         vmlal.u32       $A1xB,$Ni,${N0}[1]
 386         subs            $inner,$inner,#8
 387         vmlal.u32       $A2xB,$Ni,${N1}[0]
 388         vmlal.u32       $A3xB,$Ni,${N1}[1]
 389
 390         vmlal.u32       $A4xB,$Ni,${N2}[0]
 391          vld1.32        {$N0-$N1}, [$nptr]!
 392         vmlal.u32       $A5xB,$Ni,${N2}[1]
 393          vst1.64        {$A0xB-$A1xB}, [$toutptr,:256]!
 394         vmlal.u32       $A6xB,$Ni,${N3}[0]
 395         vmlal.u32       $A7xB,$Ni,${N3}[1]
 396          vst1.64        {$A2xB-$A3xB}, [$toutptr,:256]!
 397
 398         vmull.u32       $A0xB,$Bi,${A0}[0]
 399          vld1.32        {$N2-$N3}, [$nptr]!
 400         vmull.u32       $A1xB,$Bi,${A0}[1]
 401          vst1.64        {$A4xB-$A5xB}, [$toutptr,:256]!
 402         vmull.u32       $A2xB,$Bi,${A1}[0]
 403         vmull.u32       $A3xB,$Bi,${A1}[1]
 404          vst1.64        {$A6xB-$A7xB}, [$toutptr,:256]!
 405
 406         vmull.u32       $A4xB,$Bi,${A2}[0]
 407         vmull.u32       $A5xB,$Bi,${A2}[1]
 408         vmull.u32       $A6xB,$Bi,${A3}[0]
 409         vmull.u32       $A7xB,$Bi,${A3}[1]
 410
 411         bne     .LNEON_1st
 412
 413         vmlal.u32       $A0xB,$Ni,${N0}[0]
 414         add             $tinptr,sp,#16
 415         vmlal.u32       $A1xB,$Ni,${N0}[1]
 416         sub             $aptr,$aptr,$num,lsl#2          @ rewind $aptr
 417         vmlal.u32       $A2xB,$Ni,${N1}[0]
 418          vld1.64        {$Temp}, [sp,:128]
 419         vmlal.u32       $A3xB,$Ni,${N1}[1]
 420         sub             $outer,$num,#1
 421
 422         vmlal.u32       $A4xB,$Ni,${N2}[0]
 423         vst1.64         {$A0xB-$A1xB}, [$toutptr,:256]!
 424         vmlal.u32       $A5xB,$Ni,${N2}[1]
 425         vshr.u64        $temp,$temp,#16
 426          vld1.64        {$A0xB},       [$tinptr, :128]!
 427         vmlal.u32       $A6xB,$Ni,${N3}[0]
 428         vst1.64         {$A2xB-$A3xB}, [$toutptr,:256]!
 429         vmlal.u32       $A7xB,$Ni,${N3}[1]
 430
 431         vst1.64         {$A4xB-$A5xB}, [$toutptr,:256]!
 432         vadd.u64        $temp,$temp,`&Dhi("$Temp")`
 433         veor            $Z,$Z,$Z
 434         vst1.64         {$A6xB-$A7xB}, [$toutptr,:256]!
 435          vld1.64        {$A1xB-$A2xB}, [$tinptr, :256]!
 436         vst1.64         {$Z},          [$toutptr,:128]
 437         vshr.u64        $temp,$temp,#16
 438
 439         b               .LNEON_outer
 440
 441 .align  4
 442 .LNEON_outer:
 443         vld1.32         {${Bi}[0]}, [$bptr,:32]!
 444         sub             $nptr,$nptr,$num,lsl#2          @ rewind $nptr
 445         vld1.32         {$A0-$A3},  [$aptr]!
 446         veor            $zero,$zero,$zero
 447         mov             $toutptr,sp
 448         vzip.16         $Bi,$zero
 449         sub             $inner,$num,#8
 450         vadd.u64        `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
 451
 452         vmlal.u32       $A0xB,$Bi,${A0}[0]
 453          vld1.64        {$A3xB-$A4xB},[$tinptr,:256]!
 454         vmlal.u32       $A1xB,$Bi,${A0}[1]
 455         vmlal.u32       $A2xB,$Bi,${A1}[0]
 456          vld1.64        {$A5xB-$A6xB},[$tinptr,:256]!
 457         vmlal.u32       $A3xB,$Bi,${A1}[1]
 458
 459         vshl.i64        $temp,`&Dhi("$A0xB")`,#16
 460         veor            $zero,$zero,$zero
 461         vadd.u64        $temp,$temp,`&Dlo("$A0xB")`
 462          vld1.64        {$A7xB},[$tinptr,:128]!
 463         vmul.u32        $Ni,$temp,$M0
 464
 465         vmlal.u32       $A4xB,$Bi,${A2}[0]
 466          vld1.32        {$N0-$N3}, [$nptr]!
 467         vmlal.u32       $A5xB,$Bi,${A2}[1]
 468         vmlal.u32       $A6xB,$Bi,${A3}[0]
 469         vzip.16         $Ni,$zero
 470         vmlal.u32       $A7xB,$Bi,${A3}[1]
 471
 472 .LNEON_inner:
 473         vmlal.u32       $A0xB,$Ni,${N0}[0]
 474          vld1.32        {$A0-$A3}, [$aptr]!
 475         vmlal.u32       $A1xB,$Ni,${N0}[1]
 476          subs           $inner,$inner,#8
 477         vmlal.u32       $A2xB,$Ni,${N1}[0]
 478         vmlal.u32       $A3xB,$Ni,${N1}[1]
 479         vst1.64         {$A0xB-$A1xB}, [$toutptr,:256]!
 480
 481         vmlal.u32       $A4xB,$Ni,${N2}[0]
 482          vld1.64        {$A0xB},       [$tinptr, :128]!
 483         vmlal.u32       $A5xB,$Ni,${N2}[1]
 484         vst1.64         {$A2xB-$A3xB}, [$toutptr,:256]!
 485         vmlal.u32       $A6xB,$Ni,${N3}[0]
 486          vld1.64        {$A1xB-$A2xB}, [$tinptr, :256]!
 487         vmlal.u32       $A7xB,$Ni,${N3}[1]
 488         vst1.64         {$A4xB-$A5xB}, [$toutptr,:256]!
 489
 490         vmlal.u32       $A0xB,$Bi,${A0}[0]
 491          vld1.64        {$A3xB-$A4xB}, [$tinptr, :256]!
 492         vmlal.u32       $A1xB,$Bi,${A0}[1]
 493         vst1.64         {$A6xB-$A7xB}, [$toutptr,:256]!
 494         vmlal.u32       $A2xB,$Bi,${A1}[0]
 495          vld1.64        {$A5xB-$A6xB}, [$tinptr, :256]!
 496         vmlal.u32       $A3xB,$Bi,${A1}[1]
 497          vld1.32        {$N0-$N3}, [$nptr]!
 498
 499         vmlal.u32       $A4xB,$Bi,${A2}[0]
 500          vld1.64        {$A7xB},       [$tinptr, :128]!
 501         vmlal.u32       $A5xB,$Bi,${A2}[1]
 502         vmlal.u32       $A6xB,$Bi,${A3}[0]
 503         vmlal.u32       $A7xB,$Bi,${A3}[1]
 504
 505         bne     .LNEON_inner
 506
 507         vmlal.u32       $A0xB,$Ni,${N0}[0]
 508         add             $tinptr,sp,#16
 509         vmlal.u32       $A1xB,$Ni,${N0}[1]
 510         sub             $aptr,$aptr,$num,lsl#2          @ rewind $aptr
 511         vmlal.u32       $A2xB,$Ni,${N1}[0]
 512          vld1.64        {$Temp}, [sp,:128]
 513         vmlal.u32       $A3xB,$Ni,${N1}[1]
 514         subs            $outer,$outer,#1
 515
 516         vmlal.u32       $A4xB,$Ni,${N2}[0]
 517         vst1.64         {$A0xB-$A1xB}, [$toutptr,:256]!
 518         vmlal.u32       $A5xB,$Ni,${N2}[1]
 519          vld1.64        {$A0xB},       [$tinptr, :128]!
 520         vshr.u64        $temp,$temp,#16
 521         vst1.64         {$A2xB-$A3xB}, [$toutptr,:256]!
 522         vmlal.u32       $A6xB,$Ni,${N3}[0]
 523          vld1.64        {$A1xB-$A2xB}, [$tinptr, :256]!
 524         vmlal.u32       $A7xB,$Ni,${N3}[1]
 525
 526         vst1.64         {$A4xB-$A5xB}, [$toutptr,:256]!
 527         vadd.u64        $temp,$temp,`&Dhi("$Temp")`
 528         vst1.64         {$A6xB-$A7xB}, [$toutptr,:256]!
 529         vshr.u64        $temp,$temp,#16
 530
 531         bne     .LNEON_outer
 532
 533         mov             $toutptr,sp
 534         mov             $inner,$num
 535
 536 .LNEON_tail:
 537         vadd.u64        `&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
 538         vld1.64         {$A3xB-$A4xB}, [$tinptr, :256]!
 539         vshr.u64        $temp,`&Dlo("$A0xB")`,#16
 540         vadd.u64        `&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp
 541         vld1.64         {$A5xB-$A6xB}, [$tinptr, :256]!
 542         vshr.u64        $temp,`&Dhi("$A0xB")`,#16
 543         vld1.64         {$A7xB},       [$tinptr, :128]!
 544         vzip.16         `&Dlo("$A0xB")`,`&Dhi("$A0xB")`
 545
 546 .LNEON_tail2:
 547         vadd.u64        `&Dlo("$A1xB")`,`&Dlo("$A1xB")`,$temp
 548         vst1.32         {`&Dlo("$A0xB")`[0]}, [$toutptr, :32]!
 549         vshr.u64        $temp,`&Dlo("$A1xB")`,#16
 550         vadd.u64        `&Dhi("$A1xB")`,`&Dhi("$A1xB")`,$temp
 551         vshr.u64        $temp,`&Dhi("$A1xB")`,#16
 552         vzip.16         `&Dlo("$A1xB")`,`&Dhi("$A1xB")`
 553
 554         vadd.u64        `&Dlo("$A2xB")`,`&Dlo("$A2xB")`,$temp
 555         vst1.32         {`&Dlo("$A1xB")`[0]}, [$toutptr, :32]!
 556         vshr.u64        $temp,`&Dlo("$A2xB")`,#16
 557         vadd.u64        `&Dhi("$A2xB")`,`&Dhi("$A2xB")`,$temp
 558         vshr.u64        $temp,`&Dhi("$A2xB")`,#16
 559         vzip.16         `&Dlo("$A2xB")`,`&Dhi("$A2xB")`
 560
 561         vadd.u64        `&Dlo("$A3xB")`,`&Dlo("$A3xB")`,$temp
 562         vst1.32         {`&Dlo("$A2xB")`[0]}, [$toutptr, :32]!
 563         vshr.u64        $temp,`&Dlo("$A3xB")`,#16
 564         vadd.u64        `&Dhi("$A3xB")`,`&Dhi("$A3xB")`,$temp
 565         vshr.u64        $temp,`&Dhi("$A3xB")`,#16
 566         vzip.16         `&Dlo("$A3xB")`,`&Dhi("$A3xB")`
 567
 568         vadd.u64        `&Dlo("$A4xB")`,`&Dlo("$A4xB")`,$temp
 569         vst1.32         {`&Dlo("$A3xB")`[0]}, [$toutptr, :32]!
 570         vshr.u64        $temp,`&Dlo("$A4xB")`,#16
 571         vadd.u64        `&Dhi("$A4xB")`,`&Dhi("$A4xB")`,$temp
 572         vshr.u64        $temp,`&Dhi("$A4xB")`,#16
 573         vzip.16         `&Dlo("$A4xB")`,`&Dhi("$A4xB")`
 574
 575         vadd.u64        `&Dlo("$A5xB")`,`&Dlo("$A5xB")`,$temp
 576         vst1.32         {`&Dlo("$A4xB")`[0]}, [$toutptr, :32]!
 577         vshr.u64        $temp,`&Dlo("$A5xB")`,#16
 578         vadd.u64        `&Dhi("$A5xB")`,`&Dhi("$A5xB")`,$temp
 579         vshr.u64        $temp,`&Dhi("$A5xB")`,#16
 580         vzip.16         `&Dlo("$A5xB")`,`&Dhi("$A5xB")`
 581
 582         vadd.u64        `&Dlo("$A6xB")`,`&Dlo("$A6xB")`,$temp
 583         vst1.32         {`&Dlo("$A5xB")`[0]}, [$toutptr, :32]!
 584         vshr.u64        $temp,`&Dlo("$A6xB")`,#16
 585         vadd.u64        `&Dhi("$A6xB")`,`&Dhi("$A6xB")`,$temp
 586         vld1.64         {$A0xB}, [$tinptr, :128]!
 587         vshr.u64        $temp,`&Dhi("$A6xB")`,#16
 588         vzip.16         `&Dlo("$A6xB")`,`&Dhi("$A6xB")`
 589
 590         vadd.u64        `&Dlo("$A7xB")`,`&Dlo("$A7xB")`,$temp
 591         vst1.32         {`&Dlo("$A6xB")`[0]}, [$toutptr, :32]!
 592         vshr.u64        $temp,`&Dlo("$A7xB")`,#16
 593         vadd.u64        `&Dhi("$A7xB")`,`&Dhi("$A7xB")`,$temp
 594         vld1.64         {$A1xB-$A2xB},  [$tinptr, :256]!
 595         vshr.u64        $temp,`&Dhi("$A7xB")`,#16
 596         vzip.16         `&Dlo("$A7xB")`,`&Dhi("$A7xB")`
 597         subs            $inner,$inner,#8
 598         vst1.32         {`&Dlo("$A7xB")`[0]}, [$toutptr, :32]!
 599
 600         bne     .LNEON_tail
 601
 602         vst1.32 {${temp}[0]}, [$toutptr, :32]           @ top-most bit
 603         sub     $nptr,$nptr,$num,lsl#2                  @ rewind $nptr
 604         subs    $aptr,sp,#0                             @ clear carry flag
 605         add     $bptr,sp,$num,lsl#2
 606
 607 .LNEON_sub:
 608         ldmia   $aptr!, {r4-r7}
 609         ldmia   $nptr!, {r8-r11}
 610         sbcs    r8, r4,r8
 611         sbcs    r9, r5,r9
 612         sbcs    r10,r6,r10
 613         sbcs    r11,r7,r11
 614         teq     $aptr,$bptr                             @ preserves carry
 615         stmia   $rptr!, {r8-r11}
 616         bne     .LNEON_sub
 617
 618         ldr     r10, [$aptr]                            @ load top-most bit
 619         veor    q0,q0,q0
 620         sub     r11,$bptr,sp                            @ this is num*4
 621         veor    q1,q1,q1
 622         mov     $aptr,sp
 623         sub     $rptr,$rptr,r11                         @ rewind $rptr
 624         mov     $nptr,$bptr                             @ second 3/4th of frame
 625         sbcs    r10,r10,#0                              @ result is carry flag
 626
 627 .LNEON_copy_n_zap:
 628         ldmia   $aptr!, {r4-r7}
 629         ldmia   $rptr,  {r8-r11}
 630         movcc   r8, r4
 631         vst1.64 {q0-q1}, [$nptr,:256]!                  @ wipe
 632         movcc   r9, r5
 633         movcc   r10,r6
 634         vst1.64 {q0-q1}, [$nptr,:256]!                  @ wipe
 635         movcc   r11,r7
 636         ldmia   $aptr, {r4-r7}
 637         stmia   $rptr!, {r8-r11}
 638         sub     $aptr,$aptr,#16
 639         ldmia   $rptr, {r8-r11}
 640         movcc   r8, r4
 641         vst1.64 {q0-q1}, [$aptr,:256]!                  @ wipe
 642         movcc   r9, r5
 643         movcc   r10,r6
 644         vst1.64 {q0-q1}, [$nptr,:256]!                  @ wipe
 645         movcc   r11,r7
 646         teq     $aptr,$bptr                             @ preserves carry
 647         stmia   $rptr!, {r8-r11}
 648         bne     .LNEON_copy_n_zap
 649
 650         sub     sp,ip,#96
 651         vldmia  sp!,{d8-d15}
 652         ldmia   sp!,{r4-r11}
 653         bx      lr
 654 .size   bn_mul8x_mont_neon,.-bn_mul8x_mont_neon
 655 #endif
 656 ___
 657 }
 658 $code.=<<___;
 659 .asciz  "Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
 660 .align  2
 661 #if __ARM_ARCH__>=7
 662 .comm   OPENSSL_armcap_P,4,4
 663 #endif
 664 ___
 665
 666 $code =~ s/\`([^\`]*)\`/eval $1/gem;
 667 $code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm;    # make it possible to compile with -march=armv4
 668 print $code;
 669 close STDOUT;