crypto/ec/asm/ecp_nistz256-x86.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # ECP_NISTZ256 module for x86/SSE2.
  18 #
  19 # October 2014.
  20 #
  21 # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
  22 # http://eprint.iacr.org/2013/816. In the process of adaptation
  23 # original .c module was made 32-bit savvy in order to make this
  24 # implementation possible.
  25 #
  26 #               with/without -DECP_NISTZ256_ASM
  27 # Pentium       +66-163%
  28 # PIII          +72-172%
  29 # P4            +65-132%
  30 # Core2         +90-215%
  31 # Sandy Bridge  +105-265% (contemporary i[57]-* are all close to this)
  32 # Atom          +65-155%
  33 # Opteron       +54-110%
  34 # Bulldozer     +99-240%
  35 # VIA Nano      +93-290%
  36 #
  37 # Ranges denote minimum and maximum improvement coefficients depending
  38 # on benchmark. Lower coefficients are for ECDSA sign, server-side
  39 # operation. Keep in mind that +200% means 3x improvement.
  40
  41 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  42 push(@INC,"${dir}","${dir}../../perlasm");
  43 require "x86asm.pl";
  44
  45 $output=pop and open STDOUT,">$output";
  46
  47 &asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
  48
  49 $sse2=0;
  50 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
  51
  52 &external_label("OPENSSL_ia32cap_P") if ($sse2);
  53
  54
  55 ########################################################################
  56 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
  57 #
  58 open TABLE,"<ecp_nistz256_table.c"              or
  59 open TABLE,"<${dir}../ecp_nistz256_table.c"     or
  60 die "failed to open ecp_nistz256_table.c:",$!;
  61
  62 use integer;
  63
  64 foreach(<TABLE>) {
  65         s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
  66 }
  67 close TABLE;
  68
  69 # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
  70 # 64*16*37-1 is because $#arr returns last valid index or @arr, not
  71 # amount of elements.
  72 die "insane number of elements" if ($#arr != 64*16*37-1);
  73
  74 &public_label("ecp_nistz256_precomputed");
  75 &align(4096);
  76 &set_label("ecp_nistz256_precomputed");
  77
  78 ########################################################################
  79 # this conversion smashes P256_POINT_AFFINE by individual bytes with
  80 # 64 byte interval, similar to
  81 #       1111222233334444
  82 #       1234123412341234
  83 for(1..37) {
  84         @tbl = splice(@arr,0,64*16);
  85         for($i=0;$i<64;$i++) {
  86                 undef @line;
  87                 for($j=0;$j<64;$j++) {
  88                         push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
  89                 }
  90                 &data_byte(join(',',map { sprintf "0x%02x",$_} @line));
  91         }
  92 }
  93
  94 ########################################################################
  95 # Keep in mind that constants are stored least to most significant word
  96 &static_label("RR");
  97 &set_label("RR",64);
  98 &data_word(3,0,-1,-5,-2,-1,-3,4);       # 2^512 mod P-256
  99
 100 &static_label("ONE_mont");
 101 &set_label("ONE_mont");
 102 &data_word(1,0,0,-1,-1,-1,-2,0);
 103
 104 &static_label("ONE");
 105 &set_label("ONE");
 106 &data_word(1,0,0,0,0,0,0,0);
 107 &asciz("ECP_NISZ256 for x86/SSE2, CRYPTOGAMS by <appro\@openssl.org>");
 108 &align(64);
 109
 110 ########################################################################
 111 # void ecp_nistz256_mul_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
 112 &function_begin("ecp_nistz256_mul_by_2");
 113         &mov    ("esi",&wparam(1));
 114         &mov    ("edi",&wparam(0));
 115         &mov    ("ebp","esi");
 116 ########################################################################
 117 # common pattern for internal functions is that %edi is result pointer,
 118 # %esi and %ebp are input ones, %ebp being optional. %edi is preserved.
 119         &call   ("_ecp_nistz256_add");
 120 &function_end("ecp_nistz256_mul_by_2");
 121
 122 ########################################################################
 123 # void ecp_nistz256_mul_by_3(BN_ULONG edi[8],const BN_ULONG esi[8]);
 124 &function_begin("ecp_nistz256_mul_by_3");
 125         &mov    ("esi",&wparam(1));
 126                                         # multiplication by 3 is performed
 127                                         # as 2*n+n, but we can't use output
 128                                         # to store 2*n, because if output
 129                                         # pointer equals to input, then
 130                                         # we'll get 2*n+2*n.
 131         &stack_push(8);                 # therefore we need to allocate
 132                                         # 256-bit intermediate buffer.
 133         &mov    ("edi","esp");
 134         &mov    ("ebp","esi");
 135         &call   ("_ecp_nistz256_add");
 136         &lea    ("esi",&DWP(0,"edi"));
 137         &mov    ("ebp",&wparam(1));
 138         &mov    ("edi",&wparam(0));
 139         &call   ("_ecp_nistz256_add");
 140         &stack_pop(8);
 141 &function_end("ecp_nistz256_mul_by_3");
 142
 143 ########################################################################
 144 # void ecp_nistz256_div_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
 145 &function_begin("ecp_nistz256_div_by_2");
 146         &mov    ("esi",&wparam(1));
 147         &mov    ("edi",&wparam(0));
 148         &call   ("_ecp_nistz256_div_by_2");
 149 &function_end("ecp_nistz256_div_by_2");
 150
 151 &function_begin_B("_ecp_nistz256_div_by_2");
 152         # tmp = a is odd ? a+mod : a
 153         #
 154         # note that because mod has special form, i.e. consists of
 155         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 156         # assigning least significant bit of input to one register,
 157         # %ebp, and its negative to another, %edx.
 158
 159         &mov    ("ebp",&DWP(0,"esi"));
 160         &xor    ("edx","edx");
 161         &mov    ("ebx",&DWP(4,"esi"));
 162         &mov    ("eax","ebp");
 163         &and    ("ebp",1);
 164         &mov    ("ecx",&DWP(8,"esi"));
 165         &sub    ("edx","ebp");
 166
 167         &add    ("eax","edx");
 168         &adc    ("ebx","edx");
 169         &mov    (&DWP(0,"edi"),"eax");
 170         &adc    ("ecx","edx");
 171         &mov    (&DWP(4,"edi"),"ebx");
 172         &mov    (&DWP(8,"edi"),"ecx");
 173
 174         &mov    ("eax",&DWP(12,"esi"));
 175         &mov    ("ebx",&DWP(16,"esi"));
 176         &adc    ("eax",0);
 177         &mov    ("ecx",&DWP(20,"esi"));
 178         &adc    ("ebx",0);
 179         &mov    (&DWP(12,"edi"),"eax");
 180         &adc    ("ecx",0);
 181         &mov    (&DWP(16,"edi"),"ebx");
 182         &mov    (&DWP(20,"edi"),"ecx");
 183
 184         &mov    ("eax",&DWP(24,"esi"));
 185         &mov    ("ebx",&DWP(28,"esi"));
 186         &adc    ("eax","ebp");
 187         &adc    ("ebx","edx");
 188         &mov    (&DWP(24,"edi"),"eax");
 189         &sbb    ("esi","esi");                  # broadcast carry bit
 190         &mov    (&DWP(28,"edi"),"ebx");
 191
 192         # ret = tmp >> 1
 193
 194         &mov    ("eax",&DWP(0,"edi"));
 195         &mov    ("ebx",&DWP(4,"edi"));
 196         &mov    ("ecx",&DWP(8,"edi"));
 197         &mov    ("edx",&DWP(12,"edi"));
 198
 199         &shr    ("eax",1);
 200         &mov    ("ebp","ebx");
 201         &shl    ("ebx",31);
 202         &or     ("eax","ebx");
 203
 204         &shr    ("ebp",1);
 205         &mov    ("ebx","ecx");
 206         &shl    ("ecx",31);
 207         &mov    (&DWP(0,"edi"),"eax");
 208         &or     ("ebp","ecx");
 209         &mov    ("eax",&DWP(16,"edi"));
 210
 211         &shr    ("ebx",1);
 212         &mov    ("ecx","edx");
 213         &shl    ("edx",31);
 214         &mov    (&DWP(4,"edi"),"ebp");
 215         &or     ("ebx","edx");
 216         &mov    ("ebp",&DWP(20,"edi"));
 217
 218         &shr    ("ecx",1);
 219         &mov    ("edx","eax");
 220         &shl    ("eax",31);
 221         &mov    (&DWP(8,"edi"),"ebx");
 222         &or     ("ecx","eax");
 223         &mov    ("ebx",&DWP(24,"edi"));
 224
 225         &shr    ("edx",1);
 226         &mov    ("eax","ebp");
 227         &shl    ("ebp",31);
 228         &mov    (&DWP(12,"edi"),"ecx");
 229         &or     ("edx","ebp");
 230         &mov    ("ecx",&DWP(28,"edi"));
 231
 232         &shr    ("eax",1);
 233         &mov    ("ebp","ebx");
 234         &shl    ("ebx",31);
 235         &mov    (&DWP(16,"edi"),"edx");
 236         &or     ("eax","ebx");
 237
 238         &shr    ("ebp",1);
 239         &mov    ("ebx","ecx");
 240         &shl    ("ecx",31);
 241         &mov    (&DWP(20,"edi"),"eax");
 242         &or     ("ebp","ecx");
 243
 244         &shr    ("ebx",1);
 245         &shl    ("esi",31);
 246         &mov    (&DWP(24,"edi"),"ebp");
 247         &or     ("ebx","esi");                  # handle top-most carry bit
 248         &mov    (&DWP(28,"edi"),"ebx");
 249
 250         &ret    ();
 251 &function_end_B("_ecp_nistz256_div_by_2");
 252
 253 ########################################################################
 254 # void ecp_nistz256_add(BN_ULONG edi[8],const BN_ULONG esi[8],
 255 #                                       const BN_ULONG ebp[8]);
 256 &function_begin("ecp_nistz256_add");
 257         &mov    ("esi",&wparam(1));
 258         &mov    ("ebp",&wparam(2));
 259         &mov    ("edi",&wparam(0));
 260         &call   ("_ecp_nistz256_add");
 261 &function_end("ecp_nistz256_add");
 262
 263 &function_begin_B("_ecp_nistz256_add");
 264         &mov    ("eax",&DWP(0,"esi"));
 265         &mov    ("ebx",&DWP(4,"esi"));
 266         &mov    ("ecx",&DWP(8,"esi"));
 267         &add    ("eax",&DWP(0,"ebp"));
 268         &mov    ("edx",&DWP(12,"esi"));
 269         &adc    ("ebx",&DWP(4,"ebp"));
 270         &mov    (&DWP(0,"edi"),"eax");
 271         &adc    ("ecx",&DWP(8,"ebp"));
 272         &mov    (&DWP(4,"edi"),"ebx");
 273         &adc    ("edx",&DWP(12,"ebp"));
 274         &mov    (&DWP(8,"edi"),"ecx");
 275         &mov    (&DWP(12,"edi"),"edx");
 276
 277         &mov    ("eax",&DWP(16,"esi"));
 278         &mov    ("ebx",&DWP(20,"esi"));
 279         &mov    ("ecx",&DWP(24,"esi"));
 280         &adc    ("eax",&DWP(16,"ebp"));
 281         &mov    ("edx",&DWP(28,"esi"));
 282         &adc    ("ebx",&DWP(20,"ebp"));
 283         &mov    (&DWP(16,"edi"),"eax");
 284         &adc    ("ecx",&DWP(24,"ebp"));
 285         &mov    (&DWP(20,"edi"),"ebx");
 286         &mov    ("esi",0);
 287         &adc    ("edx",&DWP(28,"ebp"));
 288         &mov    (&DWP(24,"edi"),"ecx");
 289         &adc    ("esi",0);
 290         &mov    (&DWP(28,"edi"),"edx");
 291
 292         # if a+b >= modulus, subtract modulus.
 293         #
 294         # But since comparison implies subtraction, we subtract modulus
 295         # to see if it borrows, and then subtract it for real if
 296         # subtraction didn't borrow.
 297
 298         &mov    ("eax",&DWP(0,"edi"));
 299         &mov    ("ebx",&DWP(4,"edi"));
 300         &mov    ("ecx",&DWP(8,"edi"));
 301         &sub    ("eax",-1);
 302         &mov    ("edx",&DWP(12,"edi"));
 303         &sbb    ("ebx",-1);
 304         &mov    ("eax",&DWP(16,"edi"));
 305         &sbb    ("ecx",-1);
 306         &mov    ("ebx",&DWP(20,"edi"));
 307         &sbb    ("edx",0);
 308         &mov    ("ecx",&DWP(24,"edi"));
 309         &sbb    ("eax",0);
 310         &mov    ("edx",&DWP(28,"edi"));
 311         &sbb    ("ebx",0);
 312         &sbb    ("ecx",1);
 313         &sbb    ("edx",-1);
 314         &sbb    ("esi",0);
 315
 316         # Note that because mod has special form, i.e. consists of
 317         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 318         # by using borrow.
 319
 320         &not    ("esi");
 321         &mov    ("eax",&DWP(0,"edi"));
 322         &mov    ("ebp","esi");
 323         &mov    ("ebx",&DWP(4,"edi"));
 324         &shr    ("ebp",31);
 325         &mov    ("ecx",&DWP(8,"edi"));
 326         &sub    ("eax","esi");
 327         &mov    ("edx",&DWP(12,"edi"));
 328         &sbb    ("ebx","esi");
 329         &mov    (&DWP(0,"edi"),"eax");
 330         &sbb    ("ecx","esi");
 331         &mov    (&DWP(4,"edi"),"ebx");
 332         &sbb    ("edx",0);
 333         &mov    (&DWP(8,"edi"),"ecx");
 334         &mov    (&DWP(12,"edi"),"edx");
 335
 336         &mov    ("eax",&DWP(16,"edi"));
 337         &mov    ("ebx",&DWP(20,"edi"));
 338         &mov    ("ecx",&DWP(24,"edi"));
 339         &sbb    ("eax",0);
 340         &mov    ("edx",&DWP(28,"edi"));
 341         &sbb    ("ebx",0);
 342         &mov    (&DWP(16,"edi"),"eax");
 343         &sbb    ("ecx","ebp");
 344         &mov    (&DWP(20,"edi"),"ebx");
 345         &sbb    ("edx","esi");
 346         &mov    (&DWP(24,"edi"),"ecx");
 347         &mov    (&DWP(28,"edi"),"edx");
 348
 349         &ret    ();
 350 &function_end_B("_ecp_nistz256_add");
 351
 352 ########################################################################
 353 # void ecp_nistz256_sub(BN_ULONG edi[8],const BN_ULONG esi[8],
 354 #                                       const BN_ULONG ebp[8]);
 355 &function_begin("ecp_nistz256_sub");
 356         &mov    ("esi",&wparam(1));
 357         &mov    ("ebp",&wparam(2));
 358         &mov    ("edi",&wparam(0));
 359         &call   ("_ecp_nistz256_sub");
 360 &function_end("ecp_nistz256_sub");
 361
 362 &function_begin_B("_ecp_nistz256_sub");
 363         &mov    ("eax",&DWP(0,"esi"));
 364         &mov    ("ebx",&DWP(4,"esi"));
 365         &mov    ("ecx",&DWP(8,"esi"));
 366         &sub    ("eax",&DWP(0,"ebp"));
 367         &mov    ("edx",&DWP(12,"esi"));
 368         &sbb    ("ebx",&DWP(4,"ebp"));
 369         &mov    (&DWP(0,"edi"),"eax");
 370         &sbb    ("ecx",&DWP(8,"ebp"));
 371         &mov    (&DWP(4,"edi"),"ebx");
 372         &sbb    ("edx",&DWP(12,"ebp"));
 373         &mov    (&DWP(8,"edi"),"ecx");
 374         &mov    (&DWP(12,"edi"),"edx");
 375
 376         &mov    ("eax",&DWP(16,"esi"));
 377         &mov    ("ebx",&DWP(20,"esi"));
 378         &mov    ("ecx",&DWP(24,"esi"));
 379         &sbb    ("eax",&DWP(16,"ebp"));
 380         &mov    ("edx",&DWP(28,"esi"));
 381         &sbb    ("ebx",&DWP(20,"ebp"));
 382         &sbb    ("ecx",&DWP(24,"ebp"));
 383         &mov    (&DWP(16,"edi"),"eax");
 384         &sbb    ("edx",&DWP(28,"ebp"));
 385         &mov    (&DWP(20,"edi"),"ebx");
 386         &sbb    ("esi","esi");                  # broadcast borrow bit
 387         &mov    (&DWP(24,"edi"),"ecx");
 388         &mov    (&DWP(28,"edi"),"edx");
 389
 390         # if a-b borrows, add modulus.
 391         #
 392         # Note that because mod has special form, i.e. consists of
 393         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 394         # assigning borrow bit to one register, %ebp, and its negative
 395         # to another, %esi. But we started by calculating %esi...
 396
 397         &mov    ("eax",&DWP(0,"edi"));
 398         &mov    ("ebp","esi");
 399         &mov    ("ebx",&DWP(4,"edi"));
 400         &shr    ("ebp",31);
 401         &mov    ("ecx",&DWP(8,"edi"));
 402         &add    ("eax","esi");
 403         &mov    ("edx",&DWP(12,"edi"));
 404         &adc    ("ebx","esi");
 405         &mov    (&DWP(0,"edi"),"eax");
 406         &adc    ("ecx","esi");
 407         &mov    (&DWP(4,"edi"),"ebx");
 408         &adc    ("edx",0);
 409         &mov    (&DWP(8,"edi"),"ecx");
 410         &mov    (&DWP(12,"edi"),"edx");
 411
 412         &mov    ("eax",&DWP(16,"edi"));
 413         &mov    ("ebx",&DWP(20,"edi"));
 414         &mov    ("ecx",&DWP(24,"edi"));
 415         &adc    ("eax",0);
 416         &mov    ("edx",&DWP(28,"edi"));
 417         &adc    ("ebx",0);
 418         &mov    (&DWP(16,"edi"),"eax");
 419         &adc    ("ecx","ebp");
 420         &mov    (&DWP(20,"edi"),"ebx");
 421         &adc    ("edx","esi");
 422         &mov    (&DWP(24,"edi"),"ecx");
 423         &mov    (&DWP(28,"edi"),"edx");
 424
 425         &ret    ();
 426 &function_end_B("_ecp_nistz256_sub");
 427
 428 ########################################################################
 429 # void ecp_nistz256_neg(BN_ULONG edi[8],const BN_ULONG esi[8]);
 430 &function_begin("ecp_nistz256_neg");
 431         &mov    ("ebp",&wparam(1));
 432         &mov    ("edi",&wparam(0));
 433
 434         &xor    ("eax","eax");
 435         &stack_push(8);
 436         &mov    (&DWP(0,"esp"),"eax");
 437         &mov    ("esi","esp");
 438         &mov    (&DWP(4,"esp"),"eax");
 439         &mov    (&DWP(8,"esp"),"eax");
 440         &mov    (&DWP(12,"esp"),"eax");
 441         &mov    (&DWP(16,"esp"),"eax");
 442         &mov    (&DWP(20,"esp"),"eax");
 443         &mov    (&DWP(24,"esp"),"eax");
 444         &mov    (&DWP(28,"esp"),"eax");
 445
 446         &call   ("_ecp_nistz256_sub");
 447
 448         &stack_pop(8);
 449 &function_end("ecp_nistz256_neg");
 450
 451 &function_begin_B("_picup_eax");
 452         &mov    ("eax",&DWP(0,"esp"));
 453         &ret    ();
 454 &function_end_B("_picup_eax");
 455
 456 ########################################################################
 457 # void ecp_nistz256_to_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
 458 &function_begin("ecp_nistz256_to_mont");
 459         &mov    ("esi",&wparam(1));
 460         &call   ("_picup_eax");
 461     &set_label("pic");
 462         &lea    ("ebp",&DWP(&label("RR")."-".&label("pic"),"eax"));
 463                                                 if ($sse2) {
 464         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 465         &mov    ("eax",&DWP(0,"eax"));          }
 466         &mov    ("edi",&wparam(0));
 467         &call   ("_ecp_nistz256_mul_mont");
 468 &function_end("ecp_nistz256_to_mont");
 469
 470 ########################################################################
 471 # void ecp_nistz256_from_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
 472 &function_begin("ecp_nistz256_from_mont");
 473         &mov    ("esi",&wparam(1));
 474         &call   ("_picup_eax");
 475     &set_label("pic");
 476         &lea    ("ebp",&DWP(&label("ONE")."-".&label("pic"),"eax"));
 477                                                 if ($sse2) {
 478         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 479         &mov    ("eax",&DWP(0,"eax"));          }
 480         &mov    ("edi",&wparam(0));
 481         &call   ("_ecp_nistz256_mul_mont");
 482 &function_end("ecp_nistz256_from_mont");
 483
 484 ########################################################################
 485 # void ecp_nistz256_mul_mont(BN_ULONG edi[8],const BN_ULONG esi[8],
 486 #                                            const BN_ULONG ebp[8]);
 487 &function_begin("ecp_nistz256_mul_mont");
 488         &mov    ("esi",&wparam(1));
 489         &mov    ("ebp",&wparam(2));
 490                                                 if ($sse2) {
 491         &call   ("_picup_eax");
 492     &set_label("pic");
 493         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 494         &mov    ("eax",&DWP(0,"eax"));          }
 495         &mov    ("edi",&wparam(0));
 496         &call   ("_ecp_nistz256_mul_mont");
 497 &function_end("ecp_nistz256_mul_mont");
 498
 499 ########################################################################
 500 # void ecp_nistz256_sqr_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
 501 &function_begin("ecp_nistz256_sqr_mont");
 502         &mov    ("esi",&wparam(1));
 503                                                 if ($sse2) {
 504         &call   ("_picup_eax");
 505     &set_label("pic");
 506         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 507         &mov    ("eax",&DWP(0,"eax"));          }
 508         &mov    ("edi",&wparam(0));
 509         &mov    ("ebp","esi");
 510         &call   ("_ecp_nistz256_mul_mont");
 511 &function_end("ecp_nistz256_sqr_mont");
 512
 513 &function_begin_B("_ecp_nistz256_mul_mont");
 514                                                 if ($sse2) {
 515         &and    ("eax",1<<24|1<<26);
 516         &cmp    ("eax",1<<24|1<<26);            # see if XMM+SSE2 is on
 517         &jne    (&label("mul_mont_ialu"));
 518
 519         ########################################
 520         # SSE2 code path featuring 32x16-bit
 521         # multiplications is ~2x faster than
 522         # IALU counterpart (except on Atom)...
 523         ########################################
 524         # stack layout:
 525         # +------------------------------------+< %esp
 526         # | 7 16-byte temporary XMM words,     |
 527         # | "sliding" toward lower address     |
 528         # .                                    .
 529         # +------------------------------------+
 530         # | unused XMM word                    |
 531         # +------------------------------------+< +128,%ebx
 532         # | 8 16-byte XMM words holding copies |
 533         # | of a[i]<<64|a[i]                   |
 534         # .                                    .
 535         # .                                    .
 536         # +------------------------------------+< +256
 537         &mov    ("edx","esp");
 538         &sub    ("esp",0x100);
 539
 540         &movd   ("xmm7",&DWP(0,"ebp"));         # b[0] -> 0000.00xy
 541         &lea    ("ebp",&DWP(4,"ebp"));
 542         &pcmpeqd("xmm6","xmm6");
 543         &psrlq  ("xmm6",48);                    # compose 0xffff<<64|0xffff
 544
 545         &pshuflw("xmm7","xmm7",0b11011100);     # 0000.00xy -> 0000.0x0y
 546         &and    ("esp",-64);
 547         &pshufd ("xmm7","xmm7",0b11011100);     # 0000.0x0y -> 000x.000y
 548         &lea    ("ebx",&DWP(0x80,"esp"));
 549
 550         &movd   ("xmm0",&DWP(4*0,"esi"));       # a[0] -> 0000.00xy
 551         &pshufd ("xmm0","xmm0",0b11001100);     # 0000.00xy -> 00xy.00xy
 552         &movd   ("xmm1",&DWP(4*1,"esi"));       # a[1] -> ...
 553         &movdqa (&QWP(0x00,"ebx"),"xmm0");      # offload converted a[0]
 554         &pmuludq("xmm0","xmm7");                # a[0]*b[0]
 555
 556         &movd   ("xmm2",&DWP(4*2,"esi"));
 557         &pshufd ("xmm1","xmm1",0b11001100);
 558         &movdqa (&QWP(0x10,"ebx"),"xmm1");
 559         &pmuludq("xmm1","xmm7");                # a[1]*b[0]
 560
 561          &movq  ("xmm4","xmm0");                # clear upper 64 bits
 562          &pslldq("xmm4",6);
 563          &paddq ("xmm4","xmm0");
 564          &movdqa("xmm5","xmm4");
 565          &psrldq("xmm4",10);                    # upper 32 bits of a[0]*b[0]
 566          &pand  ("xmm5","xmm6");                # lower 32 bits of a[0]*b[0]
 567
 568         # Upper half of a[0]*b[i] is carried into next multiplication
 569         # iteration, while lower one "participates" in actual reduction.
 570         # Normally latter is done by accumulating result of multiplication
 571         # of modulus by "magic" digit, but thanks to special form of modulus
 572         # and "magic" digit it can be performed only with additions and
 573         # subtractions (see note in IALU section below). Note that we are
 574         # not bothered with carry bits, they are accumulated in "flatten"
 575         # phase after all multiplications and reductions.
 576
 577         &movd   ("xmm3",&DWP(4*3,"esi"));
 578         &pshufd ("xmm2","xmm2",0b11001100);
 579         &movdqa (&QWP(0x20,"ebx"),"xmm2");
 580         &pmuludq("xmm2","xmm7");                # a[2]*b[0]
 581          &paddq ("xmm1","xmm4");                # a[1]*b[0]+hw(a[0]*b[0]), carry
 582         &movdqa (&QWP(0x00,"esp"),"xmm1");      # t[0]
 583
 584         &movd   ("xmm0",&DWP(4*4,"esi"));
 585         &pshufd ("xmm3","xmm3",0b11001100);
 586         &movdqa (&QWP(0x30,"ebx"),"xmm3");
 587         &pmuludq("xmm3","xmm7");                # a[3]*b[0]
 588         &movdqa (&QWP(0x10,"esp"),"xmm2");
 589
 590         &movd   ("xmm1",&DWP(4*5,"esi"));
 591         &pshufd ("xmm0","xmm0",0b11001100);
 592         &movdqa (&QWP(0x40,"ebx"),"xmm0");
 593         &pmuludq("xmm0","xmm7");                # a[4]*b[0]
 594          &paddq ("xmm3","xmm5");                # a[3]*b[0]+lw(a[0]*b[0]), reduction step
 595         &movdqa (&QWP(0x20,"esp"),"xmm3");
 596
 597         &movd   ("xmm2",&DWP(4*6,"esi"));
 598         &pshufd ("xmm1","xmm1",0b11001100);
 599         &movdqa (&QWP(0x50,"ebx"),"xmm1");
 600         &pmuludq("xmm1","xmm7");                # a[5]*b[0]
 601         &movdqa (&QWP(0x30,"esp"),"xmm0");
 602          &pshufd("xmm4","xmm5",0b10110001);     # xmm4 = xmm5<<32, reduction step
 603
 604         &movd   ("xmm3",&DWP(4*7,"esi"));
 605         &pshufd ("xmm2","xmm2",0b11001100);
 606         &movdqa (&QWP(0x60,"ebx"),"xmm2");
 607         &pmuludq("xmm2","xmm7");                # a[6]*b[0]
 608         &movdqa (&QWP(0x40,"esp"),"xmm1");
 609          &psubq ("xmm4","xmm5");                # xmm4 = xmm5*0xffffffff, reduction step
 610
 611         &movd   ("xmm0",&DWP(0,"ebp"));         # b[1] -> 0000.00xy
 612         &pshufd ("xmm3","xmm3",0b11001100);
 613         &movdqa (&QWP(0x70,"ebx"),"xmm3");
 614         &pmuludq("xmm3","xmm7");                # a[7]*b[0]
 615
 616         &pshuflw("xmm7","xmm0",0b11011100);     # 0000.00xy -> 0000.0x0y
 617         &movdqa ("xmm0",&QWP(0x00,"ebx"));      # pre-load converted a[0]
 618         &pshufd ("xmm7","xmm7",0b11011100);     # 0000.0x0y -> 000x.000y
 619
 620         &mov    ("ecx",6);
 621         &lea    ("ebp",&DWP(4,"ebp"));
 622         &jmp    (&label("madd_sse2"));
 623
 624 &set_label("madd_sse2",16);
 625          &paddq ("xmm2","xmm5");                # a[6]*b[i-1]+lw(a[0]*b[i-1]), reduction step [modulo-scheduled]
 626          &paddq ("xmm3","xmm4");                # a[7]*b[i-1]+lw(a[0]*b[i-1])*0xffffffff, reduction step [modulo-scheduled]
 627         &movdqa ("xmm1",&QWP(0x10,"ebx"));
 628         &pmuludq("xmm0","xmm7");                # a[0]*b[i]
 629          &movdqa(&QWP(0x50,"esp"),"xmm2");
 630
 631         &movdqa ("xmm2",&QWP(0x20,"ebx"));
 632         &pmuludq("xmm1","xmm7");                # a[1]*b[i]
 633          &movdqa(&QWP(0x60,"esp"),"xmm3");
 634         &paddq  ("xmm0",&QWP(0x00,"esp"));
 635
 636         &movdqa ("xmm3",&QWP(0x30,"ebx"));
 637         &pmuludq("xmm2","xmm7");                # a[2]*b[i]
 638          &movq  ("xmm4","xmm0");                # clear upper 64 bits
 639          &pslldq("xmm4",6);
 640         &paddq  ("xmm1",&QWP(0x10,"esp"));
 641          &paddq ("xmm4","xmm0");
 642          &movdqa("xmm5","xmm4");
 643          &psrldq("xmm4",10);                    # upper 33 bits of a[0]*b[i]+t[0]
 644
 645         &movdqa ("xmm0",&QWP(0x40,"ebx"));
 646         &pmuludq("xmm3","xmm7");                # a[3]*b[i]
 647          &paddq ("xmm1","xmm4");                # a[1]*b[i]+hw(a[0]*b[i]), carry
 648         &paddq  ("xmm2",&QWP(0x20,"esp"));
 649         &movdqa (&QWP(0x00,"esp"),"xmm1");
 650
 651         &movdqa ("xmm1",&QWP(0x50,"ebx"));
 652         &pmuludq("xmm0","xmm7");                # a[4]*b[i]
 653         &paddq  ("xmm3",&QWP(0x30,"esp"));
 654         &movdqa (&QWP(0x10,"esp"),"xmm2");
 655          &pand  ("xmm5","xmm6");                # lower 32 bits of a[0]*b[i]
 656
 657         &movdqa ("xmm2",&QWP(0x60,"ebx"));
 658         &pmuludq("xmm1","xmm7");                # a[5]*b[i]
 659          &paddq ("xmm3","xmm5");                # a[3]*b[i]+lw(a[0]*b[i]), reduction step
 660         &paddq  ("xmm0",&QWP(0x40,"esp"));
 661         &movdqa (&QWP(0x20,"esp"),"xmm3");
 662          &pshufd("xmm4","xmm5",0b10110001);     # xmm4 = xmm5<<32, reduction step
 663
 664         &movdqa ("xmm3","xmm7");
 665         &pmuludq("xmm2","xmm7");                # a[6]*b[i]
 666          &movd  ("xmm7",&DWP(0,"ebp"));         # b[i++] -> 0000.00xy
 667          &lea   ("ebp",&DWP(4,"ebp"));
 668         &paddq  ("xmm1",&QWP(0x50,"esp"));
 669          &psubq ("xmm4","xmm5");                # xmm4 = xmm5*0xffffffff, reduction step
 670         &movdqa (&QWP(0x30,"esp"),"xmm0");
 671          &pshuflw("xmm7","xmm7",0b11011100);    # 0000.00xy -> 0000.0x0y
 672
 673         &pmuludq("xmm3",&QWP(0x70,"ebx"));      # a[7]*b[i]
 674          &pshufd("xmm7","xmm7",0b11011100);     # 0000.0x0y -> 000x.000y
 675          &movdqa("xmm0",&QWP(0x00,"ebx"));      # pre-load converted a[0]
 676         &movdqa (&QWP(0x40,"esp"),"xmm1");
 677         &paddq  ("xmm2",&QWP(0x60,"esp"));
 678
 679         &dec    ("ecx");
 680         &jnz    (&label("madd_sse2"));
 681
 682          &paddq ("xmm2","xmm5");                # a[6]*b[6]+lw(a[0]*b[6]), reduction step [modulo-scheduled]
 683          &paddq ("xmm3","xmm4");                # a[7]*b[6]+lw(a[0]*b[6])*0xffffffff, reduction step [modulo-scheduled]
 684         &movdqa ("xmm1",&QWP(0x10,"ebx"));
 685         &pmuludq("xmm0","xmm7");                # a[0]*b[7]
 686          &movdqa(&QWP(0x50,"esp"),"xmm2");
 687
 688         &movdqa ("xmm2",&QWP(0x20,"ebx"));
 689         &pmuludq("xmm1","xmm7");                # a[1]*b[7]
 690          &movdqa(&QWP(0x60,"esp"),"xmm3");
 691         &paddq  ("xmm0",&QWP(0x00,"esp"));
 692
 693         &movdqa ("xmm3",&QWP(0x30,"ebx"));
 694         &pmuludq("xmm2","xmm7");                # a[2]*b[7]
 695          &movq  ("xmm4","xmm0");                # clear upper 64 bits
 696          &pslldq("xmm4",6);
 697         &paddq  ("xmm1",&QWP(0x10,"esp"));
 698          &paddq ("xmm4","xmm0");
 699          &movdqa("xmm5","xmm4");
 700          &psrldq("xmm4",10);                    # upper 33 bits of a[0]*b[i]+t[0]
 701
 702         &movdqa ("xmm0",&QWP(0x40,"ebx"));
 703         &pmuludq("xmm3","xmm7");                # a[3]*b[7]
 704          &paddq ("xmm1","xmm4");                # a[1]*b[7]+hw(a[0]*b[7]), carry
 705         &paddq  ("xmm2",&QWP(0x20,"esp"));
 706         &movdqa (&QWP(0x00,"esp"),"xmm1");
 707
 708         &movdqa ("xmm1",&QWP(0x50,"ebx"));
 709         &pmuludq("xmm0","xmm7");                # a[4]*b[7]
 710         &paddq  ("xmm3",&QWP(0x30,"esp"));
 711         &movdqa (&QWP(0x10,"esp"),"xmm2");
 712          &pand  ("xmm5","xmm6");                # lower 32 bits of a[0]*b[i]
 713
 714         &movdqa ("xmm2",&QWP(0x60,"ebx"));
 715         &pmuludq("xmm1","xmm7");                # a[5]*b[7]
 716          &paddq ("xmm3","xmm5");                # reduction step
 717         &paddq  ("xmm0",&QWP(0x40,"esp"));
 718         &movdqa (&QWP(0x20,"esp"),"xmm3");
 719          &pshufd("xmm4","xmm5",0b10110001);     # xmm4 = xmm5<<32, reduction step
 720
 721         &movdqa ("xmm3",&QWP(0x70,"ebx"));
 722         &pmuludq("xmm2","xmm7");                # a[6]*b[7]
 723         &paddq  ("xmm1",&QWP(0x50,"esp"));
 724          &psubq ("xmm4","xmm5");                # xmm4 = xmm5*0xffffffff, reduction step
 725         &movdqa (&QWP(0x30,"esp"),"xmm0");
 726
 727         &pmuludq("xmm3","xmm7");                # a[7]*b[7]
 728         &pcmpeqd("xmm7","xmm7");
 729         &movdqa ("xmm0",&QWP(0x00,"esp"));
 730         &pslldq ("xmm7",8);
 731         &movdqa (&QWP(0x40,"esp"),"xmm1");
 732         &paddq  ("xmm2",&QWP(0x60,"esp"));
 733
 734          &paddq ("xmm2","xmm5");                # a[6]*b[7]+lw(a[0]*b[7]), reduction step
 735          &paddq ("xmm3","xmm4");                # a[6]*b[7]+lw(a[0]*b[7])*0xffffffff, reduction step
 736          &movdqa(&QWP(0x50,"esp"),"xmm2");
 737          &movdqa(&QWP(0x60,"esp"),"xmm3");
 738
 739         &movdqa ("xmm1",&QWP(0x10,"esp"));
 740         &movdqa ("xmm2",&QWP(0x20,"esp"));
 741         &movdqa ("xmm3",&QWP(0x30,"esp"));
 742
 743         &movq   ("xmm4","xmm0");                # "flatten"
 744         &pand   ("xmm0","xmm7");
 745         &xor    ("ebp","ebp");
 746         &pslldq ("xmm4",6);
 747          &movq  ("xmm5","xmm1");
 748         &paddq  ("xmm0","xmm4");
 749          &pand  ("xmm1","xmm7");
 750         &psrldq ("xmm0",6);
 751         &movd   ("eax","xmm0");
 752         &psrldq ("xmm0",4);
 753
 754         &paddq  ("xmm5","xmm0");
 755         &movdqa ("xmm0",&QWP(0x40,"esp"));
 756         &sub    ("eax",-1);                     # start subtracting modulus,
 757                                                 # this is used to determine
 758                                                 # if result is larger/smaller
 759                                                 # than modulus (see below)
 760         &pslldq ("xmm5",6);
 761          &movq  ("xmm4","xmm2");
 762         &paddq  ("xmm1","xmm5");
 763          &pand  ("xmm2","xmm7");
 764         &psrldq ("xmm1",6);
 765         &mov    (&DWP(4*0,"edi"),"eax");
 766         &movd   ("eax","xmm1");
 767         &psrldq ("xmm1",4);
 768
 769         &paddq  ("xmm4","xmm1");
 770         &movdqa ("xmm1",&QWP(0x50,"esp"));
 771         &sbb    ("eax",-1);
 772         &pslldq ("xmm4",6);
 773          &movq  ("xmm5","xmm3");
 774         &paddq  ("xmm2","xmm4");
 775          &pand  ("xmm3","xmm7");
 776         &psrldq ("xmm2",6);
 777         &mov    (&DWP(4*1,"edi"),"eax");
 778         &movd   ("eax","xmm2");
 779         &psrldq ("xmm2",4);
 780
 781         &paddq  ("xmm5","xmm2");
 782         &movdqa ("xmm2",&QWP(0x60,"esp"));
 783         &sbb    ("eax",-1);
 784         &pslldq ("xmm5",6);
 785          &movq  ("xmm4","xmm0");
 786         &paddq  ("xmm3","xmm5");
 787          &pand  ("xmm0","xmm7");
 788         &psrldq ("xmm3",6);
 789         &mov    (&DWP(4*2,"edi"),"eax");
 790         &movd   ("eax","xmm3");
 791         &psrldq ("xmm3",4);
 792
 793         &paddq  ("xmm4","xmm3");
 794         &sbb    ("eax",0);
 795         &pslldq ("xmm4",6);
 796          &movq  ("xmm5","xmm1");
 797         &paddq  ("xmm0","xmm4");
 798          &pand  ("xmm1","xmm7");
 799         &psrldq ("xmm0",6);
 800         &mov    (&DWP(4*3,"edi"),"eax");
 801         &movd   ("eax","xmm0");
 802         &psrldq ("xmm0",4);
 803
 804         &paddq  ("xmm5","xmm0");
 805         &sbb    ("eax",0);
 806         &pslldq ("xmm5",6);
 807          &movq  ("xmm4","xmm2");
 808         &paddq  ("xmm1","xmm5");
 809          &pand  ("xmm2","xmm7");
 810         &psrldq ("xmm1",6);
 811         &movd   ("ebx","xmm1");
 812         &psrldq ("xmm1",4);
 813         &mov    ("esp","edx");
 814
 815         &paddq  ("xmm4","xmm1");
 816         &pslldq ("xmm4",6);
 817         &paddq  ("xmm2","xmm4");
 818         &psrldq ("xmm2",6);
 819         &movd   ("ecx","xmm2");
 820         &psrldq ("xmm2",4);
 821         &sbb    ("ebx",0);
 822         &movd   ("edx","xmm2");
 823         &pextrw ("esi","xmm2",2);               # top-most overflow bit
 824         &sbb    ("ecx",1);
 825         &sbb    ("edx",-1);
 826         &sbb    ("esi",0);                      # borrow from subtraction
 827
 828         # Final step is "if result > mod, subtract mod", and at this point
 829         # we have result - mod written to output buffer, as well as borrow
 830         # bit from this subtraction, and if borrow bit is set, we add
 831         # modulus back.
 832         #
 833         # Note that because mod has special form, i.e. consists of
 834         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 835         # assigning borrow bit to one register, %ebp, and its negative
 836         # to another, %esi. But we started by calculating %esi...
 837
 838         &sub    ("ebp","esi");
 839         &add    (&DWP(4*0,"edi"),"esi");        # add modulus or zero
 840         &adc    (&DWP(4*1,"edi"),"esi");
 841         &adc    (&DWP(4*2,"edi"),"esi");
 842         &adc    (&DWP(4*3,"edi"),0);
 843         &adc    ("eax",0);
 844         &adc    ("ebx",0);
 845         &mov    (&DWP(4*4,"edi"),"eax");
 846         &adc    ("ecx","ebp");
 847         &mov    (&DWP(4*5,"edi"),"ebx");
 848         &adc    ("edx","esi");
 849         &mov    (&DWP(4*6,"edi"),"ecx");
 850         &mov    (&DWP(4*7,"edi"),"edx");
 851
 852         &ret    ();
 853
 854 &set_label("mul_mont_ialu",16);                 }
 855
 856         ########################################
 857         # IALU code path suitable for all CPUs.
 858         ########################################
 859         # stack layout:
 860         # +------------------------------------+< %esp
 861         # | 8 32-bit temporary words, accessed |
 862         # | as circular buffer                 |
 863         # .                                    .
 864         # .                                    .
 865         # +------------------------------------+< +32
 866         # | offloaded destination pointer      |
 867         # +------------------------------------+
 868         # | unused                             |
 869         # +------------------------------------+< +40
 870         &sub    ("esp",10*4);
 871
 872         &mov    ("eax",&DWP(0*4,"esi"));                # a[0]
 873         &mov    ("ebx",&DWP(0*4,"ebp"));                # b[0]
 874         &mov    (&DWP(8*4,"esp"),"edi");                # off-load dst ptr
 875
 876         &mul    ("ebx");                                # a[0]*b[0]
 877         &mov    (&DWP(0*4,"esp"),"eax");                # t[0]
 878         &mov    ("eax",&DWP(1*4,"esi"));
 879         &mov    ("ecx","edx")
 880
 881         &mul    ("ebx");                                # a[1]*b[0]
 882         &add    ("ecx","eax");
 883         &mov    ("eax",&DWP(2*4,"esi"));
 884         &adc    ("edx",0);
 885         &mov    (&DWP(1*4,"esp"),"ecx");                # t[1]
 886         &mov    ("ecx","edx");
 887
 888         &mul    ("ebx");                                # a[2]*b[0]
 889         &add    ("ecx","eax");
 890         &mov    ("eax",&DWP(3*4,"esi"));
 891         &adc    ("edx",0);
 892         &mov    (&DWP(2*4,"esp"),"ecx");                # t[2]
 893         &mov    ("ecx","edx");
 894
 895         &mul    ("ebx");                                # a[3]*b[0]
 896         &add    ("ecx","eax");
 897         &mov    ("eax",&DWP(4*4,"esi"));
 898         &adc    ("edx",0);
 899         &mov    (&DWP(3*4,"esp"),"ecx");                # t[3]
 900         &mov    ("ecx","edx");
 901
 902         &mul    ("ebx");                                # a[4]*b[0]
 903         &add    ("ecx","eax");
 904         &mov    ("eax",&DWP(5*4,"esi"));
 905         &adc    ("edx",0);
 906         &mov    (&DWP(4*4,"esp"),"ecx");                # t[4]
 907         &mov    ("ecx","edx");
 908
 909         &mul    ("ebx");                                # a[5]*b[0]
 910         &add    ("ecx","eax");
 911         &mov    ("eax",&DWP(6*4,"esi"));
 912         &adc    ("edx",0);
 913         &mov    (&DWP(5*4,"esp"),"ecx");                # t[5]
 914         &mov    ("ecx","edx");
 915
 916         &mul    ("ebx");                                # a[6]*b[0]
 917         &add    ("ecx","eax");
 918         &mov    ("eax",&DWP(7*4,"esi"));
 919         &adc    ("edx",0);
 920         &mov    (&DWP(6*4,"esp"),"ecx");                # t[6]
 921         &mov    ("ecx","edx");
 922
 923         &xor    ("edi","edi");                          # initial top-most carry
 924         &mul    ("ebx");                                # a[7]*b[0]
 925         &add    ("ecx","eax");                          # t[7]
 926         &mov    ("eax",&DWP(0*4,"esp"));                # t[0]
 927         &adc    ("edx",0);                              # t[8]
 928
 929 for ($i=0;$i<7;$i++) {
 930         my $j=$i+1;
 931
 932         # Reduction iteration is normally performed by accumulating
 933         # result of multiplication of modulus by "magic" digit [and
 934         # omitting least significant word, which is guaranteed to
 935         # be 0], but thanks to special form of modulus and "magic"
 936         # digit being equal to least significant word, it can be
 937         # performed with additions and subtractions alone. Indeed:
 938         #
 939         #        ffff.0001.0000.0000.0000.ffff.ffff.ffff
 940         # *                                         abcd
 941         # + xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
 942         #
 943         # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
 944         # rewrite above as:
 945         #
 946         #   xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
 947         # + abcd.0000.abcd.0000.0000.abcd.0000.0000.0000
 948         # -      abcd.0000.0000.0000.0000.0000.0000.abcd
 949         #
 950         # or marking redundant operations:
 951         #
 952         #   xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.----
 953         # + abcd.0000.abcd.0000.0000.abcd.----.----.----
 954         # -      abcd.----.----.----.----.----.----.----
 955
 956         &add    (&DWP((($i+3)%8)*4,"esp"),"eax");       # t[3]+=t[0]
 957         &adc    (&DWP((($i+4)%8)*4,"esp"),0);           # t[4]+=0
 958         &adc    (&DWP((($i+5)%8)*4,"esp"),0);           # t[5]+=0
 959         &adc    (&DWP((($i+6)%8)*4,"esp"),"eax");       # t[6]+=t[0]
 960         &adc    ("ecx",0);                              # t[7]+=0
 961         &adc    ("edx","eax");                          # t[8]+=t[0]
 962         &adc    ("edi",0);                              # top-most carry
 963          &mov   ("ebx",&DWP($j*4,"ebp"));               # b[i]
 964         &sub    ("ecx","eax");                          # t[7]-=t[0]
 965          &mov   ("eax",&DWP(0*4,"esi"));                # a[0]
 966         &sbb    ("edx",0);                              # t[8]-=0
 967         &mov    (&DWP((($i+7)%8)*4,"esp"),"ecx");
 968         &sbb    ("edi",0);                              # top-most carry,
 969                                                         # keep in mind that
 970                                                         # netto result is
 971                                                         # *addition* of value
 972                                                         # with (abcd<<32)-abcd
 973                                                         # on top, so that
 974                                                         # underflow is
 975                                                         # impossible, because
 976                                                         # (abcd<<32)-abcd
 977                                                         # doesn't underflow
 978         &mov    (&DWP((($i+8)%8)*4,"esp"),"edx");
 979
 980         &mul    ("ebx");                                # a[0]*b[i]
 981         &add    ("eax",&DWP((($j+0)%8)*4,"esp"));
 982         &adc    ("edx",0);
 983         &mov    (&DWP((($j+0)%8)*4,"esp"),"eax");
 984         &mov    ("eax",&DWP(1*4,"esi"));
 985         &mov    ("ecx","edx")
 986
 987         &mul    ("ebx");                                # a[1]*b[i]
 988         &add    ("ecx",&DWP((($j+1)%8)*4,"esp"));
 989         &adc    ("edx",0);
 990         &add    ("ecx","eax");
 991         &adc    ("edx",0);
 992         &mov    ("eax",&DWP(2*4,"esi"));
 993         &mov    (&DWP((($j+1)%8)*4,"esp"),"ecx");
 994         &mov    ("ecx","edx");
 995
 996         &mul    ("ebx");                                # a[2]*b[i]
 997         &add    ("ecx",&DWP((($j+2)%8)*4,"esp"));
 998         &adc    ("edx",0);
 999         &add    ("ecx","eax");
1000         &adc    ("edx",0);
1001         &mov    ("eax",&DWP(3*4,"esi"));
1002         &mov    (&DWP((($j+2)%8)*4,"esp"),"ecx");
1003         &mov    ("ecx","edx");
1004
1005         &mul    ("ebx");                                # a[3]*b[i]
1006         &add    ("ecx",&DWP((($j+3)%8)*4,"esp"));
1007         &adc    ("edx",0);
1008         &add    ("ecx","eax");
1009         &adc    ("edx",0);
1010         &mov    ("eax",&DWP(4*4,"esi"));
1011         &mov    (&DWP((($j+3)%8)*4,"esp"),"ecx");
1012         &mov    ("ecx","edx");
1013
1014         &mul    ("ebx");                                # a[4]*b[i]
1015         &add    ("ecx",&DWP((($j+4)%8)*4,"esp"));
1016         &adc    ("edx",0);
1017         &add    ("ecx","eax");
1018         &adc    ("edx",0);
1019         &mov    ("eax",&DWP(5*4,"esi"));
1020         &mov    (&DWP((($j+4)%8)*4,"esp"),"ecx");
1021         &mov    ("ecx","edx");
1022
1023         &mul    ("ebx");                                # a[5]*b[i]
1024         &add    ("ecx",&DWP((($j+5)%8)*4,"esp"));
1025         &adc    ("edx",0);
1026         &add    ("ecx","eax");
1027         &adc    ("edx",0);
1028         &mov    ("eax",&DWP(6*4,"esi"));
1029         &mov    (&DWP((($j+5)%8)*4,"esp"),"ecx");
1030         &mov    ("ecx","edx");
1031
1032         &mul    ("ebx");                                # a[6]*b[i]
1033         &add    ("ecx",&DWP((($j+6)%8)*4,"esp"));
1034         &adc    ("edx",0);
1035         &add    ("ecx","eax");
1036         &adc    ("edx",0);
1037         &mov    ("eax",&DWP(7*4,"esi"));
1038         &mov    (&DWP((($j+6)%8)*4,"esp"),"ecx");
1039         &mov    ("ecx","edx");
1040
1041         &mul    ("ebx");                                # a[7]*b[i]
1042         &add    ("ecx",&DWP((($j+7)%8)*4,"esp"));
1043         &adc    ("edx",0);
1044         &add    ("ecx","eax");                          # t[7]
1045         &mov    ("eax",&DWP((($j+0)%8)*4,"esp"));       # t[0]
1046         &adc    ("edx","edi");                          # t[8]
1047         &mov    ("edi",0);
1048         &adc    ("edi",0);                              # top-most carry
1049 }
1050         &mov    ("ebp",&DWP(8*4,"esp"));                # restore dst ptr
1051         &xor    ("esi","esi");
1052         my $j=$i+1;
1053
1054         # last multiplication-less reduction
1055         &add    (&DWP((($i+3)%8)*4,"esp"),"eax");       # t[3]+=t[0]
1056         &adc    (&DWP((($i+4)%8)*4,"esp"),0);           # t[4]+=0
1057         &adc    (&DWP((($i+5)%8)*4,"esp"),0);           # t[5]+=0
1058         &adc    (&DWP((($i+6)%8)*4,"esp"),"eax");       # t[6]+=t[0]
1059         &adc    ("ecx",0);                              # t[7]+=0
1060         &adc    ("edx","eax");                          # t[8]+=t[0]
1061         &adc    ("edi",0);                              # top-most carry
1062          &mov   ("ebx",&DWP((($j+1)%8)*4,"esp"));
1063         &sub    ("ecx","eax");                          # t[7]-=t[0]
1064          &mov   ("eax",&DWP((($j+0)%8)*4,"esp"));
1065         &sbb    ("edx",0);                              # t[8]-=0
1066         &mov    (&DWP((($i+7)%8)*4,"esp"),"ecx");
1067         &sbb    ("edi",0);                              # top-most carry
1068         &mov    (&DWP((($i+8)%8)*4,"esp"),"edx");
1069
1070         # Final step is "if result > mod, subtract mod", but we do it
1071         # "other way around", namely write result - mod to output buffer
1072         # and if subtraction borrowed, add modulus back.
1073
1074         &mov    ("ecx",&DWP((($j+2)%8)*4,"esp"));
1075         &sub    ("eax",-1);
1076         &mov    ("edx",&DWP((($j+3)%8)*4,"esp"));
1077         &sbb    ("ebx",-1);
1078         &mov    (&DWP(0*4,"ebp"),"eax");
1079         &sbb    ("ecx",-1);
1080         &mov    (&DWP(1*4,"ebp"),"ebx");
1081         &sbb    ("edx",0);
1082         &mov    (&DWP(2*4,"ebp"),"ecx");
1083         &mov    (&DWP(3*4,"ebp"),"edx");
1084
1085         &mov    ("eax",&DWP((($j+4)%8)*4,"esp"));
1086         &mov    ("ebx",&DWP((($j+5)%8)*4,"esp"));
1087         &mov    ("ecx",&DWP((($j+6)%8)*4,"esp"));
1088         &sbb    ("eax",0);
1089         &mov    ("edx",&DWP((($j+7)%8)*4,"esp"));
1090         &sbb    ("ebx",0);
1091         &sbb    ("ecx",1);
1092         &sbb    ("edx",-1);
1093         &sbb    ("edi",0);
1094
1095         # Note that because mod has special form, i.e. consists of
1096         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
1097         # assigning borrow bit to one register, %ebp, and its negative
1098         # to another, %esi. But we started by calculating %esi...
1099
1100         &sub    ("esi","edi");
1101         &add    (&DWP(0*4,"ebp"),"edi");                # add modulus or zero
1102         &adc    (&DWP(1*4,"ebp"),"edi");
1103         &adc    (&DWP(2*4,"ebp"),"edi");
1104         &adc    (&DWP(3*4,"ebp"),0);
1105         &adc    ("eax",0);
1106         &adc    ("ebx",0);
1107         &mov    (&DWP(4*4,"ebp"),"eax");
1108         &adc    ("ecx","esi");
1109         &mov    (&DWP(5*4,"ebp"),"ebx");
1110         &adc    ("edx","edi");
1111         &mov    (&DWP(6*4,"ebp"),"ecx");
1112         &mov    ("edi","ebp");                          # fulfill contract
1113         &mov    (&DWP(7*4,"ebp"),"edx");
1114
1115         &add    ("esp",10*4);
1116         &ret    ();
1117 &function_end_B("_ecp_nistz256_mul_mont");
1118
1119 ########################################################################
1120 # void ecp_nistz256_scatter_w5(void *edi,const P256_POINT *esi,
1121 #                                        int ebp);
1122 &function_begin("ecp_nistz256_scatter_w5");
1123         &mov    ("edi",&wparam(0));
1124         &mov    ("esi",&wparam(1));
1125         &mov    ("ebp",&wparam(2));
1126
1127         &lea    ("edi",&DWP(128-4,"edi","ebp",4));
1128         &mov    ("ebp",96/16);
1129 &set_label("scatter_w5_loop");
1130         &mov    ("eax",&DWP(0,"esi"));
1131         &mov    ("ebx",&DWP(4,"esi"));
1132         &mov    ("ecx",&DWP(8,"esi"));
1133         &mov    ("edx",&DWP(12,"esi"));
1134         &lea    ("esi",&DWP(16,"esi"));
1135         &mov    (&DWP(64*0-128,"edi"),"eax");
1136         &mov    (&DWP(64*1-128,"edi"),"ebx");
1137         &mov    (&DWP(64*2-128,"edi"),"ecx");
1138         &mov    (&DWP(64*3-128,"edi"),"edx");
1139         &lea    ("edi",&DWP(64*4,"edi"));
1140         &dec    ("ebp");
1141         &jnz    (&label("scatter_w5_loop"));
1142 &function_end("ecp_nistz256_scatter_w5");
1143
1144 ########################################################################
1145 # void ecp_nistz256_gather_w5(P256_POINT *edi,const void *esi,
1146 #                                             int ebp);
1147 &function_begin("ecp_nistz256_gather_w5");
1148         &mov    ("esi",&wparam(1));
1149         &mov    ("ebp",&wparam(2));
1150
1151         &lea    ("esi",&DWP(0,"esi","ebp",4));
1152         &neg    ("ebp");
1153         &sar    ("ebp",31);
1154         &mov    ("edi",&wparam(0));
1155         &lea    ("esi",&DWP(0,"esi","ebp",4));
1156
1157     for($i=0;$i<24;$i+=4) {
1158         &mov    ("eax",&DWP(64*($i+0),"esi"));
1159         &mov    ("ebx",&DWP(64*($i+1),"esi"));
1160         &mov    ("ecx",&DWP(64*($i+2),"esi"));
1161         &mov    ("edx",&DWP(64*($i+3),"esi"));
1162         &and    ("eax","ebp");
1163         &and    ("ebx","ebp");
1164         &and    ("ecx","ebp");
1165         &and    ("edx","ebp");
1166         &mov    (&DWP(4*($i+0),"edi"),"eax");
1167         &mov    (&DWP(4*($i+1),"edi"),"ebx");
1168         &mov    (&DWP(4*($i+2),"edi"),"ecx");
1169         &mov    (&DWP(4*($i+3),"edi"),"edx");
1170     }
1171 &function_end("ecp_nistz256_gather_w5");
1172
1173 ########################################################################
1174 # void ecp_nistz256_scatter_w7(void *edi,const P256_POINT_AFFINE *esi,
1175 #                                        int ebp);
1176 &function_begin("ecp_nistz256_scatter_w7");
1177         &mov    ("edi",&wparam(0));
1178         &mov    ("esi",&wparam(1));
1179         &mov    ("ebp",&wparam(2));
1180
1181         &lea    ("edi",&DWP(0,"edi","ebp"));
1182         &mov    ("ebp",64/4);
1183 &set_label("scatter_w7_loop");
1184         &mov    ("eax",&DWP(0,"esi"));
1185         &lea    ("esi",&DWP(4,"esi"));
1186         &mov    (&BP(64*0,"edi"),"al");
1187         &mov    (&BP(64*1,"edi"),"ah");
1188         &shr    ("eax",16);
1189         &mov    (&BP(64*2,"edi"),"al");
1190         &mov    (&BP(64*3,"edi"),"ah");
1191         &lea    ("edi",&DWP(64*4,"edi"));
1192         &dec    ("ebp");
1193         &jnz    (&label("scatter_w7_loop"));
1194 &function_end("ecp_nistz256_scatter_w7");
1195
1196 ########################################################################
1197 # void ecp_nistz256_gather_w7(P256_POINT_AFFINE *edi,const void *esi,
1198 #                                                    int ebp);
1199 &function_begin("ecp_nistz256_gather_w7");
1200         &mov    ("esi",&wparam(1));
1201         &mov    ("ebp",&wparam(2));
1202
1203         &add    ("esi","ebp");
1204         &neg    ("ebp"),
1205         &sar    ("ebp",31);
1206         &mov    ("edi",&wparam(0));
1207         &lea    ("esi",&DWP(0,"esi","ebp"));
1208
1209     for($i=0;$i<64;$i+=4) {
1210         &movz   ("eax",&BP(64*($i+0),"esi"));
1211         &movz   ("ebx",&BP(64*($i+1),"esi"));
1212         &movz   ("ecx",&BP(64*($i+2),"esi"));
1213         &and    ("eax","ebp");
1214         &movz   ("edx",&BP(64*($i+3),"esi"));
1215         &and    ("ebx","ebp");
1216         &mov    (&BP($i+0,"edi"),"al");
1217         &and    ("ecx","ebp");
1218         &mov    (&BP($i+1,"edi"),"bl");
1219         &and    ("edx","ebp");
1220         &mov    (&BP($i+2,"edi"),"cl");
1221         &mov    (&BP($i+3,"edi"),"dl");
1222     }
1223 &function_end("ecp_nistz256_gather_w7");
1224
1225 ########################################################################
1226 # following subroutines are "literal" implementation of those found in
1227 # ecp_nistz256.c
1228 #
1229 ########################################################################
1230 # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
1231 #
1232 &static_label("point_double_shortcut");
1233 &function_begin("ecp_nistz256_point_double");
1234 {   my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
1235
1236         &mov    ("esi",&wparam(1));
1237
1238         # above map() describes stack layout with 5 temporary
1239         # 256-bit vectors on top, then we take extra word for
1240         # OPENSSL_ia32cap_P copy.
1241         &stack_push(8*5+1);
1242                                                 if ($sse2) {
1243         &call   ("_picup_eax");
1244     &set_label("pic");
1245         &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
1246         &mov    ("ebp",&DWP(0,"edx"));          }
1247
1248 &set_label("point_double_shortcut");
1249         &mov    ("eax",&DWP(0,"esi"));          # copy in_x
1250         &mov    ("ebx",&DWP(4,"esi"));
1251         &mov    ("ecx",&DWP(8,"esi"));
1252         &mov    ("edx",&DWP(12,"esi"));
1253         &mov    (&DWP($in_x+0,"esp"),"eax");
1254         &mov    (&DWP($in_x+4,"esp"),"ebx");
1255         &mov    (&DWP($in_x+8,"esp"),"ecx");
1256         &mov    (&DWP($in_x+12,"esp"),"edx");
1257         &mov    ("eax",&DWP(16,"esi"));
1258         &mov    ("ebx",&DWP(20,"esi"));
1259         &mov    ("ecx",&DWP(24,"esi"));
1260         &mov    ("edx",&DWP(28,"esi"));
1261         &mov    (&DWP($in_x+16,"esp"),"eax");
1262         &mov    (&DWP($in_x+20,"esp"),"ebx");
1263         &mov    (&DWP($in_x+24,"esp"),"ecx");
1264         &mov    (&DWP($in_x+28,"esp"),"edx");
1265         &mov    (&DWP(32*5,"esp"),"ebp");       # OPENSSL_ia32cap_P copy
1266
1267         &lea    ("ebp",&DWP(32,"esi"));
1268         &lea    ("esi",&DWP(32,"esi"));
1269         &lea    ("edi",&DWP($S,"esp"));
1270         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(S, in_y);
1271
1272         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1273         &mov    ("esi",64);
1274         &add    ("esi",&wparam(1));
1275         &lea    ("edi",&DWP($Zsqr,"esp"));
1276         &mov    ("ebp","esi");
1277         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Zsqr, in_z);
1278
1279         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1280         &lea    ("esi",&DWP($S,"esp"));
1281         &lea    ("ebp",&DWP($S,"esp"));
1282         &lea    ("edi",&DWP($S,"esp"));
1283         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(S, S);
1284
1285         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1286         &mov    ("ebp",&wparam(1));
1287         &lea    ("esi",&DWP(32,"ebp"));
1288         &lea    ("ebp",&DWP(64,"ebp"));
1289         &lea    ("edi",&DWP($tmp0,"esp"));
1290         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(tmp0, in_z, in_y);
1291
1292         &lea    ("esi",&DWP($in_x,"esp"));
1293         &lea    ("ebp",&DWP($Zsqr,"esp"));
1294         &lea    ("edi",&DWP($M,"esp"));
1295         &call   ("_ecp_nistz256_add");          # p256_add(M, in_x, Zsqr);
1296
1297         &mov    ("edi",64);
1298         &lea    ("esi",&DWP($tmp0,"esp"));
1299         &lea    ("ebp",&DWP($tmp0,"esp"));
1300         &add    ("edi",&wparam(0));
1301         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(res_z, tmp0);
1302
1303         &lea    ("esi",&DWP($in_x,"esp"));
1304         &lea    ("ebp",&DWP($Zsqr,"esp"));
1305         &lea    ("edi",&DWP($Zsqr,"esp"));
1306         &call   ("_ecp_nistz256_sub");          # p256_sub(Zsqr, in_x, Zsqr);
1307
1308         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1309         &lea    ("esi",&DWP($S,"esp"));
1310         &lea    ("ebp",&DWP($S,"esp"));
1311         &lea    ("edi",&DWP($tmp0,"esp"));
1312         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(tmp0, S);
1313
1314         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1315         &lea    ("esi",&DWP($M,"esp"));
1316         &lea    ("ebp",&DWP($Zsqr,"esp"));
1317         &lea    ("edi",&DWP($M,"esp"));
1318         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(M, M, Zsqr);
1319
1320         &mov    ("edi",32);
1321         &lea    ("esi",&DWP($tmp0,"esp"));
1322         &add    ("edi",&wparam(0));
1323         &call   ("_ecp_nistz256_div_by_2");     # p256_div_by_2(res_y, tmp0);
1324
1325         &lea    ("esi",&DWP($M,"esp"));
1326         &lea    ("ebp",&DWP($M,"esp"));
1327         &lea    ("edi",&DWP($tmp0,"esp"));
1328         &call   ("_ecp_nistz256_add");          # 1/2 p256_mul_by_3(M, M);
1329
1330         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1331         &lea    ("esi",&DWP($in_x,"esp"));
1332         &lea    ("ebp",&DWP($S,"esp"));
1333         &lea    ("edi",&DWP($S,"esp"));
1334         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S, S, in_x);
1335
1336         &lea    ("esi",&DWP($tmp0,"esp"));
1337         &lea    ("ebp",&DWP($M,"esp"));
1338         &lea    ("edi",&DWP($M,"esp"));
1339         &call   ("_ecp_nistz256_add");          # 2/2 p256_mul_by_3(M, M);
1340
1341         &lea    ("esi",&DWP($S,"esp"));
1342         &lea    ("ebp",&DWP($S,"esp"));
1343         &lea    ("edi",&DWP($tmp0,"esp"));
1344         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(tmp0, S);
1345
1346         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1347         &lea    ("esi",&DWP($M,"esp"));
1348         &lea    ("ebp",&DWP($M,"esp"));
1349         &mov    ("edi",&wparam(0));
1350         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(res_x, M);
1351
1352         &mov    ("esi","edi");                  # %edi is still res_x here
1353         &lea    ("ebp",&DWP($tmp0,"esp"));
1354         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, res_x, tmp0);
1355
1356         &lea    ("esi",&DWP($S,"esp"));
1357         &mov    ("ebp","edi");                  # %edi is still res_x
1358         &lea    ("edi",&DWP($S,"esp"));
1359         &call   ("_ecp_nistz256_sub");          # p256_sub(S, S, res_x);
1360
1361         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1362         &mov    ("esi","edi");                  # %edi is still &S
1363         &lea    ("ebp",&DWP($M,"esp"));
1364         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S, S, M);
1365
1366         &mov    ("ebp",32);
1367         &lea    ("esi",&DWP($S,"esp"));
1368         &add    ("ebp",&wparam(0));
1369         &mov    ("edi","ebp");
1370         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, S, res_y);
1371
1372         &stack_pop(8*5+1);
1373 } &function_end("ecp_nistz256_point_double");
1374
1375 ########################################################################
1376 # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
1377 #                                             const P256_POINT *in2);
1378 &function_begin("ecp_nistz256_point_add");
1379 {   my ($res_x,$res_y,$res_z,
1380         $in1_x,$in1_y,$in1_z,
1381         $in2_x,$in2_y,$in2_z,
1382         $H,$Hsqr,$R,$Rsqr,$Hcub,
1383         $U1,$U2,$S1,$S2)=map(32*$_,(0..17));
1384     my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
1385
1386         &mov    ("esi",&wparam(2));
1387
1388         # above map() describes stack layout with 18 temporary
1389         # 256-bit vectors on top, then we take extra words for
1390         # ~in1infty, ~in2infty, result of check for zero and
1391         # OPENSSL_ia32cap_P copy. [one unused word for padding]
1392         &stack_push(8*18+5);
1393                                                 if ($sse2) {
1394         &call   ("_picup_eax");
1395     &set_label("pic");
1396         &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
1397         &mov    ("ebp",&DWP(0,"edx"));          }
1398
1399         &lea    ("edi",&DWP($in2_x,"esp"));
1400     for($i=0;$i<96;$i+=16) {
1401         &mov    ("eax",&DWP($i+0,"esi"));       # copy in2
1402         &mov    ("ebx",&DWP($i+4,"esi"));
1403         &mov    ("ecx",&DWP($i+8,"esi"));
1404         &mov    ("edx",&DWP($i+12,"esi"));
1405         &mov    (&DWP($i+0,"edi"),"eax");
1406         &mov    (&DWP(32*18+12,"esp"),"ebp")    if ($i==0);
1407         &mov    ("ebp","eax")                   if ($i==64);
1408         &or     ("ebp","eax")                   if ($i>64);
1409         &mov    (&DWP($i+4,"edi"),"ebx");
1410         &or     ("ebp","ebx")                   if ($i>=64);
1411         &mov    (&DWP($i+8,"edi"),"ecx");
1412         &or     ("ebp","ecx")                   if ($i>=64);
1413         &mov    (&DWP($i+12,"edi"),"edx");
1414         &or     ("ebp","edx")                   if ($i>=64);
1415     }
1416         &xor    ("eax","eax");
1417         &mov    ("esi",&wparam(1));
1418         &sub    ("eax","ebp");
1419         &or     ("ebp","eax");
1420         &sar    ("ebp",31);
1421         &mov    (&DWP(32*18+4,"esp"),"ebp");    # ~in2infty
1422
1423         &lea    ("edi",&DWP($in1_x,"esp"));
1424     for($i=0;$i<96;$i+=16) {
1425         &mov    ("eax",&DWP($i+0,"esi"));       # copy in1
1426         &mov    ("ebx",&DWP($i+4,"esi"));
1427         &mov    ("ecx",&DWP($i+8,"esi"));
1428         &mov    ("edx",&DWP($i+12,"esi"));
1429         &mov    (&DWP($i+0,"edi"),"eax");
1430         &mov    ("ebp","eax")                   if ($i==64);
1431         &or     ("ebp","eax")                   if ($i>64);
1432         &mov    (&DWP($i+4,"edi"),"ebx");
1433         &or     ("ebp","ebx")                   if ($i>=64);
1434         &mov    (&DWP($i+8,"edi"),"ecx");
1435         &or     ("ebp","ecx")                   if ($i>=64);
1436         &mov    (&DWP($i+12,"edi"),"edx");
1437         &or     ("ebp","edx")                   if ($i>=64);
1438     }
1439         &xor    ("eax","eax");
1440         &sub    ("eax","ebp");
1441         &or     ("ebp","eax");
1442         &sar    ("ebp",31);
1443         &mov    (&DWP(32*18+0,"esp"),"ebp");    # ~in1infty
1444
1445         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1446         &lea    ("esi",&DWP($in2_z,"esp"));
1447         &lea    ("ebp",&DWP($in2_z,"esp"));
1448         &lea    ("edi",&DWP($Z2sqr,"esp"));
1449         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Z2sqr, in2_z);
1450
1451         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1452         &lea    ("esi",&DWP($in1_z,"esp"));
1453         &lea    ("ebp",&DWP($in1_z,"esp"));
1454         &lea    ("edi",&DWP($Z1sqr,"esp"));
1455         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Z1sqr, in1_z);
1456
1457         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1458         &lea    ("esi",&DWP($Z2sqr,"esp"));
1459         &lea    ("ebp",&DWP($in2_z,"esp"));
1460         &lea    ("edi",&DWP($S1,"esp"));
1461         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S1, Z2sqr, in2_z);
1462
1463         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1464         &lea    ("esi",&DWP($Z1sqr,"esp"));
1465         &lea    ("ebp",&DWP($in1_z,"esp"));
1466         &lea    ("edi",&DWP($S2,"esp"));
1467         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, Z1sqr, in1_z);
1468
1469         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1470         &lea    ("esi",&DWP($in1_y,"esp"));
1471         &lea    ("ebp",&DWP($S1,"esp"));
1472         &lea    ("edi",&DWP($S1,"esp"));
1473         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S1, S1, in1_y);
1474
1475         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1476         &lea    ("esi",&DWP($in2_y,"esp"));
1477         &lea    ("ebp",&DWP($S2,"esp"));
1478         &lea    ("edi",&DWP($S2,"esp"));
1479         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, S2, in2_y);
1480
1481         &lea    ("esi",&DWP($S2,"esp"));
1482         &lea    ("ebp",&DWP($S1,"esp"));
1483         &lea    ("edi",&DWP($R,"esp"));
1484         &call   ("_ecp_nistz256_sub");          # p256_sub(R, S2, S1);
1485
1486         &or     ("ebx","eax");                  # see if result is zero
1487         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1488         &or     ("ebx","ecx");
1489         &or     ("ebx","edx");
1490         &or     ("ebx",&DWP(0,"edi"));
1491         &or     ("ebx",&DWP(4,"edi"));
1492          &lea   ("esi",&DWP($in1_x,"esp"));
1493         &or     ("ebx",&DWP(8,"edi"));
1494          &lea   ("ebp",&DWP($Z2sqr,"esp"));
1495         &or     ("ebx",&DWP(12,"edi"));
1496          &lea   ("edi",&DWP($U1,"esp"));
1497         &mov    (&DWP(32*18+8,"esp"),"ebx");
1498
1499         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U1, in1_x, Z2sqr);
1500
1501         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1502         &lea    ("esi",&DWP($in2_x,"esp"));
1503         &lea    ("ebp",&DWP($Z1sqr,"esp"));
1504         &lea    ("edi",&DWP($U2,"esp"));
1505         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, in2_x, Z1sqr);
1506
1507         &lea    ("esi",&DWP($U2,"esp"));
1508         &lea    ("ebp",&DWP($U1,"esp"));
1509         &lea    ("edi",&DWP($H,"esp"));
1510         &call   ("_ecp_nistz256_sub");          # p256_sub(H, U2, U1);
1511
1512         &or     ("eax","ebx");                  # see if result is zero
1513         &or     ("eax","ecx");
1514         &or     ("eax","edx");
1515         &or     ("eax",&DWP(0,"edi"));
1516         &or     ("eax",&DWP(4,"edi"));
1517         &or     ("eax",&DWP(8,"edi"));
1518         &or     ("eax",&DWP(12,"edi"));         # ~is_equal(U1,U2)
1519
1520         &mov    ("ebx",&DWP(32*18+0,"esp"));    # ~in1infty
1521         &not    ("ebx");                        # -1/0 -> 0/-1
1522         &or     ("eax","ebx");
1523         &mov    ("ebx",&DWP(32*18+4,"esp"));    # ~in2infty
1524         &not    ("ebx");                        # -1/0 -> 0/-1
1525         &or     ("eax","ebx");
1526         &or     ("eax",&DWP(32*18+8,"esp"));    # ~is_equal(S1,S2)
1527
1528         # if (~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2))
1529         &data_byte(0x3e);                       # predict taken
1530         &jnz    (&label("add_proceed"));
1531
1532 &set_label("add_double",16);
1533         &mov    ("esi",&wparam(1));
1534         &mov    ("ebp",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1535         &add    ("esp",4*((8*18+5)-(8*5+1)));   # difference in frame sizes
1536         &jmp    (&label("point_double_shortcut"));
1537
1538 &set_label("add_proceed",16);
1539         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1540         &lea    ("esi",&DWP($R,"esp"));
1541         &lea    ("ebp",&DWP($R,"esp"));
1542         &lea    ("edi",&DWP($Rsqr,"esp"));
1543         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Rsqr, R);
1544
1545         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1546         &lea    ("esi",&DWP($H,"esp"));
1547         &lea    ("ebp",&DWP($in1_z,"esp"));
1548         &lea    ("edi",&DWP($res_z,"esp"));
1549         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_z, H, in1_z);
1550
1551         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1552         &lea    ("esi",&DWP($H,"esp"));
1553         &lea    ("ebp",&DWP($H,"esp"));
1554         &lea    ("edi",&DWP($Hsqr,"esp"));
1555         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Hsqr, H);
1556
1557         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1558         &lea    ("esi",&DWP($in2_z,"esp"));
1559         &lea    ("ebp",&DWP($res_z,"esp"));
1560         &lea    ("edi",&DWP($res_z,"esp"));
1561         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_z, res_z, in2_z);
1562
1563         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1564         &lea    ("esi",&DWP($Hsqr,"esp"));
1565         &lea    ("ebp",&DWP($U1,"esp"));
1566         &lea    ("edi",&DWP($U2,"esp"));
1567         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, U1, Hsqr);
1568
1569         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1570         &lea    ("esi",&DWP($H,"esp"));
1571         &lea    ("ebp",&DWP($Hsqr,"esp"));
1572         &lea    ("edi",&DWP($Hcub,"esp"));
1573         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(Hcub, Hsqr, H);
1574
1575         &lea    ("esi",&DWP($U2,"esp"));
1576         &lea    ("ebp",&DWP($U2,"esp"));
1577         &lea    ("edi",&DWP($Hsqr,"esp"));
1578         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(Hsqr, U2);
1579
1580         &lea    ("esi",&DWP($Rsqr,"esp"));
1581         &lea    ("ebp",&DWP($Hsqr,"esp"));
1582         &lea    ("edi",&DWP($res_x,"esp"));
1583         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, Rsqr, Hsqr);
1584
1585         &lea    ("esi",&DWP($res_x,"esp"));
1586         &lea    ("ebp",&DWP($Hcub,"esp"));
1587         &lea    ("edi",&DWP($res_x,"esp"));
1588         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, res_x, Hcub);
1589
1590         &lea    ("esi",&DWP($U2,"esp"));
1591         &lea    ("ebp",&DWP($res_x,"esp"));
1592         &lea    ("edi",&DWP($res_y,"esp"));
1593         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, U2, res_x);
1594
1595         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1596         &lea    ("esi",&DWP($Hcub,"esp"));
1597         &lea    ("ebp",&DWP($S1,"esp"));
1598         &lea    ("edi",&DWP($S2,"esp"));
1599         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, S1, Hcub);
1600
1601         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1602         &lea    ("esi",&DWP($R,"esp"));
1603         &lea    ("ebp",&DWP($res_y,"esp"));
1604         &lea    ("edi",&DWP($res_y,"esp"));
1605         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_y, R, res_y);
1606
1607         &lea    ("esi",&DWP($res_y,"esp"));
1608         &lea    ("ebp",&DWP($S2,"esp"));
1609         &lea    ("edi",&DWP($res_y,"esp"));
1610         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, res_y, S2);
1611
1612         &mov    ("ebp",&DWP(32*18+0,"esp"));    # ~in1infty
1613         &mov    ("esi",&DWP(32*18+4,"esp"));    # ~in2infty
1614         &mov    ("edi",&wparam(0));
1615         &mov    ("edx","ebp");
1616         &not    ("ebp");
1617         &and    ("edx","esi");                  # ~in1infty & ~in2infty
1618         &and    ("ebp","esi");                  # in1infty & ~in2infty
1619         &not    ("esi");                        # in2infty
1620
1621         ########################################
1622         # conditional moves
1623     for($i=64;$i<96;$i+=4) {
1624         &mov    ("eax","edx");                  # ~in1infty & ~in2infty
1625         &and    ("eax",&DWP($res_x+$i,"esp"));
1626         &mov    ("ebx","ebp");                  # in1infty & ~in2infty
1627         &and    ("ebx",&DWP($in2_x+$i,"esp"));
1628         &mov    ("ecx","esi");                  # in2infty
1629         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1630         &or     ("eax","ebx");
1631         &or     ("eax","ecx");
1632         &mov    (&DWP($i,"edi"),"eax");
1633     }
1634     for($i=0;$i<64;$i+=4) {
1635         &mov    ("eax","edx");                  # ~in1infty & ~in2infty
1636         &and    ("eax",&DWP($res_x+$i,"esp"));
1637         &mov    ("ebx","ebp");                  # in1infty & ~in2infty
1638         &and    ("ebx",&DWP($in2_x+$i,"esp"));
1639         &mov    ("ecx","esi");                  # in2infty
1640         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1641         &or     ("eax","ebx");
1642         &or     ("eax","ecx");
1643         &mov    (&DWP($i,"edi"),"eax");
1644     }
1645     &set_label("add_done");
1646         &stack_pop(8*18+5);
1647 } &function_end("ecp_nistz256_point_add");
1648
1649 ########################################################################
1650 # void ecp_nistz256_point_add_affine(P256_POINT *out,
1651 #                                    const P256_POINT *in1,
1652 #                                    const P256_POINT_AFFINE *in2);
1653 &function_begin("ecp_nistz256_point_add_affine");
1654 {
1655     my ($res_x,$res_y,$res_z,
1656         $in1_x,$in1_y,$in1_z,
1657         $in2_x,$in2_y,
1658         $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..14));
1659     my $Z1sqr = $S2;
1660     my @ONE_mont=(1,0,0,-1,-1,-1,-2,0);
1661
1662         &mov    ("esi",&wparam(1));
1663
1664         # above map() describes stack layout with 15 temporary
1665         # 256-bit vectors on top, then we take extra words for
1666         # ~in1infty, ~in2infty, and OPENSSL_ia32cap_P copy.
1667         &stack_push(8*15+3);
1668                                                 if ($sse2) {
1669         &call   ("_picup_eax");
1670     &set_label("pic");
1671         &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
1672         &mov    ("ebp",&DWP(0,"edx"));          }
1673
1674         &lea    ("edi",&DWP($in1_x,"esp"));
1675     for($i=0;$i<96;$i+=16) {
1676         &mov    ("eax",&DWP($i+0,"esi"));       # copy in1
1677         &mov    ("ebx",&DWP($i+4,"esi"));
1678         &mov    ("ecx",&DWP($i+8,"esi"));
1679         &mov    ("edx",&DWP($i+12,"esi"));
1680         &mov    (&DWP($i+0,"edi"),"eax");
1681         &mov    (&DWP(32*15+8,"esp"),"ebp")     if ($i==0);
1682         &mov    ("ebp","eax")                   if ($i==64);
1683         &or     ("ebp","eax")                   if ($i>64);
1684         &mov    (&DWP($i+4,"edi"),"ebx");
1685         &or     ("ebp","ebx")                   if ($i>=64);
1686         &mov    (&DWP($i+8,"edi"),"ecx");
1687         &or     ("ebp","ecx")                   if ($i>=64);
1688         &mov    (&DWP($i+12,"edi"),"edx");
1689         &or     ("ebp","edx")                   if ($i>=64);
1690     }
1691         &xor    ("eax","eax");
1692         &mov    ("esi",&wparam(2));
1693         &sub    ("eax","ebp");
1694         &or     ("ebp","eax");
1695         &sar    ("ebp",31);
1696         &mov    (&DWP(32*15+0,"esp"),"ebp");    # ~in1infty
1697
1698         &lea    ("edi",&DWP($in2_x,"esp"));
1699     for($i=0;$i<64;$i+=16) {
1700         &mov    ("eax",&DWP($i+0,"esi"));       # copy in2
1701         &mov    ("ebx",&DWP($i+4,"esi"));
1702         &mov    ("ecx",&DWP($i+8,"esi"));
1703         &mov    ("edx",&DWP($i+12,"esi"));
1704         &mov    (&DWP($i+0,"edi"),"eax");
1705         &mov    ("ebp","eax")                   if ($i==0);
1706         &or     ("ebp","eax")                   if ($i!=0);
1707         &mov    (&DWP($i+4,"edi"),"ebx");
1708         &or     ("ebp","ebx");
1709         &mov    (&DWP($i+8,"edi"),"ecx");
1710         &or     ("ebp","ecx");
1711         &mov    (&DWP($i+12,"edi"),"edx");
1712         &or     ("ebp","edx");
1713     }
1714         &xor    ("ebx","ebx");
1715         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1716         &sub    ("ebx","ebp");
1717          &lea   ("esi",&DWP($in1_z,"esp"));
1718         &or     ("ebx","ebp");
1719          &lea   ("ebp",&DWP($in1_z,"esp"));
1720         &sar    ("ebx",31);
1721          &lea   ("edi",&DWP($Z1sqr,"esp"));
1722         &mov    (&DWP(32*15+4,"esp"),"ebx");    # ~in2infty
1723
1724         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Z1sqr, in1_z);
1725
1726         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1727         &lea    ("esi",&DWP($in2_x,"esp"));
1728         &mov    ("ebp","edi");                  # %esi is stull &Z1sqr
1729         &lea    ("edi",&DWP($U2,"esp"));
1730         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, Z1sqr, in2_x);
1731
1732         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1733         &lea    ("esi",&DWP($in1_z,"esp"));
1734         &lea    ("ebp",&DWP($Z1sqr,"esp"));
1735         &lea    ("edi",&DWP($S2,"esp"));
1736         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, Z1sqr, in1_z);
1737
1738         &lea    ("esi",&DWP($U2,"esp"));
1739         &lea    ("ebp",&DWP($in1_x,"esp"));
1740         &lea    ("edi",&DWP($H,"esp"));
1741         &call   ("_ecp_nistz256_sub");          # p256_sub(H, U2, in1_x);
1742
1743         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1744         &lea    ("esi",&DWP($in2_y,"esp"));
1745         &lea    ("ebp",&DWP($S2,"esp"));
1746         &lea    ("edi",&DWP($S2,"esp"));
1747         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, S2, in2_y);
1748
1749         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1750         &lea    ("esi",&DWP($in1_z,"esp"));
1751         &lea    ("ebp",&DWP($H,"esp"));
1752         &lea    ("edi",&DWP($res_z,"esp"));
1753         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_z, H, in1_z);
1754
1755         &lea    ("esi",&DWP($S2,"esp"));
1756         &lea    ("ebp",&DWP($in1_y,"esp"));
1757         &lea    ("edi",&DWP($R,"esp"));
1758         &call   ("_ecp_nistz256_sub");          # p256_sub(R, S2, in1_y);
1759
1760         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1761         &lea    ("esi",&DWP($H,"esp"));
1762         &lea    ("ebp",&DWP($H,"esp"));
1763         &lea    ("edi",&DWP($Hsqr,"esp"));
1764         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Hsqr, H);
1765
1766         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1767         &lea    ("esi",&DWP($R,"esp"));
1768         &lea    ("ebp",&DWP($R,"esp"));
1769         &lea    ("edi",&DWP($Rsqr,"esp"));
1770         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Rsqr, R);
1771
1772         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1773         &lea    ("esi",&DWP($in1_x,"esp"));
1774         &lea    ("ebp",&DWP($Hsqr,"esp"));
1775         &lea    ("edi",&DWP($U2,"esp"));
1776         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, in1_x, Hsqr);
1777
1778         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1779         &lea    ("esi",&DWP($H,"esp"));
1780         &lea    ("ebp",&DWP($Hsqr,"esp"));
1781         &lea    ("edi",&DWP($Hcub,"esp"));
1782         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(Hcub, Hsqr, H);
1783
1784         &lea    ("esi",&DWP($U2,"esp"));
1785         &lea    ("ebp",&DWP($U2,"esp"));
1786         &lea    ("edi",&DWP($Hsqr,"esp"));
1787         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(Hsqr, U2);
1788
1789         &lea    ("esi",&DWP($Rsqr,"esp"));
1790         &lea    ("ebp",&DWP($Hsqr,"esp"));
1791         &lea    ("edi",&DWP($res_x,"esp"));
1792         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, Rsqr, Hsqr);
1793
1794         &lea    ("esi",&DWP($res_x,"esp"));
1795         &lea    ("ebp",&DWP($Hcub,"esp"));
1796         &lea    ("edi",&DWP($res_x,"esp"));
1797         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, res_x, Hcub);
1798
1799         &lea    ("esi",&DWP($U2,"esp"));
1800         &lea    ("ebp",&DWP($res_x,"esp"));
1801         &lea    ("edi",&DWP($res_y,"esp"));
1802         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, U2, res_x);
1803
1804         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1805         &lea    ("esi",&DWP($Hcub,"esp"));
1806         &lea    ("ebp",&DWP($in1_y,"esp"));
1807         &lea    ("edi",&DWP($S2,"esp"));
1808         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, Hcub, in1_y);
1809
1810         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1811         &lea    ("esi",&DWP($R,"esp"));
1812         &lea    ("ebp",&DWP($res_y,"esp"));
1813         &lea    ("edi",&DWP($res_y,"esp"));
1814         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_y, res_y, R);
1815
1816         &lea    ("esi",&DWP($res_y,"esp"));
1817         &lea    ("ebp",&DWP($S2,"esp"));
1818         &lea    ("edi",&DWP($res_y,"esp"));
1819         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, res_y, S2);
1820
1821         &mov    ("ebp",&DWP(32*15+0,"esp"));    # ~in1infty
1822         &mov    ("esi",&DWP(32*15+4,"esp"));    # ~in2infty
1823         &mov    ("edi",&wparam(0));
1824         &mov    ("edx","ebp");
1825         &not    ("ebp");
1826         &and    ("edx","esi");                  # ~in1infty & ~in2infty
1827         &and    ("ebp","esi");                  # in1infty & ~in2infty
1828         &not    ("esi");                        # in2infty
1829
1830         ########################################
1831         # conditional moves
1832     for($i=64;$i<96;$i+=4) {
1833         my $one=@ONE_mont[($i-64)/4];
1834
1835         &mov    ("eax","edx");
1836         &and    ("eax",&DWP($res_x+$i,"esp"));
1837         &mov    ("ebx","ebp")                   if ($one && $one!=-1);
1838         &and    ("ebx",$one)                    if ($one && $one!=-1);
1839         &mov    ("ecx","esi");
1840         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1841         &or     ("eax",$one==-1?"ebp":"ebx")    if ($one);
1842         &or     ("eax","ecx");
1843         &mov    (&DWP($i,"edi"),"eax");
1844     }
1845     for($i=0;$i<64;$i+=4) {
1846         &mov    ("eax","edx");                  # ~in1infty & ~in2infty
1847         &and    ("eax",&DWP($res_x+$i,"esp"));
1848         &mov    ("ebx","ebp");                  # in1infty & ~in2infty
1849         &and    ("ebx",&DWP($in2_x+$i,"esp"));
1850         &mov    ("ecx","esi");                  # in2infty
1851         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1852         &or     ("eax","ebx");
1853         &or     ("eax","ecx");
1854         &mov    (&DWP($i,"edi"),"eax");
1855     }
1856         &stack_pop(8*15+3);
1857 } &function_end("ecp_nistz256_point_add_affine");
1858
1859 &asm_finish();
1860
1861 close STDOUT or die "error closing STDOUT";