-
-
-sub RC4
- {
- local($name)=@_;
-
- &function_begin_B($name,"");
-
- &mov($ty,&wparam(1)); # len
- &cmp($ty,0);
- &jne(&label("proceed"));
- &ret();
- &set_label("proceed");
-
- &comment("");
-
- &push("ebp");
- &push("ebx");
- &push("esi");
- &xor( $x, $x); # avoid partial register stalls
- &push("edi");
- &xor( $y, $y); # avoid partial register stalls
- &mov( $d, &wparam(0)); # key
- &mov( $in, &wparam(2));
-
- &movb( &LB($x), &BP(0,$d,"",1));
- &movb( &LB($y), &BP(4,$d,"",1));
-
- &mov( $out, &wparam(3));
- &inc( &LB($x));
-
- &stack_push(3); # 3 temp variables
- &add( $d, 8);
-
- # detect compressed schedule, see commentary section in rc4_skey.c...
- &cmp(&DWP(256,$d),-1);
- &je(&label("RC4_CHAR"));
-
- &lea( $ty, &DWP(-8,$ty,$in));
-
- # check for 0 length input
-
- &mov( &swtmp(2), $ty); # this is now address to exit at
- &mov( $tx, &DWP(0,$d,$x,4));
-
- &cmp( $ty, $in);
- &jb( &label("end")); # less than 8 bytes
-
- &set_label("start");
-
- # filling DELAY SLOT
- &add( $in, 8);
-
- &RC4_loop(0,-1,0);
- &RC4_loop(1,0,0);
- &RC4_loop(2,0,0);
- &RC4_loop(3,0,0);
- &RC4_loop(4,0,0);
- &RC4_loop(5,0,0);
- &RC4_loop(6,0,0);
- &RC4_loop(7,1,0);
-
- &comment("apply the cipher text");
- # xor the cipher data with input
-
- #&add( $out, 8); #moved up into last round
-
- &mov( $tx, &swtmp(0));
- &mov( $ty, &DWP(-8,$in,"",0));
- &xor( $tx, $ty);
- &mov( $ty, &DWP(-4,$in,"",0));
- &mov( &DWP(-8,$out,"",0), $tx);
- &mov( $tx, &swtmp(1));
- &xor( $tx, $ty);
- &mov( $ty, &swtmp(2)); # load end ptr;
- &mov( &DWP(-4,$out,"",0), $tx);
- &mov( $tx, &DWP(0,$d,$x,4));
- &cmp($in, $ty);
- &jbe(&label("start"));
-
- &set_label("end");
-
- # There is quite a bit of extra crap in RC4_loop() for this
- # first round
- &RC4_loop(0,-1,1);
- &RC4_loop(1,0,1);
- &RC4_loop(2,0,1);
- &RC4_loop(3,0,1);
- &RC4_loop(4,0,1);
- &RC4_loop(5,0,1);
- &RC4_loop(6,1,1);
-
- &jmp(&label("finished"));
-
- &align(16);
- # this is essentially Intel P4 specific codepath, see rc4_skey.c...
- &set_label("RC4_CHAR");
-
- &lea ($ty,&DWP(0,$in,$ty));
- &mov (&swtmp(2),$ty);
-
+ &$func ($out,&DWP(0,$dat,$ty,4));
+}
+
+if ($alt=0) {
+ # >20% faster on Atom and Sandy Bridge[!], 8% faster on Opteron,
+ # but ~40% slower on Core2 and Westmere... Attempt to add movz
+ # brings down Opteron by 25%, Atom and Sandy Bridge by 15%, yet
+ # on Core2 with movz it's almost 20% slower than below alternative
+ # code... Yes, it's a total mess...
+ my @XX=($xx,$out);
+ $RC4_loop_mmx = sub { # SSE actually...
+ my $i=shift;
+ my $j=$i<=0?0:$i>>1;
+ my $mm=$i<=0?"mm0":"mm".($i&1);
+
+ &add (&LB($yy),&LB($tx));
+ &lea (@XX[1],&DWP(1,@XX[0]));
+ &pxor ("mm2","mm0") if ($i==0);
+ &psllq ("mm1",8) if ($i==0);
+ &and (@XX[1],0xff);
+ &pxor ("mm0","mm0") if ($i<=0);
+ &mov ($ty,&DWP(0,$dat,$yy,4));
+ &mov (&DWP(0,$dat,$yy,4),$tx);
+ &pxor ("mm1","mm2") if ($i==0);
+ &mov (&DWP(0,$dat,$XX[0],4),$ty);
+ &add (&LB($ty),&LB($tx));
+ &movd (@XX[0],"mm7") if ($i==0);
+ &mov ($tx,&DWP(0,$dat,@XX[1],4));
+ &pxor ("mm1","mm1") if ($i==1);
+ &movq ("mm2",&QWP(0,$inp)) if ($i==1);
+ &movq (&QWP(-8,(@XX[0],$inp)),"mm1") if ($i==0);
+ &pinsrw ($mm,&DWP(0,$dat,$ty,4),$j);
+
+ push (@XX,shift(@XX)) if ($i>=0);
+ }
+} else {
+ # Using pinsrw here improves performane on Intel CPUs by 2-3%, but
+ # brings down AMD by 7%...
+ $RC4_loop_mmx = sub {
+ my $i=shift;
+
+ &add (&LB($yy),&LB($tx));
+ &psllq ("mm1",8*(($i-1)&7)) if (abs($i)!=1);
+ &mov ($ty,&DWP(0,$dat,$yy,4));
+ &mov (&DWP(0,$dat,$yy,4),$tx);
+ &mov (&DWP(0,$dat,$xx,4),$ty);
+ &inc ($xx);
+ &add ($ty,$tx);
+ &movz ($xx,&LB($xx)); # (*)
+ &movz ($ty,&LB($ty)); # (*)
+ &pxor ("mm2",$i==1?"mm0":"mm1") if ($i>=0);
+ &movq ("mm0",&QWP(0,$inp)) if ($i<=0);
+ &movq (&QWP(-8,($out,$inp)),"mm2") if ($i==0);
+ &mov ($tx,&DWP(0,$dat,$xx,4));
+ &movd ($i>0?"mm1":"mm2",&DWP(0,$dat,$ty,4));
+
+ # (*) This is the key to Core2 and Westmere performance.
+ # Without movz out-of-order execution logic confuses
+ # itself and fails to reorder loads and stores. Problem
+ # appears to be fixed in Sandy Bridge...
+ }
+}
+
+&external_label("OPENSSL_ia32cap_P");
+
+# void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out);
+&function_begin("RC4");
+ &mov ($dat,&wparam(0)); # load key schedule pointer
+ &mov ($ty, &wparam(1)); # load len
+ &mov ($inp,&wparam(2)); # load inp
+ &mov ($out,&wparam(3)); # load out
+
+ &xor ($xx,$xx); # avoid partial register stalls
+ &xor ($yy,$yy);
+
+ &cmp ($ty,0); # safety net
+ &je (&label("abort"));
+
+ &mov (&LB($xx),&BP(0,$dat)); # load key->x
+ &mov (&LB($yy),&BP(4,$dat)); # load key->y
+ &add ($dat,8);
+
+ &lea ($tx,&DWP(0,$inp,$ty));
+ &sub ($out,$inp); # re-bias out
+ &mov (&wparam(1),$tx); # save input+len
+
+ &inc (&LB($xx));
+
+ # detect compressed key schedule...
+ &cmp (&DWP(256,$dat),-1);
+ &je (&label("RC4_CHAR"));
+
+ &mov ($tx,&DWP(0,$dat,$xx,4));
+
+ &and ($ty,-4); # how many 4-byte chunks?
+ &jz (&label("loop1"));
+
+ &mov (&wparam(3),$out); # $out as accumulator in these loops
+ if ($x86only) {
+ &jmp (&label("go4loop4"));
+ } else {
+ &test ($ty,-8);
+ &jz (&label("go4loop4"));
+
+ &picmeup($out,"OPENSSL_ia32cap_P");
+ &bt (&DWP(0,$out),26); # check SSE2 bit [could have been MMX]
+ &jnc (&label("go4loop4"));
+
+ &mov ($out,&wparam(3)) if (!$alt);
+ &movd ("mm7",&wparam(3)) if ($alt);
+ &and ($ty,-8);
+ &lea ($ty,&DWP(-8,$inp,$ty));
+ &mov (&DWP(-4,$dat),$ty); # save input+(len/8)*8-8
+
+ &$RC4_loop_mmx(-1);
+ &jmp(&label("loop_mmx_enter"));
+
+ &set_label("loop_mmx",16);
+ &$RC4_loop_mmx(0);
+ &set_label("loop_mmx_enter");
+ for ($i=1;$i<8;$i++) { &$RC4_loop_mmx($i); }
+ &mov ($ty,$yy);
+ &xor ($yy,$yy); # this is second key to Core2
+ &mov (&LB($yy),&LB($ty)); # and Westmere performance...
+ &cmp ($inp,&DWP(-4,$dat));
+ &lea ($inp,&DWP(8,$inp));
+ &jb (&label("loop_mmx"));
+
+ if ($alt) {
+ &movd ($out,"mm7");
+ &pxor ("mm2","mm0");
+ &psllq ("mm1",8);
+ &pxor ("mm1","mm2");
+ &movq (&QWP(-8,$out,$inp),"mm1");
+ } else {
+ &psllq ("mm1",56);
+ &pxor ("mm2","mm1");
+ &movq (&QWP(-8,$out,$inp),"mm2");
+ }
+ &emms ();
+
+ &cmp ($inp,&wparam(1)); # compare to input+len
+ &je (&label("done"));
+ &jmp (&label("loop1"));
+ }
+
+&set_label("go4loop4",16);
+ &lea ($ty,&DWP(-4,$inp,$ty));
+ &mov (&wparam(2),$ty); # save input+(len/4)*4-4
+
+ &set_label("loop4");
+ for ($i=0;$i<4;$i++) { RC4_loop($i); }
+ &ror ($out,8);
+ &xor ($out,&DWP(0,$inp));
+ &cmp ($inp,&wparam(2)); # compare to input+(len/4)*4-4
+ &mov (&DWP(0,$tx,$inp),$out);# $tx holds re-biased out here
+ &lea ($inp,&DWP(4,$inp));
+ &mov ($tx,&DWP(0,$dat,$xx,4));
+ &jb (&label("loop4"));
+
+ &cmp ($inp,&wparam(1)); # compare to input+len
+ &je (&label("done"));
+ &mov ($out,&wparam(3)); # restore $out
+
+ &set_label("loop1",16);
+ &add (&LB($yy),&LB($tx));
+ &mov ($ty,&DWP(0,$dat,$yy,4));
+ &mov (&DWP(0,$dat,$yy,4),$tx);
+ &mov (&DWP(0,$dat,$xx,4),$ty);
+ &add ($ty,$tx);
+ &inc (&LB($xx));
+ &and ($ty,0xff);
+ &mov ($ty,&DWP(0,$dat,$ty,4));
+ &xor (&LB($ty),&BP(0,$inp));
+ &lea ($inp,&DWP(1,$inp));
+ &mov ($tx,&DWP(0,$dat,$xx,4));
+ &cmp ($inp,&wparam(1)); # compare to input+len
+ &mov (&BP(-1,$out,$inp),&LB($ty));
+ &jb (&label("loop1"));
+
+ &jmp (&label("done"));
+
+# this is essentially Intel P4 specific codepath...
+&set_label("RC4_CHAR",16);
+ &movz ($tx,&BP(0,$dat,$xx));