2 # Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # The module implements "4-bit" GCM GHASH function and underlying
20 # single multiplication operation in GF(2^128). "4-bit" means that it
21 # uses 256 bytes per-key table [+128 bytes shared table]. Performance
22 # was measured to be ~18 cycles per processed byte on z10, which is
23 # almost 40% better than gcc-generated code. It should be noted that
24 # 18 cycles is worse result than expected: loop is scheduled for 12
25 # and the result should be close to 12. In the lack of instruction-
26 # level profiling data it's impossible to tell why...
30 # Adapt for -m31 build. If kernel supports what's called "highgprs"
31 # feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
32 # instructions and achieve "64-bit" performance even in 31-bit legacy
33 # application context. The feature is not specific to any particular
34 # processor, as long as it's "z-CPU". Latter implies that the code
35 # remains z/Architecture specific. On z990 it was measured to perform
36 # 2.8x better than 32-bit code generated by gcc 4.3.
40 # Support for hardware KIMD-GHASH is verified to produce correct
41 # result and therefore is engaged. On z196 it was measured to process
42 # 8KB buffer ~7 faster than software implementation. It's not as
43 # impressive for smaller buffer sizes and for smallest 16-bytes buffer
44 # it's actually almost 2 times slower. Which is the reason why
45 # KIMD-GHASH is not used in gcm_gmult_4bit.
47 # $output is the last argument if it looks like a file (it has an extension)
48 # $flavour is the first argument if it doesn't look like a file
49 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
50 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
52 if ($flavour =~ /3[12]/) {
60 $output and open STDOUT,">$output";
67 $Xi="%r2"; # argument block
72 $rem0="%r6"; # variables
85 #include "s390x_arch.h"
93 $code.=<<___ if(!$softonly && 0); # hardware is slow for single block...
94 larl %r1,OPENSSL_s390xcap_P
96 lg %r1,S390X_KIMD+8(%r1) # load second word of kimd capabilities
98 tmhh %r1,0x4000 # check for function 65
100 stg %r0,16($sp) # arrange 16 bytes of zero input
102 lghi %r0,S390X_GHASH # function 65
103 la %r1,0($Xi) # H lies right after Xi in gcm128_context
106 .long 0xb93e0004 # kimd %r0,$inp
107 brc 1,.-4 # pay attention to "partial completion"
113 stm${g} %r6,%r14,6*$SIZE_T($sp)
118 larl $rem_4bit,rem_4bit
120 lg $Zlo,8+1($Xi) # Xi
122 .type gcm_gmult_4bit,\@function
123 .size gcm_gmult_4bit,(.-gcm_gmult_4bit)
125 .globl gcm_ghash_4bit
129 $code.=<<___ if(!$softonly);
130 larl %r1,OPENSSL_s390xcap_P
131 lg %r0,S390X_KIMD+8(%r1) # load second word of kimd capabilities
133 tmhh %r0,0x4000 # check for function 65
135 lghi %r0,S390X_GHASH # function 65
136 la %r1,0($Xi) # H lies right after Xi in gcm128_context
137 .long 0xb93e0004 # kimd %r0,$inp
138 brc 1,.-4 # pay attention to "partial completion"
143 $code.=<<___ if ($flavour =~ /3[12]/);
147 stm${g} %r6,%r14,6*$SIZE_T($sp)
152 larl $rem_4bit,rem_4bit
154 lg $Zlo,8+1($Xi) # Xi
158 xg $Zhi,0($inp) # Xi ^= inp
167 srlg $xi,$Zlo,8 # extract second byte
173 lg $Zlo,8($nlo,$Htbl)
174 lg $Zhi,0($nlo,$Htbl)
185 xg $Zlo,8($nhi,$Htbl)
186 xg $Zhi,0($nhi,$Htbl)
197 xg $Zlo,8($nlo,$Htbl)
199 xg $Zhi,0($nlo,$Htbl)
201 xg $Zhi,0($rem0,$rem_4bit)
211 xg $Zlo,8($nhi,$Htbl)
212 xg $Zhi,0($nhi,$Htbl)
214 xg $Zhi,0($rem1,$rem_4bit)
219 brct $cnt,.Lghash_inner
223 xg $Zlo,8($nlo,$Htbl)
224 xg $Zhi,0($nlo,$Htbl)
226 xg $Zhi,0($rem0,$rem_4bit)
233 xg $Zlo,8($nhi,$Htbl)
234 xg $Zhi,0($nhi,$Htbl)
236 xg $Zhi,0($rem1,$rem_4bit)
238 lg $tmp,0($xi,$rem_4bit)
240 sllg $tmp,$tmp,4 # correct last rem_4bit[rem]
246 lm${g} %r6,%r14,6*$SIZE_T($sp)
248 .type gcm_ghash_4bit,\@function
249 .size gcm_ghash_4bit,(.-gcm_ghash_4bit)
253 .long `0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0
254 .long `0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0
255 .long `0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0
256 .long `0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0
257 .type rem_4bit,\@object
258 .size rem_4bit,(.-rem_4bit)
259 .string "GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>"
262 $code =~ s/\`([^\`]*)\`/eval $1/gem;
264 close STDOUT or die "error closing STDOUT";