android / platform / external / openssl / e67c7edd65f69f58535fb358a8665a23e5de62a4 / . / crypto / bn / asm / s390x-mont.pl

#!/usr/bin/env perl | |

# ==================================================================== | |

# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL | |

# project. The module is, however, dual licensed under OpenSSL and | |

# CRYPTOGAMS licenses depending on where you obtain it. For further | |

# details see http://www.openssl.org/~appro/cryptogams/. | |

# ==================================================================== | |

# April 2007. | |

# | |

# Performance improvement over vanilla C code varies from 85% to 45% | |

# depending on key length and benchmark. Unfortunately in this context | |

# these are not very impressive results [for code that utilizes "wide" | |

# 64x64=128-bit multiplication, which is not commonly available to C | |

# programmers], at least hand-coded bn_asm.c replacement is known to | |

# provide 30-40% better results for longest keys. Well, on a second | |

# thought it's not very surprising, because z-CPUs are single-issue | |

# and _strictly_ in-order execution, while bn_mul_mont is more or less | |

# dependent on CPU ability to pipe-line instructions and have several | |

# of them "in-flight" at the same time. I mean while other methods, | |

# for example Karatsuba, aim to minimize amount of multiplications at | |

# the cost of other operations increase, bn_mul_mont aim to neatly | |

# "overlap" multiplications and the other operations [and on most | |

# platforms even minimize the amount of the other operations, in | |

# particular references to memory]. But it's possible to improve this | |

# module performance by implementing dedicated squaring code-path and | |

# possibly by unrolling loops... | |

# January 2009. | |

# | |

# Reschedule to minimize/avoid Address Generation Interlock hazard, | |

# make inner loops counter-based. | |

# November 2010. | |

# | |

# Adapt for -m31 build. If kernel supports what's called "highgprs" | |

# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit | |

# instructions and achieve "64-bit" performance even in 31-bit legacy | |

# application context. The feature is not specific to any particular | |

# processor, as long as it's "z-CPU". Latter implies that the code | |

# remains z/Architecture specific. Compatibility with 32-bit BN_ULONG | |

# is achieved by swapping words after 64-bit loads, follow _dswap-s. | |

# On z990 it was measured to perform 2.6-2.2 times better than | |

# compiler-generated code, less for longer keys... | |

$flavour = shift; | |

if ($flavour =~ /3[12]/) { | |

$SIZE_T=4; | |

$g=""; | |

} else { | |

$SIZE_T=8; | |

$g="g"; | |

} | |

while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} | |

open STDOUT,">$output"; | |

$stdframe=16*$SIZE_T+4*8; | |

$mn0="%r0"; | |

$num="%r1"; | |

# int bn_mul_mont( | |

$rp="%r2"; # BN_ULONG *rp, | |

$ap="%r3"; # const BN_ULONG *ap, | |

$bp="%r4"; # const BN_ULONG *bp, | |

$np="%r5"; # const BN_ULONG *np, | |

$n0="%r6"; # const BN_ULONG *n0, | |

#$num="160(%r15)" # int num); | |

$bi="%r2"; # zaps rp | |

$j="%r7"; | |

$ahi="%r8"; | |

$alo="%r9"; | |

$nhi="%r10"; | |

$nlo="%r11"; | |

$AHI="%r12"; | |

$NHI="%r13"; | |

$count="%r14"; | |

$sp="%r15"; | |

$code.=<<___; | |

.text | |

.globl bn_mul_mont | |

.type bn_mul_mont,\@function | |

bn_mul_mont: | |

lgf $num,`$stdframe+$SIZE_T-4`($sp) # pull $num | |

sla $num,`log($SIZE_T)/log(2)` # $num to enumerate bytes | |

la $bp,0($num,$bp) | |

st${g} %r2,2*$SIZE_T($sp) | |

cghi $num,16 # | |

lghi %r2,0 # | |

blr %r14 # if($num<16) return 0; | |

___ | |

$code.=<<___ if ($flavour =~ /3[12]/); | |

tmll $num,4 | |

bnzr %r14 # if ($num&1) return 0; | |

___ | |

$code.=<<___ if ($flavour !~ /3[12]/); | |

cghi $num,96 # | |

bhr %r14 # if($num>96) return 0; | |

___ | |

$code.=<<___; | |

stm${g} %r3,%r15,3*$SIZE_T($sp) | |

lghi $rp,-$stdframe-8 # leave room for carry bit | |

lcgr $j,$num # -$num | |

lgr %r0,$sp | |

la $rp,0($rp,$sp) | |

la $sp,0($j,$rp) # alloca | |

st${g} %r0,0($sp) # back chain | |

sra $num,3 # restore $num | |

la $bp,0($j,$bp) # restore $bp | |

ahi $num,-1 # adjust $num for inner loop | |

lg $n0,0($n0) # pull n0 | |

_dswap $n0 | |

lg $bi,0($bp) | |

_dswap $bi | |

lg $alo,0($ap) | |

_dswap $alo | |

mlgr $ahi,$bi # ap[0]*bp[0] | |

lgr $AHI,$ahi | |

lgr $mn0,$alo # "tp[0]"*n0 | |

msgr $mn0,$n0 | |

lg $nlo,0($np) # | |

_dswap $nlo | |

mlgr $nhi,$mn0 # np[0]*m1 | |

algr $nlo,$alo # +="tp[0]" | |

lghi $NHI,0 | |

alcgr $NHI,$nhi | |

la $j,8(%r0) # j=1 | |

lr $count,$num | |

.align 16 | |

.L1st: | |

lg $alo,0($j,$ap) | |

_dswap $alo | |

mlgr $ahi,$bi # ap[j]*bp[0] | |

algr $alo,$AHI | |

lghi $AHI,0 | |

alcgr $AHI,$ahi | |

lg $nlo,0($j,$np) | |

_dswap $nlo | |

mlgr $nhi,$mn0 # np[j]*m1 | |

algr $nlo,$NHI | |

lghi $NHI,0 | |

alcgr $nhi,$NHI # +="tp[j]" | |

algr $nlo,$alo | |

alcgr $NHI,$nhi | |

stg $nlo,$stdframe-8($j,$sp) # tp[j-1]= | |

la $j,8($j) # j++ | |

brct $count,.L1st | |

algr $NHI,$AHI | |

lghi $AHI,0 | |

alcgr $AHI,$AHI # upmost overflow bit | |

stg $NHI,$stdframe-8($j,$sp) | |

stg $AHI,$stdframe($j,$sp) | |

la $bp,8($bp) # bp++ | |

.Louter: | |

lg $bi,0($bp) # bp[i] | |

_dswap $bi | |

lg $alo,0($ap) | |

_dswap $alo | |

mlgr $ahi,$bi # ap[0]*bp[i] | |

alg $alo,$stdframe($sp) # +=tp[0] | |

lghi $AHI,0 | |

alcgr $AHI,$ahi | |

lgr $mn0,$alo | |

msgr $mn0,$n0 # tp[0]*n0 | |

lg $nlo,0($np) # np[0] | |

_dswap $nlo | |

mlgr $nhi,$mn0 # np[0]*m1 | |

algr $nlo,$alo # +="tp[0]" | |

lghi $NHI,0 | |

alcgr $NHI,$nhi | |

la $j,8(%r0) # j=1 | |

lr $count,$num | |

.align 16 | |

.Linner: | |

lg $alo,0($j,$ap) | |

_dswap $alo | |

mlgr $ahi,$bi # ap[j]*bp[i] | |

algr $alo,$AHI | |

lghi $AHI,0 | |

alcgr $ahi,$AHI | |

alg $alo,$stdframe($j,$sp)# +=tp[j] | |

alcgr $AHI,$ahi | |

lg $nlo,0($j,$np) | |

_dswap $nlo | |

mlgr $nhi,$mn0 # np[j]*m1 | |

algr $nlo,$NHI | |

lghi $NHI,0 | |

alcgr $nhi,$NHI | |

algr $nlo,$alo # +="tp[j]" | |

alcgr $NHI,$nhi | |

stg $nlo,$stdframe-8($j,$sp) # tp[j-1]= | |

la $j,8($j) # j++ | |

brct $count,.Linner | |

algr $NHI,$AHI | |

lghi $AHI,0 | |

alcgr $AHI,$AHI | |

alg $NHI,$stdframe($j,$sp)# accumulate previous upmost overflow bit | |

lghi $ahi,0 | |

alcgr $AHI,$ahi # new upmost overflow bit | |

stg $NHI,$stdframe-8($j,$sp) | |

stg $AHI,$stdframe($j,$sp) | |

la $bp,8($bp) # bp++ | |

cl${g} $bp,`$stdframe+8+4*$SIZE_T`($j,$sp) # compare to &bp[num] | |

jne .Louter | |

l${g} $rp,`$stdframe+8+2*$SIZE_T`($j,$sp) # reincarnate rp | |

la $ap,$stdframe($sp) | |

ahi $num,1 # restore $num, incidentally clears "borrow" | |

la $j,0(%r0) | |

lr $count,$num | |

.Lsub: lg $alo,0($j,$ap) | |

lg $nlo,0($j,$np) | |

_dswap $nlo | |

slbgr $alo,$nlo | |

stg $alo,0($j,$rp) | |

la $j,8($j) | |

brct $count,.Lsub | |

lghi $ahi,0 | |

slbgr $AHI,$ahi # handle upmost carry | |

ngr $ap,$AHI | |

lghi $np,-1 | |

xgr $np,$AHI | |

ngr $np,$rp | |

ogr $ap,$np # ap=borrow?tp:rp | |

la $j,0(%r0) | |

lgr $count,$num | |

.Lcopy: lg $alo,0($j,$ap) # copy or in-place refresh | |

_dswap $alo | |

stg $j,$stdframe($j,$sp) # zap tp | |

stg $alo,0($j,$rp) | |

la $j,8($j) | |

brct $count,.Lcopy | |

la %r1,`$stdframe+8+6*$SIZE_T`($j,$sp) | |

lm${g} %r6,%r15,0(%r1) | |

lghi %r2,1 # signal "processed" | |

br %r14 | |

.size bn_mul_mont,.-bn_mul_mont | |

.string "Montgomery Multiplication for s390x, CRYPTOGAMS by <appro\@openssl.org>" | |

___ | |

foreach (split("\n",$code)) { | |

s/\`([^\`]*)\`/eval $1/ge; | |

s/_dswap\s+(%r[0-9]+)/sprintf("rllg\t%s,%s,32",$1,$1) if($SIZE_T==4)/e; | |

print $_,"\n"; | |

} | |

close STDOUT; |