D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
usr
/
src
/
linux-headers-5.4.0-113
/
drivers
/
crypto
/
vmx
/
Filename :
aesp8-ppc.pl
back
Copy
#! /usr/bin/env perl # SPDX-License-Identifier: GPL-2.0 # This code is taken from CRYPTOGAMs[1] and is included here using the option # in the license to distribute the code under the GPL. Therefore this program # is free software; you can redistribute it and/or modify it under the terms of # the GNU General Public License version 2 as published by the Free Software # Foundation. # # [1] https://www.openssl.org/~appro/cryptogams/ # Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org> # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain copyright notices, # this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials # provided with the distribution. # # * Neither the name of the CRYPTOGAMS nor the names of its # copyright holder and contributors may be used to endorse or # promote products derived from this software without specific # prior written permission. # # ALTERNATIVELY, provided that this notice is retained in full, this # product may be distributed under the terms of the GNU General Public # License (GPL), in which case the provisions of the GPL apply INSTEAD OF # those given above. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # ==================================================================== # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL # project. The module is, however, dual licensed under OpenSSL and # CRYPTOGAMS licenses depending on where you obtain it. For further # details see http://www.openssl.org/~appro/cryptogams/. # ==================================================================== # # This module implements support for AES instructions as per PowerISA # specification version 2.07, first implemented by POWER8 processor. # The module is endian-agnostic in sense that it supports both big- # and little-endian cases. Data alignment in parallelizable modes is # handled with VSX loads and stores, which implies MSR.VSX flag being # set. It should also be noted that ISA specification doesn't prohibit # alignment exceptions for these instructions on page boundaries. # Initially alignment was handled in pure AltiVec/VMX way [when data # is aligned programmatically, which in turn guarantees exception- # free execution], but it turned to hamper performance when vcipher # instructions are interleaved. It's reckoned that eventual # misalignment penalties at page boundaries are in average lower # than additional overhead in pure AltiVec approach. # # May 2016 # # Add XTS subroutine, 9x on little- and 12x improvement on big-endian # systems were measured. # ###################################################################### # Current large-block performance in cycles per byte processed with # 128-bit key (less is better). # # CBC en-/decrypt CTR XTS # POWER8[le] 3.96/0.72 0.74 1.1 # POWER8[be] 3.75/0.65 0.66 1.0 $flavour = shift; if ($flavour =~ /64/) { $SIZE_T =8; $LRSAVE =2*$SIZE_T; $STU ="stdu"; $POP ="ld"; $PUSH ="std"; $UCMP ="cmpld"; $SHL ="sldi"; } elsif ($flavour =~ /32/) { $SIZE_T =4; $LRSAVE =$SIZE_T; $STU ="stwu"; $POP ="lwz"; $PUSH ="stw"; $UCMP ="cmplw"; $SHL ="slwi"; } else { die "nonsense $flavour"; } $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0; $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or die "can't locate ppc-xlate.pl"; open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!"; $FRAME=8*$SIZE_T; $prefix="aes_p8"; $sp="r1"; $vrsave="r12"; ######################################################################### {{{ # Key setup procedures # my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8)); my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6)); my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11)); $code.=<<___; .machine "any" .text .align 7 rcon: .long 0x01000000, 0x01000000, 0x01000000, 0x01000000 ?rev .long 0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000 ?rev .long 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c ?rev .long 0,0,0,0 ?asis Lconsts: mflr r0 bcl 20,31,\$+4 mflr $ptr #vvvvv "distance between . and rcon addi $ptr,$ptr,-0x48 mtlr r0 blr .long 0 .byte 0,12,0x14,0,0,0,0,0 .asciz "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>" .globl .${prefix}_set_encrypt_key Lset_encrypt_key: mflr r11 $PUSH r11,$LRSAVE($sp) li $ptr,-1 ${UCMP}i $inp,0 beq- Lenc_key_abort # if ($inp==0) return -1; ${UCMP}i $out,0 beq- Lenc_key_abort # if ($out==0) return -1; li $ptr,-2 cmpwi $bits,128 blt- Lenc_key_abort cmpwi $bits,256 bgt- Lenc_key_abort andi. r0,$bits,0x3f bne- Lenc_key_abort lis r0,0xfff0 mfspr $vrsave,256 mtspr 256,r0 bl Lconsts mtlr r11 neg r9,$inp lvx $in0,0,$inp addi $inp,$inp,15 # 15 is not typo lvsr $key,0,r9 # borrow $key li r8,0x20 cmpwi $bits,192 lvx $in1,0,$inp le?vspltisb $mask,0x0f # borrow $mask lvx $rcon,0,$ptr le?vxor $key,$key,$mask # adjust for byte swap lvx $mask,r8,$ptr addi $ptr,$ptr,0x10 vperm $in0,$in0,$in1,$key # align [and byte swap in LE] li $cnt,8 vxor $zero,$zero,$zero mtctr $cnt ?lvsr $outperm,0,$out vspltisb $outmask,-1 lvx $outhead,0,$out ?vperm $outmask,$zero,$outmask,$outperm blt Loop128 addi $inp,$inp,8 beq L192 addi $inp,$inp,8 b L256 .align 4 Loop128: vperm $key,$in0,$in0,$mask # rotate-n-splat vsldoi $tmp,$zero,$in0,12 # >>32 vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail vcipherlast $key,$key,$rcon stvx $stage,0,$out addi $out,$out,16 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vadduwm $rcon,$rcon,$rcon vxor $in0,$in0,$key bdnz Loop128 lvx $rcon,0,$ptr # last two round keys vperm $key,$in0,$in0,$mask # rotate-n-splat vsldoi $tmp,$zero,$in0,12 # >>32 vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail vcipherlast $key,$key,$rcon stvx $stage,0,$out addi $out,$out,16 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vadduwm $rcon,$rcon,$rcon vxor $in0,$in0,$key vperm $key,$in0,$in0,$mask # rotate-n-splat vsldoi $tmp,$zero,$in0,12 # >>32 vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail vcipherlast $key,$key,$rcon stvx $stage,0,$out addi $out,$out,16 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vxor $in0,$in0,$key vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail stvx $stage,0,$out addi $inp,$out,15 # 15 is not typo addi $out,$out,0x50 li $rounds,10 b Ldone .align 4 L192: lvx $tmp,0,$inp li $cnt,4 vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail stvx $stage,0,$out addi $out,$out,16 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE] vspltisb $key,8 # borrow $key mtctr $cnt vsububm $mask,$mask,$key # adjust the mask Loop192: vperm $key,$in1,$in1,$mask # roate-n-splat vsldoi $tmp,$zero,$in0,12 # >>32 vcipherlast $key,$key,$rcon vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vsldoi $stage,$zero,$in1,8 vspltw $tmp,$in0,3 vxor $tmp,$tmp,$in1 vsldoi $in1,$zero,$in1,12 # >>32 vadduwm $rcon,$rcon,$rcon vxor $in1,$in1,$tmp vxor $in0,$in0,$key vxor $in1,$in1,$key vsldoi $stage,$stage,$in0,8 vperm $key,$in1,$in1,$mask # rotate-n-splat vsldoi $tmp,$zero,$in0,12 # >>32 vperm $outtail,$stage,$stage,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail vcipherlast $key,$key,$rcon stvx $stage,0,$out addi $out,$out,16 vsldoi $stage,$in0,$in1,8 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vperm $outtail,$stage,$stage,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp stvx $stage,0,$out addi $out,$out,16 vspltw $tmp,$in0,3 vxor $tmp,$tmp,$in1 vsldoi $in1,$zero,$in1,12 # >>32 vadduwm $rcon,$rcon,$rcon vxor $in1,$in1,$tmp vxor $in0,$in0,$key vxor $in1,$in1,$key vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail stvx $stage,0,$out addi $inp,$out,15 # 15 is not typo addi $out,$out,16 bdnz Loop192 li $rounds,12 addi $out,$out,0x20 b Ldone .align 4 L256: lvx $tmp,0,$inp li $cnt,7 li $rounds,14 vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail stvx $stage,0,$out addi $out,$out,16 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE] mtctr $cnt Loop256: vperm $key,$in1,$in1,$mask # rotate-n-splat vsldoi $tmp,$zero,$in0,12 # >>32 vperm $outtail,$in1,$in1,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail vcipherlast $key,$key,$rcon stvx $stage,0,$out addi $out,$out,16 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in0,$in0,$tmp vadduwm $rcon,$rcon,$rcon vxor $in0,$in0,$key vperm $outtail,$in0,$in0,$outperm # rotate vsel $stage,$outhead,$outtail,$outmask vmr $outhead,$outtail stvx $stage,0,$out addi $inp,$out,15 # 15 is not typo addi $out,$out,16 bdz Ldone vspltw $key,$in0,3 # just splat vsldoi $tmp,$zero,$in1,12 # >>32 vsbox $key,$key vxor $in1,$in1,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in1,$in1,$tmp vsldoi $tmp,$zero,$tmp,12 # >>32 vxor $in1,$in1,$tmp vxor $in1,$in1,$key b Loop256 .align 4 Ldone: lvx $in1,0,$inp # redundant in aligned case vsel $in1,$outhead,$in1,$outmask stvx $in1,0,$inp li $ptr,0 mtspr 256,$vrsave stw $rounds,0($out) Lenc_key_abort: mr r3,$ptr blr .long 0 .byte 0,12,0x14,1,0,0,3,0 .long 0 .size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key .globl .${prefix}_set_decrypt_key $STU $sp,-$FRAME($sp) mflr r10 $PUSH r10,$FRAME+$LRSAVE($sp) bl Lset_encrypt_key mtlr r10 cmpwi r3,0 bne- Ldec_key_abort slwi $cnt,$rounds,4 subi $inp,$out,240 # first round key srwi $rounds,$rounds,1 add $out,$inp,$cnt # last round key mtctr $rounds Ldeckey: lwz r0, 0($inp) lwz r6, 4($inp) lwz r7, 8($inp) lwz r8, 12($inp) addi $inp,$inp,16 lwz r9, 0($out) lwz r10,4($out) lwz r11,8($out) lwz r12,12($out) stw r0, 0($out) stw r6, 4($out) stw r7, 8($out) stw r8, 12($out) subi $out,$out,16 stw r9, -16($inp) stw r10,-12($inp) stw r11,-8($inp) stw r12,-4($inp) bdnz Ldeckey xor r3,r3,r3 # return value Ldec_key_abort: addi $sp,$sp,$FRAME blr .long 0 .byte 0,12,4,1,0x80,0,3,0 .long 0 .size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key ___ }}} ######################################################################### {{{ # Single block en- and decrypt procedures # sub gen_block () { my $dir = shift; my $n = $dir eq "de" ? "n" : ""; my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7)); $code.=<<___; .globl .${prefix}_${dir}crypt lwz $rounds,240($key) lis r0,0xfc00 mfspr $vrsave,256 li $idx,15 # 15 is not typo mtspr 256,r0 lvx v0,0,$inp neg r11,$out lvx v1,$idx,$inp lvsl v2,0,$inp # inpperm le?vspltisb v4,0x0f ?lvsl v3,0,r11 # outperm le?vxor v2,v2,v4 li $idx,16 vperm v0,v0,v1,v2 # align [and byte swap in LE] lvx v1,0,$key ?lvsl v5,0,$key # keyperm srwi $rounds,$rounds,1 lvx v2,$idx,$key addi $idx,$idx,16 subi $rounds,$rounds,1 ?vperm v1,v1,v2,v5 # align round key vxor v0,v0,v1 lvx v1,$idx,$key addi $idx,$idx,16 mtctr $rounds Loop_${dir}c: ?vperm v2,v2,v1,v5 v${n}cipher v0,v0,v2 lvx v2,$idx,$key addi $idx,$idx,16 ?vperm v1,v1,v2,v5 v${n}cipher v0,v0,v1 lvx v1,$idx,$key addi $idx,$idx,16 bdnz Loop_${dir}c ?vperm v2,v2,v1,v5 v${n}cipher v0,v0,v2 lvx v2,$idx,$key ?vperm v1,v1,v2,v5 v${n}cipherlast v0,v0,v1 vspltisb v2,-1 vxor v1,v1,v1 li $idx,15 # 15 is not typo ?vperm v2,v1,v2,v3 # outmask le?vxor v3,v3,v4 lvx v1,0,$out # outhead vperm v0,v0,v0,v3 # rotate [and byte swap in LE] vsel v1,v1,v0,v2 lvx v4,$idx,$out stvx v1,0,$out vsel v0,v0,v4,v2 stvx v0,$idx,$out mtspr 256,$vrsave blr .long 0 .byte 0,12,0x14,0,0,0,3,0 .long 0 .size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt ___ } &gen_block("en"); &gen_block("de"); }}} ######################################################################### {{{ # CBC en- and decrypt procedures # my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10)); my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3)); my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)= map("v$_",(4..10)); $code.=<<___; .globl .${prefix}_cbc_encrypt ${UCMP}i $len,16 bltlr- cmpwi $enc,0 # test direction lis r0,0xffe0 mfspr $vrsave,256 mtspr 256,r0 li $idx,15 vxor $rndkey0,$rndkey0,$rndkey0 le?vspltisb $tmp,0x0f lvx $ivec,0,$ivp # load [unaligned] iv lvsl $inpperm,0,$ivp lvx $inptail,$idx,$ivp le?vxor $inpperm,$inpperm,$tmp vperm $ivec,$ivec,$inptail,$inpperm neg r11,$inp ?lvsl $keyperm,0,$key # prepare for unaligned key lwz $rounds,240($key) lvsr $inpperm,0,r11 # prepare for unaligned load lvx $inptail,0,$inp addi $inp,$inp,15 # 15 is not typo le?vxor $inpperm,$inpperm,$tmp ?lvsr $outperm,0,$out # prepare for unaligned store vspltisb $outmask,-1 lvx $outhead,0,$out ?vperm $outmask,$rndkey0,$outmask,$outperm le?vxor $outperm,$outperm,$tmp srwi $rounds,$rounds,1 li $idx,16 subi $rounds,$rounds,1 beq Lcbc_dec Lcbc_enc: vmr $inout,$inptail lvx $inptail,0,$inp addi $inp,$inp,16 mtctr $rounds subi $len,$len,16 # len-=16 lvx $rndkey0,0,$key vperm $inout,$inout,$inptail,$inpperm lvx $rndkey1,$idx,$key addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key addi $idx,$idx,16 vxor $inout,$inout,$ivec Loop_cbc_enc: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipher $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key addi $idx,$idx,16 bdnz Loop_cbc_enc ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key li $idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipherlast $ivec,$inout,$rndkey0 ${UCMP}i $len,16 vperm $tmp,$ivec,$ivec,$outperm vsel $inout,$outhead,$tmp,$outmask vmr $outhead,$tmp stvx $inout,0,$out addi $out,$out,16 bge Lcbc_enc b Lcbc_done .align 4 Lcbc_dec: ${UCMP}i $len,128 bge _aesp8_cbc_decrypt8x vmr $tmp,$inptail lvx $inptail,0,$inp addi $inp,$inp,16 mtctr $rounds subi $len,$len,16 # len-=16 lvx $rndkey0,0,$key vperm $tmp,$tmp,$inptail,$inpperm lvx $rndkey1,$idx,$key addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $inout,$tmp,$rndkey0 lvx $rndkey0,$idx,$key addi $idx,$idx,16 Loop_cbc_dec: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vncipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vncipher $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key addi $idx,$idx,16 bdnz Loop_cbc_dec ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vncipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key li $idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vncipherlast $inout,$inout,$rndkey0 ${UCMP}i $len,16 vxor $inout,$inout,$ivec vmr $ivec,$tmp vperm $tmp,$inout,$inout,$outperm vsel $inout,$outhead,$tmp,$outmask vmr $outhead,$tmp stvx $inout,0,$out addi $out,$out,16 bge Lcbc_dec Lcbc_done: addi $out,$out,-1 lvx $inout,0,$out # redundant in aligned case vsel $inout,$outhead,$inout,$outmask stvx $inout,0,$out neg $enc,$ivp # write [unaligned] iv li $idx,15 # 15 is not typo vxor $rndkey0,$rndkey0,$rndkey0 vspltisb $outmask,-1 le?vspltisb $tmp,0x0f ?lvsl $outperm,0,$enc ?vperm $outmask,$rndkey0,$outmask,$outperm le?vxor $outperm,$outperm,$tmp lvx $outhead,0,$ivp vperm $ivec,$ivec,$ivec,$outperm vsel $inout,$outhead,$ivec,$outmask lvx $inptail,$idx,$ivp stvx $inout,0,$ivp vsel $inout,$ivec,$inptail,$outmask stvx $inout,$idx,$ivp mtspr 256,$vrsave blr .long 0 .byte 0,12,0x14,0,0,0,6,0 .long 0 ___ ######################################################################### {{ # Optimized CBC decrypt procedure # my $key_="r11"; my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31)); my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13)); my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21)); my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys # v26-v31 last 6 round keys my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment $code.=<<___; .align 5 _aesp8_cbc_decrypt8x: $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) li r10,`$FRAME+8*16+15` li r11,`$FRAME+8*16+31` stvx v20,r10,$sp # ABI says so addi r10,r10,32 stvx v21,r11,$sp addi r11,r11,32 stvx v22,r10,$sp addi r10,r10,32 stvx v23,r11,$sp addi r11,r11,32 stvx v24,r10,$sp addi r10,r10,32 stvx v25,r11,$sp addi r11,r11,32 stvx v26,r10,$sp addi r10,r10,32 stvx v27,r11,$sp addi r11,r11,32 stvx v28,r10,$sp addi r10,r10,32 stvx v29,r11,$sp addi r11,r11,32 stvx v30,r10,$sp stvx v31,r11,$sp li r0,-1 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave li $x10,0x10 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) li $x20,0x20 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) li $x30,0x30 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) li $x40,0x40 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) li $x50,0x50 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) li $x60,0x60 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) li $x70,0x70 mtspr 256,r0 subi $rounds,$rounds,3 # -4 in total subi $len,$len,128 # bias lvx $rndkey0,$x00,$key # load key schedule lvx v30,$x10,$key addi $key,$key,0x20 lvx v31,$x00,$key ?vperm $rndkey0,$rndkey0,v30,$keyperm addi $key_,$sp,$FRAME+15 mtctr $rounds Load_cbc_dec_key: ?vperm v24,v30,v31,$keyperm lvx v30,$x10,$key addi $key,$key,0x20 stvx v24,$x00,$key_ # off-load round[1] ?vperm v25,v31,v30,$keyperm lvx v31,$x00,$key stvx v25,$x10,$key_ # off-load round[2] addi $key_,$key_,0x20 bdnz Load_cbc_dec_key lvx v26,$x10,$key ?vperm v24,v30,v31,$keyperm lvx v27,$x20,$key stvx v24,$x00,$key_ # off-load round[3] ?vperm v25,v31,v26,$keyperm lvx v28,$x30,$key stvx v25,$x10,$key_ # off-load round[4] addi $key_,$sp,$FRAME+15 # rewind $key_ ?vperm v26,v26,v27,$keyperm lvx v29,$x40,$key ?vperm v27,v27,v28,$keyperm lvx v30,$x50,$key ?vperm v28,v28,v29,$keyperm lvx v31,$x60,$key ?vperm v29,v29,v30,$keyperm lvx $out0,$x70,$key # borrow $out0 ?vperm v30,v30,v31,$keyperm lvx v24,$x00,$key_ # pre-load round[1] ?vperm v31,v31,$out0,$keyperm lvx v25,$x10,$key_ # pre-load round[2] #lvx $inptail,0,$inp # "caller" already did this #addi $inp,$inp,15 # 15 is not typo subi $inp,$inp,15 # undo "caller" le?li $idx,8 lvx_u $in0,$x00,$inp # load first 8 "words" le?lvsl $inpperm,0,$idx le?vspltisb $tmp,0x0f lvx_u $in1,$x10,$inp le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u lvx_u $in2,$x20,$inp le?vperm $in0,$in0,$in0,$inpperm lvx_u $in3,$x30,$inp le?vperm $in1,$in1,$in1,$inpperm lvx_u $in4,$x40,$inp le?vperm $in2,$in2,$in2,$inpperm vxor $out0,$in0,$rndkey0 lvx_u $in5,$x50,$inp le?vperm $in3,$in3,$in3,$inpperm vxor $out1,$in1,$rndkey0 lvx_u $in6,$x60,$inp le?vperm $in4,$in4,$in4,$inpperm vxor $out2,$in2,$rndkey0 lvx_u $in7,$x70,$inp addi $inp,$inp,0x80 le?vperm $in5,$in5,$in5,$inpperm vxor $out3,$in3,$rndkey0 le?vperm $in6,$in6,$in6,$inpperm vxor $out4,$in4,$rndkey0 le?vperm $in7,$in7,$in7,$inpperm vxor $out5,$in5,$rndkey0 vxor $out6,$in6,$rndkey0 vxor $out7,$in7,$rndkey0 mtctr $rounds b Loop_cbc_dec8x .align 5 Loop_cbc_dec8x: vncipher $out0,$out0,v24 vncipher $out1,$out1,v24 vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vncipher $out4,$out4,v24 vncipher $out5,$out5,v24 vncipher $out6,$out6,v24 vncipher $out7,$out7,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vncipher $out0,$out0,v25 vncipher $out1,$out1,v25 vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vncipher $out4,$out4,v25 vncipher $out5,$out5,v25 vncipher $out6,$out6,v25 vncipher $out7,$out7,v25 lvx v25,$x10,$key_ # round[4] bdnz Loop_cbc_dec8x subic $len,$len,128 # $len-=128 vncipher $out0,$out0,v24 vncipher $out1,$out1,v24 vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vncipher $out4,$out4,v24 vncipher $out5,$out5,v24 vncipher $out6,$out6,v24 vncipher $out7,$out7,v24 subfe. r0,r0,r0 # borrow?-1:0 vncipher $out0,$out0,v25 vncipher $out1,$out1,v25 vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vncipher $out4,$out4,v25 vncipher $out5,$out5,v25 vncipher $out6,$out6,v25 vncipher $out7,$out7,v25 and r0,r0,$len vncipher $out0,$out0,v26 vncipher $out1,$out1,v26 vncipher $out2,$out2,v26 vncipher $out3,$out3,v26 vncipher $out4,$out4,v26 vncipher $out5,$out5,v26 vncipher $out6,$out6,v26 vncipher $out7,$out7,v26 add $inp,$inp,r0 # $inp is adjusted in such # way that at exit from the # loop inX-in7 are loaded # with last "words" vncipher $out0,$out0,v27 vncipher $out1,$out1,v27 vncipher $out2,$out2,v27 vncipher $out3,$out3,v27 vncipher $out4,$out4,v27 vncipher $out5,$out5,v27 vncipher $out6,$out6,v27 vncipher $out7,$out7,v27 addi $key_,$sp,$FRAME+15 # rewind $key_ vncipher $out0,$out0,v28 vncipher $out1,$out1,v28 vncipher $out2,$out2,v28 vncipher $out3,$out3,v28 vncipher $out4,$out4,v28 vncipher $out5,$out5,v28 vncipher $out6,$out6,v28 vncipher $out7,$out7,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vncipher $out0,$out0,v29 vncipher $out1,$out1,v29 vncipher $out2,$out2,v29 vncipher $out3,$out3,v29 vncipher $out4,$out4,v29 vncipher $out5,$out5,v29 vncipher $out6,$out6,v29 vncipher $out7,$out7,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vncipher $out0,$out0,v30 vxor $ivec,$ivec,v31 # xor with last round key vncipher $out1,$out1,v30 vxor $in0,$in0,v31 vncipher $out2,$out2,v30 vxor $in1,$in1,v31 vncipher $out3,$out3,v30 vxor $in2,$in2,v31 vncipher $out4,$out4,v30 vxor $in3,$in3,v31 vncipher $out5,$out5,v30 vxor $in4,$in4,v31 vncipher $out6,$out6,v30 vxor $in5,$in5,v31 vncipher $out7,$out7,v30 vxor $in6,$in6,v31 vncipherlast $out0,$out0,$ivec vncipherlast $out1,$out1,$in0 lvx_u $in0,$x00,$inp # load next input block vncipherlast $out2,$out2,$in1 lvx_u $in1,$x10,$inp vncipherlast $out3,$out3,$in2 le?vperm $in0,$in0,$in0,$inpperm lvx_u $in2,$x20,$inp vncipherlast $out4,$out4,$in3 le?vperm $in1,$in1,$in1,$inpperm lvx_u $in3,$x30,$inp vncipherlast $out5,$out5,$in4 le?vperm $in2,$in2,$in2,$inpperm lvx_u $in4,$x40,$inp vncipherlast $out6,$out6,$in5 le?vperm $in3,$in3,$in3,$inpperm lvx_u $in5,$x50,$inp vncipherlast $out7,$out7,$in6 le?vperm $in4,$in4,$in4,$inpperm lvx_u $in6,$x60,$inp vmr $ivec,$in7 le?vperm $in5,$in5,$in5,$inpperm lvx_u $in7,$x70,$inp addi $inp,$inp,0x80 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out le?vperm $in6,$in6,$in6,$inpperm vxor $out0,$in0,$rndkey0 le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x10,$out le?vperm $in7,$in7,$in7,$inpperm vxor $out1,$in1,$rndkey0 le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x20,$out vxor $out2,$in2,$rndkey0 le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x30,$out vxor $out3,$in3,$rndkey0 le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x40,$out vxor $out4,$in4,$rndkey0 le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x50,$out vxor $out5,$in5,$rndkey0 le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x60,$out vxor $out6,$in6,$rndkey0 stvx_u $out7,$x70,$out addi $out,$out,0x80 vxor $out7,$in7,$rndkey0 mtctr $rounds beq Loop_cbc_dec8x # did $len-=128 borrow? addic. $len,$len,128 beq Lcbc_dec8x_done nop nop Loop_cbc_dec8x_tail: # up to 7 "words" tail... vncipher $out1,$out1,v24 vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vncipher $out4,$out4,v24 vncipher $out5,$out5,v24 vncipher $out6,$out6,v24 vncipher $out7,$out7,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vncipher $out1,$out1,v25 vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vncipher $out4,$out4,v25 vncipher $out5,$out5,v25 vncipher $out6,$out6,v25 vncipher $out7,$out7,v25 lvx v25,$x10,$key_ # round[4] bdnz Loop_cbc_dec8x_tail vncipher $out1,$out1,v24 vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vncipher $out4,$out4,v24 vncipher $out5,$out5,v24 vncipher $out6,$out6,v24 vncipher $out7,$out7,v24 vncipher $out1,$out1,v25 vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vncipher $out4,$out4,v25 vncipher $out5,$out5,v25 vncipher $out6,$out6,v25 vncipher $out7,$out7,v25 vncipher $out1,$out1,v26 vncipher $out2,$out2,v26 vncipher $out3,$out3,v26 vncipher $out4,$out4,v26 vncipher $out5,$out5,v26 vncipher $out6,$out6,v26 vncipher $out7,$out7,v26 vncipher $out1,$out1,v27 vncipher $out2,$out2,v27 vncipher $out3,$out3,v27 vncipher $out4,$out4,v27 vncipher $out5,$out5,v27 vncipher $out6,$out6,v27 vncipher $out7,$out7,v27 vncipher $out1,$out1,v28 vncipher $out2,$out2,v28 vncipher $out3,$out3,v28 vncipher $out4,$out4,v28 vncipher $out5,$out5,v28 vncipher $out6,$out6,v28 vncipher $out7,$out7,v28 vncipher $out1,$out1,v29 vncipher $out2,$out2,v29 vncipher $out3,$out3,v29 vncipher $out4,$out4,v29 vncipher $out5,$out5,v29 vncipher $out6,$out6,v29 vncipher $out7,$out7,v29 vncipher $out1,$out1,v30 vxor $ivec,$ivec,v31 # last round key vncipher $out2,$out2,v30 vxor $in1,$in1,v31 vncipher $out3,$out3,v30 vxor $in2,$in2,v31 vncipher $out4,$out4,v30 vxor $in3,$in3,v31 vncipher $out5,$out5,v30 vxor $in4,$in4,v31 vncipher $out6,$out6,v30 vxor $in5,$in5,v31 vncipher $out7,$out7,v30 vxor $in6,$in6,v31 cmplwi $len,32 # switch($len) blt Lcbc_dec8x_one nop beq Lcbc_dec8x_two cmplwi $len,64 blt Lcbc_dec8x_three nop beq Lcbc_dec8x_four cmplwi $len,96 blt Lcbc_dec8x_five nop beq Lcbc_dec8x_six Lcbc_dec8x_seven: vncipherlast $out1,$out1,$ivec vncipherlast $out2,$out2,$in1 vncipherlast $out3,$out3,$in2 vncipherlast $out4,$out4,$in3 vncipherlast $out5,$out5,$in4 vncipherlast $out6,$out6,$in5 vncipherlast $out7,$out7,$in6 vmr $ivec,$in7 le?vperm $out1,$out1,$out1,$inpperm le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x00,$out le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x10,$out le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x20,$out le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x30,$out le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x40,$out le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x50,$out stvx_u $out7,$x60,$out addi $out,$out,0x70 b Lcbc_dec8x_done .align 5 Lcbc_dec8x_six: vncipherlast $out2,$out2,$ivec vncipherlast $out3,$out3,$in2 vncipherlast $out4,$out4,$in3 vncipherlast $out5,$out5,$in4 vncipherlast $out6,$out6,$in5 vncipherlast $out7,$out7,$in6 vmr $ivec,$in7 le?vperm $out2,$out2,$out2,$inpperm le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x00,$out le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x10,$out le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x20,$out le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x30,$out le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x40,$out stvx_u $out7,$x50,$out addi $out,$out,0x60 b Lcbc_dec8x_done .align 5 Lcbc_dec8x_five: vncipherlast $out3,$out3,$ivec vncipherlast $out4,$out4,$in3 vncipherlast $out5,$out5,$in4 vncipherlast $out6,$out6,$in5 vncipherlast $out7,$out7,$in6 vmr $ivec,$in7 le?vperm $out3,$out3,$out3,$inpperm le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x00,$out le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x10,$out le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x20,$out le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x30,$out stvx_u $out7,$x40,$out addi $out,$out,0x50 b Lcbc_dec8x_done .align 5 Lcbc_dec8x_four: vncipherlast $out4,$out4,$ivec vncipherlast $out5,$out5,$in4 vncipherlast $out6,$out6,$in5 vncipherlast $out7,$out7,$in6 vmr $ivec,$in7 le?vperm $out4,$out4,$out4,$inpperm le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x00,$out le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x10,$out le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x20,$out stvx_u $out7,$x30,$out addi $out,$out,0x40 b Lcbc_dec8x_done .align 5 Lcbc_dec8x_three: vncipherlast $out5,$out5,$ivec vncipherlast $out6,$out6,$in5 vncipherlast $out7,$out7,$in6 vmr $ivec,$in7 le?vperm $out5,$out5,$out5,$inpperm le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x00,$out le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x10,$out stvx_u $out7,$x20,$out addi $out,$out,0x30 b Lcbc_dec8x_done .align 5 Lcbc_dec8x_two: vncipherlast $out6,$out6,$ivec vncipherlast $out7,$out7,$in6 vmr $ivec,$in7 le?vperm $out6,$out6,$out6,$inpperm le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x00,$out stvx_u $out7,$x10,$out addi $out,$out,0x20 b Lcbc_dec8x_done .align 5 Lcbc_dec8x_one: vncipherlast $out7,$out7,$ivec vmr $ivec,$in7 le?vperm $out7,$out7,$out7,$inpperm stvx_u $out7,0,$out addi $out,$out,0x10 Lcbc_dec8x_done: le?vperm $ivec,$ivec,$ivec,$inpperm stvx_u $ivec,0,$ivp # write [unaligned] iv li r10,`$FRAME+15` li r11,`$FRAME+31` stvx $inpperm,r10,$sp # wipe copies of round keys addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 stvx $inpperm,r10,$sp addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 stvx $inpperm,r10,$sp addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 stvx $inpperm,r10,$sp addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 mtspr 256,$vrsave lvx v20,r10,$sp # ABI says so addi r10,r10,32 lvx v21,r11,$sp addi r11,r11,32 lvx v22,r10,$sp addi r10,r10,32 lvx v23,r11,$sp addi r11,r11,32 lvx v24,r10,$sp addi r10,r10,32 lvx v25,r11,$sp addi r11,r11,32 lvx v26,r10,$sp addi r10,r10,32 lvx v27,r11,$sp addi r11,r11,32 lvx v28,r10,$sp addi r10,r10,32 lvx v29,r11,$sp addi r11,r11,32 lvx v30,r10,$sp lvx v31,r11,$sp $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` blr .long 0 .byte 0,12,0x14,0,0x80,6,6,0 .long 0 .size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt ___ }} }}} ######################################################################### {{{ # CTR procedure[s] # ####################### WARNING: Here be dragons! ####################### # # This code is written as 'ctr32', based on a 32-bit counter used # upstream. The kernel does *not* use a 32-bit counter. The kernel uses # a 128-bit counter. # # This leads to subtle changes from the upstream code: the counter # is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in # both the bulk (8 blocks at a time) path, and in the individual block # path. Be aware of this when doing updates. # # See: # 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug") # 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword") # https://github.com/openssl/openssl/pull/8942 # ######################################################################### my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10)); my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3)); my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)= map("v$_",(4..11)); my $dat=$tmp; $code.=<<___; .globl .${prefix}_ctr32_encrypt_blocks ${UCMP}i $len,1 bltlr- lis r0,0xfff0 mfspr $vrsave,256 mtspr 256,r0 li $idx,15 vxor $rndkey0,$rndkey0,$rndkey0 le?vspltisb $tmp,0x0f lvx $ivec,0,$ivp # load [unaligned] iv lvsl $inpperm,0,$ivp lvx $inptail,$idx,$ivp vspltisb $one,1 le?vxor $inpperm,$inpperm,$tmp vperm $ivec,$ivec,$inptail,$inpperm vsldoi $one,$rndkey0,$one,1 neg r11,$inp ?lvsl $keyperm,0,$key # prepare for unaligned key lwz $rounds,240($key) lvsr $inpperm,0,r11 # prepare for unaligned load lvx $inptail,0,$inp addi $inp,$inp,15 # 15 is not typo le?vxor $inpperm,$inpperm,$tmp srwi $rounds,$rounds,1 li $idx,16 subi $rounds,$rounds,1 ${UCMP}i $len,8 bge _aesp8_ctr32_encrypt8x ?lvsr $outperm,0,$out # prepare for unaligned store vspltisb $outmask,-1 lvx $outhead,0,$out ?vperm $outmask,$rndkey0,$outmask,$outperm le?vxor $outperm,$outperm,$tmp lvx $rndkey0,0,$key mtctr $rounds lvx $rndkey1,$idx,$key addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $inout,$ivec,$rndkey0 lvx $rndkey0,$idx,$key addi $idx,$idx,16 b Loop_ctr32_enc .align 5 Loop_ctr32_enc: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipher $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key addi $idx,$idx,16 bdnz Loop_ctr32_enc vadduqm $ivec,$ivec,$one # Kernel change for 128-bit vmr $dat,$inptail lvx $inptail,0,$inp addi $inp,$inp,16 subic. $len,$len,1 # blocks-- ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key vperm $dat,$dat,$inptail,$inpperm li $idx,16 ?vperm $rndkey1,$rndkey0,$rndkey1,$keyperm lvx $rndkey0,0,$key vxor $dat,$dat,$rndkey1 # last round key vcipherlast $inout,$inout,$dat lvx $rndkey1,$idx,$key addi $idx,$idx,16 vperm $inout,$inout,$inout,$outperm vsel $dat,$outhead,$inout,$outmask mtctr $rounds ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vmr $outhead,$inout vxor $inout,$ivec,$rndkey0 lvx $rndkey0,$idx,$key addi $idx,$idx,16 stvx $dat,0,$out addi $out,$out,16 bne Loop_ctr32_enc addi $out,$out,-1 lvx $inout,0,$out # redundant in aligned case vsel $inout,$outhead,$inout,$outmask stvx $inout,0,$out mtspr 256,$vrsave blr .long 0 .byte 0,12,0x14,0,0,0,6,0 .long 0 ___ ######################################################################### {{ # Optimized CTR procedure # my $key_="r11"; my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31)); my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14)); my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22)); my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys # v26-v31 last 6 round keys my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment my ($two,$three,$four)=($outhead,$outperm,$outmask); $code.=<<___; .align 5 _aesp8_ctr32_encrypt8x: $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) li r10,`$FRAME+8*16+15` li r11,`$FRAME+8*16+31` stvx v20,r10,$sp # ABI says so addi r10,r10,32 stvx v21,r11,$sp addi r11,r11,32 stvx v22,r10,$sp addi r10,r10,32 stvx v23,r11,$sp addi r11,r11,32 stvx v24,r10,$sp addi r10,r10,32 stvx v25,r11,$sp addi r11,r11,32 stvx v26,r10,$sp addi r10,r10,32 stvx v27,r11,$sp addi r11,r11,32 stvx v28,r10,$sp addi r10,r10,32 stvx v29,r11,$sp addi r11,r11,32 stvx v30,r10,$sp stvx v31,r11,$sp li r0,-1 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave li $x10,0x10 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) li $x20,0x20 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) li $x30,0x30 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) li $x40,0x40 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) li $x50,0x50 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) li $x60,0x60 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) li $x70,0x70 mtspr 256,r0 subi $rounds,$rounds,3 # -4 in total lvx $rndkey0,$x00,$key # load key schedule lvx v30,$x10,$key addi $key,$key,0x20 lvx v31,$x00,$key ?vperm $rndkey0,$rndkey0,v30,$keyperm addi $key_,$sp,$FRAME+15 mtctr $rounds Load_ctr32_enc_key: ?vperm v24,v30,v31,$keyperm lvx v30,$x10,$key addi $key,$key,0x20 stvx v24,$x00,$key_ # off-load round[1] ?vperm v25,v31,v30,$keyperm lvx v31,$x00,$key stvx v25,$x10,$key_ # off-load round[2] addi $key_,$key_,0x20 bdnz Load_ctr32_enc_key lvx v26,$x10,$key ?vperm v24,v30,v31,$keyperm lvx v27,$x20,$key stvx v24,$x00,$key_ # off-load round[3] ?vperm v25,v31,v26,$keyperm lvx v28,$x30,$key stvx v25,$x10,$key_ # off-load round[4] addi $key_,$sp,$FRAME+15 # rewind $key_ ?vperm v26,v26,v27,$keyperm lvx v29,$x40,$key ?vperm v27,v27,v28,$keyperm lvx v30,$x50,$key ?vperm v28,v28,v29,$keyperm lvx v31,$x60,$key ?vperm v29,v29,v30,$keyperm lvx $out0,$x70,$key # borrow $out0 ?vperm v30,v30,v31,$keyperm lvx v24,$x00,$key_ # pre-load round[1] ?vperm v31,v31,$out0,$keyperm lvx v25,$x10,$key_ # pre-load round[2] vadduqm $two,$one,$one subi $inp,$inp,15 # undo "caller" $SHL $len,$len,4 vadduqm $out1,$ivec,$one # counter values ... vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit) vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] le?li $idx,8 vadduqm $out3,$out1,$two vxor $out1,$out1,$rndkey0 le?lvsl $inpperm,0,$idx vadduqm $out4,$out2,$two vxor $out2,$out2,$rndkey0 le?vspltisb $tmp,0x0f vadduqm $out5,$out3,$two vxor $out3,$out3,$rndkey0 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u vadduqm $out6,$out4,$two vxor $out4,$out4,$rndkey0 vadduqm $out7,$out5,$two vxor $out5,$out5,$rndkey0 vadduqm $ivec,$out6,$two # next counter value vxor $out6,$out6,$rndkey0 vxor $out7,$out7,$rndkey0 mtctr $rounds b Loop_ctr32_enc8x .align 5 Loop_ctr32_enc8x: vcipher $out0,$out0,v24 vcipher $out1,$out1,v24 vcipher $out2,$out2,v24 vcipher $out3,$out3,v24 vcipher $out4,$out4,v24 vcipher $out5,$out5,v24 vcipher $out6,$out6,v24 vcipher $out7,$out7,v24 Loop_ctr32_enc8x_middle: lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vcipher $out0,$out0,v25 vcipher $out1,$out1,v25 vcipher $out2,$out2,v25 vcipher $out3,$out3,v25 vcipher $out4,$out4,v25 vcipher $out5,$out5,v25 vcipher $out6,$out6,v25 vcipher $out7,$out7,v25 lvx v25,$x10,$key_ # round[4] bdnz Loop_ctr32_enc8x subic r11,$len,256 # $len-256, borrow $key_ vcipher $out0,$out0,v24 vcipher $out1,$out1,v24 vcipher $out2,$out2,v24 vcipher $out3,$out3,v24 vcipher $out4,$out4,v24 vcipher $out5,$out5,v24 vcipher $out6,$out6,v24 vcipher $out7,$out7,v24 subfe r0,r0,r0 # borrow?-1:0 vcipher $out0,$out0,v25 vcipher $out1,$out1,v25 vcipher $out2,$out2,v25 vcipher $out3,$out3,v25 vcipher $out4,$out4,v25 vcipher $out5,$out5,v25 vcipher $out6,$out6,v25 vcipher $out7,$out7,v25 and r0,r0,r11 addi $key_,$sp,$FRAME+15 # rewind $key_ vcipher $out0,$out0,v26 vcipher $out1,$out1,v26 vcipher $out2,$out2,v26 vcipher $out3,$out3,v26 vcipher $out4,$out4,v26 vcipher $out5,$out5,v26 vcipher $out6,$out6,v26 vcipher $out7,$out7,v26 lvx v24,$x00,$key_ # re-pre-load round[1] subic $len,$len,129 # $len-=129 vcipher $out0,$out0,v27 addi $len,$len,1 # $len-=128 really vcipher $out1,$out1,v27 vcipher $out2,$out2,v27 vcipher $out3,$out3,v27 vcipher $out4,$out4,v27 vcipher $out5,$out5,v27 vcipher $out6,$out6,v27 vcipher $out7,$out7,v27 lvx v25,$x10,$key_ # re-pre-load round[2] vcipher $out0,$out0,v28 lvx_u $in0,$x00,$inp # load input vcipher $out1,$out1,v28 lvx_u $in1,$x10,$inp vcipher $out2,$out2,v28 lvx_u $in2,$x20,$inp vcipher $out3,$out3,v28 lvx_u $in3,$x30,$inp vcipher $out4,$out4,v28 lvx_u $in4,$x40,$inp vcipher $out5,$out5,v28 lvx_u $in5,$x50,$inp vcipher $out6,$out6,v28 lvx_u $in6,$x60,$inp vcipher $out7,$out7,v28 lvx_u $in7,$x70,$inp addi $inp,$inp,0x80 vcipher $out0,$out0,v29 le?vperm $in0,$in0,$in0,$inpperm vcipher $out1,$out1,v29 le?vperm $in1,$in1,$in1,$inpperm vcipher $out2,$out2,v29 le?vperm $in2,$in2,$in2,$inpperm vcipher $out3,$out3,v29 le?vperm $in3,$in3,$in3,$inpperm vcipher $out4,$out4,v29 le?vperm $in4,$in4,$in4,$inpperm vcipher $out5,$out5,v29 le?vperm $in5,$in5,$in5,$inpperm vcipher $out6,$out6,v29 le?vperm $in6,$in6,$in6,$inpperm vcipher $out7,$out7,v29 le?vperm $in7,$in7,$in7,$inpperm add $inp,$inp,r0 # $inp is adjusted in such # way that at exit from the # loop inX-in7 are loaded # with last "words" subfe. r0,r0,r0 # borrow?-1:0 vcipher $out0,$out0,v30 vxor $in0,$in0,v31 # xor with last round key vcipher $out1,$out1,v30 vxor $in1,$in1,v31 vcipher $out2,$out2,v30 vxor $in2,$in2,v31 vcipher $out3,$out3,v30 vxor $in3,$in3,v31 vcipher $out4,$out4,v30 vxor $in4,$in4,v31 vcipher $out5,$out5,v30 vxor $in5,$in5,v31 vcipher $out6,$out6,v30 vxor $in6,$in6,v31 vcipher $out7,$out7,v30 vxor $in7,$in7,v31 bne Lctr32_enc8x_break # did $len-129 borrow? vcipherlast $in0,$out0,$in0 vcipherlast $in1,$out1,$in1 vadduqm $out1,$ivec,$one # counter values ... vcipherlast $in2,$out2,$in2 vadduqm $out2,$ivec,$two vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] vcipherlast $in3,$out3,$in3 vadduqm $out3,$out1,$two vxor $out1,$out1,$rndkey0 vcipherlast $in4,$out4,$in4 vadduqm $out4,$out2,$two vxor $out2,$out2,$rndkey0 vcipherlast $in5,$out5,$in5 vadduqm $out5,$out3,$two vxor $out3,$out3,$rndkey0 vcipherlast $in6,$out6,$in6 vadduqm $out6,$out4,$two vxor $out4,$out4,$rndkey0 vcipherlast $in7,$out7,$in7 vadduqm $out7,$out5,$two vxor $out5,$out5,$rndkey0 le?vperm $in0,$in0,$in0,$inpperm vadduqm $ivec,$out6,$two # next counter value vxor $out6,$out6,$rndkey0 le?vperm $in1,$in1,$in1,$inpperm vxor $out7,$out7,$rndkey0 mtctr $rounds vcipher $out0,$out0,v24 stvx_u $in0,$x00,$out le?vperm $in2,$in2,$in2,$inpperm vcipher $out1,$out1,v24 stvx_u $in1,$x10,$out le?vperm $in3,$in3,$in3,$inpperm vcipher $out2,$out2,v24 stvx_u $in2,$x20,$out le?vperm $in4,$in4,$in4,$inpperm vcipher $out3,$out3,v24 stvx_u $in3,$x30,$out le?vperm $in5,$in5,$in5,$inpperm vcipher $out4,$out4,v24 stvx_u $in4,$x40,$out le?vperm $in6,$in6,$in6,$inpperm vcipher $out5,$out5,v24 stvx_u $in5,$x50,$out le?vperm $in7,$in7,$in7,$inpperm vcipher $out6,$out6,v24 stvx_u $in6,$x60,$out vcipher $out7,$out7,v24 stvx_u $in7,$x70,$out addi $out,$out,0x80 b Loop_ctr32_enc8x_middle .align 5 Lctr32_enc8x_break: cmpwi $len,-0x60 blt Lctr32_enc8x_one nop beq Lctr32_enc8x_two cmpwi $len,-0x40 blt Lctr32_enc8x_three nop beq Lctr32_enc8x_four cmpwi $len,-0x20 blt Lctr32_enc8x_five nop beq Lctr32_enc8x_six cmpwi $len,0x00 blt Lctr32_enc8x_seven Lctr32_enc8x_eight: vcipherlast $out0,$out0,$in0 vcipherlast $out1,$out1,$in1 vcipherlast $out2,$out2,$in2 vcipherlast $out3,$out3,$in3 vcipherlast $out4,$out4,$in4 vcipherlast $out5,$out5,$in5 vcipherlast $out6,$out6,$in6 vcipherlast $out7,$out7,$in7 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x20,$out le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x30,$out le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x40,$out le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x50,$out le?vperm $out7,$out7,$out7,$inpperm stvx_u $out6,$x60,$out stvx_u $out7,$x70,$out addi $out,$out,0x80 b Lctr32_enc8x_done .align 5 Lctr32_enc8x_seven: vcipherlast $out0,$out0,$in1 vcipherlast $out1,$out1,$in2 vcipherlast $out2,$out2,$in3 vcipherlast $out3,$out3,$in4 vcipherlast $out4,$out4,$in5 vcipherlast $out5,$out5,$in6 vcipherlast $out6,$out6,$in7 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x20,$out le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x30,$out le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x40,$out le?vperm $out6,$out6,$out6,$inpperm stvx_u $out5,$x50,$out stvx_u $out6,$x60,$out addi $out,$out,0x70 b Lctr32_enc8x_done .align 5 Lctr32_enc8x_six: vcipherlast $out0,$out0,$in2 vcipherlast $out1,$out1,$in3 vcipherlast $out2,$out2,$in4 vcipherlast $out3,$out3,$in5 vcipherlast $out4,$out4,$in6 vcipherlast $out5,$out5,$in7 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x20,$out le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x30,$out le?vperm $out5,$out5,$out5,$inpperm stvx_u $out4,$x40,$out stvx_u $out5,$x50,$out addi $out,$out,0x60 b Lctr32_enc8x_done .align 5 Lctr32_enc8x_five: vcipherlast $out0,$out0,$in3 vcipherlast $out1,$out1,$in4 vcipherlast $out2,$out2,$in5 vcipherlast $out3,$out3,$in6 vcipherlast $out4,$out4,$in7 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x20,$out le?vperm $out4,$out4,$out4,$inpperm stvx_u $out3,$x30,$out stvx_u $out4,$x40,$out addi $out,$out,0x50 b Lctr32_enc8x_done .align 5 Lctr32_enc8x_four: vcipherlast $out0,$out0,$in4 vcipherlast $out1,$out1,$in5 vcipherlast $out2,$out2,$in6 vcipherlast $out3,$out3,$in7 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$inpperm stvx_u $out2,$x20,$out stvx_u $out3,$x30,$out addi $out,$out,0x40 b Lctr32_enc8x_done .align 5 Lctr32_enc8x_three: vcipherlast $out0,$out0,$in5 vcipherlast $out1,$out1,$in6 vcipherlast $out2,$out2,$in7 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out le?vperm $out2,$out2,$out2,$inpperm stvx_u $out1,$x10,$out stvx_u $out2,$x20,$out addi $out,$out,0x30 b Lctr32_enc8x_done .align 5 Lctr32_enc8x_two: vcipherlast $out0,$out0,$in6 vcipherlast $out1,$out1,$in7 le?vperm $out0,$out0,$out0,$inpperm le?vperm $out1,$out1,$out1,$inpperm stvx_u $out0,$x00,$out stvx_u $out1,$x10,$out addi $out,$out,0x20 b Lctr32_enc8x_done .align 5 Lctr32_enc8x_one: vcipherlast $out0,$out0,$in7 le?vperm $out0,$out0,$out0,$inpperm stvx_u $out0,0,$out addi $out,$out,0x10 Lctr32_enc8x_done: li r10,`$FRAME+15` li r11,`$FRAME+31` stvx $inpperm,r10,$sp # wipe copies of round keys addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 stvx $inpperm,r10,$sp addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 stvx $inpperm,r10,$sp addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 stvx $inpperm,r10,$sp addi r10,r10,32 stvx $inpperm,r11,$sp addi r11,r11,32 mtspr 256,$vrsave lvx v20,r10,$sp # ABI says so addi r10,r10,32 lvx v21,r11,$sp addi r11,r11,32 lvx v22,r10,$sp addi r10,r10,32 lvx v23,r11,$sp addi r11,r11,32 lvx v24,r10,$sp addi r10,r10,32 lvx v25,r11,$sp addi r11,r11,32 lvx v26,r10,$sp addi r10,r10,32 lvx v27,r11,$sp addi r11,r11,32 lvx v28,r10,$sp addi r10,r10,32 lvx v29,r11,$sp addi r11,r11,32 lvx v30,r10,$sp lvx v31,r11,$sp $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` blr .long 0 .byte 0,12,0x14,0,0x80,6,6,0 .long 0 .size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks ___ }} }}} ######################################################################### {{{ # XTS procedures # # int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, # # const AES_KEY *key1, const AES_KEY *key2, # # [const] unsigned char iv[16]); # # If $key2 is NULL, then a "tweak chaining" mode is engaged, in which # # input tweak value is assumed to be encrypted already, and last tweak # # value, one suitable for consecutive call on same chunk of data, is # # written back to original buffer. In addition, in "tweak chaining" # # mode only complete input blocks are processed. # my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10)); my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2)); my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7)); my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12)); my $taillen = $key2; ($inp,$idx) = ($idx,$inp); # reassign $code.=<<___; .globl .${prefix}_xts_encrypt mr $inp,r3 # reassign li r3,-1 ${UCMP}i $len,16 bltlr- lis r0,0xfff0 mfspr r12,256 # save vrsave li r11,0 mtspr 256,r0 vspltisb $seven,0x07 # 0x070707..07 le?lvsl $leperm,r11,r11 le?vspltisb $tmp,0x0f le?vxor $leperm,$leperm,$seven li $idx,15 lvx $tweak,0,$ivp # load [unaligned] iv lvsl $inpperm,0,$ivp lvx $inptail,$idx,$ivp le?vxor $inpperm,$inpperm,$tmp vperm $tweak,$tweak,$inptail,$inpperm neg r11,$inp lvsr $inpperm,0,r11 # prepare for unaligned load lvx $inout,0,$inp addi $inp,$inp,15 # 15 is not typo le?vxor $inpperm,$inpperm,$tmp ${UCMP}i $key2,0 # key2==NULL? beq Lxts_enc_no_key2 ?lvsl $keyperm,0,$key2 # prepare for unaligned key lwz $rounds,240($key2) srwi $rounds,$rounds,1 subi $rounds,$rounds,1 li $idx,16 lvx $rndkey0,0,$key2 lvx $rndkey1,$idx,$key2 addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $tweak,$tweak,$rndkey0 lvx $rndkey0,$idx,$key2 addi $idx,$idx,16 mtctr $rounds Ltweak_xts_enc: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $tweak,$tweak,$rndkey1 lvx $rndkey1,$idx,$key2 addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipher $tweak,$tweak,$rndkey0 lvx $rndkey0,$idx,$key2 addi $idx,$idx,16 bdnz Ltweak_xts_enc ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $tweak,$tweak,$rndkey1 lvx $rndkey1,$idx,$key2 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipherlast $tweak,$tweak,$rndkey0 li $ivp,0 # don't chain the tweak b Lxts_enc Lxts_enc_no_key2: li $idx,-16 and $len,$len,$idx # in "tweak chaining" # mode only complete # blocks are processed Lxts_enc: lvx $inptail,0,$inp addi $inp,$inp,16 ?lvsl $keyperm,0,$key1 # prepare for unaligned key lwz $rounds,240($key1) srwi $rounds,$rounds,1 subi $rounds,$rounds,1 li $idx,16 vslb $eighty7,$seven,$seven # 0x808080..80 vor $eighty7,$eighty7,$seven # 0x878787..87 vspltisb $tmp,1 # 0x010101..01 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01 ${UCMP}i $len,96 bge _aesp8_xts_encrypt6x andi. $taillen,$len,15 subic r0,$len,32 subi $taillen,$taillen,16 subfe r0,r0,r0 and r0,r0,$taillen add $inp,$inp,r0 lvx $rndkey0,0,$key1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 vperm $inout,$inout,$inptail,$inpperm ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $inout,$inout,$tweak vxor $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 mtctr $rounds b Loop_xts_enc .align 5 Loop_xts_enc: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipher $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 bdnz Loop_xts_enc ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key1 li $idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $rndkey0,$rndkey0,$tweak vcipherlast $output,$inout,$rndkey0 le?vperm $tmp,$output,$output,$leperm be?nop le?stvx_u $tmp,0,$out be?stvx_u $output,0,$out addi $out,$out,16 subic. $len,$len,16 beq Lxts_enc_done vmr $inout,$inptail lvx $inptail,0,$inp addi $inp,$inp,16 lvx $rndkey0,0,$key1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 subic r0,$len,32 subfe r0,r0,r0 and r0,r0,$taillen add $inp,$inp,r0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vand $tmp,$tmp,$eighty7 vxor $tweak,$tweak,$tmp vperm $inout,$inout,$inptail,$inpperm ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $inout,$inout,$tweak vxor $output,$output,$rndkey0 # just in case $len<16 vxor $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 mtctr $rounds ${UCMP}i $len,16 bge Loop_xts_enc vxor $output,$output,$tweak lvsr $inpperm,0,$len # $inpperm is no longer needed vxor $inptail,$inptail,$inptail # $inptail is no longer needed vspltisb $tmp,-1 vperm $inptail,$inptail,$tmp,$inpperm vsel $inout,$inout,$output,$inptail subi r11,$out,17 subi $out,$out,16 mtctr $len li $len,16 Loop_xts_enc_steal: lbzu r0,1(r11) stb r0,16(r11) bdnz Loop_xts_enc_steal mtctr $rounds b Loop_xts_enc # one more time... Lxts_enc_done: ${UCMP}i $ivp,0 beq Lxts_enc_ret vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vand $tmp,$tmp,$eighty7 vxor $tweak,$tweak,$tmp le?vperm $tweak,$tweak,$tweak,$leperm stvx_u $tweak,0,$ivp Lxts_enc_ret: mtspr 256,r12 # restore vrsave li r3,0 blr .long 0 .byte 0,12,0x04,0,0x80,6,6,0 .long 0 .size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt .globl .${prefix}_xts_decrypt mr $inp,r3 # reassign li r3,-1 ${UCMP}i $len,16 bltlr- lis r0,0xfff8 mfspr r12,256 # save vrsave li r11,0 mtspr 256,r0 andi. r0,$len,15 neg r0,r0 andi. r0,r0,16 sub $len,$len,r0 vspltisb $seven,0x07 # 0x070707..07 le?lvsl $leperm,r11,r11 le?vspltisb $tmp,0x0f le?vxor $leperm,$leperm,$seven li $idx,15 lvx $tweak,0,$ivp # load [unaligned] iv lvsl $inpperm,0,$ivp lvx $inptail,$idx,$ivp le?vxor $inpperm,$inpperm,$tmp vperm $tweak,$tweak,$inptail,$inpperm neg r11,$inp lvsr $inpperm,0,r11 # prepare for unaligned load lvx $inout,0,$inp addi $inp,$inp,15 # 15 is not typo le?vxor $inpperm,$inpperm,$tmp ${UCMP}i $key2,0 # key2==NULL? beq Lxts_dec_no_key2 ?lvsl $keyperm,0,$key2 # prepare for unaligned key lwz $rounds,240($key2) srwi $rounds,$rounds,1 subi $rounds,$rounds,1 li $idx,16 lvx $rndkey0,0,$key2 lvx $rndkey1,$idx,$key2 addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $tweak,$tweak,$rndkey0 lvx $rndkey0,$idx,$key2 addi $idx,$idx,16 mtctr $rounds Ltweak_xts_dec: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $tweak,$tweak,$rndkey1 lvx $rndkey1,$idx,$key2 addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipher $tweak,$tweak,$rndkey0 lvx $rndkey0,$idx,$key2 addi $idx,$idx,16 bdnz Ltweak_xts_dec ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vcipher $tweak,$tweak,$rndkey1 lvx $rndkey1,$idx,$key2 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vcipherlast $tweak,$tweak,$rndkey0 li $ivp,0 # don't chain the tweak b Lxts_dec Lxts_dec_no_key2: neg $idx,$len andi. $idx,$idx,15 add $len,$len,$idx # in "tweak chaining" # mode only complete # blocks are processed Lxts_dec: lvx $inptail,0,$inp addi $inp,$inp,16 ?lvsl $keyperm,0,$key1 # prepare for unaligned key lwz $rounds,240($key1) srwi $rounds,$rounds,1 subi $rounds,$rounds,1 li $idx,16 vslb $eighty7,$seven,$seven # 0x808080..80 vor $eighty7,$eighty7,$seven # 0x878787..87 vspltisb $tmp,1 # 0x010101..01 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01 ${UCMP}i $len,96 bge _aesp8_xts_decrypt6x lvx $rndkey0,0,$key1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 vperm $inout,$inout,$inptail,$inpperm ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $inout,$inout,$tweak vxor $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 mtctr $rounds ${UCMP}i $len,16 blt Ltail_xts_dec be?b Loop_xts_dec .align 5 Loop_xts_dec: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vncipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vncipher $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 bdnz Loop_xts_dec ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vncipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key1 li $idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $rndkey0,$rndkey0,$tweak vncipherlast $output,$inout,$rndkey0 le?vperm $tmp,$output,$output,$leperm be?nop le?stvx_u $tmp,0,$out be?stvx_u $output,0,$out addi $out,$out,16 subic. $len,$len,16 beq Lxts_dec_done vmr $inout,$inptail lvx $inptail,0,$inp addi $inp,$inp,16 lvx $rndkey0,0,$key1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vand $tmp,$tmp,$eighty7 vxor $tweak,$tweak,$tmp vperm $inout,$inout,$inptail,$inpperm ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $inout,$inout,$tweak vxor $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 mtctr $rounds ${UCMP}i $len,16 bge Loop_xts_dec Ltail_xts_dec: vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak1,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vand $tmp,$tmp,$eighty7 vxor $tweak1,$tweak1,$tmp subi $inp,$inp,16 add $inp,$inp,$len vxor $inout,$inout,$tweak # :-( vxor $inout,$inout,$tweak1 # :-) Loop_xts_dec_short: ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vncipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vncipher $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 bdnz Loop_xts_dec_short ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm vncipher $inout,$inout,$rndkey1 lvx $rndkey1,$idx,$key1 li $idx,16 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm vxor $rndkey0,$rndkey0,$tweak1 vncipherlast $output,$inout,$rndkey0 le?vperm $tmp,$output,$output,$leperm be?nop le?stvx_u $tmp,0,$out be?stvx_u $output,0,$out vmr $inout,$inptail lvx $inptail,0,$inp #addi $inp,$inp,16 lvx $rndkey0,0,$key1 lvx $rndkey1,$idx,$key1 addi $idx,$idx,16 vperm $inout,$inout,$inptail,$inpperm ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm lvsr $inpperm,0,$len # $inpperm is no longer needed vxor $inptail,$inptail,$inptail # $inptail is no longer needed vspltisb $tmp,-1 vperm $inptail,$inptail,$tmp,$inpperm vsel $inout,$inout,$output,$inptail vxor $rndkey0,$rndkey0,$tweak vxor $inout,$inout,$rndkey0 lvx $rndkey0,$idx,$key1 addi $idx,$idx,16 subi r11,$out,1 mtctr $len li $len,16 Loop_xts_dec_steal: lbzu r0,1(r11) stb r0,16(r11) bdnz Loop_xts_dec_steal mtctr $rounds b Loop_xts_dec # one more time... Lxts_dec_done: ${UCMP}i $ivp,0 beq Lxts_dec_ret vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vand $tmp,$tmp,$eighty7 vxor $tweak,$tweak,$tmp le?vperm $tweak,$tweak,$tweak,$leperm stvx_u $tweak,0,$ivp Lxts_dec_ret: mtspr 256,r12 # restore vrsave li r3,0 blr .long 0 .byte 0,12,0x04,0,0x80,6,6,0 .long 0 .size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt ___ ######################################################################### {{ # Optimized XTS procedures # my $key_=$key2; my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31)); $x00=0 if ($flavour =~ /osx/); my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5)); my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16)); my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22)); my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys # v26-v31 last 6 round keys my ($keyperm)=($out0); # aliases with "caller", redundant assignment my $taillen=$x70; $code.=<<___; .align 5 _aesp8_xts_encrypt6x: $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) mflr r11 li r7,`$FRAME+8*16+15` li r3,`$FRAME+8*16+31` $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp) stvx v20,r7,$sp # ABI says so addi r7,r7,32 stvx v21,r3,$sp addi r3,r3,32 stvx v22,r7,$sp addi r7,r7,32 stvx v23,r3,$sp addi r3,r3,32 stvx v24,r7,$sp addi r7,r7,32 stvx v25,r3,$sp addi r3,r3,32 stvx v26,r7,$sp addi r7,r7,32 stvx v27,r3,$sp addi r3,r3,32 stvx v28,r7,$sp addi r7,r7,32 stvx v29,r3,$sp addi r3,r3,32 stvx v30,r7,$sp stvx v31,r3,$sp li r0,-1 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave li $x10,0x10 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) li $x20,0x20 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) li $x30,0x30 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) li $x40,0x40 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) li $x50,0x50 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) li $x60,0x60 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) li $x70,0x70 mtspr 256,r0 subi $rounds,$rounds,3 # -4 in total lvx $rndkey0,$x00,$key1 # load key schedule lvx v30,$x10,$key1 addi $key1,$key1,0x20 lvx v31,$x00,$key1 ?vperm $rndkey0,$rndkey0,v30,$keyperm addi $key_,$sp,$FRAME+15 mtctr $rounds Load_xts_enc_key: ?vperm v24,v30,v31,$keyperm lvx v30,$x10,$key1 addi $key1,$key1,0x20 stvx v24,$x00,$key_ # off-load round[1] ?vperm v25,v31,v30,$keyperm lvx v31,$x00,$key1 stvx v25,$x10,$key_ # off-load round[2] addi $key_,$key_,0x20 bdnz Load_xts_enc_key lvx v26,$x10,$key1 ?vperm v24,v30,v31,$keyperm lvx v27,$x20,$key1 stvx v24,$x00,$key_ # off-load round[3] ?vperm v25,v31,v26,$keyperm lvx v28,$x30,$key1 stvx v25,$x10,$key_ # off-load round[4] addi $key_,$sp,$FRAME+15 # rewind $key_ ?vperm v26,v26,v27,$keyperm lvx v29,$x40,$key1 ?vperm v27,v27,v28,$keyperm lvx v30,$x50,$key1 ?vperm v28,v28,v29,$keyperm lvx v31,$x60,$key1 ?vperm v29,v29,v30,$keyperm lvx $twk5,$x70,$key1 # borrow $twk5 ?vperm v30,v30,v31,$keyperm lvx v24,$x00,$key_ # pre-load round[1] ?vperm v31,v31,$twk5,$keyperm lvx v25,$x10,$key_ # pre-load round[2] vperm $in0,$inout,$inptail,$inpperm subi $inp,$inp,31 # undo "caller" vxor $twk0,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vand $tmp,$tmp,$eighty7 vxor $out0,$in0,$twk0 vxor $tweak,$tweak,$tmp lvx_u $in1,$x10,$inp vxor $twk1,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in1,$in1,$in1,$leperm vand $tmp,$tmp,$eighty7 vxor $out1,$in1,$twk1 vxor $tweak,$tweak,$tmp lvx_u $in2,$x20,$inp andi. $taillen,$len,15 vxor $twk2,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in2,$in2,$in2,$leperm vand $tmp,$tmp,$eighty7 vxor $out2,$in2,$twk2 vxor $tweak,$tweak,$tmp lvx_u $in3,$x30,$inp sub $len,$len,$taillen vxor $twk3,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in3,$in3,$in3,$leperm vand $tmp,$tmp,$eighty7 vxor $out3,$in3,$twk3 vxor $tweak,$tweak,$tmp lvx_u $in4,$x40,$inp subi $len,$len,0x60 vxor $twk4,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in4,$in4,$in4,$leperm vand $tmp,$tmp,$eighty7 vxor $out4,$in4,$twk4 vxor $tweak,$tweak,$tmp lvx_u $in5,$x50,$inp addi $inp,$inp,0x60 vxor $twk5,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in5,$in5,$in5,$leperm vand $tmp,$tmp,$eighty7 vxor $out5,$in5,$twk5 vxor $tweak,$tweak,$tmp vxor v31,v31,$rndkey0 mtctr $rounds b Loop_xts_enc6x .align 5 Loop_xts_enc6x: vcipher $out0,$out0,v24 vcipher $out1,$out1,v24 vcipher $out2,$out2,v24 vcipher $out3,$out3,v24 vcipher $out4,$out4,v24 vcipher $out5,$out5,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vcipher $out0,$out0,v25 vcipher $out1,$out1,v25 vcipher $out2,$out2,v25 vcipher $out3,$out3,v25 vcipher $out4,$out4,v25 vcipher $out5,$out5,v25 lvx v25,$x10,$key_ # round[4] bdnz Loop_xts_enc6x subic $len,$len,96 # $len-=96 vxor $in0,$twk0,v31 # xor with last round key vcipher $out0,$out0,v24 vcipher $out1,$out1,v24 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk0,$tweak,$rndkey0 vaddubm $tweak,$tweak,$tweak vcipher $out2,$out2,v24 vcipher $out3,$out3,v24 vsldoi $tmp,$tmp,$tmp,15 vcipher $out4,$out4,v24 vcipher $out5,$out5,v24 subfe. r0,r0,r0 # borrow?-1:0 vand $tmp,$tmp,$eighty7 vcipher $out0,$out0,v25 vcipher $out1,$out1,v25 vxor $tweak,$tweak,$tmp vcipher $out2,$out2,v25 vcipher $out3,$out3,v25 vxor $in1,$twk1,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk1,$tweak,$rndkey0 vcipher $out4,$out4,v25 vcipher $out5,$out5,v25 and r0,r0,$len vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vcipher $out0,$out0,v26 vcipher $out1,$out1,v26 vand $tmp,$tmp,$eighty7 vcipher $out2,$out2,v26 vcipher $out3,$out3,v26 vxor $tweak,$tweak,$tmp vcipher $out4,$out4,v26 vcipher $out5,$out5,v26 add $inp,$inp,r0 # $inp is adjusted in such # way that at exit from the # loop inX-in5 are loaded # with last "words" vxor $in2,$twk2,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk2,$tweak,$rndkey0 vaddubm $tweak,$tweak,$tweak vcipher $out0,$out0,v27 vcipher $out1,$out1,v27 vsldoi $tmp,$tmp,$tmp,15 vcipher $out2,$out2,v27 vcipher $out3,$out3,v27 vand $tmp,$tmp,$eighty7 vcipher $out4,$out4,v27 vcipher $out5,$out5,v27 addi $key_,$sp,$FRAME+15 # rewind $key_ vxor $tweak,$tweak,$tmp vcipher $out0,$out0,v28 vcipher $out1,$out1,v28 vxor $in3,$twk3,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk3,$tweak,$rndkey0 vcipher $out2,$out2,v28 vcipher $out3,$out3,v28 vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vcipher $out4,$out4,v28 vcipher $out5,$out5,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vand $tmp,$tmp,$eighty7 vcipher $out0,$out0,v29 vcipher $out1,$out1,v29 vxor $tweak,$tweak,$tmp vcipher $out2,$out2,v29 vcipher $out3,$out3,v29 vxor $in4,$twk4,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk4,$tweak,$rndkey0 vcipher $out4,$out4,v29 vcipher $out5,$out5,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vcipher $out0,$out0,v30 vcipher $out1,$out1,v30 vand $tmp,$tmp,$eighty7 vcipher $out2,$out2,v30 vcipher $out3,$out3,v30 vxor $tweak,$tweak,$tmp vcipher $out4,$out4,v30 vcipher $out5,$out5,v30 vxor $in5,$twk5,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk5,$tweak,$rndkey0 vcipherlast $out0,$out0,$in0 lvx_u $in0,$x00,$inp # load next input block vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vcipherlast $out1,$out1,$in1 lvx_u $in1,$x10,$inp vcipherlast $out2,$out2,$in2 le?vperm $in0,$in0,$in0,$leperm lvx_u $in2,$x20,$inp vand $tmp,$tmp,$eighty7 vcipherlast $out3,$out3,$in3 le?vperm $in1,$in1,$in1,$leperm lvx_u $in3,$x30,$inp vcipherlast $out4,$out4,$in4 le?vperm $in2,$in2,$in2,$leperm lvx_u $in4,$x40,$inp vxor $tweak,$tweak,$tmp vcipherlast $tmp,$out5,$in5 # last block might be needed # in stealing mode le?vperm $in3,$in3,$in3,$leperm lvx_u $in5,$x50,$inp addi $inp,$inp,0x60 le?vperm $in4,$in4,$in4,$leperm le?vperm $in5,$in5,$in5,$leperm le?vperm $out0,$out0,$out0,$leperm le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output vxor $out0,$in0,$twk0 le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out vxor $out1,$in1,$twk1 le?vperm $out3,$out3,$out3,$leperm stvx_u $out2,$x20,$out vxor $out2,$in2,$twk2 le?vperm $out4,$out4,$out4,$leperm stvx_u $out3,$x30,$out vxor $out3,$in3,$twk3 le?vperm $out5,$tmp,$tmp,$leperm stvx_u $out4,$x40,$out vxor $out4,$in4,$twk4 le?stvx_u $out5,$x50,$out be?stvx_u $tmp, $x50,$out vxor $out5,$in5,$twk5 addi $out,$out,0x60 mtctr $rounds beq Loop_xts_enc6x # did $len-=96 borrow? addic. $len,$len,0x60 beq Lxts_enc6x_zero cmpwi $len,0x20 blt Lxts_enc6x_one nop beq Lxts_enc6x_two cmpwi $len,0x40 blt Lxts_enc6x_three nop beq Lxts_enc6x_four Lxts_enc6x_five: vxor $out0,$in1,$twk0 vxor $out1,$in2,$twk1 vxor $out2,$in3,$twk2 vxor $out3,$in4,$twk3 vxor $out4,$in5,$twk4 bl _aesp8_xts_enc5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk5 # unused tweak le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$leperm stvx_u $out2,$x20,$out vxor $tmp,$out4,$twk5 # last block prep for stealing le?vperm $out4,$out4,$out4,$leperm stvx_u $out3,$x30,$out stvx_u $out4,$x40,$out addi $out,$out,0x50 bne Lxts_enc6x_steal b Lxts_enc6x_done .align 4 Lxts_enc6x_four: vxor $out0,$in2,$twk0 vxor $out1,$in3,$twk1 vxor $out2,$in4,$twk2 vxor $out3,$in5,$twk3 vxor $out4,$out4,$out4 bl _aesp8_xts_enc5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk4 # unused tweak le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out vxor $tmp,$out3,$twk4 # last block prep for stealing le?vperm $out3,$out3,$out3,$leperm stvx_u $out2,$x20,$out stvx_u $out3,$x30,$out addi $out,$out,0x40 bne Lxts_enc6x_steal b Lxts_enc6x_done .align 4 Lxts_enc6x_three: vxor $out0,$in3,$twk0 vxor $out1,$in4,$twk1 vxor $out2,$in5,$twk2 vxor $out3,$out3,$out3 vxor $out4,$out4,$out4 bl _aesp8_xts_enc5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk3 # unused tweak le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output vxor $tmp,$out2,$twk3 # last block prep for stealing le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out stvx_u $out2,$x20,$out addi $out,$out,0x30 bne Lxts_enc6x_steal b Lxts_enc6x_done .align 4 Lxts_enc6x_two: vxor $out0,$in4,$twk0 vxor $out1,$in5,$twk1 vxor $out2,$out2,$out2 vxor $out3,$out3,$out3 vxor $out4,$out4,$out4 bl _aesp8_xts_enc5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk2 # unused tweak vxor $tmp,$out1,$twk2 # last block prep for stealing le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output stvx_u $out1,$x10,$out addi $out,$out,0x20 bne Lxts_enc6x_steal b Lxts_enc6x_done .align 4 Lxts_enc6x_one: vxor $out0,$in5,$twk0 nop Loop_xts_enc1x: vcipher $out0,$out0,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vcipher $out0,$out0,v25 lvx v25,$x10,$key_ # round[4] bdnz Loop_xts_enc1x add $inp,$inp,$taillen cmpwi $taillen,0 vcipher $out0,$out0,v24 subi $inp,$inp,16 vcipher $out0,$out0,v25 lvsr $inpperm,0,$taillen vcipher $out0,$out0,v26 lvx_u $in0,0,$inp vcipher $out0,$out0,v27 addi $key_,$sp,$FRAME+15 # rewind $key_ vcipher $out0,$out0,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vcipher $out0,$out0,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vxor $twk0,$twk0,v31 le?vperm $in0,$in0,$in0,$leperm vcipher $out0,$out0,v30 vperm $in0,$in0,$in0,$inpperm vcipherlast $out0,$out0,$twk0 vmr $twk0,$twk1 # unused tweak vxor $tmp,$out0,$twk1 # last block prep for stealing le?vperm $out0,$out0,$out0,$leperm stvx_u $out0,$x00,$out # store output addi $out,$out,0x10 bne Lxts_enc6x_steal b Lxts_enc6x_done .align 4 Lxts_enc6x_zero: cmpwi $taillen,0 beq Lxts_enc6x_done add $inp,$inp,$taillen subi $inp,$inp,16 lvx_u $in0,0,$inp lvsr $inpperm,0,$taillen # $in5 is no more le?vperm $in0,$in0,$in0,$leperm vperm $in0,$in0,$in0,$inpperm vxor $tmp,$tmp,$twk0 Lxts_enc6x_steal: vxor $in0,$in0,$twk0 vxor $out0,$out0,$out0 vspltisb $out1,-1 vperm $out0,$out0,$out1,$inpperm vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember? subi r30,$out,17 subi $out,$out,16 mtctr $taillen Loop_xts_enc6x_steal: lbzu r0,1(r30) stb r0,16(r30) bdnz Loop_xts_enc6x_steal li $taillen,0 mtctr $rounds b Loop_xts_enc1x # one more time... .align 4 Lxts_enc6x_done: ${UCMP}i $ivp,0 beq Lxts_enc6x_ret vxor $tweak,$twk0,$rndkey0 le?vperm $tweak,$tweak,$tweak,$leperm stvx_u $tweak,0,$ivp Lxts_enc6x_ret: mtlr r11 li r10,`$FRAME+15` li r11,`$FRAME+31` stvx $seven,r10,$sp # wipe copies of round keys addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 stvx $seven,r10,$sp addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 stvx $seven,r10,$sp addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 stvx $seven,r10,$sp addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 mtspr 256,$vrsave lvx v20,r10,$sp # ABI says so addi r10,r10,32 lvx v21,r11,$sp addi r11,r11,32 lvx v22,r10,$sp addi r10,r10,32 lvx v23,r11,$sp addi r11,r11,32 lvx v24,r10,$sp addi r10,r10,32 lvx v25,r11,$sp addi r11,r11,32 lvx v26,r10,$sp addi r10,r10,32 lvx v27,r11,$sp addi r11,r11,32 lvx v28,r10,$sp addi r10,r10,32 lvx v29,r11,$sp addi r11,r11,32 lvx v30,r10,$sp lvx v31,r11,$sp $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` blr .long 0 .byte 0,12,0x04,1,0x80,6,6,0 .long 0 .align 5 _aesp8_xts_enc5x: vcipher $out0,$out0,v24 vcipher $out1,$out1,v24 vcipher $out2,$out2,v24 vcipher $out3,$out3,v24 vcipher $out4,$out4,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vcipher $out0,$out0,v25 vcipher $out1,$out1,v25 vcipher $out2,$out2,v25 vcipher $out3,$out3,v25 vcipher $out4,$out4,v25 lvx v25,$x10,$key_ # round[4] bdnz _aesp8_xts_enc5x add $inp,$inp,$taillen cmpwi $taillen,0 vcipher $out0,$out0,v24 vcipher $out1,$out1,v24 vcipher $out2,$out2,v24 vcipher $out3,$out3,v24 vcipher $out4,$out4,v24 subi $inp,$inp,16 vcipher $out0,$out0,v25 vcipher $out1,$out1,v25 vcipher $out2,$out2,v25 vcipher $out3,$out3,v25 vcipher $out4,$out4,v25 vxor $twk0,$twk0,v31 vcipher $out0,$out0,v26 lvsr $inpperm,r0,$taillen # $in5 is no more vcipher $out1,$out1,v26 vcipher $out2,$out2,v26 vcipher $out3,$out3,v26 vcipher $out4,$out4,v26 vxor $in1,$twk1,v31 vcipher $out0,$out0,v27 lvx_u $in0,0,$inp vcipher $out1,$out1,v27 vcipher $out2,$out2,v27 vcipher $out3,$out3,v27 vcipher $out4,$out4,v27 vxor $in2,$twk2,v31 addi $key_,$sp,$FRAME+15 # rewind $key_ vcipher $out0,$out0,v28 vcipher $out1,$out1,v28 vcipher $out2,$out2,v28 vcipher $out3,$out3,v28 vcipher $out4,$out4,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vxor $in3,$twk3,v31 vcipher $out0,$out0,v29 le?vperm $in0,$in0,$in0,$leperm vcipher $out1,$out1,v29 vcipher $out2,$out2,v29 vcipher $out3,$out3,v29 vcipher $out4,$out4,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vxor $in4,$twk4,v31 vcipher $out0,$out0,v30 vperm $in0,$in0,$in0,$inpperm vcipher $out1,$out1,v30 vcipher $out2,$out2,v30 vcipher $out3,$out3,v30 vcipher $out4,$out4,v30 vcipherlast $out0,$out0,$twk0 vcipherlast $out1,$out1,$in1 vcipherlast $out2,$out2,$in2 vcipherlast $out3,$out3,$in3 vcipherlast $out4,$out4,$in4 blr .long 0 .byte 0,12,0x14,0,0,0,0,0 .align 5 _aesp8_xts_decrypt6x: $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) mflr r11 li r7,`$FRAME+8*16+15` li r3,`$FRAME+8*16+31` $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp) stvx v20,r7,$sp # ABI says so addi r7,r7,32 stvx v21,r3,$sp addi r3,r3,32 stvx v22,r7,$sp addi r7,r7,32 stvx v23,r3,$sp addi r3,r3,32 stvx v24,r7,$sp addi r7,r7,32 stvx v25,r3,$sp addi r3,r3,32 stvx v26,r7,$sp addi r7,r7,32 stvx v27,r3,$sp addi r3,r3,32 stvx v28,r7,$sp addi r7,r7,32 stvx v29,r3,$sp addi r3,r3,32 stvx v30,r7,$sp stvx v31,r3,$sp li r0,-1 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave li $x10,0x10 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) li $x20,0x20 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) li $x30,0x30 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) li $x40,0x40 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) li $x50,0x50 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) li $x60,0x60 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) li $x70,0x70 mtspr 256,r0 subi $rounds,$rounds,3 # -4 in total lvx $rndkey0,$x00,$key1 # load key schedule lvx v30,$x10,$key1 addi $key1,$key1,0x20 lvx v31,$x00,$key1 ?vperm $rndkey0,$rndkey0,v30,$keyperm addi $key_,$sp,$FRAME+15 mtctr $rounds Load_xts_dec_key: ?vperm v24,v30,v31,$keyperm lvx v30,$x10,$key1 addi $key1,$key1,0x20 stvx v24,$x00,$key_ # off-load round[1] ?vperm v25,v31,v30,$keyperm lvx v31,$x00,$key1 stvx v25,$x10,$key_ # off-load round[2] addi $key_,$key_,0x20 bdnz Load_xts_dec_key lvx v26,$x10,$key1 ?vperm v24,v30,v31,$keyperm lvx v27,$x20,$key1 stvx v24,$x00,$key_ # off-load round[3] ?vperm v25,v31,v26,$keyperm lvx v28,$x30,$key1 stvx v25,$x10,$key_ # off-load round[4] addi $key_,$sp,$FRAME+15 # rewind $key_ ?vperm v26,v26,v27,$keyperm lvx v29,$x40,$key1 ?vperm v27,v27,v28,$keyperm lvx v30,$x50,$key1 ?vperm v28,v28,v29,$keyperm lvx v31,$x60,$key1 ?vperm v29,v29,v30,$keyperm lvx $twk5,$x70,$key1 # borrow $twk5 ?vperm v30,v30,v31,$keyperm lvx v24,$x00,$key_ # pre-load round[1] ?vperm v31,v31,$twk5,$keyperm lvx v25,$x10,$key_ # pre-load round[2] vperm $in0,$inout,$inptail,$inpperm subi $inp,$inp,31 # undo "caller" vxor $twk0,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vand $tmp,$tmp,$eighty7 vxor $out0,$in0,$twk0 vxor $tweak,$tweak,$tmp lvx_u $in1,$x10,$inp vxor $twk1,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in1,$in1,$in1,$leperm vand $tmp,$tmp,$eighty7 vxor $out1,$in1,$twk1 vxor $tweak,$tweak,$tmp lvx_u $in2,$x20,$inp andi. $taillen,$len,15 vxor $twk2,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in2,$in2,$in2,$leperm vand $tmp,$tmp,$eighty7 vxor $out2,$in2,$twk2 vxor $tweak,$tweak,$tmp lvx_u $in3,$x30,$inp sub $len,$len,$taillen vxor $twk3,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in3,$in3,$in3,$leperm vand $tmp,$tmp,$eighty7 vxor $out3,$in3,$twk3 vxor $tweak,$tweak,$tmp lvx_u $in4,$x40,$inp subi $len,$len,0x60 vxor $twk4,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in4,$in4,$in4,$leperm vand $tmp,$tmp,$eighty7 vxor $out4,$in4,$twk4 vxor $tweak,$tweak,$tmp lvx_u $in5,$x50,$inp addi $inp,$inp,0x60 vxor $twk5,$tweak,$rndkey0 vsrab $tmp,$tweak,$seven # next tweak value vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 le?vperm $in5,$in5,$in5,$leperm vand $tmp,$tmp,$eighty7 vxor $out5,$in5,$twk5 vxor $tweak,$tweak,$tmp vxor v31,v31,$rndkey0 mtctr $rounds b Loop_xts_dec6x .align 5 Loop_xts_dec6x: vncipher $out0,$out0,v24 vncipher $out1,$out1,v24 vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vncipher $out4,$out4,v24 vncipher $out5,$out5,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vncipher $out0,$out0,v25 vncipher $out1,$out1,v25 vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vncipher $out4,$out4,v25 vncipher $out5,$out5,v25 lvx v25,$x10,$key_ # round[4] bdnz Loop_xts_dec6x subic $len,$len,96 # $len-=96 vxor $in0,$twk0,v31 # xor with last round key vncipher $out0,$out0,v24 vncipher $out1,$out1,v24 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk0,$tweak,$rndkey0 vaddubm $tweak,$tweak,$tweak vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vsldoi $tmp,$tmp,$tmp,15 vncipher $out4,$out4,v24 vncipher $out5,$out5,v24 subfe. r0,r0,r0 # borrow?-1:0 vand $tmp,$tmp,$eighty7 vncipher $out0,$out0,v25 vncipher $out1,$out1,v25 vxor $tweak,$tweak,$tmp vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vxor $in1,$twk1,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk1,$tweak,$rndkey0 vncipher $out4,$out4,v25 vncipher $out5,$out5,v25 and r0,r0,$len vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vncipher $out0,$out0,v26 vncipher $out1,$out1,v26 vand $tmp,$tmp,$eighty7 vncipher $out2,$out2,v26 vncipher $out3,$out3,v26 vxor $tweak,$tweak,$tmp vncipher $out4,$out4,v26 vncipher $out5,$out5,v26 add $inp,$inp,r0 # $inp is adjusted in such # way that at exit from the # loop inX-in5 are loaded # with last "words" vxor $in2,$twk2,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk2,$tweak,$rndkey0 vaddubm $tweak,$tweak,$tweak vncipher $out0,$out0,v27 vncipher $out1,$out1,v27 vsldoi $tmp,$tmp,$tmp,15 vncipher $out2,$out2,v27 vncipher $out3,$out3,v27 vand $tmp,$tmp,$eighty7 vncipher $out4,$out4,v27 vncipher $out5,$out5,v27 addi $key_,$sp,$FRAME+15 # rewind $key_ vxor $tweak,$tweak,$tmp vncipher $out0,$out0,v28 vncipher $out1,$out1,v28 vxor $in3,$twk3,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk3,$tweak,$rndkey0 vncipher $out2,$out2,v28 vncipher $out3,$out3,v28 vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vncipher $out4,$out4,v28 vncipher $out5,$out5,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vand $tmp,$tmp,$eighty7 vncipher $out0,$out0,v29 vncipher $out1,$out1,v29 vxor $tweak,$tweak,$tmp vncipher $out2,$out2,v29 vncipher $out3,$out3,v29 vxor $in4,$twk4,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk4,$tweak,$rndkey0 vncipher $out4,$out4,v29 vncipher $out5,$out5,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vncipher $out0,$out0,v30 vncipher $out1,$out1,v30 vand $tmp,$tmp,$eighty7 vncipher $out2,$out2,v30 vncipher $out3,$out3,v30 vxor $tweak,$tweak,$tmp vncipher $out4,$out4,v30 vncipher $out5,$out5,v30 vxor $in5,$twk5,v31 vsrab $tmp,$tweak,$seven # next tweak value vxor $twk5,$tweak,$rndkey0 vncipherlast $out0,$out0,$in0 lvx_u $in0,$x00,$inp # load next input block vaddubm $tweak,$tweak,$tweak vsldoi $tmp,$tmp,$tmp,15 vncipherlast $out1,$out1,$in1 lvx_u $in1,$x10,$inp vncipherlast $out2,$out2,$in2 le?vperm $in0,$in0,$in0,$leperm lvx_u $in2,$x20,$inp vand $tmp,$tmp,$eighty7 vncipherlast $out3,$out3,$in3 le?vperm $in1,$in1,$in1,$leperm lvx_u $in3,$x30,$inp vncipherlast $out4,$out4,$in4 le?vperm $in2,$in2,$in2,$leperm lvx_u $in4,$x40,$inp vxor $tweak,$tweak,$tmp vncipherlast $out5,$out5,$in5 le?vperm $in3,$in3,$in3,$leperm lvx_u $in5,$x50,$inp addi $inp,$inp,0x60 le?vperm $in4,$in4,$in4,$leperm le?vperm $in5,$in5,$in5,$leperm le?vperm $out0,$out0,$out0,$leperm le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output vxor $out0,$in0,$twk0 le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out vxor $out1,$in1,$twk1 le?vperm $out3,$out3,$out3,$leperm stvx_u $out2,$x20,$out vxor $out2,$in2,$twk2 le?vperm $out4,$out4,$out4,$leperm stvx_u $out3,$x30,$out vxor $out3,$in3,$twk3 le?vperm $out5,$out5,$out5,$leperm stvx_u $out4,$x40,$out vxor $out4,$in4,$twk4 stvx_u $out5,$x50,$out vxor $out5,$in5,$twk5 addi $out,$out,0x60 mtctr $rounds beq Loop_xts_dec6x # did $len-=96 borrow? addic. $len,$len,0x60 beq Lxts_dec6x_zero cmpwi $len,0x20 blt Lxts_dec6x_one nop beq Lxts_dec6x_two cmpwi $len,0x40 blt Lxts_dec6x_three nop beq Lxts_dec6x_four Lxts_dec6x_five: vxor $out0,$in1,$twk0 vxor $out1,$in2,$twk1 vxor $out2,$in3,$twk2 vxor $out3,$in4,$twk3 vxor $out4,$in5,$twk4 bl _aesp8_xts_dec5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk5 # unused tweak vxor $twk1,$tweak,$rndkey0 le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output vxor $out0,$in0,$twk1 le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$leperm stvx_u $out2,$x20,$out le?vperm $out4,$out4,$out4,$leperm stvx_u $out3,$x30,$out stvx_u $out4,$x40,$out addi $out,$out,0x50 bne Lxts_dec6x_steal b Lxts_dec6x_done .align 4 Lxts_dec6x_four: vxor $out0,$in2,$twk0 vxor $out1,$in3,$twk1 vxor $out2,$in4,$twk2 vxor $out3,$in5,$twk3 vxor $out4,$out4,$out4 bl _aesp8_xts_dec5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk4 # unused tweak vmr $twk1,$twk5 le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output vxor $out0,$in0,$twk5 le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out le?vperm $out3,$out3,$out3,$leperm stvx_u $out2,$x20,$out stvx_u $out3,$x30,$out addi $out,$out,0x40 bne Lxts_dec6x_steal b Lxts_dec6x_done .align 4 Lxts_dec6x_three: vxor $out0,$in3,$twk0 vxor $out1,$in4,$twk1 vxor $out2,$in5,$twk2 vxor $out3,$out3,$out3 vxor $out4,$out4,$out4 bl _aesp8_xts_dec5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk3 # unused tweak vmr $twk1,$twk4 le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output vxor $out0,$in0,$twk4 le?vperm $out2,$out2,$out2,$leperm stvx_u $out1,$x10,$out stvx_u $out2,$x20,$out addi $out,$out,0x30 bne Lxts_dec6x_steal b Lxts_dec6x_done .align 4 Lxts_dec6x_two: vxor $out0,$in4,$twk0 vxor $out1,$in5,$twk1 vxor $out2,$out2,$out2 vxor $out3,$out3,$out3 vxor $out4,$out4,$out4 bl _aesp8_xts_dec5x le?vperm $out0,$out0,$out0,$leperm vmr $twk0,$twk2 # unused tweak vmr $twk1,$twk3 le?vperm $out1,$out1,$out1,$leperm stvx_u $out0,$x00,$out # store output vxor $out0,$in0,$twk3 stvx_u $out1,$x10,$out addi $out,$out,0x20 bne Lxts_dec6x_steal b Lxts_dec6x_done .align 4 Lxts_dec6x_one: vxor $out0,$in5,$twk0 nop Loop_xts_dec1x: vncipher $out0,$out0,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vncipher $out0,$out0,v25 lvx v25,$x10,$key_ # round[4] bdnz Loop_xts_dec1x subi r0,$taillen,1 vncipher $out0,$out0,v24 andi. r0,r0,16 cmpwi $taillen,0 vncipher $out0,$out0,v25 sub $inp,$inp,r0 vncipher $out0,$out0,v26 lvx_u $in0,0,$inp vncipher $out0,$out0,v27 addi $key_,$sp,$FRAME+15 # rewind $key_ vncipher $out0,$out0,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vncipher $out0,$out0,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vxor $twk0,$twk0,v31 le?vperm $in0,$in0,$in0,$leperm vncipher $out0,$out0,v30 mtctr $rounds vncipherlast $out0,$out0,$twk0 vmr $twk0,$twk1 # unused tweak vmr $twk1,$twk2 le?vperm $out0,$out0,$out0,$leperm stvx_u $out0,$x00,$out # store output addi $out,$out,0x10 vxor $out0,$in0,$twk2 bne Lxts_dec6x_steal b Lxts_dec6x_done .align 4 Lxts_dec6x_zero: cmpwi $taillen,0 beq Lxts_dec6x_done lvx_u $in0,0,$inp le?vperm $in0,$in0,$in0,$leperm vxor $out0,$in0,$twk1 Lxts_dec6x_steal: vncipher $out0,$out0,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vncipher $out0,$out0,v25 lvx v25,$x10,$key_ # round[4] bdnz Lxts_dec6x_steal add $inp,$inp,$taillen vncipher $out0,$out0,v24 cmpwi $taillen,0 vncipher $out0,$out0,v25 lvx_u $in0,0,$inp vncipher $out0,$out0,v26 lvsr $inpperm,0,$taillen # $in5 is no more vncipher $out0,$out0,v27 addi $key_,$sp,$FRAME+15 # rewind $key_ vncipher $out0,$out0,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vncipher $out0,$out0,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vxor $twk1,$twk1,v31 le?vperm $in0,$in0,$in0,$leperm vncipher $out0,$out0,v30 vperm $in0,$in0,$in0,$inpperm vncipherlast $tmp,$out0,$twk1 le?vperm $out0,$tmp,$tmp,$leperm le?stvx_u $out0,0,$out be?stvx_u $tmp,0,$out vxor $out0,$out0,$out0 vspltisb $out1,-1 vperm $out0,$out0,$out1,$inpperm vsel $out0,$in0,$tmp,$out0 vxor $out0,$out0,$twk0 subi r30,$out,1 mtctr $taillen Loop_xts_dec6x_steal: lbzu r0,1(r30) stb r0,16(r30) bdnz Loop_xts_dec6x_steal li $taillen,0 mtctr $rounds b Loop_xts_dec1x # one more time... .align 4 Lxts_dec6x_done: ${UCMP}i $ivp,0 beq Lxts_dec6x_ret vxor $tweak,$twk0,$rndkey0 le?vperm $tweak,$tweak,$tweak,$leperm stvx_u $tweak,0,$ivp Lxts_dec6x_ret: mtlr r11 li r10,`$FRAME+15` li r11,`$FRAME+31` stvx $seven,r10,$sp # wipe copies of round keys addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 stvx $seven,r10,$sp addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 stvx $seven,r10,$sp addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 stvx $seven,r10,$sp addi r10,r10,32 stvx $seven,r11,$sp addi r11,r11,32 mtspr 256,$vrsave lvx v20,r10,$sp # ABI says so addi r10,r10,32 lvx v21,r11,$sp addi r11,r11,32 lvx v22,r10,$sp addi r10,r10,32 lvx v23,r11,$sp addi r11,r11,32 lvx v24,r10,$sp addi r10,r10,32 lvx v25,r11,$sp addi r11,r11,32 lvx v26,r10,$sp addi r10,r10,32 lvx v27,r11,$sp addi r11,r11,32 lvx v28,r10,$sp addi r10,r10,32 lvx v29,r11,$sp addi r11,r11,32 lvx v30,r10,$sp lvx v31,r11,$sp $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` blr .long 0 .byte 0,12,0x04,1,0x80,6,6,0 .long 0 .align 5 _aesp8_xts_dec5x: vncipher $out0,$out0,v24 vncipher $out1,$out1,v24 vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vncipher $out4,$out4,v24 lvx v24,$x20,$key_ # round[3] addi $key_,$key_,0x20 vncipher $out0,$out0,v25 vncipher $out1,$out1,v25 vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vncipher $out4,$out4,v25 lvx v25,$x10,$key_ # round[4] bdnz _aesp8_xts_dec5x subi r0,$taillen,1 vncipher $out0,$out0,v24 vncipher $out1,$out1,v24 vncipher $out2,$out2,v24 vncipher $out3,$out3,v24 vncipher $out4,$out4,v24 andi. r0,r0,16 cmpwi $taillen,0 vncipher $out0,$out0,v25 vncipher $out1,$out1,v25 vncipher $out2,$out2,v25 vncipher $out3,$out3,v25 vncipher $out4,$out4,v25 vxor $twk0,$twk0,v31 sub $inp,$inp,r0 vncipher $out0,$out0,v26 vncipher $out1,$out1,v26 vncipher $out2,$out2,v26 vncipher $out3,$out3,v26 vncipher $out4,$out4,v26 vxor $in1,$twk1,v31 vncipher $out0,$out0,v27 lvx_u $in0,0,$inp vncipher $out1,$out1,v27 vncipher $out2,$out2,v27 vncipher $out3,$out3,v27 vncipher $out4,$out4,v27 vxor $in2,$twk2,v31 addi $key_,$sp,$FRAME+15 # rewind $key_ vncipher $out0,$out0,v28 vncipher $out1,$out1,v28 vncipher $out2,$out2,v28 vncipher $out3,$out3,v28 vncipher $out4,$out4,v28 lvx v24,$x00,$key_ # re-pre-load round[1] vxor $in3,$twk3,v31 vncipher $out0,$out0,v29 le?vperm $in0,$in0,$in0,$leperm vncipher $out1,$out1,v29 vncipher $out2,$out2,v29 vncipher $out3,$out3,v29 vncipher $out4,$out4,v29 lvx v25,$x10,$key_ # re-pre-load round[2] vxor $in4,$twk4,v31 vncipher $out0,$out0,v30 vncipher $out1,$out1,v30 vncipher $out2,$out2,v30 vncipher $out3,$out3,v30 vncipher $out4,$out4,v30 vncipherlast $out0,$out0,$twk0 vncipherlast $out1,$out1,$in1 vncipherlast $out2,$out2,$in2 vncipherlast $out3,$out3,$in3 vncipherlast $out4,$out4,$in4 mtctr $rounds blr .long 0 .byte 0,12,0x14,0,0,0,0,0 ___ }} }}} my $consts=1; foreach(split("\n",$code)) { s/\`([^\`]*)\`/eval($1)/geo; # constants table endian-specific conversion if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) { my $conv=$3; my @bytes=(); # convert to endian-agnostic format if ($1 eq "long") { foreach (split(/,\s*/,$2)) { my $l = /^0/?oct:int; push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff; } } else { @bytes = map(/^0/?oct:int,split(/,\s*/,$2)); } # little-endian conversion if ($flavour =~ /le$/o) { SWITCH: for($conv) { /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; }; /\?rev/ && do { @bytes=reverse(@bytes); last; }; } } #emit print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n"; next; } $consts=0 if (m/Lconsts:/o); # end of table # instructions prefixed with '?' are endian-specific and need # to be adjusted accordingly... if ($flavour =~ /le$/o) { # little-endian s/le\?//o or s/be\?/#be#/o or s/\?lvsr/lvsl/o or s/\?lvsl/lvsr/o or s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o; } else { # big-endian s/le\?/#le#/o or s/be\?//o or s/\?([a-z]+)/$1/o; } print $_,"\n"; } close STDOUT;