Source code
Revision control
Copy as Markdown
Other Tools
# This submission to NSS is to be made available under the terms of the
# Mozilla Public License, v. 2.0. You can obtain one at //mozilla.org/MPL/2.0/
# Copyright(c) 2021, Niels Möller and Mamone Tarsha
# Registers:
.set SP, 1
.set TOCP, 2
.macro VEC_LOAD_DATA VR, DATA, GPR
addis \GPR, 2, \DATA@got@ha
ld \GPR, \DATA@got@l(\GPR)
lvx \VR, 0, \GPR
.endm
.macro VEC_LOAD VR, GPR, IDX
lxvd2x \VR+32, \IDX, \GPR
vperm \VR, \VR, \VR, SWAP_MASK
.endm
.macro VEC_LOAD_INC VR, GPR, IDX
lxvd2x \VR+32, \IDX, \GPR
addi \IDX,\IDX,16
vperm \VR, \VR, \VR, SWAP_MASK
.endm
.macro VEC_STORE VR, GPR, IDX
vperm \VR, \VR, \VR, SWAP_MASK
stxvd2x \VR+32, \IDX, \GPR
.endm
# 0 < LEN < 16, pad the remaining bytes with zeros
.macro LOAD_LEN DATA, LEN, VAL1, VAL0, TMP0, TMP1, TMP2
li \TMP0, 0
li \VAL1, 0
li \VAL0, 0
andi. \TMP1, \LEN, 8
beq 1f
ldbrx \VAL1, 0, \DATA
li \TMP0, 8
1:
andi. \TMP1, \LEN, 7
beq 3f
li \TMP1, 56
2:
lbzx \TMP2, \TMP0, \DATA
sld \TMP2, \TMP2, \TMP1
subi \TMP1, \TMP1, 8
or \VAL0, \VAL0, \TMP2
addi \TMP0, \TMP0, 1
cmpld \TMP0, \LEN
bne 2b
andi. \TMP1, \LEN, 8
bne 3f
mr \VAL1, \VAL0
li \VAL0, 0
3:
.endm
# 0 < LEN < 16
.macro STORE_LEN DATA, LEN, VAL1, VAL0, TMP0, TMP1, TMP2
andi. \TMP1, \LEN, 8
beq 1f
stdbrx \VAL1, 0, \DATA
li \TMP0, 8
b 2f
1:
li \TMP0, 0
mr \VAL0, \VAL1
2:
andi. \TMP1, \LEN, 7
beq 4f
li \TMP1, 56
3:
srd \TMP2, \VAL0, \TMP1
subi \TMP1, \TMP1, 8
stbx \TMP2, \TMP0, \DATA
addi \TMP0, \TMP0, 1
cmpld \TMP0, \LEN
bne 3b
4:
.endm
.text
################################################################################
# Generates the H table
# void ppc_aes_gcmINIT(uint8_t Htbl[16*8], uint32_t *KS, int NR);
.globl ppc_aes_gcmINIT
.type ppc_aes_gcmINIT,@function
.align 5
ppc_aes_gcmINIT:
addis TOCP,12,(.TOC.-ppc_aes_gcmINIT)@ha
addi TOCP,TOCP,(.TOC.-ppc_aes_gcmINIT)@l
.localentry ppc_aes_gcmINIT, .-ppc_aes_gcmINIT
.set Htbl, 3
.set KS, 4
.set NR, 5
.set ZERO, 19
.set MSB, 18
.set ONE, 17
.set SWAP_MASK, 0
.set POLY, 1
.set K, 2
.set H, 3
.set H2, 4
.set H3, 5
.set H4, 6
.set HP, 7
.set HS, 8
.set R, 9
.set F, 10
.set T, 11
.set H1M, 12
.set H1L, 13
.set H2M, 14
.set H2L, 15
.set H3M, 16
.set H3L, 17
.set H4M, 18
.set H4L, 19
VEC_LOAD_DATA SWAP_MASK, .Ldb_bswap_mask, 6
VEC_LOAD_DATA POLY, .Lpoly, 6
li 6, 0
VEC_LOAD_INC H, KS, 6
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
cmpwi NR, 10
beq .LH_done
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
cmpwi NR, 12
beq .LH_done
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
VEC_LOAD_INC K, KS, 6
vcipher H, H, K
.LH_done:
VEC_LOAD K, KS, 6
vcipherlast H, H, K
vupkhsb MSB, H
vspltisb ONE, 1
vspltb MSB, MSB, 0
vsl H, H, ONE
vand MSB, MSB, POLY
vxor ZERO, ZERO, ZERO
vxor H, H, MSB
vsldoi POLY, ZERO, POLY, 8
vpmsumd HP, H, POLY
vsldoi HS, H, H, 8
vxor HP, HP, HS
vsldoi H1L, HP, HS, 8
vsldoi H1M, HS, HP, 8
vsldoi H1L, H1L, H1L, 8
# calculate H^2
vpmsumd F, H, H1L
vpmsumd R, H, H1M
vpmsumd T, F, POLY
vsldoi H2, F, F, 8
vxor R, R, T
vxor H2, H2, R
vpmsumd HP, H2, POLY
vsldoi HS, H2, H2, 8
vxor HP, HP, HS
vsldoi H2L, HP, HS, 8
vsldoi H2M, HS, HP, 8
vsldoi H2L, H2L, H2L, 8
# calculate H^3
vpmsumd F, H2, H1L
vpmsumd R, H2, H1M
vpmsumd T, F, POLY
vsldoi H3, F, F, 8
vxor R, R, T
vxor H3, H3, R
vpmsumd HP, H3, POLY
vsldoi HS, H3, H3, 8
vxor HP, HP, HS
vsldoi H3L, HP, HS, 8
vsldoi H3M, HS, HP, 8
vsldoi H3L, H3L, H3L, 8
# calculate H^4
vpmsumd F, H2, H2L
vpmsumd R, H2, H2M
vpmsumd T, F, POLY
vsldoi H4, F, F, 8
vxor R, R, T
vxor H4, H4, R
vpmsumd HP, H4, POLY
vsldoi HS, H4, H4, 8
vxor HP, HP, HS
vsldoi H4L, HP, HS, 8
vsldoi H4M, HS, HP, 8
vsldoi H4L, H4L, H4L, 8
li 8, 16*1
li 9, 16*2
li 10, 16*3
stxvd2x H1L+32, 0, Htbl
stxvd2x H1M+32, 8, Htbl
stxvd2x H2L+32, 9, Htbl
stxvd2x H2M+32, 10, Htbl
li 7, 16*4
li 8, 16*5
li 9, 16*6
li 10, 16*7
stxvd2x H3L+32, 7, Htbl
stxvd2x H3M+32, 8, Htbl
stxvd2x H4L+32, 9, Htbl
stxvd2x H4M+32, 10, Htbl
blr
.size ppc_aes_gcmINIT, . - ppc_aes_gcmINIT
################################################################################
# Authenticate only
# void ppc_aes_gcmHASH(uint8_t Htbl[16*8], uint8_t *AAD, uint64_t Alen, uint8_t *Tp);
.globl ppc_aes_gcmHASH
.type ppc_aes_gcmHASH,@function
.align 5
ppc_aes_gcmHASH:
addis TOCP,12,(.TOC.-ppc_aes_gcmHASH)@ha
addi TOCP,TOCP,(.TOC.-ppc_aes_gcmHASH)@l
.localentry ppc_aes_gcmHASH, .-ppc_aes_gcmHASH
.set Htbl, 3
.set AAD, 4
.set Alen, 5
.set Tp, 6
.set SWAP_MASK, 0
.set POLY, 1
.set D, 2
.set C0, 3
.set C1, 4
.set C2, 5
.set C3, 6
.set T, 7
.set R, 8
.set F, 9
.set R2, 10
.set F2, 11
.set R3, 12
.set F3, 13
.set R4, 14
.set F4, 15
.set H1M, 16
.set H1L, 17
.set H2M, 18
.set H2L, 19
.set H3M, 28
.set H3L, 29
.set H4M, 30
.set H4L, 31
# store non-volatile vector registers
addi 7, SP, -16
stvx 31, 0, 7
addi 7, SP, -32
stvx 30, 0, 7
addi 7, SP, -48
stvx 29, 0, 7
addi 7, SP, -64
stvx 28, 0, 7
VEC_LOAD_DATA SWAP_MASK, .Ldb_bswap_mask, 7
VEC_LOAD_DATA POLY, .Lpoly_r, 7
VEC_LOAD D, Tp, 0
# --- process 4 blocks ---
srdi. 7, Alen, 6 # 4-blocks loop count
beq .L2x
mtctr 7 # set counter register
# load table elements
li 8, 1*16
li 9, 2*16
li 10, 3*16
lxvd2x H1L+32, 0, Htbl
lxvd2x H1M+32, 8, Htbl
lxvd2x H2L+32, 9, Htbl
lxvd2x H2M+32, 10, Htbl
li 7, 4*16
li 8, 5*16
li 9, 6*16
li 10, 7*16
lxvd2x H3L+32, 7, Htbl
lxvd2x H3M+32, 8, Htbl
lxvd2x H4L+32, 9, Htbl
lxvd2x H4M+32, 10, Htbl
li 8, 0x10
li 9, 0x20
li 10, 0x30
.align 5
.L4x_loop:
# load input
lxvd2x C0+32, 0, AAD
lxvd2x C1+32, 8, AAD
lxvd2x C2+32, 9, AAD
lxvd2x C3+32, 10, AAD
vperm C0, C0, C0, SWAP_MASK
vperm C1, C1, C1, SWAP_MASK
vperm C2, C2, C2, SWAP_MASK
vperm C3, C3, C3, SWAP_MASK
# digest combining
vxor C0, C0, D
# polynomial multiplication
vpmsumd F2, H3L, C1
vpmsumd R2, H3M, C1
vpmsumd F3, H2L, C2
vpmsumd R3, H2M, C2
vpmsumd F4, H1L, C3
vpmsumd R4, H1M, C3
vpmsumd F, H4L, C0
vpmsumd R, H4M, C0
# deferred recombination of partial products
vxor F3, F3, F4
vxor R3, R3, R4
vxor F, F, F2
vxor R, R, R2
vxor F, F, F3
vxor R, R, R3
# reduction
vpmsumd T, F, POLY
vsldoi D, F, F, 8
vxor R, R, T
vxor D, R, D
addi AAD, AAD, 0x40
bdnz .L4x_loop
clrldi Alen, Alen, 58
.L2x:
# --- process 2 blocks ---
srdi. 7, Alen, 5
beq .L1x
# load table elements
li 8, 1*16
li 9, 2*16
li 10, 3*16
lxvd2x H1L+32, 0, Htbl
lxvd2x H1M+32, 8, Htbl
lxvd2x H2L+32, 9, Htbl
lxvd2x H2M+32, 10, Htbl
# load input
li 10, 0x10
lxvd2x C0+32, 0, AAD
lxvd2x C1+32, 10, AAD
vperm C0, C0, C0, SWAP_MASK
vperm C1, C1, C1, SWAP_MASK
# previous digest combining
vxor C0, C0, D
# polynomial multiplication
vpmsumd F2, H1L, C1
vpmsumd R2, H1M, C1
vpmsumd F, H2L, C0
vpmsumd R, H2M, C0
# deferred recombination of partial products
vxor F, F, F2
vxor R, R, R2
# reduction
vpmsumd T, F, POLY
vsldoi D, F, F, 8
vxor R, R, T
vxor D, R, D
addi AAD, AAD, 0x20
clrldi Alen, Alen, 59
.L1x:
# --- process 1 block ---
srdi. 7, Alen, 4
beq .Ltail
# load table elements
li 8, 1*16
lxvd2x H1L+32, 0, Htbl
lxvd2x H1M+32, 8, Htbl
# load input
lxvd2x C0+32, 0, AAD
vperm C0, C0, C0, SWAP_MASK
# previous digest combining
vxor C0, C0, D
# polynomial multiplication
vpmsumd F, H1L, C0
vpmsumd R, H1M, C0
# reduction
vpmsumd T, F, POLY
vsldoi D, F, F, 8
vxor R, R, T
vxor D, R, D
addi AAD, AAD, 0x10
clrldi Alen, Alen, 60
.Ltail:
cmpldi Alen, 0
beq .Lh_done
# --- process the final partial block ---
# load table elements
li 8, 1*16
lxvd2x H1L+32, 0, Htbl
lxvd2x H1M+32, 8, Htbl
LOAD_LEN AAD, Alen, 10, 9, 3, 7, 8
mtvrd C0, 10
mtvrd C1, 9
xxmrghd C0+32, C0+32, C1+32
# previous digest combining
vxor C0, C0, D
# polynomial multiplication
vpmsumd F, H1L, C0
vpmsumd R, H1M, C0
# reduction
vpmsumd T, F, POLY
vsldoi D, F, F, 8
vxor R, R, T
vxor D, R, D
.Lh_done:
VEC_STORE D, Tp, 0
# restore non-volatile vector registers
addi 7, SP, -16
lvx 31, 0, 7
addi 7, SP, -32
lvx 30, 0, 7
addi 7, SP, -48
lvx 29, 0, 7
addi 7, SP, -64
lvx 28, 0, 7
blr
.size ppc_aes_gcmHASH, . - ppc_aes_gcmHASH
################################################################################
# Generates the final GCM tag
# void ppc_aes_gcmTAG(uint8_t Htbl[16*8], uint8_t *Tp, uint64_t Mlen, uint64_t Alen, uint8_t* X0, uint8_t* TAG);
.globl ppc_aes_gcmTAG
.type ppc_aes_gcmTAG,@function
.align 5
ppc_aes_gcmTAG:
addis TOCP,12,(.TOC.-ppc_aes_gcmTAG)@ha
addi TOCP,TOCP,(.TOC.-ppc_aes_gcmTAG)@l
.localentry ppc_aes_gcmTAG, .-ppc_aes_gcmTAG
.set Htbl, 3
.set Tp, 4
.set Mlen, 5
.set Alen, 6
.set X0, 7
.set TAG, 8
.set SWAP_MASK, 0
.set POLY, 1
.set D, 2
.set C0, 3
.set C1, 4
.set T, 5
.set R, 6
.set F, 7
.set H1M, 8
.set H1L, 9
.set X, 10
VEC_LOAD_DATA SWAP_MASK, .Ldb_bswap_mask, 9
VEC_LOAD_DATA POLY, .Lpoly_r, 9
VEC_LOAD D, Tp, 0
# load table elements
li 9, 1*16
lxvd2x H1L+32, 0, Htbl
lxvd2x H1M+32, 9, Htbl
sldi Alen, Alen, 3
sldi Mlen, Mlen, 3
mtvrd C0, Alen
mtvrd C1, Mlen
xxmrghd C0+32, C0+32, C1+32
# previous digest combining
vxor C0, C0, D
# polynomial multiplication
vpmsumd F, H1L, C0
vpmsumd R, H1M, C0
# reduction
vpmsumd T, F, POLY
vsldoi D, F, F, 8
vxor R, R, T
vxor D, R, D
lxvd2x X+32, 0, X0
vperm D, D, D, SWAP_MASK
vxor X, X, D
stxvd2x X+32, 0, TAG
blr
.size ppc_aes_gcmTAG, . - ppc_aes_gcmTAG
################################################################################
# Crypt only
# void ppc_aes_gcmCRYPT(const uint8_t* PT, uint8_t* CT, uint64_t LEN, uint8_t *CTRP, uint32_t *KS, int NR);
.globl ppc_aes_gcmCRYPT
.type ppc_aes_gcmCRYPT,@function
.align 5
ppc_aes_gcmCRYPT:
addis TOCP,12,(.TOC.-ppc_aes_gcmCRYPT)@ha
addi TOCP,TOCP,(.TOC.-ppc_aes_gcmCRYPT)@l
.localentry ppc_aes_gcmCRYPT, .-ppc_aes_gcmCRYPT
.set PT, 3
.set CT, 4
.set LEN, 5
.set CTRP, 6
.set KS, 7
.set NR, 8
.set SWAP_MASK, 0
.set K, 1
.set CTR, 2
.set CTR0, 3
.set CTR1, 4
.set CTR2, 5
.set CTR3, 6
.set CTR4, 7
.set CTR5, 8
.set CTR6, 9
.set CTR7, 10
.set ZERO, 11
.set I1, 12
.set I2, 13
.set I3, 14
.set I4, 15
.set I5, 16
.set I6, 17
.set I7, 18
.set I8, 19
.set IN0, 24
.set IN1, 25
.set IN2, 26
.set IN3, 27
.set IN4, 28
.set IN5, 29
.set IN6, 30
.set IN7, 31
.macro ROUND_8
VEC_LOAD_INC K, KS, 10
vcipher CTR0, CTR0, K
vcipher CTR1, CTR1, K
vcipher CTR2, CTR2, K
vcipher CTR3, CTR3, K
vcipher CTR4, CTR4, K
vcipher CTR5, CTR5, K
vcipher CTR6, CTR6, K
vcipher CTR7, CTR7, K
.endm
.macro ROUND_4
VEC_LOAD_INC K, KS, 10
vcipher CTR0, CTR0, K
vcipher CTR1, CTR1, K
vcipher CTR2, CTR2, K
vcipher CTR3, CTR3, K
.endm
.macro ROUND_2
VEC_LOAD_INC K, KS, 10
vcipher CTR0, CTR0, K
vcipher CTR1, CTR1, K
.endm
.macro ROUND_1
VEC_LOAD_INC K, KS, 10
vcipher CTR0, CTR0, K
.endm
# store non-volatile general registers
std 31,-8(SP);
std 30,-16(SP);
std 29,-24(SP);
std 28,-32(SP);
std 27,-40(SP);
std 26,-48(SP);
std 25,-56(SP);
# store non-volatile vector registers
addi 9, SP, -80
stvx 31, 0, 9
addi 9, SP, -96
stvx 30, 0, 9
addi 9, SP, -112
stvx 29, 0, 9
addi 9, SP, -128
stvx 28, 0, 9
addi 9, SP, -144
stvx 27, 0, 9
addi 9, SP, -160
stvx 26, 0, 9
addi 9, SP, -176
stvx 25, 0, 9
addi 9, SP, -192
stvx 24, 0, 9
VEC_LOAD_DATA SWAP_MASK, .Ldb_bswap_mask, 9
vxor ZERO, ZERO, ZERO
vspltisb I1, 1
vspltisb I2, 2
vspltisb I3, 3
vspltisb I4, 4
vspltisb I5, 5
vspltisb I6, 6
vspltisb I7, 7
vspltisb I8, 8
vsldoi I1, ZERO, I1, 1
vsldoi I2, ZERO, I2, 1
vsldoi I3, ZERO, I3, 1
vsldoi I4, ZERO, I4, 1
vsldoi I5, ZERO, I5, 1
vsldoi I6, ZERO, I6, 1
vsldoi I7, ZERO, I7, 1
vsldoi I8, ZERO, I8, 1
VEC_LOAD CTR, CTRP, 0
srdi. 9, LEN, 7
beq .Lctr_4x
mtctr 9
li 25, 0x10
li 26, 0x20
li 27, 0x30
li 28, 0x40
li 29, 0x50
li 30, 0x60
li 31, 0x70
.align 5
.L8x_loop:
li 10, 0
VEC_LOAD_INC K, KS, 10
vadduwm CTR1, CTR, I1
vadduwm CTR2, CTR, I2
vadduwm CTR3, CTR, I3
vadduwm CTR4, CTR, I4
vadduwm CTR5, CTR, I5
vadduwm CTR6, CTR, I6
vadduwm CTR7, CTR, I7
vxor CTR0, CTR, K
vxor CTR1, CTR1, K
vxor CTR2, CTR2, K
vxor CTR3, CTR3, K
vxor CTR4, CTR4, K
vxor CTR5, CTR5, K
vxor CTR6, CTR6, K
vxor CTR7, CTR7, K
ROUND_8
ROUND_8
ROUND_8
ROUND_8
ROUND_8
ROUND_8
ROUND_8
ROUND_8
ROUND_8
cmpwi NR, 10
beq .Llast_8
ROUND_8
ROUND_8
cmpwi NR, 12
beq .Llast_8
ROUND_8
ROUND_8
.Llast_8:
VEC_LOAD K, KS, 10
vcipherlast CTR0, CTR0, K
vcipherlast CTR1, CTR1, K
vcipherlast CTR2, CTR2, K
vcipherlast CTR3, CTR3, K
vcipherlast CTR4, CTR4, K
vcipherlast CTR5, CTR5, K
vcipherlast CTR6, CTR6, K
vcipherlast CTR7, CTR7, K
lxvd2x IN0+32, 0, PT
lxvd2x IN1+32, 25, PT
lxvd2x IN2+32, 26, PT
lxvd2x IN3+32, 27, PT
lxvd2x IN4+32, 28, PT
lxvd2x IN5+32, 29, PT
lxvd2x IN6+32, 30, PT
lxvd2x IN7+32, 31, PT
vperm CTR0, CTR0, CTR0, SWAP_MASK
vperm CTR1, CTR1, CTR1, SWAP_MASK
vperm CTR2, CTR2, CTR2, SWAP_MASK
vperm CTR3, CTR3, CTR3, SWAP_MASK
vperm CTR4, CTR4, CTR4, SWAP_MASK
vperm CTR5, CTR5, CTR5, SWAP_MASK
vperm CTR6, CTR6, CTR6, SWAP_MASK
vperm CTR7, CTR7, CTR7, SWAP_MASK
vxor IN0, IN0, CTR0
vxor IN1, IN1, CTR1
vxor IN2, IN2, CTR2
vxor IN3, IN3, CTR3
vxor IN4, IN4, CTR4
vxor IN5, IN5, CTR5
vxor IN6, IN6, CTR6
vxor IN7, IN7, CTR7
stxvd2x IN0+32, 0, CT
stxvd2x IN1+32, 25, CT
stxvd2x IN2+32, 26, CT
stxvd2x IN3+32, 27, CT
stxvd2x IN4+32, 28, CT
stxvd2x IN5+32, 29, CT
stxvd2x IN6+32, 30, CT
stxvd2x IN7+32, 31, CT
vadduwm CTR, CTR, I8
addi PT, PT, 0x80
addi CT, CT, 0x80
bdnz .L8x_loop
clrldi LEN, LEN, 57
.Lctr_4x:
srdi. 9, LEN, 6
beq .Lctr_2x
li 10, 0
li 29, 0x10
li 30, 0x20
li 31, 0x30
VEC_LOAD_INC K, KS, 10
vadduwm CTR1, CTR, I1
vadduwm CTR2, CTR, I2
vadduwm CTR3, CTR, I3
vxor CTR0, CTR, K
vxor CTR1, CTR1, K
vxor CTR2, CTR2, K
vxor CTR3, CTR3, K
ROUND_4
ROUND_4
ROUND_4
ROUND_4
ROUND_4
ROUND_4
ROUND_4
ROUND_4
ROUND_4
cmpwi NR, 10
beq .Llast_4
ROUND_4
ROUND_4
cmpwi NR, 12
beq .Llast_4
ROUND_4
ROUND_4
.Llast_4:
VEC_LOAD K, KS, 10
vcipherlast CTR0, CTR0, K
vcipherlast CTR1, CTR1, K
vcipherlast CTR2, CTR2, K
vcipherlast CTR3, CTR3, K
lxvd2x IN0+32, 0, PT
lxvd2x IN1+32, 29, PT
lxvd2x IN2+32, 30, PT
lxvd2x IN3+32, 31, PT
vperm CTR0, CTR0, CTR0, SWAP_MASK
vperm CTR1, CTR1, CTR1, SWAP_MASK
vperm CTR2, CTR2, CTR2, SWAP_MASK
vperm CTR3, CTR3, CTR3, SWAP_MASK
vxor IN0, IN0, CTR0
vxor IN1, IN1, CTR1
vxor IN2, IN2, CTR2
vxor IN3, IN3, CTR3
stxvd2x IN0+32, 0, CT
stxvd2x IN1+32, 29, CT
stxvd2x IN2+32, 30, CT
stxvd2x IN3+32, 31, CT
vadduwm CTR, CTR, I4
addi PT, PT, 0x40
addi CT, CT, 0x40
clrldi LEN, LEN, 58
.Lctr_2x:
srdi. 9, LEN, 5
beq .Lctr_1x
li 10, 0
li 31, 0x10
VEC_LOAD_INC K, KS, 10
vadduwm CTR1, CTR, I1
vxor CTR0, CTR, K
vxor CTR1, CTR1, K
ROUND_2
ROUND_2
ROUND_2
ROUND_2
ROUND_2
ROUND_2
ROUND_2
ROUND_2
ROUND_2
cmpwi NR, 10
beq .Llast_2
ROUND_2
ROUND_2
cmpwi NR, 12
beq .Llast_2
ROUND_2
ROUND_2
.Llast_2:
VEC_LOAD K, KS, 10
vcipherlast CTR0, CTR0, K
vcipherlast CTR1, CTR1, K
lxvd2x IN0+32, 0, PT
lxvd2x IN1+32, 31, PT
vperm CTR0, CTR0, CTR0, SWAP_MASK
vperm CTR1, CTR1, CTR1, SWAP_MASK
vxor IN0, IN0, CTR0
vxor IN1, IN1, CTR1
stxvd2x IN0+32, 0, CT
stxvd2x IN1+32, 31, CT
vadduwm CTR, CTR, I2
addi PT, PT, 0x20
addi CT, CT, 0x20
clrldi LEN, LEN, 59
.Lctr_1x:
srdi. 9, LEN, 4
beq .Lctr_tail
li 10, 0
VEC_LOAD_INC K, KS, 10
vxor CTR0, CTR, K
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
cmpwi NR, 10
beq .Llast_1
ROUND_1
ROUND_1
cmpwi NR, 12
beq .Llast_1
ROUND_1
ROUND_1
.Llast_1:
VEC_LOAD K, KS, 10
vcipherlast CTR0, CTR0, K
lxvd2x IN0+32, 0, PT
vperm CTR0, CTR0, CTR0, SWAP_MASK
vxor IN0, IN0, CTR0
stxvd2x IN0+32, 0, CT
vadduwm CTR, CTR, I1
addi PT, PT, 0x10
addi CT, CT, 0x10
clrldi LEN, LEN, 60
.Lctr_tail:
cmpldi LEN, 0
beq .Lc_done
li 10, 0
VEC_LOAD_INC K, KS, 10
vxor CTR0, CTR, K
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
ROUND_1
cmpwi NR, 10
beq .Llast_tail
ROUND_1
ROUND_1
cmpwi NR, 12
beq .Llast_tail
ROUND_1
ROUND_1
.Llast_tail:
VEC_LOAD K, KS, 10
vcipherlast CTR0, CTR0, K
LOAD_LEN PT, LEN, 10, 9, 29, 30, 31
vsldoi CTR1, CTR0, CTR0, 8
mfvrd 31, CTR0
mfvrd 30, CTR1
xor 10, 10, 31
xor 9, 9, 30
STORE_LEN CT, LEN, 10, 9, 29, 30, 31
vadduwm CTR, CTR, I1
.Lc_done:
VEC_STORE CTR, CTRP, 0
# restore non-volatile vector registers
addi 9, SP, -80
lvx 31, 0, 9
addi 9, SP, -96
lvx 30, 0, 9
addi 9, SP, -112
lvx 29, 0, 9
addi 9, SP, -128
lvx 28, 0, 9
addi 9, SP, -144
lvx 27, 0, 9
addi 9, SP, -160
lvx 26, 0, 9
addi 9, SP, -176
lvx 25, 0, 9
addi 9, SP, -192
lvx 24, 0, 9
# restore non-volatile general registers
ld 31,-8(SP);
ld 30,-16(SP);
ld 29,-24(SP);
ld 28,-32(SP);
ld 27,-40(SP);
ld 26,-48(SP);
ld 25,-56(SP);
blr
.size ppc_aes_gcmCRYPT, . - ppc_aes_gcmCRYPT
.data
.align 4
.Lpoly:
.byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
.Lpoly_r:
.byte 0,0,0,0,0,0,0,0xc2,0,0,0,0,0,0,0,0
.Ldb_bswap_mask:
.byte 8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7