Source code
Revision control
Copy as Markdown
Other Tools
// Copyright (c) 2022, ARM Inc.
// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
// Author: Hanno Becker <hannobecker@posteo.de>
// This file was derived from the assembly generated from aesv8-gcm-armv8.pl,
// written by Fangming Fang <fangming.fang@arm.com> for the OpenSSL project,
// and derived from https: // github.com/ARM-software/AArch64cryptolib, original
// author Samuel Lee <Samuel.Lee@arm.com>.
// The code below is a 'clean' AArch64 implementation of AES-GCM emphasizing
// the logic of the computation. It is meant as the input to manual audits /
// formal verification, as well as automated micro-optimization such as done
// by the SLOTHY superoptimizer (https: // github.com/slothy-optimizer/slothy).
#if !defined(__has_feature)
#define __has_feature(x) 0
#endif
#if __has_feature(memory_sanitizer) && !defined(OPENSSL_NO_ASM)
#define OPENSSL_NO_ASM
#endif
#include "aarch64-gcm-asm-compat.h"
#if !defined(OPENSSL_NO_ASM) && defined(__AARCH64EL__)
#if defined(__ELF__)
.arch armv8-a+crypto
.text
.globl aes_gcm_dec_kernel_slothy_base_256
.hidden aes_gcm_dec_kernel_slothy_base_256
.type aes_gcm_dec_kernel_slothy_base_256,%function
#elif defined(__APPLE__)
.text
.globl _aes_gcm_dec_kernel_slothy_base_256
.private_extern _aes_gcm_dec_kernel_slothy_base_256
#else
#error Unknown configuration
#endif
#if __ARM_MAX_ARCH__ >= 8
// Arguments
input .req x0
len_bits .req x1
output .req x2
tag_ptr .req x3
ivec .req x4
key .req x5
Htable .req x6
byte_len .req x15
constant_temp .req x25
count .req x1
full_blocks .req x7
remainder .req x9
unroll .req x10
aes_st .req v0
aes_st_q .req q0
res .req v0
res_q .req q0
ghash_hi .req v9
ghash_lo .req v8
ghash_mid .req v10
ghash_mid_d .req d10
ghash_tmp .req v11
ghash_tmp_d .req d11
ghash_mod .req v7
ghash_mod_d .req d7
modulo_tmp0 .req v0
modulo_tmp1 .req v1
Ht1q .req q12
Ht2q .req q13
Ht12q .req q14
Ht1 .req v12
Ht2 .req v13
Ht12 .req v14
Ht3q .req Ht1q
Ht4q .req Ht2q
Ht34q .req Ht12q
Ht3 .req Ht1
Ht4 .req Ht2
Ht34 .req Ht12
rk0q .req q18
rk1q .req q19
rk2q .req q20
rk3q .req q21
rk4q .req q22
rk5q .req q23
rk6q .req q24
rk7q .req q25
rk8q .req q26
rk9q .req q27
rk10q .req q28
rk11q .req q15
rk12q .req q16
rk13q .req q17
rk14q .req q2
rk0 .req v18
rk1 .req v19
rk2 .req v20
rk3 .req v21
rk4 .req v22
rk5 .req v23
rk6 .req v24
rk7 .req v25
rk8 .req v26
rk9 .req v27
rk10 .req v28
rk11 .req v15
rk12 .req v16
rk13 .req v17
rk14 .req v2
plain .req v29
plain_q .req q29
rctr_inc .req v30
rtmp_ctr .req v31
rtmp_ctr_q .req q31
tag .req v11
tag_q .req q11
#define UNROLL 4
#define STACK_SIZE_GPRS (6*16)
#define STACK_SIZE_VREGS (4*16)
#define STACK_SIZE (STACK_SIZE_GPRS + STACK_SIZE_VREGS)
#define STACK_BASE_GPRS (0)
#define STACK_BASE_VREGS (STACK_SIZE_GPRS)
/********************************************************************/
/* Generic preamble/postamble macros */
/********************************************************************/
.macro save_vregs
stp d8, d9, [sp, #(STACK_BASE_VREGS + 16*0)]
stp d10, d11, [sp, #(STACK_BASE_VREGS + 16*1)]
stp d12, d13, [sp, #(STACK_BASE_VREGS + 16*2)]
stp d14, d15, [sp, #(STACK_BASE_VREGS + 16*3)]
.endm
.macro restore_vregs
ldp d8, d9, [sp, #(STACK_BASE_VREGS + 16*0)]
ldp d10, d11, [sp, #(STACK_BASE_VREGS + 16*1)]
ldp d12, d13, [sp, #(STACK_BASE_VREGS + 16*2)]
ldp d14, d15, [sp, #(STACK_BASE_VREGS + 16*3)]
.endm
.macro save_gprs
stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)]
stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)]
stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)]
stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)]
stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)]
stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)]
.endm
.macro restore_gprs
ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)]
ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)]
ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)]
ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)]
ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)]
ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)]
.endm
// Derive number of iterations of unrolled loop and single-block loop
.macro prepare_loop_counts
mov unroll, #UNROLL
// Number of AES Blocks (16b each)
lsr full_blocks, byte_len, #4
// Number of iterations of the unrolled loop
udiv count, full_blocks, unroll
// Number of iterations for the tail loop handling 1 block each
msub remainder, count, unroll, full_blocks
.endm
/********************************************************************/
/* AES related macros */
/********************************************************************/
.macro load_iv
ldr rtmp_ctr_q, [ivec]
// set up counter increment
mov constant_temp, #0x100000000
movi rctr_inc.16b, #0x0
fmov rctr_inc.d[1], constant_temp
rev32 rtmp_ctr.16b, rtmp_ctr.16b
.endm
// Increase AES counter
.macro aes_ctr_inc
add rtmp_ctr.4s, rtmp_ctr.4s, rctr_inc.4s
.endm
// Increase AES counter and initialize new AES state
.macro next_ctr_init_aes aes_st
rev32 \aes_st\().16b, rtmp_ctr.16b
aes_ctr_inc
.endm
// A single AES round
// Prevent SLOTHY from unfolding because uArchs tend to fuse AESMC+AESE
.macro aesr data, key // @slothy:no-unfold
aese \data, \key
aesmc \data, \data
.endm
.macro aesr_0_8 data, key
aesr \data\().16b, \key\()0.16b
aesr \data\().16b, \key\()1.16b
aesr \data\().16b, \key\()2.16b
aesr \data\().16b, \key\()3.16b
aesr \data\().16b, \key\()4.16b
aesr \data\().16b, \key\()5.16b
aesr \data\().16b, \key\()6.16b
aesr \data\().16b, \key\()7.16b
aesr \data\().16b, \key\()8.16b
.endm
.macro aesr_9_10 data, key
aesr \data\().16b, \key\()9.16b
aesr \data\().16b, \key\()10.16b
.endm
.macro aesr_11_12 data, key
aesr \data\().16b, \key\()11.16b
aesr \data\().16b, \key\()12.16b
.endm
// Destructs inA
.macro eor3 out, inA, inB, inC
eor \inA, \inA, \inB
eor \out, \inA, \inC
.endm
.macro aesr_final aes_st, plain, out
aese \aes_st\().16b, rk13.16b
eor3 \out\().16b, \aes_st\().16b, rk14.16b, \plain\().16b
.endm
.macro aes_full_block aes_st, input, output
next_ctr_init_aes \aes_st
aesr_0_8 \aes_st\(), rk
aesr_9_10 \aes_st\(), rk
aesr_11_12 \aes_st\(), rk
aesr_final \aes_st, \input, \output
.endm
.macro load_round_key i
ldr rk\i\()q, [key, #((\i)*16)]
.endm
.macro load_round_keys
load_round_key 0
load_round_key 1
load_round_key 2
load_round_key 3
load_round_key 4
load_round_key 5
load_round_key 6
load_round_key 7
load_round_key 8
load_round_key 9
load_round_key 10
load_round_key 11
load_round_key 12
load_round_key 13
load_round_key 14
.endm
/********************************************************************/
/* Loading of H-table (precomputed H-powers for GHASH) */
/********************************************************************/
// This has to be synchronized with the H-table generation
.macro load_h1 dst, dst_q
ldr \dst_q, [Htable]
.endm
.macro load_h2 dst, dst_q
ldr \dst_q, [Htable, #32]
.endm
.macro load_h3 dst, dst_q
ldr \dst_q, [Htable, #48]
.endm
.macro load_h4 dst, dst_q
ldr \dst_q, [Htable, #80]
.endm
.macro load_h12 dst, dst_q
ldr \dst_q, [Htable, #16]
.endm
.macro load_h34 dst, dst_q
ldr \dst_q, [Htable, #64]
.endm
.macro load_full_htable
load_h1 Ht1, Ht1q
load_h2 Ht2, Ht2q
load_h3 Ht3, Ht3q
load_h4 Ht4, Ht4q
load_h12 Ht12, Ht12q
load_h34 Ht34, Ht34q
.endm
.macro load_htable_12
load_h1 Ht1, Ht1q
load_h2 Ht2, Ht2q
load_h12 Ht12, Ht12q
.endm
.macro load_htable_34
load_h3 Ht3, Ht3q
load_h4 Ht4, Ht4q
load_h34 Ht34, Ht34q
.endm
/********************************************************************/
/* Macros for GHASH udpate */
/********************************************************************/
.macro ghash_init_0 input, Hk, Hk_mid, tag
rev64 \input\().16b, \input\().16b
eor \input\().16b, \input\().16b, \tag\().16b
// Low product
pmull ghash_lo.1q, \input\().1d, \Hk\().1d
// High product
pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d
// Middle product
mov ghash_tmp_d, \input\().d[1]
eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b
pmull ghash_mid.1q, ghash_tmp.1d, \Hk_mid\().1d
.endm
.macro ghash_init_1 input, Hk, Hk_mid, tag
rev64 \input\().16b, \input\().16b
eor \input\().16b, \input\().16b, \tag\().16b
// Low product
pmull ghash_lo.1q, \input\().1d, \Hk\().1d
// High product
pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d
// Middle product
ext ghash_tmp.16b, \input\().16b, \input\().16b, #8
eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b
pmull2 ghash_mid.1q, ghash_tmp.2d, \Hk_mid\().2d
.endm
.macro ghash_acc_0 input, Hk, Hk_mid
rev64 \input\().16b, \input\().16b
// Low product
pmull ghash_tmp.1q, \input\().1d, \Hk\().1d
eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b
// High product
pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d
eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b
// Middle product
mov ghash_tmp_d, \input\().d[1]
eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b
pmull ghash_tmp.1q, ghash_tmp.1d, \Hk_mid\().1d
eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b
.endm
.macro ghash_acc_1 input, Hk, Hk_mid
rev64 \input\().16b, \input\().16b
// Low product
pmull ghash_tmp.1q, \input\().1d, \Hk\().1d
eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b
// High product
pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d
eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b
// Middle product
ext ghash_tmp.16b, \input\().16b, \input\().16b, #8
eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b
pmull2 ghash_tmp.1q, ghash_tmp.2d, \Hk_mid\().2d
eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b
.endm
.macro ghash_finalize tag
eor modulo_tmp0.16b, ghash_lo.16b, ghash_hi.16b
pmull modulo_tmp1.1q, ghash_hi.1d, ghash_mod.1d
ext ghash_hi.16b, ghash_hi.16b, ghash_hi.16b, #8
eor ghash_mid.16b, ghash_mid.16b, modulo_tmp0.16b
eor modulo_tmp1.16b, ghash_hi.16b, modulo_tmp1.16b
eor ghash_mid.16b, ghash_mid.16b, modulo_tmp1.16b
pmull ghash_hi.1q, ghash_mid.1d, ghash_mod.1d
eor ghash_lo.16b, ghash_lo.16b, ghash_hi.16b
ext ghash_mid.16b, ghash_mid.16b, ghash_mid.16b, #8
eor \tag\().16b, ghash_lo.16b, ghash_mid.16b
ext \tag\().16b, \tag\().16b, \tag\().16b, #8
.endm
.macro load_tag
ldr tag_q, [tag_ptr]
rev64 tag.16b, tag.16b
.endm
.macro prepare_ghash
// Prepare constant for modular reduction
movi ghash_mod.8b, #0xc2
shl ghash_mod_d, ghash_mod_d, #56
.endm
/********************************************************************/
/* Core routine */
/********************************************************************/
.align 4
_aes_gcm_dec_kernel_slothy_base_256:
aes_gcm_dec_kernel_slothy_base_256:
#ifdef BORINGSSL_DISPATCH_TEST
adrp x9,_BORINGSSL_function_hit@PAGE
add x9, x9, _BORINGSSL_function_hit@PAGEOFF
mov w10, #1
strb w10, [x9,#15] // kFlag_aes_gcm_slothy
#endif
AARCH64_VALID_CALL_TARGET
sub sp, sp, #STACK_SIZE
Ldec_preamble_start:
save_gprs
save_vregs
lsr byte_len, len_bits, #3
load_round_keys
load_tag
load_iv
prepare_loop_counts
prepare_ghash
Ldec_preamble_end:
cbz count, Lloop_unrolled_end
// Instructions: 97
// Expected cycles: 65
// Expected IPC: 1.49
//
// Cycle bound: 65.0
// IPC bound: 1.49
//
// Wall time: 1.59s
// User time: 1.59s
//
// ----------------------- cycle (expected) ----------------------->
// 0 25 50
// |------------------------|------------------------|--------------
rev32 v5.16B, v31.16B // *................................................................
ldr q4, [x6, #48] // *................................................................
add v9.4S, v31.4S, v30.4S // .*...............................................................
ldr q12, [x0, #32] // .*...............................................................
aesr v5.16b, v18.16b // ..*..............................................................
ldr q0, [x0], #(4*16) // ..*..............................................................
rev32 v1.16B, v9.16B // ...*.............................................................
add v31.4S, v9.4S, v30.4S // ...*.............................................................
aesr v5.16b, v19.16b // ....*............................................................
ldr q13, [x0, #-48] // ....*............................................................
aesr v1.16b, v18.16b // .....*...........................................................
add v8.4S, v31.4S, v30.4S // .....*...........................................................
aesr v5.16b, v20.16b // ......*..........................................................
aesr v1.16b, v19.16b // .......*.........................................................
rev32 v10.16B, v8.16B // .......*.........................................................
aesr v5.16b, v21.16b // ........*........................................................
rev64 v29.16B, v13.16B // ........*........................................................
aesr v1.16b, v20.16b // .........*.......................................................
rev64 v14.16B, v0.16B // .........*.......................................................
aesr v5.16b, v22.16b // ..........*......................................................
aesr v1.16b, v21.16b // ...........*.....................................................
aesr v5.16b, v23.16b // ............*....................................................
aesr v1.16b, v22.16b // .............*...................................................
aesr v5.16b, v24.16b // ..............*..................................................
aesr v1.16b, v23.16b // ...............*.................................................
aesr v5.16b, v25.16b // ................*................................................
aesr v1.16b, v24.16b // .................*...............................................
aesr v5.16b, v26.16b // ..................*..............................................
aesr v1.16b, v25.16b // ...................*.............................................
aesr v5.16b, v27.16b // ....................*............................................
aesr v1.16b, v26.16b // .....................*...........................................
aesr v5.16b, v28.16b // ......................*..........................................
aesr v1.16b, v27.16b // .......................*.........................................
aesr v5.16b, v15.16b // ........................*........................................
aesr v1.16b, v28.16b // .........................*.......................................
aesr v10.16b, v18.16b // ..........................*......................................
aesr v1.16b, v15.16b // ...........................*.....................................
aesr v10.16b, v19.16b // ............................*....................................
aesr v1.16b, v16.16b // .............................*...................................
aesr v10.16b, v20.16b // ..............................*..................................
aese v1.16b, v17.16b // ...............................*.................................
aesr v5.16b, v16.16b // ................................*................................
eor v1.16B, v1.16B, v2.16B // .................................*...............................
aesr v10.16b, v21.16b // .................................*...............................
aese v5.16b, v17.16b // ..................................*..............................
eor v1.16B, v1.16B, v13.16B // ...................................*.............................
aesr v10.16b, v22.16b // ...................................*.............................
rev32 v9.16B, v31.16B // ....................................*............................
pmull2 v6.1q, v29.2d, v4.2d // ....................................*............................
eor v5.16B, v5.16B, v2.16B // .....................................*...........................
aesr v10.16b, v23.16b // .....................................*...........................
add v31.4S, v8.4S, v30.4S // ......................................*..........................
aesr v9.16b, v18.16b // ......................................*..........................
eor v8.16B, v5.16B, v0.16B // .......................................*.........................
aesr v10.16b, v24.16b // .......................................*.........................
aesr v9.16b, v19.16b // ........................................*........................
str q8, [x2], #(4*16) // .........................................*.......................
aesr v10.16b, v25.16b // .........................................*.......................
aesr v9.16b, v20.16b // ..........................................*......................
ldr q3, [x6, #80] // ..........................................*......................
str q1, [x2, #-48] // ...........................................*.....................
aesr v10.16b, v26.16b // ...........................................*.....................
eor v1.16B, v14.16B, v11.16B // ............................................*....................
aesr v9.16b, v21.16b // ............................................*....................
aesr v10.16b, v27.16b // .............................................*...................
aesr v9.16b, v22.16b // ..............................................*..................
ext v11.16B, v1.16B, v1.16B, #8 // ..............................................*..................
pmull2 v13.1q, v1.2d, v3.2d // ...............................................*.................
aesr v9.16b, v23.16b // ................................................*................
eor v0.16B, v13.16B, v6.16B // .................................................*...............
aesr v10.16b, v28.16b // .................................................*...............
ldr q6, [x6, #64] // ..................................................*..............
aesr v9.16b, v24.16b // ..................................................*..............
aesr v10.16b, v15.16b // ...................................................*.............
rev64 v8.16B, v12.16B // ...................................................*.............
aesr v9.16b, v25.16b // ....................................................*............
eor v5.16B, v11.16B, v1.16B // ....................................................*............
ldr q14, [x6, #32] // .....................................................*...........
aesr v10.16b, v16.16b // .....................................................*...........
aesr v9.16b, v26.16b // ......................................................*..........
aese v10.16b, v17.16b // .......................................................*.........
aesr v9.16b, v27.16b // ........................................................*........
eor v13.16B, v10.16B, v2.16B // .........................................................*.......
pmull2 v10.1q, v8.2d, v14.2d // .........................................................*.......
aesr v9.16b, v28.16b // ..........................................................*......
pmull v3.1q, v1.1d, v3.1d // ...........................................................*.....
eor v1.16B, v0.16B, v10.16B // ...........................................................*.....
mov d10, v29.d[1] // ............................................................*....
aesr v9.16b, v15.16b // ............................................................*....
pmull2 v0.1q, v5.2d, v6.2d // .............................................................*...
ext v11.16B, v8.16B, v8.16B, #8 // .............................................................*...
eor v5.8B, v10.8B, v29.8B // ..............................................................*..
aesr v9.16b, v16.16b // ..............................................................*..
eor v10.16B, v11.16B, v8.16B // ...............................................................*.
pmull v8.1q, v8.1d, v14.1d // ...............................................................*.
aese v9.16b, v17.16b // ................................................................*
ldr q11, [x6, #16] // ................................................................*
// ----------------------- cycle (expected) ----------------------->
// 0 25 50
// |------------------------|------------------------|--------------
// rev32 V<ssa_1>.16B, v31.16B // *................................................................
// add V<ssa_3>.4S, v31.4S, v30.4S // .*...............................................................
// aesr V<ssa_1>.16b, v18.16b // ..*..............................................................
// rev32 V<ssa_6>.16B, V<ssa_3>.16B // ...*.............................................................
// aesr V<ssa_1>.16b, v19.16b // ....*............................................................
// aesr V<ssa_6>.16b, v18.16b // .....*...........................................................
// aesr V<ssa_1>.16b, v20.16b // ......*..........................................................
// aesr V<ssa_6>.16b, v19.16b // .......*.........................................................
// aesr V<ssa_1>.16b, v21.16b // ........*........................................................
// aesr V<ssa_6>.16b, v20.16b // .........*.......................................................
// aesr V<ssa_1>.16b, v22.16b // ..........*......................................................
// aesr V<ssa_6>.16b, v21.16b // ...........*.....................................................
// aesr V<ssa_6>.16b, v22.16b // .............*...................................................
// ldr Q<ssa_16>, [x0, #16] // ....*............................................................
// aesr V<ssa_1>.16b, v23.16b // ............*....................................................
// aesr V<ssa_6>.16b, v23.16b // ...............*.................................................
// rev64 v29.16B, V<ssa_16>.16B // ........*........................................................
// aesr V<ssa_6>.16b, v24.16b // .................*...............................................
// add V<ssa_23>.4S, V<ssa_3>.4S, v30.4S // ...*.............................................................
// aesr V<ssa_1>.16b, v24.16b // ..............*..................................................
// aesr V<ssa_6>.16b, v25.16b // ...................*.............................................
// add V<ssa_25>.4S, V<ssa_23>.4S, v30.4S // .....*...........................................................
// aesr V<ssa_1>.16b, v25.16b // ................*................................................
// aesr V<ssa_6>.16b, v26.16b // .....................*...........................................
// aesr V<ssa_1>.16b, v26.16b // ..................*..............................................
// aesr V<ssa_6>.16b, v27.16b // .......................*.........................................
// ldr Q<ssa_30>, [x0], #(4*16) // ..*..............................................................
// aesr V<ssa_6>.16b, v28.16b // .........................*.......................................
// rev32 V<ssa_31>.16B, V<ssa_25>.16B // .......*.........................................................
// aesr V<ssa_1>.16b, v27.16b // ....................*............................................
// aesr V<ssa_6>.16b, v15.16b // ...........................*.....................................
// aesr V<ssa_1>.16b, v28.16b // ......................*..........................................
// aesr V<ssa_6>.16b, v16.16b // .............................*...................................
// rev64 V<ssa_34>.16B, V<ssa_30>.16B // .........*.......................................................
// aesr V<ssa_1>.16b, v15.16b // ........................*........................................
// aese V<ssa_6>.16b, v17.16b // ...............................*.................................
// aesr V<ssa_31>.16b, v18.16b // ..........................*......................................
// eor V<ssa_38>.16B, V<ssa_6>.16B, v2.16B // .................................*...............................
// aesr V<ssa_1>.16b, v16.16b // ................................*................................
// eor V<ssa_40>.16B, V<ssa_38>.16B, V<ssa_16>.16B // ...................................*.............................
// aesr V<ssa_31>.16b, v19.16b // ............................*....................................
// rev32 v9.16B, V<ssa_23>.16B // ....................................*............................
// aese V<ssa_1>.16b, v17.16b // ..................................*..............................
// aesr V<ssa_31>.16b, v20.16b // ..............................*..................................
// aesr v9.16b, v18.16b // ......................................*..........................
// eor V<ssa_42>.16B, V<ssa_1>.16B, v2.16B // .....................................*...........................
// ldr q12, [x0, #-32] // .*...............................................................
// aesr V<ssa_31>.16b, v21.16b // .................................*...............................
// aesr v9.16b, v19.16b // ........................................*........................
// add v31.4S, V<ssa_25>.4S, v30.4S // ......................................*..........................
// aesr V<ssa_31>.16b, v22.16b // ...................................*.............................
// eor V<ssa_44>.16B, V<ssa_42>.16B, V<ssa_30>.16B // .......................................*.........................
// aesr v9.16b, v20.16b // ..........................................*......................
// aesr V<ssa_31>.16b, v23.16b // .....................................*...........................
// ldr q4, [x6, #48] // *................................................................
// str Q<ssa_44>, [x2], #(4*16) // .........................................*.......................
// aesr v9.16b, v21.16b // ............................................*....................
// aesr V<ssa_31>.16b, v24.16b // .......................................*.........................
// ldr Q<ssa_45>, [x6, #80] // ..........................................*......................
// eor V<ssa_46>.16B, V<ssa_34>.16B, v11.16B // ............................................*....................
// aesr v9.16b, v22.16b // ..............................................*..................
// str Q<ssa_40>, [x2, #-48] // ...........................................*.....................
// aesr V<ssa_31>.16b, v25.16b // .........................................*.......................
// ext V<ssa_47>.16B, V<ssa_46>.16B, V<ssa_46>.16B, #8 // ..............................................*..................
// pmull2 V<ssa_48>.1q, v29.2d, v4.2d // ....................................*............................
// aesr V<ssa_31>.16b, v26.16b // ...........................................*.....................
// ldr q6, [x6, #64] // ..................................................*..............
// eor V<ssa_49>.16B, V<ssa_47>.16B, V<ssa_46>.16B // ....................................................*............
// aesr v9.16b, v23.16b // ................................................*................
// aesr V<ssa_31>.16b, v27.16b // .............................................*...................
// aesr v9.16b, v24.16b // ..................................................*..............
// pmull2 V<ssa_50>.1q, V<ssa_46>.2d, V<ssa_45>.2d // ...............................................*.................
// aesr V<ssa_31>.16b, v28.16b // .................................................*...............
// aesr v9.16b, v25.16b // ....................................................*............
// eor V<ssa_51>.16B, V<ssa_50>.16B, V<ssa_48>.16B // .................................................*...............
// aesr V<ssa_31>.16b, v15.16b // ...................................................*.............
// ldr Q<ssa_52>, [x6, #32] // .....................................................*...........
// aesr v9.16b, v26.16b // ......................................................*..........
// rev64 V<ssa_53>.16B, v12.16B // ...................................................*.............
// aesr V<ssa_31>.16b, v16.16b // .....................................................*...........
// aesr v9.16b, v27.16b // ........................................................*........
// aese V<ssa_31>.16b, v17.16b // .......................................................*.........
// pmull2 V<ssa_54>.1q, V<ssa_53>.2d, V<ssa_52>.2d // .........................................................*.......
// aesr v9.16b, v28.16b // ..........................................................*......
// eor v13.16B, V<ssa_31>.16B, v2.16B // .........................................................*.......
// eor v1.16B, V<ssa_51>.16B, V<ssa_54>.16B // ...........................................................*.....
// pmull v3.1q, V<ssa_46>.1d, V<ssa_45>.1d // ...........................................................*.....
// aesr v9.16b, v15.16b // ............................................................*....
// mov D<ssa_55>, v29.d[1] // ............................................................*....
// pmull v8.1q, V<ssa_53>.1d, V<ssa_52>.1d // ...............................................................*.
// ext V<ssa_56>.16B, V<ssa_53>.16B, V<ssa_53>.16B, #8 // .............................................................*...
// eor v5.8B, V<ssa_55>.8B, v29.8B // ..............................................................*..
// aesr v9.16b, v16.16b // ..............................................................*..
// pmull2 v0.1q, V<ssa_49>.2d, v6.2d // .............................................................*...
// eor v10.16B, V<ssa_56>.16B, V<ssa_53>.16B // ...............................................................*.
// aese v9.16b, v17.16b // ................................................................*
// ldr q11, [x6, #16] // ................................................................*
sub count, count, #1
cbz count, Lloop_unrolled_start_postamble
Lloop_unrolled_start:
// Instructions: 131
// Expected cycles: 70
// Expected IPC: 1.87
//
// Cycle bound: 70.0
// IPC bound: 1.87
//
// Wall time: 14848.71s
// User time: 14848.71s
//
// ------------------------- cycle (expected) -------------------------->
// 0 25 50
// |------------------------|------------------------|-------------------
pmull v29.1q, v29.1d, v4.1d // *.....................................................................
rev32 v4.16B, v31.16B // e.....................................................................
pmull v5.1q, v5.1d, v6.1d // .*....................................................................
add v6.4S, v31.4S, v30.4S // .e....................................................................
aesr v4.16b, v18.16b // ..e...................................................................
eor v14.16B, v9.16B, v2.16B // ..*...................................................................
pmull2 v9.1q, v10.2d, v11.2d // ...*..................................................................
rev32 v10.16B, v6.16B // ...e..................................................................
aesr v4.16b, v19.16b // ....e.................................................................
ldr q31, [x0, #-16] // ....*.................................................................
aesr v10.16b, v18.16b // .....e................................................................
eor v14.16B, v14.16B, v12.16B // .....*................................................................
eor v12.16B, v0.16B, v5.16B // ......*...............................................................
aesr v4.16b, v20.16b // ......e...............................................................
aesr v10.16b, v19.16b // .......e..............................................................
ldr q5, [x6] // .......*..............................................................
aesr v4.16b, v21.16b // ........e.............................................................
eor v13.16B, v13.16B, v31.16B // ........*.............................................................
eor v29.16B, v3.16B, v29.16B // .........*............................................................
aesr v10.16b, v20.16b // .........e............................................................
aesr v4.16b, v22.16b // ..........e...........................................................
rev64 v3.16B, v31.16B // ..........*...........................................................
str q13, [x2, #-16] // ...........*..........................................................
aesr v10.16b, v21.16b // ...........e..........................................................
eor v9.16B, v12.16B, v9.16B // ............*.........................................................
pmull2 v0.1q, v3.2d, v5.2d // ............*.........................................................
aesr v10.16b, v22.16b // .............e........................................................
ldr q12, [x0, #16] // .............e........................................................
eor v31.16B, v29.16B, v8.16B // ..............*.......................................................
pmull v5.1q, v3.1d, v5.1d // ..............*.......................................................
eor v13.16B, v1.16B, v0.16B // ...............*......................................................
aesr v4.16b, v23.16b // ...............e......................................................
eor v5.16B, v31.16B, v5.16B // ................*.....................................................
aesr v10.16b, v23.16b // ................e.....................................................
rev64 v29.16B, v12.16B // .................e....................................................
pmull v1.1q, v13.1d, v7.1d // .................*....................................................
aesr v10.16b, v24.16b // ..................e...................................................
mov d8, v3.d[1] // ..................*...................................................
add v31.4S, v6.4S, v30.4S // ...................e..................................................
aesr v4.16b, v24.16b // ...................e..................................................
aesr v10.16b, v25.16b // ....................e.................................................
ext v6.16B, v13.16B, v13.16B, #8 // ....................*.................................................
add v0.4S, v31.4S, v30.4S // .....................e................................................
aesr v4.16b, v25.16b // .....................e................................................
eor v6.16B, v6.16B, v1.16B // ......................*...............................................
aesr v10.16b, v26.16b // ......................e...............................................
eor v1.8B, v8.8B, v3.8B // .......................*..............................................
aesr v4.16b, v26.16b // .......................e..............................................
eor v13.16B, v5.16B, v13.16B // ........................*.............................................
aesr v10.16b, v27.16b // ........................e.............................................
pmull v3.1q, v1.1d, v11.1d // .........................*............................................
ldr q8, [x0], #(4*16) // .........................e............................................
aesr v10.16b, v28.16b // ..........................e...........................................
rev32 v1.16B, v0.16B // ..........................e...........................................
eor v9.16B, v9.16B, v3.16B // ...........................*..........................................
aesr v4.16b, v27.16b // ...........................e..........................................
aesr v10.16b, v15.16b // ............................e.........................................
eor v9.16B, v9.16B, v13.16B // .............................*........................................
aesr v4.16b, v28.16b // .............................e........................................
aesr v10.16b, v16.16b // ..............................e.......................................
rev64 v13.16B, v8.16B // ..............................e.......................................
eor v11.16B, v9.16B, v6.16B // ...............................*......................................
aesr v4.16b, v15.16b // ...............................e......................................
aese v10.16b, v17.16b // ................................e.....................................
str q14, [x2, #-32] // ................................*.....................................
pmull v6.1q, v11.1d, v7.1d // .................................*....................................
ext v11.16B, v11.16B, v11.16B, #8 // .................................*....................................
aesr v1.16b, v18.16b // ..................................e...................................
eor v9.16B, v10.16B, v2.16B // ..................................e...................................
eor v3.16B, v5.16B, v6.16B // ...................................*..................................
aesr v4.16b, v16.16b // ...................................e..................................
eor v14.16B, v9.16B, v12.16B // ....................................e.................................
aesr v1.16b, v19.16b // ....................................e.................................
rev32 v9.16B, v31.16B // .....................................e................................
aese v4.16b, v17.16b // .....................................e................................
eor v11.16B, v3.16B, v11.16B // ......................................*...............................
aesr v1.16b, v20.16b // ......................................e...............................
aesr v9.16b, v18.16b // .......................................e..............................
eor v10.16B, v4.16B, v2.16B // .......................................e..............................
ldr q12, [x0, #-32] // ........................................e.............................
aesr v1.16b, v21.16b // ........................................e.............................
ext v11.16B, v11.16B, v11.16B, #8 // .........................................*............................
aesr v9.16b, v19.16b // .........................................e............................
add v31.4S, v0.4S, v30.4S // ..........................................e...........................
aesr v1.16b, v22.16b // ..........................................e...........................
eor v0.16B, v10.16B, v8.16B // ...........................................e..........................
aesr v9.16b, v20.16b // ...........................................e..........................
aesr v1.16b, v23.16b // ............................................e.........................
ldr q4, [x6, #48] // ............................................e.........................
str q0, [x2], #(4*16) // .............................................e........................
aesr v9.16b, v21.16b // .............................................e........................
aesr v1.16b, v24.16b // ..............................................e.......................
ldr q8, [x6, #80] // ..............................................e.......................
eor v5.16B, v13.16B, v11.16B // ...............................................e......................
aesr v9.16b, v22.16b // ...............................................e......................
str q14, [x2, #-48] // ................................................e.....................
aesr v1.16b, v25.16b // ................................................e.....................
ext v13.16B, v5.16B, v5.16B, #8 // .................................................e....................
pmull2 v10.1q, v29.2d, v4.2d // .................................................e....................
aesr v1.16b, v26.16b // ..................................................e...................
ldr q6, [x6, #64] // ..................................................e...................
eor v0.16B, v13.16B, v5.16B // ...................................................e..................
aesr v9.16b, v23.16b // ...................................................e..................
aesr v1.16b, v27.16b // ....................................................e.................
aesr v9.16b, v24.16b // .....................................................e................
pmull2 v11.1q, v5.2d, v8.2d // ......................................................e...............
aesr v1.16b, v28.16b // .......................................................e..............
aesr v9.16b, v25.16b // ........................................................e.............
eor v14.16B, v11.16B, v10.16B // ........................................................e.............
aesr v1.16b, v15.16b // .........................................................e............
ldr q11, [x6, #32] // .........................................................e............
aesr v9.16b, v26.16b // ..........................................................e...........
rev64 v10.16B, v12.16B // ..........................................................e...........
aesr v1.16b, v16.16b // ...........................................................e..........
aesr v9.16b, v27.16b // ............................................................e.........
aese v1.16b, v17.16b // .............................................................e........
pmull2 v3.1q, v10.2d, v11.2d // ..............................................................e.......
aesr v9.16b, v28.16b // ...............................................................e......
eor v13.16B, v1.16B, v2.16B // ...............................................................e......
eor v1.16B, v14.16B, v3.16B // ................................................................e.....
pmull v3.1q, v5.1d, v8.1d // ................................................................e.....
aesr v9.16b, v15.16b // .................................................................e....
mov d5, v29.d[1] // .................................................................e....
pmull v8.1q, v10.1d, v11.1d // ..................................................................e...
ext v11.16B, v10.16B, v10.16B, #8 // ..................................................................e...
eor v5.8B, v5.8B, v29.8B // ...................................................................e..
aesr v9.16b, v16.16b // ...................................................................e..
pmull2 v0.1q, v0.2d, v6.2d // ....................................................................e.
eor v10.16B, v11.16B, v10.16B // ....................................................................e.
aese v9.16b, v17.16b // .....................................................................e
ldr q11, [x6, #16] // .....................................................................e
// ---------------------------------------------- cycle (expected) ----------------------------------------------->
// 0 25 50 75 100
// |------------------------|------------------------|------------------------|------------------------|-----------
// ldr q29, [x0], #(4*16) // .........................e............................................'........................~................
// rev32 v0.16b, v31.16b // e.....................................................................~.........................................
// add v31.4s, v31.4s, v30.4s // .e....................................................................'~........................................
// aesr v0.16b, v18.16b // ..e...................................................................'.~.......................................
// aesr v0.16b, v19.16b // ....e.................................................................'...~.....................................
// aesr v0.16b, v20.16b // ......e...............................................................'.....~...................................
// aesr v0.16b, v21.16b // ........e.............................................................'.......~.................................
// aesr v0.16b, v22.16b // ..........e...........................................................'.........~...............................
// aesr v0.16b, v23.16b // ...............e......................................................'..............~..........................
// aesr v0.16b, v24.16b // ...................e..................................................'..................~......................
// aesr v0.16b, v25.16b // .....................e................................................'....................~....................
// aesr v0.16b, v26.16b // .......................e..............................................'......................~..................
// aesr v0.16b, v27.16b // ...........................e..........................................'..........................~..............
// aesr v0.16b, v28.16b // .............................e........................................'............................~............
// aesr v0.16b, v15.16b // ...............................e......................................'..............................~..........
// aesr v0.16b, v16.16b // ...................................e..................................'..................................~......
// aese v0.16b, v17.16b // .....................................e................................'....................................~....
// eor v0.16b, v0.16b, v2.16b // .......................................e..............................'......................................~..
// eor v0.16b, v0.16b, v29.16b // ...........................................e..........................'.........................................
// str q0, [x2], #(4*16) // .............................................e........................'.........................................
// ldr q12, [x6, #48] // ............................................e.........................'.........................................
// ldr q13, [x6, #80] // ..............................................e.......................'.........................................
// ldr q14, [x6, #64] // ..................................................e...................'.........................................
// rev64 v29.16b, v29.16b // ..............................e.......................................'.............................~...........
// eor v29.16b, v29.16b, v11.16b // ...............................................e......................'.........................................
// pmull v8.1q, v29.1d, v13.1d // ................................................................e.....'.........................................
// pmull2 v9.1q, v29.2d, v13.2d // ......................................................e...............'.........................................
// ext v11.16b, v29.16b, v29.16b, #8 // .................................................e....................'.........................................
// eor v11.16b, v11.16b, v29.16b // ...................................................e..................'.........................................
// pmull2 v10.1q, v11.2d, v14.2d // ....................................................................e.'.........................................
// ldr q29, [x0, #(-3*16)] // .............e........................................................'............~............................
// rev32 v0.16b, v31.16b // ...e..................................................................'..~......................................
// add v31.4s, v31.4s, v30.4s // ...................e..................................................'..................~......................
// aesr v0.16b, v18.16b // .....e................................................................'....~....................................
// aesr v0.16b, v19.16b // .......e..............................................................'......~..................................
// aesr v0.16b, v20.16b // .........e............................................................'........~................................
// aesr v0.16b, v21.16b // ...........e..........................................................'..........~..............................
// aesr v0.16b, v22.16b // .............e........................................................'............~............................
// aesr v0.16b, v23.16b // ................e.....................................................'...............~.........................
// aesr v0.16b, v24.16b // ..................e...................................................'.................~.......................
// aesr v0.16b, v25.16b // ....................e.................................................'...................~.....................
// aesr v0.16b, v26.16b // ......................e...............................................'.....................~...................
// aesr v0.16b, v27.16b // ........................e.............................................'.......................~.................
// aesr v0.16b, v28.16b // ..........................e...........................................'.........................~...............
// aesr v0.16b, v15.16b // ............................e.........................................'...........................~.............
// aesr v0.16b, v16.16b // ..............................e.......................................'.............................~...........
// aese v0.16b, v17.16b // ................................e.....................................'...............................~.........
// eor v0.16b, v0.16b, v2.16b // ..................................e...................................'.................................~.......
// eor v0.16b, v0.16b, v29.16b // ....................................e.................................'...................................~.....
// str q0, [x2, #(-3*16)] // ................................................e.....................'.........................................
// rev64 v29.16b, v29.16b // .................e....................................................'................~........................
// pmull v11.1q, v29.1d, v12.1d // ~.....................................................................*.........................................
// eor v8.16b, v8.16b, v11.16b // .........~............................................................'........*................................
// pmull2 v11.1q, v29.2d, v12.2d // .................................................e....................'.........................................
// eor v9.16b, v9.16b, v11.16b // ........................................................e.............'.........................................
// mov d11, v29.d[1] // .................................................................e....'.........................................
// eor v11.8b, v11.8b, v29.8b // ...................................................................e..'.........................................
// pmull v11.1q, v11.1d, v14.1d // .~....................................................................'*........................................
// eor v10.16b, v10.16b, v11.16b // ......~...............................................................'.....*...................................
// ldr q29, [x0, #(-2*16)] // ........................................e.............................'.......................................~.
// rev32 v0.16b, v31.16b // .....................................e................................'....................................~....
// add v31.4s, v31.4s, v30.4s // .....................e................................................'....................~....................
// aesr v0.16b, v18.16b // .......................................e..............................'......................................~..
// aesr v0.16b, v19.16b // .........................................e............................'.........................................
// aesr v0.16b, v20.16b // ...........................................e..........................'.........................................
// aesr v0.16b, v21.16b // .............................................e........................'.........................................
// aesr v0.16b, v22.16b // ...............................................e......................'.........................................
// aesr v0.16b, v23.16b // ...................................................e..................'.........................................
// aesr v0.16b, v24.16b // .....................................................e................'.........................................
// aesr v0.16b, v25.16b // ........................................................e.............'.........................................
// aesr v0.16b, v26.16b // ..........................................................e...........'.........................................
// aesr v0.16b, v27.16b // ............................................................e.........'.........................................
// aesr v0.16b, v28.16b // ...............................................................e......'.........................................
// aesr v0.16b, v15.16b // .................................................................e....'.........................................
// aesr v0.16b, v16.16b // ...................................................................e..'.........................................
// aese v0.16b, v17.16b // .....................................................................e'.........................................
// eor v0.16b, v0.16b, v2.16b // ..~...................................................................'.*.......................................
// eor v0.16b, v0.16b, v29.16b // .....~................................................................'....*....................................
// str q0, [x2, #(-2*16)] // ................................~.....................................'...............................*.........
// ldr q12, [x6] // .......~..............................................................'......*..................................
// ldr q13, [x6, #32] // .........................................................e............'.........................................
// ldr q14, [x6, #16] // .....................................................................e'.........................................
// rev64 v29.16b, v29.16b // ..........................................................e...........'.........................................
// pmull v11.1q, v29.1d, v13.1d // ..................................................................e...'.........................................
// eor v8.16b, v8.16b, v11.16b // ..............~.......................................................'.............*...........................
// pmull2 v11.1q, v29.2d, v13.2d // ..............................................................e.......'.........................................
// eor v9.16b, v9.16b, v11.16b // ................................................................e.....'.........................................
// ext v11.16b, v29.16b, v29.16b, #8 // ..................................................................e...'.........................................
// eor v11.16b, v11.16b, v29.16b // ....................................................................e.'.........................................
// pmull2 v11.1q, v11.2d, v14.2d // ...~..................................................................'..*......................................
// eor v10.16b, v10.16b, v11.16b // ............~.........................................................'...........*.............................
// ldr q29, [x0, #(-1*16)] // ....~.................................................................'...*.....................................
// rev32 v0.16b, v31.16b // ..........................e...........................................'.........................~...............
// add v31.4s, v31.4s, v30.4s // ..........................................e...........................'.........................................
// aesr v0.16b, v18.16b // ..................................e...................................'.................................~.......
// aesr v0.16b, v19.16b // ....................................e.................................'...................................~.....
// aesr v0.16b, v20.16b // ......................................e...............................'.....................................~...
// aesr v0.16b, v21.16b // ........................................e.............................'.......................................~.
// aesr v0.16b, v22.16b // ..........................................e...........................'.........................................
// aesr v0.16b, v23.16b // ............................................e.........................'.........................................
// aesr v0.16b, v24.16b // ..............................................e.......................'.........................................
// aesr v0.16b, v25.16b // ................................................e.....................'.........................................
// aesr v0.16b, v26.16b // ..................................................e...................'.........................................
// aesr v0.16b, v27.16b // ....................................................e.................'.........................................
// aesr v0.16b, v28.16b // .......................................................e..............'.........................................
// aesr v0.16b, v15.16b // .........................................................e............'.........................................
// aesr v0.16b, v16.16b // ...........................................................e..........'.........................................
// aese v0.16b, v17.16b // .............................................................e........'.........................................
// eor v0.16b, v0.16b, v2.16b // ...............................................................e......'.........................................
// eor v0.16b, v0.16b, v29.16b // ........~.............................................................'.......*.................................
// str q0, [x2, #(-1*16)] // ...........~..........................................................'..........*..............................
// rev64 v29.16b, v29.16b // ..........~...........................................................'.........*...............................
// pmull v11.1q, v29.1d, v12.1d // ..............~.......................................................'.............*...........................
// eor v8.16b, v8.16b, v11.16b // ................~.....................................................'...............*.........................
// pmull2 v11.1q, v29.2d, v12.2d // ............~.........................................................'...........*.............................
// eor v9.16b, v9.16b, v11.16b // ...............~......................................................'..............*..........................
// mov d11, v29.d[1] // ..................~...................................................'.................*.......................
// eor v11.8b, v11.8b, v29.8b // .......................~..............................................'......................*..................
// pmull v11.1q, v11.1d, v14.1d // .........................~............................................'........................*................
// eor v10.16b, v10.16b, v11.16b // ...........................~..........................................'..........................*..............
// eor v0.16b, v8.16b, v9.16b // ........................~.............................................'.......................*.................
// pmull v1.1q, v9.1d, v7.1d // .................~....................................................'................*........................
// ext v9.16b, v9.16b, v9.16b, #8 // ....................~.................................................'...................*.....................
// eor v10.16b, v10.16b, v0.16b // .............................~........................................'............................*............
// eor v1.16b, v9.16b, v1.16b // ......................~...............................................'.....................*...................
// eor v10.16b, v10.16b, v1.16b // ...............................~......................................'..............................*..........
// pmull v9.1q, v10.1d, v7.1d // .................................~....................................'................................*........
// eor v8.16b, v8.16b, v9.16b // ...................................~..................................'..................................*......
// ext v10.16b, v10.16b, v10.16b, #8 // .................................~....................................'................................*........
// eor v11.16b, v8.16b, v10.16b // ......................................~...............................'.....................................*...
// ext v11.16b, v11.16b, v11.16b, #8 // .........................................~............................'........................................*
sub count, count, #1
cbnz count, Lloop_unrolled_start
Lloop_unrolled_start_postamble:// end of loop kernel
// Instructions: 34
// Expected cycles: 25
// Expected IPC: 1.36
//
// Cycle bound: 25.0
// IPC bound: 1.36
//
// Wall time: 0.37s
// User time: 0.37s
//
// ----- cycle (expected) ------>
// 0 25
// |------------------------|----
pmull v4.1q, v29.1d, v4.1d // *.............................
ldr q29, [x0, #-16] // *.............................
pmull v14.1q, v5.1d, v6.1d // .*............................
ldr q6, [x6] // .*............................
eor v4.16B, v3.16B, v4.16B // ..*...........................
eor v3.16B, v0.16B, v14.16B // ...*..........................
pmull2 v14.1q, v10.2d, v11.2d // ...*..........................
eor v0.16B, v4.16B, v8.16B // ....*.........................
rev64 v10.16B, v29.16B // ....*.........................
eor v4.16B, v3.16B, v14.16B // .....*........................
eor v14.16B, v9.16B, v2.16B // .....*........................
pmull2 v8.1q, v10.2d, v6.2d // ......*.......................
mov d3, v10.d[1] // ......*.......................
pmull v9.1q, v10.1d, v6.1d // .......*......................
eor v8.16B, v1.16B, v8.16B // ........*.....................
eor v1.8B, v3.8B, v10.8B // ........*.....................
eor v3.16B, v0.16B, v9.16B // .........*....................
pmull v0.1q, v1.1d, v11.1d // ..........*...................
ext v9.16B, v8.16B, v8.16B, #8 // ...........*..................
eor v10.16B, v3.16B, v8.16B // ...........*..................
eor v4.16B, v4.16B, v0.16B // ............*.................
pmull v8.1q, v8.1d, v7.1d // ............*.................
eor v11.16B, v14.16B, v12.16B // .............*................
eor v14.16B, v9.16B, v8.16B // ..............*...............
eor v8.16B, v4.16B, v10.16B // ..............*...............
str q11, [x2, #-32] // ...............*..............
eor v11.16B, v13.16B, v29.16B // ...............*..............
eor v13.16B, v8.16B, v14.16B // ................*.............
str q11, [x2, #-16] // .................*............
pmull v10.1q, v13.1d, v7.1d // ..................*...........
ext v5.16B, v13.16B, v13.16B, #8 // ...................*..........
eor v6.16B, v3.16B, v10.16B // ....................*.........
eor v8.16B, v6.16B, v5.16B // ......................*.......
ext v11.16B, v8.16B, v8.16B, #8 // ........................*.....
// ------ cycle (expected) ------>
// 0 25
// |------------------------|-----
// pmull V<ssa_0>.1q, v29.1d, v4.1d // *..............................
// pmull V<ssa_2>.1q, v5.1d, v6.1d // .*.............................
// eor V<ssa_4>.16B, v9.16B, v2.16B // .....*.........................
// pmull2 V<ssa_5>.1q, v10.2d, v11.2d // ...*...........................
// ldr Q<ssa_7>, [x0, #-16] // *..............................
// eor V<ssa_8>.16B, V<ssa_4>.16B, v12.16B // .............*.................
// eor V<ssa_9>.16B, v0.16B, V<ssa_2>.16B // ...*...........................
// ldr Q<ssa_10>, [x6] // .*.............................
// eor V<ssa_11>.16B, v13.16B, V<ssa_7>.16B // ...............*...............
// eor V<ssa_12>.16B, v3.16B, V<ssa_0>.16B // ..*............................
// rev64 V<ssa_13>.16B, V<ssa_7>.16B // ....*..........................
// str Q<ssa_11>, [x2, #-16] // .................*.............
// eor V<ssa_14>.16B, V<ssa_9>.16B, V<ssa_5>.16B // .....*.........................
// pmull2 V<ssa_15>.1q, V<ssa_13>.2d, V<ssa_10>.2d // ......*........................
// eor V<ssa_17>.16B, V<ssa_12>.16B, v8.16B // ....*..........................
// pmull V<ssa_18>.1q, V<ssa_13>.1d, V<ssa_10>.1d // .......*.......................
// eor V<ssa_19>.16B, v1.16B, V<ssa_15>.16B // ........*......................
// eor V<ssa_20>.16B, V<ssa_17>.16B, V<ssa_18>.16B // .........*.....................
// pmull V<ssa_21>.1q, V<ssa_19>.1d, v7.1d // ............*..................
// mov D<ssa_22>, V<ssa_13>.d[1] // ......*........................
// ext V<ssa_24>.16B, V<ssa_19>.16B, V<ssa_19>.16B, #8 // ...........*...................
// eor V<ssa_26>.16B, V<ssa_24>.16B, V<ssa_21>.16B // ..............*................
// eor V<ssa_27>.8B, V<ssa_22>.8B, V<ssa_13>.8B // ........*......................
// eor V<ssa_28>.16B, V<ssa_20>.16B, V<ssa_19>.16B // ...........*...................
// pmull V<ssa_29>.1q, V<ssa_27>.1d, v11.1d // ..........*....................
// eor V<ssa_32>.16B, V<ssa_14>.16B, V<ssa_29>.16B // ............*..................
// eor V<ssa_33>.16B, V<ssa_32>.16B, V<ssa_28>.16B // ..............*................
// eor V<ssa_35>.16B, V<ssa_33>.16B, V<ssa_26>.16B // ................*..............
// str Q<ssa_8>, [x2, #-32] // ...............*...............
// pmull V<ssa_36>.1q, V<ssa_35>.1d, v7.1d // ..................*............
// ext V<ssa_37>.16B, V<ssa_35>.16B, V<ssa_35>.16B, #8 // ...................*...........
// eor V<ssa_39>.16B, V<ssa_20>.16B, V<ssa_36>.16B // ....................*..........
// eor V<ssa_41>.16B, V<ssa_39>.16B, V<ssa_37>.16B // ......................*........
// ext v11.16B, V<ssa_41>.16B, V<ssa_41>.16B, #8 // ........................*......
b Lloop_unrolled_start_end
Lloop_unrolled_start_end:
Lloop_unrolled_end:
load_htable_12
cbz remainder, Lloop_1x_end
Lloop_1x_start:
ldr plain_q, [input], #16
aes_full_block aes_st, plain, res
str res_q, [output], #16
ghash_init_0 plain, Ht1, Ht12, tag
ghash_finalize tag
sub remainder, remainder, #1
cbnz remainder, Lloop_1x_start
Lloop_1x_end:
// Return number of bytes processed
mov x0, byte_len
// Store new authentication tag
rev64 tag.16b, tag.16b
str tag_q, [tag_ptr]
// Store updated counter
rev32 rtmp_ctr.16b, rtmp_ctr.16b
str rtmp_ctr_q, [ivec]
restore_vregs
restore_gprs
Ldec_postamble_end:
add sp, sp, #STACK_SIZE
ret
#endif
#endif // !OPENSSL_NO_ASM && defined(__AARCH64EL__) && defined(__APPLE__)
#if defined(__ELF__)
// See https: // www.airs.com/blog/archives/518.
.section .note.GNU-stack,"",%progbits
#endif