Source code
Revision control
Copy as Markdown
Other Tools
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#ifdef FREEBL_NO_DEPEND
#include "stubs.h"
#endif
#include "blapii.h"
#include "mpi.h"
#include "secerr.h"
#include "prtypes.h"
#include "prinit.h"
#include "prenv.h"
#if defined(_MSC_VER) && !defined(_M_IX86)
#include <intrin.h> /* for _xgetbv() */
#endif
#if defined(_WIN64) && defined(__aarch64__)
#include <windows.h>
#endif
#if defined(DARWIN)
#include <TargetConditionals.h>
#endif
static PRCallOnceType coFreeblInit;
/* State variables. */
static PRBool aesni_support_ = PR_FALSE;
static PRBool clmul_support_ = PR_FALSE;
static PRBool sha_support_ = PR_FALSE;
static PRBool avx_support_ = PR_FALSE;
static PRBool avx2_support_ = PR_FALSE;
static PRBool adx_support_ = PR_FALSE;
static PRBool ssse3_support_ = PR_FALSE;
static PRBool sse4_1_support_ = PR_FALSE;
static PRBool sse4_2_support_ = PR_FALSE;
static PRBool arm_neon_support_ = PR_FALSE;
static PRBool arm_aes_support_ = PR_FALSE;
static PRBool arm_sha1_support_ = PR_FALSE;
static PRBool arm_sha2_support_ = PR_FALSE;
static PRBool arm_pmull_support_ = PR_FALSE;
static PRBool ppc_crypto_support_ = PR_FALSE;
#ifdef NSS_X86_OR_X64
/*
* Adapted from the example code in "How to detect New Instruction support in
* the 4th generation Intel Core processor family" by Max Locktyukhin.
*
* XGETBV:
* Reads an extended control register (XCR) specified by ECX into EDX:EAX.
*/
static PRBool
check_xcr0_ymm()
{
PRUint32 xcr0;
#if defined(_MSC_VER)
#if defined(_M_IX86)
__asm {
mov ecx, 0
xgetbv
mov xcr0, eax
}
#else
xcr0 = (PRUint32)_xgetbv(0); /* Requires VS2010 SP1 or later. */
#endif /* _M_IX86 */
#else /* _MSC_VER */
/* Old OSX compilers don't support xgetbv. Use byte form. */
__asm__(".byte 0x0F, 0x01, 0xd0"
: "=a"(xcr0)
: "c"(0)
: "%edx");
#endif /* _MSC_VER */
/* Check if xmm and ymm state are enabled in XCR0. */
return (xcr0 & 6) == 6;
}
#define ECX_AESNI (1 << 25)
#define ECX_CLMUL (1 << 1)
#define ECX_XSAVE (1 << 26)
#define ECX_OSXSAVE (1 << 27)
#define ECX_AVX (1 << 28)
#define EBX_AVX2 (1 << 5)
#define EBX_ADX (1 << 19)
#define EBX_BMI1 (1 << 3)
#define EBX_BMI2 (1 << 8)
#define EBX_SHA (1 << 29)
#define ECX_FMA (1 << 12)
#define ECX_MOVBE (1 << 22)
#define ECX_SSSE3 (1 << 9)
#define ECX_SSE4_1 (1 << 19)
#define ECX_SSE4_2 (1 << 20)
#define AVX_BITS (ECX_XSAVE | ECX_OSXSAVE | ECX_AVX)
#define AVX2_EBX_BITS (EBX_AVX2 | EBX_BMI1 | EBX_BMI2)
#define AVX2_ECX_BITS (ECX_FMA | ECX_MOVBE)
void
CheckX86CPUSupport()
{
unsigned long eax, ebx, ecx, edx;
unsigned long eax7, ebx7, ecx7, edx7;
char *disable_hw_aes = PR_GetEnvSecure("NSS_DISABLE_HW_AES");
char *disable_pclmul = PR_GetEnvSecure("NSS_DISABLE_PCLMUL");
char *disable_hw_sha = PR_GetEnvSecure("NSS_DISABLE_HW_SHA");
char *disable_avx = PR_GetEnvSecure("NSS_DISABLE_AVX");
char *disable_avx2 = PR_GetEnvSecure("NSS_DISABLE_AVX2");
char *disable_adx = PR_GetEnvSecure("NSS_DISABLE_ADX");
char *disable_ssse3 = PR_GetEnvSecure("NSS_DISABLE_SSSE3");
char *disable_sse4_1 = PR_GetEnvSecure("NSS_DISABLE_SSE4_1");
char *disable_sse4_2 = PR_GetEnvSecure("NSS_DISABLE_SSE4_2");
freebl_cpuid(1, &eax, &ebx, &ecx, &edx);
freebl_cpuid(7, &eax7, &ebx7, &ecx7, &edx7);
aesni_support_ = (PRBool)((ecx & ECX_AESNI) != 0 && disable_hw_aes == NULL);
clmul_support_ = (PRBool)((ecx & ECX_CLMUL) != 0 && disable_pclmul == NULL);
sha_support_ = (PRBool)((ebx7 & EBX_SHA) != 0 && disable_hw_sha == NULL);
/* For AVX we ensure that:
* - The AVX, OSXSAVE, and XSAVE bits of ECX from CPUID(EAX=1) are set, and
* - the SSE and AVX state bits of XCR0 are set (check_xcr0_ymm).
*/
avx_support_ = (PRBool)((ecx & AVX_BITS) == AVX_BITS) && check_xcr0_ymm() &&
disable_avx == NULL;
/* For AVX2 we ensure that:
* - AVX is supported,
* - the AVX2, BMI1, and BMI2 bits of EBX from CPUID(EAX=7) are set, and
* - the FMA, and MOVBE bits of ECX from CPUID(EAX=1) are set.
* We do not check for LZCNT support.
*/
avx2_support_ = (PRBool)(avx_support_ == PR_TRUE &&
(ebx7 & AVX2_EBX_BITS) == AVX2_EBX_BITS &&
(ecx & AVX2_ECX_BITS) == AVX2_ECX_BITS &&
disable_avx2 == NULL);
/* CPUID.(EAX=07H, ECX=0H):EBX.ADX[bit 19]=1 indicates
the processor supports ADCX and ADOX instructions.*/
adx_support_ = (PRBool)((ebx7 & EBX_ADX) != 0 && disable_adx == NULL);
ssse3_support_ = (PRBool)((ecx & ECX_SSSE3) != 0 &&
disable_ssse3 == NULL);
sse4_1_support_ = (PRBool)((ecx & ECX_SSE4_1) != 0 &&
disable_sse4_1 == NULL);
sse4_2_support_ = (PRBool)((ecx & ECX_SSE4_2) != 0 &&
disable_sse4_2 == NULL);
}
#endif /* NSS_X86_OR_X64 */
/* clang-format off */
#if (defined(__aarch64__) || defined(__arm__)) && !defined(TARGET_OS_IPHONE)
#ifndef __has_include
#define __has_include(x) 0
#endif
#if (__has_include(<sys/auxv.h>) || defined(__linux__)) && \
defined(__GNUC__) && __GNUC__ >= 2 && defined(__ELF__)
/* This might be conflict with host compiler */
#if !defined(__ANDROID__)
#include <sys/auxv.h>
#endif
extern unsigned long getauxval(unsigned long type) __attribute__((weak));
#elif defined(__arm__) || (!defined(__OpenBSD__) && !defined(_WIN64))
static unsigned long (*getauxval)(unsigned long) = NULL;
#endif /* defined(__GNUC__) && __GNUC__ >= 2 && defined(__ELF__)*/
#if defined(__FreeBSD__) && !defined(__aarch64__) && __has_include(<sys/auxv.h>)
/* Avoid conflict with static declaration above */
#define getauxval freebl_getauxval
static unsigned long getauxval(unsigned long type)
{
/* Only AT_HWCAP* return unsigned long */
if (type != AT_HWCAP && type != AT_HWCAP2) {
return 0;
}
unsigned long ret = 0;
elf_aux_info(type, &ret, sizeof(ret));
return ret;
}
#endif
#ifndef AT_HWCAP2
#define AT_HWCAP2 26
#endif
#ifndef AT_HWCAP
#define AT_HWCAP 16
#endif
#endif /* defined(__aarch64__) || defined(__arm__) */
/* clang-format on */
#if defined(__aarch64__)
#if defined(__linux__)
// Defines from hwcap.h in Linux kernel - ARM64
#ifndef HWCAP_AES
#define HWCAP_AES (1 << 3)
#endif
#ifndef HWCAP_PMULL
#define HWCAP_PMULL (1 << 4)
#endif
#ifndef HWCAP_SHA1
#define HWCAP_SHA1 (1 << 5)
#endif
#ifndef HWCAP_SHA2
#define HWCAP_SHA2 (1 << 6)
#endif
#endif /* defined(__linux__) */
#if defined(__FreeBSD__)
#include <stdint.h>
#include <machine/armreg.h>
// Support for older version of armreg.h
#ifndef ID_AA64ISAR0_AES_VAL
#define ID_AA64ISAR0_AES_VAL ID_AA64ISAR0_AES
#endif
#ifndef ID_AA64ISAR0_SHA1_VAL
#define ID_AA64ISAR0_SHA1_VAL ID_AA64ISAR0_SHA1
#endif
#ifndef ID_AA64ISAR0_SHA2_VAL
#define ID_AA64ISAR0_SHA2_VAL ID_AA64ISAR0_SHA2
#endif
#endif /* defined(__FreeBSD__) */
#if defined(__OpenBSD__)
#include <sys/sysctl.h>
#include <machine/cpu.h>
#include <machine/armreg.h>
#endif /* defined(__OpenBSD__) */
void
CheckARMSupport()
{
#if defined(_WIN64)
BOOL arm_crypto_support = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE);
arm_aes_support_ = arm_crypto_support;
arm_pmull_support_ = arm_crypto_support;
arm_sha1_support_ = arm_crypto_support;
arm_sha2_support_ = arm_crypto_support;
#elif defined(__linux__)
if (getauxval) {
long hwcaps = getauxval(AT_HWCAP);
arm_aes_support_ = (hwcaps & HWCAP_AES) == HWCAP_AES;
arm_pmull_support_ = (hwcaps & HWCAP_PMULL) == HWCAP_PMULL;
arm_sha1_support_ = (hwcaps & HWCAP_SHA1) == HWCAP_SHA1;
arm_sha2_support_ = (hwcaps & HWCAP_SHA2) == HWCAP_SHA2;
}
#elif defined(__FreeBSD__)
/* qemu-user does not support register access from userspace */
if (PR_GetEnvSecure("QEMU_EMULATING") == NULL) {
uint64_t isar0 = READ_SPECIALREG(id_aa64isar0_el1);
arm_aes_support_ = ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_BASE;
arm_pmull_support_ = ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_PMULL;
arm_sha1_support_ = ID_AA64ISAR0_SHA1_VAL(isar0) >= ID_AA64ISAR0_SHA1_BASE;
arm_sha2_support_ = ID_AA64ISAR0_SHA2_VAL(isar0) >= ID_AA64ISAR0_SHA2_BASE;
}
#elif defined(__OpenBSD__)
const int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
uint64_t isar0;
size_t len = sizeof(isar0);
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) < 0)
return;
arm_aes_support_ = ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_BASE;
arm_pmull_support_ = ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_PMULL;
arm_sha1_support_ = ID_AA64ISAR0_SHA1(isar0) >= ID_AA64ISAR0_SHA1_BASE;
arm_sha2_support_ = ID_AA64ISAR0_SHA2(isar0) >= ID_AA64ISAR0_SHA2_BASE;
#elif defined(__ARM_FEATURE_CRYPTO)
/*
* Although no feature detection, default compiler option allows ARM
* Crypto Extension.
*/
arm_aes_support_ = PR_TRUE;
arm_pmull_support_ = PR_TRUE;
arm_sha1_support_ = PR_TRUE;
arm_sha2_support_ = PR_TRUE;
#endif
/* aarch64 must support NEON. */
arm_neon_support_ = PR_GetEnvSecure("NSS_DISABLE_ARM_NEON") == NULL;
arm_aes_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_AES") == NULL;
arm_pmull_support_ &= PR_GetEnvSecure("NSS_DISABLE_PMULL") == NULL;
arm_sha1_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA1") == NULL;
arm_sha2_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA2") == NULL;
}
#endif /* defined(__aarch64__) */
#if defined(__arm__)
// Defines from hwcap.h in Linux kernel - ARM
/*
* HWCAP flags - for elf_hwcap (in kernel) and AT_HWCAP
*/
#ifndef HWCAP_NEON
#define HWCAP_NEON (1 << 12)
#endif
/*
* HWCAP2 flags - for elf_hwcap2 (in kernel) and AT_HWCAP2
*/
#ifndef HWCAP2_AES
#define HWCAP2_AES (1 << 0)
#endif
#ifndef HWCAP2_PMULL
#define HWCAP2_PMULL (1 << 1)
#endif
#ifndef HWCAP2_SHA1
#define HWCAP2_SHA1 (1 << 2)
#endif
#ifndef HWCAP2_SHA2
#define HWCAP2_SHA2 (1 << 3)
#endif
PRBool
GetNeonSupport()
{
char *disable_arm_neon = PR_GetEnvSecure("NSS_DISABLE_ARM_NEON");
if (disable_arm_neon) {
return PR_FALSE;
}
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
// Compiler generates NEON instruction as default option.
// If no getauxval, compiler generate NEON instruction by default,
// we should allow NOEN support.
return PR_TRUE;
#elif !defined(__ANDROID__)
// Android's cpu-features.c detects features by the following logic
//
// - Call getauxval(AT_HWCAP)
// - Parse /proc/self/auxv if getauxval is nothing or returns 0
// - Parse /proc/cpuinfo if both cannot detect features
//
// But we don't use it for Android since Android document
// one problem with AT_HWCAP sometimes devices (Nexus 4 and emulator)
// are mistaken for IDIV.
if (getauxval) {
return (getauxval(AT_HWCAP) & HWCAP_NEON);
}
#endif /* defined(__ARM_NEON) || defined(__ARM_NEON__) */
return PR_FALSE;
}
#ifdef __linux__
static long
ReadCPUInfoForHWCAP2()
{
FILE *cpuinfo;
char buf[512];
char *p;
long hwcap2 = 0;
cpuinfo = fopen("/proc/cpuinfo", "r");
if (!cpuinfo) {
return 0;
}
while (fgets(buf, 511, cpuinfo)) {
if (!memcmp(buf, "Features", 8)) {
p = strstr(buf, " aes");
if (p && (p[4] == ' ' || p[4] == '\n')) {
hwcap2 |= HWCAP2_AES;
}
p = strstr(buf, " sha1");
if (p && (p[5] == ' ' || p[5] == '\n')) {
hwcap2 |= HWCAP2_SHA1;
}
p = strstr(buf, " sha2");
if (p && (p[5] == ' ' || p[5] == '\n')) {
hwcap2 |= HWCAP2_SHA2;
}
p = strstr(buf, " pmull");
if (p && (p[6] == ' ' || p[6] == '\n')) {
hwcap2 |= HWCAP2_PMULL;
}
break;
}
}
fclose(cpuinfo);
return hwcap2;
}
#endif /* __linux__ */
void
CheckARMSupport()
{
char *disable_hw_aes = PR_GetEnvSecure("NSS_DISABLE_HW_AES");
if (getauxval) {
// Android's cpu-features.c uses AT_HWCAP2 for newer features.
// AT_HWCAP2 is implemented on newer devices / kernel, so we can trust
// it since cpu-features.c doesn't have workaround / fallback.
// Also, AT_HWCAP2 is supported by glibc 2.18+ on Linux/arm, If
// AT_HWCAP2 isn't supported by glibc or Linux kernel, getauxval will
// returns 0.
long hwcaps = getauxval(AT_HWCAP2);
#ifdef __linux__
if (!hwcaps) {
// Some ARMv8 devices may not implement AT_HWCAP2. So we also
// read /proc/cpuinfo if AT_HWCAP2 is 0.
hwcaps = ReadCPUInfoForHWCAP2();
}
#endif
arm_aes_support_ = hwcaps & HWCAP2_AES && disable_hw_aes == NULL;
arm_pmull_support_ = hwcaps & HWCAP2_PMULL;
arm_sha1_support_ = hwcaps & HWCAP2_SHA1;
arm_sha2_support_ = hwcaps & HWCAP2_SHA2;
}
arm_neon_support_ = GetNeonSupport();
arm_sha1_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA1") == NULL;
arm_sha2_support_ &= PR_GetEnvSecure("NSS_DISABLE_HW_SHA2") == NULL;
}
#endif /* defined(__arm__) */
// Enable when Firefox can use it for Android API 16 and 17.
// #if defined(__ANDROID__) && (defined(__arm__) || defined(__aarch64__))
// #include <cpu-features.h>
// void
// CheckARMSupport()
// {
// char *disable_arm_neon = PR_GetEnvSecure("NSS_DISABLE_ARM_NEON");
// char *disable_hw_aes = PR_GetEnvSecure("NSS_DISABLE_HW_AES");
// AndroidCpuFamily family = android_getCpuFamily();
// uint64_t features = android_getCpuFeatures();
// if (family == ANDROID_CPU_FAMILY_ARM64) {
// arm_aes_support_ = features & ANDROID_CPU_ARM64_FEATURE_AES &&
// disable_hw_aes == NULL;
// arm_pmull_support_ = features & ANDROID_CPU_ARM64_FEATURE_PMULL;
// arm_sha1_support_ = features & ANDROID_CPU_ARM64_FEATURE_SHA1;
// arm_sha2_support_ = features & ANDROID_CPU_ARM64_FEATURE_SHA2;
// arm_neon_support_ = disable_arm_neon == NULL;
// }
// if (family == ANDROID_CPU_FAMILY_ARM) {
// arm_aes_support_ = features & ANDROID_CPU_ARM_FEATURE_AES &&
// disable_hw_aes == NULL;
// arm_pmull_support_ = features & ANDROID_CPU_ARM_FEATURE_PMULL;
// arm_sha1_support_ = features & ANDROID_CPU_ARM_FEATURE_SHA1;
// arm_sha2_support_ = features & ANDROID_CPU_ARM_FEATURE_SHA2;
// arm_neon_support_ = hwcaps & ANDROID_CPU_ARM_FEATURE_NEON &&
// disable_arm_neon == NULL;
// }
// }
// #endif /* defined(__ANDROID__) && (defined(__arm__) || defined(__aarch64__)) */
PRBool
aesni_support()
{
return aesni_support_;
}
PRBool
clmul_support()
{
return clmul_support_;
}
PRBool
sha_support()
{
return sha_support_;
}
PRBool
avx_support()
{
return avx_support_;
}
PRBool
avx2_support()
{
return avx2_support_;
}
PRBool
adx_support()
{
return adx_support_;
}
PRBool
ssse3_support()
{
return ssse3_support_;
}
PRBool
sse4_1_support()
{
return sse4_1_support_;
}
PRBool
sse4_2_support()
{
return sse4_2_support_;
}
PRBool
arm_neon_support()
{
return arm_neon_support_;
}
PRBool
arm_aes_support()
{
return arm_aes_support_;
}
PRBool
arm_pmull_support()
{
return arm_pmull_support_;
}
PRBool
arm_sha1_support()
{
return arm_sha1_support_;
}
PRBool
arm_sha2_support()
{
return arm_sha2_support_;
}
PRBool
ppc_crypto_support()
{
return ppc_crypto_support_;
}
#if defined(__powerpc__)
#ifndef __has_include
#define __has_include(x) 0
#endif
/* clang-format off */
#if defined(__linux__) || (defined(__FreeBSD__) && __FreeBSD__ >= 12)
#if __has_include(<sys/auxv.h>)
#include <sys/auxv.h>
#endif
#elif (defined(__FreeBSD__) && __FreeBSD__ < 12)
#include <sys/sysctl.h>
#endif
// Defines from cputable.h in Linux kernel - PPC, letting us build on older kernels
#ifndef PPC_FEATURE2_VEC_CRYPTO
#define PPC_FEATURE2_VEC_CRYPTO 0x02000000
#endif
static void
CheckPPCSupport()
{
char *disable_hw_crypto = PR_GetEnvSecure("NSS_DISABLE_PPC_GHASH");
unsigned long hwcaps = 0;
#if defined(__linux__)
#if __has_include(<sys/auxv.h>)
hwcaps = getauxval(AT_HWCAP2);
#endif
#elif defined(__FreeBSD__)
#if __FreeBSD__ >= 12
#if __has_include(<sys/auxv.h>)
elf_aux_info(AT_HWCAP2, &hwcaps, sizeof(hwcaps));
#endif
#else
size_t len = sizeof(hwcaps);
sysctlbyname("hw.cpu_features2", &hwcaps, &len, NULL, 0);
#endif
#endif
ppc_crypto_support_ = hwcaps & PPC_FEATURE2_VEC_CRYPTO && disable_hw_crypto == NULL;
}
/* clang-format on */
#endif /* __powerpc__ */
static PRStatus
FreeblInit(void)
{
#ifdef NSS_X86_OR_X64
CheckX86CPUSupport();
#elif (defined(__aarch64__) || defined(__arm__))
CheckARMSupport();
#elif (defined(__powerpc__))
CheckPPCSupport();
#endif
return PR_SUCCESS;
}
SECStatus
BL_Init(void)
{
if (PR_CallOnce(&coFreeblInit, FreeblInit) != PR_SUCCESS) {
PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
return SECFailure;
}
RSA_Init();
return SECSuccess;
}