Source code
Revision control
Copy as Markdown
Other Tools
// Copyright 2020 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef HIGHWAY_HWY_CACHE_CONTROL_H_
#define HIGHWAY_HWY_CACHE_CONTROL_H_
#include "hwy/base.h"
// Requires SSE2; fails to compile on 32-bit Clang 7 (see
#if !defined(__SSE2__) || (HWY_COMPILER_CLANG && HWY_ARCH_X86_32)
#undef HWY_DISABLE_CACHE_CONTROL
#define HWY_DISABLE_CACHE_CONTROL
#endif
#ifndef HWY_DISABLE_CACHE_CONTROL
// intrin.h is sufficient on MSVC and already included by base.h.
#if HWY_ARCH_X86 && !HWY_COMPILER_MSVC
#include <emmintrin.h> // SSE2
#include <xmmintrin.h> // _mm_prefetch
#elif HWY_ARCH_ARM_A64
#include <arm_acle.h>
#endif
#endif // HWY_DISABLE_CACHE_CONTROL
namespace hwy {
// Even if N*sizeof(T) is smaller, Stream may write a multiple of this size.
#define HWY_STREAM_MULTIPLE 16
// The following functions may also require an attribute.
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) && !HWY_COMPILER_MSVC
#define HWY_ATTR_CACHE __attribute__((target("sse2")))
#else
#define HWY_ATTR_CACHE
#endif
// Windows.h #defines this, which causes infinite recursion. Temporarily
// undefine to avoid conflict with our function.
// TODO(janwas): remove when this function is removed.
#pragma push_macro("LoadFence")
#undef LoadFence
// Delays subsequent loads until prior loads are visible. Beware of potentially
// differing behavior across architectures and vendors: on Intel but not
// AMD CPUs, also serves as a full fence (waits for all prior instructions to
// complete).
HWY_INLINE HWY_ATTR_CACHE void LoadFence() {
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL)
_mm_lfence();
#endif
}
// TODO(janwas): remove when this function is removed. (See above.)
#pragma pop_macro("LoadFence")
// Ensures values written by previous `Stream` calls are visible on the current
// core. This is NOT sufficient for synchronizing across cores; when `Stream`
// outputs are to be consumed by other core(s), the producer must publish
// availability (e.g. via mutex or atomic_flag) after `FlushStream`.
HWY_INLINE HWY_ATTR_CACHE void FlushStream() {
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL)
_mm_sfence();
#endif
}
// Optionally begins loading the cache line containing "p" to reduce latency of
// subsequent actual loads.
template <typename T>
HWY_INLINE HWY_ATTR_CACHE void Prefetch(const T* p) {
(void)p;
#ifndef HWY_DISABLE_CACHE_CONTROL
#if HWY_ARCH_X86
_mm_prefetch(reinterpret_cast<const char*>(p), _MM_HINT_T0);
#elif HWY_COMPILER_GCC // includes clang
// Hint=0 (NTA) behavior differs, but skipping outer caches is probably not
// desirable, so use the default 3 (keep in caches).
__builtin_prefetch(p, /*write=*/0, /*hint=*/3);
#endif
#endif // HWY_DISABLE_CACHE_CONTROL
}
// Invalidates and flushes the cache line containing "p", if possible.
HWY_INLINE HWY_ATTR_CACHE void FlushCacheline(const void* p) {
#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL)
_mm_clflush(p);
#else
(void)p;
#endif
}
// Hints that we are inside a spin loop and potentially reduces power
// consumption and coherency traffic. For example, x86 avoids multiple
// outstanding load requests, which reduces the memory order violation penalty
// when exiting the loop.
HWY_INLINE HWY_ATTR_CACHE void Pause() {
#ifndef HWY_DISABLE_CACHE_CONTROL
#if HWY_ARCH_X86
_mm_pause();
#elif HWY_ARCH_ARM_A64 && HWY_COMPILER_CLANG
// This is documented in ACLE and the YIELD instruction is also available in
// Armv7, but the intrinsic is broken for Armv7 clang, hence A64 only.
__yield();
#elif HWY_ARCH_ARM && HWY_COMPILER_GCC // includes clang
__asm__ volatile("yield" ::: "memory");
#elif HWY_ARCH_PPC && HWY_COMPILER_GCC // includes clang
__asm__ volatile("or 27,27,27" ::: "memory");
#endif
#endif // HWY_DISABLE_CACHE_CONTROL
}
} // namespace hwy
#endif // HIGHWAY_HWY_CACHE_CONTROL_H_