Source code
Revision control
Copy as Markdown
Other Tools
//! The `basic` API flavor provides barebones UTF-8 checking at the highest speed.
//!
//! It is fastest on valid UTF-8, but only checks for errors after processing the whole byte sequence
//! and does not provide detailed information if the data is not valid UTF-8. [`Utf8Error`] is a zero-sized error struct.
//!
//! If you need detailed error information use the functions from the [`crate::compat`] module instead.
use core::str::{from_utf8_unchecked, from_utf8_unchecked_mut};
use crate::implementation::validate_utf8_basic;
/// Simple zero-sized UTF-8 error.
///
/// No information is provided where the error occurred or how long the invalid byte
/// byte sequence is.
#[derive(Copy, Eq, PartialEq, Clone, Debug)]
pub struct Utf8Error;
impl core::fmt::Display for Utf8Error {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.write_str("invalid utf-8 sequence")
}
}
#[cfg(feature = "std")]
impl std::error::Error for Utf8Error {}
/// Analogue to [`std::str::from_utf8()`].
///
/// Checks if the passed byte sequence is valid UTF-8 and returns an
/// [`std::str`] reference to the passed byte slice wrapped in `Ok()` if it is.
///
/// # Errors
/// Will return the zero-sized Err([`Utf8Error`]) on if the input contains invalid UTF-8.
#[inline]
pub fn from_utf8(input: &[u8]) -> Result<&str, Utf8Error> {
unsafe {
validate_utf8_basic(input)?;
Ok(from_utf8_unchecked(input))
}
}
/// Analogue to [`std::str::from_utf8_mut()`].
///
/// Checks if the passed mutable byte sequence is valid UTF-8 and returns a mutable
/// [`std::str`] reference to the passed byte slice wrapped in `Ok()` if it is.
///
/// # Errors
/// Will return the zero-sized Err([`Utf8Error`]) on if the input contains invalid UTF-8.
#[inline]
pub fn from_utf8_mut(input: &mut [u8]) -> Result<&mut str, Utf8Error> {
unsafe {
validate_utf8_basic(input)?;
Ok(from_utf8_unchecked_mut(input))
}
}
/// Allows direct access to the platform-specific unsafe validation implementations.
#[cfg(feature = "public_imp")]
pub mod imp {
use crate::basic;
/// A low-level interface for streaming validation of UTF-8 data. It is meant to be integrated
/// in high-performance data processing pipelines.
///
/// Data can be streamed in arbitrarily-sized chunks using the [`Self::update()`] method. There is
/// no way to find out if the input so far was valid UTF-8 during the validation. Only when
/// the validation is completed with the [`Self::finalize()`] method the result of the validation is
/// returned. Use [`ChunkedUtf8Validator`] if possible for highest performance.
///
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to use it if the required CPU features are not available which
/// is why all trait methods are `unsafe`.
///
/// General usage:
/// ```rust
/// use simdutf8::basic::imp::Utf8Validator;
/// use std::io::{stdin, Read, Result};
///
/// # #[cfg(target_arch = "x86_64")]
/// fn main() -> Result<()> {
/// unsafe {
/// if !std::is_x86_feature_detected!("avx2") {
/// panic!("This example only works with CPUs supporting AVX 2");
/// }
///
/// let mut validator = simdutf8::basic::imp::x86::avx2::Utf8ValidatorImp::new();
/// let mut buf = vec![0; 8192];
/// loop {
/// let bytes_read = stdin().read(buf.as_mut())?;
/// if bytes_read == 0 {
/// break;
/// }
/// validator.update(&buf);
/// }
///
/// if validator.finalize().is_ok() {
/// println!("Input is valid UTF-8");
/// } else {
/// println!("Input is not valid UTF-8");
/// }
/// }
///
/// Ok(())
/// }
///
/// # #[cfg(not(target_arch = "x86_64"))]
/// # fn main() { }
/// ```
///
pub trait Utf8Validator {
/// Creates a new validator.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
#[must_use]
unsafe fn new() -> Self
where
Self: Sized;
/// Updates the validator with `input`.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn update(&mut self, input: &[u8]);
/// Finishes the validation and returns `Ok(())` if the input was valid UTF-8.
///
/// # Errors
/// A [`basic::Utf8Error`] is returned if the input was not valid UTF-8. No
/// further information about the location of the error is provided.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn finalize(self) -> core::result::Result<(), basic::Utf8Error>;
}
/// Like [`Utf8Validator`] this low-level API is for streaming validation of UTF-8 data.
///
/// It has additional restrictions imposed on how the input is passed in to allow
/// validation with as little overhead as possible.
///
/// To feed it data you need to call the [`Self::update_from_chunks()`] method which takes slices which
/// have to be a multiple of 64 bytes long. The method will panic otherwise. There is
/// no way to find out if the input so far was valid UTF-8 during the validation. Only when
/// the validation is completed with the [`Self::finalize()`] method the result of the validation is
/// returned.
///
/// The `Self::finalize()` method can be fed the rest of the data. There is no restriction on the
/// data passed to it.
///
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to use it if the required CPU features are not available which
/// is why all trait methods are `unsafe`.
pub trait ChunkedUtf8Validator {
/// Creates a new validator.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
#[must_use]
unsafe fn new() -> Self
where
Self: Sized;
/// Updates the validator with `input`.
///
/// # Panics
/// If `input.len()` is not a multiple of 64.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn update_from_chunks(&mut self, input: &[u8]);
/// Updates the validator with remaining input if any. There is no restriction on the
/// data provided.
///
/// Finishes the validation and returns `Ok(())` if the input was valid UTF-8.
///
/// # Errors
/// A [`basic::Utf8Error`] is returned if the input was not valid UTF-8. No
/// further information about the location of the error is provided.
///
/// # Safety
/// This implementation requires CPU SIMD features specified by the module it resides in.
/// It is undefined behavior to call it if the required CPU features are not available.
unsafe fn finalize(
self,
remaining_input: core::option::Option<&[u8]>,
) -> core::result::Result<(), basic::Utf8Error>;
}
/// Includes the x86/x86-64 SIMD implementations.
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
pub mod x86 {
/// Includes the validation implementation for AVX 2-compatible CPUs.
///
/// Using the provided functionality on CPUs which do not support AVX 2 is undefined
/// behavior and will very likely cause a crash.
pub mod avx2 {
pub use crate::implementation::x86::avx2::validate_utf8_basic as validate_utf8;
pub use crate::implementation::x86::avx2::ChunkedUtf8ValidatorImp;
pub use crate::implementation::x86::avx2::Utf8ValidatorImp;
}
/// Includes the validation implementation for SSE 4.2-compatible CPUs.
///
/// Using the provided functionality on CPUs which do not support SSE 4.2 is undefined
/// behavior and will very likely cause a crash.
pub mod sse42 {
pub use crate::implementation::x86::sse42::validate_utf8_basic as validate_utf8;
pub use crate::implementation::x86::sse42::ChunkedUtf8ValidatorImp;
pub use crate::implementation::x86::sse42::Utf8ValidatorImp;
}
}
/// Includes the aarch64 SIMD implementations.
#[cfg(all(feature = "aarch64_neon", target_arch = "aarch64"))]
pub mod aarch64 {
/// Includes the Neon-based validation implementation for aarch64 CPUs.
///
/// Should be supported on all ARM64 CPUSs. If it is not supported by the operating
/// system using it is undefined behavior and will likely cause a crash.
pub mod neon {
pub use crate::implementation::aarch64::neon::validate_utf8_basic as validate_utf8;
pub use crate::implementation::aarch64::neon::ChunkedUtf8ValidatorImp;
pub use crate::implementation::aarch64::neon::Utf8ValidatorImp;
}
}
/// Includes the wasm32 SIMD implementations.
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
pub mod wasm32 {
/// Includes the simd128-based validation implementation for WASM runtimes.
///
/// Using the provided functionality on WASM runtimes that do not support SIMD
/// instructions will likely cause a crash.
pub mod simd128 {
pub use crate::implementation::wasm32::simd128::validate_utf8_basic as validate_utf8;
pub use crate::implementation::wasm32::simd128::ChunkedUtf8ValidatorImp;
pub use crate::implementation::wasm32::simd128::Utf8ValidatorImp;
}
}
}