Source code
Revision control
Copy as Markdown
Other Tools
//! Linux `futex`.
//!
//! Futex is a very low-level mechanism for implementing concurrency primitives
//! such as mutexes, rwlocks, and condvars.
//!
//! # Examples
//!
//! ```
//! use rustix::thread::futex;
//! use std::sync::atomic::AtomicU32;
//!
//! # fn test(futex: &AtomicU32) -> rustix::io::Result<()> {
//! // Wake up one waiter.
//! futex::wake(futex, futex::Flags::PRIVATE, 1)?;
//! # Ok(())
//! # }
//! ```
//!
//! # References
//! - [Linux `futex` system call]
//! - [Linux `futex` feature]
//!
#![allow(unsafe_code)]
use core::num::NonZeroU32;
use core::ptr;
use core::sync::atomic::AtomicU32;
use crate::backend::thread::futex::Operation;
use crate::backend::thread::syscalls::{futex_timeout, futex_val2};
use crate::fd::{FromRawFd, OwnedFd, RawFd};
use crate::utils::option_as_ptr;
use crate::{backend, io};
pub use crate::timespec::{Nsecs, Secs, Timespec};
pub use backend::thread::futex::{Flags, OWNER_DIED, WAITERS};
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT, val, timeout, NULL, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn wait(
uaddr: &AtomicU32,
flags: Flags,
val: u32,
timeout: Option<Timespec>,
) -> io::Result<()> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_timeout(
uaddr,
Operation::Wait,
flags,
val,
option_as_ptr(timeout.as_ref()),
ptr::null(),
0,
)
.map(|val| {
debug_assert_eq!(
val, 0,
"The return value should always equal zero, if the call is successful"
);
})
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE, val, NULL, NULL, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn wake(uaddr: &AtomicU32, flags: Flags, val: u32) -> io::Result<usize> {
// SAFETY: The raw pointers come from references or null.
unsafe { futex_val2(uaddr, Operation::Wake, flags, val, 0, ptr::null(), 0) }
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_FD, val, NULL, NULL, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn fd(uaddr: &AtomicU32, flags: Flags, val: u32) -> io::Result<OwnedFd> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_val2(uaddr, Operation::Fd, flags, val, 0, ptr::null(), 0).map(|val| {
let fd = val as RawFd;
debug_assert_eq!(fd as usize, val, "return value should be a valid fd");
OwnedFd::from_raw_fd(fd)
})
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_REQUEUE, val, val2, uaddr2, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn requeue(
uaddr: &AtomicU32,
flags: Flags,
val: u32,
val2: u32,
uaddr2: &AtomicU32,
) -> io::Result<usize> {
// SAFETY: The raw pointers come from references or null.
unsafe { futex_val2(uaddr, Operation::Requeue, flags, val, val2, uaddr2, 0) }
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE, val, val2, uaddr2, val3)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn cmp_requeue(
uaddr: &AtomicU32,
flags: Flags,
val: u32,
val2: u32,
uaddr2: &AtomicU32,
val3: u32,
) -> io::Result<usize> {
// SAFETY: The raw pointers come from references or null.
unsafe { futex_val2(uaddr, Operation::CmpRequeue, flags, val, val2, uaddr2, val3) }
}
/// `FUTEX_OP_*` operations for use with [`wake_op`].
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[repr(u32)]
#[allow(clippy::identity_op)]
pub enum WakeOp {
/// `FUTEX_OP_SET`: `uaddr2 = oparg;`
Set = 0,
/// `FUTEX_OP_ADD`: `uaddr2 += oparg;`
Add = 1,
/// `FUTEX_OP_OR`: `uaddr2 |= oparg;`
Or = 2,
/// `FUTEX_OP_ANDN`: `uaddr2 &= ~oparg;`
AndN = 3,
/// `FUTEX_OP_XOR`: `uaddr2 ^= oparg;`
XOr = 4,
/// `FUTEX_OP_SET | FUTEX_OP_ARG_SHIFT`: `uaddr2 = (oparg << 1);`
SetShift = 0 | 8,
/// `FUTEX_OP_ADD | FUTEX_OP_ARG_SHIFT`: `uaddr2 += (oparg << 1);`
AddShift = 1 | 8,
/// `FUTEX_OP_OR | FUTEX_OP_ARG_SHIFT`: `uaddr2 |= (oparg << 1);`
OrShift = 2 | 8,
/// `FUTEX_OP_ANDN | FUTEX_OP_ARG_SHIFT`: `uaddr2 &= !(oparg << 1);`
AndNShift = 3 | 8,
/// `FUTEX_OP_XOR | FUTEX_OP_ARG_SHIFT`: `uaddr2 ^= (oparg << 1);`
XOrShift = 4 | 8,
}
/// `FUTEX_OP_CMP_*` operations for use with [`wake_op`].
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[repr(u32)]
pub enum WakeOpCmp {
/// `FUTEX_OP_CMP_EQ`: `if oldval == cmparg { wake(); }`
Eq = 0,
/// `FUTEX_OP_CMP_EQ`: `if oldval != cmparg { wake(); }`
Ne = 1,
/// `FUTEX_OP_CMP_EQ`: `if oldval < cmparg { wake(); }`
Lt = 2,
/// `FUTEX_OP_CMP_EQ`: `if oldval <= cmparg { wake(); }`
Le = 3,
/// `FUTEX_OP_CMP_EQ`: `if oldval > cmparg { wake(); }`
Gt = 4,
/// `FUTEX_OP_CMP_EQ`: `if oldval >= cmparg { wake(); }`
Ge = 5,
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE_OP, val, val2, uaddr2, val3)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
#[allow(clippy::too_many_arguments)]
pub fn wake_op(
uaddr: &AtomicU32,
flags: Flags,
val: u32,
val2: u32,
uaddr2: &AtomicU32,
op: WakeOp,
cmp: WakeOpCmp,
oparg: u16,
cmparg: u16,
) -> io::Result<usize> {
if oparg >= 1 << 12 || cmparg >= 1 << 12 {
return Err(io::Errno::INVAL);
}
let val3 =
((op as u32) << 28) | ((cmp as u32) << 24) | ((oparg as u32) << 12) | (cmparg as u32);
// SAFETY: The raw pointers come from references or null.
unsafe { futex_val2(uaddr, Operation::WakeOp, flags, val, val2, uaddr2, val3) }
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI, 0, timeout, NULL, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn lock_pi(uaddr: &AtomicU32, flags: Flags, timeout: Option<Timespec>) -> io::Result<()> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_timeout(
uaddr,
Operation::LockPi,
flags,
0,
option_as_ptr(timeout.as_ref()),
ptr::null(),
0,
)
.map(|val| {
debug_assert_eq!(
val, 0,
"The return value should always equal zero, if the call is successful"
);
})
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_UNLOCK_PI, 0, NULL, NULL, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn unlock_pi(uaddr: &AtomicU32, flags: Flags) -> io::Result<()> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_val2(uaddr, Operation::UnlockPi, flags, 0, 0, ptr::null(), 0).map(|val| {
debug_assert_eq!(
val, 0,
"The return value should always equal zero, if the call is successful"
);
})
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_TRYLOCK_PI, 0, NULL, NULL, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn trylock_pi(uaddr: &AtomicU32, flags: Flags) -> io::Result<bool> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_val2(uaddr, Operation::TrylockPi, flags, 0, 0, ptr::null(), 0).map(|ret| ret == 0)
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT_BITSET, val, timeout, NULL, val3)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn wait_bitset(
uaddr: &AtomicU32,
flags: Flags,
val: u32,
timeout: Option<Timespec>,
val3: NonZeroU32,
) -> io::Result<()> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_timeout(
uaddr,
Operation::WaitBitset,
flags,
val,
option_as_ptr(timeout.as_ref()),
ptr::null(),
val3.get(),
)
.map(|val| {
debug_assert_eq!(
val, 0,
"The return value should always equal zero, if the call is successful"
);
})
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE_BITSET, val, NULL, NULL, val3)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn wake_bitset(
uaddr: &AtomicU32,
flags: Flags,
val: u32,
val3: NonZeroU32,
) -> io::Result<usize> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_val2(
uaddr,
Operation::WakeBitset,
flags,
val,
0,
ptr::null(),
val3.get(),
)
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT_REQUEUE_PI, val, timeout, uaddr2, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn wait_requeue_pi(
uaddr: &AtomicU32,
flags: Flags,
val: u32,
timeout: Option<Timespec>,
uaddr2: &AtomicU32,
) -> io::Result<()> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_timeout(
uaddr,
Operation::WaitRequeuePi,
flags,
val,
option_as_ptr(timeout.as_ref()),
uaddr2,
0,
)
.map(|val| {
debug_assert_eq!(
val, 0,
"The return value should always equal zero, if the call is successful"
);
})
}
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE_PI, 1, val2, uaddr2, val3)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn cmp_requeue_pi(
uaddr: &AtomicU32,
flags: Flags,
val2: u32,
uaddr2: &AtomicU32,
val3: u32,
) -> io::Result<usize> {
// SAFETY: The raw pointers come from references or null.
unsafe { futex_val2(uaddr, Operation::CmpRequeuePi, flags, 1, val2, uaddr2, val3) }
}
/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI2, 0, timeout, NULL, 0)`
///
/// This is a very low-level feature for implementing synchronization
/// primitives. See the references links.
///
/// # References
/// - [Linux `futex` system call]
/// - [Linux `futex` feature]
///
#[inline]
pub fn lock_pi2(uaddr: &AtomicU32, flags: Flags, timeout: Option<Timespec>) -> io::Result<()> {
// SAFETY: The raw pointers come from references or null.
unsafe {
futex_timeout(
uaddr,
Operation::LockPi2,
flags,
0,
option_as_ptr(timeout.as_ref()),
ptr::null(),
0,
)
.map(|val| {
debug_assert_eq!(
val, 0,
"The return value should always equal zero, if the call is successful"
);
})
}
}