1147 lines
35 KiB
Rust
1147 lines
35 KiB
Rust
//! Linux networking backend.
|
|
|
|
use std::cell::Cell;
|
|
use std::ffi::c_void;
|
|
use std::future::Future;
|
|
use std::io;
|
|
use std::mem::MaybeUninit;
|
|
use std::net::{
|
|
Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, SocketAddrV4, SocketAddrV6, ToSocketAddrs,
|
|
};
|
|
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd};
|
|
use std::pin::Pin;
|
|
use std::thread;
|
|
use std::time::Duration;
|
|
|
|
thread_local! {
|
|
// None = untested, Some(true) = io_uring works, Some(false) = use offload.
|
|
// After the first successful IORING_OP_SEND, the clone in send() is skipped
|
|
// for all subsequent calls on the same thread.
|
|
static SEND_URING_SUPPORTED: Cell<Option<bool>> = const { Cell::new(None) };
|
|
}
|
|
|
|
use crate::op::completion::completion_for_current_thread;
|
|
use crate::op::net::{AcceptedSocket, NetOp, ReceivedDatagram};
|
|
use crate::platform::linux_x86_64::runtime::with_current_driver;
|
|
use crate::platform::linux_x86_64::uring::{
|
|
IORING_OP_ACCEPT, IORING_OP_BIND, IORING_OP_CLOSE, IORING_OP_CONNECT, IORING_OP_LISTEN,
|
|
IORING_OP_RECV, IORING_OP_RECVMSG, IORING_OP_SEND, IORING_OP_SENDMSG, IORING_OP_SHUTDOWN,
|
|
IORING_OP_SOCKET, IoUringCqe, IoUringSqe,
|
|
};
|
|
|
|
const DEFAULT_LISTENER_BACKLOG: i32 = 1024;
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub enum ExecutionPath {
|
|
IoUring,
|
|
Offload,
|
|
}
|
|
|
|
pub fn execution_path(op: &NetOp) -> ExecutionPath {
|
|
match op {
|
|
NetOp::Socket { .. }
|
|
| NetOp::Connect { .. }
|
|
| NetOp::Bind { .. }
|
|
| NetOp::Listen { .. }
|
|
| NetOp::Accept { .. }
|
|
| NetOp::Send { .. }
|
|
| NetOp::Recv { .. }
|
|
| NetOp::Shutdown { .. }
|
|
| NetOp::Close { .. } => ExecutionPath::IoUring,
|
|
NetOp::SendTo { .. } | NetOp::RecvFrom { .. } => ExecutionPath::IoUring,
|
|
}
|
|
}
|
|
|
|
pub async fn resolve_addrs<A>(addr: A) -> io::Result<Vec<SocketAddr>>
|
|
where
|
|
A: ToSocketAddrs + Send + 'static,
|
|
{
|
|
offload(move || {
|
|
let addrs = addr.to_socket_addrs()?.collect::<Vec<_>>();
|
|
if addrs.is_empty() {
|
|
Err(io::Error::new(
|
|
io::ErrorKind::InvalidInput,
|
|
"address resolved to no socket addresses",
|
|
))
|
|
} else {
|
|
Ok(addrs)
|
|
}
|
|
})
|
|
.await
|
|
}
|
|
|
|
pub async fn socket(op: NetOp) -> io::Result<OwnedFd> {
|
|
let NetOp::Socket {
|
|
domain,
|
|
socket_type,
|
|
protocol,
|
|
flags,
|
|
} = op
|
|
else {
|
|
unreachable!("socket backend called with non-socket op");
|
|
};
|
|
|
|
match submit_uring::<OwnedFd, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_SOCKET;
|
|
sqe.fd = domain;
|
|
sqe.off = socket_type as u64;
|
|
sqe.len = protocol as u32;
|
|
sqe.op_flags = flags;
|
|
},
|
|
move |cqe| cqe_to_result(cqe).map(|fd| unsafe { OwnedFd::from_raw_fd(fd as RawFd) }),
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
offload(move || socket_sync(domain, socket_type, protocol, flags)).await
|
|
}
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn connect(op: NetOp) -> io::Result<()> {
|
|
let NetOp::Connect { fd, addr } = op else {
|
|
unreachable!("connect backend called with non-connect op");
|
|
};
|
|
|
|
let raw_addr = RawSocketAddr::from_socket_addr(addr);
|
|
let fallback_addr = raw_addr;
|
|
let addr_ptr = raw_addr.as_ptr();
|
|
let addr_len = raw_addr.len();
|
|
match submit_uring::<(), _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_CONNECT;
|
|
sqe.fd = fd;
|
|
sqe.addr = addr_ptr as u64;
|
|
sqe.off = addr_len as u64;
|
|
},
|
|
move |cqe| {
|
|
let _raw_addr = raw_addr;
|
|
cqe_to_result(cqe).map(|_| ())
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
offload(move || connect_sync(fd, fallback_addr)).await
|
|
}
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn bind(op: NetOp) -> io::Result<()> {
|
|
let NetOp::Bind { fd, addr } = op else {
|
|
unreachable!("bind backend called with non-bind op");
|
|
};
|
|
|
|
let raw_addr = RawSocketAddr::from_socket_addr(addr);
|
|
let fallback_addr = raw_addr;
|
|
let addr_ptr = raw_addr.as_ptr();
|
|
let addr_len = raw_addr.len();
|
|
match submit_uring::<(), _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_BIND;
|
|
sqe.fd = fd;
|
|
sqe.addr = addr_ptr as u64;
|
|
sqe.off = addr_len as u64;
|
|
},
|
|
move |cqe| {
|
|
let _raw_addr = raw_addr;
|
|
cqe_to_result(cqe).map(|_| ())
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
offload(move || bind_sync(fd, fallback_addr)).await
|
|
}
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn listen(op: NetOp) -> io::Result<()> {
|
|
let NetOp::Listen { fd, backlog } = op else {
|
|
unreachable!("listen backend called with non-listen op");
|
|
};
|
|
|
|
match submit_uring::<(), _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_LISTEN;
|
|
sqe.fd = fd;
|
|
sqe.len = backlog as u32;
|
|
},
|
|
move |cqe| cqe_to_result(cqe).map(|_| ()),
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
offload(move || listen_sync(fd, backlog)).await
|
|
}
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn accept(op: NetOp) -> io::Result<AcceptedSocket> {
|
|
let NetOp::Accept { fd } = op else {
|
|
unreachable!("accept backend called with non-accept op");
|
|
};
|
|
|
|
let mut storage = Box::new(MaybeUninit::<libc::sockaddr_storage>::zeroed());
|
|
let mut addr_len = Box::new(std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t);
|
|
let storage_ptr = storage.as_mut_ptr();
|
|
let addr_len_ptr = addr_len.as_mut() as *mut libc::socklen_t;
|
|
|
|
match submit_uring::<AcceptedSocket, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_ACCEPT;
|
|
sqe.fd = fd;
|
|
sqe.addr = storage_ptr as u64;
|
|
sqe.off = addr_len_ptr as u64;
|
|
},
|
|
move |cqe| {
|
|
let accepted_fd = cqe_to_result(cqe)? as RawFd;
|
|
let storage = unsafe { storage.assume_init() };
|
|
let peer_addr = socket_addr_from_storage(&storage, *addr_len)?;
|
|
Ok(AcceptedSocket {
|
|
fd: accepted_fd,
|
|
peer_addr,
|
|
})
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => offload(move || accept_sync(fd)).await,
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn send(op: NetOp) -> io::Result<usize> {
|
|
let NetOp::Send { fd, data, flags } = op else {
|
|
unreachable!("send backend called with non-send op");
|
|
};
|
|
|
|
// Capability known: io_uring SEND is not supported — skip to offload directly.
|
|
if SEND_URING_SUPPORTED.with(|c| c.get()) == Some(false) {
|
|
return offload(move || send_sync(fd, data, flags)).await;
|
|
}
|
|
|
|
// Capability known: io_uring SEND works — submit without a fallback clone.
|
|
if SEND_URING_SUPPORTED.with(|c| c.get()) == Some(true) {
|
|
let data_ptr = data.as_ptr();
|
|
let data_len = data.len();
|
|
return submit_uring::<usize, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_SEND;
|
|
sqe.fd = fd;
|
|
sqe.addr = data_ptr as u64;
|
|
sqe.len = data_len as u32;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
move |cqe| {
|
|
let _data = data;
|
|
cqe_to_result(cqe).map(|written| written as usize)
|
|
},
|
|
)
|
|
.await;
|
|
}
|
|
|
|
// Capability unknown: probe with a one-time clone. Cache the result.
|
|
let fallback_data = data.clone();
|
|
let data_ptr = data.as_ptr();
|
|
let data_len = data.len();
|
|
match submit_uring::<usize, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_SEND;
|
|
sqe.fd = fd;
|
|
sqe.addr = data_ptr as u64;
|
|
sqe.len = data_len as u32;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
move |cqe| {
|
|
let _data = data;
|
|
cqe_to_result(cqe).map(|written| written as usize)
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
SEND_URING_SUPPORTED.with(|c| c.set(Some(false)));
|
|
offload(move || send_sync(fd, fallback_data, flags)).await
|
|
}
|
|
result => {
|
|
if result.is_ok() {
|
|
SEND_URING_SUPPORTED.with(|c| c.set(Some(true)));
|
|
}
|
|
result
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn send_to(op: NetOp) -> io::Result<usize> {
|
|
let NetOp::SendTo {
|
|
fd,
|
|
target,
|
|
data,
|
|
flags,
|
|
} = op
|
|
else {
|
|
unreachable!("send_to backend called with non-send_to op");
|
|
};
|
|
|
|
let raw_addr = Box::new(RawSocketAddr::from_socket_addr(target));
|
|
let mut iov = Box::new(libc::iovec {
|
|
iov_base: data.as_ptr() as *mut c_void,
|
|
iov_len: data.len(),
|
|
});
|
|
let mut msg = Box::new(unsafe { std::mem::zeroed::<libc::msghdr>() });
|
|
msg.msg_name = raw_addr.as_ptr() as *mut c_void;
|
|
msg.msg_namelen = raw_addr.len();
|
|
msg.msg_iov = iov.as_mut() as *mut libc::iovec;
|
|
msg.msg_iovlen = 1;
|
|
let msg_ptr = msg.as_mut() as *mut libc::msghdr as u64;
|
|
let iov = SendIovec(iov);
|
|
let msg = SendMsghdr(msg);
|
|
|
|
submit_uring::<usize, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_SENDMSG;
|
|
sqe.fd = fd;
|
|
sqe.addr = msg_ptr;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
move |cqe| {
|
|
let _raw_addr = raw_addr;
|
|
let _iov = iov;
|
|
let _msg = msg;
|
|
let _data = data;
|
|
cqe_to_result(cqe).map(|written| written as usize)
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn recv(op: NetOp) -> io::Result<Vec<u8>> {
|
|
let NetOp::Recv { fd, len, flags } = op else {
|
|
unreachable!("recv backend called with non-recv op");
|
|
};
|
|
|
|
let mut buffer = vec![0; len];
|
|
let buffer_ptr = buffer.as_mut_ptr();
|
|
let buffer_len = buffer.len();
|
|
match submit_uring::<Vec<u8>, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_RECV;
|
|
sqe.fd = fd;
|
|
sqe.addr = buffer_ptr as u64;
|
|
sqe.len = buffer_len as u32;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
move |cqe| {
|
|
let read = cqe_to_result(cqe)? as usize;
|
|
buffer.truncate(read);
|
|
Ok(buffer)
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
offload(move || recv_sync(fd, len, flags)).await
|
|
}
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn recv_from(op: NetOp) -> io::Result<ReceivedDatagram> {
|
|
let NetOp::RecvFrom { fd, len, flags } = op else {
|
|
unreachable!("recv_from backend called with non-recv_from op");
|
|
};
|
|
|
|
let mut data = vec![0u8; len];
|
|
let mut storage = Box::new(MaybeUninit::<libc::sockaddr_storage>::zeroed());
|
|
let mut iov = Box::new(libc::iovec {
|
|
iov_base: data.as_mut_ptr() as *mut c_void,
|
|
iov_len: data.len(),
|
|
});
|
|
let mut msg = Box::new(unsafe { std::mem::zeroed::<libc::msghdr>() });
|
|
msg.msg_name = storage.as_mut_ptr() as *mut c_void;
|
|
msg.msg_namelen = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
|
|
msg.msg_iov = iov.as_mut() as *mut libc::iovec;
|
|
msg.msg_iovlen = 1;
|
|
let msg_ptr = msg.as_mut() as *mut libc::msghdr as u64;
|
|
let iov = SendIovec(iov);
|
|
let msg = SendMsghdr(msg);
|
|
|
|
match submit_uring::<ReceivedDatagram, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_RECVMSG;
|
|
sqe.fd = fd;
|
|
sqe.addr = msg_ptr;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
move |cqe| {
|
|
let _iov = iov;
|
|
let addr_len = msg.0.msg_namelen;
|
|
drop(msg);
|
|
let read = cqe_to_result(cqe)? as usize;
|
|
data.truncate(read);
|
|
let storage = unsafe { storage.assume_init() };
|
|
let peer_addr = socket_addr_from_storage(&storage, addr_len)?;
|
|
Ok(ReceivedDatagram { data, peer_addr })
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
offload(move || recv_from_sync(fd, len, flags)).await
|
|
}
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn shutdown(op: NetOp) -> io::Result<()> {
|
|
let NetOp::Shutdown { fd, how } = op else {
|
|
unreachable!("shutdown backend called with non-shutdown op");
|
|
};
|
|
|
|
let fallback_how = how;
|
|
match submit_uring::<(), _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_SHUTDOWN;
|
|
sqe.fd = fd;
|
|
sqe.len = shutdown_how(how) as u32;
|
|
},
|
|
move |cqe| cqe_to_result(cqe).map(|_| ()),
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => {
|
|
offload(move || shutdown_sync(fd, fallback_how)).await
|
|
}
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn close(op: NetOp) -> io::Result<()> {
|
|
let NetOp::Close { fd } = op else {
|
|
unreachable!("close backend called with non-close op");
|
|
};
|
|
|
|
match submit_uring::<(), _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_CLOSE;
|
|
sqe.fd = fd;
|
|
},
|
|
move |cqe| cqe_to_result(cqe).map(|_| ()),
|
|
)
|
|
.await
|
|
{
|
|
Err(error) if should_fallback_to_offload(&error) => offload(move || close_sync(fd)).await,
|
|
result => result,
|
|
}
|
|
}
|
|
|
|
pub async fn connect_stream(addr: SocketAddr) -> io::Result<OwnedFd> {
|
|
let socket = socket(NetOp::Socket {
|
|
domain: socket_domain(addr),
|
|
socket_type: libc::SOCK_STREAM,
|
|
protocol: 0,
|
|
flags: libc::SOCK_CLOEXEC as u32,
|
|
})
|
|
.await?;
|
|
|
|
let connect_result = connect(NetOp::Connect {
|
|
fd: socket.as_raw_fd(),
|
|
addr,
|
|
})
|
|
.await;
|
|
match connect_result {
|
|
Ok(()) => Ok(socket),
|
|
Err(error) => Err(error),
|
|
}
|
|
}
|
|
|
|
pub async fn bind_listener(addr: SocketAddr, backlog: Option<i32>) -> io::Result<OwnedFd> {
|
|
let listener = socket(NetOp::Socket {
|
|
domain: socket_domain(addr),
|
|
socket_type: libc::SOCK_STREAM,
|
|
protocol: 0,
|
|
flags: libc::SOCK_CLOEXEC as u32,
|
|
})
|
|
.await?;
|
|
|
|
set_reuse_addr(listener.as_raw_fd(), true)?;
|
|
|
|
bind(NetOp::Bind {
|
|
fd: listener.as_raw_fd(),
|
|
addr,
|
|
})
|
|
.await?;
|
|
listen(NetOp::Listen {
|
|
fd: listener.as_raw_fd(),
|
|
backlog: backlog.unwrap_or(DEFAULT_LISTENER_BACKLOG),
|
|
})
|
|
.await?;
|
|
Ok(listener)
|
|
}
|
|
|
|
pub async fn bind_datagram(addr: SocketAddr) -> io::Result<OwnedFd> {
|
|
let socket = socket(NetOp::Socket {
|
|
domain: socket_domain(addr),
|
|
socket_type: libc::SOCK_DGRAM,
|
|
protocol: 0,
|
|
flags: libc::SOCK_CLOEXEC as u32,
|
|
})
|
|
.await?;
|
|
|
|
bind(NetOp::Bind {
|
|
fd: socket.as_raw_fd(),
|
|
addr,
|
|
})
|
|
.await?;
|
|
Ok(socket)
|
|
}
|
|
|
|
pub async fn duplicate(fd: RawFd) -> io::Result<OwnedFd> {
|
|
offload(move || {
|
|
let duplicated = cvt(unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 0) })?;
|
|
Ok(unsafe { OwnedFd::from_raw_fd(duplicated) })
|
|
})
|
|
.await
|
|
}
|
|
|
|
pub async fn recv_timeout(
|
|
fd: RawFd,
|
|
len: usize,
|
|
flags: i32,
|
|
timeout: Duration,
|
|
) -> io::Result<Vec<u8>> {
|
|
let mut buffer = vec![0u8; len];
|
|
let buffer_ptr = buffer.as_mut_ptr();
|
|
let buffer_len = buffer.len();
|
|
submit_uring_with_linked_timeout::<Vec<u8>, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_RECV;
|
|
sqe.fd = fd;
|
|
sqe.addr = buffer_ptr as u64;
|
|
sqe.len = buffer_len as u32;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
timeout,
|
|
move |cqe| {
|
|
let read = cqe_to_timed_result(cqe)? as usize;
|
|
buffer.truncate(read);
|
|
Ok(buffer)
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn send_timeout(
|
|
fd: RawFd,
|
|
data: Vec<u8>,
|
|
flags: i32,
|
|
timeout: Duration,
|
|
) -> io::Result<usize> {
|
|
let data_ptr = data.as_ptr();
|
|
let data_len = data.len();
|
|
submit_uring_with_linked_timeout::<usize, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_SEND;
|
|
sqe.fd = fd;
|
|
sqe.addr = data_ptr as u64;
|
|
sqe.len = data_len as u32;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
timeout,
|
|
move |cqe| {
|
|
let _data = data;
|
|
cqe_to_timed_result(cqe).map(|written| written as usize)
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn recv_from_timeout(
|
|
fd: RawFd,
|
|
len: usize,
|
|
flags: i32,
|
|
timeout: Duration,
|
|
) -> io::Result<ReceivedDatagram> {
|
|
let mut data = vec![0u8; len];
|
|
let mut storage = Box::new(MaybeUninit::<libc::sockaddr_storage>::zeroed());
|
|
let mut iov = Box::new(libc::iovec {
|
|
iov_base: data.as_mut_ptr() as *mut c_void,
|
|
iov_len: data.len(),
|
|
});
|
|
let mut msg = Box::new(unsafe { std::mem::zeroed::<libc::msghdr>() });
|
|
msg.msg_name = storage.as_mut_ptr() as *mut c_void;
|
|
msg.msg_namelen = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
|
|
msg.msg_iov = iov.as_mut() as *mut libc::iovec;
|
|
msg.msg_iovlen = 1;
|
|
let msg_ptr = msg.as_mut() as *mut libc::msghdr as u64;
|
|
let iov = SendIovec(iov);
|
|
let msg = SendMsghdr(msg);
|
|
|
|
submit_uring_with_linked_timeout::<ReceivedDatagram, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_RECVMSG;
|
|
sqe.fd = fd;
|
|
sqe.addr = msg_ptr;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
timeout,
|
|
move |cqe| {
|
|
let _iov = iov;
|
|
let addr_len = msg.0.msg_namelen;
|
|
drop(msg);
|
|
let read = cqe_to_timed_result(cqe)? as usize;
|
|
data.truncate(read);
|
|
let storage = unsafe { storage.assume_init() };
|
|
let peer_addr = socket_addr_from_storage(&storage, addr_len)?;
|
|
Ok(ReceivedDatagram { data, peer_addr })
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn send_to_timeout(
|
|
fd: RawFd,
|
|
data: Vec<u8>,
|
|
target: SocketAddr,
|
|
flags: i32,
|
|
timeout: Duration,
|
|
) -> io::Result<usize> {
|
|
let raw_addr = Box::new(RawSocketAddr::from_socket_addr(target));
|
|
let mut iov = Box::new(libc::iovec {
|
|
iov_base: data.as_ptr() as *mut c_void,
|
|
iov_len: data.len(),
|
|
});
|
|
let mut msg = Box::new(unsafe { std::mem::zeroed::<libc::msghdr>() });
|
|
msg.msg_name = raw_addr.as_ptr() as *mut c_void;
|
|
msg.msg_namelen = raw_addr.len();
|
|
msg.msg_iov = iov.as_mut() as *mut libc::iovec;
|
|
msg.msg_iovlen = 1;
|
|
let msg_ptr = msg.as_mut() as *mut libc::msghdr as u64;
|
|
let iov = SendIovec(iov);
|
|
let msg = SendMsghdr(msg);
|
|
|
|
submit_uring_with_linked_timeout::<usize, _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_SENDMSG;
|
|
sqe.fd = fd;
|
|
sqe.addr = msg_ptr;
|
|
sqe.op_flags = flags as u32;
|
|
},
|
|
timeout,
|
|
move |cqe| {
|
|
let _raw_addr = raw_addr;
|
|
let _iov = iov;
|
|
let _msg = msg;
|
|
let _data = data;
|
|
cqe_to_timed_result(cqe).map(|written| written as usize)
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn connect_stream_timeout(addr: SocketAddr, timeout: Duration) -> io::Result<OwnedFd> {
|
|
let socket = socket(NetOp::Socket {
|
|
domain: socket_domain(addr),
|
|
socket_type: libc::SOCK_STREAM,
|
|
protocol: 0,
|
|
flags: libc::SOCK_CLOEXEC as u32,
|
|
})
|
|
.await?;
|
|
|
|
let fd = socket.as_raw_fd();
|
|
let raw_addr = RawSocketAddr::from_socket_addr(addr);
|
|
let addr_ptr = raw_addr.as_ptr();
|
|
let addr_len = raw_addr.len();
|
|
|
|
submit_uring_with_linked_timeout::<(), _>(
|
|
move |sqe| {
|
|
sqe.opcode = IORING_OP_CONNECT;
|
|
sqe.fd = fd;
|
|
sqe.addr = addr_ptr as u64;
|
|
sqe.off = addr_len as u64;
|
|
},
|
|
timeout,
|
|
move |cqe| {
|
|
let _raw_addr = raw_addr;
|
|
cqe_to_timed_result(cqe).map(|_| ())
|
|
},
|
|
)
|
|
.await?;
|
|
|
|
Ok(socket)
|
|
}
|
|
|
|
pub fn local_addr(fd: RawFd) -> io::Result<SocketAddr> {
|
|
socket_addr_with(libc::getsockname, fd)
|
|
}
|
|
|
|
pub fn peer_addr(fd: RawFd) -> io::Result<SocketAddr> {
|
|
socket_addr_with(libc::getpeername, fd)
|
|
}
|
|
|
|
pub fn nodelay(fd: RawFd) -> io::Result<bool> {
|
|
let mut value = 0;
|
|
let mut len = std::mem::size_of::<libc::c_int>() as libc::socklen_t;
|
|
cvt(unsafe {
|
|
libc::getsockopt(
|
|
fd,
|
|
libc::IPPROTO_TCP,
|
|
libc::TCP_NODELAY,
|
|
&mut value as *mut libc::c_int as *mut c_void,
|
|
&mut len,
|
|
)
|
|
})?;
|
|
Ok(value != 0)
|
|
}
|
|
|
|
pub fn broadcast(fd: RawFd) -> io::Result<bool> {
|
|
getsockopt_int(fd, libc::SOL_SOCKET, libc::SO_BROADCAST).map(|value| value != 0)
|
|
}
|
|
|
|
pub fn set_broadcast(fd: RawFd, enabled: bool) -> io::Result<()> {
|
|
setsockopt_int(fd, libc::SOL_SOCKET, libc::SO_BROADCAST, enabled.into())
|
|
}
|
|
|
|
pub fn ttl(fd: RawFd) -> io::Result<u32> {
|
|
match socket_family(fd)? {
|
|
libc::AF_INET => {
|
|
getsockopt_int(fd, libc::IPPROTO_IP, libc::IP_TTL).map(|value| value as u32)
|
|
}
|
|
libc::AF_INET6 => getsockopt_int(fd, libc::IPPROTO_IPV6, libc::IPV6_UNICAST_HOPS)
|
|
.map(|value| value as u32),
|
|
family => Err(io::Error::new(
|
|
io::ErrorKind::InvalidInput,
|
|
format!("unsupported socket family {family} for TTL"),
|
|
)),
|
|
}
|
|
}
|
|
|
|
pub fn set_ttl(fd: RawFd, ttl: u32) -> io::Result<()> {
|
|
let ttl = i32::try_from(ttl)
|
|
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "TTL exceeds i32 range"))?;
|
|
match socket_family(fd)? {
|
|
libc::AF_INET => setsockopt_int(fd, libc::IPPROTO_IP, libc::IP_TTL, ttl),
|
|
libc::AF_INET6 => setsockopt_int(fd, libc::IPPROTO_IPV6, libc::IPV6_UNICAST_HOPS, ttl),
|
|
family => Err(io::Error::new(
|
|
io::ErrorKind::InvalidInput,
|
|
format!("unsupported socket family {family} for TTL"),
|
|
)),
|
|
}
|
|
}
|
|
|
|
pub fn set_nodelay(fd: RawFd, enabled: bool) -> io::Result<()> {
|
|
let value: libc::c_int = enabled.into();
|
|
cvt(unsafe {
|
|
libc::setsockopt(
|
|
fd,
|
|
libc::IPPROTO_TCP,
|
|
libc::TCP_NODELAY,
|
|
&value as *const libc::c_int as *const c_void,
|
|
std::mem::size_of_val(&value) as libc::socklen_t,
|
|
)
|
|
})
|
|
.map(|_| ())
|
|
}
|
|
|
|
pub type RecvFuture = Pin<Box<dyn Future<Output = io::Result<Vec<u8>>> + 'static>>;
|
|
pub type SendFuture = Pin<Box<dyn Future<Output = io::Result<usize>> + 'static>>;
|
|
pub type ShutdownFuture = Pin<Box<dyn Future<Output = io::Result<()>> + 'static>>;
|
|
|
|
pub fn recv_future(fd: RawFd, len: usize) -> RecvFuture {
|
|
Box::pin(recv(NetOp::Recv { fd, len, flags: 0 }))
|
|
}
|
|
|
|
pub fn send_future(fd: RawFd, data: Vec<u8>) -> SendFuture {
|
|
Box::pin(send(NetOp::Send { fd, data, flags: 0 }))
|
|
}
|
|
|
|
pub fn shutdown_future(fd: RawFd, how: Shutdown) -> ShutdownFuture {
|
|
Box::pin(shutdown(NetOp::Shutdown { fd, how }))
|
|
}
|
|
|
|
async fn submit_uring<T: Send + 'static, M>(
|
|
fill: impl FnOnce(&mut IoUringSqe),
|
|
map: M,
|
|
) -> io::Result<T>
|
|
where
|
|
M: FnOnce(IoUringCqe) -> io::Result<T> + Send + 'static,
|
|
{
|
|
let (future, handle) = completion_for_current_thread::<io::Result<T>>();
|
|
let callback_handle = handle.clone();
|
|
let token = with_current_driver(|driver| {
|
|
driver.submit_operation(fill, move |cqe| {
|
|
callback_handle.complete(map(cqe));
|
|
})
|
|
})?;
|
|
|
|
handle.set_cancel(move || {
|
|
let _ = with_current_driver(|driver| driver.cancel_operation(token));
|
|
});
|
|
|
|
future.await
|
|
}
|
|
|
|
/// Like [`submit_uring`] but pairs the main SQE with an `IORING_OP_LINK_TIMEOUT`.
|
|
///
|
|
/// If the timeout elapses before the main op completes, the completion callback
|
|
/// receives a CQE with `res = -ECANCELED`. Callers should map `-ECANCELED` to
|
|
/// `io::ErrorKind::TimedOut`.
|
|
async fn submit_uring_with_linked_timeout<T: Send + 'static, M>(
|
|
fill: impl FnOnce(&mut IoUringSqe),
|
|
timeout: Duration,
|
|
map: M,
|
|
) -> io::Result<T>
|
|
where
|
|
M: FnOnce(IoUringCqe) -> io::Result<T> + Send + 'static,
|
|
{
|
|
let (future, handle) = completion_for_current_thread::<io::Result<T>>();
|
|
let callback_handle = handle.clone();
|
|
let token = with_current_driver(|driver| {
|
|
driver.submit_operation_with_linked_timeout(fill, timeout, move |cqe| {
|
|
callback_handle.complete(map(cqe));
|
|
})
|
|
})?;
|
|
|
|
handle.set_cancel(move || {
|
|
let _ = with_current_driver(|driver| driver.cancel_operation(token));
|
|
});
|
|
|
|
future.await
|
|
}
|
|
|
|
async fn offload<T: Send + 'static>(
|
|
task: impl FnOnce() -> io::Result<T> + Send + 'static,
|
|
) -> io::Result<T> {
|
|
let (future, handle) = completion_for_current_thread::<io::Result<T>>();
|
|
thread::Builder::new()
|
|
.name("ruin-runtime-net-offload".into())
|
|
.spawn(move || handle.complete(task()))
|
|
.map_err(io::Error::other)?;
|
|
future.await
|
|
}
|
|
|
|
fn socket_domain(addr: SocketAddr) -> i32 {
|
|
match addr {
|
|
SocketAddr::V4(_) => libc::AF_INET,
|
|
SocketAddr::V6(_) => libc::AF_INET6,
|
|
}
|
|
}
|
|
|
|
fn shutdown_how(how: Shutdown) -> i32 {
|
|
match how {
|
|
Shutdown::Read => libc::SHUT_RD,
|
|
Shutdown::Write => libc::SHUT_WR,
|
|
Shutdown::Both => libc::SHUT_RDWR,
|
|
}
|
|
}
|
|
|
|
fn socket_addr_with(
|
|
op: unsafe extern "C" fn(RawFd, *mut libc::sockaddr, *mut libc::socklen_t) -> libc::c_int,
|
|
fd: RawFd,
|
|
) -> io::Result<SocketAddr> {
|
|
let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
|
|
let mut len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
|
|
cvt(unsafe { op(fd, storage.as_mut_ptr().cast::<libc::sockaddr>(), &mut len) })?;
|
|
let storage = unsafe { storage.assume_init() };
|
|
socket_addr_from_storage(&storage, len)
|
|
}
|
|
|
|
fn set_reuse_addr(fd: RawFd, enabled: bool) -> io::Result<()> {
|
|
setsockopt_int(fd, libc::SOL_SOCKET, libc::SO_REUSEADDR, enabled.into())
|
|
}
|
|
|
|
fn socket_family(fd: RawFd) -> io::Result<i32> {
|
|
let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
|
|
let mut len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
|
|
cvt(unsafe { libc::getsockname(fd, storage.as_mut_ptr().cast::<libc::sockaddr>(), &mut len) })?;
|
|
let storage = unsafe { storage.assume_init() };
|
|
Ok(storage.ss_family as i32)
|
|
}
|
|
|
|
fn getsockopt_int(fd: RawFd, level: i32, name: i32) -> io::Result<i32> {
|
|
let mut value = 0;
|
|
let mut len = std::mem::size_of::<libc::c_int>() as libc::socklen_t;
|
|
cvt(unsafe {
|
|
libc::getsockopt(
|
|
fd,
|
|
level,
|
|
name,
|
|
&mut value as *mut libc::c_int as *mut c_void,
|
|
&mut len,
|
|
)
|
|
})?;
|
|
Ok(value)
|
|
}
|
|
|
|
fn setsockopt_int(fd: RawFd, level: i32, name: i32, value: i32) -> io::Result<()> {
|
|
cvt(unsafe {
|
|
libc::setsockopt(
|
|
fd,
|
|
level,
|
|
name,
|
|
&value as *const libc::c_int as *const c_void,
|
|
std::mem::size_of_val(&value) as libc::socklen_t,
|
|
)
|
|
})
|
|
.map(|_| ())
|
|
}
|
|
|
|
fn socket_addr_from_storage(
|
|
storage: &libc::sockaddr_storage,
|
|
len: libc::socklen_t,
|
|
) -> io::Result<SocketAddr> {
|
|
match storage.ss_family as i32 {
|
|
libc::AF_INET => {
|
|
if len < std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t {
|
|
return Err(io::Error::new(
|
|
io::ErrorKind::InvalidData,
|
|
"short IPv4 socket address from kernel",
|
|
));
|
|
}
|
|
|
|
let addr = unsafe { *(storage as *const _ as *const libc::sockaddr_in) };
|
|
Ok(SocketAddr::V4(SocketAddrV4::new(
|
|
Ipv4Addr::from(addr.sin_addr.s_addr.to_ne_bytes()),
|
|
u16::from_be(addr.sin_port),
|
|
)))
|
|
}
|
|
libc::AF_INET6 => {
|
|
if len < std::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t {
|
|
return Err(io::Error::new(
|
|
io::ErrorKind::InvalidData,
|
|
"short IPv6 socket address from kernel",
|
|
));
|
|
}
|
|
|
|
let addr = unsafe { *(storage as *const _ as *const libc::sockaddr_in6) };
|
|
Ok(SocketAddr::V6(SocketAddrV6::new(
|
|
Ipv6Addr::from(addr.sin6_addr.s6_addr),
|
|
u16::from_be(addr.sin6_port),
|
|
addr.sin6_flowinfo,
|
|
addr.sin6_scope_id,
|
|
)))
|
|
}
|
|
family => Err(io::Error::new(
|
|
io::ErrorKind::InvalidData,
|
|
format!("unsupported socket address family {family}"),
|
|
)),
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
struct RawSocketAddr {
|
|
storage: libc::sockaddr_storage,
|
|
len: libc::socklen_t,
|
|
}
|
|
|
|
impl RawSocketAddr {
|
|
fn from_socket_addr(addr: SocketAddr) -> Self {
|
|
match addr {
|
|
SocketAddr::V4(addr) => {
|
|
let sockaddr = libc::sockaddr_in {
|
|
sin_family: libc::AF_INET as libc::sa_family_t,
|
|
sin_port: addr.port().to_be(),
|
|
sin_addr: libc::in_addr {
|
|
s_addr: u32::from_ne_bytes(addr.ip().octets()),
|
|
},
|
|
sin_zero: [0; 8],
|
|
};
|
|
let mut storage =
|
|
unsafe { MaybeUninit::<libc::sockaddr_storage>::zeroed().assume_init() };
|
|
unsafe {
|
|
std::ptr::write(
|
|
&mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_in,
|
|
sockaddr,
|
|
);
|
|
}
|
|
Self {
|
|
storage,
|
|
len: std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
|
|
}
|
|
}
|
|
SocketAddr::V6(addr) => {
|
|
let sockaddr = libc::sockaddr_in6 {
|
|
sin6_family: libc::AF_INET6 as libc::sa_family_t,
|
|
sin6_port: addr.port().to_be(),
|
|
sin6_flowinfo: addr.flowinfo(),
|
|
sin6_addr: libc::in6_addr {
|
|
s6_addr: addr.ip().octets(),
|
|
},
|
|
sin6_scope_id: addr.scope_id(),
|
|
};
|
|
let mut storage =
|
|
unsafe { MaybeUninit::<libc::sockaddr_storage>::zeroed().assume_init() };
|
|
unsafe {
|
|
std::ptr::write(
|
|
&mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_in6,
|
|
sockaddr,
|
|
);
|
|
}
|
|
Self {
|
|
storage,
|
|
len: std::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn as_ptr(&self) -> *const libc::sockaddr {
|
|
&self.storage as *const libc::sockaddr_storage as *const libc::sockaddr
|
|
}
|
|
|
|
fn len(&self) -> libc::socklen_t {
|
|
self.len
|
|
}
|
|
}
|
|
|
|
fn cqe_to_result(cqe: IoUringCqe) -> io::Result<i32> {
|
|
if cqe.res < 0 {
|
|
Err(io::Error::from_raw_os_error(-cqe.res))
|
|
} else {
|
|
Ok(cqe.res)
|
|
}
|
|
}
|
|
|
|
/// Like [`cqe_to_result`] but maps `-ECANCELED` (timeout fired) to `TimedOut`.
|
|
fn cqe_to_timed_result(cqe: IoUringCqe) -> io::Result<i32> {
|
|
if cqe.res == -libc::ECANCELED {
|
|
return Err(io::Error::new(
|
|
io::ErrorKind::TimedOut,
|
|
"socket operation timed out",
|
|
));
|
|
}
|
|
cqe_to_result(cqe)
|
|
}
|
|
|
|
fn cvt(value: libc::c_int) -> io::Result<libc::c_int> {
|
|
if value == -1 {
|
|
Err(io::Error::last_os_error())
|
|
} else {
|
|
Ok(value)
|
|
}
|
|
}
|
|
|
|
fn should_fallback_to_offload(error: &io::Error) -> bool {
|
|
matches!(
|
|
error.raw_os_error(),
|
|
Some(libc::EINVAL | libc::ENOSYS | libc::EOPNOTSUPP)
|
|
)
|
|
}
|
|
|
|
fn socket_sync(domain: i32, socket_type: i32, protocol: i32, flags: u32) -> io::Result<OwnedFd> {
|
|
let fd = cvt(unsafe { libc::socket(domain, socket_type | flags as i32, protocol) })?;
|
|
Ok(unsafe { OwnedFd::from_raw_fd(fd) })
|
|
}
|
|
|
|
fn connect_sync(fd: RawFd, addr: RawSocketAddr) -> io::Result<()> {
|
|
cvt(unsafe { libc::connect(fd, addr.as_ptr(), addr.len()) }).map(|_| ())
|
|
}
|
|
|
|
fn bind_sync(fd: RawFd, addr: RawSocketAddr) -> io::Result<()> {
|
|
cvt(unsafe { libc::bind(fd, addr.as_ptr(), addr.len()) }).map(|_| ())
|
|
}
|
|
|
|
fn listen_sync(fd: RawFd, backlog: i32) -> io::Result<()> {
|
|
cvt(unsafe { libc::listen(fd, backlog) }).map(|_| ())
|
|
}
|
|
|
|
fn accept_sync(fd: RawFd) -> io::Result<AcceptedSocket> {
|
|
let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
|
|
let mut len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
|
|
let accepted_fd = cvt(unsafe {
|
|
libc::accept4(
|
|
fd,
|
|
storage.as_mut_ptr().cast::<libc::sockaddr>(),
|
|
&mut len,
|
|
libc::SOCK_CLOEXEC,
|
|
)
|
|
})?;
|
|
let storage = unsafe { storage.assume_init() };
|
|
let peer_addr = socket_addr_from_storage(&storage, len)?;
|
|
Ok(AcceptedSocket {
|
|
fd: accepted_fd,
|
|
peer_addr,
|
|
})
|
|
}
|
|
|
|
fn send_sync(fd: RawFd, data: Vec<u8>, flags: i32) -> io::Result<usize> {
|
|
let written = unsafe { libc::send(fd, data.as_ptr().cast::<c_void>(), data.len(), flags) };
|
|
cvt_long(written).map(|written| written as usize)
|
|
}
|
|
|
|
fn recv_sync(fd: RawFd, len: usize, flags: i32) -> io::Result<Vec<u8>> {
|
|
let mut buffer = vec![0; len];
|
|
let read = unsafe {
|
|
libc::recv(
|
|
fd,
|
|
buffer.as_mut_ptr().cast::<c_void>(),
|
|
buffer.len(),
|
|
flags,
|
|
)
|
|
};
|
|
let read = cvt_long(read)? as usize;
|
|
buffer.truncate(read);
|
|
Ok(buffer)
|
|
}
|
|
|
|
fn recv_from_sync(fd: RawFd, len: usize, flags: i32) -> io::Result<ReceivedDatagram> {
|
|
let mut buffer = vec![0; len];
|
|
let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
|
|
let mut addr_len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
|
|
let read = unsafe {
|
|
libc::recvfrom(
|
|
fd,
|
|
buffer.as_mut_ptr().cast::<c_void>(),
|
|
buffer.len(),
|
|
flags,
|
|
storage.as_mut_ptr().cast::<libc::sockaddr>(),
|
|
&mut addr_len,
|
|
)
|
|
};
|
|
let read = cvt_long(read)? as usize;
|
|
buffer.truncate(read);
|
|
let storage = unsafe { storage.assume_init() };
|
|
let peer_addr = socket_addr_from_storage(&storage, addr_len)?;
|
|
Ok(ReceivedDatagram {
|
|
data: buffer,
|
|
peer_addr,
|
|
})
|
|
}
|
|
|
|
fn shutdown_sync(fd: RawFd, how: Shutdown) -> io::Result<()> {
|
|
cvt(unsafe { libc::shutdown(fd, shutdown_how(how)) }).map(|_| ())
|
|
}
|
|
|
|
fn close_sync(fd: RawFd) -> io::Result<()> {
|
|
cvt(unsafe { libc::close(fd) }).map(|_| ())
|
|
}
|
|
|
|
/// Wrapper making `Box<libc::iovec>` sendable across the async CQE boundary.
|
|
///
|
|
/// Safety: `iov_base` points into a `Vec<u8>` that is owned by the same
|
|
/// closure, so the pointer is valid until the CQE fires and the closure drops.
|
|
struct SendIovec(#[allow(dead_code)] Box<libc::iovec>);
|
|
unsafe impl Send for SendIovec {}
|
|
|
|
/// Wrapper making `Box<libc::msghdr>` sendable across the async CQE boundary.
|
|
///
|
|
/// Safety: all raw pointers inside the `msghdr` (`msg_name`, `msg_iov`) point
|
|
/// into heap storage owned by the same closure, so they are valid until the
|
|
/// CQE fires and the closure drops.
|
|
struct SendMsghdr(Box<libc::msghdr>);
|
|
unsafe impl Send for SendMsghdr {}
|
|
|
|
fn cvt_long(value: libc::ssize_t) -> io::Result<libc::ssize_t> {
|
|
if value == -1 {
|
|
Err(io::Error::last_os_error())
|
|
} else {
|
|
Ok(value)
|
|
}
|
|
}
|