Restaged repo, allocator and runtime implemented, ioring-backed async fs/net/channel/timer primitives

lib/runtime/src/sys/linux/net.rs

@@ -0,0 +1,974 @@
+//! Linux networking backend.
+
+use std::ffi::c_void;
+use std::future::Future;
+use std::io;
+use std::mem::MaybeUninit;
+use std::net::{
+    Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, SocketAddrV4, SocketAddrV6, ToSocketAddrs,
+};
+use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd};
+use std::pin::Pin;
+use std::thread;
+use std::time::Duration;
+
+use crate::op::completion::completion_for_current_thread;
+use crate::op::net::{AcceptedSocket, NetOp, ReceivedDatagram};
+use crate::platform::linux_x86_64::runtime::with_current_reactor;
+use crate::platform::linux_x86_64::uring::{
+    IORING_OP_ACCEPT, IORING_OP_BIND, IORING_OP_CLOSE, IORING_OP_CONNECT, IORING_OP_LISTEN,
+    IORING_OP_RECV, IORING_OP_SEND, IORING_OP_SHUTDOWN, IORING_OP_SOCKET, IoUringCqe, IoUringSqe,
+};
+
+const DEFAULT_LISTENER_BACKLOG: i32 = 1024;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ExecutionPath {
+    IoUring,
+    Offload,
+}
+
+pub fn execution_path(op: &NetOp) -> ExecutionPath {
+    match op {
+        NetOp::Socket { .. }
+        | NetOp::Connect { .. }
+        | NetOp::Bind { .. }
+        | NetOp::Listen { .. }
+        | NetOp::Accept { .. }
+        | NetOp::Send { .. }
+        | NetOp::Recv { .. }
+        | NetOp::Shutdown { .. }
+        | NetOp::Close { .. } => ExecutionPath::IoUring,
+        NetOp::SendTo { .. } | NetOp::RecvFrom { .. } => ExecutionPath::Offload,
+    }
+}
+
+pub async fn resolve_addrs<A>(addr: A) -> io::Result<Vec<SocketAddr>>
+where
+    A: ToSocketAddrs + Send + 'static,
+{
+    offload(move || {
+        let addrs = addr.to_socket_addrs()?.collect::<Vec<_>>();
+        if addrs.is_empty() {
+            Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "address resolved to no socket addresses",
+            ))
+        } else {
+            Ok(addrs)
+        }
+    })
+    .await
+}
+
+pub async fn socket(op: NetOp) -> io::Result<OwnedFd> {
+    let NetOp::Socket {
+        domain,
+        socket_type,
+        protocol,
+        flags,
+    } = op
+    else {
+        unreachable!("socket backend called with non-socket op");
+    };
+
+    match submit_uring::<OwnedFd, _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_SOCKET;
+            sqe.fd = domain;
+            sqe.off = socket_type as u64;
+            sqe.len = protocol as u32;
+            sqe.op_flags = flags;
+        },
+        move |cqe| cqe_to_result(cqe).map(|fd| unsafe { OwnedFd::from_raw_fd(fd as RawFd) }),
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => {
+            offload(move || socket_sync(domain, socket_type, protocol, flags)).await
+        }
+        result => result,
+    }
+}
+
+pub async fn connect(op: NetOp) -> io::Result<()> {
+    let NetOp::Connect { fd, addr } = op else {
+        unreachable!("connect backend called with non-connect op");
+    };
+
+    let raw_addr = RawSocketAddr::from_socket_addr(addr);
+    let fallback_addr = raw_addr;
+    let addr_ptr = raw_addr.as_ptr();
+    let addr_len = raw_addr.len();
+    match submit_uring::<(), _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_CONNECT;
+            sqe.fd = fd;
+            sqe.addr = addr_ptr as u64;
+            sqe.off = addr_len as u64;
+        },
+        move |cqe| {
+            let _raw_addr = raw_addr;
+            cqe_to_result(cqe).map(|_| ())
+        },
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => {
+            offload(move || connect_sync(fd, fallback_addr)).await
+        }
+        result => result,
+    }
+}
+
+pub async fn bind(op: NetOp) -> io::Result<()> {
+    let NetOp::Bind { fd, addr } = op else {
+        unreachable!("bind backend called with non-bind op");
+    };
+
+    let raw_addr = RawSocketAddr::from_socket_addr(addr);
+    let fallback_addr = raw_addr;
+    let addr_ptr = raw_addr.as_ptr();
+    let addr_len = raw_addr.len();
+    match submit_uring::<(), _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_BIND;
+            sqe.fd = fd;
+            sqe.addr = addr_ptr as u64;
+            sqe.off = addr_len as u64;
+        },
+        move |cqe| {
+            let _raw_addr = raw_addr;
+            cqe_to_result(cqe).map(|_| ())
+        },
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => {
+            offload(move || bind_sync(fd, fallback_addr)).await
+        }
+        result => result,
+    }
+}
+
+pub async fn listen(op: NetOp) -> io::Result<()> {
+    let NetOp::Listen { fd, backlog } = op else {
+        unreachable!("listen backend called with non-listen op");
+    };
+
+    match submit_uring::<(), _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_LISTEN;
+            sqe.fd = fd;
+            sqe.len = backlog as u32;
+        },
+        move |cqe| cqe_to_result(cqe).map(|_| ()),
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => {
+            offload(move || listen_sync(fd, backlog)).await
+        }
+        result => result,
+    }
+}
+
+pub async fn accept(op: NetOp) -> io::Result<AcceptedSocket> {
+    let NetOp::Accept { fd } = op else {
+        unreachable!("accept backend called with non-accept op");
+    };
+
+    let mut storage = Box::new(MaybeUninit::<libc::sockaddr_storage>::zeroed());
+    let mut addr_len = Box::new(std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t);
+    let storage_ptr = storage.as_mut_ptr();
+    let addr_len_ptr = addr_len.as_mut() as *mut libc::socklen_t;
+
+    match submit_uring::<AcceptedSocket, _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_ACCEPT;
+            sqe.fd = fd;
+            sqe.addr = storage_ptr as u64;
+            sqe.off = addr_len_ptr as u64;
+        },
+        move |cqe| {
+            let accepted_fd = cqe_to_result(cqe)? as RawFd;
+            let storage = unsafe { storage.assume_init() };
+            let peer_addr = socket_addr_from_storage(&storage, *addr_len)?;
+            Ok(AcceptedSocket {
+                fd: accepted_fd,
+                peer_addr,
+            })
+        },
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => offload(move || accept_sync(fd)).await,
+        result => result,
+    }
+}
+
+pub async fn send(op: NetOp) -> io::Result<usize> {
+    let NetOp::Send { fd, data, flags } = op else {
+        unreachable!("send backend called with non-send op");
+    };
+
+    let fallback_data = data.clone();
+    let data_ptr = data.as_ptr();
+    let data_len = data.len();
+    match submit_uring::<usize, _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_SEND;
+            sqe.fd = fd;
+            sqe.addr = data_ptr as u64;
+            sqe.len = data_len as u32;
+            sqe.op_flags = flags as u32;
+        },
+        move |cqe| {
+            let _data = data;
+            cqe_to_result(cqe).map(|written| written as usize)
+        },
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => {
+            offload(move || send_sync(fd, fallback_data, flags)).await
+        }
+        result => result,
+    }
+}
+
+pub async fn send_to(op: NetOp) -> io::Result<usize> {
+    let NetOp::SendTo {
+        fd,
+        target,
+        data,
+        flags,
+    } = op
+    else {
+        unreachable!("send_to backend called with non-send_to op");
+    };
+
+    let raw_addr = RawSocketAddr::from_socket_addr(target);
+    offload(move || send_to_sync(fd, data, raw_addr, flags)).await
+}
+
+pub async fn recv(op: NetOp) -> io::Result<Vec<u8>> {
+    let NetOp::Recv { fd, len, flags } = op else {
+        unreachable!("recv backend called with non-recv op");
+    };
+
+    let mut buffer = vec![0; len];
+    let buffer_ptr = buffer.as_mut_ptr();
+    let buffer_len = buffer.len();
+    match submit_uring::<Vec<u8>, _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_RECV;
+            sqe.fd = fd;
+            sqe.addr = buffer_ptr as u64;
+            sqe.len = buffer_len as u32;
+            sqe.op_flags = flags as u32;
+        },
+        move |cqe| {
+            let read = cqe_to_result(cqe)? as usize;
+            buffer.truncate(read);
+            Ok(buffer)
+        },
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => {
+            offload(move || recv_sync(fd, len, flags)).await
+        }
+        result => result,
+    }
+}
+
+pub async fn recv_from(op: NetOp) -> io::Result<ReceivedDatagram> {
+    let NetOp::RecvFrom { fd, len, flags } = op else {
+        unreachable!("recv_from backend called with non-recv_from op");
+    };
+
+    offload(move || recv_from_sync(fd, len, flags)).await
+}
+
+pub async fn shutdown(op: NetOp) -> io::Result<()> {
+    let NetOp::Shutdown { fd, how } = op else {
+        unreachable!("shutdown backend called with non-shutdown op");
+    };
+
+    let fallback_how = how;
+    match submit_uring::<(), _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_SHUTDOWN;
+            sqe.fd = fd;
+            sqe.len = shutdown_how(how) as u32;
+        },
+        move |cqe| cqe_to_result(cqe).map(|_| ()),
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => {
+            offload(move || shutdown_sync(fd, fallback_how)).await
+        }
+        result => result,
+    }
+}
+
+pub async fn close(op: NetOp) -> io::Result<()> {
+    let NetOp::Close { fd } = op else {
+        unreachable!("close backend called with non-close op");
+    };
+
+    match submit_uring::<(), _>(
+        move |sqe| {
+            sqe.opcode = IORING_OP_CLOSE;
+            sqe.fd = fd;
+        },
+        move |cqe| cqe_to_result(cqe).map(|_| ()),
+    )
+    .await
+    {
+        Err(error) if should_fallback_to_offload(&error) => offload(move || close_sync(fd)).await,
+        result => result,
+    }
+}
+
+pub async fn connect_stream(addr: SocketAddr) -> io::Result<OwnedFd> {
+    let socket = socket(NetOp::Socket {
+        domain: socket_domain(addr),
+        socket_type: libc::SOCK_STREAM,
+        protocol: 0,
+        flags: libc::SOCK_CLOEXEC as u32,
+    })
+    .await?;
+
+    let connect_result = connect(NetOp::Connect {
+        fd: socket.as_raw_fd(),
+        addr,
+    })
+    .await;
+    match connect_result {
+        Ok(()) => Ok(socket),
+        Err(error) => Err(error),
+    }
+}
+
+pub async fn bind_listener(addr: SocketAddr, backlog: Option<i32>) -> io::Result<OwnedFd> {
+    let listener = socket(NetOp::Socket {
+        domain: socket_domain(addr),
+        socket_type: libc::SOCK_STREAM,
+        protocol: 0,
+        flags: libc::SOCK_CLOEXEC as u32,
+    })
+    .await?;
+
+    set_reuse_addr(listener.as_raw_fd(), true)?;
+
+    bind(NetOp::Bind {
+        fd: listener.as_raw_fd(),
+        addr,
+    })
+    .await?;
+    listen(NetOp::Listen {
+        fd: listener.as_raw_fd(),
+        backlog: backlog.unwrap_or(DEFAULT_LISTENER_BACKLOG),
+    })
+    .await?;
+    Ok(listener)
+}
+
+pub async fn bind_datagram(addr: SocketAddr) -> io::Result<OwnedFd> {
+    let socket = socket(NetOp::Socket {
+        domain: socket_domain(addr),
+        socket_type: libc::SOCK_DGRAM,
+        protocol: 0,
+        flags: libc::SOCK_CLOEXEC as u32,
+    })
+    .await?;
+
+    bind(NetOp::Bind {
+        fd: socket.as_raw_fd(),
+        addr,
+    })
+    .await?;
+    Ok(socket)
+}
+
+pub async fn duplicate(fd: RawFd) -> io::Result<OwnedFd> {
+    offload(move || {
+        let duplicated = cvt(unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 0) })?;
+        Ok(unsafe { OwnedFd::from_raw_fd(duplicated) })
+    })
+    .await
+}
+
+pub async fn recv_timeout(
+    fd: RawFd,
+    len: usize,
+    flags: i32,
+    timeout: Duration,
+) -> io::Result<Vec<u8>> {
+    offload(move || {
+        wait_socket(fd, libc::POLLIN, timeout)?;
+        recv_sync(fd, len, flags)
+    })
+    .await
+}
+
+pub async fn send_timeout(
+    fd: RawFd,
+    data: Vec<u8>,
+    flags: i32,
+    timeout: Duration,
+) -> io::Result<usize> {
+    offload(move || {
+        wait_socket(fd, libc::POLLOUT, timeout)?;
+        send_sync(fd, data, flags)
+    })
+    .await
+}
+
+pub async fn recv_from_timeout(
+    fd: RawFd,
+    len: usize,
+    flags: i32,
+    timeout: Duration,
+) -> io::Result<ReceivedDatagram> {
+    offload(move || {
+        wait_socket(fd, libc::POLLIN, timeout)?;
+        recv_from_sync(fd, len, flags)
+    })
+    .await
+}
+
+pub async fn send_to_timeout(
+    fd: RawFd,
+    data: Vec<u8>,
+    target: SocketAddr,
+    flags: i32,
+    timeout: Duration,
+) -> io::Result<usize> {
+    offload(move || {
+        wait_socket(fd, libc::POLLOUT, timeout)?;
+        send_to_sync(fd, data, RawSocketAddr::from_socket_addr(target), flags)
+    })
+    .await
+}
+
+pub async fn connect_stream_timeout(addr: SocketAddr, timeout: Duration) -> io::Result<OwnedFd> {
+    offload(move || connect_stream_timeout_sync(addr, timeout)).await
+}
+
+pub fn local_addr(fd: RawFd) -> io::Result<SocketAddr> {
+    socket_addr_with(libc::getsockname, fd)
+}
+
+pub fn peer_addr(fd: RawFd) -> io::Result<SocketAddr> {
+    socket_addr_with(libc::getpeername, fd)
+}
+
+pub fn nodelay(fd: RawFd) -> io::Result<bool> {
+    let mut value = 0;
+    let mut len = std::mem::size_of::<libc::c_int>() as libc::socklen_t;
+    cvt(unsafe {
+        libc::getsockopt(
+            fd,
+            libc::IPPROTO_TCP,
+            libc::TCP_NODELAY,
+            &mut value as *mut libc::c_int as *mut c_void,
+            &mut len,
+        )
+    })?;
+    Ok(value != 0)
+}
+
+pub fn broadcast(fd: RawFd) -> io::Result<bool> {
+    getsockopt_int(fd, libc::SOL_SOCKET, libc::SO_BROADCAST).map(|value| value != 0)
+}
+
+pub fn set_broadcast(fd: RawFd, enabled: bool) -> io::Result<()> {
+    setsockopt_int(fd, libc::SOL_SOCKET, libc::SO_BROADCAST, enabled.into())
+}
+
+pub fn ttl(fd: RawFd) -> io::Result<u32> {
+    match socket_family(fd)? {
+        libc::AF_INET => {
+            getsockopt_int(fd, libc::IPPROTO_IP, libc::IP_TTL).map(|value| value as u32)
+        }
+        libc::AF_INET6 => getsockopt_int(fd, libc::IPPROTO_IPV6, libc::IPV6_UNICAST_HOPS)
+            .map(|value| value as u32),
+        family => Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            format!("unsupported socket family {family} for TTL"),
+        )),
+    }
+}
+
+pub fn set_ttl(fd: RawFd, ttl: u32) -> io::Result<()> {
+    let ttl = i32::try_from(ttl)
+        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "TTL exceeds i32 range"))?;
+    match socket_family(fd)? {
+        libc::AF_INET => setsockopt_int(fd, libc::IPPROTO_IP, libc::IP_TTL, ttl),
+        libc::AF_INET6 => setsockopt_int(fd, libc::IPPROTO_IPV6, libc::IPV6_UNICAST_HOPS, ttl),
+        family => Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            format!("unsupported socket family {family} for TTL"),
+        )),
+    }
+}
+
+pub fn set_nodelay(fd: RawFd, enabled: bool) -> io::Result<()> {
+    let value: libc::c_int = enabled.into();
+    cvt(unsafe {
+        libc::setsockopt(
+            fd,
+            libc::IPPROTO_TCP,
+            libc::TCP_NODELAY,
+            &value as *const libc::c_int as *const c_void,
+            std::mem::size_of_val(&value) as libc::socklen_t,
+        )
+    })
+    .map(|_| ())
+}
+
+pub type RecvFuture = Pin<Box<dyn Future<Output = io::Result<Vec<u8>>> + 'static>>;
+pub type SendFuture = Pin<Box<dyn Future<Output = io::Result<usize>> + 'static>>;
+pub type ShutdownFuture = Pin<Box<dyn Future<Output = io::Result<()>> + 'static>>;
+
+pub fn recv_future(fd: RawFd, len: usize) -> RecvFuture {
+    Box::pin(recv(NetOp::Recv { fd, len, flags: 0 }))
+}
+
+pub fn send_future(fd: RawFd, data: Vec<u8>) -> SendFuture {
+    Box::pin(send(NetOp::Send { fd, data, flags: 0 }))
+}
+
+pub fn shutdown_future(fd: RawFd, how: Shutdown) -> ShutdownFuture {
+    Box::pin(shutdown(NetOp::Shutdown { fd, how }))
+}
+
+async fn submit_uring<T: Send + 'static, M>(
+    fill: impl FnOnce(&mut IoUringSqe),
+    map: M,
+) -> io::Result<T>
+where
+    M: FnOnce(IoUringCqe) -> io::Result<T> + Send + 'static,
+{
+    let (future, handle) = completion_for_current_thread::<io::Result<T>>();
+    let callback_handle = handle.clone();
+    let token = with_current_reactor(|reactor| {
+        reactor.submit_operation(fill, move |cqe| {
+            callback_handle.complete(map(cqe));
+        })
+    })?;
+
+    handle.set_cancel(move || {
+        let _ = with_current_reactor(|reactor| reactor.cancel_operation(token));
+    });
+
+    future.await
+}
+
+async fn offload<T: Send + 'static>(
+    task: impl FnOnce() -> io::Result<T> + Send + 'static,
+) -> io::Result<T> {
+    let (future, handle) = completion_for_current_thread::<io::Result<T>>();
+    thread::Builder::new()
+        .name("ruin-runtime-net-offload".into())
+        .spawn(move || handle.complete(task()))
+        .map_err(io::Error::other)?;
+    future.await
+}
+
+fn socket_domain(addr: SocketAddr) -> i32 {
+    match addr {
+        SocketAddr::V4(_) => libc::AF_INET,
+        SocketAddr::V6(_) => libc::AF_INET6,
+    }
+}
+
+fn shutdown_how(how: Shutdown) -> i32 {
+    match how {
+        Shutdown::Read => libc::SHUT_RD,
+        Shutdown::Write => libc::SHUT_WR,
+        Shutdown::Both => libc::SHUT_RDWR,
+    }
+}
+
+fn socket_addr_with(
+    op: unsafe extern "C" fn(RawFd, *mut libc::sockaddr, *mut libc::socklen_t) -> libc::c_int,
+    fd: RawFd,
+) -> io::Result<SocketAddr> {
+    let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
+    let mut len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
+    cvt(unsafe { op(fd, storage.as_mut_ptr().cast::<libc::sockaddr>(), &mut len) })?;
+    let storage = unsafe { storage.assume_init() };
+    socket_addr_from_storage(&storage, len)
+}
+
+fn set_reuse_addr(fd: RawFd, enabled: bool) -> io::Result<()> {
+    setsockopt_int(fd, libc::SOL_SOCKET, libc::SO_REUSEADDR, enabled.into())
+}
+
+fn socket_family(fd: RawFd) -> io::Result<i32> {
+    let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
+    let mut len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
+    cvt(unsafe { libc::getsockname(fd, storage.as_mut_ptr().cast::<libc::sockaddr>(), &mut len) })?;
+    let storage = unsafe { storage.assume_init() };
+    Ok(storage.ss_family as i32)
+}
+
+fn getsockopt_int(fd: RawFd, level: i32, name: i32) -> io::Result<i32> {
+    let mut value = 0;
+    let mut len = std::mem::size_of::<libc::c_int>() as libc::socklen_t;
+    cvt(unsafe {
+        libc::getsockopt(
+            fd,
+            level,
+            name,
+            &mut value as *mut libc::c_int as *mut c_void,
+            &mut len,
+        )
+    })?;
+    Ok(value)
+}
+
+fn setsockopt_int(fd: RawFd, level: i32, name: i32, value: i32) -> io::Result<()> {
+    cvt(unsafe {
+        libc::setsockopt(
+            fd,
+            level,
+            name,
+            &value as *const libc::c_int as *const c_void,
+            std::mem::size_of_val(&value) as libc::socklen_t,
+        )
+    })
+    .map(|_| ())
+}
+
+fn socket_addr_from_storage(
+    storage: &libc::sockaddr_storage,
+    len: libc::socklen_t,
+) -> io::Result<SocketAddr> {
+    match storage.ss_family as i32 {
+        libc::AF_INET => {
+            if len < std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    "short IPv4 socket address from kernel",
+                ));
+            }
+
+            let addr = unsafe { *(storage as *const _ as *const libc::sockaddr_in) };
+            Ok(SocketAddr::V4(SocketAddrV4::new(
+                Ipv4Addr::from(addr.sin_addr.s_addr.to_ne_bytes()),
+                u16::from_be(addr.sin_port),
+            )))
+        }
+        libc::AF_INET6 => {
+            if len < std::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    "short IPv6 socket address from kernel",
+                ));
+            }
+
+            let addr = unsafe { *(storage as *const _ as *const libc::sockaddr_in6) };
+            Ok(SocketAddr::V6(SocketAddrV6::new(
+                Ipv6Addr::from(addr.sin6_addr.s6_addr),
+                u16::from_be(addr.sin6_port),
+                addr.sin6_flowinfo,
+                addr.sin6_scope_id,
+            )))
+        }
+        family => Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!("unsupported socket address family {family}"),
+        )),
+    }
+}
+
+#[derive(Clone, Copy)]
+struct RawSocketAddr {
+    storage: libc::sockaddr_storage,
+    len: libc::socklen_t,
+}
+
+impl RawSocketAddr {
+    fn from_socket_addr(addr: SocketAddr) -> Self {
+        match addr {
+            SocketAddr::V4(addr) => {
+                let sockaddr = libc::sockaddr_in {
+                    sin_family: libc::AF_INET as libc::sa_family_t,
+                    sin_port: addr.port().to_be(),
+                    sin_addr: libc::in_addr {
+                        s_addr: u32::from_ne_bytes(addr.ip().octets()),
+                    },
+                    sin_zero: [0; 8],
+                };
+                let mut storage =
+                    unsafe { MaybeUninit::<libc::sockaddr_storage>::zeroed().assume_init() };
+                unsafe {
+                    std::ptr::write(
+                        &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_in,
+                        sockaddr,
+                    );
+                }
+                Self {
+                    storage,
+                    len: std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
+                }
+            }
+            SocketAddr::V6(addr) => {
+                let sockaddr = libc::sockaddr_in6 {
+                    sin6_family: libc::AF_INET6 as libc::sa_family_t,
+                    sin6_port: addr.port().to_be(),
+                    sin6_flowinfo: addr.flowinfo(),
+                    sin6_addr: libc::in6_addr {
+                        s6_addr: addr.ip().octets(),
+                    },
+                    sin6_scope_id: addr.scope_id(),
+                };
+                let mut storage =
+                    unsafe { MaybeUninit::<libc::sockaddr_storage>::zeroed().assume_init() };
+                unsafe {
+                    std::ptr::write(
+                        &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_in6,
+                        sockaddr,
+                    );
+                }
+                Self {
+                    storage,
+                    len: std::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t,
+                }
+            }
+        }
+    }
+
+    fn as_ptr(&self) -> *const libc::sockaddr {
+        &self.storage as *const libc::sockaddr_storage as *const libc::sockaddr
+    }
+
+    fn len(&self) -> libc::socklen_t {
+        self.len
+    }
+}
+
+fn cqe_to_result(cqe: IoUringCqe) -> io::Result<i32> {
+    if cqe.res < 0 {
+        Err(io::Error::from_raw_os_error(-cqe.res))
+    } else {
+        Ok(cqe.res)
+    }
+}
+
+fn cvt(value: libc::c_int) -> io::Result<libc::c_int> {
+    if value == -1 {
+        Err(io::Error::last_os_error())
+    } else {
+        Ok(value)
+    }
+}
+
+fn should_fallback_to_offload(error: &io::Error) -> bool {
+    matches!(
+        error.raw_os_error(),
+        Some(libc::EINVAL | libc::ENOSYS | libc::EOPNOTSUPP)
+    )
+}
+
+fn socket_sync(domain: i32, socket_type: i32, protocol: i32, flags: u32) -> io::Result<OwnedFd> {
+    let fd = cvt(unsafe { libc::socket(domain, socket_type | flags as i32, protocol) })?;
+    Ok(unsafe { OwnedFd::from_raw_fd(fd) })
+}
+
+fn connect_sync(fd: RawFd, addr: RawSocketAddr) -> io::Result<()> {
+    cvt(unsafe { libc::connect(fd, addr.as_ptr(), addr.len()) }).map(|_| ())
+}
+
+fn bind_sync(fd: RawFd, addr: RawSocketAddr) -> io::Result<()> {
+    cvt(unsafe { libc::bind(fd, addr.as_ptr(), addr.len()) }).map(|_| ())
+}
+
+fn listen_sync(fd: RawFd, backlog: i32) -> io::Result<()> {
+    cvt(unsafe { libc::listen(fd, backlog) }).map(|_| ())
+}
+
+fn accept_sync(fd: RawFd) -> io::Result<AcceptedSocket> {
+    let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
+    let mut len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
+    let accepted_fd = cvt(unsafe {
+        libc::accept4(
+            fd,
+            storage.as_mut_ptr().cast::<libc::sockaddr>(),
+            &mut len,
+            libc::SOCK_CLOEXEC,
+        )
+    })?;
+    let storage = unsafe { storage.assume_init() };
+    let peer_addr = socket_addr_from_storage(&storage, len)?;
+    Ok(AcceptedSocket {
+        fd: accepted_fd,
+        peer_addr,
+    })
+}
+
+fn send_sync(fd: RawFd, data: Vec<u8>, flags: i32) -> io::Result<usize> {
+    let written = unsafe { libc::send(fd, data.as_ptr().cast::<c_void>(), data.len(), flags) };
+    cvt_long(written).map(|written| written as usize)
+}
+
+fn send_to_sync(fd: RawFd, data: Vec<u8>, target: RawSocketAddr, flags: i32) -> io::Result<usize> {
+    let written = unsafe {
+        libc::sendto(
+            fd,
+            data.as_ptr().cast::<c_void>(),
+            data.len(),
+            flags,
+            target.as_ptr(),
+            target.len(),
+        )
+    };
+    cvt_long(written).map(|written| written as usize)
+}
+
+fn recv_sync(fd: RawFd, len: usize, flags: i32) -> io::Result<Vec<u8>> {
+    let mut buffer = vec![0; len];
+    let read = unsafe {
+        libc::recv(
+            fd,
+            buffer.as_mut_ptr().cast::<c_void>(),
+            buffer.len(),
+            flags,
+        )
+    };
+    let read = cvt_long(read)? as usize;
+    buffer.truncate(read);
+    Ok(buffer)
+}
+
+fn recv_from_sync(fd: RawFd, len: usize, flags: i32) -> io::Result<ReceivedDatagram> {
+    let mut buffer = vec![0; len];
+    let mut storage = MaybeUninit::<libc::sockaddr_storage>::zeroed();
+    let mut addr_len = std::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
+    let read = unsafe {
+        libc::recvfrom(
+            fd,
+            buffer.as_mut_ptr().cast::<c_void>(),
+            buffer.len(),
+            flags,
+            storage.as_mut_ptr().cast::<libc::sockaddr>(),
+            &mut addr_len,
+        )
+    };
+    let read = cvt_long(read)? as usize;
+    buffer.truncate(read);
+    let storage = unsafe { storage.assume_init() };
+    let peer_addr = socket_addr_from_storage(&storage, addr_len)?;
+    Ok(ReceivedDatagram {
+        data: buffer,
+        peer_addr,
+    })
+}
+
+fn shutdown_sync(fd: RawFd, how: Shutdown) -> io::Result<()> {
+    cvt(unsafe { libc::shutdown(fd, shutdown_how(how)) }).map(|_| ())
+}
+
+fn close_sync(fd: RawFd) -> io::Result<()> {
+    cvt(unsafe { libc::close(fd) }).map(|_| ())
+}
+
+fn connect_stream_timeout_sync(addr: SocketAddr, timeout: Duration) -> io::Result<OwnedFd> {
+    let fd = cvt(unsafe {
+        libc::socket(
+            socket_domain(addr),
+            libc::SOCK_STREAM | libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK,
+            0,
+        )
+    })?;
+    let raw_addr = RawSocketAddr::from_socket_addr(addr);
+    let connect_result = unsafe { libc::connect(fd, raw_addr.as_ptr(), raw_addr.len()) };
+    if connect_result == 0 {
+        set_nonblocking(fd, false)?;
+        return Ok(unsafe { OwnedFd::from_raw_fd(fd) });
+    }
+
+    let error = io::Error::last_os_error();
+    if error.raw_os_error() != Some(libc::EINPROGRESS) {
+        let _ = close_sync(fd);
+        return Err(error);
+    }
+
+    let completion = wait_socket(fd, libc::POLLOUT, timeout)
+        .and_then(|_| getsockopt_int(fd, libc::SOL_SOCKET, libc::SO_ERROR));
+    match completion {
+        Ok(0) => {
+            set_nonblocking(fd, false)?;
+            Ok(unsafe { OwnedFd::from_raw_fd(fd) })
+        }
+        Ok(code) => {
+            let _ = close_sync(fd);
+            Err(io::Error::from_raw_os_error(code))
+        }
+        Err(error) => {
+            let _ = close_sync(fd);
+            Err(error)
+        }
+    }
+}
+
+fn set_nonblocking(fd: RawFd, enabled: bool) -> io::Result<()> {
+    let flags = cvt(unsafe { libc::fcntl(fd, libc::F_GETFL) })?;
+    let new_flags = if enabled {
+        flags | libc::O_NONBLOCK
+    } else {
+        flags & !libc::O_NONBLOCK
+    };
+    cvt(unsafe { libc::fcntl(fd, libc::F_SETFL, new_flags) }).map(|_| ())
+}
+
+fn wait_socket(fd: RawFd, events: i16, timeout: Duration) -> io::Result<()> {
+    let timeout_ms = timeout
+        .as_millis()
+        .min(i32::MAX as u128)
+        .try_into()
+        .unwrap_or(i32::MAX);
+
+    loop {
+        let mut poll_fd = libc::pollfd {
+            fd,
+            events,
+            revents: 0,
+        };
+        let result = unsafe { libc::poll(&mut poll_fd, 1, timeout_ms) };
+        if result == 0 {
+            return Err(io::Error::new(
+                io::ErrorKind::TimedOut,
+                "socket operation timed out",
+            ));
+        }
+        if result < 0 {
+            let error = io::Error::last_os_error();
+            if error.kind() == io::ErrorKind::Interrupted {
+                continue;
+            }
+            return Err(error);
+        }
+        if poll_fd.revents & (libc::POLLERR | libc::POLLHUP | libc::POLLNVAL) != 0 {
+            let socket_error = getsockopt_int(fd, libc::SOL_SOCKET, libc::SO_ERROR).unwrap_or(0);
+            if socket_error != 0 {
+                return Err(io::Error::from_raw_os_error(socket_error));
+            }
+        }
+        return Ok(());
+    }
+}
+
+fn cvt_long(value: libc::ssize_t) -> io::Result<libc::ssize_t> {
+    if value == -1 {
+        Err(io::Error::last_os_error())
+    } else {
+        Ok(value)
+    }
+}