Merge pull request #125 from expressvpn/CVPN-1590-uring-configure-sqpoll

lightway-app-utils: iouring: Make sqpoll idle time configurable

lightway-app-utils/examples/udprelay.rs

@@ -11,6 +11,7 @@ use pnet::packet::ipv4::MutableIpv4Packet;
 
 use std::net::{Ipv4Addr, SocketAddr};
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::net::UdpSocket;
 use tokio_tun::Tun;
 
@@ -196,7 +197,14 @@ struct TunIOUring {
 
 impl TunIOUring {
     async fn new(tun: Tun, ring_size: usize, channel_size: usize) -> Result<Self> {
-        let tun_iouring = IOUring::new(Arc::new(tun), ring_size, channel_size, TUN_MTU).await?;
+        let tun_iouring = IOUring::new(
+            Arc::new(tun),
+            ring_size,
+            channel_size,
+            TUN_MTU,
+            Duration::from_millis(100),
+        )
+        .await?;
 
         Ok(Self { tun_iouring })
     }

lightway-app-utils/src/iouring.rs

@@ -5,13 +5,14 @@ use thiserror::Error;
 
 use crate::metrics;
 use io_uring::{
-    cqueue::Entry as CEntry, opcode, squeue::Entry as SEntry, types::Fixed, IoUring,
+    cqueue::Entry as CEntry, opcode, squeue::Entry as SEntry, types::Fixed, Builder, IoUring,
     SubmissionQueue, Submitter,
 };
 use std::{
     os::fd::{AsRawFd, RawFd},
     sync::Arc,
     thread,
+    time::Duration,
 };
 use tokio::{
     io::AsyncReadExt,
@@ -20,7 +21,6 @@ use tokio::{
 use tokio_eventfd::EventFd;
 
 const REGISTERED_FD_INDEX: u32 = 0;
-const IOURING_SQPOLL_IDLE_TIME: u32 = 100;
 
 /// IO-uring Struct
 pub struct IOUring<T: AsRawFd> {
@@ -52,6 +52,7 @@ impl<T: AsRawFd> IOUring<T> {
         ring_size: usize,
         channel_size: usize,
         mtu: usize,
+        sqpoll_idle_time: Duration,
     ) -> Result<Self> {
         let fd = owned_fd.as_raw_fd();
 
@@ -68,6 +69,7 @@ impl<T: AsRawFd> IOUring<T> {
                         fd,
                         ring_size,
                         mtu,
+                        sqpoll_idle_time,
                         tx_queue_receiver,
                         rx_queue_sender,
                     ))
@@ -266,13 +268,22 @@ async fn iouring_task(
     fd: RawFd,
     ring_size: usize,
     mtu: usize,
+    sqpoll_idle_time: Duration,
     mut tx_queue: mpsc::Receiver<Bytes>,
     rx_queue: mpsc::Sender<BytesMut>,
 ) -> Result<()> {
     let mut event_fd: EventFd = EventFd::new(0, false)?;
-    let mut ring: IoUring<SEntry, CEntry> = IoUring::builder()
+    let mut builder: Builder<SEntry, CEntry> = IoUring::builder();
+
+    if sqpoll_idle_time.as_millis() > 0 {
+        let idle_time: u32 = sqpoll_idle_time
+            .as_millis()
+            .try_into()
+            .with_context(|| "invalid sqpoll idle time")?;
         // This setting makes CPU go 100% when there is continuous traffic
-        .setup_sqpoll(IOURING_SQPOLL_IDLE_TIME) // Needs 5.13
+        builder.setup_sqpoll(idle_time); // Needs 5.13
+    }
+    let mut ring = builder
         .build(ring_size as u32)
         .inspect_err(|e| tracing::error!("iouring setup failed: {e}"))?;
 
@@ -284,7 +295,12 @@ async fn iouring_task(
 
     // Using half of total io-uring size for rx and half for tx
     let nr_tx_rx_slots = (ring_size / 2) as isize;
-    tracing::info!(ring_size, nr_tx_rx_slots, "uring main task");
+    tracing::info!(
+        ring_size,
+        nr_tx_rx_slots,
+        ?sqpoll_idle_time,
+        "uring main task"
+    );
 
     let mut rx_slots: Vec<_> = (0..nr_tx_rx_slots).map(SlotIdx::Rx).collect();
     let mut rx_state: Vec<_> = rx_slots

lightway-app-utils/src/tun.rs

@@ -2,10 +2,13 @@ use anyhow::Result;
 use bytes::BytesMut;
 use lightway_core::IOCallbackResult;
 
+#[cfg(feature = "io-uring")]
+use std::time::Duration;
 use std::{
     ops::Deref,
     os::fd::{AsRawFd, RawFd},
 };
+
 use tun::{AbstractDevice, AsyncDevice as TokioTun};
 
 pub use tun::Configuration as TunConfig;
@@ -33,8 +36,14 @@ impl Tun {
 
     /// Create new `Tun` instance with iouring read/write
     #[cfg(feature = "io-uring")]
-    pub async fn iouring(config: TunConfig, ring_size: usize) -> Result<Self> {
-        Ok(Self::IoUring(TunIoUring::new(config, ring_size).await?))
+    pub async fn iouring(
+        config: TunConfig,
+        ring_size: usize,
+        sqpoll_idle_time: Duration,
+    ) -> Result<Self> {
+        Ok(Self::IoUring(
+            TunIoUring::new(config, ring_size, sqpoll_idle_time).await?,
+        ))
     }
 
     /// Recv a packet from `Tun`
@@ -143,10 +152,15 @@ pub struct TunIoUring {
 #[cfg(feature = "io-uring")]
 impl TunIoUring {
     /// Create `TunIoUring` struct
-    pub async fn new(config: TunConfig, ring_size: usize) -> Result<Self> {
+    pub async fn new(
+        config: TunConfig,
+        ring_size: usize,
+        sqpoll_idle_time: Duration,
+    ) -> Result<Self> {
         let tun = TunDirect::new(config)?;
         let mtu = tun.mtu();
-        let tun_io_uring = IOUring::new(Arc::new(tun), ring_size, ring_size, mtu).await?;
+        let tun_io_uring =
+            IOUring::new(Arc::new(tun), ring_size, ring_size, mtu, sqpoll_idle_time).await?;
 
         Ok(TunIoUring { tun_io_uring })
     }

lightway-client/src/args.rs

@@ -102,6 +102,12 @@ pub struct Config {
     #[clap(long, default_value_t = 1024)]
     pub iouring_entry_count: usize,
 
+    /// IO-uring sqpoll idle time. If non-zero use a kernel thread to
+    /// perform submission queue polling. After the given idle time
+    /// the thread will go to sleep.
+    #[clap(long, default_value = "100ms")]
+    pub iouring_sqpoll_idle_time: Duration,
+
     /// Server domain name
     #[clap(long, default_value = None)]
     pub server_dn: Option<String>,

lightway-client/src/io/inside/tun.rs

@@ -1,4 +1,6 @@
 use std::net::Ipv4Addr;
+#[cfg(feature = "io-uring")]
+use std::time::Duration;
 
 use anyhow::Result;
 use async_trait::async_trait;
@@ -24,11 +26,13 @@ impl Tun {
         tun: TunConfig,
         ip: Ipv4Addr,
         dns_ip: Ipv4Addr,
-        #[cfg(feature = "io-uring")] iouring: Option<usize>,
+        #[cfg(feature = "io-uring")] iouring: Option<(usize, Duration)>,
     ) -> Result<Self> {
         #[cfg(feature = "io-uring")]
         let tun = match iouring {
-            Some(ring_size) => AppUtilsTun::iouring(tun, ring_size).await?,
+            Some((ring_size, sqpoll_idle_time)) => {
+                AppUtilsTun::iouring(tun, ring_size, sqpoll_idle_time).await?
+            }
             None => AppUtilsTun::direct(tun).await?,
         };
         #[cfg(not(feature = "io-uring"))]

lightway-client/src/lib.rs

@@ -134,6 +134,12 @@ pub struct ClientConfig<'cert, A: 'static + Send + EventCallback> {
     #[cfg(feature = "io-uring")]
     pub iouring_entry_count: usize,
 
+    /// IO-uring sqpoll idle time. If non-zero use a kernel thread to
+    /// perform submission queue polling. After the given idle time
+    /// the thread will go to sleep.
+    #[cfg(feature = "io-uring")]
+    pub iouring_sqpoll_idle_time: Duration,
+
     /// Server domain name to validate
     pub server_dn: Option<String>,
 
@@ -388,7 +394,7 @@ pub async fn client<A: 'static + Send + EventCallback>(
 
     #[cfg(feature = "io-uring")]
     let iouring = if config.enable_tun_iouring {
-        Some(config.iouring_entry_count)
+        Some((config.iouring_entry_count, config.iouring_sqpoll_idle_time))
     } else {
         None
     };

lightway-client/src/main.rs

@@ -96,6 +96,8 @@ async fn main() -> Result<()> {
         enable_tun_iouring: config.enable_tun_iouring,
         #[cfg(feature = "io-uring")]
         iouring_entry_count: config.iouring_entry_count,
+        #[cfg(feature = "io-uring")]
+        iouring_sqpoll_idle_time: config.iouring_sqpoll_idle_time.into(),
         server_dn: config.server_dn,
         server: config.server,
         inside_plugins: Default::default(),

lightway-server/src/args.rs

@@ -81,6 +81,12 @@ pub struct Config {
     #[clap(long, default_value_t = 1024)]
     pub iouring_entry_count: usize,
 
+    /// IO-uring sqpoll idle time. If non-zero use a kernel thread to
+    /// perform submission queue polling. After the given idle time
+    /// the thread will go to sleep.
+    #[clap(long, default_value = "100ms")]
+    pub iouring_sqpoll_idle_time: Duration,
+
     /// Log format
     #[clap(long, value_enum, default_value_t = LogFormat::Full)]
     pub log_format: LogFormat,

lightway-server/src/io/inside/tun.rs

@@ -10,13 +10,16 @@ use lightway_core::{
 };
 use std::os::fd::{AsRawFd, RawFd};
 use std::sync::Arc;
+use std::time::Duration;
 
 pub(crate) struct Tun(AppUtilsTun);
 
 impl Tun {
-    pub async fn new(tun: TunConfig, iouring: Option<usize>) -> Result<Self> {
+    pub async fn new(tun: TunConfig, iouring: Option<(usize, Duration)>) -> Result<Self> {
         let tun = match iouring {
-            Some(ring_size) => AppUtilsTun::iouring(tun, ring_size).await?,
+            Some((ring_size, sqpoll_idle_time)) => {
+                AppUtilsTun::iouring(tun, ring_size, sqpoll_idle_time).await?
+            }
             None => AppUtilsTun::direct(tun).await?,
         };
         Ok(Tun(tun))

lightway-server/src/lib.rs

@@ -118,6 +118,9 @@ pub struct ServerConfig<SA: for<'a> ServerAuth<AuthState<'a>>> {
     /// IO-uring submission queue count
     pub iouring_entry_count: usize,
 
+    /// IO-uring sqpoll idle time.
+    pub iouring_sqpoll_idle_time: Duration,
+
     /// The key update interval for DTLS/TLS 1.3 connections
     pub key_update_interval: Duration,
 
@@ -173,7 +176,7 @@ pub async fn server<SA: for<'a> ServerAuth<AuthState<'a>> + Sync + Send + 'stati
     let auth = Arc::new(AuthAdapter(config.auth));
 
     let iouring = if config.enable_tun_iouring {
-        Some(config.iouring_entry_count)
+        Some((config.iouring_entry_count, config.iouring_sqpoll_idle_time))
     } else {
         None
     };

lightway-server/src/main.rs

@@ -132,6 +132,7 @@ async fn main() -> Result<()> {
         enable_pqc: config.enable_pqc,
         enable_tun_iouring: config.enable_tun_iouring,
         iouring_entry_count: config.iouring_entry_count,
+        iouring_sqpoll_idle_time: config.iouring_sqpoll_idle_time.into(),
         key_update_interval: config.key_update_interval.into(),
         inside_plugins: Default::default(),
         outside_plugins: Default::default(),

tests/client/client_config.yaml

@@ -15,6 +15,7 @@ outside_mtu: 1500
 cipher: aes256
 enable_pqc: false
 enable_tun_iouring: false
+# iouring_sqpoll_idle_time: 100ms
 iouring_entry_count: 1024
 keepalive_interval: 0s
 keepalive_timeout: 0s

tests/server/server_config.yaml

@@ -18,6 +18,7 @@ log_format: full
 log_level: info
 enable_pqc: false
 enable_tun_iouring: false
+# iouring_sqpoll_idle_time: 100ms
 iouring_entry_count: 1024
 key_update_interval: 15m
 user_db: "tests/server/lwpasswd"