From 59f47d5786e3dcd8f2c3825623d67bf92b77d15f Mon Sep 17 00:00:00 2001 From: Sijie Yang Date: Thu, 4 Jul 2024 10:36:17 +0800 Subject: [PATCH 1/7] Fix stream operations that should mark conn as tickable (#305) --- src/connection/connection.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/connection/connection.rs b/src/connection/connection.rs index 9b0622fd..c95986c6 100644 --- a/src/connection/connection.rs +++ b/src/connection/connection.rs @@ -3635,11 +3635,13 @@ impl Connection { /// Set want write flag for a stream. pub fn stream_want_write(&mut self, stream_id: u64, want: bool) -> Result<()> { + self.mark_tickable(true); self.streams.want_write(stream_id, want) } /// Set want read flag for a stream. pub fn stream_want_read(&mut self, stream_id: u64, want: bool) -> Result<()> { + self.mark_tickable(true); self.streams.want_read(stream_id, want) } From 05bc265b810b84c6a1f30ffb4d8240940e11f439 Mon Sep 17 00:00:00 2001 From: Sijie Yang Date: Thu, 4 Jul 2024 18:01:52 +0800 Subject: [PATCH 2/7] Update config API for congestion control (#308) --- include/tquic.h | 36 +++++++++++ src/congestion_control/bbr.rs | 52 +++++++++++++--- src/congestion_control/bbr3.rs | 16 ++--- src/congestion_control/congestion_control.rs | 37 ++--------- src/congestion_control/copa.rs | 15 +++-- src/congestion_control/cubic.rs | 30 +++++---- src/congestion_control/dummy.rs | 49 +++++++++++++-- src/ffi.rs | 42 +++++++++++++ src/lib.rs | 64 ++++++++++++++++++++ 9 files changed, 272 insertions(+), 69 deletions(-) diff --git a/include/tquic.h b/include/tquic.h index 2b0dd586..c11fc511 100644 --- a/include/tquic.h +++ b/include/tquic.h @@ -430,6 +430,42 @@ void quic_config_set_initial_congestion_window(struct quic_config_t *config, uin */ void quic_config_set_min_congestion_window(struct quic_config_t *config, uint64_t v); +/** + * Set the threshold for slow start in packets. + * The default value is the maximum value of u64. + */ +void quic_config_set_slow_start_thresh(struct quic_config_t *config, uint64_t v); + +/** + * Set the minimum duration for BBR ProbeRTT state in milliseconds. + * The default value is 200 milliseconds. + */ +void quic_config_set_bbr_probe_rtt_duration(struct quic_config_t *config, uint64_t v); + +/** + * Enable using a cwnd based on bdp during ProbeRTT state. + * The default value is false. + */ +void quic_config_enable_bbr_probe_rtt_based_on_bdp(struct quic_config_t *config, bool v); + +/** + * Set the cwnd gain for BBR ProbeRTT state. + * The default value is 0.75 + */ +void quic_config_set_bbr_probe_rtt_cwnd_gain(struct quic_config_t *config, double v); + +/** + * Set the length of the BBR RTProp min filter window in milliseconds. + * The default value is 10000 milliseconds. + */ +void quic_config_set_bbr_rtprop_filter_len(struct quic_config_t *config, uint64_t v); + +/** + * Set the cwnd gain for BBR ProbeBW state. + * The default value is 2.0 + */ +void quic_config_set_bbr_probe_bw_cwnd_gain(struct quic_config_t *config, double v); + /** * Set the initial RTT in milliseconds. The default value is 333ms. * The configuration should be changed with caution. Setting a value less than the default diff --git a/src/congestion_control/bbr.rs b/src/congestion_control/bbr.rs index ba84caf6..0dcb4bf7 100644 --- a/src/congestion_control/bbr.rs +++ b/src/congestion_control/bbr.rs @@ -35,6 +35,7 @@ use super::minmax::MinMax; use super::{CongestionController, CongestionStats}; use crate::connection::rtt::RttEstimator; use crate::connection::space::{RateSamplePacketState, SentPacket}; +use crate::RecoveryConfig; /// BBR configurable parameters. #[derive(Debug)] @@ -48,21 +49,43 @@ pub struct BbrConfig { /// Initial Smoothed rtt. initial_rtt: Option, + /// The minimum duration for ProbeRTT state + probe_rtt_duration: Duration, + + /// If true, use a cwnd of `probe_rtt_cwnd_gain*BDP` during ProbeRtt state + /// instead of minimal window. + probe_rtt_based_on_bdp: bool, + + /// The cwnd gain for ProbeRTT state + probe_rtt_cwnd_gain: f64, + + /// The length of the RTProp min filter window + rtprop_filter_len: Duration, + + /// The cwnd gain for ProbeBW state + probe_bw_cwnd_gain: f64, + /// Max datagram size in bytes. max_datagram_size: u64, } impl BbrConfig { - pub fn new( - min_cwnd: u64, - initial_cwnd: u64, - initial_rtt: Option, - max_datagram_size: u64, - ) -> Self { + pub fn from(conf: &RecoveryConfig) -> Self { + let max_datagram_size = conf.max_datagram_size as u64; + let min_cwnd = conf.min_congestion_window.saturating_mul(max_datagram_size); + let initial_cwnd = conf + .initial_congestion_window + .saturating_mul(max_datagram_size); + Self { min_cwnd, initial_cwnd, - initial_rtt, + initial_rtt: Some(conf.initial_rtt), + probe_rtt_duration: conf.bbr_probe_rtt_duration, + probe_rtt_based_on_bdp: conf.bbr_probe_rtt_based_on_bdp, + probe_rtt_cwnd_gain: conf.bbr_probe_rtt_cwnd_gain, + rtprop_filter_len: conf.bbr_rtprop_filter_len, + probe_bw_cwnd_gain: conf.bbr_probe_bw_cwnd_gain, max_datagram_size, } } @@ -74,6 +97,11 @@ impl Default for BbrConfig { min_cwnd: 4 * crate::DEFAULT_SEND_UDP_PAYLOAD_SIZE as u64, initial_cwnd: 80 * crate::DEFAULT_SEND_UDP_PAYLOAD_SIZE as u64, initial_rtt: Some(crate::INITIAL_RTT), + probe_rtt_duration: PROBE_RTT_DURATION, + probe_rtt_based_on_bdp: false, + probe_rtt_cwnd_gain: 0.75, + rtprop_filter_len: RTPROP_FILTER_LEN, + probe_bw_cwnd_gain: 2.0, max_datagram_size: crate::DEFAULT_SEND_UDP_PAYLOAD_SIZE as u64, } } @@ -547,7 +575,7 @@ impl Bbr { fn enter_probe_bw(&mut self, now: Instant) { self.state = BbrStateMachine::ProbeBW; self.pacing_gain = 1.0; - self.cwnd_gain = 2.0; + self.cwnd_gain = self.config.probe_bw_cwnd_gain; // Gain Cycling Randomization. // To improve mixing and fairness, and to reduce queues when multiple @@ -643,6 +671,10 @@ impl Bbr { /// Return cwnd for ProbeRTT state. fn probe_rtt_cwnd(&self) -> u64 { + if self.config.probe_rtt_based_on_bdp { + return self.inflight(self.config.probe_rtt_cwnd_gain); + } + self.config.min_cwnd } @@ -700,7 +732,7 @@ impl Bbr { self.exit_probe_rtt(now); } } else if bytes_in_flight <= self.probe_rtt_cwnd() { - self.probe_rtt_done_stamp = Some(now + PROBE_RTT_DURATION); + self.probe_rtt_done_stamp = Some(now + self.config.probe_rtt_duration); // ProbeRTT round passed. self.probe_rtt_round_done = false; self.round.next_round_delivered = self.delivery_rate_estimator.delivered(); @@ -760,7 +792,7 @@ impl Bbr { let sample_rtt = self.delivery_rate_estimator.sample_rtt(); self.is_rtprop_expired = - now.saturating_duration_since(self.rtprop_stamp) > RTPROP_FILTER_LEN; + now.saturating_duration_since(self.rtprop_stamp) > self.config.rtprop_filter_len; // Use the same state to track BBR.RTprop and ProbeRTT timing. // In section-4.1.2.3, a zero packet.rtt is allowed, but it makes no sense. diff --git a/src/congestion_control/bbr3.rs b/src/congestion_control/bbr3.rs index f2a40994..53830c90 100644 --- a/src/congestion_control/bbr3.rs +++ b/src/congestion_control/bbr3.rs @@ -38,6 +38,7 @@ use super::minmax::MinMax; use super::{CongestionController, CongestionStats}; use crate::connection::rtt::RttEstimator; use crate::connection::space::{RateSamplePacketState, SentPacket}; +use crate::RecoveryConfig; /// BBR configurable parameters. #[derive(Debug)] @@ -82,16 +83,17 @@ pub struct Bbr3Config { } impl Bbr3Config { - pub fn new( - min_cwnd: u64, - initial_cwnd: u64, - initial_rtt: Option, - max_datagram_size: u64, - ) -> Self { + pub fn from(conf: &RecoveryConfig) -> Self { + let max_datagram_size = conf.max_datagram_size as u64; + let min_cwnd = conf.min_congestion_window.saturating_mul(max_datagram_size); + let initial_cwnd = conf + .initial_congestion_window + .saturating_mul(max_datagram_size); + Self { min_cwnd, initial_cwnd, - initial_rtt, + initial_rtt: Some(conf.initial_rtt), max_datagram_size, ..Self::default() } diff --git a/src/congestion_control/congestion_control.rs b/src/congestion_control/congestion_control.rs index 91811f4c..20102fe7 100644 --- a/src/congestion_control/congestion_control.rs +++ b/src/congestion_control/congestion_control.rs @@ -33,6 +33,7 @@ pub use copa::CopaConfig; pub use cubic::Cubic; pub use cubic::CubicConfig; pub use dummy::Dummy; +pub use dummy::DummyConfig; pub use hystart_plus_plus::HystartPlusPlus; pub use pacing::Pacer; @@ -185,38 +186,12 @@ impl fmt::Debug for dyn CongestionController { /// Build a congestion controller. pub fn build_congestion_controller(conf: &RecoveryConfig) -> Box { - let max_datagram_size: u64 = conf.max_datagram_size as u64; - let min_cwnd = conf.min_congestion_window.saturating_mul(max_datagram_size); - let initial_cwnd = conf - .initial_congestion_window - .saturating_mul(max_datagram_size); - match conf.congestion_control_algorithm { - CongestionControlAlgorithm::Cubic => Box::new(Cubic::new(CubicConfig::new( - min_cwnd, - initial_cwnd, - Some(conf.initial_rtt), - max_datagram_size, - ))), - CongestionControlAlgorithm::Bbr => Box::new(Bbr::new(BbrConfig::new( - min_cwnd, - initial_cwnd, - Some(conf.initial_rtt), - max_datagram_size, - ))), - CongestionControlAlgorithm::Bbr3 => Box::new(Bbr3::new(Bbr3Config::new( - min_cwnd, - initial_cwnd, - Some(conf.initial_rtt), - max_datagram_size, - ))), - CongestionControlAlgorithm::Copa => Box::new(Copa::new(CopaConfig::new( - min_cwnd, - initial_cwnd, - Some(conf.initial_rtt), - max_datagram_size, - ))), - CongestionControlAlgorithm::Dummy => Box::new(Dummy::new(initial_cwnd)), + CongestionControlAlgorithm::Cubic => Box::new(Cubic::new(CubicConfig::from(conf))), + CongestionControlAlgorithm::Bbr => Box::new(Bbr::new(BbrConfig::from(conf))), + CongestionControlAlgorithm::Bbr3 => Box::new(Bbr3::new(Bbr3Config::from(conf))), + CongestionControlAlgorithm::Copa => Box::new(Copa::new(CopaConfig::from(conf))), + CongestionControlAlgorithm::Dummy => Box::new(Dummy::new(DummyConfig::from(conf))), } } diff --git a/src/congestion_control/copa.rs b/src/congestion_control/copa.rs index e459d081..c3390df1 100644 --- a/src/congestion_control/copa.rs +++ b/src/congestion_control/copa.rs @@ -33,6 +33,7 @@ use super::CongestionStats; use crate::connection::rtt::RttEstimator; use crate::connection::space::RateSamplePacketState; use crate::connection::space::SentPacket; +use crate::RecoveryConfig; /// Delta: determines how much to weigh delay compared to throughput. const COPA_DELTA: f64 = 0.04; @@ -86,12 +87,14 @@ pub struct CopaConfig { } impl CopaConfig { - pub fn new( - min_cwnd: u64, - initial_cwnd: u64, - initial_rtt: Option, - max_datagram_size: u64, - ) -> Self { + pub fn from(conf: &RecoveryConfig) -> Self { + let max_datagram_size = conf.max_datagram_size as u64; + let min_cwnd = conf.min_congestion_window.saturating_mul(max_datagram_size); + let initial_cwnd = conf + .initial_congestion_window + .saturating_mul(max_datagram_size); + let initial_rtt = Some(conf.initial_rtt); + Self { min_cwnd, initial_cwnd, diff --git a/src/congestion_control/cubic.rs b/src/congestion_control/cubic.rs index 1bdbe9e4..f65225e8 100644 --- a/src/congestion_control/cubic.rs +++ b/src/congestion_control/cubic.rs @@ -23,6 +23,7 @@ use super::CongestionStats; use super::HystartPlusPlus; use crate::connection::rtt::RttEstimator; use crate::connection::space::SentPacket; +use crate::RecoveryConfig; /// Cubic constant C. /// @@ -65,6 +66,9 @@ pub struct CubicConfig { /// Initial congestion window in bytes. initial_congestion_window: u64, + /// The threshold for slow start in bytes. + slow_start_thresh: u64, + /// Max datagram size in bytes. max_datagram_size: u64, @@ -79,19 +83,21 @@ pub struct CubicConfig { } impl CubicConfig { - pub fn new( - min_congestion_window: u64, - initial_congestion_window: u64, - initial_rtt: Option, - max_datagram_size: u64, - ) -> Self { - let c = C; - let beta = BETA; + pub fn from(conf: &RecoveryConfig) -> Self { + let max_datagram_size = conf.max_datagram_size as u64; + let min_congestion_window = conf.min_congestion_window.saturating_mul(max_datagram_size); + let initial_congestion_window = conf + .initial_congestion_window + .saturating_mul(max_datagram_size); + let slow_start_thresh = conf.slow_start_thresh.saturating_mul(max_datagram_size); + let initial_rtt = Some(conf.initial_rtt); + Self { - c, - beta, + c: C, + beta: BETA, min_congestion_window, initial_congestion_window, + slow_start_thresh, initial_rtt, max_datagram_size, hystart_enabled: true, @@ -149,6 +155,7 @@ impl Default for CubicConfig { beta: BETA, min_congestion_window: 2 * crate::DEFAULT_SEND_UDP_PAYLOAD_SIZE as u64, initial_congestion_window: 10 * crate::DEFAULT_SEND_UDP_PAYLOAD_SIZE as u64, + slow_start_thresh: u64::MAX, initial_rtt: Some(crate::INITIAL_RTT), max_datagram_size: crate::DEFAULT_SEND_UDP_PAYLOAD_SIZE as u64, hystart_enabled: true, @@ -209,6 +216,7 @@ pub struct Cubic { impl Cubic { pub fn new(config: CubicConfig) -> Self { let initial_cwnd = config.initial_congestion_window; + let ssthresh = config.slow_start_thresh; let initial_rtt = std::cmp::max( config.initial_rtt.unwrap_or(crate::INITIAL_RTT), Duration::from_micros(1), @@ -220,7 +228,7 @@ impl Cubic { config, hystart: HystartPlusPlus::new(hystart_enabled), cwnd: initial_cwnd, - ssthresh: u64::MAX, + ssthresh, w_max: 0_f64, k: 0_f64, alpha, diff --git a/src/congestion_control/dummy.rs b/src/congestion_control/dummy.rs index 2bcdf461..3ad14745 100644 --- a/src/congestion_control/dummy.rs +++ b/src/congestion_control/dummy.rs @@ -20,6 +20,35 @@ use super::CongestionController; use super::CongestionStats; use crate::connection::rtt::RttEstimator; use crate::connection::space::SentPacket; +use crate::RecoveryConfig; + +/// Dummpy Configuration. +#[derive(Debug)] +pub struct DummyConfig { + /// Initial congestion window in bytes. + initial_congestion_window: u64, +} + +impl DummyConfig { + pub fn from(conf: &RecoveryConfig) -> Self { + let max_datagram_size = conf.max_datagram_size as u64; + let initial_congestion_window = conf + .initial_congestion_window + .saturating_mul(max_datagram_size); + + Self { + initial_congestion_window, + } + } +} + +impl Default for DummyConfig { + fn default() -> Self { + Self { + initial_congestion_window: 10 * crate::DEFAULT_SEND_UDP_PAYLOAD_SIZE as u64, + } + } +} /// Dummy is a simple congestion controller with a static congestion window. /// It is intended to be used for testing and experiments. @@ -33,9 +62,9 @@ pub struct Dummy { } impl Dummy { - pub fn new(initial_cwnd: u64) -> Self { + pub fn new(conf: DummyConfig) -> Self { Self { - cwnd: initial_cwnd, + cwnd: conf.initial_congestion_window, stats: Default::default(), } } @@ -122,7 +151,13 @@ mod tests { #[test] fn dummy_init() { - let d = Dummy::new(1200 * 10); + let conf = RecoveryConfig { + initial_congestion_window: 10, + max_datagram_size: 1200, + ..RecoveryConfig::default() + }; + let d = Dummy::new(DummyConfig::from(&conf)); + assert_eq!(d.name(), "DUMMY"); assert_eq!(d.congestion_window(), 1200 * 10); assert_eq!(d.initial_window(), 1200 * 10); @@ -136,7 +171,13 @@ mod tests { #[test] fn dummy_stats() { - let mut d = Dummy::new(1200 * 10); + let conf = RecoveryConfig { + initial_congestion_window: 10, + max_datagram_size: 1200, + ..RecoveryConfig::default() + }; + let mut d = Dummy::new(DummyConfig::from(&conf)); + let rtt = Duration::from_millis(100); let rtt_estimator = RttEstimator::new(rtt); let now = Instant::now(); diff --git a/src/ffi.rs b/src/ffi.rs index 632ebe7e..fdd9dca3 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -244,6 +244,48 @@ pub extern "C" fn quic_config_set_min_congestion_window(config: &mut Config, v: config.set_min_congestion_window(v); } +/// Set the threshold for slow start in packets. +/// The default value is the maximum value of u64. +#[no_mangle] +pub extern "C" fn quic_config_set_slow_start_thresh(config: &mut Config, v: u64) { + config.set_slow_start_thresh(v); +} + +/// Set the minimum duration for BBR ProbeRTT state in milliseconds. +/// The default value is 200 milliseconds. +#[no_mangle] +pub extern "C" fn quic_config_set_bbr_probe_rtt_duration(config: &mut Config, v: u64) { + config.set_bbr_probe_rtt_duration(v); +} + +/// Enable using a cwnd based on bdp during ProbeRTT state. +/// The default value is false. +#[no_mangle] +pub extern "C" fn quic_config_enable_bbr_probe_rtt_based_on_bdp(config: &mut Config, v: bool) { + config.enable_bbr_probe_rtt_based_on_bdp(v); +} + +/// Set the cwnd gain for BBR ProbeRTT state. +/// The default value is 0.75 +#[no_mangle] +pub extern "C" fn quic_config_set_bbr_probe_rtt_cwnd_gain(config: &mut Config, v: f64) { + config.set_bbr_probe_rtt_cwnd_gain(v); +} + +/// Set the length of the BBR RTProp min filter window in milliseconds. +/// The default value is 10000 milliseconds. +#[no_mangle] +pub extern "C" fn quic_config_set_bbr_rtprop_filter_len(config: &mut Config, v: u64) { + config.set_bbr_rtprop_filter_len(v); +} + +/// Set the cwnd gain for BBR ProbeBW state. +/// The default value is 2.0 +#[no_mangle] +pub extern "C" fn quic_config_set_bbr_probe_bw_cwnd_gain(config: &mut Config, v: f64) { + config.set_bbr_probe_bw_cwnd_gain(v); +} + /// Set the initial RTT in milliseconds. The default value is 333ms. /// The configuration should be changed with caution. Setting a value less than the default /// will cause retransmission of handshake packets to be more aggressive. diff --git a/src/lib.rs b/src/lib.rs index eff2d558..42a59468 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -518,6 +518,46 @@ impl Config { self.recovery.min_congestion_window = packets } + /// Set the threshold for slow start in packets. + /// The default value is the maximum value of u64. + pub fn set_slow_start_thresh(&mut self, packets: u64) { + self.recovery.slow_start_thresh = packets + } + + /// Set the minimum duration for BBR ProbeRTT state in milliseconds. + /// The default value is 200 milliseconds. + pub fn set_bbr_probe_rtt_duration(&mut self, millis: u64) { + self.recovery.bbr_probe_rtt_duration = + cmp::max(Duration::from_millis(millis), TIMER_GRANULARITY); + } + + /// Enable using a cwnd based on bdp during ProbeRTT state. + /// The default value is false. + pub fn enable_bbr_probe_rtt_based_on_bdp(&mut self, v: bool) { + self.recovery.bbr_probe_rtt_based_on_bdp = v; + } + + /// Set the cwnd gain for BBR ProbeRTT state. + /// This option is meaningful only when `bbr_probe_rtt_based_on_bdp` option + /// is set to true. + /// The default value is 0.75 + pub fn set_bbr_probe_rtt_cwnd_gain(&mut self, v: f64) { + self.recovery.bbr_probe_rtt_cwnd_gain = v; + } + + /// Set the length of the BBR RTProp min filter window in milliseconds. + /// The default value is 10000 milliseconds. + pub fn set_bbr_rtprop_filter_len(&mut self, millis: u64) { + self.recovery.bbr_rtprop_filter_len = + cmp::max(Duration::from_millis(millis), TIMER_GRANULARITY); + } + + /// Set the cwnd gain for BBR ProbeBW state. + /// The default value is 2.0 + pub fn set_bbr_probe_bw_cwnd_gain(&mut self, v: f64) { + self.recovery.bbr_probe_bw_cwnd_gain = v; + } + /// Set the initial RTT in milliseconds. The default value is 333ms. /// /// The configuration should be changed with caution. Setting a value less than the default @@ -723,6 +763,24 @@ pub struct RecoveryConfig { /// See RFC 9002 Section 7.2 pub initial_congestion_window: u64, + /// The threshold for slow start in packets. + pub slow_start_thresh: u64, + + /// The minimum duration for BBR ProbeRTT state + pub bbr_probe_rtt_duration: Duration, + + /// Enable using a cwnd based on bdp during ProbeRTT state. + pub bbr_probe_rtt_based_on_bdp: bool, + + /// The cwnd gain for BBR ProbeRTT state + pub bbr_probe_rtt_cwnd_gain: f64, + + /// The length of the RTProp min filter window + pub bbr_rtprop_filter_len: Duration, + + /// The cwnd gain for ProbeBW state + pub bbr_probe_bw_cwnd_gain: f64, + /// The initial rtt, used before real rtt is estimated. pub initial_rtt: Duration, @@ -745,6 +803,12 @@ impl Default for RecoveryConfig { congestion_control_algorithm: CongestionControlAlgorithm::Bbr, min_congestion_window: 2_u64, initial_congestion_window: 10_u64, + slow_start_thresh: u64::MAX, + bbr_probe_rtt_duration: Duration::from_millis(200), + bbr_probe_rtt_based_on_bdp: false, + bbr_probe_rtt_cwnd_gain: 0.75, + bbr_rtprop_filter_len: Duration::from_secs(10), + bbr_probe_bw_cwnd_gain: 2.0, initial_rtt: INITIAL_RTT, enable_pacing: true, pto_linear_factor: DEFAULT_PTO_LINEAR_FACTOR, From 410372d50afdea3e37a82310de10d19fc3fc5bce Mon Sep 17 00:00:00 2001 From: xiaofei0800 <54530729+xiaofei0800@users.noreply.github.com> Date: Sun, 7 Jul 2024 12:22:45 +0800 Subject: [PATCH 3/7] Tweak comments for application protos in FFI (#309) --- include/tquic.h | 5 +++++ src/ffi.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/include/tquic.h b/include/tquic.h index c11fc511..977e76bb 100644 --- a/include/tquic.h +++ b/include/tquic.h @@ -598,6 +598,7 @@ struct quic_tls_config_t *quic_tls_config_new_with_ssl_ctx(SSL_CTX *ssl_ctx); * Create a new client side TlsConfig. * The caller is responsible for the memory of the TlsConfig and should properly * destroy it by calling `quic_tls_config_free`. + * For more information about `protos`, please see `quic_tls_config_set_application_protos`. */ struct quic_tls_config_t *quic_tls_config_new_client_config(const char *const *protos, intptr_t proto_num, @@ -607,6 +608,7 @@ struct quic_tls_config_t *quic_tls_config_new_client_config(const char *const *p * Create a new server side TlsConfig. * The caller is responsible for the memory of the TlsConfig and should properly * destroy it by calling `quic_tls_config_free`. + * For more information about `protos`, please see `quic_tls_config_set_application_protos`. */ struct quic_tls_config_t *quic_tls_config_new_server_config(const char *cert_file, const char *key_file, @@ -626,6 +628,9 @@ void quic_tls_config_set_early_data_enabled(struct quic_tls_config_t *tls_config /** * Set the list of supported application protocols. + * The `protos` is a pointer that points to an array, where each element of the array is a string + * pointer representing an application protocol identifier. For example, you can define it as + * follows: const char* const protos[2] = {"h3", "http/0.9"}. */ int quic_tls_config_set_application_protos(struct quic_tls_config_t *tls_config, const char *const *protos, diff --git a/src/ffi.rs b/src/ffi.rs index fdd9dca3..86562f3a 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -485,6 +485,7 @@ fn convert_application_protos(protos: *const *const c_char, proto_num: isize) -> /// Create a new client side TlsConfig. /// The caller is responsible for the memory of the TlsConfig and should properly /// destroy it by calling `quic_tls_config_free`. +/// For more information about `protos`, please see `quic_tls_config_set_application_protos`. #[no_mangle] pub extern "C" fn quic_tls_config_new_client_config( protos: *const *const c_char, @@ -505,6 +506,7 @@ pub extern "C" fn quic_tls_config_new_client_config( /// Create a new server side TlsConfig. /// The caller is responsible for the memory of the TlsConfig and should properly /// destroy it by calling `quic_tls_config_free`. +/// For more information about `protos`, please see `quic_tls_config_set_application_protos`. #[no_mangle] pub extern "C" fn quic_tls_config_new_server_config( cert_file: *const c_char, @@ -552,6 +554,9 @@ pub extern "C" fn quic_tls_config_set_early_data_enabled(tls_config: &mut TlsCon } /// Set the list of supported application protocols. +/// The `protos` is a pointer that points to an array, where each element of the array is a string +/// pointer representing an application protocol identifier. For example, you can define it as +/// follows: const char* const protos[2] = {"h3", "http/0.9"}. #[no_mangle] pub extern "C" fn quic_tls_config_set_application_protos( tls_config: &mut TlsConfig, From 6e35e186193b42376c592d8400d3cde2375c0c43 Mon Sep 17 00:00:00 2001 From: Sijie Yang Date: Tue, 9 Jul 2024 11:02:33 +0800 Subject: [PATCH 4/7] Update cbindgen.toml and generated header file (#314) --- cbindgen.toml | 5 +- include/tquic.h | 144 ++++++-------------------------------------- include/tquic_def.h | 116 +++++++++++++++++++++++++++++++++++ src/lib.rs | 8 ++- src/tls/tls.rs | 1 - 5 files changed, 145 insertions(+), 129 deletions(-) create mode 100644 include/tquic_def.h diff --git a/cbindgen.toml b/cbindgen.toml index 0b291166..d9f8670b 100644 --- a/cbindgen.toml +++ b/cbindgen.toml @@ -16,10 +16,10 @@ documentation = true # A list of headers to #include (with quotes) sys_includes = ["sys/socket.h", "sys/types.h"] -includes = ["openssl/ssl.h"] +includes = ["openssl/ssl.h", "tquic_def.h"] [export] -exclude = ["MAX_CID_LEN", "MIN_CLIENT_INITIAL_LEN"] +exclude = ["MAX_CID_LEN", "MIN_CLIENT_INITIAL_LEN", "VINT_MAX"] [export.rename] "Config" = "quic_config_t" @@ -42,6 +42,7 @@ exclude = ["MAX_CID_LEN", "MIN_CLIENT_INITIAL_LEN"] "TlsConfigSelectMethods" = "quic_tls_config_select_methods_t" "TlsConfigSelectorContext" = "quic_tls_config_select_context_t" "CongestionControlAlgorithm" = "quic_congestion_control_algorithm" +"MultipathAlgorithm" = "quic_multipath_algorithm" "LevelFilter" = "quic_log_level" "Http3Connection" = "http3_conn_t" "Http3Config" = "http3_config_t" diff --git a/include/tquic.h b/include/tquic.h index 977e76bb..f2eebfd6 100644 --- a/include/tquic.h +++ b/include/tquic.h @@ -10,6 +10,7 @@ #include #include #include "openssl/ssl.h" +#include "tquic_def.h" /** * The current QUIC wire version. @@ -22,13 +23,13 @@ #define QUIC_VERSION_V1 1 /** - * Available congestion control algorithm + * Available congestion control algorithms. */ typedef enum quic_congestion_control_algorithm { /** * CUBIC uses a cubic function instead of a linear window increase function * of the current TCP standards to improve scalability and stability under - * fast and long-distance networks.. + * fast and long-distance networks. */ QUIC_CONGESTION_CONTROL_ALGORITHM_CUBIC, /** @@ -52,6 +53,11 @@ typedef enum quic_congestion_control_algorithm { * (Experimental) */ QUIC_CONGESTION_CONTROL_ALGORITHM_COPA, + /** + * Dummy is a simple congestion controller with a static congestion window. + * It is intended to be used for testing and experiments. + */ + QUIC_CONGESTION_CONTROL_ALGORITHM_DUMMY, } quic_congestion_control_algorithm; /** @@ -64,7 +70,7 @@ typedef enum quic_multipath_algorithm { * load balancing, making it particularly advantageous for bulk transfer * applications in heterogeneous networks. */ - MULTIPATH_ALGORITHM_MIN_RTT, + QUIC_MULTIPATH_ALGORITHM_MIN_RTT, /** * The scheduler sends all packets redundantly on all available paths. It * utilizes additional bandwidth to minimize latency, thereby reducing the @@ -74,14 +80,14 @@ typedef enum quic_multipath_algorithm { * present, it ensures a goodput at least equivalent to the best single * path. */ - MULTIPATH_ALGORITHM_REDUNDANT, + QUIC_MULTIPATH_ALGORITHM_REDUNDANT, /** * The scheduler sends packets over available paths in a round robin * manner. It aims to fully utilize the capacity of each path as the * distribution across all path is equal. It is only used for testing * purposes. */ - MULTIPATH_ALGORITHM_ROUND_ROBIN, + QUIC_MULTIPATH_ALGORITHM_ROUND_ROBIN, } quic_multipath_algorithm; /** @@ -358,7 +364,6 @@ void enable_dplpmtud(struct quic_config_t *config, bool v); /** * Set the maximum outgoing UDP payload size in bytes. * It corresponds to the maximum datagram size that DPLPMTUD tries to discovery. - * * The default value is `1200` which means let DPLPMTUD choose a value. */ void quic_config_set_send_udp_payload_size(struct quic_config_t *config, uintptr_t v); @@ -1156,6 +1161,13 @@ int64_t http3_stream_new_with_priority(struct http3_conn_t *conn, struct quic_conn_t *quic_conn, const struct http3_priority_t *priority); +/** + * Close the given HTTP/3 stream. + */ +int http3_stream_close(struct http3_conn_t *conn, + struct quic_conn_t *quic_conn, + uint64_t stream_id); + /** * Set priority for an HTTP/3 stream. */ @@ -1164,13 +1176,6 @@ int http3_stream_set_priority(struct http3_conn_t *conn, uint64_t stream_id, const struct http3_priority_t *priority); -/** - * Close the given HTTP/3 stream. - */ -int http3_stream_close(struct http3_conn_t *conn, - struct quic_conn_t *quic_conn, - uint64_t stream_id); - /** * Send HTTP/3 request or response headers on the given stream. */ @@ -1226,40 +1231,10 @@ int http3_take_priority_update(struct http3_conn_t *conn, void *argp), void *argp); -/** - * An enum representing the available verbosity level filters of the logger. - */ -typedef enum quic_log_level { - /** - * A level lower than all log levels. - */ - QUIC_LOG_LEVEL_OFF, - /** - * Corresponds to the `Error` log level. - */ - QUIC_LOG_LEVEL_ERROR, - /** - * Corresponds to the `Warn` log level. - */ - QUIC_LOG_LEVEL_WARN, - /** - * Corresponds to the `Info` log level. - */ - QUIC_LOG_LEVEL_INFO, - /** - * Corresponds to the `Debug` log level. - */ - QUIC_LOG_LEVEL_DEBUG, - /** - * Corresponds to the `Trace` log level. - */ - QUIC_LOG_LEVEL_TRACE, -} quic_log_level; - /** * Set logger. * `cb` is a callback function that will be called for each log message. - * `line` is a null-terminated log message and `argp` is user-defined data that will be passed to + * `data` is a '\n' terminated log message and `argp` is user-defined data that will be passed to * the callback. * `level` represents the log level. */ @@ -1267,87 +1242,6 @@ void quic_set_logger(void (*cb)(const uint8_t *data, size_t data_len, void *argp void *argp, quic_log_level level); -typedef enum http3_error { - HTTP3_NO_ERROR = 0, - - // There is no error or no work to do - HTTP3_ERR_DONE = -1, - - // The endpoint detected an error in the protocol - HTTP3_ERR_GENERAL_PROTOCOL_ERROR = -2, - - // The endpoint encountered an internal error and cannot continue with the - // connection - HTTP3_ERR_INTERNAL_ERROR = -3, - - // The endpoint detected that its peer created a stream that it will not - // accept - HTTP3_ERR_STREAM_CREATION_ERROR = -4, - - // A stream required by the connection was closed or reset - HTTP3_ERR_CLOSED_CRITICAL_STREAM = -5, - - // A frame was received which is not permitted in the current state or on - // the current stream - HTTP3_ERR_FRAME_UNEXPECTED = -6, - - // A frame that fails to satisfy layout requirements or with an invalid - // size was received - HTTP3_ERR_FRAME_ERROR = -7, - - // The endpoint detected that its peer is exhibiting a behavior that might - // be generating excessive load - HTTP3_ERR_EXCESSIVE_LOAD = -8, - - // A stream ID or push ID was used incorrectly, such as exceeding a limit, - // reducing a limit, or being reused - HTTP3_ERR_ID_ERROR = -9, - - // An endpoint detected an error in the payload of a SETTINGS frame - HTTP3_ERR_SETTINGS_ERROR = -10, - - // No SETTINGS frame was received at the beginning of the control stream - HTTP3_ERR_MISSING_SETTINGS = -11, - - // -12 reserved - - // The stream is blocked - HTTP3_ERR_STREAM_BLOCKED = -13, - - // The server rejected the request without performing any application - // processing - HTTP3_ERR_REQUEST_REJECTED = -14, - - // The request or its response (including pushed response) is cancelled - HTTP3_ERR_REQUEST_CANCELLED = -15, - - // The client's stream terminated without containing a fully-formed request - HTTP3_ERR_REQUEST_INCOMPLETE = -16, - - // An HTTP message was malformed and cannot be processed - HTTP3_ERR_MESSAGE_ERROR = -17, - - // The TCP connection established in response to a CONNECT request was - // reset or abnormally closed - HTTP3_ERR_CONNECT_ERROR = -18, - - // The requested operation cannot be served over HTTP/3. The peer should - // retry over HTTP/1.1 - HTTP3_ERR_VERSION_FALLBACK = -19, - - // The decoder failed to interpret an encoded field section and is not - // able to continue decoding that field section - HTTP3_ERR_QPACK_DECOMPRESSION_FAILED = -20, - - // The decoder failed to interpret an encoder instruction received on the - // encoder stream - HTTP3_ERR_QPACK_ENCODER_STREAM_ERROR = -21, - - // The encoder failed to interpret a decoder instruction received on the - // decoder stream - HTTP3_ERR_QPACK_DECODER_STREAM_ERROR = -22, -} http3_error; - #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/include/tquic_def.h b/include/tquic_def.h new file mode 100644 index 00000000..b792d37c --- /dev/null +++ b/include/tquic_def.h @@ -0,0 +1,116 @@ +#ifndef _TQUIC_DEF_H_ +#define _TQUIC_DEF_H_ + +/** + * An enum representing the available verbosity level filters of the logger. + */ +typedef enum quic_log_level { + /** + * A level lower than all log levels. + */ + QUIC_LOG_LEVEL_OFF, + /** + * Corresponds to the `Error` log level. + */ + QUIC_LOG_LEVEL_ERROR, + /** + * Corresponds to the `Warn` log level. + */ + QUIC_LOG_LEVEL_WARN, + /** + * Corresponds to the `Info` log level. + */ + QUIC_LOG_LEVEL_INFO, + /** + * Corresponds to the `Debug` log level. + */ + QUIC_LOG_LEVEL_DEBUG, + /** + * Corresponds to the `Trace` log level. + */ + QUIC_LOG_LEVEL_TRACE, +} quic_log_level; + + +typedef enum http3_error { + HTTP3_NO_ERROR = 0, + + // There is no error or no work to do + HTTP3_ERR_DONE = -1, + + // The endpoint detected an error in the protocol + HTTP3_ERR_GENERAL_PROTOCOL_ERROR = -2, + + // The endpoint encountered an internal error and cannot continue with the + // connection + HTTP3_ERR_INTERNAL_ERROR = -3, + + // The endpoint detected that its peer created a stream that it will not + // accept + HTTP3_ERR_STREAM_CREATION_ERROR = -4, + + // A stream required by the connection was closed or reset + HTTP3_ERR_CLOSED_CRITICAL_STREAM = -5, + + // A frame was received which is not permitted in the current state or on + // the current stream + HTTP3_ERR_FRAME_UNEXPECTED = -6, + + // A frame that fails to satisfy layout requirements or with an invalid + // size was received + HTTP3_ERR_FRAME_ERROR = -7, + + // The endpoint detected that its peer is exhibiting a behavior that might + // be generating excessive load + HTTP3_ERR_EXCESSIVE_LOAD = -8, + + // A stream ID or push ID was used incorrectly, such as exceeding a limit, + // reducing a limit, or being reused + HTTP3_ERR_ID_ERROR = -9, + + // An endpoint detected an error in the payload of a SETTINGS frame + HTTP3_ERR_SETTINGS_ERROR = -10, + + // No SETTINGS frame was received at the beginning of the control stream + HTTP3_ERR_MISSING_SETTINGS = -11, + + // -12 reserved + + // The stream is blocked + HTTP3_ERR_STREAM_BLOCKED = -13, + + // The server rejected the request without performing any application + // processing + HTTP3_ERR_REQUEST_REJECTED = -14, + + // The request or its response (including pushed response) is cancelled + HTTP3_ERR_REQUEST_CANCELLED = -15, + + // The client's stream terminated without containing a fully-formed request + HTTP3_ERR_REQUEST_INCOMPLETE = -16, + + // An HTTP message was malformed and cannot be processed + HTTP3_ERR_MESSAGE_ERROR = -17, + + // The TCP connection established in response to a CONNECT request was + // reset or abnormally closed + HTTP3_ERR_CONNECT_ERROR = -18, + + // The requested operation cannot be served over HTTP/3. The peer should + // retry over HTTP/1.1 + HTTP3_ERR_VERSION_FALLBACK = -19, + + // The decoder failed to interpret an encoded field section and is not + // able to continue decoding that field section + HTTP3_ERR_QPACK_DECOMPRESSION_FAILED = -20, + + // The decoder failed to interpret an encoder instruction received on the + // encoder stream + HTTP3_ERR_QPACK_ENCODER_STREAM_ERROR = -21, + + // The encoder failed to interpret a decoder instruction received on the + // decoder stream + HTTP3_ERR_QPACK_DECODER_STREAM_ERROR = -22, +} http3_error; + +#endif /* _TQUIC_DEF_H_ */ diff --git a/src/lib.rs b/src/lib.rs index 42a59468..c9d47d22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,7 +80,7 @@ use crate::trans_param::TransportParams; pub const QUIC_VERSION: u32 = QUIC_VERSION_V1; /// The QUIC Version 1 -const QUIC_VERSION_V1: u32 = 0x0000_0001; +pub const QUIC_VERSION_V1: u32 = 0x0000_0001; /// The Connection ID MUST NOT exceed 20 bytes in QUIC version 1. /// See RFC 9000 Section 17.2 @@ -1182,6 +1182,12 @@ mod qlog; #[cfg(feature = "ffi")] mod ffi; +// Note: Workaround for the module path issue in cbindgen. +// DON'T enable this feature when building with cargo. +#[cfg(feature = "cbindgen")] +#[path = "h3/connection.rs"] +mod h3_connection; + mod codec; pub mod endpoint; pub mod error; diff --git a/src/tls/tls.rs b/src/tls/tls.rs index 14974bb5..42165a6a 100644 --- a/src/tls/tls.rs +++ b/src/tls/tls.rs @@ -77,7 +77,6 @@ where } } -#[repr(C)] pub struct TlsConfig { /// Boringssl SSL context. tls_ctx: boringssl::tls::Context, From 9d7f1477256804d4953a37fb0c9de4d4334960d5 Mon Sep 17 00:00:00 2001 From: XYT874264076 <53042955+XYT874264076@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:35:42 +0800 Subject: [PATCH 5/7] Fix the problem for occasionally experiences connection timeout due to pacing --- src/endpoint.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/endpoint.rs b/src/endpoint.rs index dbe4635d..48498dee 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -710,6 +710,9 @@ impl Endpoint { } Err(Error::Done) => { self.packets.put_buffer(buf); + // Return Err(Error::Done) it means we can not send packets right now, + // but we still need to update timers for other events (e.g. Timer::Pacer) + sent.insert(idx); // When a connection runs out of packets to send, // it is removed from the sendable queue. conn.mark_sendable(false); From a778c1eba87efb4a6a5ec89d3c53a7f60f40f043 Mon Sep 17 00:00:00 2001 From: Sijie Yang Date: Thu, 11 Jul 2024 12:01:15 +0800 Subject: [PATCH 6/7] Fix the issue with sending MAX_DATA frames (#318) --- src/connection/connection.rs | 4 +--- src/connection/stream.rs | 8 +++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/connection/connection.rs b/src/connection/connection.rs index c95986c6..3c03134b 100644 --- a/src/connection/connection.rs +++ b/src/connection/connection.rs @@ -2158,9 +2158,7 @@ impl Connection { } // Create MAX_DATA frame if needed. - if self.streams.rx_almost_full - && self.streams.max_rx_data() < self.streams.max_rx_data_next() - { + if self.streams.need_send_max_data() { // Adjust the connection window size automatically. self.streams .autotune_window(now, path.recovery.rtt.smoothed_rtt()); diff --git a/src/connection/stream.rs b/src/connection/stream.rs index ed8bb69c..d171bc7a 100644 --- a/src/connection/stream.rs +++ b/src/connection/stream.rs @@ -708,10 +708,16 @@ impl StreamMap { } } + /// Return true if we should send `MAX_DATA` frame to peer to update + /// the connection level flow control limit. + pub fn need_send_max_data(&self) -> bool { + self.rx_almost_full && self.max_rx_data() < self.max_rx_data_next() + } + /// Return true if need to send stream frames. pub fn need_send_stream_frames(&self) -> bool { self.has_sendable_streams() - || self.rx_almost_full + || self.need_send_max_data() || self.data_blocked_at().is_some() || self.should_send_max_streams() || self.has_almost_full_streams() From 52d8e9e8ab33a7c9e80e9efe82f6085035b49349 Mon Sep 17 00:00:00 2001 From: Sijie Yang Date: Thu, 11 Jul 2024 15:11:28 +0800 Subject: [PATCH 7/7] Update VERSION and CHANGELOG --- CHANGELOG.md | 17 +++++++++++++++++ Cargo.toml | 2 +- tools/Cargo.toml | 4 ++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5f1db0f..7a6f7205 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v0.14.0] - 2024-07-11 + +### Added +- Update config API for congestion control +- Update cbindgen.toml and the generated header file +- Tweak comments for application protos in FFI + +### Changed +- Rename enum members of `quic_multipath_algorithm` in `tquic.h` + +### Fixed +- Fix stream operations that should mark conn as tickable +- Fix the issue with sending MAX_DATA frames +- Fix the issue with pacer timer that occasionally leads to a connection timeout error + + ## [v0.13.0] - 2024-06-25 ### Added @@ -246,6 +262,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Provide example clients and servers. +[v0.14.0]: https://github.com/tencent/tquic/compare/v0.13.0...v0.14.0 [v0.13.0]: https://github.com/tencent/tquic/compare/v0.12.0...v0.13.0 [v0.12.0]: https://github.com/tencent/tquic/compare/v0.11.0...v0.12.0 [v0.11.0]: https://github.com/tencent/tquic/compare/v0.10.0...v0.11.0 diff --git a/Cargo.toml b/Cargo.toml index d317f8bc..cc22db06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tquic" -version = "0.13.0" +version = "0.14.0" edition = "2021" rust-version = "1.70.0" license = "Apache-2.0" diff --git a/tools/Cargo.toml b/tools/Cargo.toml index ade4dfaf..c004404d 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tquic_tools" -version = "0.13.0" +version = "0.14.0" edition = "2021" rust-version = "1.70.0" license = "Apache-2.0" @@ -23,7 +23,7 @@ rand = "0.8.5" statrs = "0.16" jemallocator = { version = "0.5", package = "tikv-jemallocator" } signal-hook = "0.3.17" -tquic = { path = "..", version = "0.13.0"} +tquic = { path = "..", version = "0.14.0"} [lib] crate-type = ["lib"]