diff --git a/Cargo.lock b/Cargo.lock index 25ee386..a69aae9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -456,7 +456,7 @@ dependencies = [ [[package]] name = "ngx" -version = "0.3.0-beta" +version = "0.4.0-beta" dependencies = [ "nginx-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 6eac1e0..6fed388 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "ngx" -version = "0.3.0-beta" +version = "0.4.0-beta" edition = "2021" autoexamples = false categories = ["api-bindings", "network-programming"] diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 7759359..713f991 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -22,13 +22,16 @@ name = "awssig" path = "awssig.rs" crate-type = ["cdylib"] - - [[example]] name = "httporigdst" path = "httporigdst.rs" crate-type = ["cdylib"] required-features = ["linux"] +[[example]] +name = "upstream" +path = "upstream.rs" +crate-type = ["cdylib"] + [features] linux = [] diff --git a/examples/README.md b/examples/README.md index aabf24c..e8a1dfb 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,6 +8,11 @@ - [Embedded Variables](#embedded-variables) - [Usage](#usage) - [Caveats](#caveats) + - [UPSTREAM - Example upstream module for HTTP](#upstream---example-upstream-module-for-http) + - [Attributions](#attributions) + - [Example Configuration](#example-configuration) + - [HTTP](#http) + - [Usage](#usage) # Examples @@ -98,7 +103,7 @@ The following embedded variables are provided: 1. Clone the git repository. ``` - https://github.com/nginxinc/ngx-rust + git clone git@github.com:nginxinc/ngx-rust.git ``` 2. Compile the module from the cloned repo. @@ -150,3 +155,69 @@ The following embedded variables are provided: ### Caveats This module only supports IPv4. + +## UPSTREAM - Example upstream module for HTTP + +This module simply proxies requests through a custom load balancer to the previously configured balancer. This is for demonstration purposes only. As a module writer, you can start with this structure and adjust to your needs, then implement the proper algorithm for your usage. + +The module replaces the `peer` callback functions with its own, logs, and then calls through to the originally saved `peer` functions. This may look confusing at first, but rest assured, it's intentionally not implementing an algorithm of its own. + +### Attributions + +This module was converted from https://github.com/gabihodoroaga/nginx-upstream-module. + +### Example Configuration +#### HTTP + +```nginx configuration +load_module "modules/upstream.so" + +http { + upstream backend { + server localhost:8081; + + custom 32; + } + + server { + listen 8080; + server_name _; + + location / { + proxy_pass http://backend; + } + } +} +``` + +### Usage + +1. Clone the git repository. + ``` + git clone git@github.com:nginxinc/ngx-rust.git + ``` + +2. Compile the module from the cloned repo. + ``` + cd ${CLONED_DIRECTORY}/ngx-rust + cargo build --package=examples --example=upstream + ``` + +3. Copy the shared object to the modules directory, /etc/nginx/modules. + ``` + cp ./target/debug/examples/libupstream.so /etc/nginx/modules + ``` + +4. Add the `load_module` directive to your configuration. + ``` + load_module "modules/libupstream.so"; + ``` + +5. Add the example `server` and `upstream` block from the example above. + +6. Reload NGINX. + ``` + nginx -t && nginx -s reload + ``` + +7. Test with `curl`. Traffic should pass to your listener on port 8081 (this could be another NGINX server for example). With debug logging enabled you should notice the "custom" log messages (see the source code for log examples). diff --git a/examples/upstream.rs b/examples/upstream.rs new file mode 100644 index 0000000..d627cb6 --- /dev/null +++ b/examples/upstream.rs @@ -0,0 +1,357 @@ +/* + * This example is based on: + * https://github.com/gabihodoroaga/nginx-upstream-module + * + * The NGINX authors are grateful to @gabihodoroaga for their contributions + * to the community at large. + * https://github.com/gabihodoroaga + */ +use ngx::{ + core::{Pool, Status}, + ffi::{ + nginx_version, ngx_atoi, ngx_command_t, ngx_conf_log_error, ngx_conf_t, ngx_connection_t, + ngx_event_free_peer_pt, ngx_event_get_peer_pt, ngx_http_module_t, ngx_http_request_t, + ngx_http_upstream_init_peer_pt, ngx_http_upstream_init_pt, ngx_http_upstream_init_round_robin, + ngx_http_upstream_srv_conf_t, ngx_http_upstream_t, ngx_int_t, ngx_module_t, ngx_peer_connection_t, ngx_str_t, + ngx_uint_t, NGX_CONF_NOARGS, NGX_CONF_TAKE1, NGX_CONF_UNSET, NGX_ERROR, NGX_HTTP_MODULE, NGX_HTTP_UPS_CONF, + NGX_LOG_DEBUG_HTTP, NGX_LOG_EMERG, NGX_RS_HTTP_SRV_CONF_OFFSET, NGX_RS_MODULE_SIGNATURE, + }, + http::{ + ngx_http_conf_get_module_srv_conf, ngx_http_conf_upstream_srv_conf_immutable, + ngx_http_conf_upstream_srv_conf_mutable, HTTPModule, Merge, MergeConfigError, Request, + }, + http_upstream_peer_init, ngx_log_debug, ngx_log_debug_http, ngx_log_debug_mask, ngx_modules, ngx_null_command, + ngx_string, +}; +use std::{ + mem, + os::raw::{c_char, c_void}, + slice, +}; + +#[derive(Clone, Copy, Debug)] +#[repr(C)] +struct SrvConfig { + max: u32, + + original_init_upstream: ngx_http_upstream_init_pt, + original_init_peer: ngx_http_upstream_init_peer_pt, +} + +impl Default for SrvConfig { + fn default() -> Self { + SrvConfig { + max: u32::MAX, + original_init_upstream: None, + original_init_peer: None, + } + } +} + +impl Merge for SrvConfig { + fn merge(&mut self, _prev: &SrvConfig) -> Result<(), MergeConfigError> { + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] +#[repr(C)] +struct UpstreamPeerData { + conf: Option<*const SrvConfig>, + upstream: Option<*mut ngx_http_upstream_t>, + client_connection: Option<*mut ngx_connection_t>, + original_get_peer: ngx_event_get_peer_pt, + original_free_peer: ngx_event_free_peer_pt, + data: *mut c_void, +} + +impl Default for UpstreamPeerData { + fn default() -> Self { + UpstreamPeerData { + conf: None, + upstream: None, + client_connection: None, + original_get_peer: None, + original_free_peer: None, + data: std::ptr::null_mut(), + } + } +} + +#[no_mangle] +static ngx_http_upstream_custom_ctx: ngx_http_module_t = ngx_http_module_t { + preconfiguration: Some(Module::preconfiguration), + postconfiguration: Some(Module::postconfiguration), + create_main_conf: Some(Module::create_main_conf), + init_main_conf: Some(Module::init_main_conf), + create_srv_conf: Some(Module::create_srv_conf), + merge_srv_conf: Some(Module::merge_srv_conf), + create_loc_conf: Some(Module::create_loc_conf), + merge_loc_conf: Some(Module::merge_loc_conf), +}; + +#[no_mangle] +static mut ngx_http_upstream_custom_commands: [ngx_command_t; 2] = [ + ngx_command_t { + name: ngx_string!("custom"), + type_: (NGX_HTTP_UPS_CONF | NGX_CONF_NOARGS | NGX_CONF_TAKE1) as ngx_uint_t, + set: Some(ngx_http_upstream_commands_set_custom), + conf: NGX_RS_HTTP_SRV_CONF_OFFSET, + offset: 0, + post: std::ptr::null_mut(), + }, + ngx_null_command!(), +]; + +ngx_modules!(ngx_http_upstream_custom_module); + +#[no_mangle] +pub static mut ngx_http_upstream_custom_module: ngx_module_t = ngx_module_t { + ctx_index: ngx_uint_t::max_value(), + index: ngx_uint_t::max_value(), + name: std::ptr::null_mut(), + spare0: 0, + spare1: 0, + version: nginx_version as ngx_uint_t, + signature: NGX_RS_MODULE_SIGNATURE.as_ptr() as *const c_char, + + ctx: &ngx_http_upstream_custom_ctx as *const _ as *mut _, + commands: unsafe { &ngx_http_upstream_custom_commands[0] as *const _ as *mut _ }, + type_: NGX_HTTP_MODULE as ngx_uint_t, + + init_master: None, + init_module: None, + init_process: None, + init_thread: None, + exit_thread: None, + exit_process: None, + exit_master: None, + + spare_hook0: 0, + spare_hook1: 0, + spare_hook2: 0, + spare_hook3: 0, + spare_hook4: 0, + spare_hook5: 0, + spare_hook6: 0, + spare_hook7: 0, +}; + +// http_upstream_init_custom_peer +// The module's custom peer.init callback. On HTTP request the peer upstream get and free callbacks +// are saved into peer data and replaced with this module's custom callbacks. +http_upstream_peer_init!( + http_upstream_init_custom_peer, + |request: &mut Request, us: *mut ngx_http_upstream_srv_conf_t| { + ngx_log_debug_http!(request, "CUSTOM UPSTREAM request peer init"); + + let mut hcpd = request.pool().alloc_type::(); + if hcpd.is_null() { + return Status::NGX_ERROR; + } + + let maybe_conf: Option<*const SrvConfig> = + ngx_http_conf_upstream_srv_conf_immutable(us, &ngx_http_upstream_custom_module); + if maybe_conf.is_none() { + return Status::NGX_ERROR; + } + + let hccf = maybe_conf.unwrap(); + let original_init_peer = (*hccf).original_init_peer.unwrap(); + if original_init_peer(request.into(), us) != Status::NGX_OK.into() { + return Status::NGX_ERROR; + } + + let maybe_upstream = request.upstream(); + if maybe_upstream.is_none() { + return Status::NGX_ERROR; + } + let upstream_ptr = maybe_upstream.unwrap(); + + (*hcpd).conf = Some(hccf); + (*hcpd).upstream = maybe_upstream; + (*hcpd).data = (*upstream_ptr).peer.data; + (*hcpd).client_connection = Some(request.connection()); + (*hcpd).original_get_peer = (*upstream_ptr).peer.get; + (*hcpd).original_free_peer = (*upstream_ptr).peer.free; + + (*upstream_ptr).peer.data = hcpd as *mut c_void; + (*upstream_ptr).peer.get = Some(ngx_http_upstream_get_custom_peer); + (*upstream_ptr).peer.free = Some(ngx_http_upstream_free_custom_peer); + + ngx_log_debug_http!(request, "CUSTOM UPSTREAM end request peer init"); + Status::NGX_OK + } +); + +// ngx_http_usptream_get_custom_peer +// For demonstration purposes, use the original get callback, but log this callback proxies through +// to the original. +#[no_mangle] +unsafe extern "C" fn ngx_http_upstream_get_custom_peer(pc: *mut ngx_peer_connection_t, data: *mut c_void) -> ngx_int_t { + let hcpd: *mut UpstreamPeerData = unsafe { mem::transmute(data) }; + + ngx_log_debug_mask!( + NGX_LOG_DEBUG_HTTP, + (*pc).log, + "CUSTOM UPSTREAM get peer, try: {}, conn: {:p}", + (*pc).tries, + (*hcpd).client_connection.unwrap(), + ); + + let original_get_peer = (*hcpd).original_get_peer.unwrap(); + let rc = original_get_peer(pc, (*hcpd).data); + + if rc != Status::NGX_OK.into() { + return rc; + } + + ngx_log_debug!((*pc).log, "CUSTOM UPSTREAM end get peer"); + Status::NGX_OK.into() +} + +// ngx_http_upstream_free_custom_peer +// For demonstration purposes, use the original free callback, but log this callback proxies +// through to the original. +#[no_mangle] +unsafe extern "C" fn ngx_http_upstream_free_custom_peer( + pc: *mut ngx_peer_connection_t, + data: *mut c_void, + state: ngx_uint_t, +) { + ngx_log_debug_mask!(NGX_LOG_DEBUG_HTTP, (*pc).log, "CUSTOM UPSTREAM free peer"); + + let hcpd: *mut UpstreamPeerData = unsafe { mem::transmute(data) }; + + let original_free_peer = (*hcpd).original_free_peer.unwrap(); + + original_free_peer(pc, (*hcpd).data, state); + + ngx_log_debug!((*pc).log, "CUSTOM UPSTREAM end free peer"); +} + +// ngx_http_upstream_init_custom +// The module's custom `peer.init_upstream` callback. +// The original callback is saved in our SrvConfig data and reset to this module's `peer.init`. +#[no_mangle] +unsafe extern "C" fn ngx_http_upstream_init_custom( + cf: *mut ngx_conf_t, + us: *mut ngx_http_upstream_srv_conf_t, +) -> ngx_int_t { + ngx_log_debug_mask!(NGX_LOG_DEBUG_HTTP, (*cf).log, "CUSTOM UPSTREAM peer init_upstream"); + + let maybe_conf: Option<*mut SrvConfig> = + ngx_http_conf_upstream_srv_conf_mutable(us, &ngx_http_upstream_custom_module); + if let None = maybe_conf { + ngx_conf_log_error( + NGX_LOG_EMERG as usize, + cf, + 0, + "CUSTOM UPSTREAM no upstream srv_conf".as_bytes().as_ptr() as *const i8, + ); + return isize::from(Status::NGX_ERROR); + } else { + let hccf = maybe_conf.unwrap(); + // NOTE: ngx_conf_init_uint_value macro is unavailable + if (*hccf).max == u32::MAX { + (*hccf).max = 100; + } + + let init_upstream_ptr = (*hccf).original_init_upstream.unwrap(); + if init_upstream_ptr(cf, us) != Status::NGX_OK.into() { + ngx_conf_log_error( + NGX_LOG_EMERG as usize, + cf, + 0, + "CUSTOM UPSTREAM failed calling init_upstream".as_bytes().as_ptr() as *const i8, + ); + return isize::from(Status::NGX_ERROR); + } + + (*hccf).original_init_peer = (*us).peer.init; + (*us).peer.init = Some(http_upstream_init_custom_peer); + } + + ngx_log_debug!((*cf).log, "CUSTOM UPSTREAM end peer init_upstream"); + isize::from(Status::NGX_OK) +} + +// ngx_http_upstream_commands_set_custom +// Entry point for the module, if this command is set our custom upstreams take effect. +// The original upstream initializer function is saved and replaced with this module's initializer. +#[no_mangle] +unsafe extern "C" fn ngx_http_upstream_commands_set_custom( + cf: *mut ngx_conf_t, + cmd: *mut ngx_command_t, + conf: *mut c_void, +) -> *mut c_char { + ngx_log_debug_mask!(NGX_LOG_DEBUG_HTTP, (*cf).log, "CUSTOM UPSTREAM module init"); + + ngx_log_debug!((*cf).log, "CUSTOM DEBUG !MASK LOG"); + + let mut ccf = &mut (*(conf as *mut SrvConfig)); + + if (*(*cf).args).nelts == 2 { + let value: &[ngx_str_t] = slice::from_raw_parts((*(*cf).args).elts as *const ngx_str_t, (*(*cf).args).nelts); + let n = ngx_atoi(value[1].data, value[1].len); + if n == (NGX_ERROR as isize) || n == 0 { + ngx_conf_log_error( + NGX_LOG_EMERG as usize, + cf, + 0, + "invalid value \"%V\" in \"%V\" directive".as_bytes().as_ptr() as *const i8, + value[1], + &(*cmd).name, + ); + return usize::MAX as *mut i8; + } + ccf.max = n as u32; + } + + let uscf: *mut ngx_http_upstream_srv_conf_t = + ngx_http_conf_get_module_srv_conf(cf, &ngx_http_upstream_custom_module) as *mut ngx_http_upstream_srv_conf_t; + + ccf.original_init_upstream = if (*uscf).peer.init_upstream.is_some() { + (*uscf).peer.init_upstream + } else { + Some(ngx_http_upstream_init_round_robin) + }; + + (*uscf).peer.init_upstream = Some(ngx_http_upstream_init_custom); + + ngx_log_debug!((*cf).log, "CUSTOM UPSTREAM end module init"); + // NGX_CONF_OK + std::ptr::null_mut() +} + +// The upstream module. +// Only server blocks are supported to trigger the module command; therefore, the only callback +// implemented is our `create_srv_conf` method. +struct Module; + +impl HTTPModule for Module { + type MainConf = (); + type SrvConf = SrvConfig; + type LocConf = (); + + unsafe extern "C" fn create_srv_conf(cf: *mut ngx_conf_t) -> *mut c_void { + let mut pool = Pool::from_ngx_pool((*cf).pool); + let conf = pool.alloc_type::(); + if conf.is_null() { + ngx_conf_log_error( + NGX_LOG_EMERG as usize, + cf, + 0, + "CUSTOM UPSTREAM could not allocate memory for config" + .as_bytes() + .as_ptr() as *const i8, + ); + return std::ptr::null_mut(); + } + + (*conf).max = NGX_CONF_UNSET as u32; + + ngx_log_debug!((*cf).log, "CUSTOM UPSTREAM end create_srv_conf"); + conf as *mut c_void + } +} diff --git a/src/http/conf.rs b/src/http/conf.rs index 4761bb0..f26cffb 100644 --- a/src/http/conf.rs +++ b/src/http/conf.rs @@ -31,3 +31,35 @@ pub unsafe fn ngx_http_conf_get_module_loc_conf( let http_conf_ctx = (*cf).ctx as *mut ngx_http_conf_ctx_t; *(*http_conf_ctx).loc_conf.add(module.ctx_index) as *mut ngx_http_core_loc_conf_t } + +/// # Safety +/// +/// The caller has provided a value `ngx_http_upstream_srv_conf_t. If the `us` argument is null, a +/// None Option is returned; however, if the `us` internal fields are invalid or the module index +/// is out of bounds failures may still occur. +pub unsafe fn ngx_http_conf_upstream_srv_conf_immutable( + us: *const ngx_http_upstream_srv_conf_t, + module: &ngx_module_t, +) -> Option<*const T> { + if us.is_null() { + return None; + } + let cf: *const T = (*us).srv_conf.offset(module.ctx_index as isize) as *const T; + Some(cf) +} + +/// # Safety +/// +/// The caller has provided a value `ngx_http_upstream_srv_conf_t. If the `us` argument is null, a +/// None Option is returned; however, if the `us` internal fields are invalid or the module index +/// is out of bounds failures may still occur. +pub unsafe fn ngx_http_conf_upstream_srv_conf_mutable( + us: *const ngx_http_upstream_srv_conf_t, + module: &ngx_module_t, +) -> Option<*mut T> { + if us.is_null() { + return None; + } + let cf: *mut T = (*us).srv_conf.offset(module.ctx_index as isize) as *mut T; + Some(cf) +} diff --git a/src/http/mod.rs b/src/http/mod.rs index 230ce0b..024ad1f 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -2,8 +2,10 @@ mod conf; mod module; mod request; mod status; +mod upstream; pub use conf::*; pub use module::*; pub use request::*; pub use status::*; +pub use upstream::*; diff --git a/src/http/request.rs b/src/http/request.rs index 88607fd..c86f7c4 100644 --- a/src/http/request.rs +++ b/src/http/request.rs @@ -79,6 +79,17 @@ macro_rules! http_variable_get { #[repr(transparent)] pub struct Request(ngx_http_request_t); +impl<'a> From<&'a Request> for *const ngx_http_request_t { + fn from(request: &'a Request) -> Self { + &request.0 as *const _ + } +} +impl<'a> From<&'a mut Request> for *mut ngx_http_request_t { + fn from(request: &'a mut Request) -> Self { + &request.0 as *const _ as *mut _ + } +} + impl Request { /// Create a [`Request`] from an [`ngx_http_request_t`]. /// @@ -104,6 +115,17 @@ impl Request { unsafe { Pool::from_ngx_pool(self.0.pool) } } + /// Returns the result as an `Option` if it exists, otherwise `None`. + /// + /// The option wraps a pointer to a [`ngx_http_upstream_t`] upstream server object. + /// + /// [`ngx_http_upstream_t`]: is best described in + /// https://nginx.org/en/docs/dev/development_guide.html#http_request + /// https://nginx.org/en/docs/dev/development_guide.html#http_load_balancing + pub fn upstream(&self) -> Option<*mut ngx_http_upstream_t> { + Some(self.0.upstream) + } + /// Pointer to a [`ngx_connection_t`] client connection object. /// /// [`ngx_connection_t`]: https://nginx.org/en/docs/dev/development_guide.html#connection diff --git a/src/http/upstream.rs b/src/http/upstream.rs new file mode 100644 index 0000000..186ea73 --- /dev/null +++ b/src/http/upstream.rs @@ -0,0 +1,15 @@ +/// Define a static upstream peer initializer +/// +/// Initializes the upstream 'get', 'free', and 'session' callbacks and gives the module writer an +/// opportunity to set custom data. +/// Load Balancing: +#[macro_export] +macro_rules! http_upstream_peer_init { + ( $name: ident, $handler: expr ) => { + #[no_mangle] + unsafe extern "C" fn $name(r: *mut ngx_http_request_t, us: *mut ngx_http_upstream_srv_conf_t) -> ngx_int_t { + let status: Status = $handler(unsafe { &mut Request::from_ngx_http_request(r) }, us); + status.0 + } + }; +} diff --git a/src/log.rs b/src/log.rs index 37d4ac4..5271957 100644 --- a/src/log.rs +++ b/src/log.rs @@ -27,3 +27,26 @@ macro_rules! ngx_log_debug_http { $crate::ngx_log_debug!(log, $($arg)*); } } + +/// Log with appropriate debug mask. +/// +/// When the request logger is available `ngx_log_debug_http` can be used for `NGX_LOG_DEBUG_HTTP` masks. +/// This macro is useful when other masks are necessary or when the request logger is not +/// conveniently accessible. +/// +/// See https://nginx.org/en/docs/dev/development_guide.html#logging for details and available +/// masks. +#[macro_export] +macro_rules! ngx_log_debug_mask { + ( $mask:expr, $log:expr, $($arg:tt)* ) => { + let log_level = unsafe { (*$log).log_level }; + if log_level & $mask as usize != 0 { + let level = $mask as $crate::ffi::ngx_uint_t; + let fmt = ::std::ffi::CString::new("%s").unwrap(); + let c_message = ::std::ffi::CString::new(format!($($arg)*)).unwrap(); + unsafe { + $crate::ffi::ngx_log_error_core(level, $log, 0, fmt.as_ptr(), c_message.as_ptr()); + } + } + } +}