From 9fcef2d51104f8a17ed6cddb540babc4a92edf7a Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Wed, 28 Jun 2023 10:33:15 +0900 Subject: [PATCH 01/43] deps and submodule --- Cargo.toml | 8 ++++---- quinn | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8d955ac..1a0be43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,9 +18,9 @@ sticky-cookie = ["base64", "sha2", "chrono"] [dependencies] anyhow = "1.0.71" -clap = { version = "4.3.4", features = ["std", "cargo", "wrap_help"] } +clap = { version = "4.3.8", features = ["std", "cargo", "wrap_help"] } rand = "0.8.5" -toml = { version = "0.7.4", default-features = false, features = ["parse"] } +toml = { version = "0.7.5", default-features = false, features = ["parse"] } rustc-hash = "1.1.0" serde = { version = "1.0.164", default-features = false, features = ["derive"] } bytes = "1.4.0" @@ -28,7 +28,7 @@ thiserror = "1.0.40" x509-parser = "0.15.0" derive_builder = "0.12.0" futures = { version = "0.3.28", features = ["alloc", "async-await"] } -tokio = { version = "1.28.2", default-features = false, features = [ +tokio = { version = "1.29.0", default-features = false, features = [ "net", "rt-multi-thread", "parking_lot", @@ -38,7 +38,7 @@ tokio = { version = "1.28.2", default-features = false, features = [ ] } # http and tls -hyper = { version = "0.14.26", default-features = false, features = [ +hyper = { version = "0.14.27", default-features = false, features = [ "server", "http1", "http2", diff --git a/quinn b/quinn index 7914468..b30711f 160000 --- a/quinn +++ b/quinn @@ -1 +1 @@ -Subproject commit 7914468e27621633a8399c8d02fbf3f557d54df2 +Subproject commit b30711f5595983989b60bbbad0ac3f067be7a596 From b9c768247fe06ee9b18cb81819ae106d4b259455 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 4 Jul 2023 00:17:03 +0900 Subject: [PATCH 02/43] deps --- Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1a0be43..78e651a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,17 +18,17 @@ sticky-cookie = ["base64", "sha2", "chrono"] [dependencies] anyhow = "1.0.71" -clap = { version = "4.3.8", features = ["std", "cargo", "wrap_help"] } +clap = { version = "4.3.10", features = ["std", "cargo", "wrap_help"] } rand = "0.8.5" toml = { version = "0.7.5", default-features = false, features = ["parse"] } rustc-hash = "1.1.0" -serde = { version = "1.0.164", default-features = false, features = ["derive"] } +serde = { version = "1.0.165", default-features = false, features = ["derive"] } bytes = "1.4.0" thiserror = "1.0.40" x509-parser = "0.15.0" derive_builder = "0.12.0" futures = { version = "0.3.28", features = ["alloc", "async-await"] } -tokio = { version = "1.29.0", default-features = false, features = [ +tokio = { version = "1.29.1", default-features = false, features = [ "net", "rt-multi-thread", "parking_lot", @@ -51,7 +51,7 @@ hyper-rustls = { version = "0.24.0", default-features = false, features = [ "http2", ] } tokio-rustls = { version = "0.24.1", features = ["early-data"] } -rustls-pemfile = "1.0.2" +rustls-pemfile = "1.0.3" rustls = { version = "0.21.2", default-features = false } webpki = "0.22.0" From b5a6509ddbe5d97689bae87146f929b09779b065 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 7 Jul 2023 10:21:07 +0900 Subject: [PATCH 03/43] deps --- Cargo.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 78e651a..c73533e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,13 +18,13 @@ sticky-cookie = ["base64", "sha2", "chrono"] [dependencies] anyhow = "1.0.71" -clap = { version = "4.3.10", features = ["std", "cargo", "wrap_help"] } +clap = { version = "4.3.11", features = ["std", "cargo", "wrap_help"] } rand = "0.8.5" -toml = { version = "0.7.5", default-features = false, features = ["parse"] } +toml = { version = "0.7.6", default-features = false, features = ["parse"] } rustc-hash = "1.1.0" -serde = { version = "1.0.165", default-features = false, features = ["derive"] } +serde = { version = "1.0.167", default-features = false, features = ["derive"] } bytes = "1.4.0" -thiserror = "1.0.40" +thiserror = "1.0.43" x509-parser = "0.15.0" derive_builder = "0.12.0" futures = { version = "0.3.28", features = ["alloc", "async-await"] } @@ -44,7 +44,7 @@ hyper = { version = "0.14.27", default-features = false, features = [ "http2", "stream", ] } -hyper-rustls = { version = "0.24.0", default-features = false, features = [ +hyper-rustls = { version = "0.24.1", default-features = false, features = [ "tokio-runtime", "webpki-tokio", "http1", @@ -52,7 +52,7 @@ hyper-rustls = { version = "0.24.0", default-features = false, features = [ ] } tokio-rustls = { version = "0.24.1", features = ["early-data"] } rustls-pemfile = "1.0.3" -rustls = { version = "0.21.2", default-features = false } +rustls = { version = "0.21.3", default-features = false } webpki = "0.22.0" # logging From 80e10d5ccd5b006e7c4b5a05326e7f710126d0c3 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 7 Jul 2023 20:24:38 +0900 Subject: [PATCH 04/43] refactor: make globals simple --- src/config/parse.rs | 2 +- src/config/toml.rs | 13 ++++++++----- src/error.rs | 3 +++ src/globals.rs | 38 +++++++++++++++++++++++++------------- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/config/parse.rs b/src/config/parse.rs index 8e4ddf7..9c4fd86 100644 --- a/src/config/parse.rs +++ b/src/config/parse.rs @@ -205,7 +205,7 @@ fn get_reverse_proxy( let mut upstream: HashMap = HashMap::default(); rp_settings.iter().for_each(|rpo| { - let upstream_vec: Vec = rpo.upstream.iter().map(|x| x.to_upstream().unwrap()).collect(); + let upstream_vec: Vec = rpo.upstream.iter().map(|x| x.try_into().unwrap()).collect(); // let upstream_iter = rpo.upstream.iter().map(|x| x.to_upstream().unwrap()); // let lb_upstream_num = vec_upstream.len(); let elem = UpstreamGroupBuilder::default() diff --git a/src/config/toml.rs b/src/config/toml.rs index 6ce48b2..29e76cc 100644 --- a/src/config/toml.rs +++ b/src/config/toml.rs @@ -65,8 +65,11 @@ pub struct UpstreamParams { pub location: String, pub tls: Option, } -impl UpstreamParams { - pub fn to_upstream(&self) -> Result { + +impl TryInto for &UpstreamParams { + type Error = RpxyError; + + fn try_into(self) -> std::result::Result { let mut scheme = "http"; if let Some(t) = self.tls { if t { @@ -81,9 +84,9 @@ impl UpstreamParams { } impl ConfigToml { - pub fn new(config_file: &str) -> std::result::Result { - let config_str = fs::read_to_string(config_file).context("Failed to read config file")?; + pub fn new(config_file: &str) -> std::result::Result { + let config_str = fs::read_to_string(config_file).map_err(RpxyError::Io)?; - toml::from_str(&config_str).context("Failed to parse toml config") + toml::from_str(&config_str).map_err(RpxyError::TomlDe) } } diff --git a/src/error.rs b/src/error.rs index 3fb3474..18b4307 100644 --- a/src/error.rs +++ b/src/error.rs @@ -29,6 +29,9 @@ pub enum RpxyError { #[error("I/O Error")] Io(#[from] io::Error), + #[error("Toml Deserialization Error")] + TomlDe(#[from] toml::de::Error), + #[cfg(feature = "http3")] #[error("Quic Connection Error")] QuicConn(#[from] quinn::ConnectionError), diff --git a/src/globals.rs b/src/globals.rs index 0bd06a6..579695f 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -6,25 +6,24 @@ use std::sync::{ }; use tokio::time::Duration; +/// Global object containing proxy configurations and shared object like counters. +/// But note that in Globals, we do not have Mutex and RwLock. It is indeed, the context shared among async tasks. pub struct Globals { - pub listen_sockets: Vec, - pub http_port: Option, - pub https_port: Option, + pub listen_sockets: Vec, // when instantiate server + pub http_port: Option, // when instantiate server + pub https_port: Option, // when instantiate server - pub proxy_timeout: Duration, - pub upstream_timeout: Duration, + pub proxy_timeout: Duration, // when serving requests at Proxy + pub upstream_timeout: Duration, // when serving requests at Handler - pub max_clients: usize, - pub request_count: RequestCount, - pub max_concurrent_streams: u32, - pub keepalive: bool, - - pub runtime_handle: tokio::runtime::Handle, - pub backends: Backends, + pub max_clients: usize, // when serving requests + pub max_concurrent_streams: u32, // when instantiate server + pub keepalive: bool, // when instantiate server // experimentals - pub sni_consistency: bool, + pub sni_consistency: bool, // Handler + // All need to make packet acceptor #[cfg(feature = "http3")] pub http3: bool, #[cfg(feature = "http3")] @@ -39,9 +38,22 @@ pub struct Globals { pub h3_max_concurrent_connections: u32, #[cfg(feature = "http3")] pub h3_max_idle_timeout: Option, + + // Shared context + // Backend application objects to which http request handler forward incoming requests + pub backends: Backends, + // Counter for serving requests + pub request_count: RequestCount, + // Async task runtime handler + pub runtime_handle: tokio::runtime::Handle, } +// // TODO: Implement default for default values +// #[derive(Debug, Clone)] +// pub struct ProxyConfig {} + #[derive(Debug, Clone, Default)] +/// Counter for serving requests pub struct RequestCount(Arc); impl RequestCount { From f8d37f7846df329af82ce0833283d6289a227f55 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 7 Jul 2023 21:54:56 +0900 Subject: [PATCH 05/43] refactor: make some config parameters in globals belong to other struct --- src/backend/mod.rs | 9 ++++++ src/config/parse.rs | 50 ++++++++++++++--------------- src/globals.rs | 63 +++++++++++++++++++++++++++++-------- src/handler/handler_main.rs | 14 ++++----- src/main.rs | 50 ++++------------------------- src/proxy/proxy_h3.rs | 6 ++-- src/proxy/proxy_main.rs | 8 ++--- src/proxy/proxy_tls.rs | 10 +++--- 8 files changed, 109 insertions(+), 101 deletions(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index b7923c5..4aa5f7a 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -218,6 +218,15 @@ pub struct Backends { pub default_server_name_bytes: Option, // for plaintext http } +impl Default for Backends { + fn default() -> Self { + Self { + default_server_name_bytes: None, + apps: HashMap::::default(), + } + } +} + pub type SniServerCryptoMap = HashMap>; pub struct ServerCrypto { // For Quic/HTTP3, only servers with no client authentication diff --git a/src/config/parse.rs b/src/config/parse.rs index 9c4fd86..1b13d30 100644 --- a/src/config/parse.rs +++ b/src/config/parse.rs @@ -30,11 +30,11 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro }; // listen port and socket - globals.http_port = config.listen_port; - globals.https_port = config.listen_port_tls; + globals.proxy_config.http_port = config.listen_port; + globals.proxy_config.https_port = config.listen_port_tls; ensure!( - { globals.http_port.is_some() || globals.https_port.is_some() } && { - if let (Some(p), Some(t)) = (globals.http_port, globals.https_port) { + { globals.proxy_config.http_port.is_some() || globals.proxy_config.https_port.is_some() } && { + if let (Some(p), Some(t)) = (globals.proxy_config.http_port, globals.proxy_config.https_port) { p != t } else { true @@ -53,32 +53,32 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro LISTEN_ADDRESSES_V4.to_vec() } }; - globals.listen_sockets = listen_addresses + globals.proxy_config.listen_sockets = listen_addresses .iter() .flat_map(|x| { let mut v: Vec = vec![]; - if let Some(p) = globals.http_port { + if let Some(p) = globals.proxy_config.http_port { v.push(format!("{x}:{p}").parse().unwrap()); } - if let Some(p) = globals.https_port { + if let Some(p) = globals.proxy_config.https_port { v.push(format!("{x}:{p}").parse().unwrap()); } v }) .collect(); - if globals.http_port.is_some() { - info!("Listen port: {}", globals.http_port.unwrap()); + if globals.proxy_config.http_port.is_some() { + info!("Listen port: {}", globals.proxy_config.http_port.unwrap()); } - if globals.https_port.is_some() { - info!("Listen port: {} (for TLS)", globals.https_port.unwrap()); + if globals.proxy_config.https_port.is_some() { + info!("Listen port: {} (for TLS)", globals.proxy_config.https_port.unwrap()); } // max values if let Some(c) = config.max_clients { - globals.max_clients = c as usize; + globals.proxy_config.max_clients = c as usize; } if let Some(c) = config.max_concurrent_streams { - globals.max_concurrent_streams = c; + globals.proxy_config.max_concurrent_streams = c; } // backend apps @@ -90,7 +90,7 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro for (app_name, app) in apps.0.iter() { ensure!(app.server_name.is_some(), "Missing server_name"); let server_name_string = app.server_name.as_ref().unwrap(); - if globals.http_port.is_none() { + if globals.proxy_config.http_port.is_none() { // if only https_port is specified, tls must be configured ensure!(app.tls.is_some()) } @@ -108,7 +108,7 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro // TLS settings and build backend instance let backend = if app.tls.is_none() { - ensure!(globals.http_port.is_some(), "Required HTTP port"); + ensure!(globals.proxy_config.http_port.is_some(), "Required HTTP port"); backend_builder.build()? } else { let tls = app.tls.as_ref().unwrap(); @@ -117,7 +117,7 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro let https_redirection = if tls.https_redirection.is_none() { Some(true) // Default true } else { - ensure!(globals.https_port.is_some()); // only when both https ports are configured. + ensure!(globals.proxy_config.https_port.is_some()); // only when both https ports are configured. tls.https_redirection }; @@ -159,28 +159,28 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro #[cfg(feature = "http3")] { if let Some(h3option) = exp.h3 { - globals.http3 = true; + globals.proxy_config.http3 = true; info!("Experimental HTTP/3.0 is enabled. Note it is still very unstable."); if let Some(x) = h3option.alt_svc_max_age { - globals.h3_alt_svc_max_age = x; + globals.proxy_config.h3_alt_svc_max_age = x; } if let Some(x) = h3option.request_max_body_size { - globals.h3_request_max_body_size = x; + globals.proxy_config.h3_request_max_body_size = x; } if let Some(x) = h3option.max_concurrent_connections { - globals.h3_max_concurrent_connections = x; + globals.proxy_config.h3_max_concurrent_connections = x; } if let Some(x) = h3option.max_concurrent_bidistream { - globals.h3_max_concurrent_bidistream = x.into(); + globals.proxy_config.h3_max_concurrent_bidistream = x.into(); } if let Some(x) = h3option.max_concurrent_unistream { - globals.h3_max_concurrent_unistream = x.into(); + globals.proxy_config.h3_max_concurrent_unistream = x.into(); } if let Some(x) = h3option.max_idle_timeout { if x == 0u64 { - globals.h3_max_idle_timeout = None; + globals.proxy_config.h3_max_idle_timeout = None; } else { - globals.h3_max_idle_timeout = + globals.proxy_config.h3_max_idle_timeout = Some(quinn::IdleTimeout::try_from(tokio::time::Duration::from_secs(x)).unwrap()) } } @@ -188,7 +188,7 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro } if let Some(b) = exp.ignore_sni_consistency { - globals.sni_consistency = !b; + globals.proxy_config.sni_consistency = !b; if b { info!("Ignore consistency between TLS SNI and Host header (or Request line). Note it violates RFC."); } diff --git a/src/globals.rs b/src/globals.rs index 579695f..b5c4a46 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -1,4 +1,4 @@ -use crate::backend::Backends; +use crate::{backend::Backends, constants::*}; use std::net::SocketAddr; use std::sync::{ atomic::{AtomicUsize, Ordering}, @@ -9,6 +9,21 @@ use tokio::time::Duration; /// Global object containing proxy configurations and shared object like counters. /// But note that in Globals, we do not have Mutex and RwLock. It is indeed, the context shared among async tasks. pub struct Globals { + /// Configuration parameters for proxy transport and request handlers + pub proxy_config: ProxyConfig, + + /// Shared context - Backend application objects to which http request handler forward incoming requests + pub backends: Backends, + + /// Shared context - Counter for serving requests + pub request_count: RequestCount, + + /// Shared context - Async task runtime handler + pub runtime_handle: tokio::runtime::Handle, +} + +/// Configuration parameters for proxy transport and request handlers +pub struct ProxyConfig { pub listen_sockets: Vec, // when instantiate server pub http_port: Option, // when instantiate server pub https_port: Option, // when instantiate server @@ -22,7 +37,6 @@ pub struct Globals { // experimentals pub sni_consistency: bool, // Handler - // All need to make packet acceptor #[cfg(feature = "http3")] pub http3: bool, @@ -38,19 +52,42 @@ pub struct Globals { pub h3_max_concurrent_connections: u32, #[cfg(feature = "http3")] pub h3_max_idle_timeout: Option, - - // Shared context - // Backend application objects to which http request handler forward incoming requests - pub backends: Backends, - // Counter for serving requests - pub request_count: RequestCount, - // Async task runtime handler - pub runtime_handle: tokio::runtime::Handle, } -// // TODO: Implement default for default values -// #[derive(Debug, Clone)] -// pub struct ProxyConfig {} +impl Default for ProxyConfig { + fn default() -> Self { + Self { + listen_sockets: Vec::new(), + http_port: None, + https_port: None, + + // TODO: Reconsider each timeout values + proxy_timeout: Duration::from_secs(PROXY_TIMEOUT_SEC), + upstream_timeout: Duration::from_secs(UPSTREAM_TIMEOUT_SEC), + + max_clients: MAX_CLIENTS, + max_concurrent_streams: MAX_CONCURRENT_STREAMS, + keepalive: true, + + sni_consistency: true, + + #[cfg(feature = "http3")] + http3: false, + #[cfg(feature = "http3")] + h3_alt_svc_max_age: H3::ALT_SVC_MAX_AGE, + #[cfg(feature = "http3")] + h3_request_max_body_size: H3::REQUEST_MAX_BODY_SIZE, + #[cfg(feature = "http3")] + h3_max_concurrent_connections: H3::MAX_CONCURRENT_CONNECTIONS, + #[cfg(feature = "http3")] + h3_max_concurrent_bidistream: H3::MAX_CONCURRENT_BIDISTREAM.into(), + #[cfg(feature = "http3")] + h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM.into(), + #[cfg(feature = "http3")] + h3_max_idle_timeout: Some(quinn::IdleTimeout::try_from(Duration::from_secs(H3::MAX_IDLE_TIMEOUT)).unwrap()), + } + } +} #[derive(Debug, Clone, Default)] /// Counter for serving requests diff --git a/src/handler/handler_main.rs b/src/handler/handler_main.rs index a73dcbc..d2a47be 100644 --- a/src/handler/handler_main.rs +++ b/src/handler/handler_main.rs @@ -56,7 +56,7 @@ where }; // check consistency of between TLS SNI and HOST/Request URI Line. #[allow(clippy::collapsible_if)] - if tls_enabled && self.globals.sni_consistency { + if tls_enabled && self.globals.proxy_config.sni_consistency { if server_name != tls_server_name.unwrap_or_default() { return self.return_with_error_log(StatusCode::MISDIRECTED_REQUEST, &mut log_data); } @@ -75,7 +75,7 @@ where if !tls_enabled && backend.https_redirection.unwrap_or(false) { debug!("Redirect to secure connection: {}", &backend.server_name); log_data.status_code(&StatusCode::PERMANENT_REDIRECT).output(); - return secure_redirection(&backend.server_name, self.globals.https_port, &req); + return secure_redirection(&backend.server_name, self.globals.proxy_config.https_port, &req); } // Find reverse proxy for given path and choose one of upstream host @@ -112,7 +112,7 @@ where // Forward request to let mut res_backend = { - match timeout(self.globals.upstream_timeout, self.forwarder.request(req)).await { + match timeout(self.globals.proxy_config.upstream_timeout, self.forwarder.request(req)).await { Err(_) => { return self.return_with_error_log(StatusCode::GATEWAY_TIMEOUT, &mut log_data); } @@ -207,14 +207,14 @@ where #[cfg(feature = "http3")] { // TODO: Workaround for avoid h3 for client authentication - if self.globals.http3 && chosen_backend.client_ca_cert_path.is_none() { - if let Some(port) = self.globals.https_port { + if self.globals.proxy_config.http3 && chosen_backend.client_ca_cert_path.is_none() { + if let Some(port) = self.globals.proxy_config.https_port { add_header_entry_overwrite_if_exist( headers, header::ALT_SVC.as_str(), format!( "h3=\":{}\"; ma={}, h3-29=\":{}\"; ma={}", - port, self.globals.h3_alt_svc_max_age, port, self.globals.h3_alt_svc_max_age + port, self.globals.proxy_config.h3_alt_svc_max_age, port, self.globals.proxy_config.h3_alt_svc_max_age ), )?; } @@ -225,7 +225,7 @@ where } #[cfg(not(feature = "http3"))] { - if let Some(port) = self.globals.https_port { + if let Some(port) = self.globals.proxy_config.https_port { headers.remove(header::ALT_SVC.as_str()); } } diff --git a/src/main.rs b/src/main.rs index ea60fa7..2b5e1fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,22 +16,13 @@ mod proxy; mod utils; use crate::{ - backend::{Backend, Backends}, - config::parse_opts, - constants::*, - error::*, - globals::*, - handler::HttpMessageHandlerBuilder, - log::*, + backend::Backends, config::parse_opts, error::*, globals::*, handler::HttpMessageHandlerBuilder, log::*, proxy::ProxyBuilder, - utils::ServerNameBytesExp, }; use futures::future::select_all; use hyper::Client; // use hyper_trust_dns::TrustDnsResolver; -use rustc_hash::FxHashMap as HashMap; use std::sync::Arc; -use tokio::time::Duration; fn main() { init_logger(); @@ -43,41 +34,12 @@ fn main() { runtime.block_on(async { let mut globals = Globals { - listen_sockets: Vec::new(), - http_port: None, - https_port: None, + // TODO: proxy configはarcに包んでこいつだけ使いまわせばいいように変えていく。backendsも？ + proxy_config: ProxyConfig::default(), + backends: Backends::default(), - // TODO: Reconsider each timeout values - proxy_timeout: Duration::from_secs(PROXY_TIMEOUT_SEC), - upstream_timeout: Duration::from_secs(UPSTREAM_TIMEOUT_SEC), - - max_clients: MAX_CLIENTS, request_count: Default::default(), - max_concurrent_streams: MAX_CONCURRENT_STREAMS, - keepalive: true, - runtime_handle: runtime.handle().clone(), - backends: Backends { - default_server_name_bytes: None, - apps: HashMap::::default(), - }, - - sni_consistency: true, - - #[cfg(feature = "http3")] - http3: false, - #[cfg(feature = "http3")] - h3_alt_svc_max_age: H3::ALT_SVC_MAX_AGE, - #[cfg(feature = "http3")] - h3_request_max_body_size: H3::REQUEST_MAX_BODY_SIZE, - #[cfg(feature = "http3")] - h3_max_concurrent_connections: H3::MAX_CONCURRENT_CONNECTIONS, - #[cfg(feature = "http3")] - h3_max_concurrent_bidistream: H3::MAX_CONCURRENT_BIDISTREAM.into(), - #[cfg(feature = "http3")] - h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM.into(), - #[cfg(feature = "http3")] - h3_max_idle_timeout: Some(quinn::IdleTimeout::try_from(Duration::from_secs(H3::MAX_IDLE_TIMEOUT)).unwrap()), }; if let Err(e) = parse_opts(&mut globals) { @@ -105,10 +67,10 @@ async fn entrypoint(globals: Arc) -> Result<()> { .globals(globals.clone()) .build()?; - let addresses = globals.listen_sockets.clone(); + let addresses = globals.proxy_config.listen_sockets.clone(); let futures = select_all(addresses.into_iter().map(|addr| { let mut tls_enabled = false; - if let Some(https_port) = globals.https_port { + if let Some(https_port) = globals.proxy_config.https_port { tls_enabled = https_port == addr.port() } diff --git a/src/proxy/proxy_h3.rs b/src/proxy/proxy_h3.rs index d5a6c88..12ebd7d 100644 --- a/src/proxy/proxy_h3.rs +++ b/src/proxy/proxy_h3.rs @@ -43,7 +43,7 @@ where // We consider the connection count separately from the stream count. // Max clients for h1/h2 = max 'stream' for h3. let request_count = self.globals.request_count.clone(); - if request_count.increment() > self.globals.max_clients { + if request_count.increment() > self.globals.proxy_config.max_clients { request_count.decrement(); h3_conn.shutdown(0).await?; break; @@ -54,7 +54,7 @@ where let tls_server_name_inner = tls_server_name.clone(); self.globals.runtime_handle.spawn(async move { if let Err(e) = timeout( - self_inner.globals.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2 + self_inner.globals.proxy_config.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2 self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner), ) .await @@ -97,7 +97,7 @@ where // Buffering and sending body through channel for protocol conversion like h3 -> h2/http1.1 // The underling buffering, i.e., buffer given by the API recv_data.await?, is handled by quinn. - let max_body_size = self.globals.h3_request_max_body_size; + let max_body_size = self.globals.proxy_config.h3_request_max_body_size; self.globals.runtime_handle.spawn(async move { let mut sender = body_sender; let mut size = 0usize; diff --git a/src/proxy/proxy_main.rs b/src/proxy/proxy_main.rs index 722ef3c..a0f9660 100644 --- a/src/proxy/proxy_main.rs +++ b/src/proxy/proxy_main.rs @@ -56,7 +56,7 @@ where I: AsyncRead + AsyncWrite + Send + Unpin + 'static, { let request_count = self.globals.request_count.clone(); - if request_count.increment() > self.globals.max_clients { + if request_count.increment() > self.globals.proxy_config.max_clients { request_count.decrement(); return; } @@ -64,7 +64,7 @@ where self.globals.runtime_handle.clone().spawn(async move { timeout( - self.globals.proxy_timeout + Duration::from_secs(1), + self.globals.proxy_config.proxy_timeout + Duration::from_secs(1), server .serve_connection( stream, @@ -103,8 +103,8 @@ where pub async fn start(self) -> Result<()> { let mut server = Http::new(); - server.http1_keep_alive(self.globals.keepalive); - server.http2_max_concurrent_streams(self.globals.max_concurrent_streams); + server.http1_keep_alive(self.globals.proxy_config.keepalive); + server.http2_max_concurrent_streams(self.globals.proxy_config.max_concurrent_streams); server.pipeline_flush(true); let executor = LocalExecutor::new(self.globals.runtime_handle.clone()); let server = server.with_executor(executor); diff --git a/src/proxy/proxy_tls.rs b/src/proxy/proxy_tls.rs index de18e0c..aa9c69f 100644 --- a/src/proxy/proxy_tls.rs +++ b/src/proxy/proxy_tls.rs @@ -129,13 +129,13 @@ where let mut transport_config_quic = TransportConfig::default(); transport_config_quic - .max_concurrent_bidi_streams(self.globals.h3_max_concurrent_bidistream) - .max_concurrent_uni_streams(self.globals.h3_max_concurrent_unistream) - .max_idle_timeout(self.globals.h3_max_idle_timeout); + .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream) + .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream) + .max_idle_timeout(self.globals.proxy_config.h3_max_idle_timeout); let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config)); server_config_h3.transport = Arc::new(transport_config_quic); - server_config_h3.concurrent_connections(self.globals.h3_max_concurrent_connections); + server_config_h3.concurrent_connections(self.globals.proxy_config.h3_max_concurrent_connections); let endpoint = Endpoint::server(server_config_h3, self.listening_on)?; let mut server_crypto: Option> = None; @@ -212,7 +212,7 @@ where } #[cfg(feature = "http3")] { - if self.globals.http3 { + if self.globals.proxy_config.http3 { tokio::select! { _= self.cert_service(tx) => { error!("Cert service for TLS exited"); From fab28e8609550d8936d4996cde77aa9b9e77e462 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 10 Jul 2023 18:04:15 +0900 Subject: [PATCH 06/43] make globals more simple --- src/backend/mod.rs | 10 +- src/config/mod.rs | 2 +- src/config/parse.rs | 252 +++++++++++--------------------------------- src/config/toml.rs | 200 ++++++++++++++++++++++++++++++++--- src/globals.rs | 2 +- src/main.rs | 23 ++-- 6 files changed, 261 insertions(+), 228 deletions(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 4aa5f7a..97753f2 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -212,21 +212,13 @@ impl Backend { } } +#[derive(Default)] /// HashMap and some meta information for multiple Backend structs. pub struct Backends { pub apps: HashMap, // hyper::uriで抜いたhostで引っ掛ける pub default_server_name_bytes: Option, // for plaintext http } -impl Default for Backends { - fn default() -> Self { - Self { - default_server_name_bytes: None, - apps: HashMap::::default(), - } - } -} - pub type SniServerCryptoMap = HashMap>; pub struct ServerCrypto { // For Quic/HTTP3, only servers with no client authentication diff --git a/src/config/mod.rs b/src/config/mod.rs index 6e8123c..54b2600 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,4 +1,4 @@ mod parse; mod toml; -pub use parse::parse_opts; +pub use parse::build_globals; diff --git a/src/config/parse.rs b/src/config/parse.rs index 1b13d30..1593aba 100644 --- a/src/config/parse.rs +++ b/src/config/parse.rs @@ -1,17 +1,9 @@ -use super::toml::{ConfigToml, ReverseProxyOption}; -use crate::{ - backend::{BackendBuilder, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption}, - constants::*, - error::*, - globals::*, - log::*, - utils::{BytesName, PathNameBytesExp}, -}; +use super::toml::ConfigToml; +use crate::{backend::Backends, error::*, globals::*, log::*, utils::BytesName}; use clap::Arg; -use rustc_hash::FxHashMap as HashMap; -use std::net::SocketAddr; +use tokio::runtime::Handle; -pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Error> { +pub fn build_globals(runtime_handle: Handle) -> std::result::Result { let _ = include_str!("../../Cargo.toml"); let options = clap::command!().arg( Arg::new("config_file") @@ -22,6 +14,7 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro ); let matches = options.get_matches(); + /////////////////////////////////// let config = if let Some(config_file_path) = matches.get_one::("config_file") { ConfigToml::new(config_file_path)? } else { @@ -29,117 +22,67 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro ConfigToml::default() }; - // listen port and socket - globals.proxy_config.http_port = config.listen_port; - globals.proxy_config.https_port = config.listen_port_tls; - ensure!( - { globals.proxy_config.http_port.is_some() || globals.proxy_config.https_port.is_some() } && { - if let (Some(p), Some(t)) = (globals.proxy_config.http_port, globals.proxy_config.https_port) { - p != t - } else { - true - } - }, - anyhow!("Wrong port spec.") - ); - // NOTE: when [::]:xx is bound, both v4 and v6 listeners are enabled. - let listen_addresses: Vec<&str> = match config.listen_ipv6 { - Some(true) => { - info!("Listen both IPv4 and IPv6"); - LISTEN_ADDRESSES_V6.to_vec() - } - Some(false) | None => { - info!("Listen IPv4"); - LISTEN_ADDRESSES_V4.to_vec() - } - }; - globals.proxy_config.listen_sockets = listen_addresses - .iter() - .flat_map(|x| { - let mut v: Vec = vec![]; - if let Some(p) = globals.proxy_config.http_port { - v.push(format!("{x}:{p}").parse().unwrap()); - } - if let Some(p) = globals.proxy_config.https_port { - v.push(format!("{x}:{p}").parse().unwrap()); - } - v - }) - .collect(); - if globals.proxy_config.http_port.is_some() { - info!("Listen port: {}", globals.proxy_config.http_port.unwrap()); + /////////////////////////////////// + // build proxy config + let proxy_config: ProxyConfig = (&config).try_into()?; + // For loggings + if proxy_config.listen_sockets.iter().any(|addr| addr.is_ipv6()) { + info!("Listen both IPv4 and IPv6") + } else { + info!("Listen IPv4") } - if globals.proxy_config.https_port.is_some() { - info!("Listen port: {} (for TLS)", globals.proxy_config.https_port.unwrap()); + if proxy_config.http_port.is_some() { + info!("Listen port: {}", proxy_config.http_port.unwrap()); + } + if proxy_config.https_port.is_some() { + info!("Listen port: {} (for TLS)", proxy_config.https_port.unwrap()); + } + if proxy_config.http3 { + info!("Experimental HTTP/3.0 is enabled. Note it is still very unstable."); + } + if !proxy_config.sni_consistency { + info!("Ignore consistency between TLS SNI and Host header (or Request line). Note it violates RFC."); } - // max values - if let Some(c) = config.max_clients { - globals.proxy_config.max_clients = c as usize; - } - if let Some(c) = config.max_concurrent_streams { - globals.proxy_config.max_concurrent_streams = c; - } + /////////////////////////////////// + // backend_apps + let apps = config.apps.ok_or(anyhow!("Missing application spec"))?; - // backend apps - ensure!(config.apps.is_some(), "Missing application spec."); - let apps = config.apps.unwrap(); + // assertions for all backend apps ensure!(!apps.0.is_empty(), "Wrong application spec."); + // if only https_port is specified, tls must be configured for all apps + if proxy_config.http_port.is_none() { + ensure!( + apps.0.iter().all(|(_, app)| app.tls.is_some()), + "Some apps serves only plaintext HTTP" + ); + } + // https redirection can be configured if both ports are active + if !(proxy_config.https_port.is_some() && proxy_config.http_port.is_some()) { + ensure!( + apps.0.iter().all(|(_, app)| { + if let Some(tls) = app.tls.as_ref() { + tls.https_redirection.is_none() + } else { + true + } + }), + "https_redirection can be specified only when both http_port and https_port are specified" + ); + } - // each app + // build backends + let mut backends = Backends::default(); for (app_name, app) in apps.0.iter() { - ensure!(app.server_name.is_some(), "Missing server_name"); - let server_name_string = app.server_name.as_ref().unwrap(); - if globals.proxy_config.http_port.is_none() { - // if only https_port is specified, tls must be configured - ensure!(app.tls.is_some()) - } - - // backend builder - let mut backend_builder = BackendBuilder::default(); - // reverse proxy settings - ensure!(app.reverse_proxy.is_some(), "Missing reverse_proxy"); - let reverse_proxy = get_reverse_proxy(server_name_string, app.reverse_proxy.as_ref().unwrap())?; - - backend_builder - .app_name(server_name_string) - .server_name(server_name_string) - .reverse_proxy(reverse_proxy); - - // TLS settings and build backend instance - let backend = if app.tls.is_none() { - ensure!(globals.proxy_config.http_port.is_some(), "Required HTTP port"); - backend_builder.build()? - } else { - let tls = app.tls.as_ref().unwrap(); - ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some()); - - let https_redirection = if tls.https_redirection.is_none() { - Some(true) // Default true - } else { - ensure!(globals.proxy_config.https_port.is_some()); // only when both https ports are configured. - tls.https_redirection - }; - - backend_builder - .tls_cert_path(&tls.tls_cert_path) - .tls_cert_key_path(&tls.tls_cert_key_path) - .https_redirection(https_redirection) - .client_ca_cert_path(&tls.client_ca_cert_path) - .build()? - }; - - globals - .backends - .apps - .insert(server_name_string.to_server_name_vec(), backend); + let server_name_string = app.server_name.as_ref().ok_or(anyhow!("No server name"))?; + let backend = app.try_into()?; + backends.apps.insert(server_name_string.to_server_name_vec(), backend); info!("Registering application: {} ({})", app_name, server_name_string); } // default backend application for plaintext http requests if let Some(d) = config.default_app { - let d_sn: Vec<&str> = globals - .backends + let d_sn: Vec<&str> = backends .apps .iter() .filter(|(_k, v)| v.app_name == d) @@ -150,86 +93,17 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro "Serving plaintext http for requests to unconfigured server_name by app {} (server_name: {}).", d, d_sn[0] ); - globals.backends.default_server_name_bytes = Some(d_sn[0].to_server_name_vec()); + backends.default_server_name_bytes = Some(d_sn[0].to_server_name_vec()); } } - // experimental - if let Some(exp) = config.experimental { - #[cfg(feature = "http3")] - { - if let Some(h3option) = exp.h3 { - globals.proxy_config.http3 = true; - info!("Experimental HTTP/3.0 is enabled. Note it is still very unstable."); - if let Some(x) = h3option.alt_svc_max_age { - globals.proxy_config.h3_alt_svc_max_age = x; - } - if let Some(x) = h3option.request_max_body_size { - globals.proxy_config.h3_request_max_body_size = x; - } - if let Some(x) = h3option.max_concurrent_connections { - globals.proxy_config.h3_max_concurrent_connections = x; - } - if let Some(x) = h3option.max_concurrent_bidistream { - globals.proxy_config.h3_max_concurrent_bidistream = x.into(); - } - if let Some(x) = h3option.max_concurrent_unistream { - globals.proxy_config.h3_max_concurrent_unistream = x.into(); - } - if let Some(x) = h3option.max_idle_timeout { - if x == 0u64 { - globals.proxy_config.h3_max_idle_timeout = None; - } else { - globals.proxy_config.h3_max_idle_timeout = - Some(quinn::IdleTimeout::try_from(tokio::time::Duration::from_secs(x)).unwrap()) - } - } - } - } + /////////////////////////////////// + let globals = Globals { + proxy_config, + backends, + request_count: Default::default(), + runtime_handle, + }; - if let Some(b) = exp.ignore_sni_consistency { - globals.proxy_config.sni_consistency = !b; - if b { - info!("Ignore consistency between TLS SNI and Host header (or Request line). Note it violates RFC."); - } - } - } - - Ok(()) -} - -fn get_reverse_proxy( - server_name_string: &str, - rp_settings: &[ReverseProxyOption], -) -> std::result::Result { - let mut upstream: HashMap = HashMap::default(); - - rp_settings.iter().for_each(|rpo| { - let upstream_vec: Vec = rpo.upstream.iter().map(|x| x.try_into().unwrap()).collect(); - // let upstream_iter = rpo.upstream.iter().map(|x| x.to_upstream().unwrap()); - // let lb_upstream_num = vec_upstream.len(); - let elem = UpstreamGroupBuilder::default() - .upstream(&upstream_vec) - .path(&rpo.path) - .replace_path(&rpo.replace_path) - .lb(&rpo.load_balance, &upstream_vec, server_name_string, &rpo.path) - .opts(&rpo.upstream_options) - .build() - .unwrap(); - - upstream.insert(elem.path.clone(), elem); - }); - ensure!( - rp_settings.iter().filter(|rpo| rpo.path.is_none()).count() < 2, - "Multiple default reverse proxy setting" - ); - ensure!( - upstream - .iter() - .all(|(_, elem)| !(elem.opts.contains(&UpstreamOption::ConvertHttpsTo11) - && elem.opts.contains(&UpstreamOption::ConvertHttpsTo2))), - "either one of force_http11 or force_http2 can be enabled" - ); - - Ok(ReverseProxy { upstream }) + Ok(globals) } diff --git a/src/config/toml.rs b/src/config/toml.rs index 29e76cc..b883f6a 100644 --- a/src/config/toml.rs +++ b/src/config/toml.rs @@ -1,7 +1,13 @@ -use crate::{backend::Upstream, error::*}; +use crate::{ + backend::{Backend, BackendBuilder, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption}, + constants::*, + error::*, + globals::ProxyConfig, + utils::PathNameBytesExp, +}; use rustc_hash::FxHashMap as HashMap; use serde::Deserialize; -use std::fs; +use std::{fs, net::SocketAddr}; #[derive(Deserialize, Debug, Default)] pub struct ConfigToml { @@ -66,20 +72,93 @@ pub struct UpstreamParams { pub tls: Option, } -impl TryInto for &UpstreamParams { - type Error = RpxyError; +impl TryInto for &ConfigToml { + type Error = anyhow::Error; - fn try_into(self) -> std::result::Result { - let mut scheme = "http"; - if let Some(t) = self.tls { - if t { - scheme = "https"; + fn try_into(self) -> std::result::Result { + let mut proxy_config = ProxyConfig { + // listen port and socket + http_port: self.listen_port, + https_port: self.listen_port_tls, + ..Default::default() + }; + ensure!( + proxy_config.http_port.is_some() || proxy_config.https_port.is_some(), + anyhow!("Either/Both of http_port or https_port must be specified") + ); + if proxy_config.http_port.is_some() && proxy_config.https_port.is_some() { + ensure!( + proxy_config.http_port.unwrap() != proxy_config.https_port.unwrap(), + anyhow!("http_port and https_port must be different") + ); + } + + // NOTE: when [::]:xx is bound, both v4 and v6 listeners are enabled. + let listen_addresses: Vec<&str> = if let Some(true) = self.listen_ipv6 { + LISTEN_ADDRESSES_V6.to_vec() + } else { + LISTEN_ADDRESSES_V4.to_vec() + }; + proxy_config.listen_sockets = listen_addresses + .iter() + .flat_map(|addr| { + let mut v: Vec = vec![]; + if let Some(port) = proxy_config.http_port { + v.push(format!("{addr}:{port}").parse().unwrap()); + } + if let Some(port) = proxy_config.https_port { + v.push(format!("{addr}:{port}").parse().unwrap()); + } + v + }) + .collect(); + + // max values + if let Some(c) = self.max_clients { + proxy_config.max_clients = c as usize; + } + if let Some(c) = self.max_concurrent_streams { + proxy_config.max_concurrent_streams = c; + } + + // experimental + if let Some(exp) = &self.experimental { + #[cfg(feature = "http3")] + { + if let Some(h3option) = &exp.h3 { + proxy_config.http3 = true; + if let Some(x) = h3option.alt_svc_max_age { + proxy_config.h3_alt_svc_max_age = x; + } + if let Some(x) = h3option.request_max_body_size { + proxy_config.h3_request_max_body_size = x; + } + if let Some(x) = h3option.max_concurrent_connections { + proxy_config.h3_max_concurrent_connections = x; + } + if let Some(x) = h3option.max_concurrent_bidistream { + proxy_config.h3_max_concurrent_bidistream = x.into(); + } + if let Some(x) = h3option.max_concurrent_unistream { + proxy_config.h3_max_concurrent_unistream = x.into(); + } + if let Some(x) = h3option.max_idle_timeout { + if x == 0u64 { + proxy_config.h3_max_idle_timeout = None; + } else { + proxy_config.h3_max_idle_timeout = + Some(quinn::IdleTimeout::try_from(tokio::time::Duration::from_secs(x)).unwrap()) + } + } + } + } + + if let Some(ignore) = exp.ignore_sni_consistency { + proxy_config.sni_consistency = !ignore; } } - let location = format!("{}://{}", scheme, self.location); - Ok(Upstream { - uri: location.parse::().map_err(|e| anyhow!("{}", e))?, - }) + + Ok(proxy_config) } } @@ -90,3 +169,98 @@ impl ConfigToml { toml::from_str(&config_str).map_err(RpxyError::TomlDe) } } + +impl TryInto for &Application { + type Error = anyhow::Error; + + fn try_into(self) -> std::result::Result { + let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; + + // backend builder + let mut backend_builder = BackendBuilder::default(); + // reverse proxy settings + let reverse_proxy = self.try_into()?; + + backend_builder + .app_name(server_name_string) + .server_name(server_name_string) + .reverse_proxy(reverse_proxy); + + // TLS settings and build backend instance + let backend = if self.tls.is_none() { + backend_builder.build()? + } else { + let tls = self.tls.as_ref().unwrap(); + ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some()); + + let https_redirection = if tls.https_redirection.is_none() { + Some(true) // Default true + } else { + tls.https_redirection + }; + + backend_builder + .tls_cert_path(&tls.tls_cert_path) + .tls_cert_key_path(&tls.tls_cert_key_path) + .https_redirection(https_redirection) + .client_ca_cert_path(&tls.client_ca_cert_path) + .build()? + }; + Ok(backend) + } +} + +impl TryInto for &Application { + type Error = anyhow::Error; + + fn try_into(self) -> std::result::Result { + let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; + let rp_settings = self.reverse_proxy.as_ref().ok_or(anyhow!("Missing reverse_proxy"))?; + + let mut upstream: HashMap = HashMap::default(); + + rp_settings.iter().for_each(|rpo| { + let upstream_vec: Vec = rpo.upstream.iter().map(|x| x.try_into().unwrap()).collect(); + // let upstream_iter = rpo.upstream.iter().map(|x| x.to_upstream().unwrap()); + // let lb_upstream_num = vec_upstream.len(); + let elem = UpstreamGroupBuilder::default() + .upstream(&upstream_vec) + .path(&rpo.path) + .replace_path(&rpo.replace_path) + .lb(&rpo.load_balance, &upstream_vec, server_name_string, &rpo.path) + .opts(&rpo.upstream_options) + .build() + .unwrap(); + + upstream.insert(elem.path.clone(), elem); + }); + ensure!( + rp_settings.iter().filter(|rpo| rpo.path.is_none()).count() < 2, + "Multiple default reverse proxy setting" + ); + ensure!( + upstream + .iter() + .all(|(_, elem)| !(elem.opts.contains(&UpstreamOption::ConvertHttpsTo11) + && elem.opts.contains(&UpstreamOption::ConvertHttpsTo2))), + "either one of force_http11 or force_http2 can be enabled" + ); + + Ok(ReverseProxy { upstream }) + } +} + +impl TryInto for &UpstreamParams { + type Error = RpxyError; + + fn try_into(self) -> std::result::Result { + let scheme = match self.tls { + Some(true) => "https", + _ => "http", + }; + let location = format!("{}://{}", scheme, self.location); + Ok(Upstream { + uri: location.parse::().map_err(|e| anyhow!("{}", e))?, + }) + } +} diff --git a/src/globals.rs b/src/globals.rs index b5c4a46..cd47611 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -10,7 +10,7 @@ use tokio::time::Duration; /// But note that in Globals, we do not have Mutex and RwLock. It is indeed, the context shared among async tasks. pub struct Globals { /// Configuration parameters for proxy transport and request handlers - pub proxy_config: ProxyConfig, + pub proxy_config: ProxyConfig, // TODO: proxy configはarcに包んでこいつだけ使いまわせばいいように変えていく。backendsも？ /// Shared context - Backend application objects to which http request handler forward incoming requests pub backends: Backends, diff --git a/src/main.rs b/src/main.rs index 2b5e1fa..77b5d1b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,8 +16,7 @@ mod proxy; mod utils; use crate::{ - backend::Backends, config::parse_opts, error::*, globals::*, handler::HttpMessageHandlerBuilder, log::*, - proxy::ProxyBuilder, + config::build_globals, error::*, globals::*, handler::HttpMessageHandlerBuilder, log::*, proxy::ProxyBuilder, }; use futures::future::select_all; use hyper::Client; @@ -33,23 +32,17 @@ fn main() { let runtime = runtime_builder.build().unwrap(); runtime.block_on(async { - let mut globals = Globals { - // TODO: proxy configはarcに包んでこいつだけ使いまわせばいいように変えていく。backendsも？ - proxy_config: ProxyConfig::default(), - backends: Backends::default(), - - request_count: Default::default(), - runtime_handle: runtime.handle().clone(), - }; - - if let Err(e) = parse_opts(&mut globals) { - error!("Invalid configuration: {}", e); - std::process::exit(1); + let globals = match build_globals(runtime.handle().clone()) { + Ok(g) => g, + Err(e) => { + error!("Invalid configuration: {}", e); + std::process::exit(1); + } }; entrypoint(Arc::new(globals)).await.unwrap() }); - warn!("Exit the program"); + warn!("rpxy exited!"); } // entrypoint creates and spawns tasks of proxy services From 1f98b69c7ed2987f38d7e876582deb48e65c9424 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 10 Jul 2023 18:35:02 +0900 Subject: [PATCH 07/43] refactor --- Cargo.toml | 3 +-- quinn | 2 +- src/globals.rs | 2 +- src/proxy/proxy_tls.rs | 23 +++++++++-------------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c73533e..5defb89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ clap = { version = "4.3.11", features = ["std", "cargo", "wrap_help"] } rand = "0.8.5" toml = { version = "0.7.6", default-features = false, features = ["parse"] } rustc-hash = "1.1.0" -serde = { version = "1.0.167", default-features = false, features = ["derive"] } +serde = { version = "1.0.171", default-features = false, features = ["derive"] } bytes = "1.4.0" thiserror = "1.0.43" x509-parser = "0.15.0" @@ -31,7 +31,6 @@ futures = { version = "0.3.28", features = ["alloc", "async-await"] } tokio = { version = "1.29.1", default-features = false, features = [ "net", "rt-multi-thread", - "parking_lot", "time", "sync", "macros", diff --git a/quinn b/quinn index b30711f..e652b6d 160000 --- a/quinn +++ b/quinn @@ -1 +1 @@ -Subproject commit b30711f5595983989b60bbbad0ac3f067be7a596 +Subproject commit e652b6d999f053ffe21eeea247854882ae480281 diff --git a/src/globals.rs b/src/globals.rs index cd47611..64f9d8d 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -12,7 +12,7 @@ pub struct Globals { /// Configuration parameters for proxy transport and request handlers pub proxy_config: ProxyConfig, // TODO: proxy configはarcに包んでこいつだけ使いまわせばいいように変えていく。backendsも？ - /// Shared context - Backend application objects to which http request handler forward incoming requests + /// Backend application objects to which http request handler forward incoming requests pub backends: Backends, /// Shared context - Counter for serving requests diff --git a/src/proxy/proxy_tls.rs b/src/proxy/proxy_tls.rs index aa9c69f..317be7c 100644 --- a/src/proxy/proxy_tls.rs +++ b/src/proxy/proxy_tls.rs @@ -146,29 +146,24 @@ where continue; } let mut conn: quinn::Connecting = new_conn.unwrap(); - let hsd = match conn.handshake_data().await { - Ok(h) => h, - Err(_) => continue + let Ok(hsd) = conn.handshake_data().await else { + continue }; - let hsd_downcast = match hsd.downcast::() { - Ok(d) => d, - Err(_) => continue + let Ok(hsd_downcast) = hsd.downcast::() else { + continue }; - let new_server_name = match hsd_downcast.server_name { - Some(sn) => sn.to_server_name_vec(), - None => { - warn!("HTTP/3 no SNI is given"); - continue; - } + let Some(new_server_name) = hsd_downcast.server_name else { + warn!("HTTP/3 no SNI is given"); + continue; }; debug!( "HTTP/3 connection incoming (SNI {:?})", - new_server_name.0 + new_server_name ); // TODO: server_nameをここで出してどんどん深く投げていくのは効率が悪い。connecting -> connectionsの後でいいのでは？ // TODO: 通常のTLSと同じenumか何かにまとめたい - let fut = self.clone().connection_serve_h3(conn, new_server_name); + let fut = self.clone().connection_serve_h3(conn, new_server_name.to_server_name_vec()); self.globals.runtime_handle.spawn(async move { // Timeout is based on underlying quic if let Err(e) = fut.await { From 6b8b7784f87671dd2a4878680dcd27d399a45446 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 10 Jul 2023 22:57:31 +0900 Subject: [PATCH 08/43] use hot_reload to update certificates --- Cargo.toml | 2 + src/backend/mod.rs | 278 +----------------------------------- src/cert_reader.rs | 93 ++++++++++++ src/constants.rs | 1 + src/main.rs | 1 + src/proxy/crypto_service.rs | 253 ++++++++++++++++++++++++++++++++ src/proxy/mod.rs | 2 + src/proxy/proxy_tls.rs | 74 +++++----- src/utils/bytes_name.rs | 7 + 9 files changed, 399 insertions(+), 312 deletions(-) create mode 100644 src/cert_reader.rs create mode 100644 src/proxy/crypto_service.rs diff --git a/Cargo.toml b/Cargo.toml index 5defb89..5f7b0ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,8 @@ tokio = { version = "1.29.1", default-features = false, features = [ "sync", "macros", ] } +async-trait = "0.1.71" +hot_reload = "0.1.2" # reloading certs # http and tls hyper = { version = "0.14.27", default-features = false, features = [ diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 97753f2..c8298c3 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -13,26 +13,10 @@ pub use self::{ upstream::{ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder}, upstream_opts::UpstreamOption, }; -use crate::{ - log::*, - utils::{BytesName, PathNameBytesExp, ServerNameBytesExp}, -}; +use crate::utils::{BytesName, PathNameBytesExp, ServerNameBytesExp}; use derive_builder::Builder; -use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; -use rustls::{OwnedTrustAnchor, RootCertStore}; -use std::{ - borrow::Cow, - fs::File, - io::{self, BufReader, Cursor, Read}, - path::PathBuf, - sync::Arc, -}; -use tokio_rustls::rustls::{ - server::ResolvesServerCertUsingSni, - sign::{any_supported_type, CertifiedKey}, - Certificate, PrivateKey, ServerConfig, -}; -use x509_parser::prelude::*; +use rustc_hash::FxHashMap as HashMap; +use std::{borrow::Cow, path::PathBuf}; /// Struct serving information to route incoming connections, like server name to be handled and tls certs/keys settings. #[derive(Builder)] @@ -79,265 +63,9 @@ fn opt_string_to_opt_pathbuf(input: &Option) -> Option { input.to_owned().as_ref().map(PathBuf::from) } -impl Backend { - pub fn read_certs_and_key(&self) -> io::Result { - debug!("Read TLS server certificates and private key"); - let (Some(certs_path), Some(certs_keys_path)) = (self.tls_cert_path.as_ref(), self.tls_cert_key_path.as_ref()) else { - return Err(io::Error::new(io::ErrorKind::Other, "Invalid certs and keys paths")); - }; - let certs: Vec<_> = { - let certs_path_str = certs_path.display().to_string(); - let mut reader = BufReader::new(File::open(certs_path).map_err(|e| { - io::Error::new( - e.kind(), - format!("Unable to load the certificates [{certs_path_str}]: {e}"), - ) - })?); - rustls_pemfile::certs(&mut reader) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the certificates"))? - } - .drain(..) - .map(Certificate) - .collect(); - let certs_keys: Vec<_> = { - let certs_keys_path_str = certs_keys_path.display().to_string(); - let encoded_keys = { - let mut encoded_keys = vec![]; - File::open(certs_keys_path) - .map_err(|e| { - io::Error::new( - e.kind(), - format!("Unable to load the certificate keys [{certs_keys_path_str}]: {e}"), - ) - })? - .read_to_end(&mut encoded_keys)?; - encoded_keys - }; - let mut reader = Cursor::new(encoded_keys); - let pkcs8_keys = rustls_pemfile::pkcs8_private_keys(&mut reader).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidInput, - "Unable to parse the certificates private keys (PKCS8)", - ) - })?; - reader.set_position(0); - let mut rsa_keys = rustls_pemfile::rsa_private_keys(&mut reader)?; - let mut keys = pkcs8_keys; - keys.append(&mut rsa_keys); - if keys.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "No private keys found - Make sure that they are in PKCS#8/PEM format", - )); - } - keys.drain(..).map(PrivateKey).collect() - }; - let signing_key = certs_keys - .iter() - .find_map(|k| { - if let Ok(sk) = any_supported_type(k) { - Some(sk) - } else { - None - } - }) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "Unable to find a valid certificate and key", - ) - })?; - Ok(CertifiedKey::new(certs, signing_key)) - } - - fn read_client_ca_certs(&self) -> io::Result<(Vec, HashSet>)> { - debug!("Read CA certificates for client authentication"); - // Reads client certificate and returns client - let client_ca_cert_path = { - let Some(c) = self.client_ca_cert_path.as_ref() else { - return Err(io::Error::new(io::ErrorKind::Other, "Invalid certs and keys paths")); - }; - c - }; - let certs: Vec<_> = { - let certs_path_str = client_ca_cert_path.display().to_string(); - let mut reader = BufReader::new(File::open(client_ca_cert_path).map_err(|e| { - io::Error::new( - e.kind(), - format!("Unable to load the client certificates [{certs_path_str}]: {e}"), - ) - })?); - rustls_pemfile::certs(&mut reader) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the client certificates"))? - } - .drain(..) - .map(Certificate) - .collect(); - - let owned_trust_anchors: Vec<_> = certs - .iter() - .map(|v| { - // let trust_anchor = tokio_rustls::webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); - let trust_anchor = webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); - rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( - trust_anchor.subject, - trust_anchor.spki, - trust_anchor.name_constraints, - ) - }) - .collect(); - - // TODO: SKID is not used currently - let subject_key_identifiers: HashSet<_> = certs - .iter() - .filter_map(|v| { - // retrieve ca key id (subject key id) - let cert = parse_x509_certificate(&v.0).unwrap().1; - let subject_key_ids = cert - .iter_extensions() - .filter_map(|ext| match ext.parsed_extension() { - ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid), - _ => None, - }) - .collect::>(); - if !subject_key_ids.is_empty() { - Some(subject_key_ids[0].0.to_owned()) - } else { - None - } - }) - .collect(); - - Ok((owned_trust_anchors, subject_key_identifiers)) - } -} - #[derive(Default)] /// HashMap and some meta information for multiple Backend structs. pub struct Backends { pub apps: HashMap, // hyper::uriで抜いたhostで引っ掛ける pub default_server_name_bytes: Option, // for plaintext http } - -pub type SniServerCryptoMap = HashMap>; -pub struct ServerCrypto { - // For Quic/HTTP3, only servers with no client authentication - pub inner_global_no_client_auth: Arc, - // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers - pub inner_local_map: Arc, -} - -impl Backends { - pub async fn generate_server_crypto(&self) -> Result { - let mut resolver_global = ResolvesServerCertUsingSni::new(); - let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default(); - - for (server_name_bytes_exp, backend) in self.apps.iter() { - if backend.tls_cert_key_path.is_some() && backend.tls_cert_path.is_some() { - match backend.read_certs_and_key() { - Ok(certified_key) => { - let mut resolver_local = ResolvesServerCertUsingSni::new(); - let mut client_ca_roots_local = RootCertStore::empty(); - - // add server certificate and key - if let Err(e) = resolver_local.add(backend.server_name.as_str(), certified_key.to_owned()) { - error!( - "{}: Failed to read some certificates and keys {}", - backend.server_name.as_str(), - e - ) - } - - if backend.client_ca_cert_path.is_none() { - // aggregated server config for no client auth server for http3 - if let Err(e) = resolver_global.add(backend.server_name.as_str(), certified_key) { - error!( - "{}: Failed to read some certificates and keys {}", - backend.server_name.as_str(), - e - ) - } - } else { - // add client certificate if specified - match backend.read_client_ca_certs() { - Ok((owned_trust_anchors, _subject_key_ids)) => { - client_ca_roots_local.add_server_trust_anchors(owned_trust_anchors.into_iter()); - } - Err(e) => { - warn!( - "Failed to add client CA certificate for {}: {}", - backend.server_name.as_str(), - e - ); - } - } - } - - let mut server_config_local = if client_ca_roots_local.is_empty() { - // with no client auth, enable http1.1 -- 3 - #[cfg(not(feature = "http3"))] - { - ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_cert_resolver(Arc::new(resolver_local)) - } - #[cfg(feature = "http3")] - { - let mut sc = ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_cert_resolver(Arc::new(resolver_local)); - sc.alpn_protocols = vec![b"h3".to_vec(), b"hq-29".to_vec()]; // TODO: remove hq-29 later? - sc - } - } else { - // with client auth, enable only http1.1 and 2 - // let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots); - let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots_local); - ServerConfig::builder() - .with_safe_defaults() - .with_client_cert_verifier(Arc::new(client_certs_verifier)) - .with_cert_resolver(Arc::new(resolver_local)) - }; - server_config_local.alpn_protocols.push(b"h2".to_vec()); - server_config_local.alpn_protocols.push(b"http/1.1".to_vec()); - - server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local)); - } - Err(e) => { - warn!("Failed to add certificate for {}: {}", backend.server_name.as_str(), e); - } - } - } - } - // debug!("Load certificate chain for {} server_name's", cnt); - - ////////////// - let mut server_crypto_global = ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_cert_resolver(Arc::new(resolver_global)); - - ////////////////////////////// - - #[cfg(feature = "http3")] - { - server_crypto_global.alpn_protocols = vec![ - b"h3".to_vec(), - b"hq-29".to_vec(), // TODO: remove later? - b"h2".to_vec(), - b"http/1.1".to_vec(), - ]; - } - #[cfg(not(feature = "http3"))] - { - server_crypto_global.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; - } - - Ok(ServerCrypto { - inner_global_no_client_auth: Arc::new(server_crypto_global), - inner_local_map: Arc::new(server_crypto_local_map), - }) - } -} diff --git a/src/cert_reader.rs b/src/cert_reader.rs new file mode 100644 index 0000000..a52f2e2 --- /dev/null +++ b/src/cert_reader.rs @@ -0,0 +1,93 @@ +use crate::{log::*, proxy::CertsAndKeys}; +use rustls::{Certificate, PrivateKey}; +use std::{ + fs::File, + io::{self, BufReader, Cursor, Read}, + path::PathBuf, +}; + +/// Read certificates and private keys from file +pub(crate) fn read_certs_and_keys( + cert_path: &PathBuf, + cert_key_path: &PathBuf, + client_ca_cert_path: Option<&PathBuf>, +) -> Result { + debug!("Read TLS server certificates and private key"); + + let certs: Vec<_> = { + let certs_path_str = cert_path.display().to_string(); + let mut reader = BufReader::new(File::open(cert_path).map_err(|e| { + io::Error::new( + e.kind(), + format!("Unable to load the certificates [{certs_path_str}]: {e}"), + ) + })?); + rustls_pemfile::certs(&mut reader) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the certificates"))? + } + .drain(..) + .map(Certificate) + .collect(); + + let cert_keys: Vec<_> = { + let cert_key_path_str = cert_key_path.display().to_string(); + let encoded_keys = { + let mut encoded_keys = vec![]; + File::open(cert_key_path) + .map_err(|e| { + io::Error::new( + e.kind(), + format!("Unable to load the certificate keys [{cert_key_path_str}]: {e}"), + ) + })? + .read_to_end(&mut encoded_keys)?; + encoded_keys + }; + let mut reader = Cursor::new(encoded_keys); + let pkcs8_keys = rustls_pemfile::pkcs8_private_keys(&mut reader).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "Unable to parse the certificates private keys (PKCS8)", + ) + })?; + reader.set_position(0); + let mut rsa_keys = rustls_pemfile::rsa_private_keys(&mut reader)?; + let mut keys = pkcs8_keys; + keys.append(&mut rsa_keys); + if keys.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "No private keys found - Make sure that they are in PKCS#8/PEM format", + )); + } + keys.drain(..).map(PrivateKey).collect() + }; + + let client_ca_certs = if let Some(path) = client_ca_cert_path { + debug!("Read CA certificates for client authentication"); + // Reads client certificate and returns client + let certs: Vec<_> = { + let certs_path_str = path.display().to_string(); + let mut reader = BufReader::new(File::open(path).map_err(|e| { + io::Error::new( + e.kind(), + format!("Unable to load the client certificates [{certs_path_str}]: {e}"), + ) + })?); + rustls_pemfile::certs(&mut reader) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the client certificates"))? + } + .drain(..) + .map(Certificate) + .collect(); + Some(certs) + } else { + None + }; + + Ok(CertsAndKeys { + certs, + cert_keys, + client_ca_certs, + }) +} diff --git a/src/constants.rs b/src/constants.rs index a29be29..2ed14d1 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -8,6 +8,7 @@ pub const TLS_HANDSHAKE_TIMEOUT_SEC: u64 = 15; // default as with firefox browse pub const MAX_CLIENTS: usize = 512; pub const MAX_CONCURRENT_STREAMS: u32 = 64; pub const CERTS_WATCH_DELAY_SECS: u32 = 60; +pub const LOAD_CERTS_ONLY_WHEN_UPDATED: bool = true; // #[cfg(feature = "http3")] // pub const H3_RESPONSE_BUF_SIZE: usize = 65_536; // 64KB diff --git a/src/main.rs b/src/main.rs index 77b5d1b..ea47e57 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ use tikv_jemallocator::Jemalloc; static GLOBAL: Jemalloc = Jemalloc; mod backend; +mod cert_reader; mod config; mod constants; mod error; diff --git a/src/proxy/crypto_service.rs b/src/proxy/crypto_service.rs new file mode 100644 index 0000000..728a531 --- /dev/null +++ b/src/proxy/crypto_service.rs @@ -0,0 +1,253 @@ +use crate::{ + cert_reader::read_certs_and_keys, // TODO: Trait defining read_certs_and_keys and add struct implementing the trait to backend when build backend + globals::Globals, + log::*, + utils::ServerNameBytesExp, +}; +use async_trait::async_trait; +use hot_reload::*; +use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; +use rustls::{ + server::ResolvesServerCertUsingSni, + sign::{any_supported_type, CertifiedKey}, + Certificate, OwnedTrustAnchor, PrivateKey, RootCertStore, ServerConfig, +}; +use std::{io, sync::Arc}; +use x509_parser::prelude::*; + +#[derive(Clone)] +/// Reloader service for certificates and keys for TLS +pub struct CryptoReloader { + globals: Arc, +} + +/// Certificates and private keys in rustls loaded from files +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct CertsAndKeys { + pub certs: Vec, + pub cert_keys: Vec, + pub client_ca_certs: Option>, +} + +pub type SniServerCryptoMap = HashMap>; +pub struct ServerCrypto { + // For Quic/HTTP3, only servers with no client authentication + pub inner_global_no_client_auth: Arc, + // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers + pub inner_local_map: Arc, +} + +/// Reloader target for the certificate reloader service +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct ServerCryptoBase { + inner: HashMap, +} + +#[async_trait] +impl Reload for CryptoReloader { + type Source = Arc; + async fn new(source: &Self::Source) -> Result> { + Ok(Self { + globals: source.clone(), + }) + } + + async fn reload(&self) -> Result, ReloaderError> { + let mut certs_and_keys_map = ServerCryptoBase::default(); + + for (server_name_bytes_exp, backend) in self.globals.backends.apps.iter() { + if backend.tls_cert_key_path.is_some() && backend.tls_cert_path.is_some() { + let tls_cert_key_path = backend.tls_cert_key_path.as_ref().unwrap(); + let tls_cert_path = backend.tls_cert_path.as_ref().unwrap(); + let tls_client_ca_cert_path = backend.client_ca_cert_path.as_ref(); + let certs_and_keys = read_certs_and_keys(tls_cert_path, tls_cert_key_path, tls_client_ca_cert_path) + .map_err(|_e| ReloaderError::::Reload("Failed to reload cert, key or ca cert"))?; + + certs_and_keys_map + .inner + .insert(server_name_bytes_exp.to_owned(), certs_and_keys); + } + } + + Ok(Some(certs_and_keys_map)) + } +} + +impl CertsAndKeys { + fn parse_server_certs_and_keys(&self) -> Result { + // for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { + let signing_key = self + .cert_keys + .iter() + .find_map(|k| { + if let Ok(sk) = any_supported_type(k) { + Some(sk) + } else { + None + } + }) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "Unable to find a valid certificate and key", + ) + })?; + Ok(CertifiedKey::new(self.certs.clone(), signing_key)) + } + + pub fn parse_client_ca_certs(&self) -> Result<(Vec, HashSet>), anyhow::Error> { + let certs = self.client_ca_certs.as_ref().ok_or(anyhow::anyhow!("No client cert"))?; + + let owned_trust_anchors: Vec<_> = certs + .iter() + .map(|v| { + // let trust_anchor = tokio_rustls::webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); + let trust_anchor = webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); + rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( + trust_anchor.subject, + trust_anchor.spki, + trust_anchor.name_constraints, + ) + }) + .collect(); + + // TODO: SKID is not used currently + let subject_key_identifiers: HashSet<_> = certs + .iter() + .filter_map(|v| { + // retrieve ca key id (subject key id) + let cert = parse_x509_certificate(&v.0).unwrap().1; + let subject_key_ids = cert + .iter_extensions() + .filter_map(|ext| match ext.parsed_extension() { + ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid), + _ => None, + }) + .collect::>(); + if !subject_key_ids.is_empty() { + Some(subject_key_ids[0].0.to_owned()) + } else { + None + } + }) + .collect(); + + Ok((owned_trust_anchors, subject_key_identifiers)) + } +} + +impl TryInto> for &ServerCryptoBase { + type Error = anyhow::Error; + + fn try_into(self) -> Result, Self::Error> { + let mut resolver_global = ResolvesServerCertUsingSni::new(); + let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default(); + + for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { + let server_name: String = server_name_bytes_exp.try_into()?; + + // Parse server certificates and private keys + let Ok(certified_key): Result = certs_and_keys.parse_server_certs_and_keys() else { + warn!("Failed to add certificate for {}", server_name); + continue; + }; + + let mut resolver_local = ResolvesServerCertUsingSni::new(); + let mut client_ca_roots_local = RootCertStore::empty(); + + // add server certificate and key + if let Err(e) = resolver_local.add(server_name.as_str(), certified_key.to_owned()) { + error!( + "{}: Failed to read some certificates and keys {}", + server_name.as_str(), + e + ) + } + + // add client certificate if specified + if certs_and_keys.client_ca_certs.is_none() { + // aggregated server config for no client auth server for http3 + if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) { + error!( + "{}: Failed to read some certificates and keys {}", + server_name.as_str(), + e + ) + } + } else { + // add client certificate if specified + match certs_and_keys.parse_client_ca_certs() { + Ok((owned_trust_anchors, _subject_key_ids)) => { + client_ca_roots_local.add_server_trust_anchors(owned_trust_anchors.into_iter()); + } + Err(e) => { + warn!( + "Failed to add client CA certificate for {}: {}", + server_name.as_str(), + e + ); + } + } + } + + let mut server_config_local = if client_ca_roots_local.is_empty() { + // with no client auth, enable http1.1 -- 3 + #[cfg(not(feature = "http3"))] + { + ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver_local)) + } + #[cfg(feature = "http3")] + { + let mut sc = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver_local)); + sc.alpn_protocols = vec![b"h3".to_vec(), b"hq-29".to_vec()]; // TODO: remove hq-29 later? + sc + } + } else { + // with client auth, enable only http1.1 and 2 + // let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots); + let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots_local); + ServerConfig::builder() + .with_safe_defaults() + .with_client_cert_verifier(Arc::new(client_certs_verifier)) + .with_cert_resolver(Arc::new(resolver_local)) + }; + server_config_local.alpn_protocols.push(b"h2".to_vec()); + server_config_local.alpn_protocols.push(b"http/1.1".to_vec()); + + server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local)); + } + + ////////////// + let mut server_crypto_global = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver_global)); + + ////////////////////////////// + + #[cfg(feature = "http3")] + { + server_crypto_global.alpn_protocols = vec![ + b"h3".to_vec(), + b"hq-29".to_vec(), // TODO: remove later? + b"h2".to_vec(), + b"http/1.1".to_vec(), + ]; + } + #[cfg(not(feature = "http3"))] + { + server_crypto_global.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + } + + Ok(Arc::new(ServerCrypto { + inner_global_no_client_auth: Arc::new(server_crypto_global), + inner_local_map: Arc::new(server_crypto_local_map), + })) + } +} diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index 04413f5..d8fdc83 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -1,7 +1,9 @@ +mod crypto_service; mod proxy_client_cert; #[cfg(feature = "http3")] mod proxy_h3; mod proxy_main; mod proxy_tls; +pub use crypto_service::CertsAndKeys; pub use proxy_main::{Proxy, ProxyBuilder, ProxyBuilderError}; diff --git a/src/proxy/proxy_tls.rs b/src/proxy/proxy_tls.rs index 317be7c..e01f9d3 100644 --- a/src/proxy/proxy_tls.rs +++ b/src/proxy/proxy_tls.rs @@ -1,11 +1,9 @@ -use super::proxy_main::{LocalExecutor, Proxy}; -use crate::{ - backend::{ServerCrypto, SniServerCryptoMap}, - constants::*, - error::*, - log::*, - utils::BytesName, +use super::{ + crypto_service::{CryptoReloader, ServerCrypto, ServerCryptoBase, SniServerCryptoMap}, + proxy_main::{LocalExecutor, Proxy}, }; +use crate::{constants::*, error::*, log::*, utils::BytesName}; +use hot_reload::{ReloaderReceiver, ReloaderService}; use hyper::{client::connect::Connect, server::conn::Http}; #[cfg(feature = "http3")] use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerConfig, TransportConfig}; @@ -14,34 +12,18 @@ use rustls::ServerConfig; use std::sync::Arc; use tokio::{ net::TcpListener, - sync::watch, - time::{sleep, timeout, Duration}, + time::{timeout, Duration}, }; impl Proxy where T: Connect + Clone + Sync + Send + 'static, { - async fn cert_service(&self, server_crypto_tx: watch::Sender>>) { - info!("Start cert watch service"); - loop { - if let Ok(server_crypto) = self.globals.backends.generate_server_crypto().await { - if let Err(_e) = server_crypto_tx.send(Some(Arc::new(server_crypto))) { - error!("Failed to populate server crypto"); - break; - } - } else { - error!("Failed to update certs"); - } - sleep(Duration::from_secs(CERTS_WATCH_DELAY_SECS.into())).await; - } - } - // TCP Listener Service, i.e., http/2 and http/1.1 async fn listener_service( &self, server: Http, - mut server_crypto_rx: watch::Receiver>>, + mut server_crypto_rx: ReloaderReceiver, ) -> Result<()> { let tcp_listener = TcpListener::bind(&self.listening_on).await?; info!("Start TCP proxy serving with HTTPS request for configured host names"); @@ -105,9 +87,14 @@ where } _ = server_crypto_rx.changed() => { if server_crypto_rx.borrow().is_none() { + error!("Reloader is broken"); break; } - let server_crypto = server_crypto_rx.borrow().clone().unwrap(); + let cert_keys_map = server_crypto_rx.borrow().clone().unwrap(); + let Some(server_crypto): Option> = (&cert_keys_map).try_into().ok() else { + error!("Failed to update server crypto"); + break; + }; server_crypto_map = Some(server_crypto.inner_local_map.clone()); } else => break @@ -117,7 +104,7 @@ where } #[cfg(feature = "http3")] - async fn listener_service_h3(&self, mut server_crypto_rx: watch::Receiver>>) -> Result<()> { + async fn listener_service_h3(&self, mut server_crypto_rx: ReloaderReceiver) -> Result<()> { info!("Start UDP proxy serving with HTTP/3 request for configured host names"); // first set as null config server let rustls_server_config = ServerConfig::builder() @@ -173,12 +160,18 @@ where } _ = server_crypto_rx.changed() => { if server_crypto_rx.borrow().is_none() { + error!("Reloader is broken"); break; } - server_crypto = server_crypto_rx.borrow().clone(); - if server_crypto.is_some(){ - endpoint.set_server_config(Some(QuicServerConfig::with_crypto(server_crypto.clone().unwrap().inner_global_no_client_auth.clone()))); - } + let cert_keys_map = server_crypto_rx.borrow().clone().unwrap(); + + server_crypto = (&cert_keys_map).try_into().ok(); + let Some(inner) = server_crypto.clone() else { + error!("Failed to update server crypto for h3"); + break; + }; + endpoint.set_server_config(Some(QuicServerConfig::with_crypto(inner.clone().inner_global_no_client_auth.clone()))); + } else => break } @@ -188,7 +181,14 @@ where } pub async fn start_with_tls(self, server: Http) -> Result<()> { - let (tx, rx) = watch::channel::>>(None); + let (cert_reloader_service, cert_reloader_rx) = ReloaderService::::new( + &self.globals.clone(), + CERTS_WATCH_DELAY_SECS, + !LOAD_CERTS_ONLY_WHEN_UPDATED, + ) + .await + .map_err(|e| anyhow::anyhow!(e))?; + #[cfg(not(feature = "http3"))] { tokio::select! { @@ -209,13 +209,13 @@ where { if self.globals.proxy_config.http3 { tokio::select! { - _= self.cert_service(tx) => { + _= cert_reloader_service.start() => { error!("Cert service for TLS exited"); }, - _ = self.listener_service(server, rx.clone()) => { + _ = self.listener_service(server, cert_reloader_rx.clone()) => { error!("TCP proxy service for TLS exited"); }, - _= self.listener_service_h3(rx) => { + _= self.listener_service_h3(cert_reloader_rx) => { error!("UDP proxy service for QUIC exited"); }, else => { @@ -226,10 +226,10 @@ where Ok(()) } else { tokio::select! { - _= self.cert_service(tx) => { + _= cert_reloader_service.start() => { error!("Cert service for TLS exited"); }, - _ = self.listener_service(server, rx) => { + _ = self.listener_service(server, cert_reloader_rx) => { error!("TCP proxy service for TLS exited"); }, else => { diff --git a/src/utils/bytes_name.rs b/src/utils/bytes_name.rs index 16ec7ab..a093c41 100644 --- a/src/utils/bytes_name.rs +++ b/src/utils/bytes_name.rs @@ -7,6 +7,13 @@ impl From<&[u8]> for ServerNameBytesExp { Self(b.to_ascii_lowercase()) } } +impl TryInto for &ServerNameBytesExp { + type Error = anyhow::Error; + fn try_into(self) -> Result { + let s = std::str::from_utf8(&self.0)?; + Ok(s.to_string()) + } +} /// Path name, like "/path/ok", represented in bytes-based struct /// for searching hashmap or key list by exact or longest-prefix matching From 145a1dc1ee5b6f759acc81c8637dfab261feabc4 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 10 Jul 2023 23:01:34 +0900 Subject: [PATCH 09/43] refactor --- src/proxy/proxy_client_cert.rs | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/proxy/proxy_client_cert.rs b/src/proxy/proxy_client_cert.rs index adac4b7..dfba4ce 100644 --- a/src/proxy/proxy_client_cert.rs +++ b/src/proxy/proxy_client_cert.rs @@ -10,26 +10,18 @@ pub(super) fn check_client_authentication( client_certs: Option<&[Certificate]>, client_ca_keyids_set_for_sni: Option<&HashSet>>, ) -> std::result::Result<(), ClientCertsError> { - let client_ca_keyids_set = match client_ca_keyids_set_for_sni { - Some(c) => c, - None => { - // No client cert settings for given server name - return Ok(()); - } + let Some(client_ca_keyids_set) = client_ca_keyids_set_for_sni else { + // No client cert settings for given server name + return Ok(()); }; - let client_certs = match client_certs { - Some(c) => { - debug!("Incoming TLS client is (temporarily) authenticated via client cert"); - c - } - None => { - error!("Client certificate is needed for given server name"); - return Err(ClientCertsError::ClientCertRequired( - "Client certificate is needed for given server name".to_string(), - )); - } + let Some(client_certs) = client_certs else { + error!("Client certificate is needed for given server name"); + return Err(ClientCertsError::ClientCertRequired( + "Client certificate is needed for given server name".to_string(), + )); }; + debug!("Incoming TLS client is (temporarily) authenticated via client cert"); // Check client certificate key ids let mut client_certs_parsed_iter = client_certs.iter().filter_map(|d| parse_x509_certificate(&d.0).ok()); From f2327778f6c7b2db4056e19ccfd38e7e6f9fcfb0 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Wed, 12 Jul 2023 19:11:30 +0900 Subject: [PATCH 10/43] refactor --- src/{cert_reader.rs => cert_file_reader.rs} | 2 +- src/certs.rs | 17 +++++++++++++++++ src/main.rs | 3 ++- src/proxy/crypto_service.rs | 13 +++---------- src/proxy/mod.rs | 1 - 5 files changed, 23 insertions(+), 13 deletions(-) rename src/{cert_reader.rs => cert_file_reader.rs} (98%) create mode 100644 src/certs.rs diff --git a/src/cert_reader.rs b/src/cert_file_reader.rs similarity index 98% rename from src/cert_reader.rs rename to src/cert_file_reader.rs index a52f2e2..53a736e 100644 --- a/src/cert_reader.rs +++ b/src/cert_file_reader.rs @@ -1,4 +1,4 @@ -use crate::{log::*, proxy::CertsAndKeys}; +use crate::{certs::CertsAndKeys, log::*}; use rustls::{Certificate, PrivateKey}; use std::{ fs::File, diff --git a/src/certs.rs b/src/certs.rs new file mode 100644 index 0000000..3008900 --- /dev/null +++ b/src/certs.rs @@ -0,0 +1,17 @@ +use async_trait::async_trait; +use rustls::{Certificate, PrivateKey}; + +/// Certificates and private keys in rustls loaded from files +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct CertsAndKeys { + pub certs: Vec, + pub cert_keys: Vec, + pub client_ca_certs: Option>, +} + +#[async_trait] +// Trait to read certs and keys anywhere from KVS, file, sqlite, etc. +pub trait ReadCerts { + type Error; + async fn read_crypto_source(&self) -> Result; +} diff --git a/src/main.rs b/src/main.rs index ea47e57..526c290 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,8 @@ use tikv_jemallocator::Jemalloc; static GLOBAL: Jemalloc = Jemalloc; mod backend; -mod cert_reader; +mod cert_file_reader; +mod certs; mod config; mod constants; mod error; diff --git a/src/proxy/crypto_service.rs b/src/proxy/crypto_service.rs index 728a531..629119b 100644 --- a/src/proxy/crypto_service.rs +++ b/src/proxy/crypto_service.rs @@ -1,5 +1,6 @@ use crate::{ - cert_reader::read_certs_and_keys, // TODO: Trait defining read_certs_and_keys and add struct implementing the trait to backend when build backend + cert_file_reader::read_certs_and_keys, // TODO: Trait defining read_certs_and_keys and add struct implementing the trait to backend when build backend + certs::CertsAndKeys, globals::Globals, log::*, utils::ServerNameBytesExp, @@ -10,7 +11,7 @@ use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; use rustls::{ server::ResolvesServerCertUsingSni, sign::{any_supported_type, CertifiedKey}, - Certificate, OwnedTrustAnchor, PrivateKey, RootCertStore, ServerConfig, + OwnedTrustAnchor, RootCertStore, ServerConfig, }; use std::{io, sync::Arc}; use x509_parser::prelude::*; @@ -21,14 +22,6 @@ pub struct CryptoReloader { globals: Arc, } -/// Certificates and private keys in rustls loaded from files -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct CertsAndKeys { - pub certs: Vec, - pub cert_keys: Vec, - pub client_ca_certs: Option>, -} - pub type SniServerCryptoMap = HashMap>; pub struct ServerCrypto { // For Quic/HTTP3, only servers with no client authentication diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index d8fdc83..73a4002 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -5,5 +5,4 @@ mod proxy_h3; mod proxy_main; mod proxy_tls; -pub use crypto_service::CertsAndKeys; pub use proxy_main::{Proxy, ProxyBuilder, ProxyBuilderError}; From db329e38b4575f29213a9fdb946925fa8dd0b1e8 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Wed, 12 Jul 2023 19:21:43 +0900 Subject: [PATCH 11/43] refactor: define crypto source trait --- src/cert_file_reader.rs | 26 +++++++++++++++++++++++++- src/certs.rs | 4 ++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/cert_file_reader.rs b/src/cert_file_reader.rs index 53a736e..2a800b3 100644 --- a/src/cert_file_reader.rs +++ b/src/cert_file_reader.rs @@ -1,4 +1,8 @@ -use crate::{certs::CertsAndKeys, log::*}; +use crate::{ + certs::{CertsAndKeys, CryptoSource}, + log::*, +}; +use async_trait::async_trait; use rustls::{Certificate, PrivateKey}; use std::{ fs::File, @@ -6,6 +10,26 @@ use std::{ path::PathBuf, }; +/// Crypto-related file reader implementing certs::CryptoRead trait +pub struct CryptoFileSource { + /// tls settings in file + pub tls_cert_path: PathBuf, + pub tls_cert_key_path: PathBuf, + pub client_ca_cert_path: Option, +} + +#[async_trait] +impl CryptoSource for CryptoFileSource { + type Error = io::Error; + async fn read(&self) -> Result { + read_certs_and_keys( + &self.tls_cert_path, + &self.tls_cert_key_path, + self.client_ca_cert_path.as_ref(), + ) + } +} + /// Read certificates and private keys from file pub(crate) fn read_certs_and_keys( cert_path: &PathBuf, diff --git a/src/certs.rs b/src/certs.rs index 3008900..da51e14 100644 --- a/src/certs.rs +++ b/src/certs.rs @@ -11,7 +11,7 @@ pub struct CertsAndKeys { #[async_trait] // Trait to read certs and keys anywhere from KVS, file, sqlite, etc. -pub trait ReadCerts { +pub trait CryptoSource { type Error; - async fn read_crypto_source(&self) -> Result; + async fn read(&self) -> Result; } From b6073e5d12d299ba358fc245ca876228562c80dd Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Wed, 12 Jul 2023 19:51:48 +0900 Subject: [PATCH 12/43] refactor: implement tests for crypto file source reader --- src/cert_file_reader.rs | 65 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/src/cert_file_reader.rs b/src/cert_file_reader.rs index 2a800b3..c3a3310 100644 --- a/src/cert_file_reader.rs +++ b/src/cert_file_reader.rs @@ -3,6 +3,7 @@ use crate::{ log::*, }; use async_trait::async_trait; +use derive_builder::Builder; use rustls::{Certificate, PrivateKey}; use std::{ fs::File, @@ -10,14 +11,37 @@ use std::{ path::PathBuf, }; +#[derive(Builder, Debug)] /// Crypto-related file reader implementing certs::CryptoRead trait pub struct CryptoFileSource { - /// tls settings in file + #[builder(setter(custom))] + /// Always exist pub tls_cert_path: PathBuf, + + #[builder(setter(custom))] + /// Always exist pub tls_cert_key_path: PathBuf, + + #[builder(setter(custom), default)] + /// This may not exist pub client_ca_cert_path: Option, } +impl CryptoFileSourceBuilder { + pub fn tls_cert_path(&mut self, v: &str) -> &mut Self { + self.tls_cert_path = Some(PathBuf::from(v)); + self + } + pub fn tls_cert_key_path(&mut self, v: &str) -> &mut Self { + self.tls_cert_key_path = Some(PathBuf::from(v)); + self + } + pub fn client_ca_cert_path(&mut self, v: &str) -> &mut Self { + self.client_ca_cert_path = Some(Some(PathBuf::from(v))); + self + } +} + #[async_trait] impl CryptoSource for CryptoFileSource { type Error = io::Error; @@ -115,3 +139,42 @@ pub(crate) fn read_certs_and_keys( client_ca_certs, }) } + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn read_server_crt_key_files() { + let tls_cert_path = "example-certs/server.crt"; + let tls_cert_key_path = "example-certs/server.key"; + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .build(); + assert!(crypto_file_source.is_ok()); + + let crypto_file_source = crypto_file_source.unwrap(); + let crypto_elem = crypto_file_source.read().await; + assert!(crypto_elem.is_ok()); + } + + #[tokio::test] + async fn read_server_crt_key_files_with_client_ca_crt() { + let tls_cert_path = "example-certs/server.crt"; + let tls_cert_key_path = "example-certs/server.key"; + let client_ca_cert_path = "example-certs/client.ca.crt"; + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .client_ca_cert_path(client_ca_cert_path) + .build(); + assert!(crypto_file_source.is_ok()); + + let crypto_file_source = crypto_file_source.unwrap(); + let crypto_elem = crypto_file_source.read().await; + assert!(crypto_elem.is_ok()); + + let crypto_elem = crypto_elem.unwrap(); + assert!(crypto_elem.client_ca_certs.is_some()); + } +} From 6c0fd85ca5099a4e7705c170b080a57f63d2b658 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Wed, 12 Jul 2023 20:31:31 +0900 Subject: [PATCH 13/43] refactor: add cert_reader object in backend --- src/backend/mod.rs | 38 +++++++++++++++++++++++++++++++------ src/cert_file_reader.rs | 2 +- src/config/parse.rs | 9 ++++++--- src/config/toml.rs | 8 ++++++-- src/globals.rs | 8 ++++++-- src/handler/handler_main.rs | 18 ++++++++++-------- src/main.rs | 11 ++++++++--- src/proxy/crypto_service.rs | 16 +++++++++++----- src/proxy/proxy_h3.rs | 5 +++-- src/proxy/proxy_main.rs | 14 +++++++++----- src/proxy/proxy_tls.rs | 7 ++++--- 11 files changed, 96 insertions(+), 40 deletions(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index c8298c3..9bc28e5 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -13,14 +13,20 @@ pub use self::{ upstream::{ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder}, upstream_opts::UpstreamOption, }; -use crate::utils::{BytesName, PathNameBytesExp, ServerNameBytesExp}; +use crate::{ + certs::CryptoSource, + utils::{BytesName, PathNameBytesExp, ServerNameBytesExp}, +}; use derive_builder::Builder; use rustc_hash::FxHashMap as HashMap; use std::{borrow::Cow, path::PathBuf}; /// Struct serving information to route incoming connections, like server name to be handled and tls certs/keys settings. #[derive(Builder)] -pub struct Backend { +pub struct Backend +where + T: CryptoSource, +{ #[builder(setter(into))] /// backend application name, e.g., app1 pub app_name: String, @@ -39,8 +45,14 @@ pub struct Backend { pub https_redirection: Option, #[builder(setter(custom), default)] pub client_ca_cert_path: Option, + + #[builder(default)] + pub crypto_source: Option, } -impl<'a> BackendBuilder { +impl<'a, T> BackendBuilder +where + T: CryptoSource, +{ pub fn server_name(&mut self, server_name: impl Into>) -> &mut Self { self.server_name = Some(server_name.into().to_ascii_lowercase()); self @@ -63,9 +75,23 @@ fn opt_string_to_opt_pathbuf(input: &Option) -> Option { input.to_owned().as_ref().map(PathBuf::from) } -#[derive(Default)] /// HashMap and some meta information for multiple Backend structs. -pub struct Backends { - pub apps: HashMap, // hyper::uriで抜いたhostで引っ掛ける +pub struct Backends +where + T: CryptoSource, +{ + pub apps: HashMap>, // hyper::uriで抜いたhostで引っ掛ける pub default_server_name_bytes: Option, // for plaintext http } + +impl Backends +where + T: CryptoSource, +{ + pub fn new() -> Self { + Backends { + apps: HashMap::>::default(), + default_server_name_bytes: None, + } + } +} diff --git a/src/cert_file_reader.rs b/src/cert_file_reader.rs index c3a3310..dc25b09 100644 --- a/src/cert_file_reader.rs +++ b/src/cert_file_reader.rs @@ -11,7 +11,7 @@ use std::{ path::PathBuf, }; -#[derive(Builder, Debug)] +#[derive(Builder, Debug, Clone)] /// Crypto-related file reader implementing certs::CryptoRead trait pub struct CryptoFileSource { #[builder(setter(custom))] diff --git a/src/config/parse.rs b/src/config/parse.rs index 1593aba..83aa546 100644 --- a/src/config/parse.rs +++ b/src/config/parse.rs @@ -1,9 +1,12 @@ use super::toml::ConfigToml; -use crate::{backend::Backends, error::*, globals::*, log::*, utils::BytesName}; +use crate::{backend::Backends, certs::CryptoSource, error::*, globals::*, log::*, utils::BytesName}; use clap::Arg; use tokio::runtime::Handle; -pub fn build_globals(runtime_handle: Handle) -> std::result::Result { +pub fn build_globals(runtime_handle: Handle) -> std::result::Result, anyhow::Error> +where + T: CryptoSource + Clone, +{ let _ = include_str!("../../Cargo.toml"); let options = clap::command!().arg( Arg::new("config_file") @@ -72,7 +75,7 @@ pub fn build_globals(runtime_handle: Handle) -> std::result::Result for &Application { +impl TryInto> for &Application +where + T: CryptoSource + Clone, +{ type Error = anyhow::Error; - fn try_into(self) -> std::result::Result { + fn try_into(self) -> std::result::Result, Self::Error> { let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; // backend builder diff --git a/src/globals.rs b/src/globals.rs index 64f9d8d..b85733b 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -1,3 +1,4 @@ +use crate::certs::CryptoSource; use crate::{backend::Backends, constants::*}; use std::net::SocketAddr; use std::sync::{ @@ -8,12 +9,15 @@ use tokio::time::Duration; /// Global object containing proxy configurations and shared object like counters. /// But note that in Globals, we do not have Mutex and RwLock. It is indeed, the context shared among async tasks. -pub struct Globals { +pub struct Globals +where + T: CryptoSource, +{ /// Configuration parameters for proxy transport and request handlers pub proxy_config: ProxyConfig, // TODO: proxy configはarcに包んでこいつだけ使いまわせばいいように変えていく。backendsも？ /// Backend application objects to which http request handler forward incoming requests - pub backends: Backends, + pub backends: Backends, /// Shared context - Counter for serving requests pub request_count: RequestCount, diff --git a/src/handler/handler_main.rs b/src/handler/handler_main.rs index d2a47be..2016f2c 100644 --- a/src/handler/handler_main.rs +++ b/src/handler/handler_main.rs @@ -2,6 +2,7 @@ use super::{utils_headers::*, utils_request::*, utils_synth_response::*, HandlerContext}; use crate::{ backend::{Backend, UpstreamGroup}, + certs::CryptoSource, error::*, globals::Globals, log::*, @@ -18,17 +19,19 @@ use std::{env, net::SocketAddr, sync::Arc}; use tokio::{io::copy_bidirectional, time::timeout}; #[derive(Clone, Builder)] -pub struct HttpMessageHandler +pub struct HttpMessageHandler where T: Connect + Clone + Sync + Send + 'static, + U: CryptoSource + Clone, { forwarder: Arc>, - globals: Arc, + globals: Arc>, } -impl HttpMessageHandler +impl HttpMessageHandler where T: Connect + Clone + Sync + Send + 'static, + U: CryptoSource + Clone, { fn return_with_error_log(&self, status_code: StatusCode, log_data: &mut MessageLog) -> Result> { log_data.status_code(&status_code).output(); @@ -194,11 +197,10 @@ where //////////////////////////////////////////////////// // Functions to generate messages - fn generate_response_forwarded( - &self, - response: &mut Response, - chosen_backend: &Backend, - ) -> Result<()> { + fn generate_response_forwarded(&self, response: &mut Response, chosen_backend: &Backend) -> Result<()> + where + B: core::fmt::Debug, + { let headers = response.headers_mut(); remove_connection_header(headers); remove_hop_header(headers); diff --git a/src/main.rs b/src/main.rs index 526c290..7f8dcfc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use certs::CryptoSource; #[cfg(not(target_env = "msvc"))] use tikv_jemallocator::Jemalloc; @@ -18,7 +19,8 @@ mod proxy; mod utils; use crate::{ - config::build_globals, error::*, globals::*, handler::HttpMessageHandlerBuilder, log::*, proxy::ProxyBuilder, + cert_file_reader::CryptoFileSource, config::build_globals, error::*, globals::*, handler::HttpMessageHandlerBuilder, + log::*, proxy::ProxyBuilder, }; use futures::future::select_all; use hyper::Client; @@ -34,7 +36,7 @@ fn main() { let runtime = runtime_builder.build().unwrap(); runtime.block_on(async { - let globals = match build_globals(runtime.handle().clone()) { + let globals: Globals = match build_globals(runtime.handle().clone()) { Ok(g) => g, Err(e) => { error!("Invalid configuration: {}", e); @@ -48,7 +50,10 @@ fn main() { } // entrypoint creates and spawns tasks of proxy services -async fn entrypoint(globals: Arc) -> Result<()> { +async fn entrypoint(globals: Arc>) -> Result<()> +where + T: CryptoSource + Clone + Send + Sync + 'static, +{ // let connector = TrustDnsResolver::default().into_rustls_webpki_https_connector(); let connector = hyper_rustls::HttpsConnectorBuilder::new() .with_webpki_roots() diff --git a/src/proxy/crypto_service.rs b/src/proxy/crypto_service.rs index 629119b..bb582fa 100644 --- a/src/proxy/crypto_service.rs +++ b/src/proxy/crypto_service.rs @@ -1,6 +1,6 @@ use crate::{ cert_file_reader::read_certs_and_keys, // TODO: Trait defining read_certs_and_keys and add struct implementing the trait to backend when build backend - certs::CertsAndKeys, + certs::{CertsAndKeys, CryptoSource}, globals::Globals, log::*, utils::ServerNameBytesExp, @@ -18,8 +18,11 @@ use x509_parser::prelude::*; #[derive(Clone)] /// Reloader service for certificates and keys for TLS -pub struct CryptoReloader { - globals: Arc, +pub struct CryptoReloader +where + T: CryptoSource, +{ + globals: Arc>, } pub type SniServerCryptoMap = HashMap>; @@ -37,8 +40,11 @@ pub struct ServerCryptoBase { } #[async_trait] -impl Reload for CryptoReloader { - type Source = Arc; +impl Reload for CryptoReloader +where + T: CryptoSource + Sync + Send, +{ + type Source = Arc>; async fn new(source: &Self::Source) -> Result> { Ok(Self { globals: source.clone(), diff --git a/src/proxy/proxy_h3.rs b/src/proxy/proxy_h3.rs index 12ebd7d..324060f 100644 --- a/src/proxy/proxy_h3.rs +++ b/src/proxy/proxy_h3.rs @@ -1,14 +1,15 @@ use super::Proxy; -use crate::{error::*, log::*, utils::ServerNameBytesExp}; +use crate::{certs::CryptoSource, error::*, log::*, utils::ServerNameBytesExp}; use bytes::{Buf, Bytes}; use h3::{quic::BidiStream, server::RequestStream}; use hyper::{client::connect::Connect, Body, Request, Response}; use std::net::SocketAddr; use tokio::time::{timeout, Duration}; -impl Proxy +impl Proxy where T: Connect + Clone + Sync + Send + 'static, + U: CryptoSource + Clone + Sync + Send + 'static, { pub(super) async fn connection_serve_h3( self, diff --git a/src/proxy/proxy_main.rs b/src/proxy/proxy_main.rs index a0f9660..e5a02a5 100644 --- a/src/proxy/proxy_main.rs +++ b/src/proxy/proxy_main.rs @@ -1,5 +1,7 @@ // use super::proxy_handler::handle_request; -use crate::{error::*, globals::Globals, handler::HttpMessageHandler, log::*, utils::ServerNameBytesExp}; +use crate::{ + certs::CryptoSource, error::*, globals::Globals, handler::HttpMessageHandler, log::*, utils::ServerNameBytesExp, +}; use derive_builder::{self, Builder}; use hyper::{client::connect::Connect, server::conn::Http, service::service_fn, Body, Request}; use std::{net::SocketAddr, sync::Arc}; @@ -32,19 +34,21 @@ where } #[derive(Clone, Builder)] -pub struct Proxy +pub struct Proxy where T: Connect + Clone + Sync + Send + 'static, + U: CryptoSource + Clone + Sync + Send + 'static, { pub listening_on: SocketAddr, pub tls_enabled: bool, // TCP待受がTLSかどうか - pub msg_handler: HttpMessageHandler, - pub globals: Arc, + pub msg_handler: HttpMessageHandler, + pub globals: Arc>, } -impl Proxy +impl Proxy where T: Connect + Clone + Sync + Send + 'static, + U: CryptoSource + Clone + Sync + Send, { pub(super) fn client_serve( self, diff --git a/src/proxy/proxy_tls.rs b/src/proxy/proxy_tls.rs index e01f9d3..5e846f0 100644 --- a/src/proxy/proxy_tls.rs +++ b/src/proxy/proxy_tls.rs @@ -2,7 +2,7 @@ use super::{ crypto_service::{CryptoReloader, ServerCrypto, ServerCryptoBase, SniServerCryptoMap}, proxy_main::{LocalExecutor, Proxy}, }; -use crate::{constants::*, error::*, log::*, utils::BytesName}; +use crate::{certs::CryptoSource, constants::*, error::*, log::*, utils::BytesName}; use hot_reload::{ReloaderReceiver, ReloaderService}; use hyper::{client::connect::Connect, server::conn::Http}; #[cfg(feature = "http3")] @@ -15,9 +15,10 @@ use tokio::{ time::{timeout, Duration}, }; -impl Proxy +impl Proxy where T: Connect + Clone + Sync + Send + 'static, + U: CryptoSource + Clone + Sync + Send + 'static, { // TCP Listener Service, i.e., http/2 and http/1.1 async fn listener_service( @@ -181,7 +182,7 @@ where } pub async fn start_with_tls(self, server: Http) -> Result<()> { - let (cert_reloader_service, cert_reloader_rx) = ReloaderService::::new( + let (cert_reloader_service, cert_reloader_rx) = ReloaderService::, ServerCryptoBase>::new( &self.globals.clone(), CERTS_WATCH_DELAY_SECS, !LOAD_CERTS_ONLY_WHEN_UPDATED, From 05b2aab8b0ee727526aabc001ae242109430c867 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Wed, 12 Jul 2023 21:40:08 +0900 Subject: [PATCH 14/43] refactor: remove explict cert file path from backend mods and define abstracted trait for the cert source preparing librarization --- src/backend/mod.rs | 27 +++------------------------ src/cert_file_reader.rs | 15 ++++++++++----- src/certs.rs | 5 +++++ src/config/parse.rs | 14 +++++++++----- src/config/toml.rs | 31 ++++++++++++++++--------------- src/error.rs | 5 ++--- src/handler/handler_main.rs | 7 ++++++- src/proxy/crypto_service.rs | 11 ++++------- 8 files changed, 55 insertions(+), 60 deletions(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 9bc28e5..524f30b 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -19,7 +19,7 @@ use crate::{ }; use derive_builder::Builder; use rustc_hash::FxHashMap as HashMap; -use std::{borrow::Cow, path::PathBuf}; +use std::borrow::Cow; /// Struct serving information to route incoming connections, like server name to be handled and tls certs/keys settings. #[derive(Builder)] @@ -36,16 +36,11 @@ where /// struct of reverse proxy serving incoming request pub reverse_proxy: ReverseProxy, - /// tls settings - #[builder(setter(custom), default)] - pub tls_cert_path: Option, - #[builder(setter(custom), default)] - pub tls_cert_key_path: Option, + /// tls settings: https redirection with 30x #[builder(default)] pub https_redirection: Option, - #[builder(setter(custom), default)] - pub client_ca_cert_path: Option, + /// TLS settings: source meta for server cert, key, client ca cert #[builder(default)] pub crypto_source: Option, } @@ -57,22 +52,6 @@ where self.server_name = Some(server_name.into().to_ascii_lowercase()); self } - pub fn tls_cert_path(&mut self, v: &Option) -> &mut Self { - self.tls_cert_path = Some(opt_string_to_opt_pathbuf(v)); - self - } - pub fn tls_cert_key_path(&mut self, v: &Option) -> &mut Self { - self.tls_cert_key_path = Some(opt_string_to_opt_pathbuf(v)); - self - } - pub fn client_ca_cert_path(&mut self, v: &Option) -> &mut Self { - self.client_ca_cert_path = Some(opt_string_to_opt_pathbuf(v)); - self - } -} - -fn opt_string_to_opt_pathbuf(input: &Option) -> Option { - input.to_owned().as_ref().map(PathBuf::from) } /// HashMap and some meta information for multiple Backend structs. diff --git a/src/cert_file_reader.rs b/src/cert_file_reader.rs index dc25b09..e25dcf7 100644 --- a/src/cert_file_reader.rs +++ b/src/cert_file_reader.rs @@ -36,8 +36,8 @@ impl CryptoFileSourceBuilder { self.tls_cert_key_path = Some(PathBuf::from(v)); self } - pub fn client_ca_cert_path(&mut self, v: &str) -> &mut Self { - self.client_ca_cert_path = Some(Some(PathBuf::from(v))); + pub fn client_ca_cert_path(&mut self, v: &Option) -> &mut Self { + self.client_ca_cert_path = Some(v.to_owned().as_ref().map(PathBuf::from)); self } } @@ -45,6 +45,7 @@ impl CryptoFileSourceBuilder { #[async_trait] impl CryptoSource for CryptoFileSource { type Error = io::Error; + /// read crypto materials from source async fn read(&self) -> Result { read_certs_and_keys( &self.tls_cert_path, @@ -52,10 +53,14 @@ impl CryptoSource for CryptoFileSource { self.client_ca_cert_path.as_ref(), ) } + /// Returns true when mutual tls is enabled + fn is_mutual_tls(&self) -> bool { + self.client_ca_cert_path.is_some() + } } /// Read certificates and private keys from file -pub(crate) fn read_certs_and_keys( +fn read_certs_and_keys( cert_path: &PathBuf, cert_key_path: &PathBuf, client_ca_cert_path: Option<&PathBuf>, @@ -162,11 +167,11 @@ mod tests { async fn read_server_crt_key_files_with_client_ca_crt() { let tls_cert_path = "example-certs/server.crt"; let tls_cert_key_path = "example-certs/server.key"; - let client_ca_cert_path = "example-certs/client.ca.crt"; + let client_ca_cert_path = Some("example-certs/client.ca.crt".to_string()); let crypto_file_source = CryptoFileSourceBuilder::default() .tls_cert_key_path(tls_cert_key_path) .tls_cert_path(tls_cert_path) - .client_ca_cert_path(client_ca_cert_path) + .client_ca_cert_path(&client_ca_cert_path) .build(); assert!(crypto_file_source.is_ok()); diff --git a/src/certs.rs b/src/certs.rs index da51e14..2ed0198 100644 --- a/src/certs.rs +++ b/src/certs.rs @@ -13,5 +13,10 @@ pub struct CertsAndKeys { // Trait to read certs and keys anywhere from KVS, file, sqlite, etc. pub trait CryptoSource { type Error; + + /// read crypto materials from source async fn read(&self) -> Result; + + /// Returns true when mutual tls is enabled + fn is_mutual_tls(&self) -> bool; } diff --git a/src/config/parse.rs b/src/config/parse.rs index 83aa546..1dc2545 100644 --- a/src/config/parse.rs +++ b/src/config/parse.rs @@ -1,12 +1,16 @@ use super::toml::ConfigToml; -use crate::{backend::Backends, certs::CryptoSource, error::*, globals::*, log::*, utils::BytesName}; +use crate::{ + backend::Backends, + cert_file_reader::CryptoFileSource, + error::{anyhow, ensure}, + globals::*, + log::*, + utils::BytesName, +}; use clap::Arg; use tokio::runtime::Handle; -pub fn build_globals(runtime_handle: Handle) -> std::result::Result, anyhow::Error> -where - T: CryptoSource + Clone, -{ +pub fn build_globals(runtime_handle: Handle) -> std::result::Result, anyhow::Error> { let _ = include_str!("../../Cargo.toml"); let options = clap::command!().arg( Arg::new("config_file") diff --git a/src/config/toml.rs b/src/config/toml.rs index a68b82b..f33ea4d 100644 --- a/src/config/toml.rs +++ b/src/config/toml.rs @@ -1,8 +1,8 @@ use crate::{ backend::{Backend, BackendBuilder, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption}, - certs::CryptoSource, + cert_file_reader::{CryptoFileSource, CryptoFileSourceBuilder}, constants::*, - error::*, + error::{anyhow, ensure}, globals::ProxyConfig, utils::PathNameBytesExp, }; @@ -164,20 +164,17 @@ impl TryInto for &ConfigToml { } impl ConfigToml { - pub fn new(config_file: &str) -> std::result::Result { - let config_str = fs::read_to_string(config_file).map_err(RpxyError::Io)?; + pub fn new(config_file: &str) -> std::result::Result { + let config_str = fs::read_to_string(config_file)?; - toml::from_str(&config_str).map_err(RpxyError::TomlDe) + toml::from_str(&config_str).map_err(|e| anyhow!(e)) } } -impl TryInto> for &Application -where - T: CryptoSource + Clone, -{ +impl TryInto> for &Application { type Error = anyhow::Error; - fn try_into(self) -> std::result::Result, Self::Error> { + fn try_into(self) -> std::result::Result, Self::Error> { let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; // backend builder @@ -203,11 +200,15 @@ where tls.https_redirection }; - backend_builder - .tls_cert_path(&tls.tls_cert_path) - .tls_cert_key_path(&tls.tls_cert_key_path) - .https_redirection(https_redirection) + let crypto_source = CryptoFileSourceBuilder::default() + .tls_cert_path(tls.tls_cert_path.as_ref().unwrap()) + .tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap()) .client_ca_cert_path(&tls.client_ca_cert_path) + .build()?; + + backend_builder + .https_redirection(https_redirection) + .crypto_source(Some(crypto_source)) .build()? }; Ok(backend) @@ -255,7 +256,7 @@ impl TryInto for &Application { } impl TryInto for &UpstreamParams { - type Error = RpxyError; + type Error = anyhow::Error; fn try_into(self) -> std::result::Result { let scheme = match self.tls { diff --git a/src/error.rs b/src/error.rs index 18b4307..187c993 100644 --- a/src/error.rs +++ b/src/error.rs @@ -29,9 +29,8 @@ pub enum RpxyError { #[error("I/O Error")] Io(#[from] io::Error), - #[error("Toml Deserialization Error")] - TomlDe(#[from] toml::de::Error), - + // #[error("Toml Deserialization Error")] + // TomlDe(#[from] toml::de::Error), #[cfg(feature = "http3")] #[error("Quic Connection Error")] QuicConn(#[from] quinn::ConnectionError), diff --git a/src/handler/handler_main.rs b/src/handler/handler_main.rs index 2016f2c..c23fd24 100644 --- a/src/handler/handler_main.rs +++ b/src/handler/handler_main.rs @@ -209,7 +209,12 @@ where #[cfg(feature = "http3")] { // TODO: Workaround for avoid h3 for client authentication - if self.globals.proxy_config.http3 && chosen_backend.client_ca_cert_path.is_none() { + if self.globals.proxy_config.http3 + && chosen_backend + .crypto_source + .as_ref() + .is_some_and(|v| !v.is_mutual_tls()) + { if let Some(port) = self.globals.proxy_config.https_port { add_header_entry_overwrite_if_exist( headers, diff --git a/src/proxy/crypto_service.rs b/src/proxy/crypto_service.rs index bb582fa..8d7f00d 100644 --- a/src/proxy/crypto_service.rs +++ b/src/proxy/crypto_service.rs @@ -1,5 +1,4 @@ use crate::{ - cert_file_reader::read_certs_and_keys, // TODO: Trait defining read_certs_and_keys and add struct implementing the trait to backend when build backend certs::{CertsAndKeys, CryptoSource}, globals::Globals, log::*, @@ -55,13 +54,11 @@ where let mut certs_and_keys_map = ServerCryptoBase::default(); for (server_name_bytes_exp, backend) in self.globals.backends.apps.iter() { - if backend.tls_cert_key_path.is_some() && backend.tls_cert_path.is_some() { - let tls_cert_key_path = backend.tls_cert_key_path.as_ref().unwrap(); - let tls_cert_path = backend.tls_cert_path.as_ref().unwrap(); - let tls_client_ca_cert_path = backend.client_ca_cert_path.as_ref(); - let certs_and_keys = read_certs_and_keys(tls_cert_path, tls_cert_key_path, tls_client_ca_cert_path) + if let Some(crypto_source) = &backend.crypto_source { + let certs_and_keys = crypto_source + .read() + .await .map_err(|_e| ReloaderError::::Reload("Failed to reload cert, key or ca cert"))?; - certs_and_keys_map .inner .insert(server_name_bytes_exp.to_owned(), certs_and_keys); From 1da7e5bfb77d1ce4ee8d6cfc59b1c725556fc192 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Thu, 13 Jul 2023 12:01:45 +0900 Subject: [PATCH 15/43] submodule: h3 --- h3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h3 b/h3 index 3ef7c1a..a38b194 160000 --- a/h3 +++ b/h3 @@ -1 +1 @@ -Subproject commit 3ef7c1a37b635e8446322d8f8d3a68580a208ad8 +Subproject commit a38b194a2f00dc0b2b60564c299093204d349d7e From 98f974b95f0acc2c3627b01a2897eee285ae21c2 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sat, 15 Jul 2023 01:26:58 +0900 Subject: [PATCH 16/43] update benchmark result --- bench/README.md | 77 +++++++++++++++++++++++----------------- bench/docker-compose.yml | 1 + 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/bench/README.md b/bench/README.md index d8a5017..7a98c81 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,11 +1,24 @@ # Sample Benchmark Result -Using `rewrk` and Docker on a Macbook Pro 14 to simply measure the performance of several reverse proxy through HTTP1.1. +Done at Jul. 15, 2023 -``` +This test simply measures the performance of several reverse proxy through HTTP/1.1 by the following command using [`rewrk`](https://github.com/lnx-search/rewrk). + +```bash $ rewrk -c 512 -t 4 -d 15s -h http://localhost:8080 --pct ``` +## Environment + +- `rpxy` commit id: `1da7e5bfb77d1ce4ee8d6cfc59b1c725556fc192` +- Docker Desktop 4.21.1 (114176) +- ReWrk 0.3.1 +- Macbook Pro '14 (2021, M1 Max, 64GB RAM) + + + +## Result + ``` ---------------------------- Benchmark on rpxy @@ -13,23 +26,23 @@ Beginning round 1... Benchmarking 512 connections @ http://localhost:8080 for 15 second(s) Latencies: Avg Stdev Min Max - 26.81ms 11.96ms 2.96ms 226.04ms + 19.64ms 8.85ms 0.67ms 113.22ms Requests: - Total: 285390 Req/Sec: 19032.01 + Total: 390078 Req/Sec: 26011.25 Transfer: - Total: 222.85 MB Transfer Rate: 14.86 MB/Sec + Total: 304.85 MB Transfer Rate: 20.33 MB/Sec + --------------- + --------------- + | Percentile | Avg Latency | + --------------- + --------------- + -| 99.9% | 145.89ms | -| 99% | 81.33ms | -| 95% | 59.08ms | -| 90% | 51.67ms | -| 75% | 42.45ms | -| 50% | 35.39ms | +| 99.9% | 79.24ms | +| 99% | 54.28ms | +| 95% | 42.50ms | +| 90% | 37.82ms | +| 75% | 31.54ms | +| 50% | 26.37ms | + --------------- + --------------- + -767 Errors: error shutting down connection: Socket is not connected (os error 57) +721 Errors: error shutting down connection: Socket is not connected (os error 57) sleep 3 secs ---------------------------- @@ -38,23 +51,23 @@ Beginning round 1... Benchmarking 512 connections @ http://localhost:8090 for 15 second(s) Latencies: Avg Stdev Min Max - 38.39ms 21.06ms 2.91ms 248.32ms + 33.26ms 15.18ms 1.40ms 118.94ms Requests: - Total: 199210 Req/Sec: 13288.91 + Total: 230268 Req/Sec: 15356.08 Transfer: - Total: 161.46 MB Transfer Rate: 10.77 MB/Sec + Total: 186.77 MB Transfer Rate: 12.46 MB/Sec + --------------- + --------------- + | Percentile | Avg Latency | + --------------- + --------------- + -| 99.9% | 164.33ms | -| 99% | 121.55ms | -| 95% | 96.43ms | -| 90% | 85.05ms | -| 75% | 67.80ms | -| 50% | 53.85ms | +| 99.9% | 99.91ms | +| 99% | 83.74ms | +| 95% | 70.67ms | +| 90% | 64.03ms | +| 75% | 54.32ms | +| 50% | 45.19ms | + --------------- + --------------- + -736 Errors: error shutting down connection: Socket is not connected (os error 57) +677 Errors: error shutting down connection: Socket is not connected (os error 57) sleep 3 secs ---------------------------- @@ -63,21 +76,21 @@ Beginning round 1... Benchmarking 512 connections @ http://localhost:8100 for 15 second(s) Latencies: Avg Stdev Min Max - 83.17ms 73.71ms 1.24ms 734.67ms + 48.51ms 50.74ms 0.34ms 554.58ms Requests: - Total: 91685 Req/Sec: 6114.05 + Total: 157239 Req/Sec: 10485.98 Transfer: - Total: 73.20 MB Transfer Rate: 4.88 MB/Sec + Total: 125.99 MB Transfer Rate: 8.40 MB/Sec + --------------- + --------------- + | Percentile | Avg Latency | + --------------- + --------------- + -| 99.9% | 642.29ms | -| 99% | 507.21ms | -| 95% | 324.34ms | -| 90% | 249.55ms | -| 75% | 174.62ms | -| 50% | 128.85ms | +| 99.9% | 473.82ms | +| 99% | 307.16ms | +| 95% | 212.28ms | +| 90% | 169.05ms | +| 75% | 115.92ms | +| 50% | 80.24ms | + --------------- + --------------- + -740 Errors: error shutting down connection: Socket is not connected (os error 57) +708 Errors: error shutting down connection: Socket is not connected (os error 57) ``` diff --git a/bench/docker-compose.yml b/bench/docker-compose.yml index 1b4bfa0..23308aa 100644 --- a/bench/docker-compose.yml +++ b/bench/docker-compose.yml @@ -25,6 +25,7 @@ services: pull_policy: never build: context: ../ + dockerfile: docker/Dockerfile.amd64 restart: unless-stopped environment: - LOG_LEVEL=info From 629e3a1f98eaf16414c6b81739280d464af31ba2 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sat, 15 Jul 2023 01:43:20 +0900 Subject: [PATCH 17/43] submodule: h3 --- h3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h3 b/h3 index a38b194..dccb3cd 160000 --- a/h3 +++ b/h3 @@ -1 +1 @@ -Subproject commit a38b194a2f00dc0b2b60564c299093204d349d7e +Subproject commit dccb3cdae9d5a9d720fae5f774b53f0bd8a16019 From 15e865963319353a58bfa5c416a2f5e0a0bda53d Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 18 Jul 2023 15:36:35 +0900 Subject: [PATCH 18/43] fix: inappropriate location of CertsAndKeys implementations --- src/certs.rs | 87 +++++++++++++++++++++++++++++++++---- src/proxy/crypto_service.rs | 74 ++----------------------------- 2 files changed, 81 insertions(+), 80 deletions(-) diff --git a/src/certs.rs b/src/certs.rs index 2ed0198..c9cfafd 100644 --- a/src/certs.rs +++ b/src/certs.rs @@ -1,13 +1,11 @@ use async_trait::async_trait; -use rustls::{Certificate, PrivateKey}; - -/// Certificates and private keys in rustls loaded from files -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct CertsAndKeys { - pub certs: Vec, - pub cert_keys: Vec, - pub client_ca_certs: Option>, -} +use rustc_hash::FxHashSet as HashSet; +use rustls::{ + sign::{any_supported_type, CertifiedKey}, + Certificate, OwnedTrustAnchor, PrivateKey, +}; +use std::io; +use x509_parser::prelude::*; #[async_trait] // Trait to read certs and keys anywhere from KVS, file, sqlite, etc. @@ -20,3 +18,74 @@ pub trait CryptoSource { /// Returns true when mutual tls is enabled fn is_mutual_tls(&self) -> bool; } + +/// Certificates and private keys in rustls loaded from files +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct CertsAndKeys { + pub certs: Vec, + pub cert_keys: Vec, + pub client_ca_certs: Option>, +} + +impl CertsAndKeys { + pub fn parse_server_certs_and_keys(&self) -> Result { + // for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { + let signing_key = self + .cert_keys + .iter() + .find_map(|k| { + if let Ok(sk) = any_supported_type(k) { + Some(sk) + } else { + None + } + }) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "Unable to find a valid certificate and key", + ) + })?; + Ok(CertifiedKey::new(self.certs.clone(), signing_key)) + } + + pub fn parse_client_ca_certs(&self) -> Result<(Vec, HashSet>), anyhow::Error> { + let certs = self.client_ca_certs.as_ref().ok_or(anyhow::anyhow!("No client cert"))?; + + let owned_trust_anchors: Vec<_> = certs + .iter() + .map(|v| { + // let trust_anchor = tokio_rustls::webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); + let trust_anchor = webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); + rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( + trust_anchor.subject, + trust_anchor.spki, + trust_anchor.name_constraints, + ) + }) + .collect(); + + // TODO: SKID is not used currently + let subject_key_identifiers: HashSet<_> = certs + .iter() + .filter_map(|v| { + // retrieve ca key id (subject key id) + let cert = parse_x509_certificate(&v.0).unwrap().1; + let subject_key_ids = cert + .iter_extensions() + .filter_map(|ext| match ext.parsed_extension() { + ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid), + _ => None, + }) + .collect::>(); + if !subject_key_ids.is_empty() { + Some(subject_key_ids[0].0.to_owned()) + } else { + None + } + }) + .collect(); + + Ok((owned_trust_anchors, subject_key_identifiers)) + } +} diff --git a/src/proxy/crypto_service.rs b/src/proxy/crypto_service.rs index 8d7f00d..8675a1d 100644 --- a/src/proxy/crypto_service.rs +++ b/src/proxy/crypto_service.rs @@ -6,14 +6,9 @@ use crate::{ }; use async_trait::async_trait; use hot_reload::*; -use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; -use rustls::{ - server::ResolvesServerCertUsingSni, - sign::{any_supported_type, CertifiedKey}, - OwnedTrustAnchor, RootCertStore, ServerConfig, -}; -use std::{io, sync::Arc}; -use x509_parser::prelude::*; +use rustc_hash::FxHashMap as HashMap; +use rustls::{server::ResolvesServerCertUsingSni, sign::CertifiedKey, RootCertStore, ServerConfig}; +use std::sync::Arc; #[derive(Clone)] /// Reloader service for certificates and keys for TLS @@ -69,69 +64,6 @@ where } } -impl CertsAndKeys { - fn parse_server_certs_and_keys(&self) -> Result { - // for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { - let signing_key = self - .cert_keys - .iter() - .find_map(|k| { - if let Ok(sk) = any_supported_type(k) { - Some(sk) - } else { - None - } - }) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "Unable to find a valid certificate and key", - ) - })?; - Ok(CertifiedKey::new(self.certs.clone(), signing_key)) - } - - pub fn parse_client_ca_certs(&self) -> Result<(Vec, HashSet>), anyhow::Error> { - let certs = self.client_ca_certs.as_ref().ok_or(anyhow::anyhow!("No client cert"))?; - - let owned_trust_anchors: Vec<_> = certs - .iter() - .map(|v| { - // let trust_anchor = tokio_rustls::webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); - let trust_anchor = webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap(); - rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( - trust_anchor.subject, - trust_anchor.spki, - trust_anchor.name_constraints, - ) - }) - .collect(); - - // TODO: SKID is not used currently - let subject_key_identifiers: HashSet<_> = certs - .iter() - .filter_map(|v| { - // retrieve ca key id (subject key id) - let cert = parse_x509_certificate(&v.0).unwrap().1; - let subject_key_ids = cert - .iter_extensions() - .filter_map(|ext| match ext.parsed_extension() { - ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid), - _ => None, - }) - .collect::>(); - if !subject_key_ids.is_empty() { - Some(subject_key_ids[0].0.to_owned()) - } else { - None - } - }) - .collect(); - - Ok((owned_trust_anchors, subject_key_identifiers)) - } -} - impl TryInto> for &ServerCryptoBase { type Error = anyhow::Error; From 66950c611480b2dbe71e23627a48c80af02bb7c0 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 21 Jul 2023 14:04:15 +0900 Subject: [PATCH 19/43] update todo list --- TODO.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index 90fb79d..81af1d5 100644 --- a/TODO.md +++ b/TODO.md @@ -4,11 +4,18 @@ - More flexible option for rewriting path - Refactoring - Split `backend` module into three parts + - Split `backend` module into three parts - - backend(s): struct containing info, defined for each served domain with multiple paths - - upstream/upstream group: information on targeted destinations for each set of (a domain + a path) - - load-balance: load balancing mod for a domain + path + - backend(s): struct containing info, defined for each served domain with multiple paths + - upstream/upstream group: information on targeted destinations for each set of (a domain + a path) + - load-balance: load balancing mod for a domain + path + + - Split `rpxy` source codes into `rpxy-lib` and `rpxy-bin` to make the core part (reverse proxy) isolated from the misc part like toml file loader. This is in order to make the configuration-related part more flexible (related to [#33](https://github.com/junkurihara/rust-rpxy/issues/33)) + +- Cache option for the response with `Cache-Control: public` header directive ([#55](https://github.com/junkurihara/rust-rpxy/issues/55)) +- Consideration on migrating from `quinn` and `h3-quinn` to other QUIC implementations ([#57](https://github.com/junkurihara/rust-rpxy/issues/57)) +- Benchmark with other reverse proxy implementations like Sozu ([#58](https://github.com/junkurihara/rust-rpxy/issues/58)) + - Currently, Sozu can work only on `amd64` format due to its HTTP message parser limitation... Since the main developer have only `arm64` (Apple M1) laptops, so we should do that on VPS? - Unit tests - Options to serve custom http_error page. From ec85b0bb28c2c3ef0c6e908d8b4403bdffcbd4c1 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 21 Jul 2023 17:22:10 +0900 Subject: [PATCH 20/43] deps --- Cargo.toml | 12 ++++++------ h3 | 2 +- quinn | 2 +- src/globals.rs | 3 +-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5f7b0ea..e950521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,14 +17,14 @@ http3 = ["quinn", "h3", "h3-quinn"] sticky-cookie = ["base64", "sha2", "chrono"] [dependencies] -anyhow = "1.0.71" -clap = { version = "4.3.11", features = ["std", "cargo", "wrap_help"] } +anyhow = "1.0.72" +clap = { version = "4.3.17", features = ["std", "cargo", "wrap_help"] } rand = "0.8.5" toml = { version = "0.7.6", default-features = false, features = ["parse"] } rustc-hash = "1.1.0" -serde = { version = "1.0.171", default-features = false, features = ["derive"] } +serde = { version = "1.0.174", default-features = false, features = ["derive"] } bytes = "1.4.0" -thiserror = "1.0.43" +thiserror = "1.0.44" x509-parser = "0.15.0" derive_builder = "0.12.0" futures = { version = "0.3.28", features = ["alloc", "async-await"] } @@ -35,7 +35,7 @@ tokio = { version = "1.29.1", default-features = false, features = [ "sync", "macros", ] } -async-trait = "0.1.71" +async-trait = "0.1.72" hot_reload = "0.1.2" # reloading certs # http and tls @@ -53,7 +53,7 @@ hyper-rustls = { version = "0.24.1", default-features = false, features = [ ] } tokio-rustls = { version = "0.24.1", features = ["early-data"] } rustls-pemfile = "1.0.3" -rustls = { version = "0.21.3", default-features = false } +rustls = { version = "0.21.5", default-features = false } webpki = "0.22.0" # logging diff --git a/h3 b/h3 index dccb3cd..3991dca 160000 --- a/h3 +++ b/h3 @@ -1 +1 @@ -Subproject commit dccb3cdae9d5a9d720fae5f774b53f0bd8a16019 +Subproject commit 3991dcaf3801595e49d0bb7fb1649b4cf50292b7 diff --git a/quinn b/quinn index e652b6d..0ae7c60 160000 --- a/quinn +++ b/quinn @@ -1 +1 @@ -Subproject commit e652b6d999f053ffe21eeea247854882ae480281 +Subproject commit 0ae7c60b15637d7343410ba1e5cc3151e3814557 diff --git a/src/globals.rs b/src/globals.rs index b85733b..5150bff 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -1,5 +1,4 @@ -use crate::certs::CryptoSource; -use crate::{backend::Backends, constants::*}; +use crate::{backend::Backends, certs::CryptoSource, constants::*}; use std::net::SocketAddr; use std::sync::{ atomic::{AtomicUsize, Ordering}, From 13e82035a88c6add4d85f2e0431d5d4a120de08f Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 21 Jul 2023 18:48:40 +0900 Subject: [PATCH 21/43] refactor: initial implementation of separeted lib and bin --- Cargo.toml | 87 +------------------ rpxy-bin/Cargo.toml | 49 +++++++++++ {src => rpxy-bin/src}/cert_file_reader.rs | 10 +-- {src => rpxy-bin/src}/config/mod.rs | 0 {src => rpxy-bin/src}/config/parse.rs | 4 +- {src => rpxy-bin/src}/config/toml.rs | 9 +- rpxy-bin/src/constants.rs | 2 + rpxy-bin/src/error.rs | 1 + rpxy-bin/src/log.rs | 24 +++++ rpxy-bin/src/main.rs | 38 ++++++++ rpxy-lib/Cargo.toml | 77 ++++++++++++++++ {src => rpxy-lib/src}/backend/load_balance.rs | 0 .../src}/backend/load_balance_sticky.rs | 0 {src => rpxy-lib/src}/backend/mod.rs | 1 + .../src}/backend/sticky_cookie.rs | 0 {src => rpxy-lib/src}/backend/upstream.rs | 0 .../src}/backend/upstream_opts.rs | 0 {src => rpxy-lib/src}/certs.rs | 0 {src => rpxy-lib/src}/constants.rs | 4 +- {src => rpxy-lib/src}/error.rs | 0 {src => rpxy-lib/src}/globals.rs | 0 {src => rpxy-lib/src}/handler/handler_main.rs | 0 {src => rpxy-lib/src}/handler/mod.rs | 0 .../src}/handler/utils_headers.rs | 0 .../src}/handler/utils_request.rs | 0 .../src}/handler/utils_synth_response.rs | 0 src/main.rs => rpxy-lib/src/lib.rs | 50 +++-------- {src => rpxy-lib/src}/log.rs | 23 ----- {src => rpxy-lib/src}/proxy/crypto_service.rs | 0 {src => rpxy-lib/src}/proxy/mod.rs | 0 .../src}/proxy/proxy_client_cert.rs | 0 {src => rpxy-lib/src}/proxy/proxy_h3.rs | 0 {src => rpxy-lib/src}/proxy/proxy_main.rs | 0 {src => rpxy-lib/src}/proxy/proxy_tls.rs | 0 {src => rpxy-lib/src}/utils/bytes_name.rs | 3 + {src => rpxy-lib/src}/utils/mod.rs | 0 {src => rpxy-lib/src}/utils/socket_addr.rs | 0 37 files changed, 225 insertions(+), 157 deletions(-) create mode 100644 rpxy-bin/Cargo.toml rename {src => rpxy-bin/src}/cert_file_reader.rs (98%) rename {src => rpxy-bin/src}/config/mod.rs (100%) rename {src => rpxy-bin/src}/config/parse.rs (98%) rename {src => rpxy-bin/src}/config/toml.rs (97%) create mode 100644 rpxy-bin/src/constants.rs create mode 100644 rpxy-bin/src/error.rs create mode 100644 rpxy-bin/src/log.rs create mode 100644 rpxy-bin/src/main.rs create mode 100644 rpxy-lib/Cargo.toml rename {src => rpxy-lib/src}/backend/load_balance.rs (100%) rename {src => rpxy-lib/src}/backend/load_balance_sticky.rs (100%) rename {src => rpxy-lib/src}/backend/mod.rs (98%) rename {src => rpxy-lib/src}/backend/sticky_cookie.rs (100%) rename {src => rpxy-lib/src}/backend/upstream.rs (100%) rename {src => rpxy-lib/src}/backend/upstream_opts.rs (100%) rename {src => rpxy-lib/src}/certs.rs (100%) rename {src => rpxy-lib/src}/constants.rs (90%) rename {src => rpxy-lib/src}/error.rs (100%) rename {src => rpxy-lib/src}/globals.rs (100%) rename {src => rpxy-lib/src}/handler/handler_main.rs (100%) rename {src => rpxy-lib/src}/handler/mod.rs (100%) rename {src => rpxy-lib/src}/handler/utils_headers.rs (100%) rename {src => rpxy-lib/src}/handler/utils_request.rs (100%) rename {src => rpxy-lib/src}/handler/utils_synth_response.rs (100%) rename src/main.rs => rpxy-lib/src/lib.rs (56%) rename {src => rpxy-lib/src}/log.rs (80%) rename {src => rpxy-lib/src}/proxy/crypto_service.rs (100%) rename {src => rpxy-lib/src}/proxy/mod.rs (100%) rename {src => rpxy-lib/src}/proxy/proxy_client_cert.rs (100%) rename {src => rpxy-lib/src}/proxy/proxy_h3.rs (100%) rename {src => rpxy-lib/src}/proxy/proxy_main.rs (100%) rename {src => rpxy-lib/src}/proxy/proxy_tls.rs (100%) rename {src => rpxy-lib/src}/utils/bytes_name.rs (98%) rename {src => rpxy-lib/src}/utils/mod.rs (100%) rename {src => rpxy-lib/src}/utils/socket_addr.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index e950521..64d1414 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,88 +1,7 @@ -[package] -name = "rpxy" -version = "0.3.0" -authors = ["Jun Kurihara"] -homepage = "https://github.com/junkurihara/rust-rpxy" -repository = "https://github.com/junkurihara/rust-rpxy" -license = "MIT" -readme = "README.md" -edition = "2021" -publish = false - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[features] -default = ["http3", "sticky-cookie"] -http3 = ["quinn", "h3", "h3-quinn"] -sticky-cookie = ["base64", "sha2", "chrono"] - -[dependencies] -anyhow = "1.0.72" -clap = { version = "4.3.17", features = ["std", "cargo", "wrap_help"] } -rand = "0.8.5" -toml = { version = "0.7.6", default-features = false, features = ["parse"] } -rustc-hash = "1.1.0" -serde = { version = "1.0.174", default-features = false, features = ["derive"] } -bytes = "1.4.0" -thiserror = "1.0.44" -x509-parser = "0.15.0" -derive_builder = "0.12.0" -futures = { version = "0.3.28", features = ["alloc", "async-await"] } -tokio = { version = "1.29.1", default-features = false, features = [ - "net", - "rt-multi-thread", - "time", - "sync", - "macros", -] } -async-trait = "0.1.72" -hot_reload = "0.1.2" # reloading certs - -# http and tls -hyper = { version = "0.14.27", default-features = false, features = [ - "server", - "http1", - "http2", - "stream", -] } -hyper-rustls = { version = "0.24.1", default-features = false, features = [ - "tokio-runtime", - "webpki-tokio", - "http1", - "http2", -] } -tokio-rustls = { version = "0.24.1", features = ["early-data"] } -rustls-pemfile = "1.0.3" -rustls = { version = "0.21.5", default-features = false } -webpki = "0.22.0" - -# logging -tracing = { version = "0.1.37" } -tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } - -# http/3 -# quinn = { version = "0.9.3", optional = true } -quinn = { path = "./quinn/quinn", optional = true } # Tentative to support rustls-0.21 -h3 = { path = "./h3/h3/", optional = true } -# h3-quinn = { path = "./h3/h3-quinn/", optional = true } -h3-quinn = { path = "./h3-quinn/", optional = true } # Tentative to support rustls-0.21 - -# cookie handling for sticky cookie -chrono = { version = "0.4.26", default-features = false, features = [ - "unstable-locales", - "alloc", - "clock", -], optional = true } -base64 = { version = "0.21.2", optional = true } -sha2 = { version = "0.10.7", default-features = false, optional = true } - - -[target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = "0.5.0" - - -[dev-dependencies] +[workspace] +members = ["rpxy-bin", "rpxy-lib"] +exclude = ["quinn", "h3-quinn", "h3"] [profile.release] codegen-units = 1 diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml new file mode 100644 index 0000000..9f65325 --- /dev/null +++ b/rpxy-bin/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "rpxy" +version = "0.4.0" +authors = ["Jun Kurihara"] +homepage = "https://github.com/junkurihara/rust-rpxy" +repository = "https://github.com/junkurihara/rust-rpxy" +license = "MIT" +readme = "README.md" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] + +[dependencies] +rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] } + +anyhow = "1.0.72" +rustc-hash = "1.1.0" +serde = { version = "1.0.174", default-features = false, features = ["derive"] } +derive_builder = "0.12.0" +tokio = { version = "1.29.1", default-features = false, features = [ + "net", + "rt-multi-thread", + "time", + "sync", + "macros", +] } +async-trait = "0.1.72" + +# config +clap = { version = "4.3.17", features = ["std", "cargo", "wrap_help"] } +toml = { version = "0.7.6", default-features = false, features = ["parse"] } + +# reloading certs +hot_reload = "0.1.2" +rustls-pemfile = "1.0.3" + +# logging +tracing = { version = "0.1.37" } +tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } + + +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = "0.5.0" + + +[dev-dependencies] diff --git a/src/cert_file_reader.rs b/rpxy-bin/src/cert_file_reader.rs similarity index 98% rename from src/cert_file_reader.rs rename to rpxy-bin/src/cert_file_reader.rs index e25dcf7..ffe3099 100644 --- a/src/cert_file_reader.rs +++ b/rpxy-bin/src/cert_file_reader.rs @@ -1,10 +1,10 @@ -use crate::{ - certs::{CertsAndKeys, CryptoSource}, - log::*, -}; +use crate::log::*; use async_trait::async_trait; use derive_builder::Builder; -use rustls::{Certificate, PrivateKey}; +use rpxy_lib::{ + reexports::{Certificate, PrivateKey}, + CertsAndKeys, CryptoSource, +}; use std::{ fs::File, io::{self, BufReader, Cursor, Read}, diff --git a/src/config/mod.rs b/rpxy-bin/src/config/mod.rs similarity index 100% rename from src/config/mod.rs rename to rpxy-bin/src/config/mod.rs diff --git a/src/config/parse.rs b/rpxy-bin/src/config/parse.rs similarity index 98% rename from src/config/parse.rs rename to rpxy-bin/src/config/parse.rs index 1dc2545..27c8cd8 100644 --- a/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -1,13 +1,11 @@ use super::toml::ConfigToml; use crate::{ - backend::Backends, cert_file_reader::CryptoFileSource, error::{anyhow, ensure}, - globals::*, log::*, - utils::BytesName, }; use clap::Arg; +use rpxy_lib::{Backends, BytesName, Globals, ProxyConfig}; use tokio::runtime::Handle; pub fn build_globals(runtime_handle: Handle) -> std::result::Result, anyhow::Error> { diff --git a/src/config/toml.rs b/rpxy-bin/src/config/toml.rs similarity index 97% rename from src/config/toml.rs rename to rpxy-bin/src/config/toml.rs index f33ea4d..c73f8c1 100644 --- a/src/config/toml.rs +++ b/rpxy-bin/src/config/toml.rs @@ -1,10 +1,11 @@ use crate::{ - backend::{Backend, BackendBuilder, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption}, cert_file_reader::{CryptoFileSource, CryptoFileSourceBuilder}, constants::*, error::{anyhow, ensure}, - globals::ProxyConfig, - utils::PathNameBytesExp, +}; +use rpxy_lib::{ + reexports::Uri, Backend, BackendBuilder, PathNameBytesExp, ProxyConfig, ReverseProxy, Upstream, UpstreamGroup, + UpstreamGroupBuilder, UpstreamOption, }; use rustc_hash::FxHashMap as HashMap; use serde::Deserialize; @@ -265,7 +266,7 @@ impl TryInto for &UpstreamParams { }; let location = format!("{}://{}", scheme, self.location); Ok(Upstream { - uri: location.parse::().map_err(|e| anyhow!("{}", e))?, + uri: location.parse::().map_err(|e| anyhow!("{}", e))?, }) } } diff --git a/rpxy-bin/src/constants.rs b/rpxy-bin/src/constants.rs new file mode 100644 index 0000000..4181a26 --- /dev/null +++ b/rpxy-bin/src/constants.rs @@ -0,0 +1,2 @@ +pub const LISTEN_ADDRESSES_V4: &[&str] = &["0.0.0.0"]; +pub const LISTEN_ADDRESSES_V6: &[&str] = &["[::]"]; diff --git a/rpxy-bin/src/error.rs b/rpxy-bin/src/error.rs new file mode 100644 index 0000000..b559bce --- /dev/null +++ b/rpxy-bin/src/error.rs @@ -0,0 +1 @@ +pub use anyhow::{anyhow, bail, ensure, Context}; diff --git a/rpxy-bin/src/log.rs b/rpxy-bin/src/log.rs new file mode 100644 index 0000000..3fcf694 --- /dev/null +++ b/rpxy-bin/src/log.rs @@ -0,0 +1,24 @@ +pub use tracing::{debug, error, info, warn}; + +pub fn init_logger() { + use tracing_subscriber::{fmt, prelude::*, EnvFilter}; + + let format_layer = fmt::layer() + .with_line_number(false) + .with_thread_ids(false) + .with_target(false) + .with_thread_names(true) + .with_target(true) + .with_level(true) + .compact(); + + // This limits the logger to emits only rpxy crate + let level_string = std::env::var(EnvFilter::DEFAULT_ENV).unwrap_or_else(|_| "info".to_string()); + let filter_layer = EnvFilter::new(format!("{}={}", env!("CARGO_PKG_NAME"), level_string)); + // let filter_layer = EnvFilter::from_default_env(); + + tracing_subscriber::registry() + .with(format_layer) + .with(filter_layer) + .init(); +} diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs new file mode 100644 index 0000000..cd74116 --- /dev/null +++ b/rpxy-bin/src/main.rs @@ -0,0 +1,38 @@ +#[cfg(not(target_env = "msvc"))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + +mod cert_file_reader; +mod config; +mod constants; +mod error; +mod log; + +use crate::{cert_file_reader::CryptoFileSource, config::build_globals, log::*}; +use rpxy_lib::{entrypoint, Globals}; +use std::sync::Arc; + +fn main() { + init_logger(); + + let mut runtime_builder = tokio::runtime::Builder::new_multi_thread(); + runtime_builder.enable_all(); + runtime_builder.thread_name("rpxy"); + let runtime = runtime_builder.build().unwrap(); + + runtime.block_on(async { + let globals: Globals = match build_globals(runtime.handle().clone()) { + Ok(g) => g, + Err(e) => { + error!("Invalid configuration: {}", e); + std::process::exit(1); + } + }; + + entrypoint(Arc::new(globals)).await.unwrap() + }); + warn!("rpxy exited!"); +} diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml new file mode 100644 index 0000000..48cb437 --- /dev/null +++ b/rpxy-lib/Cargo.toml @@ -0,0 +1,77 @@ +[package] +name = "rpxy-lib" +version = "0.4.0" +authors = ["Jun Kurihara"] +homepage = "https://github.com/junkurihara/rust-rpxy" +repository = "https://github.com/junkurihara/rust-rpxy" +license = "MIT" +readme = "README.md" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["http3", "sticky-cookie"] +http3 = ["quinn", "h3", "h3-quinn"] +sticky-cookie = ["base64", "sha2", "chrono"] + +[dependencies] +rand = "0.8.5" +rustc-hash = "1.1.0" +bytes = "1.4.0" +derive_builder = "0.12.0" +futures = { version = "0.3.28", features = ["alloc", "async-await"] } +tokio = { version = "1.29.1", default-features = false, features = [ + "net", + "rt-multi-thread", + "time", + "sync", + "macros", +] } +async-trait = "0.1.72" +hot_reload = "0.1.2" # reloading certs + +# Error handling +anyhow = "1.0.72" +thiserror = "1.0.44" + +# http and tls +hyper = { version = "0.14.27", default-features = false, features = [ + "server", + "http1", + "http2", + "stream", +] } +hyper-rustls = { version = "0.24.1", default-features = false, features = [ + "tokio-runtime", + "webpki-tokio", + "http1", + "http2", +] } +tokio-rustls = { version = "0.24.1", features = ["early-data"] } +rustls = { version = "0.21.5", default-features = false } +webpki = "0.22.0" +x509-parser = "0.15.0" + +# logging +tracing = { version = "0.1.37" } + +# http/3 +# quinn = { version = "0.9.3", optional = true } +quinn = { path = "../quinn/quinn", optional = true } # Tentative to support rustls-0.21 +h3 = { path = "../h3/h3/", optional = true } +# h3-quinn = { path = "./h3/h3-quinn/", optional = true } +h3-quinn = { path = "../h3-quinn/", optional = true } # Tentative to support rustls-0.21 + +# cookie handling for sticky cookie +chrono = { version = "0.4.26", default-features = false, features = [ + "unstable-locales", + "alloc", + "clock", +], optional = true } +base64 = { version = "0.21.2", optional = true } +sha2 = { version = "0.10.7", default-features = false, optional = true } + + +[dev-dependencies] diff --git a/src/backend/load_balance.rs b/rpxy-lib/src/backend/load_balance.rs similarity index 100% rename from src/backend/load_balance.rs rename to rpxy-lib/src/backend/load_balance.rs diff --git a/src/backend/load_balance_sticky.rs b/rpxy-lib/src/backend/load_balance_sticky.rs similarity index 100% rename from src/backend/load_balance_sticky.rs rename to rpxy-lib/src/backend/load_balance_sticky.rs diff --git a/src/backend/mod.rs b/rpxy-lib/src/backend/mod.rs similarity index 98% rename from src/backend/mod.rs rename to rpxy-lib/src/backend/mod.rs index 524f30b..73c4466 100644 --- a/src/backend/mod.rs +++ b/rpxy-lib/src/backend/mod.rs @@ -67,6 +67,7 @@ impl Backends where T: CryptoSource, { + #[allow(clippy::new_without_default)] pub fn new() -> Self { Backends { apps: HashMap::>::default(), diff --git a/src/backend/sticky_cookie.rs b/rpxy-lib/src/backend/sticky_cookie.rs similarity index 100% rename from src/backend/sticky_cookie.rs rename to rpxy-lib/src/backend/sticky_cookie.rs diff --git a/src/backend/upstream.rs b/rpxy-lib/src/backend/upstream.rs similarity index 100% rename from src/backend/upstream.rs rename to rpxy-lib/src/backend/upstream.rs diff --git a/src/backend/upstream_opts.rs b/rpxy-lib/src/backend/upstream_opts.rs similarity index 100% rename from src/backend/upstream_opts.rs rename to rpxy-lib/src/backend/upstream_opts.rs diff --git a/src/certs.rs b/rpxy-lib/src/certs.rs similarity index 100% rename from src/certs.rs rename to rpxy-lib/src/certs.rs diff --git a/src/constants.rs b/rpxy-lib/src/constants.rs similarity index 90% rename from src/constants.rs rename to rpxy-lib/src/constants.rs index 2ed14d1..72cce78 100644 --- a/src/constants.rs +++ b/rpxy-lib/src/constants.rs @@ -1,5 +1,5 @@ -pub const LISTEN_ADDRESSES_V4: &[&str] = &["0.0.0.0"]; -pub const LISTEN_ADDRESSES_V6: &[&str] = &["[::]"]; +// pub const LISTEN_ADDRESSES_V4: &[&str] = &["0.0.0.0"]; +// pub const LISTEN_ADDRESSES_V6: &[&str] = &["[::]"]; // pub const HTTP_LISTEN_PORT: u16 = 8080; // pub const HTTPS_LISTEN_PORT: u16 = 8443; pub const PROXY_TIMEOUT_SEC: u64 = 60; diff --git a/src/error.rs b/rpxy-lib/src/error.rs similarity index 100% rename from src/error.rs rename to rpxy-lib/src/error.rs diff --git a/src/globals.rs b/rpxy-lib/src/globals.rs similarity index 100% rename from src/globals.rs rename to rpxy-lib/src/globals.rs diff --git a/src/handler/handler_main.rs b/rpxy-lib/src/handler/handler_main.rs similarity index 100% rename from src/handler/handler_main.rs rename to rpxy-lib/src/handler/handler_main.rs diff --git a/src/handler/mod.rs b/rpxy-lib/src/handler/mod.rs similarity index 100% rename from src/handler/mod.rs rename to rpxy-lib/src/handler/mod.rs diff --git a/src/handler/utils_headers.rs b/rpxy-lib/src/handler/utils_headers.rs similarity index 100% rename from src/handler/utils_headers.rs rename to rpxy-lib/src/handler/utils_headers.rs diff --git a/src/handler/utils_request.rs b/rpxy-lib/src/handler/utils_request.rs similarity index 100% rename from src/handler/utils_request.rs rename to rpxy-lib/src/handler/utils_request.rs diff --git a/src/handler/utils_synth_response.rs b/rpxy-lib/src/handler/utils_synth_response.rs similarity index 100% rename from src/handler/utils_synth_response.rs rename to rpxy-lib/src/handler/utils_synth_response.rs diff --git a/src/main.rs b/rpxy-lib/src/lib.rs similarity index 56% rename from src/main.rs rename to rpxy-lib/src/lib.rs index 7f8dcfc..7d7764a 100644 --- a/src/main.rs +++ b/rpxy-lib/src/lib.rs @@ -1,15 +1,5 @@ -use certs::CryptoSource; -#[cfg(not(target_env = "msvc"))] -use tikv_jemallocator::Jemalloc; - -#[cfg(not(target_env = "msvc"))] -#[global_allocator] -static GLOBAL: Jemalloc = Jemalloc; - mod backend; -mod cert_file_reader; mod certs; -mod config; mod constants; mod error; mod globals; @@ -18,39 +8,27 @@ mod log; mod proxy; mod utils; -use crate::{ - cert_file_reader::CryptoFileSource, config::build_globals, error::*, globals::*, handler::HttpMessageHandlerBuilder, - log::*, proxy::ProxyBuilder, -}; +use crate::{error::*, handler::HttpMessageHandlerBuilder, log::*, proxy::ProxyBuilder}; use futures::future::select_all; use hyper::Client; // use hyper_trust_dns::TrustDnsResolver; use std::sync::Arc; -fn main() { - init_logger(); - - let mut runtime_builder = tokio::runtime::Builder::new_multi_thread(); - runtime_builder.enable_all(); - runtime_builder.thread_name("rpxy"); - let runtime = runtime_builder.build().unwrap(); - - runtime.block_on(async { - let globals: Globals = match build_globals(runtime.handle().clone()) { - Ok(g) => g, - Err(e) => { - error!("Invalid configuration: {}", e); - std::process::exit(1); - } - }; - - entrypoint(Arc::new(globals)).await.unwrap() - }); - warn!("rpxy exited!"); +pub use crate::{ + backend::{ + Backend, BackendBuilder, Backends, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption, + }, + certs::{CertsAndKeys, CryptoSource}, + globals::{Globals, ProxyConfig}, // TODO: BackendConfigに変える + utils::{BytesName, PathNameBytesExp}, +}; +pub mod reexports { + pub use hyper::Uri; + pub use rustls::{Certificate, PrivateKey}; } -// entrypoint creates and spawns tasks of proxy services -async fn entrypoint(globals: Arc>) -> Result<()> +/// Entrypoint that creates and spawns tasks of reverse proxy services +pub async fn entrypoint(globals: Arc>) -> Result<()> where T: CryptoSource + Clone + Send + Sync + 'static, { diff --git a/src/log.rs b/rpxy-lib/src/log.rs similarity index 80% rename from src/log.rs rename to rpxy-lib/src/log.rs index d391607..0fb7812 100644 --- a/src/log.rs +++ b/rpxy-lib/src/log.rs @@ -95,26 +95,3 @@ impl MessageLog { ); } } - -pub fn init_logger() { - use tracing_subscriber::{fmt, prelude::*, EnvFilter}; - - let format_layer = fmt::layer() - .with_line_number(false) - .with_thread_ids(false) - .with_target(false) - .with_thread_names(true) - .with_target(true) - .with_level(true) - .compact(); - - // This limits the logger to emits only rpxy crate - let level_string = std::env::var(EnvFilter::DEFAULT_ENV).unwrap_or_else(|_| "info".to_string()); - let filter_layer = EnvFilter::new(format!("{}={}", env!("CARGO_PKG_NAME"), level_string)); - // let filter_layer = EnvFilter::from_default_env(); - - tracing_subscriber::registry() - .with(format_layer) - .with(filter_layer) - .init(); -} diff --git a/src/proxy/crypto_service.rs b/rpxy-lib/src/proxy/crypto_service.rs similarity index 100% rename from src/proxy/crypto_service.rs rename to rpxy-lib/src/proxy/crypto_service.rs diff --git a/src/proxy/mod.rs b/rpxy-lib/src/proxy/mod.rs similarity index 100% rename from src/proxy/mod.rs rename to rpxy-lib/src/proxy/mod.rs diff --git a/src/proxy/proxy_client_cert.rs b/rpxy-lib/src/proxy/proxy_client_cert.rs similarity index 100% rename from src/proxy/proxy_client_cert.rs rename to rpxy-lib/src/proxy/proxy_client_cert.rs diff --git a/src/proxy/proxy_h3.rs b/rpxy-lib/src/proxy/proxy_h3.rs similarity index 100% rename from src/proxy/proxy_h3.rs rename to rpxy-lib/src/proxy/proxy_h3.rs diff --git a/src/proxy/proxy_main.rs b/rpxy-lib/src/proxy/proxy_main.rs similarity index 100% rename from src/proxy/proxy_main.rs rename to rpxy-lib/src/proxy/proxy_main.rs diff --git a/src/proxy/proxy_tls.rs b/rpxy-lib/src/proxy/proxy_tls.rs similarity index 100% rename from src/proxy/proxy_tls.rs rename to rpxy-lib/src/proxy/proxy_tls.rs diff --git a/src/utils/bytes_name.rs b/rpxy-lib/src/utils/bytes_name.rs similarity index 98% rename from src/utils/bytes_name.rs rename to rpxy-lib/src/utils/bytes_name.rs index a093c41..5d2fef5 100644 --- a/src/utils/bytes_name.rs +++ b/rpxy-lib/src/utils/bytes_name.rs @@ -23,6 +23,9 @@ impl PathNameBytesExp { pub fn len(&self) -> usize { self.0.len() } + pub fn is_empty(&self) -> bool { + self.0.len() == 0 + } pub fn get(&self, index: I) -> Option<&I::Output> where I: std::slice::SliceIndex<[u8]>, diff --git a/src/utils/mod.rs b/rpxy-lib/src/utils/mod.rs similarity index 100% rename from src/utils/mod.rs rename to rpxy-lib/src/utils/mod.rs diff --git a/src/utils/socket_addr.rs b/rpxy-lib/src/utils/socket_addr.rs similarity index 100% rename from src/utils/socket_addr.rs rename to rpxy-lib/src/utils/socket_addr.rs From f6c4032f833986383d69069679260e5a09146535 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 21 Jul 2023 22:07:36 +0900 Subject: [PATCH 22/43] refactor: cleanup codes --- rpxy-bin/Cargo.toml | 8 +- rpxy-bin/src/cert_file_reader.rs | 10 +- rpxy-bin/src/config/mod.rs | 2 +- rpxy-bin/src/config/parse.rs | 44 +++----- rpxy-bin/src/config/toml.rs | 116 +++++++++----------- rpxy-bin/src/main.rs | 11 +- rpxy-lib/src/error.rs | 6 ++ rpxy-lib/src/globals.rs | 178 ++++++++++++++++++++++++++++++- rpxy-lib/src/lib.rs | 21 ++-- rpxy-lib/src/proxy/proxy_tls.rs | 8 +- 10 files changed, 280 insertions(+), 124 deletions(-) diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 9f65325..90094c4 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -12,6 +12,8 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] +default = ["http3"] +http3 = [] [dependencies] rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] } @@ -28,14 +30,12 @@ tokio = { version = "1.29.1", default-features = false, features = [ "macros", ] } async-trait = "0.1.72" +rustls-pemfile = "1.0.3" # config clap = { version = "4.3.17", features = ["std", "cargo", "wrap_help"] } toml = { version = "0.7.6", default-features = false, features = ["parse"] } - -# reloading certs -hot_reload = "0.1.2" -rustls-pemfile = "1.0.3" +# hot_reload = "0.1.2" # logging tracing = { version = "0.1.37" } diff --git a/rpxy-bin/src/cert_file_reader.rs b/rpxy-bin/src/cert_file_reader.rs index ffe3099..0a6a14f 100644 --- a/rpxy-bin/src/cert_file_reader.rs +++ b/rpxy-bin/src/cert_file_reader.rs @@ -150,8 +150,8 @@ mod tests { use super::*; #[tokio::test] async fn read_server_crt_key_files() { - let tls_cert_path = "example-certs/server.crt"; - let tls_cert_key_path = "example-certs/server.key"; + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; let crypto_file_source = CryptoFileSourceBuilder::default() .tls_cert_key_path(tls_cert_key_path) .tls_cert_path(tls_cert_path) @@ -165,9 +165,9 @@ mod tests { #[tokio::test] async fn read_server_crt_key_files_with_client_ca_crt() { - let tls_cert_path = "example-certs/server.crt"; - let tls_cert_key_path = "example-certs/server.key"; - let client_ca_cert_path = Some("example-certs/client.ca.crt".to_string()); + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let client_ca_cert_path = Some("../example-certs/client.ca.crt".to_string()); let crypto_file_source = CryptoFileSourceBuilder::default() .tls_cert_key_path(tls_cert_key_path) .tls_cert_path(tls_cert_path) diff --git a/rpxy-bin/src/config/mod.rs b/rpxy-bin/src/config/mod.rs index 54b2600..a71ca6e 100644 --- a/rpxy-bin/src/config/mod.rs +++ b/rpxy-bin/src/config/mod.rs @@ -1,4 +1,4 @@ mod parse; mod toml; -pub use parse::build_globals; +pub use parse::build_settings; diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 27c8cd8..109dc4b 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -5,10 +5,9 @@ use crate::{ log::*, }; use clap::Arg; -use rpxy_lib::{Backends, BytesName, Globals, ProxyConfig}; -use tokio::runtime::Handle; +use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig}; -pub fn build_globals(runtime_handle: Handle) -> std::result::Result, anyhow::Error> { +pub fn build_settings() -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { let _ = include_str!("../../Cargo.toml"); let options = clap::command!().arg( Arg::new("config_file") @@ -76,39 +75,22 @@ pub fn build_globals(runtime_handle: Handle) -> std::result::Result>::new(); + + // let mut backends = Backends::new(); for (app_name, app) in apps.0.iter() { let server_name_string = app.server_name.as_ref().ok_or(anyhow!("No server name"))?; - let backend = app.try_into()?; - backends.apps.insert(server_name_string.to_server_name_vec(), backend); + let app_config = app.try_into()?; + app_config_list_inner.push(app_config); info!("Registering application: {} ({})", app_name, server_name_string); } - // default backend application for plaintext http requests - if let Some(d) = config.default_app { - let d_sn: Vec<&str> = backends - .apps - .iter() - .filter(|(_k, v)| v.app_name == d) - .map(|(_, v)| v.server_name.as_ref()) - .collect(); - if !d_sn.is_empty() { - info!( - "Serving plaintext http for requests to unconfigured server_name by app {} (server_name: {}).", - d, d_sn[0] - ); - backends.default_server_name_bytes = Some(d_sn[0].to_server_name_vec()); - } - } - - /////////////////////////////////// - let globals = Globals { - proxy_config, - backends, - request_count: Default::default(), - runtime_handle, + let app_config_list = AppConfigList { + inner: app_config_list_inner, + default_app: config.default_app, // default backend application for plaintext http requests }; - Ok(globals) + Ok((proxy_config, app_config_list)) + // todo!() } diff --git a/rpxy-bin/src/config/toml.rs b/rpxy-bin/src/config/toml.rs index c73f8c1..55faf93 100644 --- a/rpxy-bin/src/config/toml.rs +++ b/rpxy-bin/src/config/toml.rs @@ -3,10 +3,7 @@ use crate::{ constants::*, error::{anyhow, ensure}, }; -use rpxy_lib::{ - reexports::Uri, Backend, BackendBuilder, PathNameBytesExp, ProxyConfig, ReverseProxy, Upstream, UpstreamGroup, - UpstreamGroupBuilder, UpstreamOption, -}; +use rpxy_lib::{reexports::Uri, AppConfig, ProxyConfig, ReverseProxyConfig, TlsConfig, UpstreamUri}; use rustc_hash::FxHashMap as HashMap; use serde::Deserialize; use std::{fs, net::SocketAddr}; @@ -148,8 +145,7 @@ impl TryInto for &ConfigToml { if x == 0u64 { proxy_config.h3_max_idle_timeout = None; } else { - proxy_config.h3_max_idle_timeout = - Some(quinn::IdleTimeout::try_from(tokio::time::Duration::from_secs(x)).unwrap()) + proxy_config.h3_max_idle_timeout = Some(tokio::time::Duration::from_secs(x)) } } } @@ -172,101 +168,87 @@ impl ConfigToml { } } -impl TryInto> for &Application { +impl TryInto> for &Application { type Error = anyhow::Error; - fn try_into(self) -> std::result::Result, Self::Error> { + fn try_into(self) -> std::result::Result, Self::Error> { let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; - // backend builder - let mut backend_builder = BackendBuilder::default(); // reverse proxy settings - let reverse_proxy = self.try_into()?; + let reverse_proxy_config: Vec = self.try_into()?; - backend_builder - .app_name(server_name_string) - .server_name(server_name_string) - .reverse_proxy(reverse_proxy); - - // TLS settings and build backend instance - let backend = if self.tls.is_none() { - backend_builder.build()? - } else { + // tls settings + let tls_config = if self.tls.is_some() { let tls = self.tls.as_ref().unwrap(); ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some()); - - let https_redirection = if tls.https_redirection.is_none() { - Some(true) // Default true - } else { - tls.https_redirection - }; - - let crypto_source = CryptoFileSourceBuilder::default() + let inner = CryptoFileSourceBuilder::default() .tls_cert_path(tls.tls_cert_path.as_ref().unwrap()) .tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap()) .client_ca_cert_path(&tls.client_ca_cert_path) .build()?; - backend_builder - .https_redirection(https_redirection) - .crypto_source(Some(crypto_source)) - .build()? + let https_redirection = if tls.https_redirection.is_none() { + true // Default true + } else { + tls.https_redirection.unwrap() + }; + + Some(TlsConfig { + inner, + https_redirection, + }) + } else { + None }; - Ok(backend) + + Ok(AppConfig { + server_name: server_name_string.to_owned(), + reverse_proxy: reverse_proxy_config, + tls: tls_config, + }) } } -impl TryInto for &Application { +impl TryInto> for &Application { type Error = anyhow::Error; - fn try_into(self) -> std::result::Result { - let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; + fn try_into(self) -> std::result::Result, Self::Error> { + let _server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; let rp_settings = self.reverse_proxy.as_ref().ok_or(anyhow!("Missing reverse_proxy"))?; - let mut upstream: HashMap = HashMap::default(); + let mut reverse_proxies: Vec = Vec::new(); - rp_settings.iter().for_each(|rpo| { - let upstream_vec: Vec = rpo.upstream.iter().map(|x| x.try_into().unwrap()).collect(); - // let upstream_iter = rpo.upstream.iter().map(|x| x.to_upstream().unwrap()); - // let lb_upstream_num = vec_upstream.len(); - let elem = UpstreamGroupBuilder::default() - .upstream(&upstream_vec) - .path(&rpo.path) - .replace_path(&rpo.replace_path) - .lb(&rpo.load_balance, &upstream_vec, server_name_string, &rpo.path) - .opts(&rpo.upstream_options) - .build() - .unwrap(); + for rpo in rp_settings.iter() { + let upstream_res: Vec> = rpo.upstream.iter().map(|v| v.try_into().ok()).collect(); + if !upstream_res.iter().all(|v| v.is_some()) { + return Err(anyhow!("[{}] Upstream uri is invalid", &_server_name_string)); + } + let upstream = upstream_res.into_iter().map(|v| v.unwrap()).collect(); - upstream.insert(elem.path.clone(), elem); - }); - ensure!( - rp_settings.iter().filter(|rpo| rpo.path.is_none()).count() < 2, - "Multiple default reverse proxy setting" - ); - ensure!( - upstream - .iter() - .all(|(_, elem)| !(elem.opts.contains(&UpstreamOption::ConvertHttpsTo11) - && elem.opts.contains(&UpstreamOption::ConvertHttpsTo2))), - "either one of force_http11 or force_http2 can be enabled" - ); + reverse_proxies.push(ReverseProxyConfig { + path: rpo.path.clone(), + replace_path: rpo.replace_path.clone(), + upstream, + upstream_options: rpo.upstream_options.clone(), + load_balance: rpo.load_balance.clone(), + }) + } - Ok(ReverseProxy { upstream }) + Ok(reverse_proxies) } } -impl TryInto for &UpstreamParams { +impl TryInto for &UpstreamParams { type Error = anyhow::Error; - fn try_into(self) -> std::result::Result { + fn try_into(self) -> std::result::Result { let scheme = match self.tls { Some(true) => "https", _ => "http", }; let location = format!("{}://{}", scheme, self.location); - Ok(Upstream { - uri: location.parse::().map_err(|e| anyhow!("{}", e))?, + Ok(UpstreamUri { + inner: location.parse::().map_err(|e| anyhow!("{}", e))?, }) } } diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index cd74116..ed0cba5 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -11,9 +11,8 @@ mod constants; mod error; mod log; -use crate::{cert_file_reader::CryptoFileSource, config::build_globals, log::*}; -use rpxy_lib::{entrypoint, Globals}; -use std::sync::Arc; +use crate::{config::build_settings, log::*}; +use rpxy_lib::entrypoint; fn main() { init_logger(); @@ -24,7 +23,7 @@ fn main() { let runtime = runtime_builder.build().unwrap(); runtime.block_on(async { - let globals: Globals = match build_globals(runtime.handle().clone()) { + let (proxy_conf, app_conf) = match build_settings() { Ok(g) => g, Err(e) => { error!("Invalid configuration: {}", e); @@ -32,7 +31,9 @@ fn main() { } }; - entrypoint(Arc::new(globals)).await.unwrap() + entrypoint(proxy_conf, app_conf, runtime.handle().clone()) + .await + .unwrap() }); warn!("rpxy exited!"); } diff --git a/rpxy-lib/src/error.rs b/rpxy-lib/src/error.rs index 187c993..3407e8a 100644 --- a/rpxy-lib/src/error.rs +++ b/rpxy-lib/src/error.rs @@ -10,9 +10,15 @@ pub enum RpxyError { #[error("Proxy build error")] ProxyBuild(#[from] crate::proxy::ProxyBuilderError), + #[error("Backend build error")] + BackendBuild(#[from] crate::backend::BackendBuilderError), + #[error("MessageHandler build error")] HandlerBuild(#[from] crate::handler::HttpMessageHandlerBuilderError), + #[error("Config builder error: {0}")] + ConfigBuild(&'static str), + #[error("Http Message Handler Error: {0}")] Handler(&'static str), diff --git a/rpxy-lib/src/globals.rs b/rpxy-lib/src/globals.rs index 5150bff..f97bc3b 100644 --- a/rpxy-lib/src/globals.rs +++ b/rpxy-lib/src/globals.rs @@ -1,4 +1,14 @@ -use crate::{backend::Backends, certs::CryptoSource, constants::*}; +use crate::{ + backend::{ + Backend, BackendBuilder, Backends, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption, + }, + certs::CryptoSource, + constants::*, + error::RpxyError, + log::*, + utils::{BytesName, PathNameBytesExp}, +}; +use rustc_hash::FxHashMap as HashMap; use std::net::SocketAddr; use std::sync::{ atomic::{AtomicUsize, Ordering}, @@ -26,6 +36,7 @@ where } /// Configuration parameters for proxy transport and request handlers +#[derive(PartialEq, Eq)] pub struct ProxyConfig { pub listen_sockets: Vec, // when instantiate server pub http_port: Option, // when instantiate server @@ -54,7 +65,7 @@ pub struct ProxyConfig { #[cfg(feature = "http3")] pub h3_max_concurrent_connections: u32, #[cfg(feature = "http3")] - pub h3_max_idle_timeout: Option, + pub h3_max_idle_timeout: Option, } impl Default for ProxyConfig { @@ -87,11 +98,172 @@ impl Default for ProxyConfig { #[cfg(feature = "http3")] h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM.into(), #[cfg(feature = "http3")] - h3_max_idle_timeout: Some(quinn::IdleTimeout::try_from(Duration::from_secs(H3::MAX_IDLE_TIMEOUT)).unwrap()), + h3_max_idle_timeout: Some(Duration::from_secs(H3::MAX_IDLE_TIMEOUT)), } } } +/// Configuration parameters for backend applications +#[derive(PartialEq, Eq)] +pub struct AppConfigList +where + T: CryptoSource, +{ + pub inner: Vec>, + pub default_app: Option, +} +impl TryInto> for AppConfigList +where + T: CryptoSource + Clone, +{ + type Error = RpxyError; + + fn try_into(self) -> Result, Self::Error> { + let mut backends = Backends::new(); + for app_config in self.inner.iter() { + let backend = app_config.try_into()?; + backends + .apps + .insert(app_config.server_name.clone().to_server_name_vec(), backend); + info!("Registering application: ({})", &app_config.server_name); + } + + // default backend application for plaintext http requests + if let Some(d) = self.default_app { + let d_sn: Vec<&str> = backends + .apps + .iter() + .filter(|(_k, v)| v.app_name == d) + .map(|(_, v)| v.server_name.as_ref()) + .collect(); + if !d_sn.is_empty() { + info!( + "Serving plaintext http for requests to unconfigured server_name by app {} (server_name: {}).", + d, d_sn[0] + ); + backends.default_server_name_bytes = Some(d_sn[0].to_server_name_vec()); + } + } + Ok(backends) + } +} + +/// Configuration parameters for single backend application +#[derive(PartialEq, Eq)] +pub struct AppConfig +where + T: CryptoSource, +{ + pub server_name: String, + pub reverse_proxy: Vec, + pub tls: Option>, +} +impl TryInto> for &AppConfig +where + T: CryptoSource + Clone, +{ + type Error = RpxyError; + + fn try_into(self) -> Result, Self::Error> { + // backend builder + let mut backend_builder = BackendBuilder::default(); + // reverse proxy settings + let reverse_proxy = self.try_into()?; + + backend_builder + .app_name(self.server_name.clone()) + .server_name(self.server_name.clone()) + .reverse_proxy(reverse_proxy); + + // TLS settings and build backend instance + let backend = if self.tls.is_none() { + backend_builder.build().map_err(RpxyError::BackendBuild)? + } else { + let tls = self.tls.as_ref().unwrap(); + + backend_builder + .https_redirection(Some(tls.https_redirection)) + .crypto_source(Some(tls.inner.clone())) + .build()? + }; + Ok(backend) + } +} +impl TryInto for &AppConfig +where + T: CryptoSource + Clone, +{ + type Error = RpxyError; + + fn try_into(self) -> Result { + let mut upstream: HashMap = HashMap::default(); + + self.reverse_proxy.iter().for_each(|rpo| { + let upstream_vec: Vec = rpo.upstream.iter().map(|x| x.try_into().unwrap()).collect(); + // let upstream_iter = rpo.upstream.iter().map(|x| x.to_upstream().unwrap()); + // let lb_upstream_num = vec_upstream.len(); + let elem = UpstreamGroupBuilder::default() + .upstream(&upstream_vec) + .path(&rpo.path) + .replace_path(&rpo.replace_path) + .lb(&rpo.load_balance, &upstream_vec, &self.server_name, &rpo.path) + .opts(&rpo.upstream_options) + .build() + .unwrap(); + + upstream.insert(elem.path.clone(), elem); + }); + if self.reverse_proxy.iter().filter(|rpo| rpo.path.is_none()).count() >= 2 { + error!("Multiple default reverse proxy setting"); + return Err(RpxyError::ConfigBuild("Invalid reverse proxy setting")); + } + + if !(upstream.iter().all(|(_, elem)| { + !(elem.opts.contains(&UpstreamOption::ConvertHttpsTo11) && elem.opts.contains(&UpstreamOption::ConvertHttpsTo2)) + })) { + error!("Either one of force_http11 or force_http2 can be enabled"); + return Err(RpxyError::ConfigBuild("Invalid upstream option setting")); + } + + Ok(ReverseProxy { upstream }) + } +} + +/// Configuration parameters for single reverse proxy corresponding to the path +#[derive(PartialEq, Eq)] +pub struct ReverseProxyConfig { + pub path: Option, + pub replace_path: Option, + pub upstream: Vec, + pub upstream_options: Option>, + pub load_balance: Option, +} + +/// Configuration parameters for single upstream destination from a reverse proxy +#[derive(PartialEq, Eq)] +pub struct UpstreamUri { + pub inner: hyper::Uri, +} +impl TryInto for &UpstreamUri { + type Error = anyhow::Error; + + fn try_into(self) -> std::result::Result { + Ok(Upstream { + uri: self.inner.clone(), + }) + } +} + +/// Configuration parameters on TLS for a single backend application +#[derive(PartialEq, Eq)] +pub struct TlsConfig +where + T: CryptoSource, +{ + pub inner: T, + pub https_redirection: bool, +} + #[derive(Debug, Clone, Default)] /// Counter for serving requests pub struct RequestCount(Arc); diff --git a/rpxy-lib/src/lib.rs b/rpxy-lib/src/lib.rs index 7d7764a..7820db6 100644 --- a/rpxy-lib/src/lib.rs +++ b/rpxy-lib/src/lib.rs @@ -8,19 +8,15 @@ mod log; mod proxy; mod utils; -use crate::{error::*, handler::HttpMessageHandlerBuilder, log::*, proxy::ProxyBuilder}; +use crate::{error::*, globals::Globals, handler::HttpMessageHandlerBuilder, log::*, proxy::ProxyBuilder}; use futures::future::select_all; use hyper::Client; // use hyper_trust_dns::TrustDnsResolver; use std::sync::Arc; pub use crate::{ - backend::{ - Backend, BackendBuilder, Backends, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption, - }, certs::{CertsAndKeys, CryptoSource}, - globals::{Globals, ProxyConfig}, // TODO: BackendConfigに変える - utils::{BytesName, PathNameBytesExp}, + globals::{AppConfig, AppConfigList, ProxyConfig, ReverseProxyConfig, TlsConfig, UpstreamUri}, }; pub mod reexports { pub use hyper::Uri; @@ -28,10 +24,21 @@ pub mod reexports { } /// Entrypoint that creates and spawns tasks of reverse proxy services -pub async fn entrypoint(globals: Arc>) -> Result<()> +pub async fn entrypoint( + proxy_config: ProxyConfig, + app_config_list: AppConfigList, + runtime_handle: tokio::runtime::Handle, +) -> Result<()> where T: CryptoSource + Clone + Send + Sync + 'static, { + // build global + let globals = Arc::new(Globals { + proxy_config, + backends: app_config_list.try_into()?, + request_count: Default::default(), + runtime_handle, + }); // let connector = TrustDnsResolver::default().into_rustls_webpki_https_connector(); let connector = hyper_rustls::HttpsConnectorBuilder::new() .with_webpki_roots() diff --git a/rpxy-lib/src/proxy/proxy_tls.rs b/rpxy-lib/src/proxy/proxy_tls.rs index 5e846f0..6e0a6b6 100644 --- a/rpxy-lib/src/proxy/proxy_tls.rs +++ b/rpxy-lib/src/proxy/proxy_tls.rs @@ -119,7 +119,13 @@ where transport_config_quic .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream) .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream) - .max_idle_timeout(self.globals.proxy_config.h3_max_idle_timeout); + .max_idle_timeout( + self + .globals + .proxy_config + .h3_max_idle_timeout + .map(|v| quinn::IdleTimeout::try_from(v).unwrap()), + ); let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config)); server_config_h3.transport = Arc::new(transport_config_quic); From 5e76c2b055c64120cf30b9834bbd0933d8c40ebe Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 21 Jul 2023 22:42:47 +0900 Subject: [PATCH 23/43] fix: bug of default_app for cleartext http --- rpxy-bin/src/config/parse.rs | 8 ++++---- rpxy-bin/src/config/toml.rs | 7 +++---- rpxy-lib/src/globals.rs | 8 ++++++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 109dc4b..09b24b3 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -80,15 +80,15 @@ pub fn build_settings() -> std::result::Result<(ProxyConfig, AppConfigList> for &Application { - type Error = anyhow::Error; - - fn try_into(self) -> std::result::Result, Self::Error> { +impl Application { + pub fn build_app_config(&self, app_name: &str) -> std::result::Result, anyhow::Error> { let server_name_string = self.server_name.as_ref().ok_or(anyhow!("Missing server_name"))?; // reverse proxy settings @@ -202,6 +200,7 @@ impl TryInto> for &Application { }; Ok(AppConfig { + app_name: app_name.to_owned(), server_name: server_name_string.to_owned(), reverse_proxy: reverse_proxy_config, tls: tls_config, diff --git a/rpxy-lib/src/globals.rs b/rpxy-lib/src/globals.rs index f97bc3b..6614b5c 100644 --- a/rpxy-lib/src/globals.rs +++ b/rpxy-lib/src/globals.rs @@ -125,7 +125,10 @@ where backends .apps .insert(app_config.server_name.clone().to_server_name_vec(), backend); - info!("Registering application: ({})", &app_config.server_name); + info!( + "Registering application {} ({})", + &app_config.server_name, &app_config.app_name + ); } // default backend application for plaintext http requests @@ -154,6 +157,7 @@ pub struct AppConfig where T: CryptoSource, { + pub app_name: String, pub server_name: String, pub reverse_proxy: Vec, pub tls: Option>, @@ -171,7 +175,7 @@ where let reverse_proxy = self.try_into()?; backend_builder - .app_name(self.server_name.clone()) + .app_name(self.app_name.clone()) .server_name(self.server_name.clone()) .reverse_proxy(reverse_proxy); From 58e22d33afe04115e1594915aca8c80ac8bf01f5 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 23 Jul 2023 01:42:39 +0900 Subject: [PATCH 24/43] feat: hot-reloading of config file --- config-example.toml | 3 ++ rpxy-bin/Cargo.toml | 2 +- rpxy-bin/src/config/mod.rs | 7 ++- rpxy-bin/src/config/parse.rs | 25 ++++++----- rpxy-bin/src/config/service.rs | 24 ++++++++++ rpxy-bin/src/config/toml.rs | 22 +++++---- rpxy-bin/src/constants.rs | 1 + rpxy-bin/src/main.rs | 76 +++++++++++++++++++++++++++----- rpxy-lib/Cargo.toml | 4 +- rpxy-lib/src/constants.rs | 1 + rpxy-lib/src/globals.rs | 14 +++--- rpxy-lib/src/lib.rs | 16 +++---- rpxy-lib/src/proxy/mod.rs | 1 + rpxy-lib/src/proxy/proxy_main.rs | 7 +-- rpxy-lib/src/proxy/proxy_tls.rs | 27 ++++++++---- rpxy-lib/src/proxy/socket.rs | 41 +++++++++++++++++ 16 files changed, 213 insertions(+), 58 deletions(-) create mode 100644 rpxy-bin/src/config/service.rs create mode 100644 rpxy-lib/src/proxy/socket.rs diff --git a/config-example.toml b/config-example.toml index 0382393..605067c 100644 --- a/config-example.toml +++ b/config-example.toml @@ -10,6 +10,9 @@ listen_port = 8080 listen_port_tls = 8443 +# Optional for h2 and http1.1 +tcp_listen_backlog = 1024 + # Optional for h2 and http1.1 max_concurrent_streams = 100 diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 90094c4..fb6d4aa 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -35,7 +35,7 @@ rustls-pemfile = "1.0.3" # config clap = { version = "4.3.17", features = ["std", "cargo", "wrap_help"] } toml = { version = "0.7.6", default-features = false, features = ["parse"] } -# hot_reload = "0.1.2" +hot_reload = "0.1.4" # logging tracing = { version = "0.1.37" } diff --git a/rpxy-bin/src/config/mod.rs b/rpxy-bin/src/config/mod.rs index a71ca6e..09ec2b9 100644 --- a/rpxy-bin/src/config/mod.rs +++ b/rpxy-bin/src/config/mod.rs @@ -1,4 +1,9 @@ mod parse; +mod service; mod toml; -pub use parse::build_settings; +pub use { + self::toml::ConfigToml, + parse::{build_settings, parse_opts}, + service::ConfigTomlReloader, +}; diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 09b24b3..56c6f2c 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -7,28 +7,30 @@ use crate::{ use clap::Arg; use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig}; -pub fn build_settings() -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { +pub fn parse_opts() -> Result { let _ = include_str!("../../Cargo.toml"); let options = clap::command!().arg( Arg::new("config_file") .long("config") .short('c') .value_name("FILE") - .help("Configuration file path like \"./config.toml\""), + .required(true) + .help("Configuration file path like ./config.toml"), ); let matches = options.get_matches(); /////////////////////////////////// - let config = if let Some(config_file_path) = matches.get_one::("config_file") { - ConfigToml::new(config_file_path)? - } else { - // Default config Toml - ConfigToml::default() - }; + let config_file_path = matches.get_one::("config_file").unwrap(); + Ok(config_file_path.to_string()) +} + +pub fn build_settings( + config: &ConfigToml, +) -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { /////////////////////////////////// // build proxy config - let proxy_config: ProxyConfig = (&config).try_into()?; + let proxy_config: ProxyConfig = config.try_into()?; // For loggings if proxy_config.listen_sockets.iter().any(|addr| addr.is_ipv6()) { info!("Listen both IPv4 and IPv6") @@ -50,7 +52,7 @@ pub fn build_settings() -> std::result::Result<(ProxyConfig, AppConfigList std::result::Result<(ProxyConfig, AppConfigList for ConfigTomlReloader { + type Source = String; + async fn new(source: &Self::Source) -> Result> { + Ok(Self { + config_path: source.clone(), + }) + } + + async fn reload(&self) -> Result, ReloaderError> { + let conf = ConfigToml::new(&self.config_path) + .map_err(|_e| ReloaderError::::Reload("Failed to reload config toml"))?; + Ok(Some(conf)) + } +} diff --git a/rpxy-bin/src/config/toml.rs b/rpxy-bin/src/config/toml.rs index 6868d21..84260c0 100644 --- a/rpxy-bin/src/config/toml.rs +++ b/rpxy-bin/src/config/toml.rs @@ -8,11 +8,12 @@ use rustc_hash::FxHashMap as HashMap; use serde::Deserialize; use std::{fs, net::SocketAddr}; -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct ConfigToml { pub listen_port: Option, pub listen_port_tls: Option, pub listen_ipv6: Option, + pub tcp_listen_backlog: Option, pub max_concurrent_streams: Option, pub max_clients: Option, pub apps: Option, @@ -21,7 +22,7 @@ pub struct ConfigToml { } #[cfg(feature = "http3")] -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct Http3Option { pub alt_svc_max_age: Option, pub request_max_body_size: Option, @@ -31,24 +32,24 @@ pub struct Http3Option { pub max_idle_timeout: Option, } -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct Experimental { #[cfg(feature = "http3")] pub h3: Option, pub ignore_sni_consistency: Option, } -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct Apps(pub HashMap); -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct Application { pub server_name: Option, pub reverse_proxy: Option>, pub tls: Option, } -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct TlsOption { pub tls_cert_path: Option, pub tls_cert_key_path: Option, @@ -56,7 +57,7 @@ pub struct TlsOption { pub client_ca_cert_path: Option, } -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct ReverseProxyOption { pub path: Option, pub replace_path: Option, @@ -65,7 +66,7 @@ pub struct ReverseProxyOption { pub load_balance: Option, } -#[derive(Deserialize, Debug, Default)] +#[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)] pub struct UpstreamParams { pub location: String, pub tls: Option, @@ -112,6 +113,11 @@ impl TryInto for &ConfigToml { }) .collect(); + // tcp backlog + if let Some(backlog) = self.tcp_listen_backlog { + proxy_config.tcp_listen_backlog = backlog; + } + // max values if let Some(c) = self.max_clients { proxy_config.max_clients = c as usize; diff --git a/rpxy-bin/src/constants.rs b/rpxy-bin/src/constants.rs index 4181a26..323615f 100644 --- a/rpxy-bin/src/constants.rs +++ b/rpxy-bin/src/constants.rs @@ -1,2 +1,3 @@ pub const LISTEN_ADDRESSES_V4: &[&str] = &["0.0.0.0"]; pub const LISTEN_ADDRESSES_V6: &[&str] = &["[::]"]; +pub const CONFIG_WATCH_DELAY_SECS: u32 = 20; diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index ed0cba5..8466372 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -11,7 +11,12 @@ mod constants; mod error; mod log; -use crate::{config::build_settings, log::*}; +use crate::{ + config::{build_settings, parse_opts, ConfigToml, ConfigTomlReloader}, + constants::CONFIG_WATCH_DELAY_SECS, + log::*, +}; +use hot_reload::{ReloaderReceiver, ReloaderService}; use rpxy_lib::entrypoint; fn main() { @@ -23,17 +28,68 @@ fn main() { let runtime = runtime_builder.build().unwrap(); runtime.block_on(async { - let (proxy_conf, app_conf) = match build_settings() { - Ok(g) => g, - Err(e) => { - error!("Invalid configuration: {}", e); + // Initially load config + let Ok(config_path) = parse_opts() else { + error!("Invalid toml file"); std::process::exit(1); - } }; + let (config_service, config_rx) = + ReloaderService::::new(&config_path, CONFIG_WATCH_DELAY_SECS, false) + .await + .unwrap(); - entrypoint(proxy_conf, app_conf, runtime.handle().clone()) - .await - .unwrap() + tokio::select! { + _ = config_service.start() => { + error!("config reloader service exited"); + } + _ = rpxy_service(config_rx, runtime.handle().clone()) => { + error!("rpxy service existed"); + } + } }); - warn!("rpxy exited!"); +} + +async fn rpxy_service( + mut config_rx: ReloaderReceiver, + runtime_handle: tokio::runtime::Handle, +) -> Result<(), anyhow::Error> { + // Initial loading + config_rx.changed().await?; + let config_toml = config_rx.borrow().clone().unwrap(); + let (mut proxy_conf, mut app_conf) = match build_settings(&config_toml) { + Ok(v) => v, + Err(e) => { + error!("Invalid configuration: {e}"); + return Err(anyhow::anyhow!(e)); + } + }; + + // Continuous monitoring + loop { + tokio::select! { + _ = entrypoint(&proxy_conf, &app_conf, &runtime_handle) => { + error!("rpxy entrypoint exited"); + break; + } + _ = config_rx.changed() => { + if config_rx.borrow().is_none() { + error!("Something wrong in config reloader receiver"); + break; + } + let config_toml = config_rx.borrow().clone().unwrap(); + match build_settings(&config_toml) { + Ok((p, a)) => { + (proxy_conf, app_conf) = (p, a) + }, + Err(e) => { + error!("Invalid configuration. Configuration does not updated: {e}"); + continue; + } + }; + info!("Configuration updated. Force to re-bind TCP/UDP sockets"); + } + else => break + } + } + Ok(()) } diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index 48cb437..e36bae6 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -30,7 +30,7 @@ tokio = { version = "1.29.1", default-features = false, features = [ "macros", ] } async-trait = "0.1.72" -hot_reload = "0.1.2" # reloading certs +hot_reload = "0.1.4" # reloading certs # Error handling anyhow = "1.0.72" @@ -63,6 +63,8 @@ quinn = { path = "../quinn/quinn", optional = true } # Tentative to support rust h3 = { path = "../h3/h3/", optional = true } # h3-quinn = { path = "./h3/h3-quinn/", optional = true } h3-quinn = { path = "../h3-quinn/", optional = true } # Tentative to support rustls-0.21 +# for UDP socket wit SO_REUSEADDR +socket2 = { version = "0.5.3", features = ["all"] } # cookie handling for sticky cookie chrono = { version = "0.4.26", default-features = false, features = [ diff --git a/rpxy-lib/src/constants.rs b/rpxy-lib/src/constants.rs index 72cce78..9d7fb5e 100644 --- a/rpxy-lib/src/constants.rs +++ b/rpxy-lib/src/constants.rs @@ -1,5 +1,6 @@ // pub const LISTEN_ADDRESSES_V4: &[&str] = &["0.0.0.0"]; // pub const LISTEN_ADDRESSES_V6: &[&str] = &["[::]"]; +pub const TCP_LISTEN_BACKLOG: u32 = 1024; // pub const HTTP_LISTEN_PORT: u16 = 8080; // pub const HTTPS_LISTEN_PORT: u16 = 8443; pub const PROXY_TIMEOUT_SEC: u64 = 60; diff --git a/rpxy-lib/src/globals.rs b/rpxy-lib/src/globals.rs index 6614b5c..44808dd 100644 --- a/rpxy-lib/src/globals.rs +++ b/rpxy-lib/src/globals.rs @@ -36,11 +36,12 @@ where } /// Configuration parameters for proxy transport and request handlers -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone)] pub struct ProxyConfig { pub listen_sockets: Vec, // when instantiate server pub http_port: Option, // when instantiate server pub https_port: Option, // when instantiate server + pub tcp_listen_backlog: u32, // when instantiate server pub proxy_timeout: Duration, // when serving requests at Proxy pub upstream_timeout: Duration, // when serving requests at Handler @@ -74,6 +75,7 @@ impl Default for ProxyConfig { listen_sockets: Vec::new(), http_port: None, https_port: None, + tcp_listen_backlog: TCP_LISTEN_BACKLOG, // TODO: Reconsider each timeout values proxy_timeout: Duration::from_secs(PROXY_TIMEOUT_SEC), @@ -104,7 +106,7 @@ impl Default for ProxyConfig { } /// Configuration parameters for backend applications -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone)] pub struct AppConfigList where T: CryptoSource, @@ -152,7 +154,7 @@ where } /// Configuration parameters for single backend application -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone)] pub struct AppConfig where T: CryptoSource, @@ -234,7 +236,7 @@ where } /// Configuration parameters for single reverse proxy corresponding to the path -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone)] pub struct ReverseProxyConfig { pub path: Option, pub replace_path: Option, @@ -244,7 +246,7 @@ pub struct ReverseProxyConfig { } /// Configuration parameters for single upstream destination from a reverse proxy -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone)] pub struct UpstreamUri { pub inner: hyper::Uri, } @@ -259,7 +261,7 @@ impl TryInto for &UpstreamUri { } /// Configuration parameters on TLS for a single backend application -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone)] pub struct TlsConfig where T: CryptoSource, diff --git a/rpxy-lib/src/lib.rs b/rpxy-lib/src/lib.rs index 7820db6..4fd64ce 100644 --- a/rpxy-lib/src/lib.rs +++ b/rpxy-lib/src/lib.rs @@ -25,19 +25,19 @@ pub mod reexports { /// Entrypoint that creates and spawns tasks of reverse proxy services pub async fn entrypoint( - proxy_config: ProxyConfig, - app_config_list: AppConfigList, - runtime_handle: tokio::runtime::Handle, + proxy_config: &ProxyConfig, + app_config_list: &AppConfigList, + runtime_handle: &tokio::runtime::Handle, ) -> Result<()> where T: CryptoSource + Clone + Send + Sync + 'static, { // build global let globals = Arc::new(Globals { - proxy_config, - backends: app_config_list.try_into()?, + proxy_config: proxy_config.clone(), + backends: app_config_list.clone().try_into()?, request_count: Default::default(), - runtime_handle, + runtime_handle: runtime_handle.clone(), }); // let connector = TrustDnsResolver::default().into_rustls_webpki_https_connector(); let connector = hyper_rustls::HttpsConnectorBuilder::new() @@ -71,8 +71,8 @@ where })); // wait for all future - if let (Ok(_), _, _) = futures.await { - error!("Some proxy services are down"); + if let (Ok(Err(e)), _, _) = futures.await { + error!("Some proxy services are down: {:?}", e); }; Ok(()) diff --git a/rpxy-lib/src/proxy/mod.rs b/rpxy-lib/src/proxy/mod.rs index 73a4002..749239c 100644 --- a/rpxy-lib/src/proxy/mod.rs +++ b/rpxy-lib/src/proxy/mod.rs @@ -4,5 +4,6 @@ mod proxy_client_cert; mod proxy_h3; mod proxy_main; mod proxy_tls; +mod socket; pub use proxy_main::{Proxy, ProxyBuilder, ProxyBuilderError}; diff --git a/rpxy-lib/src/proxy/proxy_main.rs b/rpxy-lib/src/proxy/proxy_main.rs index e5a02a5..166f048 100644 --- a/rpxy-lib/src/proxy/proxy_main.rs +++ b/rpxy-lib/src/proxy/proxy_main.rs @@ -1,4 +1,4 @@ -// use super::proxy_handler::handle_request; +use super::socket::bind_tcp_socket; use crate::{ certs::CryptoSource, error::*, globals::Globals, handler::HttpMessageHandler, log::*, utils::ServerNameBytesExp, }; @@ -7,7 +7,6 @@ use hyper::{client::connect::Connect, server::conn::Http, service::service_fn, B use std::{net::SocketAddr, sync::Arc}; use tokio::{ io::{AsyncRead, AsyncWrite}, - net::TcpListener, runtime::Handle, time::{timeout, Duration}, }; @@ -94,7 +93,9 @@ where async fn start_without_tls(self, server: Http) -> Result<()> { let listener_service = async { - let tcp_listener = TcpListener::bind(&self.listening_on).await?; + let tcp_socket = bind_tcp_socket(&self.listening_on)?; + let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?; + // let tcp_listener = TcpListener::bind(&self.listening_on).await?; info!("Start TCP proxy serving with HTTP request for configured host names"); while let Ok((stream, _client_addr)) = tcp_listener.accept().await { self.clone().client_serve(stream, server.clone(), _client_addr, None); diff --git a/rpxy-lib/src/proxy/proxy_tls.rs b/rpxy-lib/src/proxy/proxy_tls.rs index 6e0a6b6..5512eff 100644 --- a/rpxy-lib/src/proxy/proxy_tls.rs +++ b/rpxy-lib/src/proxy/proxy_tls.rs @@ -1,6 +1,9 @@ +#[cfg(feature = "http3")] +use super::socket::bind_udp_socket; use super::{ crypto_service::{CryptoReloader, ServerCrypto, ServerCryptoBase, SniServerCryptoMap}, proxy_main::{LocalExecutor, Proxy}, + socket::bind_tcp_socket, }; use crate::{certs::CryptoSource, constants::*, error::*, log::*, utils::BytesName}; use hot_reload::{ReloaderReceiver, ReloaderService}; @@ -10,10 +13,7 @@ use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerC #[cfg(feature = "http3")] use rustls::ServerConfig; use std::sync::Arc; -use tokio::{ - net::TcpListener, - time::{timeout, Duration}, -}; +use tokio::time::{timeout, Duration}; impl Proxy where @@ -26,7 +26,8 @@ where server: Http, mut server_crypto_rx: ReloaderReceiver, ) -> Result<()> { - let tcp_listener = TcpListener::bind(&self.listening_on).await?; + let tcp_socket = bind_tcp_socket(&self.listening_on)?; + let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?; info!("Start TCP proxy serving with HTTPS request for configured host names"); let mut server_crypto_map: Option> = None; @@ -130,7 +131,17 @@ where let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config)); server_config_h3.transport = Arc::new(transport_config_quic); server_config_h3.concurrent_connections(self.globals.proxy_config.h3_max_concurrent_connections); - let endpoint = Endpoint::server(server_config_h3, self.listening_on)?; + + // To reuse address + let udp_socket = bind_udp_socket(&self.listening_on)?; + let runtime = quinn::default_runtime() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No async runtime found"))?; + let endpoint = Endpoint::new( + quinn::EndpointConfig::default(), + Some(server_config_h3), + udp_socket, + runtime, + )?; let mut server_crypto: Option> = None; loop { @@ -199,10 +210,10 @@ where #[cfg(not(feature = "http3"))] { tokio::select! { - _= self.cert_service(tx) => { + _= cert_reloader_service.start() => { error!("Cert service for TLS exited"); }, - _ = self.listener_service(server, rx) => { + _ = self.listener_service(server, cert_reloader_rx) => { error!("TCP proxy service for TLS exited"); }, else => { diff --git a/rpxy-lib/src/proxy/socket.rs b/rpxy-lib/src/proxy/socket.rs new file mode 100644 index 0000000..8858732 --- /dev/null +++ b/rpxy-lib/src/proxy/socket.rs @@ -0,0 +1,41 @@ +use crate::{error::*, log::*}; +#[cfg(feature = "http3")] +use socket2::{Domain, Protocol, Socket, Type}; +use std::net::SocketAddr; +#[cfg(feature = "http3")] +use std::net::UdpSocket; +use tokio::net::TcpSocket; + +pub(super) fn bind_tcp_socket(listening_on: &SocketAddr) -> Result { + let tcp_socket = if listening_on.is_ipv6() { + TcpSocket::new_v6() + } else { + TcpSocket::new_v4() + }?; + tcp_socket.set_reuseaddr(true)?; + tcp_socket.set_reuseport(true)?; + if let Err(e) = tcp_socket.bind(*listening_on) { + error!("Failed to bind TCP socket: {}", e); + return Err(RpxyError::Io(e)); + }; + Ok(tcp_socket) +} + +#[cfg(feature = "http3")] +pub(super) fn bind_udp_socket(listening_on: &SocketAddr) -> Result { + let socket = if listening_on.is_ipv6() { + Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP)) + } else { + Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP)) + }?; + socket.set_reuse_address(true)?; + socket.set_reuse_port(true)?; + + if let Err(e) = socket.bind(&(*listening_on).into()) { + error!("Failed to bind UDP socket: {}", e); + return Err(RpxyError::Io(e)); + }; + let udp_socket: UdpSocket = socket.into(); + + Ok(udp_socket) +} From 16290f3173c28f7b5ea01e1f3976af715a25e637 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 23 Jul 2023 01:48:33 +0900 Subject: [PATCH 25/43] refactor --- rpxy-bin/src/config/parse.rs | 18 ------------------ rpxy-bin/src/main.rs | 1 + rpxy-lib/src/lib.rs | 19 +++++++++++++++++++ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 56c6f2c..02f5207 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -31,24 +31,6 @@ pub fn build_settings( /////////////////////////////////// // build proxy config let proxy_config: ProxyConfig = config.try_into()?; - // For loggings - if proxy_config.listen_sockets.iter().any(|addr| addr.is_ipv6()) { - info!("Listen both IPv4 and IPv6") - } else { - info!("Listen IPv4") - } - if proxy_config.http_port.is_some() { - info!("Listen port: {}", proxy_config.http_port.unwrap()); - } - if proxy_config.https_port.is_some() { - info!("Listen port: {} (for TLS)", proxy_config.https_port.unwrap()); - } - if proxy_config.http3 { - info!("Experimental HTTP/3.0 is enabled. Note it is still very unstable."); - } - if !proxy_config.sni_consistency { - info!("Ignore consistency between TLS SNI and Host header (or Request line). Note it violates RFC."); - } /////////////////////////////////// // backend_apps diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index 8466372..fabfdf3 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -53,6 +53,7 @@ async fn rpxy_service( mut config_rx: ReloaderReceiver, runtime_handle: tokio::runtime::Handle, ) -> Result<(), anyhow::Error> { + info!("Start rpxy service"); // Initial loading config_rx.changed().await?; let config_toml = config_rx.borrow().clone().unwrap(); diff --git a/rpxy-lib/src/lib.rs b/rpxy-lib/src/lib.rs index 4fd64ce..72f8a8a 100644 --- a/rpxy-lib/src/lib.rs +++ b/rpxy-lib/src/lib.rs @@ -32,6 +32,25 @@ pub async fn entrypoint( where T: CryptoSource + Clone + Send + Sync + 'static, { + // For initial message logging + if proxy_config.listen_sockets.iter().any(|addr| addr.is_ipv6()) { + info!("Listen both IPv4 and IPv6") + } else { + info!("Listen IPv4") + } + if proxy_config.http_port.is_some() { + info!("Listen port: {}", proxy_config.http_port.unwrap()); + } + if proxy_config.https_port.is_some() { + info!("Listen port: {} (for TLS)", proxy_config.https_port.unwrap()); + } + if proxy_config.http3 { + info!("Experimental HTTP/3.0 is enabled. Note it is still very unstable."); + } + if !proxy_config.sni_consistency { + info!("Ignore consistency between TLS SNI and Host header (or Request line). Note it violates RFC."); + } + // build global let globals = Arc::new(Globals { proxy_config: proxy_config.clone(), From e2b2ae0729df86efcf2341475d93bcd2e222cc02 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 23 Jul 2023 01:48:51 +0900 Subject: [PATCH 26/43] refactor --- rpxy-bin/src/config/parse.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 02f5207..3f0d4b8 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -2,7 +2,6 @@ use super::toml::ConfigToml; use crate::{ cert_file_reader::CryptoFileSource, error::{anyhow, ensure}, - log::*, }; use clap::Arg; use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig}; From c58da361cac4a011bfc2e3ea34af265959762a1a Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 23 Jul 2023 01:54:39 +0900 Subject: [PATCH 27/43] update todo and changelog --- CHANGELOG.md | 10 ++++++++++ TODO.md | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dea263..8511a16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## 0.4.0 (unreleased) +### Improvement + +- Feat: Continuous watching on a specified config file and hot-reloading the file when updated +- Feat: Enabled to specify TCP listen backlog in the config file +- Refactor: Split `rpxy` into `rpxy-lib` and `rpxy-bin` +- Refactor: lots of minor improvements + +### Bugfix + +- Fix bug to apply default backend application ## 0.3.0 diff --git a/TODO.md b/TODO.md index 81af1d5..3425d11 100644 --- a/TODO.md +++ b/TODO.md @@ -10,7 +10,8 @@ - upstream/upstream group: information on targeted destinations for each set of (a domain + a path) - load-balance: load balancing mod for a domain + path - - Split `rpxy` source codes into `rpxy-lib` and `rpxy-bin` to make the core part (reverse proxy) isolated from the misc part like toml file loader. This is in order to make the configuration-related part more flexible (related to [#33](https://github.com/junkurihara/rust-rpxy/issues/33)) + - **Almost done in version 0.4.0**: + Split `rpxy` source codes into `rpxy-lib` and `rpxy-bin` to make the core part (reverse proxy) isolated from the misc part like toml file loader. This is in order to make the configuration-related part more flexible (related to [#33](https://github.com/junkurihara/rust-rpxy/issues/33)) - Cache option for the response with `Cache-Control: public` header directive ([#55](https://github.com/junkurihara/rust-rpxy/issues/55)) - Consideration on migrating from `quinn` and `h3-quinn` to other QUIC implementations ([#57](https://github.com/junkurihara/rust-rpxy/issues/57)) From 302766987484be8ed55094f4f5518cef064c2182 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 23 Jul 2023 02:19:38 +0900 Subject: [PATCH 28/43] deps --- rpxy-bin/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index fb6d4aa..f4bdc68 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -33,7 +33,7 @@ async-trait = "0.1.72" rustls-pemfile = "1.0.3" # config -clap = { version = "4.3.17", features = ["std", "cargo", "wrap_help"] } +clap = { version = "4.3.19", features = ["std", "cargo", "wrap_help"] } toml = { version = "0.7.6", default-features = false, features = ["parse"] } hot_reload = "0.1.4" From 7c0945a5124418aa9a1024568c1989bb77cf312f Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 24 Jul 2023 15:52:54 +0900 Subject: [PATCH 29/43] submodule --- h3 | 2 +- quinn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/h3 b/h3 index dccb3cd..a57ed22 160000 --- a/h3 +++ b/h3 @@ -1 +1 @@ -Subproject commit dccb3cdae9d5a9d720fae5f774b53f0bd8a16019 +Subproject commit a57ed224ac5d17a635eb71eb6f83c1196f581a51 diff --git a/quinn b/quinn index e652b6d..0ae7c60 160000 --- a/quinn +++ b/quinn @@ -1 +1 @@ -Subproject commit e652b6d999f053ffe21eeea247854882ae480281 +Subproject commit 0ae7c60b15637d7343410ba1e5cc3151e3814557 From 7f52dce23d4985246908628240bd584ae20ffb00 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 24 Jul 2023 20:08:18 +0900 Subject: [PATCH 30/43] add benchmark result and fix Dockerfile names since it does not depends on architecture --- .github/workflows/docker_build_push.yml | 11 +- bench/README.md | 183 ++++++++++++++++++++++-- bench/bench.amd64.sh | 53 +++++++ bench/bench.sh | 2 + bench/docker-compose.amd64.yml | 96 +++++++++++++ bench/docker-compose.yml | 4 +- bench/nginx.conf | 4 +- bench/sozu-config.toml | 16 +++ docker/{Dockerfile.amd64 => Dockerfile} | 1 - docker/Dockerfile.amd64-slim | 1 - docker/docker-compose.yml | 2 +- 11 files changed, 354 insertions(+), 19 deletions(-) create mode 100644 bench/bench.amd64.sh create mode 100644 bench/docker-compose.amd64.yml create mode 100644 bench/sozu-config.toml rename docker/{Dockerfile.amd64 => Dockerfile} (98%) diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 1dfd260..305aa45 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -30,7 +30,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Release build and push x86_64 + - name: Release build and push if: ${{ env.BRANCH == 'main' }} uses: docker/build-push-action@v4 with: @@ -38,7 +38,8 @@ jobs: push: true tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest - file: ./docker/Dockerfile.amd64 + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 - name: Release build and push x86_64-slim if: ${{ env.BRANCH == 'main' }} @@ -49,8 +50,9 @@ jobs: tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim file: ./docker/Dockerfile.amd64-slim + platforms: linux/amd64 - - name: Nightly build and push x86_64 + - name: Nightly build and push if: ${{ env.BRANCH == 'develop' }} uses: docker/build-push-action@v4 with: @@ -58,4 +60,5 @@ jobs: push: true tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly - file: ./docker/Dockerfile.amd64 + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 diff --git a/bench/README.md b/bench/README.md index 7a98c81..55ca4dd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,23 +1,25 @@ -# Sample Benchmark Result - -Done at Jul. 15, 2023 +# Sample Benchmark Results This test simply measures the performance of several reverse proxy through HTTP/1.1 by the following command using [`rewrk`](https://github.com/lnx-search/rewrk). -```bash +```sh: $ rewrk -c 512 -t 4 -d 15s -h http://localhost:8080 --pct ``` -## Environment +## Tests on `linux/arm64/v8` + +Done at Jul. 15, 2023 + +### Environment - `rpxy` commit id: `1da7e5bfb77d1ce4ee8d6cfc59b1c725556fc192` - Docker Desktop 4.21.1 (114176) -- ReWrk 0.3.1 +- ReWrk 0.3.2 - Macbook Pro '14 (2021, M1 Max, 64GB RAM) +The docker images of `nginx` and `caddy` for `linux/arm64/v8` are pulled from the official registry. - -## Result +### Result for `rpxy`, `nginx` and `caddy` ``` ---------------------------- @@ -94,3 +96,168 @@ Benchmarking 512 connections @ http://localhost:8100 for 15 second(s) 708 Errors: error shutting down connection: Socket is not connected (os error 57) ``` + +## Results on `linux/amd64` + +Done at Jul. 24, 2023 + +### Environment + +- `rpxy` commit id: `7c0945a5124418aa9a1024568c1989bb77cf312f` +- Docker Desktop 4.21.1 (114176) +- ReWrk 0.3.2 and Wrk 0.4.2 +- iMac '27 (2020, 10-Core Intel Core i9, 128GB RAM) + +The docker images of `nginx` and `caddy` for `linux/amd64` were pulled from the official registry. For `Sozu`, the official docker image from its developers was still version 0.11.0 (currently the latest version is 0.15.2). So we built it by ourselves locally using the `Sozu`'s official [`Dockerfile`](https://github.com/sozu-proxy/sozu/blob/main/Dockerfile). + +Also, when `Sozu` is configured as an HTTP reverse proxy, it cannot handle HTTP request messages emit from `ReWrk` due to hostname parsing errors though it can correctly handle messages dispatched from `curl` and browsers. So, we additionally test using [`Wrk`](https://github.com/wg/wrk) to examine `Sozu` with the following command. + +```sh: +$ wrk -c 512 -t 4 -d 15s http://localhost:8110 +``` + + + +### Result + +#### With ReWrk for `rpxy`, `nginx` and `caddy` + +``` +---------------------------- +Benchmark [x86_64] with ReWrk +---------------------------- +Benchmark on rpxy +Beginning round 1... +Benchmarking 512 connections @ http://localhost:8080 for 15 second(s) + Latencies: + Avg Stdev Min Max + 20.37ms 8.95ms 1.63ms 160.27ms + Requests: + Total: 376345 Req/Sec: 25095.19 + Transfer: + Total: 295.61 MB Transfer Rate: 19.71 MB/Sec ++ --------------- + --------------- + +| Percentile | Avg Latency | ++ --------------- + --------------- + +| 99.9% | 112.50ms | +| 99% | 61.33ms | +| 95% | 44.26ms | +| 90% | 38.74ms | +| 75% | 32.00ms | +| 50% | 26.82ms | ++ --------------- + --------------- + + +626 Errors: error shutting down connection: Socket is not connected (os error 57) + +sleep 3 secs +---------------------------- +Benchmark on nginx +Beginning round 1... +Benchmarking 512 connections @ http://localhost:8090 for 15 second(s) + Latencies: + Avg Stdev Min Max + 23.45ms 12.42ms 1.18ms 154.44ms + Requests: + Total: 326685 Req/Sec: 21784.73 + Transfer: + Total: 265.22 MB Transfer Rate: 17.69 MB/Sec ++ --------------- + --------------- + +| Percentile | Avg Latency | ++ --------------- + --------------- + +| 99.9% | 96.85ms | +| 99% | 73.93ms | +| 95% | 57.57ms | +| 90% | 50.36ms | +| 75% | 40.57ms | +| 50% | 32.70ms | ++ --------------- + --------------- + + +657 Errors: error shutting down connection: Socket is not connected (os error 57) + +sleep 3 secs +---------------------------- +Benchmark on caddy +Beginning round 1... +Benchmarking 512 connections @ http://localhost:8100 for 15 second(s) + Latencies: + Avg Stdev Min Max + 45.71ms 50.47ms 0.88ms 908.49ms + Requests: + Total: 166917 Req/Sec: 11129.80 + Transfer: + Total: 133.77 MB Transfer Rate: 8.92 MB/Sec ++ --------------- + --------------- + +| Percentile | Avg Latency | ++ --------------- + --------------- + +| 99.9% | 608.92ms | +| 99% | 351.18ms | +| 95% | 210.56ms | +| 90% | 162.68ms | +| 75% | 106.97ms | +| 50% | 73.90ms | ++ --------------- + --------------- + + +646 Errors: error shutting down connection: Socket is not connected (os error 57) + +sleep 3 secs +``` + +#### With Wrk for `rpxy`, `nginx`, `caddy` and `sozu` + +``` +---------------------------- +Benchmark [x86_64] with Wrk +---------------------------- +Benchmark on rpxy +Running 15s test @ http://localhost:8080 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 18.68ms 8.09ms 122.64ms 74.03% + Req/Sec 6.95k 815.23 8.45k 83.83% + 414819 requests in 15.01s, 326.37MB read + Socket errors: connect 0, read 608, write 0, timeout 0 +Requests/sec: 27627.79 +Transfer/sec: 21.74MB + +sleep 3 secs +---------------------------- +Benchmark on nginx +Running 15s test @ http://localhost:8090 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 23.34ms 13.80ms 126.06ms 74.66% + Req/Sec 5.71k 607.41 7.07k 73.17% + 341127 requests in 15.03s, 277.50MB read + Socket errors: connect 0, read 641, write 0, timeout 0 +Requests/sec: 22701.54 +Transfer/sec: 18.47MB + +sleep 3 secs +---------------------------- +Benchmark on caddy +Running 15s test @ http://localhost:8100 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 54.19ms 55.63ms 674.53ms 88.55% + Req/Sec 2.92k 1.40k 5.57k 56.17% + 174748 requests in 15.03s, 140.61MB read + Socket errors: connect 0, read 660, write 0, timeout 0 + Non-2xx or 3xx responses: 70 +Requests/sec: 11624.63 +Transfer/sec: 9.35MB + +sleep 3 secs +---------------------------- +Benchmark on sozu +Running 15s test @ http://localhost:8110 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 19.78ms 4.89ms 98.09ms 76.88% + Req/Sec 6.49k 824.75 8.11k 76.17% + 387744 requests in 15.02s, 329.11MB read + Socket errors: connect 0, read 647, write 0, timeout 0 +Requests/sec: 25821.93 +Transfer/sec: 21.92MB +``` diff --git a/bench/bench.amd64.sh b/bench/bench.amd64.sh new file mode 100644 index 0000000..de73016 --- /dev/null +++ b/bench/bench.amd64.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +echo "----------------------------" +echo "Benchmark [x86_64] with ReWrk" + +echo "----------------------------" +echo "Benchmark on rpxy" +rewrk -c 512 -t 4 -d 15s -h http://localhost:8080 --pct + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on nginx" +rewrk -c 512 -t 4 -d 15s -h http://localhost:8090 --pct + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on caddy" +rewrk -c 512 -t 4 -d 15s -h http://localhost:8100 --pct + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark [x86_64] with Wrk" + +echo "----------------------------" +echo "Benchmark on rpxy" +wrk -c 512 -t 4 -d 15s http://localhost:8080 + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on nginx" +wrk -c 512 -t 4 -d 15s http://localhost:8090 + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on caddy" +wrk -c 512 -t 4 -d 15s http://localhost:8100 + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on sozu" +wrk -c 512 -t 4 -d 15s http://localhost:8110 diff --git a/bench/bench.sh b/bench/bench.sh index 0d5477f..912f5fa 100644 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -12,6 +12,8 @@ # echo "Benchmark on caddy" # ab -c 100 -n 10000 http://127.0.0.1:8100/index.html +echo "----------------------------" +echo "Benchmark [Any Arch]" echo "----------------------------" echo "Benchmark on rpxy" diff --git a/bench/docker-compose.amd64.yml b/bench/docker-compose.amd64.yml new file mode 100644 index 0000000..a440138 --- /dev/null +++ b/bench/docker-compose.amd64.yml @@ -0,0 +1,96 @@ +version: "3" +services: + nginx: + image: nginx:alpine + container_name: backend-nginx + restart: unless-stopped + environment: + - VIRTUAL_HOST=localhost + - VIRTUAL_PORT=80 + expose: + - 80 + # ports: + # - 127.0.0.1:8888:80 + logging: + options: + max-size: "10m" + max-file: "3" + networks: + bench-nw: + ipv4_address: 192.168.100.100 + + rpxy-rp: + image: jqtype/rpxy + container_name: proxy-rpxy + pull_policy: never + build: + context: ../ + dockerfile: docker/Dockerfile + restart: unless-stopped + environment: + - LOG_LEVEL=info + - LOG_TO_FILE=false + ports: + - 127.0.0.1:8080:8080 + tty: false + volumes: + - ./rpxy.toml:/etc/rpxy.toml:ro + networks: + bench-nw: + + nginx-rp: + image: nginx:alpine + container_name: proxy-nginx + ports: + - 127.0.0.1:8090:80 + restart: unless-stopped + tty: false + privileged: true + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - /var/run/docker.sock:/tmp/docker.sock:ro + logging: + options: + max-size: "10m" + max-file: "3" + networks: + bench-nw: + + caddy-rp: + image: caddy:2 + container_name: proxy-caddy + ports: + - 127.0.0.1:8100:80 + restart: unless-stopped + tty: false + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + networks: + bench-nw: + + # Sozu wokrs only in X86_64 (amd64) environment + # Official image from sozu developers is still version 0.11.0. + # So we built it by ourselves locally. + sozu-rp: + image: jqtype/sozu + container_name: proxy-sozu + restart: unless-stopped + ports: + - 127.0.0.1:8110:80 + logging: + options: + max-size: "10m" + max-file: "3" + volumes: + - ./sozu-config.toml:/etc/sozu/config.toml + networks: + bench-nw: + +networks: + bench-nw: + name: bench-nw + driver: bridge + ipam: + driver: default + config: + - subnet: 192.168.100.0/24 diff --git a/bench/docker-compose.yml b/bench/docker-compose.yml index 23308aa..7b2b043 100644 --- a/bench/docker-compose.yml +++ b/bench/docker-compose.yml @@ -10,7 +10,7 @@ services: expose: - 80 # ports: - # - 127.0.0.1:8888:80 + # - 127.0.0.1:8888:80 logging: options: max-size: "10m" @@ -25,7 +25,7 @@ services: pull_policy: never build: context: ../ - dockerfile: docker/Dockerfile.amd64 + dockerfile: docker/Dockerfile restart: unless-stopped environment: - LOG_LEVEL=info diff --git a/bench/nginx.conf b/bench/nginx.conf index 0359ac6..0ef89e3 100644 --- a/bench/nginx.conf +++ b/bench/nginx.conf @@ -59,7 +59,7 @@ # return 503; # } # localhost -upstream localhost { +upstream backend { ## Can be connected with "bench-nw" network # backend-nginx server 192.168.100.100:80; @@ -69,6 +69,6 @@ server { listen 80 ; # access_log /var/log/nginx/access.log vhost; location / { - proxy_pass http://localhost; + proxy_pass http://backend; } } diff --git a/bench/sozu-config.toml b/bench/sozu-config.toml new file mode 100644 index 0000000..ab76515 --- /dev/null +++ b/bench/sozu-config.toml @@ -0,0 +1,16 @@ +log_level = "info" +log_target = "stdout" +max_connections = 512 +activate_listeners = true + +[[listeners]] +protocol = "http" +# listening address +address = "0.0.0.0:80" + +[clusters] + +[clusters.backend] +protocol = "http" +frontends = [{ address = "0.0.0.0:80", hostname = "localhost" }] +backends = [{ address = "192.168.100.100:80" }] diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile similarity index 98% rename from docker/Dockerfile.amd64 rename to docker/Dockerfile index da27439..fd6d510 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile @@ -29,7 +29,6 @@ RUN apt-get update && apt-get install -qy --no-install-recommends $BUILD_DEPS && ######################################## FROM base AS runner -ENV TAG_NAME=amd64 ENV RUNTIME_DEPS logrotate ca-certificates gosu RUN apt-get update && \ diff --git a/docker/Dockerfile.amd64-slim b/docker/Dockerfile.amd64-slim index fb0246e..12cd261 100644 --- a/docker/Dockerfile.amd64-slim +++ b/docker/Dockerfile.amd64-slim @@ -18,7 +18,6 @@ RUN echo "Building rpxy from source" && \ FROM alpine:latest as runner LABEL maintainer="Jun Kurihara" -ENV TAG_NAME=amd64-slim ENV TARGET_DIR=x86_64-unknown-linux-musl ENV RUNTIME_DEPS logrotate ca-certificates su-exec diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 716d0de..d817918 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,7 +10,7 @@ services: - 127.0.0.1:8443:8443 build: context: ../ - dockerfile: ./docker/Dockerfile.amd64 + dockerfile: ./docker/Dockerfile environment: - LOG_LEVEL=debug - LOG_TO_FILE=true From 411fbaf29683b0a5fc8c942d6c0c3ee06ae6c820 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 24 Jul 2023 21:53:01 +0900 Subject: [PATCH 31/43] feat: add option to activate continuous monitoring on config file --- ...d_push.yml => docker_build_push_amd64.yml} | 6 +- .github/workflows/docker_build_push_arm64.yml | 64 ++++++++++++++++++ CHANGELOG.md | 2 + README.md | 16 ++++- docker/Dockerfile.arm64-slim | 45 +++++++++++++ rpxy-bin/src/config/parse.rs | 43 ++++++++---- rpxy-bin/src/main.rs | 66 +++++++++++++++---- rpxy-lib/src/proxy/socket.rs | 4 ++ 8 files changed, 216 insertions(+), 30 deletions(-) rename .github/workflows/{docker_build_push.yml => docker_build_push_amd64.yml} (93%) create mode 100644 .github/workflows/docker_build_push_arm64.yml create mode 100644 docker/Dockerfile.arm64-slim diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push_amd64.yml similarity index 93% rename from .github/workflows/docker_build_push.yml rename to .github/workflows/docker_build_push_amd64.yml index 305aa45..fa016a0 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push_amd64.yml @@ -1,4 +1,4 @@ -name: Build and Publish Docker +name: Build and Publish Docker x86_64 on: push: @@ -39,7 +39,7 @@ jobs: tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest file: ./docker/Dockerfile - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 - name: Release build and push x86_64-slim if: ${{ env.BRANCH == 'main' }} @@ -61,4 +61,4 @@ jobs: tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly file: ./docker/Dockerfile - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 diff --git a/.github/workflows/docker_build_push_arm64.yml b/.github/workflows/docker_build_push_arm64.yml new file mode 100644 index 0000000..b31581c --- /dev/null +++ b/.github/workflows/docker_build_push_arm64.yml @@ -0,0 +1,64 @@ +name: Build and Publish Docker Aarch64 + +on: + push: + branches: + - main + - develop + +jobs: + build_and_push: + runs-on: ubuntu-latest + env: + IMAGE_NAME: rpxy + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: GitHub Environment + run: echo "BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Release build and push + if: ${{ env.BRANCH == 'main' }} + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest + file: ./docker/Dockerfile + platforms: linux/arm64 + + - name: Release build and push x86_64-slim + if: ${{ env.BRANCH == 'main' }} + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim + file: ./docker/Dockerfile.arm64-slim + platforms: linux/arm64 + + - name: Nightly build and push + if: ${{ env.BRANCH == 'develop' }} + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly + file: ./docker/Dockerfile + platforms: linux/arm64 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8511a16..3a11094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Feat: Continuous watching on a specified config file and hot-reloading the file when updated - Feat: Enabled to specify TCP listen backlog in the config file +- Feat: Add a GitHub action to build `arm64` docker image. +- Bench: Add benchmark result on `amd64` architecture. - Refactor: Split `rpxy` into `rpxy-lib` and `rpxy-bin` - Refactor: lots of minor improvements diff --git a/README.md b/README.md index 561141a..a3d8795 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,20 @@ You can run `rpxy` with a configuration file like % ./target/release/rpxy --config config.toml ``` +If you specify `-w` option along with the config file path, `rpxy` tracks the change of `config.toml` in the real-time manner and apply the change immediately without restarting the process. + +The full help messages are given follows. + +```bash: +usage: rpxy [OPTIONS] --config + +Options: + -c, --config Configuration file path like ./config.toml + -w, --watch Activate dynamic reloading of the config file via continuous monitoring + -h, --help Print help + -V, --version Print version +``` + That's all! ## Basic Configuration @@ -217,7 +231,7 @@ Since it is currently a work-in-progress project, we are frequently adding new o ## Using Docker Image -You can also use [docker image](https://hub.docker.com/r/jqtype/rpxy) instead of directly executing the binary. There are only two docker-specific environment variables. +You can also use [docker image](https://hub.docker.com/r/jqtype/rpxy) instead of directly executing the binary. There are only several docker-specific environment variables. - `HOST_USER` (default: `user`): User name executing `rpxy` inside the container. - `HOST_UID` (default: `900`): `UID` of `HOST_USER`. diff --git a/docker/Dockerfile.arm64-slim b/docker/Dockerfile.arm64-slim new file mode 100644 index 0000000..83b2d16 --- /dev/null +++ b/docker/Dockerfile.arm64-slim @@ -0,0 +1,45 @@ +######################################## +FROM messense/rust-musl-cross:aarch64-musl as builder + +ENV TARGET_DIR=aarch64-unknown-linux-musl +ENV CFLAGS=-Ofast + +WORKDIR /tmp + +COPY . /tmp/ + +ENV RUSTFLAGS "-C link-arg=-s" + +RUN echo "Building rpxy from source" && \ + cargo build --release && \ + musl-strip --strip-all /tmp/target/${TARGET_DIR}/release/rpxy + +######################################## +FROM alpine:latest as runner +LABEL maintainer="Jun Kurihara" + +ENV TARGET_DIR=aarch64-unknown-linux-musl +ENV RUNTIME_DEPS logrotate ca-certificates su-exec + +RUN apk add --no-cache ${RUNTIME_DEPS} && \ + update-ca-certificates && \ + find / -type d -path /proc -prune -o -type f -perm /u+s -exec chmod u-s {} \; && \ + find / -type d -path /proc -prune -o -type f -perm /g+s -exec chmod g-s {} \; && \ + mkdir -p /rpxy/bin &&\ + mkdir -p /rpxy/log + +COPY --from=builder /tmp/target/${TARGET_DIR}/release/rpxy /rpxy/bin/rpxy +COPY ./docker/run.sh /rpxy +COPY ./docker/entrypoint.sh /rpxy + +RUN chmod +x /rpxy/run.sh && \ + chmod +x /rpxy/entrypoint.sh + +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +ENV SSL_CERT_DIR=/etc/ssl/certs + +EXPOSE 80 443 + +CMD ["/rpxy/entrypoint.sh"] + +ENTRYPOINT ["/rpxy/entrypoint.sh"] diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 3f0d4b8..15ff240 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -3,25 +3,44 @@ use crate::{ cert_file_reader::CryptoFileSource, error::{anyhow, ensure}, }; -use clap::Arg; +use clap::{Arg, ArgAction}; use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig}; -pub fn parse_opts() -> Result { +/// Parsed options +pub struct Opts { + pub config_file_path: String, + pub watch: bool, +} + +/// Parse arg values passed from cli +pub fn parse_opts() -> Result { let _ = include_str!("../../Cargo.toml"); - let options = clap::command!().arg( - Arg::new("config_file") - .long("config") - .short('c') - .value_name("FILE") - .required(true) - .help("Configuration file path like ./config.toml"), - ); + let options = clap::command!() + .arg( + Arg::new("config_file") + .long("config") + .short('c') + .value_name("FILE") + .required(true) + .help("Configuration file path like ./config.toml"), + ) + .arg( + Arg::new("watch") + .long("watch") + .short('w') + .action(ArgAction::SetTrue) + .help("Activate dynamic reloading of the config file via continuous monitoring"), + ); let matches = options.get_matches(); /////////////////////////////////// - let config_file_path = matches.get_one::("config_file").unwrap(); + let config_file_path = matches.get_one::("config_file").unwrap().to_owned(); + let watch = matches.get_one::("watch").unwrap().to_owned(); - Ok(config_file_path.to_string()) + Ok(Opts { + config_file_path, + watch, + }) } pub fn build_settings( diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index fabfdf3..8fe00dc 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -28,32 +28,69 @@ fn main() { let runtime = runtime_builder.build().unwrap(); runtime.block_on(async { - // Initially load config - let Ok(config_path) = parse_opts() else { + // Initially load options + let Ok(parsed_opts) = parse_opts() else { error!("Invalid toml file"); std::process::exit(1); }; - let (config_service, config_rx) = - ReloaderService::::new(&config_path, CONFIG_WATCH_DELAY_SECS, false) - .await - .unwrap(); - tokio::select! { - _ = config_service.start() => { - error!("config reloader service exited"); + if !parsed_opts.watch { + if let Err(e) = rpxy_service_without_watcher(&parsed_opts.config_file_path, runtime.handle().clone()).await { + error!("rpxy service existed: {e}"); + std::process::exit(1); } - _ = rpxy_service(config_rx, runtime.handle().clone()) => { - error!("rpxy service existed"); + } else { + let (config_service, config_rx) = ReloaderService::::new( + &parsed_opts.config_file_path, + CONFIG_WATCH_DELAY_SECS, + false, + ) + .await + .unwrap(); + + tokio::select! { + Err(e) = config_service.start() => { + error!("config reloader service exited: {e}"); + std::process::exit(1); + } + Err(e) = rpxy_service_with_watcher(config_rx, runtime.handle().clone()) => { + error!("rpxy service existed: {e}"); + std::process::exit(1); + } } } }); } -async fn rpxy_service( - mut config_rx: ReloaderReceiver, +async fn rpxy_service_without_watcher( + config_file_path: &str, runtime_handle: tokio::runtime::Handle, ) -> Result<(), anyhow::Error> { info!("Start rpxy service"); + let config_toml = match ConfigToml::new(config_file_path) { + Ok(v) => v, + Err(e) => { + error!("Invalid toml file: {e}"); + std::process::exit(1); + } + }; + let (proxy_conf, app_conf) = match build_settings(&config_toml) { + Ok(v) => v, + Err(e) => { + error!("Invalid configuration: {e}"); + return Err(anyhow::anyhow!(e)); + } + }; + entrypoint(&proxy_conf, &app_conf, &runtime_handle) + .await + .map_err(|e| anyhow::anyhow!(e)) +} + +async fn rpxy_service_with_watcher( + mut config_rx: ReloaderReceiver, + runtime_handle: tokio::runtime::Handle, +) -> Result<(), anyhow::Error> { + info!("Start rpxy service with dynamic config reloader"); // Initial loading config_rx.changed().await?; let config_toml = config_rx.borrow().clone().unwrap(); @@ -92,5 +129,6 @@ async fn rpxy_service( else => break } } - Ok(()) + + Err(anyhow::anyhow!("rpxy or continuous monitoring service exited")) } diff --git a/rpxy-lib/src/proxy/socket.rs b/rpxy-lib/src/proxy/socket.rs index 8858732..48f72e9 100644 --- a/rpxy-lib/src/proxy/socket.rs +++ b/rpxy-lib/src/proxy/socket.rs @@ -6,6 +6,8 @@ use std::net::SocketAddr; use std::net::UdpSocket; use tokio::net::TcpSocket; +/// Bind TCP socket to the given `SocketAddr`, and returns the TCP socket with `SO_REUSEADDR` and `SO_REUSEPORT` options. +/// This option is required to re-bind the socket address when the proxy instance is reconstructed. pub(super) fn bind_tcp_socket(listening_on: &SocketAddr) -> Result { let tcp_socket = if listening_on.is_ipv6() { TcpSocket::new_v6() @@ -22,6 +24,8 @@ pub(super) fn bind_tcp_socket(listening_on: &SocketAddr) -> Result { } #[cfg(feature = "http3")] +/// Bind UDP socket to the given `SocketAddr`, and returns the UDP socket with `SO_REUSEADDR` and `SO_REUSEPORT` options. +/// This option is required to re-bind the socket address when the proxy instance is reconstructed. pub(super) fn bind_udp_socket(listening_on: &SocketAddr) -> Result { let socket = if listening_on.is_ipv6() { Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP)) From 7116c0203254762e1c72eb89316849bd69024373 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 24 Jul 2023 22:02:40 +0900 Subject: [PATCH 32/43] docs: todo and readme --- README.md | 3 ++- TODO.md | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a3d8795..85b3ab9 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) ![Unit Test](https://github.com/junkurihara/rust-rpxy/actions/workflows/ci.yml/badge.svg) -![Build and Publish Docker](https://github.com/junkurihara/rust-rpxy/actions/workflows/docker_build_push.yml/badge.svg) +![Docker x86_64](https://github.com/junkurihara/rust-rpxy/actions/workflows/docker_build_push_amd64.yml/badge.svg) +![Docker aarch64](https://github.com/junkurihara/rust-rpxy/actions/workflows/docker_build_push_arm64.yml/badge.svg) ![ShiftLeft Scan](https://github.com/junkurihara/rust-rpxy/actions/workflows/shift_left.yml/badge.svg) [![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/jqtype/rpxy)](https://hub.docker.com/r/jqtype/rpxy) diff --git a/TODO.md b/TODO.md index 3425d11..c552359 100644 --- a/TODO.md +++ b/TODO.md @@ -10,13 +10,13 @@ - upstream/upstream group: information on targeted destinations for each set of (a domain + a path) - load-balance: load balancing mod for a domain + path - - **Almost done in version 0.4.0**: - Split `rpxy` source codes into `rpxy-lib` and `rpxy-bin` to make the core part (reverse proxy) isolated from the misc part like toml file loader. This is in order to make the configuration-related part more flexible (related to [#33](https://github.com/junkurihara/rust-rpxy/issues/33)) + - Done in v0.4.0: + ~~Split `rpxy` source codes into `rpxy-lib` and `rpxy-bin` to make the core part (reverse proxy) isolated from the misc part like toml file loader. This is in order to make the configuration-related part more flexible (related to [#33](https://github.com/junkurihara/rust-rpxy/issues/33))~~ - Cache option for the response with `Cache-Control: public` header directive ([#55](https://github.com/junkurihara/rust-rpxy/issues/55)) - Consideration on migrating from `quinn` and `h3-quinn` to other QUIC implementations ([#57](https://github.com/junkurihara/rust-rpxy/issues/57)) -- Benchmark with other reverse proxy implementations like Sozu ([#58](https://github.com/junkurihara/rust-rpxy/issues/58)) - - Currently, Sozu can work only on `amd64` format due to its HTTP message parser limitation... Since the main developer have only `arm64` (Apple M1) laptops, so we should do that on VPS? +- Done in v0.4.0: + ~~Benchmark with other reverse proxy implementations like Sozu ([#58](https://github.com/junkurihara/rust-rpxy/issues/58)) Currently, Sozu can work only on `amd64` format due to its HTTP message parser limitation... Since the main developer have only `arm64` (Apple M1) laptops, so we should do that on VPS?~~ - Unit tests - Options to serve custom http_error page. From f7a6395d1152dc434d2e2536dc93f95d7d3d7600 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 24 Jul 2023 22:10:34 +0900 Subject: [PATCH 33/43] typo --- .github/workflows/docker_build_push_arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_push_arm64.yml b/.github/workflows/docker_build_push_arm64.yml index b31581c..2fd59d6 100644 --- a/.github/workflows/docker_build_push_arm64.yml +++ b/.github/workflows/docker_build_push_arm64.yml @@ -41,7 +41,7 @@ jobs: file: ./docker/Dockerfile platforms: linux/arm64 - - name: Release build and push x86_64-slim + - name: Release build and push aarch64-slim if: ${{ env.BRANCH == 'main' }} uses: docker/build-push-action@v4 with: From deb7eb12027fc5ce43d5f3eabc99df1dd54c44a0 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 25 Jul 2023 00:11:44 +0900 Subject: [PATCH 34/43] fix dockerfile for arm64 --- ...d_push_amd64.yml => docker_build_push.yml} | 17 +++-- .github/workflows/docker_build_push_arm64.yml | 64 ------------------- docker/Dockerfile | 4 +- docker/Dockerfile.amd64-slim | 4 +- docker/Dockerfile.arm64-slim | 4 +- 5 files changed, 19 insertions(+), 74 deletions(-) rename .github/workflows/{docker_build_push_amd64.yml => docker_build_push.yml} (78%) delete mode 100644 .github/workflows/docker_build_push_arm64.yml diff --git a/.github/workflows/docker_build_push_amd64.yml b/.github/workflows/docker_build_push.yml similarity index 78% rename from .github/workflows/docker_build_push_amd64.yml rename to .github/workflows/docker_build_push.yml index fa016a0..4c2b5b1 100644 --- a/.github/workflows/docker_build_push_amd64.yml +++ b/.github/workflows/docker_build_push.yml @@ -21,6 +21,9 @@ jobs: - name: GitHub Environment run: echo "BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -39,9 +42,11 @@ jobs: tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest file: ./docker/Dockerfile - platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 - - name: Release build and push x86_64-slim + - name: Release build and push slim if: ${{ env.BRANCH == 'main' }} uses: docker/build-push-action@v4 with: @@ -50,7 +55,9 @@ jobs: tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim file: ./docker/Dockerfile.amd64-slim - platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 - name: Nightly build and push if: ${{ env.BRANCH == 'develop' }} @@ -61,4 +68,6 @@ jobs: tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly file: ./docker/Dockerfile - platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/docker_build_push_arm64.yml b/.github/workflows/docker_build_push_arm64.yml deleted file mode 100644 index 2fd59d6..0000000 --- a/.github/workflows/docker_build_push_arm64.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Build and Publish Docker Aarch64 - -on: - push: - branches: - - main - - develop - -jobs: - build_and_push: - runs-on: ubuntu-latest - env: - IMAGE_NAME: rpxy - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: GitHub Environment - run: echo "BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Release build and push - if: ${{ env.BRANCH == 'main' }} - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: | - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest - file: ./docker/Dockerfile - platforms: linux/arm64 - - - name: Release build and push aarch64-slim - if: ${{ env.BRANCH == 'main' }} - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: | - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim - file: ./docker/Dockerfile.arm64-slim - platforms: linux/arm64 - - - name: Nightly build and push - if: ${{ env.BRANCH == 'develop' }} - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: | - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly - file: ./docker/Dockerfile - platforms: linux/arm64 diff --git a/docker/Dockerfile b/docker/Dockerfile index fd6d510..95b76aa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ SHELL ["/bin/sh", "-x", "-c"] ENV SERIAL 2 ######################################## -FROM base as builder +FROM --platform=$BUILDPLATFORM base AS builder ENV CFLAGS=-Ofast ENV BUILD_DEPS curl make ca-certificates build-essential @@ -27,7 +27,7 @@ RUN apt-get update && apt-get install -qy --no-install-recommends $BUILD_DEPS && strip --strip-all /tmp/target/release/rpxy ######################################## -FROM base AS runner +FROM --platform=$TARGETPLATFORM base AS runner ENV RUNTIME_DEPS logrotate ca-certificates gosu diff --git a/docker/Dockerfile.amd64-slim b/docker/Dockerfile.amd64-slim index 12cd261..7d8c366 100644 --- a/docker/Dockerfile.amd64-slim +++ b/docker/Dockerfile.amd64-slim @@ -1,5 +1,5 @@ ######################################## -FROM messense/rust-musl-cross:x86_64-musl as builder +FROM --platform=$BUILDPLATFORM messense/rust-musl-cross:x86_64-musl AS builder ENV TARGET_DIR=x86_64-unknown-linux-musl ENV CFLAGS=-Ofast @@ -15,7 +15,7 @@ RUN echo "Building rpxy from source" && \ musl-strip --strip-all /tmp/target/${TARGET_DIR}/release/rpxy ######################################## -FROM alpine:latest as runner +FROM --platform=$TARGETPLATFORM alpine:latest AS runner LABEL maintainer="Jun Kurihara" ENV TARGET_DIR=x86_64-unknown-linux-musl diff --git a/docker/Dockerfile.arm64-slim b/docker/Dockerfile.arm64-slim index 83b2d16..ff433a2 100644 --- a/docker/Dockerfile.arm64-slim +++ b/docker/Dockerfile.arm64-slim @@ -1,5 +1,5 @@ ######################################## -FROM messense/rust-musl-cross:aarch64-musl as builder +FROM --platform=$BUILDPLATFORM messense/rust-musl-cross:aarch64-musl AS builder ENV TARGET_DIR=aarch64-unknown-linux-musl ENV CFLAGS=-Ofast @@ -15,7 +15,7 @@ RUN echo "Building rpxy from source" && \ musl-strip --strip-all /tmp/target/${TARGET_DIR}/release/rpxy ######################################## -FROM alpine:latest as runner +FROM --platform=$TARGETPLATFORM alpine:latest AS runner LABEL maintainer="Jun Kurihara" ENV TARGET_DIR=aarch64-unknown-linux-musl From 5bbeb92c99be0e125c598294ec240c281bcde834 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 25 Jul 2023 01:43:52 +0900 Subject: [PATCH 35/43] update cross compilation settings for github actions --- .github/workflows/docker_build_push.yml | 33 ++++++++++++-- README.md | 3 +- docker/Dockerfile | 18 +++++++- docker/Dockerfile.arm64-slim | 45 ------------------- ...{Dockerfile.amd64-slim => Dockerfile.slim} | 22 ++++++--- 5 files changed, 62 insertions(+), 59 deletions(-) delete mode 100644 docker/Dockerfile.arm64-slim rename docker/{Dockerfile.amd64-slim => Dockerfile.slim} (60%) diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 4c2b5b1..dbac31b 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -9,6 +9,12 @@ on: jobs: build_and_push: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 env: IMAGE_NAME: rpxy @@ -44,7 +50,7 @@ jobs: file: ./docker/Dockerfile cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.platform }} - name: Release build and push slim if: ${{ env.BRANCH == 'main' }} @@ -54,10 +60,13 @@ jobs: push: true tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim - file: ./docker/Dockerfile.amd64-slim + build-contexts: | + messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl + messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl + file: ./docker/Dockerfile.slim cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.platform }} - name: Nightly build and push if: ${{ env.BRANCH == 'develop' }} @@ -70,4 +79,20 @@ jobs: file: ./docker/Dockerfile cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.platform }} + + - name: Release build and push slim + if: ${{ env.BRANCH == 'develop' }} + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly-slim + build-contexts: | + messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl + messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl + file: ./docker/Dockerfile.slim + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} diff --git a/README.md b/README.md index 85b3ab9..08d1270 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) ![Unit Test](https://github.com/junkurihara/rust-rpxy/actions/workflows/ci.yml/badge.svg) -![Docker x86_64](https://github.com/junkurihara/rust-rpxy/actions/workflows/docker_build_push_amd64.yml/badge.svg) -![Docker aarch64](https://github.com/junkurihara/rust-rpxy/actions/workflows/docker_build_push_arm64.yml/badge.svg) +![Docker](https://github.com/junkurihara/rust-rpxy/actions/workflows/docker_build_push.yml/badge.svg) ![ShiftLeft Scan](https://github.com/junkurihara/rust-rpxy/actions/workflows/shift_left.yml/badge.svg) [![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/jqtype/rpxy)](https://hub.docker.com/r/jqtype/rpxy) diff --git a/docker/Dockerfile b/docker/Dockerfile index 95b76aa..7844c2f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,6 +15,17 @@ WORKDIR /tmp COPY . /tmp/ +ARG TARGETARCH + +RUN if [ $TARGETARCH = "amd64" ]; then \ + echo "x86_64" > /arch; \ + elif [ $TARGETARCH = "arm64" ]; then \ + echo "aarch64" > /arch; \ + else \ + echo "Unsupported platform: $TARGETARCH"; \ + exit 1; \ + fi + ENV RUSTFLAGS "-C link-arg=-s" RUN update-ca-certificates 2> /dev/null || true @@ -22,9 +33,12 @@ RUN update-ca-certificates 2> /dev/null || true RUN apt-get update && apt-get install -qy --no-install-recommends $BUILD_DEPS && \ curl -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain stable && \ export PATH="$HOME/.cargo/bin:$PATH" && \ + echo "Install toolchain" && \ + rustup target add $(cat /arch)-unknown-linux-gnu &&\ echo "Building rpxy from source" && \ - cargo build --release && \ - strip --strip-all /tmp/target/release/rpxy + cargo build --release --target=$(cat /arch)-unknown-linux-gnu && \ + strip --strip-all /tmp/target/$(cat /arch)-unknown-linux-gnu/release/rpxy &&\ + cp /tmp/target/$(cat /arch)-unknown-linux-gnu/release/rpxy /tmp/target/release/rpxy ######################################## FROM --platform=$TARGETPLATFORM base AS runner diff --git a/docker/Dockerfile.arm64-slim b/docker/Dockerfile.arm64-slim deleted file mode 100644 index ff433a2..0000000 --- a/docker/Dockerfile.arm64-slim +++ /dev/null @@ -1,45 +0,0 @@ -######################################## -FROM --platform=$BUILDPLATFORM messense/rust-musl-cross:aarch64-musl AS builder - -ENV TARGET_DIR=aarch64-unknown-linux-musl -ENV CFLAGS=-Ofast - -WORKDIR /tmp - -COPY . /tmp/ - -ENV RUSTFLAGS "-C link-arg=-s" - -RUN echo "Building rpxy from source" && \ - cargo build --release && \ - musl-strip --strip-all /tmp/target/${TARGET_DIR}/release/rpxy - -######################################## -FROM --platform=$TARGETPLATFORM alpine:latest AS runner -LABEL maintainer="Jun Kurihara" - -ENV TARGET_DIR=aarch64-unknown-linux-musl -ENV RUNTIME_DEPS logrotate ca-certificates su-exec - -RUN apk add --no-cache ${RUNTIME_DEPS} && \ - update-ca-certificates && \ - find / -type d -path /proc -prune -o -type f -perm /u+s -exec chmod u-s {} \; && \ - find / -type d -path /proc -prune -o -type f -perm /g+s -exec chmod g-s {} \; && \ - mkdir -p /rpxy/bin &&\ - mkdir -p /rpxy/log - -COPY --from=builder /tmp/target/${TARGET_DIR}/release/rpxy /rpxy/bin/rpxy -COPY ./docker/run.sh /rpxy -COPY ./docker/entrypoint.sh /rpxy - -RUN chmod +x /rpxy/run.sh && \ - chmod +x /rpxy/entrypoint.sh - -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -ENV SSL_CERT_DIR=/etc/ssl/certs - -EXPOSE 80 443 - -CMD ["/rpxy/entrypoint.sh"] - -ENTRYPOINT ["/rpxy/entrypoint.sh"] diff --git a/docker/Dockerfile.amd64-slim b/docker/Dockerfile.slim similarity index 60% rename from docker/Dockerfile.amd64-slim rename to docker/Dockerfile.slim index 7d8c366..6f210c1 100644 --- a/docker/Dockerfile.amd64-slim +++ b/docker/Dockerfile.slim @@ -1,7 +1,17 @@ ######################################## -FROM --platform=$BUILDPLATFORM messense/rust-musl-cross:x86_64-musl AS builder +FROM --platform=$BUILDPLATFORM messense/rust-musl-cross:${TARGETARCH}-musl AS builder + +ARG TARGETARCH + +RUN if [ $TARGETARCH = "amd64" ]; then \ + echo "x86_64" > /arch; \ + elif [ $TARGETARCH = "arm64" ]; then \ + echo "aarch64" > /arch; \ + else \ + echo "Unsupported platform: $TARGETARCH"; \ + exit 1; \ + fi -ENV TARGET_DIR=x86_64-unknown-linux-musl ENV CFLAGS=-Ofast WORKDIR /tmp @@ -11,14 +21,14 @@ COPY . /tmp/ ENV RUSTFLAGS "-C link-arg=-s" RUN echo "Building rpxy from source" && \ - cargo build --release && \ - musl-strip --strip-all /tmp/target/${TARGET_DIR}/release/rpxy + cargo build --release --target $(cat /arch)-unknown-linux-musl && \ + musl-strip --strip-all /tmp/target/$(cat /arch)-unknown-linux-musl/release/rpxy && \ + cp /tmp/target/$(cat /arch)-unknown-linux-musl/release/rpxy /tmp/target/release/rpxy ######################################## FROM --platform=$TARGETPLATFORM alpine:latest AS runner LABEL maintainer="Jun Kurihara" -ENV TARGET_DIR=x86_64-unknown-linux-musl ENV RUNTIME_DEPS logrotate ca-certificates su-exec RUN apk add --no-cache ${RUNTIME_DEPS} && \ @@ -28,7 +38,7 @@ RUN apk add --no-cache ${RUNTIME_DEPS} && \ mkdir -p /rpxy/bin &&\ mkdir -p /rpxy/log -COPY --from=builder /tmp/target/${TARGET_DIR}/release/rpxy /rpxy/bin/rpxy +COPY --from=builder /tmp/target/release/rpxy /rpxy/bin/rpxy COPY ./docker/run.sh /rpxy COPY ./docker/entrypoint.sh /rpxy From 9207b6ba3f40416fe4e1ffe5815e0d8eef766c3a Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 25 Jul 2023 01:46:34 +0900 Subject: [PATCH 36/43] fix github actions --- .github/workflows/docker_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index dbac31b..1176777 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -1,4 +1,4 @@ -name: Build and Publish Docker x86_64 +name: Build and Publish Docker on: push: From 9f0642621f0987bdf43276788179a093c0eac79d Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 25 Jul 2023 02:24:25 +0900 Subject: [PATCH 37/43] fix github actions fix github actions fix github actions fix github actions env --- .github/workflows/docker_build_push.yml | 99 +++++++++++++++++++++---- 1 file changed, 83 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 1176777..83d3d99 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -6,6 +6,9 @@ on: - main - develop +env: + REGISTRY_IMAGE: jqtype/rpxy + jobs: build_and_push: runs-on: ubuntu-latest @@ -15,8 +18,6 @@ jobs: platform: - linux/amd64 - linux/arm64 - env: - IMAGE_NAME: rpxy steps: - name: Checkout @@ -27,6 +28,12 @@ jobs: - name: GitHub Environment run: echo "BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY_IMAGE }} + - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -39,18 +46,20 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Release build and push + - name: Release build and push by digest if: ${{ env.BRANCH == 'main' }} uses: docker/build-push-action@v4 with: context: . push: true tags: | - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest + ${{ env.REGISTRY_IMAGE }}:latest file: ./docker/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.platform }}-latest + cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-latest platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Release build and push slim if: ${{ env.BRANCH == 'main' }} @@ -59,14 +68,16 @@ jobs: context: . push: true tags: | - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim + ${{ env.REGISTRY_IMAGE }}:slim, ${{ env.REGISTRY_IMAGE }}:latest-slim build-contexts: | messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl file: ./docker/Dockerfile.slim - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.platform }}-latest-slim + cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-latest-slim platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Nightly build and push if: ${{ env.BRANCH == 'develop' }} @@ -75,24 +86,80 @@ jobs: context: . push: true tags: | - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly + ${{ env.REGISTRY_IMAGE }}:nightly file: ./docker/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.platform }}-nightly + cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-nightly platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Release build and push slim + - name: Nightly build and push slim if: ${{ env.BRANCH == 'develop' }} uses: docker/build-push-action@v4 with: context: . push: true tags: | - ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly-slim + ${{ env.REGISTRY_IMAGE }}:nightly-slim build-contexts: | messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl file: ./docker/Dockerfile.slim - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.platform }}-nightly-slim + cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-nightly-slim platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v3 + with: + name: digests + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: + - build_and_push + steps: + - + name: Download digests + uses: actions/download-artifact@v3 + with: + name: digests + path: /tmp/digests + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY_IMAGE }} + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - + name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} From 2e3216019b266e3b4f9026347e2b8df15c96b08a Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 25 Jul 2023 03:26:55 +0900 Subject: [PATCH 38/43] revert github actions revert github actions --- .github/workflows/docker_build_push.yml | 88 ++++--------------------- docker/Dockerfile | 1 - docker/Dockerfile.slim | 2 + 3 files changed, 15 insertions(+), 76 deletions(-) diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 83d3d99..b3e5e88 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -12,12 +12,6 @@ env: jobs: build_and_push: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 steps: - name: Checkout @@ -46,7 +40,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Release build and push by digest + - name: Release build and push if: ${{ env.BRANCH == 'main' }} uses: docker/build-push-action@v4 with: @@ -55,11 +49,10 @@ jobs: tags: | ${{ env.REGISTRY_IMAGE }}:latest file: ./docker/Dockerfile - cache-from: type=gha,scope=${{ matrix.platform }}-latest - cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-latest - platforms: ${{ matrix.platform }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Release build and push slim if: ${{ env.BRANCH == 'main' }} @@ -73,11 +66,10 @@ jobs: messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl file: ./docker/Dockerfile.slim - cache-from: type=gha,scope=${{ matrix.platform }}-latest-slim - cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-latest-slim - platforms: ${{ matrix.platform }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Nightly build and push if: ${{ env.BRANCH == 'develop' }} @@ -88,11 +80,10 @@ jobs: tags: | ${{ env.REGISTRY_IMAGE }}:nightly file: ./docker/Dockerfile - cache-from: type=gha,scope=${{ matrix.platform }}-nightly - cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-nightly - platforms: ${{ matrix.platform }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Nightly build and push slim if: ${{ env.BRANCH == 'develop' }} @@ -106,60 +97,7 @@ jobs: messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl file: ./docker/Dockerfile.slim - cache-from: type=gha,scope=${{ matrix.platform }}-nightly-slim - cache-to: type=gha,mode=max,scope=${{ matrix.platform }}-nightly-slim - platforms: ${{ matrix.platform }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v3 - with: - name: digests - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge: - runs-on: ubuntu-latest - needs: - - build_and_push - steps: - - - name: Download digests - uses: actions/download-artifact@v3 - with: - name: digests - path: /tmp/digests - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY_IMAGE }} - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 7844c2f..456df2b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,3 @@ - FROM ubuntu:22.04 AS base LABEL maintainer="Jun Kurihara" diff --git a/docker/Dockerfile.slim b/docker/Dockerfile.slim index 6f210c1..1d77b78 100644 --- a/docker/Dockerfile.slim +++ b/docker/Dockerfile.slim @@ -1,6 +1,8 @@ ######################################## FROM --platform=$BUILDPLATFORM messense/rust-musl-cross:${TARGETARCH}-musl AS builder +LABEL maintainer="Jun Kurihara" + ARG TARGETARCH RUN if [ $TARGETARCH = "amd64" ]; then \ From 5ff285c3eca1eb166186705ae2690a112b307c5e Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 25 Jul 2023 16:52:17 +0900 Subject: [PATCH 39/43] feat: add a docker environment var for continuous watching --- README.md | 1 + docker/docker-compose.yml | 4 ++++ docker/run.sh | 17 ++++++++++++++++- rpxy-bin/Cargo.toml | 2 +- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 08d1270..c733a7e 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,7 @@ You can also use [docker image](https://hub.docker.com/r/jqtype/rpxy) instead of - `HOST_GID` (default: `900`): `GID` of `HOST_USER` - `LOG_LEVEL=debug|info|warn|error`: Log level - `LOG_TO_FILE=true|false`: Enable logging to the log file `/rpxy/log/rpxy.log` using `logrotate`. You should mount `/rpxy/log` via docker volume option if enabled. The log dir and file will be owned by the `HOST_USER` with `HOST_UID:HOST_GID` on the host machine. Hence, `HOST_USER`, `HOST_UID` and `HOST_GID` should be the same as ones of the user who executes the `rpxy` docker container on the host. +- `WATCH=true|false` (default: `false`): Activate continuous watching of the config file if true. Other than them, all you need is to mount your `config.toml` as `/etc/rpxy.toml` and certificates/private keys as you like through the docker volume option. See [`docker/docker-compose.yml`](./docker/docker-compose.yml) for the detailed configuration. Note that the file path of keys and certificates must be ones in your docker container. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d817918..886a471 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,12 +11,16 @@ services: build: context: ../ dockerfile: ./docker/Dockerfile + platforms: # Choose your platforms + - "linux/amd64" + # - "linux/arm64" environment: - LOG_LEVEL=debug - LOG_TO_FILE=true - HOST_USER=jun - HOST_UID=501 - HOST_GID=501 + # - WATCH=true tty: false privileged: true volumes: diff --git a/docker/run.sh b/docker/run.sh index 6f83ff8..25f50d6 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -7,4 +7,19 @@ if [ -z $LOG_LEVEL ]; then fi echo "rpxy: Logging with level ${LOG_LEVEL}" -RUST_LOG=${LOG_LEVEL} /rpxy/bin/rpxy --config ${CONFIG_FILE} +# continuously watch and reload the config file +if [ -z $WATCH ]; then + WATCH=false +else + if [ "$WATCH" = "true" ]; then + WATCH=true + else + WATCH=false + fi +fi + +if $WATCH ; then + RUST_LOG=${LOG_LEVEL} /rpxy/bin/rpxy --config ${CONFIG_FILE} -w +else + RUST_LOG=${LOG_LEVEL} /rpxy/bin/rpxy --config ${CONFIG_FILE} +fi diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index f4bdc68..3d7c9e4 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -20,7 +20,7 @@ rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] } anyhow = "1.0.72" rustc-hash = "1.1.0" -serde = { version = "1.0.174", default-features = false, features = ["derive"] } +serde = { version = "1.0.175", default-features = false, features = ["derive"] } derive_builder = "0.12.0" tokio = { version = "1.29.1", default-features = false, features = [ "net", From a40d8a00721246934cfac96d293359ad9e0e6843 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 28 Jul 2023 15:18:21 +0900 Subject: [PATCH 40/43] refactor http handler --- quinn | 2 +- rpxy-bin/Cargo.toml | 4 +- rpxy-lib/src/handler/handler_main.rs | 124 ++++++++++--------- rpxy-lib/src/handler/mod.rs | 1 + rpxy-lib/src/handler/utils_headers.rs | 27 ++-- rpxy-lib/src/handler/utils_request.rs | 3 + rpxy-lib/src/handler/utils_synth_response.rs | 2 + 7 files changed, 91 insertions(+), 72 deletions(-) diff --git a/quinn b/quinn index 0ae7c60..532ba7d 160000 --- a/quinn +++ b/quinn @@ -1 +1 @@ -Subproject commit 0ae7c60b15637d7343410ba1e5cc3151e3814557 +Subproject commit 532ba7d80405ad083fd05546fa71becbe5eff1a4 diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index f4bdc68..0ed54d2 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -20,7 +20,7 @@ rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] } anyhow = "1.0.72" rustc-hash = "1.1.0" -serde = { version = "1.0.174", default-features = false, features = ["derive"] } +serde = { version = "1.0.177", default-features = false, features = ["derive"] } derive_builder = "0.12.0" tokio = { version = "1.29.1", default-features = false, features = [ "net", @@ -43,7 +43,7 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = "0.5.0" +tikv-jemallocator = "0.5.4" [dev-dependencies] diff --git a/rpxy-lib/src/handler/handler_main.rs b/rpxy-lib/src/handler/handler_main.rs index c23fd24..f6c4dc7 100644 --- a/rpxy-lib/src/handler/handler_main.rs +++ b/rpxy-lib/src/handler/handler_main.rs @@ -19,6 +19,8 @@ use std::{env, net::SocketAddr, sync::Arc}; use tokio::{io::copy_bidirectional, time::timeout}; #[derive(Clone, Builder)] +/// HTTP message handler for requests from clients and responses from backend applications, +/// responsible to manipulate and forward messages to upstream backends and downstream clients. pub struct HttpMessageHandler where T: Connect + Clone + Sync + Send + 'static, @@ -33,11 +35,13 @@ where T: Connect + Clone + Sync + Send + 'static, U: CryptoSource + Clone, { + /// Return with an arbitrary status code of error and log message fn return_with_error_log(&self, status_code: StatusCode, log_data: &mut MessageLog) -> Result> { log_data.status_code(&status_code).output(); http_error(status_code) } + /// Handle incoming request message from a client pub async fn handle_request( self, mut req: Request, @@ -65,13 +69,15 @@ where } } // Find backend application for given server_name, and drop if incoming request is invalid as request. - let backend = if let Some(be) = self.globals.backends.apps.get(&server_name) { - be - } else if let Some(default_server_name) = &self.globals.backends.default_server_name_bytes { - debug!("Serving by default app"); - self.globals.backends.apps.get(default_server_name).unwrap() - } else { - return self.return_with_error_log(StatusCode::SERVICE_UNAVAILABLE, &mut log_data); + let backend = match self.globals.backends.apps.get(&server_name) { + Some(be) => be, + None => { + let Some(default_server_name) = &self.globals.backends.default_server_name_bytes else { + return self.return_with_error_log(StatusCode::SERVICE_UNAVAILABLE, &mut log_data); + }; + debug!("Serving by default app"); + self.globals.backends.apps.get(default_server_name).unwrap() + } }; // Redirect to https if !tls_enabled and redirect_to_https is true @@ -84,9 +90,8 @@ where // Find reverse proxy for given path and choose one of upstream host // Longest prefix match let path = req.uri().path(); - let upstream_group = match backend.reverse_proxy.get(path) { - Some(ug) => ug, - None => return self.return_with_error_log(StatusCode::NOT_FOUND, &mut log_data), + let Some(upstream_group) = backend.reverse_proxy.get(path) else { + return self.return_with_error_log(StatusCode::NOT_FOUND, &mut log_data) }; // Upgrade in request header @@ -113,19 +118,17 @@ where log_data.upstream(req.uri()); ////// - // Forward request to + // Forward request to a chosen backend let mut res_backend = { - match timeout(self.globals.proxy_config.upstream_timeout, self.forwarder.request(req)).await { - Err(_) => { - return self.return_with_error_log(StatusCode::GATEWAY_TIMEOUT, &mut log_data); + let Ok(result) = timeout(self.globals.proxy_config.upstream_timeout, self.forwarder.request(req)).await else { + return self.return_with_error_log(StatusCode::GATEWAY_TIMEOUT, &mut log_data); + }; + match result { + Ok(res) => res, + Err(e) => { + error!("Failed to get response from backend: {}", e); + return self.return_with_error_log(StatusCode::SERVICE_UNAVAILABLE, &mut log_data); } - Ok(x) => match x { - Ok(res) => res, - Err(e) => { - error!("Failed to get response from backend: {}", e); - return self.return_with_error_log(StatusCode::SERVICE_UNAVAILABLE, &mut log_data); - } - }, } }; @@ -141,62 +144,63 @@ where if res_backend.status() != StatusCode::SWITCHING_PROTOCOLS { // Generate response to client - if self.generate_response_forwarded(&mut res_backend, backend).is_ok() { - log_data.status_code(&res_backend.status()).output(); - return Ok(res_backend); - } else { + if self.generate_response_forwarded(&mut res_backend, backend).is_err() { return self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data); } + log_data.status_code(&res_backend.status()).output(); + return Ok(res_backend); } // Handle StatusCode::SWITCHING_PROTOCOLS in response let upgrade_in_response = extract_upgrade(res_backend.headers()); - if if let (Some(u_req), Some(u_res)) = (upgrade_in_request.as_ref(), upgrade_in_response.as_ref()) { + let should_upgrade = if let (Some(u_req), Some(u_res)) = (upgrade_in_request.as_ref(), upgrade_in_response.as_ref()) + { u_req.to_ascii_lowercase() == u_res.to_ascii_lowercase() } else { false - } { - if let Some(request_upgraded) = request_upgraded { - let Some(onupgrade) = res_backend.extensions_mut().remove::() else { - error!("Response does not have an upgrade extension"); - return self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data); - }; - - self.globals.runtime_handle.spawn(async move { - let mut response_upgraded = onupgrade.await.map_err(|e| { - error!("Failed to upgrade response: {}", e); - RpxyError::Hyper(e) - })?; - let mut request_upgraded = request_upgraded.await.map_err(|e| { - error!("Failed to upgrade request: {}", e); - RpxyError::Hyper(e) - })?; - copy_bidirectional(&mut response_upgraded, &mut request_upgraded) - .await - .map_err(|e| { - error!("Coping between upgraded connections failed: {}", e); - RpxyError::Io(e) - })?; - Ok(()) as Result<()> - }); - log_data.status_code(&res_backend.status()).output(); - Ok(res_backend) - } else { - error!("Request does not have an upgrade extension"); - self.return_with_error_log(StatusCode::BAD_REQUEST, &mut log_data) - } - } else { + }; + if !should_upgrade { error!( "Backend tried to switch to protocol {:?} when {:?} was requested", upgrade_in_response, upgrade_in_request ); - self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data) + return self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data); } + let Some(request_upgraded) = request_upgraded else { + error!("Request does not have an upgrade extension"); + return self.return_with_error_log(StatusCode::BAD_REQUEST, &mut log_data); + }; + let Some(onupgrade) = res_backend.extensions_mut().remove::() else { + error!("Response does not have an upgrade extension"); + return self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data); + }; + + self.globals.runtime_handle.spawn(async move { + let mut response_upgraded = onupgrade.await.map_err(|e| { + error!("Failed to upgrade response: {}", e); + RpxyError::Hyper(e) + })?; + let mut request_upgraded = request_upgraded.await.map_err(|e| { + error!("Failed to upgrade request: {}", e); + RpxyError::Hyper(e) + })?; + copy_bidirectional(&mut response_upgraded, &mut request_upgraded) + .await + .map_err(|e| { + error!("Coping between upgraded connections failed: {}", e); + RpxyError::Io(e) + })?; + Ok(()) as Result<()> + }); + log_data.status_code(&res_backend.status()).output(); + Ok(res_backend) } //////////////////////////////////////////////////// // Functions to generate messages + //////////////////////////////////////////////////// + /// Manipulate a response message sent from a backend application to forward downstream to a client. fn generate_response_forwarded(&self, response: &mut Response, chosen_backend: &Backend) -> Result<()> where B: core::fmt::Debug, @@ -208,7 +212,8 @@ where #[cfg(feature = "http3")] { - // TODO: Workaround for avoid h3 for client authentication + // Manipulate ALT_SVC allowing h3 in response message only when mutual TLS is not enabled + // TODO: This is a workaround for avoiding a client authentication in HTTP/3 if self.globals.proxy_config.http3 && chosen_backend .crypto_source @@ -241,6 +246,7 @@ where } #[allow(clippy::too_many_arguments)] + /// Manipulate a request message sent from a client to forward upstream to a backend application fn generate_request_forwarded( &self, client_addr: &SocketAddr, diff --git a/rpxy-lib/src/handler/mod.rs b/rpxy-lib/src/handler/mod.rs index 8bec011..aed9831 100644 --- a/rpxy-lib/src/handler/mod.rs +++ b/rpxy-lib/src/handler/mod.rs @@ -9,6 +9,7 @@ pub use handler_main::{HttpMessageHandler, HttpMessageHandlerBuilder, HttpMessag #[allow(dead_code)] #[derive(Debug)] +/// Context object to handle sticky cookies at HTTP message handler struct HandlerContext { #[cfg(feature = "sticky-cookie")] context_lb: Option, diff --git a/rpxy-lib/src/handler/utils_headers.rs b/rpxy-lib/src/handler/utils_headers.rs index 944d4d9..d09df79 100644 --- a/rpxy-lib/src/handler/utils_headers.rs +++ b/rpxy-lib/src/handler/utils_headers.rs @@ -8,7 +8,7 @@ use hyper::{ header::{self, HeaderMap, HeaderName, HeaderValue}, Uri, }; -use std::net::SocketAddr; +use std::{borrow::Cow, net::SocketAddr}; //////////////////////////////////////////////////// // Functions to manipulate headers @@ -83,6 +83,7 @@ pub(super) fn set_sticky_cookie_lb_context(headers: &mut HeaderMap, context_from Ok(()) } +/// Apply options to request header, which are specified in the configuration pub(super) fn apply_upstream_options_to_header( headers: &mut HeaderMap, _client_addr: &SocketAddr, @@ -113,7 +114,7 @@ pub(super) fn apply_upstream_options_to_header( Ok(()) } -// https://datatracker.ietf.org/doc/html/rfc9110 +/// Append header entry with comma according to [RFC9110](https://datatracker.ietf.org/doc/html/rfc9110) pub(super) fn append_header_entry_with_comma(headers: &mut HeaderMap, key: &str, value: &str) -> Result<()> { match headers.entry(HeaderName::from_bytes(key.as_bytes())?) { header::Entry::Vacant(entry) => { @@ -132,10 +133,11 @@ pub(super) fn append_header_entry_with_comma(headers: &mut HeaderMap, key: &str, Ok(()) } +/// Add header entry if not exist pub(super) fn add_header_entry_if_not_exist( headers: &mut HeaderMap, - key: impl Into>, - value: impl Into>, + key: impl Into>, + value: impl Into>, ) -> Result<()> { match headers.entry(HeaderName::from_bytes(key.into().as_bytes())?) { header::Entry::Vacant(entry) => { @@ -147,10 +149,11 @@ pub(super) fn add_header_entry_if_not_exist( Ok(()) } +/// Overwrite header entry if exist pub(super) fn add_header_entry_overwrite_if_exist( headers: &mut HeaderMap, - key: impl Into>, - value: impl Into>, + key: impl Into>, + value: impl Into>, ) -> Result<()> { match headers.entry(HeaderName::from_bytes(key.into().as_bytes())?) { header::Entry::Vacant(entry) => { @@ -164,11 +167,10 @@ pub(super) fn add_header_entry_overwrite_if_exist( Ok(()) } +/// Align cookie values in single line +/// Sometimes violates [RFC6265](https://www.rfc-editor.org/rfc/rfc6265#section-5.4) (for http/1.1). +/// This is allowed in RFC7540 (for http/2) as mentioned [here](https://stackoverflow.com/questions/4843556/in-http-specification-what-is-the-string-that-separates-cookies). pub(super) fn make_cookie_single_line(headers: &mut HeaderMap) -> Result<()> { - // Sometimes violates RFC6265 (for http/1.1). - // https://www.rfc-editor.org/rfc/rfc6265#section-5.4 - // This is allowed in RFC7540 (for http/2). - // https://stackoverflow.com/questions/4843556/in-http-specification-what-is-the-string-that-separates-cookies let cookies = headers .iter() .filter(|(k, _)| **k == hyper::header::COOKIE) @@ -182,6 +184,7 @@ pub(super) fn make_cookie_single_line(headers: &mut HeaderMap) -> Result<()> { Ok(()) } +/// Add forwarding headers like `x-forwarded-for`. pub(super) fn add_forwarding_header( headers: &mut HeaderMap, client_addr: &SocketAddr, @@ -219,6 +222,7 @@ pub(super) fn add_forwarding_header( Ok(()) } +/// Remove connection header pub(super) fn remove_connection_header(headers: &mut HeaderMap) { if let Some(values) = headers.get(header::CONNECTION) { if let Ok(v) = values.clone().to_str() { @@ -231,6 +235,7 @@ pub(super) fn remove_connection_header(headers: &mut HeaderMap) { } } +/// Hop header values which are removed at proxy const HOP_HEADERS: &[&str] = &[ "connection", "te", @@ -243,12 +248,14 @@ const HOP_HEADERS: &[&str] = &[ "upgrade", ]; +/// Remove hop headers pub(super) fn remove_hop_header(headers: &mut HeaderMap) { HOP_HEADERS.iter().for_each(|key| { headers.remove(*key); }); } +/// Extract upgrade header value if exist pub(super) fn extract_upgrade(headers: &HeaderMap) -> Option { if let Some(c) = headers.get(header::CONNECTION) { if c diff --git a/rpxy-lib/src/handler/utils_request.rs b/rpxy-lib/src/handler/utils_request.rs index 74e7be7..03e36a1 100644 --- a/rpxy-lib/src/handler/utils_request.rs +++ b/rpxy-lib/src/handler/utils_request.rs @@ -7,6 +7,7 @@ use hyper::{header, Request}; //////////////////////////////////////////////////// // Functions to manipulate request line +/// Apply upstream options in request line, specified in the configuration pub(super) fn apply_upstream_options_to_request_line(req: &mut Request, upstream: &UpstreamGroup) -> Result<()> { for opt in upstream.opts.iter() { match opt { @@ -19,10 +20,12 @@ pub(super) fn apply_upstream_options_to_request_line(req: &mut Request, up Ok(()) } +/// Trait defining parser of hostname pub trait ParseHost { fn parse_host(&self) -> Result<&[u8]>; } impl ParseHost for Request { + /// Extract hostname from either the request HOST header or request line fn parse_host(&self) -> Result<&[u8]> { let headers_host = self.headers().get(header::HOST); let uri_host = self.uri().host(); diff --git a/rpxy-lib/src/handler/utils_synth_response.rs b/rpxy-lib/src/handler/utils_synth_response.rs index e1977f8..baa6987 100644 --- a/rpxy-lib/src/handler/utils_synth_response.rs +++ b/rpxy-lib/src/handler/utils_synth_response.rs @@ -5,11 +5,13 @@ use hyper::{Body, Request, Response, StatusCode, Uri}; //////////////////////////////////////////////////// // Functions to create response (error or redirect) +/// Generate a synthetic response message of a certain error status code pub(super) fn http_error(status_code: StatusCode) -> Result> { let response = Response::builder().status(status_code).body(Body::empty())?; Ok(response) } +/// Generate synthetic response message of a redirection to https host with 301 pub(super) fn secure_redirection( server_name: &str, tls_port: Option, From 2f43986630e2c1476913ccc4f0889926fe877dd4 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Fri, 28 Jul 2023 15:23:28 +0900 Subject: [PATCH 41/43] fix --- rpxy-bin/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 870dc0f..0ed54d2 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -21,7 +21,6 @@ rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] } anyhow = "1.0.72" rustc-hash = "1.1.0" serde = { version = "1.0.177", default-features = false, features = ["derive"] } -develop derive_builder = "0.12.0" tokio = { version = "1.29.1", default-features = false, features = [ "net", From 892f5f89e03519fa8059b73d5396b625e003676a Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 30 Jul 2023 09:53:44 +0900 Subject: [PATCH 42/43] refactor: remove set_reuse_address for udpsocket --- rpxy-lib/src/proxy/socket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpxy-lib/src/proxy/socket.rs b/rpxy-lib/src/proxy/socket.rs index 48f72e9..2151710 100644 --- a/rpxy-lib/src/proxy/socket.rs +++ b/rpxy-lib/src/proxy/socket.rs @@ -32,7 +32,7 @@ pub(super) fn bind_udp_socket(listening_on: &SocketAddr) -> Result { } else { Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP)) }?; - socket.set_reuse_address(true)?; + // socket.set_reuse_address(true)?; // This isn't necessary socket.set_reuse_port(true)?; if let Err(e) = socket.bind(&(*listening_on).into()) { From 0f1eed7718f066dde3efa217aac81964302a8039 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 30 Jul 2023 10:00:35 +0900 Subject: [PATCH 43/43] chore: deps and fix typo --- rpxy-bin/Cargo.toml | 4 ++-- rpxy-lib/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 0ed54d2..0fc2ae4 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" license = "MIT" -readme = "README.md" +readme = "../README.md" edition = "2021" publish = false @@ -20,7 +20,7 @@ rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] } anyhow = "1.0.72" rustc-hash = "1.1.0" -serde = { version = "1.0.177", default-features = false, features = ["derive"] } +serde = { version = "1.0.178", default-features = false, features = ["derive"] } derive_builder = "0.12.0" tokio = { version = "1.29.1", default-features = false, features = [ "net", diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index e36bae6..3f405e8 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" license = "MIT" -readme = "README.md" +readme = "../README.md" edition = "2021" publish = false