From 0b1eb89ed10a5f101005ec5fe8fb543a4b7e7aea Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Sun, 30 Jul 2023 16:10:19 +0900 Subject: [PATCH] feat: initial support of s2n-quic --- CHANGELOG.md | 2 + README.md | 11 +- rpxy-bin/Cargo.toml | 8 +- rpxy-bin/src/config/toml.rs | 4 +- rpxy-bin/src/main.rs | 3 + rpxy-lib/Cargo.toml | 15 +- rpxy-lib/src/constants.rs | 2 +- rpxy-lib/src/error.rs | 12 +- rpxy-lib/src/globals.rs | 36 ++-- rpxy-lib/src/handler/handler_main.rs | 4 +- rpxy-lib/src/lib.rs | 5 +- rpxy-lib/src/proxy/crypto_service.rs | 155 ++++++++++++++---- rpxy-lib/src/proxy/mod.rs | 8 +- rpxy-lib/src/proxy/proxy_h3.rs | 3 + .../{proxy_quic.rs => proxy_quic_quinn.rs} | 6 +- rpxy-lib/src/proxy/proxy_quic_s2n.rs | 135 +++++++++++++++ rpxy-lib/src/proxy/proxy_tls.rs | 4 +- rpxy-lib/src/proxy/socket.rs | 6 +- 18 files changed, 343 insertions(+), 76 deletions(-) rename rpxy-lib/src/proxy/{proxy_quic.rs => proxy_quic_quinn.rs} (98%) create mode 100644 rpxy-lib/src/proxy/proxy_quic_s2n.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c6339..000a48c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Improvement +- Feat: `s2n-quic` with `s2n-quic-h3` is supported as QUIC and HTTP/3 library in addition to `quinn` with `h3-quinn`, related to #57. + ## 0.4.0 ### Improvement diff --git a/README.md b/README.md index c733a7e..eadc42c 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ `rpxy` [ahr-pik-see] is an implementation of simple and lightweight reverse-proxy with some additional features. The implementation is based on [`hyper`](https://github.com/hyperium/hyper), [`rustls`](https://github.com/rustls/rustls) and [`tokio`](https://github.com/tokio-rs/tokio), i.e., written in pure Rust. Our `rpxy` routes multiple host names to appropriate backend application servers while serving TLS connections. - As default, `rpxy` provides the *TLS connection sanitization* by correctly binding a certificate used to establish a secure channel with the backend application. Specifically, it always keeps the consistency between the given SNI (server name indication) in `ClientHello` of the underlying TLS and the domain name given by the overlaid HTTP HOST header (or URL in Request line) [^1]. Additionally, as a somewhat unstable feature, our `rpxy` can handle the brand-new HTTP/3 connection thanks to [`quinn`](https://github.com/quinn-rs/quinn) and [`hyperium/h3`](https://github.com/hyperium/h3). + As default, `rpxy` provides the *TLS connection sanitization* by correctly binding a certificate used to establish a secure channel with the backend application. Specifically, it always keeps the consistency between the given SNI (server name indication) in `ClientHello` of the underlying TLS and the domain name given by the overlaid HTTP HOST header (or URL in Request line) [^1]. Additionally, as a somewhat unstable feature, our `rpxy` can handle the brand-new HTTP/3 connection thanks to [`quinn`](https://github.com/quinn-rs/quinn), [`s2n-quic`](https://github.com/aws/s2n-quic) and [`hyperium/h3`](https://github.com/hyperium/h3).[^h3lib] + + [^h3lib]: HTTP/3 libraries are mutually exclusive. You need to explicitly specify `s2n-quic` with `--no-default-features` flag. This project is still *work-in-progress*. But it is already working in some production environments and serves a number of domain names. Furthermore it *significantly outperforms* NGINX and Caddy, e.g., *1.5x faster than NGINX*, in the setting of a very simple HTTP reverse-proxy scenario (See [`bench`](./bench/) directory). @@ -27,11 +29,14 @@ You can build an executable binary yourself by checking out this Git repository. % git clone https://github.com/junkurihara/rust-rpxy % cd rust-rpxy -# Update submodule hyperium/h3 +# Update submodules % git submodule update --init -# Build +# Build (default: QUIC and HTTP/3 is enabled using `quinn`) % cargo build --release + +# If you want to use `s2n-quic`, build as follows. +% cargo build --no-default-features --features http3-s2n --release ``` Then you have an executive binary `rust-rpxy/target/release/rpxy`. diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 3d1f90c..7e20882 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rpxy" -version = "0.4.0" +version = "0.5.0" authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" @@ -14,9 +14,13 @@ publish = false [features] default = ["http3"] http3 = ["rpxy-lib/http3"] +http3-quinn = ["rpxy-lib/http3-quinn"] +http3-s2n = ["rpxy-lib/http3-s2n"] [dependencies] -rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] } +rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [ + "sticky-cookie", +] } anyhow = "1.0.72" rustc-hash = "1.1.0" diff --git a/rpxy-bin/src/config/toml.rs b/rpxy-bin/src/config/toml.rs index 84260c0..35c5d7a 100644 --- a/rpxy-bin/src/config/toml.rs +++ b/rpxy-bin/src/config/toml.rs @@ -142,10 +142,10 @@ impl TryInto for &ConfigToml { proxy_config.h3_max_concurrent_connections = x; } if let Some(x) = h3option.max_concurrent_bidistream { - proxy_config.h3_max_concurrent_bidistream = x.into(); + proxy_config.h3_max_concurrent_bidistream = x; } if let Some(x) = h3option.max_concurrent_unistream { - proxy_config.h3_max_concurrent_unistream = x.into(); + proxy_config.h3_max_concurrent_unistream = x; } if let Some(x) = h3option.max_idle_timeout { if x == 0u64 { diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index 8fe00dc..861c3d5 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -19,6 +19,9 @@ use crate::{ use hot_reload::{ReloaderReceiver, ReloaderService}; use rpxy_lib::entrypoint; +#[cfg(all(feature = "http3-quinn", feature = "http3-s2n"))] +compile_error!("feature \"http3-quinn\" and feature \"http3-s2n\" cannot be enabled at the same time"); + fn main() { init_logger(); diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index 3f405e8..063ca13 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rpxy-lib" -version = "0.4.0" +version = "0.5.0" authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" @@ -13,7 +13,9 @@ publish = false [features] default = ["http3", "sticky-cookie"] -http3 = ["quinn", "h3", "h3-quinn"] +http3 = ["http3-s2n"] +http3-quinn = ["quinn", "h3", "h3-quinn", "socket2"] +http3-s2n = ["h3", "s2n-quic", "s2n-quic-rustls", "s2n-quic-h3"] sticky-cookie = ["base64", "sha2", "chrono"] [dependencies] @@ -63,8 +65,13 @@ quinn = { path = "../quinn/quinn", optional = true } # Tentative to support rust h3 = { path = "../h3/h3/", optional = true } # h3-quinn = { path = "./h3/h3-quinn/", optional = true } h3-quinn = { path = "../h3-quinn/", optional = true } # Tentative to support rustls-0.21 -# for UDP socket wit SO_REUSEADDR -socket2 = { version = "0.5.3", features = ["all"] } +# for UDP socket wit SO_REUSEADDR when h3 with quinn +socket2 = { version = "0.5.3", features = ["all"], optional = true } +s2n-quic = { path = "../s2n-quic/quic/s2n-quic/", features = [ + "provider-tls-rustls", +], optional = true } +s2n-quic-h3 = { path = "../s2n-quic/quic/s2n-quic-h3/", optional = true } +s2n-quic-rustls = { path = "../s2n-quic/quic/s2n-quic-rustls/", optional = true } # cookie handling for sticky cookie chrono = { version = "0.4.26", default-features = false, features = [ diff --git a/rpxy-lib/src/constants.rs b/rpxy-lib/src/constants.rs index 9d7fb5e..39a93e7 100644 --- a/rpxy-lib/src/constants.rs +++ b/rpxy-lib/src/constants.rs @@ -17,7 +17,7 @@ pub const LOAD_CERTS_ONLY_WHEN_UPDATED: bool = true; // pub const H3_REQUEST_BUF_SIZE: usize = 65_536; // 64KB // handled by quinn #[allow(non_snake_case)] -#[cfg(feature = "http3")] +#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] pub mod H3 { pub const ALT_SVC_MAX_AGE: u32 = 3600; pub const REQUEST_MAX_BODY_SIZE: usize = 268_435_456; // 256MB diff --git a/rpxy-lib/src/error.rs b/rpxy-lib/src/error.rs index 3407e8a..dd88a9a 100644 --- a/rpxy-lib/src/error.rs +++ b/rpxy-lib/src/error.rs @@ -37,14 +37,22 @@ pub enum RpxyError { // #[error("Toml Deserialization Error")] // TomlDe(#[from] toml::de::Error), - #[cfg(feature = "http3")] + #[cfg(feature = "http3-quinn")] #[error("Quic Connection Error")] QuicConn(#[from] quinn::ConnectionError), - #[cfg(feature = "http3")] + #[cfg(feature = "http3-s2n")] + #[error("Quic Connection Error [s2n-quic]")] + QUicConn(#[from] s2n_quic::connection::Error), + + #[cfg(feature = "http3-quinn")] #[error("H3 Error")] H3(#[from] h3::Error), + #[cfg(feature = "http3-s2n")] + #[error("H3 Error [s2n-quic]")] + H3(#[from] s2n_quic_h3::h3::Error), + #[error("rustls Connection Error")] Rustls(#[from] rustls::Error), diff --git a/rpxy-lib/src/globals.rs b/rpxy-lib/src/globals.rs index 44808dd..6186d84 100644 --- a/rpxy-lib/src/globals.rs +++ b/rpxy-lib/src/globals.rs @@ -53,19 +53,19 @@ pub struct ProxyConfig { // experimentals pub sni_consistency: bool, // Handler // All need to make packet acceptor - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] pub http3: bool, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] pub h3_alt_svc_max_age: u32, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] pub h3_request_max_body_size: usize, - #[cfg(feature = "http3")] - pub h3_max_concurrent_bidistream: quinn::VarInt, - #[cfg(feature = "http3")] - pub h3_max_concurrent_unistream: quinn::VarInt, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] + pub h3_max_concurrent_bidistream: u32, + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] + pub h3_max_concurrent_unistream: u32, + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] pub h3_max_concurrent_connections: u32, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] pub h3_max_idle_timeout: Option, } @@ -87,19 +87,19 @@ impl Default for ProxyConfig { sni_consistency: true, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] http3: false, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] h3_alt_svc_max_age: H3::ALT_SVC_MAX_AGE, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] h3_request_max_body_size: H3::REQUEST_MAX_BODY_SIZE, - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] h3_max_concurrent_connections: H3::MAX_CONCURRENT_CONNECTIONS, - #[cfg(feature = "http3")] - h3_max_concurrent_bidistream: H3::MAX_CONCURRENT_BIDISTREAM.into(), - #[cfg(feature = "http3")] - h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM.into(), - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] + h3_max_concurrent_bidistream: H3::MAX_CONCURRENT_BIDISTREAM, + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] + h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM, + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] h3_max_idle_timeout: Some(Duration::from_secs(H3::MAX_IDLE_TIMEOUT)), } } diff --git a/rpxy-lib/src/handler/handler_main.rs b/rpxy-lib/src/handler/handler_main.rs index f6c4dc7..0b554ae 100644 --- a/rpxy-lib/src/handler/handler_main.rs +++ b/rpxy-lib/src/handler/handler_main.rs @@ -210,7 +210,7 @@ where remove_hop_header(headers); add_header_entry_overwrite_if_exist(headers, "server", env!("CARGO_PKG_NAME"))?; - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] { // Manipulate ALT_SVC allowing h3 in response message only when mutual TLS is not enabled // TODO: This is a workaround for avoiding a client authentication in HTTP/3 @@ -235,7 +235,7 @@ where headers.remove(header::ALT_SVC.as_str()); } } - #[cfg(not(feature = "http3"))] + #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))] { if let Some(port) = self.globals.proxy_config.https_port { headers.remove(header::ALT_SVC.as_str()); diff --git a/rpxy-lib/src/lib.rs b/rpxy-lib/src/lib.rs index 1915b68..c472b05 100644 --- a/rpxy-lib/src/lib.rs +++ b/rpxy-lib/src/lib.rs @@ -23,6 +23,9 @@ pub mod reexports { pub use rustls::{Certificate, PrivateKey}; } +#[cfg(all(feature = "http3-quinn", feature = "http3-s2n"))] +compile_error!("feature \"http3-quinn\" and feature \"http3-s2n\" cannot be enabled at the same time"); + /// Entrypoint that creates and spawns tasks of reverse proxy services pub async fn entrypoint( proxy_config: &ProxyConfig, @@ -44,7 +47,7 @@ where if proxy_config.https_port.is_some() { info!("Listen port: {} (for TLS)", proxy_config.https_port.unwrap()); } - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] if proxy_config.http3 { info!("Experimental HTTP/3.0 is enabled. Note it is still very unstable."); } diff --git a/rpxy-lib/src/proxy/crypto_service.rs b/rpxy-lib/src/proxy/crypto_service.rs index 8675a1d..d6191e6 100644 --- a/rpxy-lib/src/proxy/crypto_service.rs +++ b/rpxy-lib/src/proxy/crypto_service.rs @@ -22,7 +22,10 @@ where pub type SniServerCryptoMap = HashMap>; pub struct ServerCrypto { // For Quic/HTTP3, only servers with no client authentication + #[cfg(feature = "http3-quinn")] pub inner_global_no_client_auth: Arc, + #[cfg(feature = "http3-s2n")] + pub inner_global_no_client_auth: s2n_quic_rustls::Server, // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers pub inner_local_map: Arc, } @@ -68,7 +71,22 @@ impl TryInto> for &ServerCryptoBase { type Error = anyhow::Error; fn try_into(self) -> Result, Self::Error> { - let mut resolver_global = ResolvesServerCertUsingSni::new(); + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] + let server_crypto_global = self.build_server_crypto_global()?; + let server_crypto_local_map: SniServerCryptoMap = self.build_server_crypto_local_map()?; + + Ok(Arc::new(ServerCrypto { + #[cfg(feature = "http3-quinn")] + inner_global_no_client_auth: Arc::new(server_crypto_global), + #[cfg(feature = "http3-s2n")] + inner_global_no_client_auth: server_crypto_global, + inner_local_map: Arc::new(server_crypto_local_map), + })) + } +} + +impl ServerCryptoBase { + fn build_server_crypto_local_map(&self) -> Result> { let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default(); for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { @@ -93,16 +111,7 @@ impl TryInto> for &ServerCryptoBase { } // add client certificate if specified - if certs_and_keys.client_ca_certs.is_none() { - // aggregated server config for no client auth server for http3 - if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) { - error!( - "{}: Failed to read some certificates and keys {}", - server_name.as_str(), - e - ) - } - } else { + if certs_and_keys.client_ca_certs.is_some() { // add client certificate if specified match certs_and_keys.parse_client_ca_certs() { Ok((owned_trust_anchors, _subject_key_ids)) => { @@ -120,14 +129,14 @@ impl TryInto> for &ServerCryptoBase { let mut server_config_local = if client_ca_roots_local.is_empty() { // with no client auth, enable http1.1 -- 3 - #[cfg(not(feature = "http3"))] + #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))] { ServerConfig::builder() .with_safe_defaults() .with_no_client_auth() .with_cert_resolver(Arc::new(resolver_local)) } - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] { let mut sc = ServerConfig::builder() .with_safe_defaults() @@ -150,6 +159,33 @@ impl TryInto> for &ServerCryptoBase { server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local)); } + Ok(server_crypto_local_map) + } + + #[cfg(feature = "http3-quinn")] + fn build_server_crypto_global(&self) -> Result> { + let mut resolver_global = ResolvesServerCertUsingSni::new(); + + for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { + let server_name: String = server_name_bytes_exp.try_into()?; + + // Parse server certificates and private keys + let Ok(certified_key): Result = certs_and_keys.parse_server_certs_and_keys() else { + warn!("Failed to add certificate for {}", server_name); + continue; + }; + + if certs_and_keys.client_ca_certs.is_none() { + // aggregated server config for no client auth server for http3 + if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) { + error!( + "{}: Failed to read some certificates and keys {}", + server_name.as_str(), + e + ) + } + } + } ////////////// let mut server_crypto_global = ServerConfig::builder() @@ -159,23 +195,82 @@ impl TryInto> for &ServerCryptoBase { ////////////////////////////// - #[cfg(feature = "http3")] - { - server_crypto_global.alpn_protocols = vec![ - b"h3".to_vec(), - b"hq-29".to_vec(), // TODO: remove later? - b"h2".to_vec(), - b"http/1.1".to_vec(), - ]; - } - #[cfg(not(feature = "http3"))] - { - server_crypto_global.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; - } + server_crypto_global.alpn_protocols = vec![ + b"h3".to_vec(), + b"hq-29".to_vec(), // TODO: remove later? + b"h2".to_vec(), + b"http/1.1".to_vec(), + ]; + Ok(server_crypto_global) + } - Ok(Arc::new(ServerCrypto { - inner_global_no_client_auth: Arc::new(server_crypto_global), - inner_local_map: Arc::new(server_crypto_local_map), - })) + #[cfg(feature = "http3-s2n")] + fn build_server_crypto_global(&self) -> Result> { + let mut resolver_global = s2n_quic_rustls::rustls::server::ResolvesServerCertUsingSni::new(); + + for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() { + let server_name: String = server_name_bytes_exp.try_into()?; + + // Parse server certificates and private keys + let Ok(certified_key) = parse_server_certs_and_keys_s2n(certs_and_keys) else { + warn!("Failed to add certificate for {}", server_name); + continue; + }; + + if certs_and_keys.client_ca_certs.is_none() { + // aggregated server config for no client auth server for http3 + if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) { + error!( + "{}: Failed to read some certificates and keys {}", + server_name.as_str(), + e + ) + } + } + } + let alpn = vec![ + b"h3".to_vec(), + b"hq-29".to_vec(), // TODO: remove later? + b"h2".to_vec(), + b"http/1.1".to_vec(), + ]; + let server_crypto_global = s2n_quic::provider::tls::rustls::Server::builder() + .with_cert_resolver(Arc::new(resolver_global)) + .map_err(|e| anyhow::anyhow!(e))? + .with_application_protocols(alpn.iter()) + .map_err(|e| anyhow::anyhow!(e))? + .build() + .map_err(|e| anyhow::anyhow!(e))?; + Ok(server_crypto_global) } } + +#[cfg(feature = "http3-s2n")] +/// This is workaround for the version difference between rustls and s2n-quic-rustls +fn parse_server_certs_and_keys_s2n( + certs_and_keys: &CertsAndKeys, +) -> Result { + let signing_key = certs_and_keys + .cert_keys + .iter() + .find_map(|k| { + let s2n_private_key = s2n_quic_rustls::PrivateKey(k.0.clone()); + if let Ok(sk) = s2n_quic_rustls::rustls::sign::any_supported_type(&s2n_private_key) { + Some(sk) + } else { + None + } + }) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Unable to find a valid certificate and key", + ) + })?; + let certs: Vec<_> = certs_and_keys + .certs + .iter() + .map(|c| s2n_quic_rustls::rustls::Certificate(c.0.clone())) + .collect(); + Ok(s2n_quic_rustls::rustls::sign::CertifiedKey::new(certs, signing_key)) +} diff --git a/rpxy-lib/src/proxy/mod.rs b/rpxy-lib/src/proxy/mod.rs index 05d63b0..0551b62 100644 --- a/rpxy-lib/src/proxy/mod.rs +++ b/rpxy-lib/src/proxy/mod.rs @@ -1,10 +1,12 @@ mod crypto_service; mod proxy_client_cert; -#[cfg(feature = "http3")] +#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] mod proxy_h3; mod proxy_main; -#[cfg(feature = "http3")] -mod proxy_quic; +#[cfg(feature = "http3-quinn")] +mod proxy_quic_quinn; +#[cfg(feature = "http3-s2n")] +mod proxy_quic_s2n; mod proxy_tls; mod socket; diff --git a/rpxy-lib/src/proxy/proxy_h3.rs b/rpxy-lib/src/proxy/proxy_h3.rs index eac6dbf..7773ad9 100644 --- a/rpxy-lib/src/proxy/proxy_h3.rs +++ b/rpxy-lib/src/proxy/proxy_h3.rs @@ -1,8 +1,11 @@ use super::Proxy; use crate::{certs::CryptoSource, error::*, log::*, utils::ServerNameBytesExp}; use bytes::{Buf, Bytes}; +#[cfg(feature = "http3-quinn")] use h3::{quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream}; use hyper::{client::connect::Connect, Body, Request, Response}; +#[cfg(feature = "http3-s2n")] +use s2n_quic_h3::h3::{self, quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream}; use std::net::SocketAddr; use tokio::time::{timeout, Duration}; diff --git a/rpxy-lib/src/proxy/proxy_quic.rs b/rpxy-lib/src/proxy/proxy_quic_quinn.rs similarity index 98% rename from rpxy-lib/src/proxy/proxy_quic.rs rename to rpxy-lib/src/proxy/proxy_quic_quinn.rs index 0e660c1..fb08420 100644 --- a/rpxy-lib/src/proxy/proxy_quic.rs +++ b/rpxy-lib/src/proxy/proxy_quic_quinn.rs @@ -19,7 +19,7 @@ where &self, mut server_crypto_rx: ReloaderReceiver, ) -> Result<()> { - info!("Start UDP proxy serving with HTTP/3 request for configured host names"); + info!("Start UDP proxy serving with HTTP/3 request for configured host names [quinn]"); // first set as null config server let rustls_server_config = ServerConfig::builder() .with_safe_default_cipher_suites() @@ -30,8 +30,8 @@ where let mut transport_config_quic = TransportConfig::default(); transport_config_quic - .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream) - .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream) + .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream.into()) + .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream.into()) .max_idle_timeout( self .globals diff --git a/rpxy-lib/src/proxy/proxy_quic_s2n.rs b/rpxy-lib/src/proxy/proxy_quic_s2n.rs new file mode 100644 index 0000000..e0c41a5 --- /dev/null +++ b/rpxy-lib/src/proxy/proxy_quic_s2n.rs @@ -0,0 +1,135 @@ +use super::{ + crypto_service::{ServerCrypto, ServerCryptoBase}, + proxy_main::Proxy, +}; +use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName}; +use hot_reload::ReloaderReceiver; +use hyper::client::connect::Connect; +use s2n_quic::provider; +use std::sync::Arc; + +impl Proxy +where + T: Connect + Clone + Sync + Send + 'static, + U: CryptoSource + Clone + Sync + Send + 'static, +{ + pub(super) async fn listener_service_h3( + &self, + mut server_crypto_rx: ReloaderReceiver, + ) -> Result<()> { + info!("Start UDP proxy serving with HTTP/3 request for configured host names [s2n-quic]"); + + // initially wait for receipt + let mut server_crypto: Option> = { + let _ = server_crypto_rx.changed().await; + let sc = self.receive_server_crypto(server_crypto_rx.clone())?; + Some(sc) + }; + + // event loop + loop { + tokio::select! { + v = self.serve_connection(&server_crypto) => { + if let Err(e) = v { + error!("Quic connection event loop illegally shutdown [s2n-quic] {e}"); + break; + } + } + _ = server_crypto_rx.changed() => { + server_crypto = match self.receive_server_crypto(server_crypto_rx.clone()) { + Ok(sc) => Some(sc), + Err(e) => { + error!("{e}"); + break; + } + }; + } + else => break + } + } + + Ok(()) + } + + fn receive_server_crypto(&self, server_crypto_rx: ReloaderReceiver) -> Result> { + let cert_keys_map = server_crypto_rx.borrow().clone().ok_or_else(|| { + error!("Reloader is broken"); + RpxyError::Other(anyhow!("Reloader is broken")) + })?; + + let server_crypto: Option> = (&cert_keys_map).try_into().ok(); + server_crypto.ok_or_else(|| { + error!("Failed to update server crypto for h3 [s2n-quic]"); + RpxyError::Other(anyhow!("Failed to update server crypto for h3 [s2n-quic]")) + }) + } + + async fn serve_connection(&self, server_crypto: &Option>) -> Result<()> { + // setup UDP socket + let io = provider::io::tokio::Builder::default() + .with_receive_address(self.listening_on)? + .with_reuse_port()? + .build()?; + + // setup limits + let mut limits = provider::limits::Limits::default() + .with_max_open_local_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64) + .map_err(|e| anyhow!(e))? + .with_max_open_remote_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64) + .map_err(|e| anyhow!(e))? + .with_max_open_local_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64) + .map_err(|e| anyhow!(e))? + .with_max_open_remote_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64) + .map_err(|e| anyhow!(e))? + .with_max_active_connection_ids(self.globals.proxy_config.h3_max_concurrent_connections as u64) + .map_err(|e| anyhow!(e))?; + limits = if let Some(v) = self.globals.proxy_config.h3_max_idle_timeout { + limits.with_max_idle_timeout(v).map_err(|e| anyhow!(e))? + } else { + limits + }; + + // setup tls + let Some(server_crypto) = server_crypto else { + warn!("No server crypto is given [s2n-quic]"); + return Err(RpxyError::Other(anyhow!("No server crypto is given [s2n-quic]"))); + }; + let tls = server_crypto.inner_global_no_client_auth.clone(); + + let mut server = s2n_quic::Server::builder() + .with_tls(tls) + .map_err(|e| anyhow::anyhow!(e))? + .with_io(io) + .map_err(|e| anyhow!(e))? + .with_limits(limits) + .map_err(|e| anyhow!(e))? + .start() + .map_err(|e| anyhow!(e))?; + + // quic event loop. this immediately cancels when crypto is updated by tokio::select! + while let Some(new_conn) = server.accept().await { + debug!("New QUIC connection established"); + let Ok(Some(new_server_name)) = new_conn.server_name() else { + warn!("HTTP/3 no SNI is given"); + continue; + }; + debug!("HTTP/3 connection incoming (SNI {:?})", new_server_name); + let self_clone = self.clone(); + + self.globals.runtime_handle.spawn(async move { + let client_addr = new_conn.remote_addr()?; + let quic_connection = s2n_quic_h3::Connection::new(new_conn); + // Timeout is based on underlying quic + if let Err(e) = self_clone + .connection_serve_h3(quic_connection, new_server_name.to_server_name_vec(), client_addr) + .await + { + warn!("QUIC or HTTP/3 connection failed: {}", e); + }; + Ok(()) as Result<()> + }); + } + + Ok(()) + } +} diff --git a/rpxy-lib/src/proxy/proxy_tls.rs b/rpxy-lib/src/proxy/proxy_tls.rs index b937b02..da4205e 100644 --- a/rpxy-lib/src/proxy/proxy_tls.rs +++ b/rpxy-lib/src/proxy/proxy_tls.rs @@ -108,7 +108,7 @@ where .await .map_err(|e| anyhow::anyhow!(e))?; - #[cfg(not(feature = "http3"))] + #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))] { tokio::select! { _= cert_reloader_service.start() => { @@ -124,7 +124,7 @@ where }; Ok(()) } - #[cfg(feature = "http3")] + #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] { if self.globals.proxy_config.http3 { tokio::select! { diff --git a/rpxy-lib/src/proxy/socket.rs b/rpxy-lib/src/proxy/socket.rs index 2151710..a8b9f01 100644 --- a/rpxy-lib/src/proxy/socket.rs +++ b/rpxy-lib/src/proxy/socket.rs @@ -1,8 +1,8 @@ use crate::{error::*, log::*}; -#[cfg(feature = "http3")] +#[cfg(feature = "http3-quinn")] use socket2::{Domain, Protocol, Socket, Type}; use std::net::SocketAddr; -#[cfg(feature = "http3")] +#[cfg(feature = "http3-quinn")] use std::net::UdpSocket; use tokio::net::TcpSocket; @@ -23,7 +23,7 @@ pub(super) fn bind_tcp_socket(listening_on: &SocketAddr) -> Result { Ok(tcp_socket) } -#[cfg(feature = "http3")] +#[cfg(feature = "http3-quinn")] /// Bind UDP socket to the given `SocketAddr`, and returns the UDP socket with `SO_REUSEADDR` and `SO_REUSEPORT` options. /// This option is required to re-bind the socket address when the proxy instance is reconstructed. pub(super) fn bind_udp_socket(listening_on: &SocketAddr) -> Result {