From e25c6fa81fa1ac796e192e25d54300cbf16110c5 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Tue, 28 May 2024 17:52:00 +0900 Subject: [PATCH] wip: integrate certmanager to rpxy-bin along with existing old rustls --- rpxy-bin/Cargo.toml | 8 ++- rpxy-bin/src/config/mod.rs | 2 +- rpxy-bin/src/config/parse.rs | 43 +++++++++++---- rpxy-bin/src/main.rs | 92 +++++++++++++++++++-------------- rpxy-certs/Cargo.toml | 4 +- rpxy-certs/src/error.rs | 6 +++ rpxy-certs/src/lib.rs | 11 ++-- rpxy-certs/src/server_crypto.rs | 67 ++++++++++++++++++++++-- 8 files changed, 171 insertions(+), 62 deletions(-) diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index 4242741..d6dd842 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -26,6 +26,7 @@ rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [ "sticky-cookie", ] } +mimalloc = { version = "*", default-features = false } anyhow = "1.0.86" rustc-hash = "1.1.0" serde = { version = "1.0.202", default-features = false, features = ["derive"] } @@ -39,7 +40,7 @@ tokio = { version = "1.37.0", default-features = false, features = [ ] } async-trait = "0.1.80" rustls-pemfile = "1.0.4" -mimalloc = { version = "*", default-features = false } + # config clap = { version = "4.5.4", features = ["std", "cargo", "wrap_help"] } @@ -50,5 +51,10 @@ hot_reload = "0.1.5" tracing = { version = "0.1.40" } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +################################ +# cert management +rpxy-certs = { path = "../rpxy-certs/", default-features = false, features = [ + "http3", +] } [dev-dependencies] diff --git a/rpxy-bin/src/config/mod.rs b/rpxy-bin/src/config/mod.rs index 09ec2b9..adc4ff2 100644 --- a/rpxy-bin/src/config/mod.rs +++ b/rpxy-bin/src/config/mod.rs @@ -4,6 +4,6 @@ mod toml; pub use { self::toml::ConfigToml, - parse::{build_settings, parse_opts}, + parse::{build_cert_manager, build_settings, parse_opts}, service::ConfigTomlReloader, }; diff --git a/rpxy-bin/src/config/parse.rs b/rpxy-bin/src/config/parse.rs index 15ff240..049f5ee 100644 --- a/rpxy-bin/src/config/parse.rs +++ b/rpxy-bin/src/config/parse.rs @@ -4,7 +4,10 @@ use crate::{ error::{anyhow, ensure}, }; use clap::{Arg, ArgAction}; +use hot_reload::{ReloaderReceiver, ReloaderService}; +use rpxy_certs::{build_cert_reloader, CryptoFileSourceBuilder, CryptoReloader, ServerCryptoBase}; use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig}; +use rustc_hash::FxHashMap as HashMap; /// Parsed options pub struct Opts { @@ -37,20 +40,13 @@ pub fn parse_opts() -> Result { let config_file_path = matches.get_one::("config_file").unwrap().to_owned(); let watch = matches.get_one::("watch").unwrap().to_owned(); - Ok(Opts { - config_file_path, - watch, - }) + Ok(Opts { config_file_path, watch }) } -pub fn build_settings( - config: &ConfigToml, -) -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { - /////////////////////////////////// +pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, AppConfigList), anyhow::Error> { // build proxy config let proxy_config: ProxyConfig = config.try_into()?; - /////////////////////////////////// // backend_apps let apps = config.apps.clone().ok_or(anyhow!("Missing application spec"))?; @@ -95,3 +91,32 @@ pub fn build_settings( Ok((proxy_config, app_config_list)) } + +/* ----------------------- */ +/// Build cert map +pub async fn build_cert_manager( + config: &ConfigToml, +) -> Result< + ( + ReloaderService, + ReloaderReceiver, + ), + anyhow::Error, +> { + let apps = config.apps.as_ref().ok_or(anyhow!("No apps"))?; + let mut crypto_source_map = HashMap::default(); + for app in apps.0.values() { + if let Some(tls) = app.tls.as_ref() { + ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some()); + let server_name = app.server_name.as_ref().ok_or(anyhow!("No server name"))?; + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_path(tls.tls_cert_path.as_ref().unwrap()) + .tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap()) + .client_ca_cert_path(tls.client_ca_cert_path.as_deref()) + .build()?; + crypto_source_map.insert(server_name.to_owned(), crypto_file_source); + } + } + let res = build_cert_reloader(&crypto_source_map, None).await?; + Ok(res) +} diff --git a/rpxy-bin/src/main.rs b/rpxy-bin/src/main.rs index 9aeb971..114e6db 100644 --- a/rpxy-bin/src/main.rs +++ b/rpxy-bin/src/main.rs @@ -8,8 +8,9 @@ mod error; mod log; use crate::{ - config::{build_settings, parse_opts, ConfigToml, ConfigTomlReloader}, + config::{build_cert_manager, build_settings, parse_opts, ConfigToml, ConfigTomlReloader}, constants::CONFIG_WATCH_DELAY_SECS, + error::*, log::*, }; use hot_reload::{ReloaderReceiver, ReloaderService}; @@ -36,13 +37,10 @@ fn main() { std::process::exit(1); } } else { - let (config_service, config_rx) = ReloaderService::::new( - &parsed_opts.config_file_path, - CONFIG_WATCH_DELAY_SECS, - false, - ) - .await - .unwrap(); + let (config_service, config_rx) = + ReloaderService::::new(&parsed_opts.config_file_path, CONFIG_WATCH_DELAY_SECS, false) + .await + .unwrap(); tokio::select! { Err(e) = config_service.start() => { @@ -53,6 +51,9 @@ fn main() { error!("rpxy service existed: {e}"); std::process::exit(1); } + else => { + std::process::exit(0); + } } } }); @@ -63,23 +64,22 @@ async fn rpxy_service_without_watcher( runtime_handle: tokio::runtime::Handle, ) -> Result<(), anyhow::Error> { info!("Start rpxy service"); - let config_toml = match ConfigToml::new(config_file_path) { - Ok(v) => v, - Err(e) => { - error!("Invalid toml file: {e}"); - std::process::exit(1); - } - }; - let (proxy_conf, app_conf) = match build_settings(&config_toml) { - Ok(v) => v, - Err(e) => { - error!("Invalid configuration: {e}"); - return Err(anyhow::anyhow!(e)); - } - }; - entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) + let config_toml = ConfigToml::new(config_file_path).map_err(|e| anyhow!("Invalid toml file: {e}"))?; + let (proxy_conf, app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?; + let (cert_service, cert_rx) = build_cert_manager(&config_toml) .await - .map_err(|e| anyhow::anyhow!(e)) + .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?; + + tokio::select! { + rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) => { + error!("rpxy entrypoint exited"); + rpxy_res.map_err(|e| anyhow!(e)) + } + cert_res = cert_service.start() => { + error!("cert reloader service exited"); + cert_res.map_err(|e| anyhow!(e)) + } + } } async fn rpxy_service_with_watcher( @@ -89,14 +89,15 @@ async fn rpxy_service_with_watcher( info!("Start rpxy service with dynamic config reloader"); // Initial loading config_rx.changed().await?; - let config_toml = config_rx.borrow().clone().unwrap(); - let (mut proxy_conf, mut app_conf) = match build_settings(&config_toml) { - Ok(v) => v, - Err(e) => { - error!("Invalid configuration: {e}"); - return Err(anyhow::anyhow!(e)); - } - }; + let config_toml = config_rx + .borrow() + .clone() + .ok_or(anyhow!("Something wrong in config reloader receiver"))?; + let (mut proxy_conf, mut app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?; + + let (mut cert_service, mut cert_rx) = build_cert_manager(&config_toml) + .await + .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?; // Notifier for proxy service termination let term_notify = std::sync::Arc::new(tokio::sync::Notify::new()); @@ -104,16 +105,15 @@ async fn rpxy_service_with_watcher( // Continuous monitoring loop { tokio::select! { - _ = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => { + rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => { error!("rpxy entrypoint exited"); - break; + return rpxy_res.map_err(|e| anyhow!(e)); } _ = config_rx.changed() => { - if config_rx.borrow().is_none() { + let Some(config_toml) = config_rx.borrow().clone() else { error!("Something wrong in config reloader receiver"); - break; - } - let config_toml = config_rx.borrow().clone().unwrap(); + return Err(anyhow!("Something wrong in config reloader receiver")); + }; match build_settings(&config_toml) { Ok((p, a)) => { (proxy_conf, app_conf) = (p, a) @@ -123,13 +123,27 @@ async fn rpxy_service_with_watcher( continue; } }; + match build_cert_manager(&config_toml).await { + Ok((c, r)) => { + (cert_service, cert_rx) = (c, r) + }, + Err(e) => { + error!("Invalid cert configuration. Configuration does not updated: {e}"); + continue; + } + }; + info!("Configuration updated. Terminate all spawned proxy services and force to re-bind TCP/UDP sockets"); term_notify.notify_waiters(); // tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; } + cert_res = cert_service.start() => { + error!("cert reloader service exited"); + return cert_res.map_err(|e| anyhow!(e)); + } else => break } } - Err(anyhow::anyhow!("rpxy or continuous monitoring service exited")) + Ok(()) } diff --git a/rpxy-certs/Cargo.toml b/rpxy-certs/Cargo.toml index 732901d..2205ebb 100644 --- a/rpxy-certs/Cargo.toml +++ b/rpxy-certs/Cargo.toml @@ -22,6 +22,7 @@ thiserror = { version = "1.0.61" } hot_reload = { version = "0.1.5" } async-trait = { version = "0.1.80" } rustls = { version = "0.23.8", default-features = false, features = [ + "std", "aws_lc_rs", ] } rustls-pemfile = { version = "2.1.2" } @@ -33,9 +34,6 @@ x509-parser = { version = "0.16.0" } [dev-dependencies] tokio = { version = "1.37.0", default-features = false, features = [ - # "net", "rt-multi-thread", - # "time", - # "sync", "macros", ] } diff --git a/rpxy-certs/src/error.rs b/rpxy-certs/src/error.rs index 26e16b7..7b0ebe8 100644 --- a/rpxy-certs/src/error.rs +++ b/rpxy-certs/src/error.rs @@ -18,4 +18,10 @@ pub enum RpxyCertError { /// Error when converting server name bytes to string #[error("Failed to convert server name bytes to string: {0}")] ServerNameBytesToString(#[from] std::string::FromUtf8Error), + /// Rustls error + #[error("Rustls error: {0}")] + RustlsError(#[from] rustls::Error), + /// Rustls CryptoProvider error + #[error("Rustls No default CryptoProvider error")] + NoDefaultCryptoProvider, } diff --git a/rpxy-certs/src/lib.rs b/rpxy-certs/src/lib.rs index 106dbdd..6a262f0 100644 --- a/rpxy-certs/src/lib.rs +++ b/rpxy-certs/src/lib.rs @@ -9,18 +9,17 @@ mod log { pub(super) use tracing::{debug, error, info, warn}; } -use crate::{ - error::*, - reloader_service::{CryptoReloader, DynCryptoSource}, -}; +use crate::{error::*, log::*, reloader_service::DynCryptoSource}; use hot_reload::{ReloaderReceiver, ReloaderService}; use rustc_hash::FxHashMap as HashMap; +use rustls::crypto::{aws_lc_rs, CryptoProvider}; use std::sync::Arc; /* ------------------------------------------------ */ pub use crate::{ certs::SingleServerCertsKeys, crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource}, + reloader_service::CryptoReloader, server_crypto::{ServerCrypto, ServerCryptoBase}, }; @@ -44,6 +43,10 @@ pub async fn build_cert_reloader( where T: CryptoSource + Send + Sync + Clone + 'static, { + info!("Building certificate reloader service"); + // Install aws_lc_rs as default crypto provider for rustls + let _ = CryptoProvider::install_default(aws_lc_rs::default_provider()); + let source = crypto_source_map .iter() .map(|(k, v)| { diff --git a/rpxy-certs/src/server_crypto.rs b/rpxy-certs/src/server_crypto.rs index 53fbcc3..72932d6 100644 --- a/rpxy-certs/src/server_crypto.rs +++ b/rpxy-certs/src/server_crypto.rs @@ -1,6 +1,7 @@ use crate::{certs::SingleServerCertsKeys, error::*, log::*}; use rustc_hash::FxHashMap as HashMap; use rustls::{ + crypto::CryptoProvider, server::{ResolvesServerCertUsingSni, WebPkiClientVerifier}, RootCertStore, ServerConfig, }; @@ -40,7 +41,6 @@ impl TryInto> for &ServerCryptoBase { fn try_into(self) -> Result, Self::Error> { let aggregated = self.build_aggrated_server_crypto()?; let individual = self.build_individual_server_crypto_map()?; - Ok(Arc::new(ServerCrypto { aggregated_config_no_client_auth: Arc::new(aggregated), individual_config_map: Arc::new(individual), @@ -53,6 +53,9 @@ impl ServerCryptoBase { fn build_individual_server_crypto_map(&self) -> Result { let mut server_crypto_map: ServerNameCryptoMap = HashMap::default(); + // AWS LC provider by default + let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?; + for (server_name_bytes, certs_keys) in self.inner.iter() { let server_name = server_name_bytes_to_string(server_name_bytes)?; @@ -69,9 +72,11 @@ impl ServerCryptoBase { // With no client authentication case if !certs_keys.is_mutual_tls() { - let mut server_crypto_local = ServerConfig::builder() + let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions()? .with_no_client_auth() .with_cert_resolver(Arc::new(resolver_local)); + #[cfg(feature = "http3")] { server_crypto_local.alpn_protocols = vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()]; @@ -93,11 +98,14 @@ impl ServerCryptoBase { let trust_anchors_without_skid = trust_anchors.values().map(|ta| ta.to_owned()); client_ca_roots_local.extend(trust_anchors_without_skid); - let Ok(client_cert_verifier) = WebPkiClientVerifier::builder(Arc::new(client_ca_roots_local)).build() else { + let Ok(client_cert_verifier) = + WebPkiClientVerifier::builder_with_provider(Arc::new(client_ca_roots_local), provider.clone()).build() + else { warn!("Failed to build client CA certificate verifier for {server_name}"); continue; }; - let mut server_crypto_local = ServerConfig::builder() + let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions()? .with_client_cert_verifier(client_cert_verifier) .with_cert_resolver(Arc::new(resolver_local)); server_crypto_local.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; @@ -112,6 +120,9 @@ impl ServerCryptoBase { fn build_aggrated_server_crypto(&self) -> Result { let mut resolver_global = ResolvesServerCertUsingSni::new(); + // AWS LC provider by default + let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?; + for (server_name_bytes, certs_keys) in self.inner.iter() { let server_name = server_name_bytes_to_string(server_name_bytes)?; @@ -129,7 +140,8 @@ impl ServerCryptoBase { } } - let mut server_crypto_global = ServerConfig::builder() + let mut server_crypto_global = ServerConfig::builder_with_provider(provider.clone()) + .with_safe_default_protocol_versions()? .with_no_client_auth() .with_cert_resolver(Arc::new(resolver_global)); @@ -145,3 +157,48 @@ impl ServerCryptoBase { Ok(server_crypto_global) } } + +/* ------------------------------------------------ */ +#[cfg(test)] +mod tests { + use super::*; + use crate::{CryptoFileSourceBuilder, CryptoSource}; + use std::convert::TryInto; + + async fn read_file_source() -> SingleServerCertsKeys { + let tls_cert_path = "../example-certs/server.crt"; + let tls_cert_key_path = "../example-certs/server.key"; + let client_ca_cert_path = Some("../example-certs/client.ca.crt"); + let crypto_file_source = CryptoFileSourceBuilder::default() + .tls_cert_key_path(tls_cert_key_path) + .tls_cert_path(tls_cert_path) + .client_ca_cert_path(client_ca_cert_path) + .build(); + crypto_file_source.unwrap().read().await.unwrap() + } + + #[tokio::test] + async fn test_server_crypto_base_try_into() { + let mut server_crypto_base = ServerCryptoBase::default(); + + let single_certs_keys = read_file_source().await; + server_crypto_base.inner.insert(b"localhost".to_vec(), single_certs_keys); + let server_crypto: Arc = (&server_crypto_base).try_into().unwrap(); + assert_eq!(server_crypto.individual_config_map.len(), 1); + + #[cfg(feature = "http3")] + { + assert_eq!( + server_crypto.aggregated_config_no_client_auth.alpn_protocols, + vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()] + ); + } + #[cfg(not(feature = "http3"))] + { + assert_eq!( + server_crypto.aggregated_config_no_client_auth.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec()] + ); + } + } +}