wip: integrate certmanager to rpxy-bin along with existing old rustls

rpxy-bin/Cargo.toml

@@ -26,6 +26,7 @@ rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [
   "sticky-cookie",
 ] }
 
+mimalloc = { version = "*", default-features = false }
 anyhow = "1.0.86"
 rustc-hash = "1.1.0"
 serde = { version = "1.0.202", default-features = false, features = ["derive"] }
@@ -39,7 +40,7 @@ tokio = { version = "1.37.0", default-features = false, features = [
 ] }
 async-trait = "0.1.80"
 rustls-pemfile = "1.0.4"
-mimalloc = { version = "*", default-features = false }
+
 
 # config
 clap = { version = "4.5.4", features = ["std", "cargo", "wrap_help"] }
@@ -50,5 +51,10 @@ hot_reload = "0.1.5"
 tracing = { version = "0.1.40" }
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 
+################################
+# cert management
+rpxy-certs = { path = "../rpxy-certs/", default-features = false, features = [
+  "http3",
+] }
 
 [dev-dependencies]

rpxy-bin/src/config/mod.rs

@@ -4,6 +4,6 @@ mod toml;
 
 pub use {
   self::toml::ConfigToml,
-  parse::{build_settings, parse_opts},
+  parse::{build_cert_manager, build_settings, parse_opts},
   service::ConfigTomlReloader,
 };

rpxy-bin/src/config/parse.rs

@@ -4,7 +4,10 @@ use crate::{
   error::{anyhow, ensure},
 };
 use clap::{Arg, ArgAction};
+use hot_reload::{ReloaderReceiver, ReloaderService};
+use rpxy_certs::{build_cert_reloader, CryptoFileSourceBuilder, CryptoReloader, ServerCryptoBase};
 use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig};
+use rustc_hash::FxHashMap as HashMap;
 
 /// Parsed options
 pub struct Opts {
@@ -37,20 +40,13 @@ pub fn parse_opts() -> Result<Opts, anyhow::Error> {
   let config_file_path = matches.get_one::<String>("config_file").unwrap().to_owned();
   let watch = matches.get_one::<bool>("watch").unwrap().to_owned();
 
-  Ok(Opts {
+  Ok(Opts { config_file_path, watch })
-    config_file_path,
-    watch,
-  })
 }
 
-pub fn build_settings(
+pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, AppConfigList<CryptoFileSource>), anyhow::Error> {
-  config: &ConfigToml,
-) -> std::result::Result<(ProxyConfig, AppConfigList<CryptoFileSource>), anyhow::Error> {
-  ///////////////////////////////////
   // build proxy config
   let proxy_config: ProxyConfig = config.try_into()?;
 
-  ///////////////////////////////////
   // backend_apps
   let apps = config.apps.clone().ok_or(anyhow!("Missing application spec"))?;
 
@@ -95,3 +91,32 @@ pub fn build_settings(
 
   Ok((proxy_config, app_config_list))
 }
+
+/* ----------------------- */
+/// Build cert map
+pub async fn build_cert_manager(
+  config: &ConfigToml,
+) -> Result<
+  (
+    ReloaderService<CryptoReloader, ServerCryptoBase>,
+    ReloaderReceiver<ServerCryptoBase>,
+  ),
+  anyhow::Error,
+> {
+  let apps = config.apps.as_ref().ok_or(anyhow!("No apps"))?;
+  let mut crypto_source_map = HashMap::default();
+  for app in apps.0.values() {
+    if let Some(tls) = app.tls.as_ref() {
+      ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some());
+      let server_name = app.server_name.as_ref().ok_or(anyhow!("No server name"))?;
+      let crypto_file_source = CryptoFileSourceBuilder::default()
+        .tls_cert_path(tls.tls_cert_path.as_ref().unwrap())
+        .tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap())
+        .client_ca_cert_path(tls.client_ca_cert_path.as_deref())
+        .build()?;
+      crypto_source_map.insert(server_name.to_owned(), crypto_file_source);
+    }
+  }
+  let res = build_cert_reloader(&crypto_source_map, None).await?;
+  Ok(res)
+}

rpxy-bin/src/main.rs

@@ -8,8 +8,9 @@ mod error;
 mod log;
 
 use crate::{
-  config::{build_settings, parse_opts, ConfigToml, ConfigTomlReloader},
+  config::{build_cert_manager, build_settings, parse_opts, ConfigToml, ConfigTomlReloader},
   constants::CONFIG_WATCH_DELAY_SECS,
+  error::*,
   log::*,
 };
 use hot_reload::{ReloaderReceiver, ReloaderService};
@@ -36,13 +37,10 @@ fn main() {
         std::process::exit(1);
       }
     } else {
-      let (config_service, config_rx) = ReloaderService::<ConfigTomlReloader, ConfigToml>::new(
+      let (config_service, config_rx) =
-        &parsed_opts.config_file_path,
+        ReloaderService::<ConfigTomlReloader, ConfigToml>::new(&parsed_opts.config_file_path, CONFIG_WATCH_DELAY_SECS, false)
-        CONFIG_WATCH_DELAY_SECS,
+          .await
-        false,
+          .unwrap();
-      )
-      .await
-      .unwrap();
 
       tokio::select! {
         Err(e) = config_service.start() => {
@@ -53,6 +51,9 @@ fn main() {
           error!("rpxy service existed: {e}");
           std::process::exit(1);
         }
+        else => {
+          std::process::exit(0);
+        }
       }
     }
   });
@@ -63,23 +64,22 @@ async fn rpxy_service_without_watcher(
   runtime_handle: tokio::runtime::Handle,
 ) -> Result<(), anyhow::Error> {
   info!("Start rpxy service");
-  let config_toml = match ConfigToml::new(config_file_path) {
+  let config_toml = ConfigToml::new(config_file_path).map_err(|e| anyhow!("Invalid toml file: {e}"))?;
-    Ok(v) => v,
+  let (proxy_conf, app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?;
-    Err(e) => {
+  let (cert_service, cert_rx) = build_cert_manager(&config_toml)
-      error!("Invalid toml file: {e}");
-      std::process::exit(1);
-    }
-  };
-  let (proxy_conf, app_conf) = match build_settings(&config_toml) {
-    Ok(v) => v,
-    Err(e) => {
-      error!("Invalid configuration: {e}");
-      return Err(anyhow::anyhow!(e));
-    }
-  };
-  entrypoint(&proxy_conf, &app_conf, &runtime_handle, None)
     .await
-    .map_err(|e| anyhow::anyhow!(e))
+    .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?;
+
+  tokio::select! {
+    rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) => {
+      error!("rpxy entrypoint exited");
+      rpxy_res.map_err(|e| anyhow!(e))
+    }
+    cert_res = cert_service.start() => {
+      error!("cert reloader service exited");
+      cert_res.map_err(|e| anyhow!(e))
+    }
+  }
 }
 
 async fn rpxy_service_with_watcher(
@@ -89,14 +89,15 @@ async fn rpxy_service_with_watcher(
   info!("Start rpxy service with dynamic config reloader");
   // Initial loading
   config_rx.changed().await?;
-  let config_toml = config_rx.borrow().clone().unwrap();
+  let config_toml = config_rx
-  let (mut proxy_conf, mut app_conf) = match build_settings(&config_toml) {
+    .borrow()
-    Ok(v) => v,
+    .clone()
-    Err(e) => {
+    .ok_or(anyhow!("Something wrong in config reloader receiver"))?;
-      error!("Invalid configuration: {e}");
+  let (mut proxy_conf, mut app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?;
-      return Err(anyhow::anyhow!(e));
+
-    }
+  let (mut cert_service, mut cert_rx) = build_cert_manager(&config_toml)
-  };
+    .await
+    .map_err(|e| anyhow!("Invalid cert configuration: {e}"))?;
 
   // Notifier for proxy service termination
   let term_notify = std::sync::Arc::new(tokio::sync::Notify::new());
@@ -104,16 +105,15 @@ async fn rpxy_service_with_watcher(
   // Continuous monitoring
   loop {
     tokio::select! {
-      _ = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => {
+      rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => {
         error!("rpxy entrypoint exited");
-        break;
+        return rpxy_res.map_err(|e| anyhow!(e));
       }
       _ = config_rx.changed() => {
-        if config_rx.borrow().is_none() {
+        let Some(config_toml) = config_rx.borrow().clone() else {
           error!("Something wrong in config reloader receiver");
-          break;
+          return Err(anyhow!("Something wrong in config reloader receiver"));
-        }
+        };
-        let config_toml = config_rx.borrow().clone().unwrap();
         match build_settings(&config_toml) {
           Ok((p, a)) => {
             (proxy_conf, app_conf) = (p, a)
@@ -123,13 +123,27 @@ async fn rpxy_service_with_watcher(
             continue;
           }
         };
+        match build_cert_manager(&config_toml).await {
+          Ok((c, r)) => {
+            (cert_service, cert_rx) = (c, r)
+          },
+          Err(e) => {
+            error!("Invalid cert configuration. Configuration does not updated: {e}");
+            continue;
+          }
+        };
+
         info!("Configuration updated. Terminate all spawned proxy services and force to re-bind TCP/UDP sockets");
         term_notify.notify_waiters();
         // tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
       }
+      cert_res = cert_service.start() => {
+        error!("cert reloader service exited");
+        return cert_res.map_err(|e| anyhow!(e));
+      }
       else => break
     }
   }
 
-  Err(anyhow::anyhow!("rpxy or continuous monitoring service exited"))
+  Ok(())
 }

rpxy-certs/Cargo.toml

@@ -22,6 +22,7 @@ thiserror = { version = "1.0.61" }
 hot_reload = { version = "0.1.5" }
 async-trait = { version = "0.1.80" }
 rustls = { version = "0.23.8", default-features = false, features = [
+  "std",
   "aws_lc_rs",
 ] }
 rustls-pemfile = { version = "2.1.2" }
@@ -33,9 +34,6 @@ x509-parser = { version = "0.16.0" }
 
 [dev-dependencies]
 tokio = { version = "1.37.0", default-features = false, features = [
-  # "net",
   "rt-multi-thread",
-  # "time",
-  # "sync",
   "macros",
 ] }

rpxy-certs/src/error.rs

@@ -18,4 +18,10 @@ pub enum RpxyCertError {
   /// Error when converting server name bytes to string
   #[error("Failed to convert server name bytes to string: {0}")]
   ServerNameBytesToString(#[from] std::string::FromUtf8Error),
+  /// Rustls error
+  #[error("Rustls error: {0}")]
+  RustlsError(#[from] rustls::Error),
+  /// Rustls CryptoProvider error
+  #[error("Rustls No default CryptoProvider error")]
+  NoDefaultCryptoProvider,
 }

rpxy-certs/src/lib.rs

@@ -9,18 +9,17 @@ mod log {
   pub(super) use tracing::{debug, error, info, warn};
 }
 
-use crate::{
+use crate::{error::*, log::*, reloader_service::DynCryptoSource};
-  error::*,
-  reloader_service::{CryptoReloader, DynCryptoSource},
-};
 use hot_reload::{ReloaderReceiver, ReloaderService};
 use rustc_hash::FxHashMap as HashMap;
+use rustls::crypto::{aws_lc_rs, CryptoProvider};
 use std::sync::Arc;
 
 /* ------------------------------------------------ */
 pub use crate::{
   certs::SingleServerCertsKeys,
   crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource},
+  reloader_service::CryptoReloader,
   server_crypto::{ServerCrypto, ServerCryptoBase},
 };
 
@@ -44,6 +43,10 @@ pub async fn build_cert_reloader<T>(
 where
   T: CryptoSource<Error = RpxyCertError> + Send + Sync + Clone + 'static,
 {
+  info!("Building certificate reloader service");
+  // Install aws_lc_rs as default crypto provider for rustls
+  let _ = CryptoProvider::install_default(aws_lc_rs::default_provider());
+
   let source = crypto_source_map
     .iter()
     .map(|(k, v)| {

rpxy-certs/src/server_crypto.rs

@@ -1,6 +1,7 @@
 use crate::{certs::SingleServerCertsKeys, error::*, log::*};
 use rustc_hash::FxHashMap as HashMap;
 use rustls::{
+  crypto::CryptoProvider,
   server::{ResolvesServerCertUsingSni, WebPkiClientVerifier},
   RootCertStore, ServerConfig,
 };
@@ -40,7 +41,6 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
   fn try_into(self) -> Result<Arc<ServerCrypto>, Self::Error> {
     let aggregated = self.build_aggrated_server_crypto()?;
     let individual = self.build_individual_server_crypto_map()?;
-
     Ok(Arc::new(ServerCrypto {
       aggregated_config_no_client_auth: Arc::new(aggregated),
       individual_config_map: Arc::new(individual),
@@ -53,6 +53,9 @@ impl ServerCryptoBase {
   fn build_individual_server_crypto_map(&self) -> Result<ServerNameCryptoMap, RpxyCertError> {
     let mut server_crypto_map: ServerNameCryptoMap = HashMap::default();
 
+    // AWS LC provider by default
+    let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?;
+
     for (server_name_bytes, certs_keys) in self.inner.iter() {
       let server_name = server_name_bytes_to_string(server_name_bytes)?;
 
@@ -69,9 +72,11 @@ impl ServerCryptoBase {
 
       // With no client authentication case
       if !certs_keys.is_mutual_tls() {
-        let mut server_crypto_local = ServerConfig::builder()
+        let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone())
+          .with_safe_default_protocol_versions()?
           .with_no_client_auth()
           .with_cert_resolver(Arc::new(resolver_local));
+
         #[cfg(feature = "http3")]
         {
           server_crypto_local.alpn_protocols = vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()];
@@ -93,11 +98,14 @@ impl ServerCryptoBase {
       let trust_anchors_without_skid = trust_anchors.values().map(|ta| ta.to_owned());
       client_ca_roots_local.extend(trust_anchors_without_skid);
 
-      let Ok(client_cert_verifier) = WebPkiClientVerifier::builder(Arc::new(client_ca_roots_local)).build() else {
+      let Ok(client_cert_verifier) =
+        WebPkiClientVerifier::builder_with_provider(Arc::new(client_ca_roots_local), provider.clone()).build()
+      else {
         warn!("Failed to build client CA certificate verifier for {server_name}");
         continue;
       };
-      let mut server_crypto_local = ServerConfig::builder()
+      let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone())
+        .with_safe_default_protocol_versions()?
         .with_client_cert_verifier(client_cert_verifier)
         .with_cert_resolver(Arc::new(resolver_local));
       server_crypto_local.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
@@ -112,6 +120,9 @@ impl ServerCryptoBase {
   fn build_aggrated_server_crypto(&self) -> Result<ServerConfig, RpxyCertError> {
     let mut resolver_global = ResolvesServerCertUsingSni::new();
 
+    // AWS LC provider by default
+    let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?;
+
     for (server_name_bytes, certs_keys) in self.inner.iter() {
       let server_name = server_name_bytes_to_string(server_name_bytes)?;
 
@@ -129,7 +140,8 @@ impl ServerCryptoBase {
       }
     }
 
-    let mut server_crypto_global = ServerConfig::builder()
+    let mut server_crypto_global = ServerConfig::builder_with_provider(provider.clone())
+      .with_safe_default_protocol_versions()?
       .with_no_client_auth()
       .with_cert_resolver(Arc::new(resolver_global));
 
@@ -145,3 +157,48 @@ impl ServerCryptoBase {
     Ok(server_crypto_global)
   }
 }
+
+/* ------------------------------------------------ */
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use crate::{CryptoFileSourceBuilder, CryptoSource};
+  use std::convert::TryInto;
+
+  async fn read_file_source() -> SingleServerCertsKeys {
+    let tls_cert_path = "../example-certs/server.crt";
+    let tls_cert_key_path = "../example-certs/server.key";
+    let client_ca_cert_path = Some("../example-certs/client.ca.crt");
+    let crypto_file_source = CryptoFileSourceBuilder::default()
+      .tls_cert_key_path(tls_cert_key_path)
+      .tls_cert_path(tls_cert_path)
+      .client_ca_cert_path(client_ca_cert_path)
+      .build();
+    crypto_file_source.unwrap().read().await.unwrap()
+  }
+
+  #[tokio::test]
+  async fn test_server_crypto_base_try_into() {
+    let mut server_crypto_base = ServerCryptoBase::default();
+
+    let single_certs_keys = read_file_source().await;
+    server_crypto_base.inner.insert(b"localhost".to_vec(), single_certs_keys);
+    let server_crypto: Arc<ServerCrypto> = (&server_crypto_base).try_into().unwrap();
+    assert_eq!(server_crypto.individual_config_map.len(), 1);
+
+    #[cfg(feature = "http3")]
+    {
+      assert_eq!(
+        server_crypto.aggregated_config_no_client_auth.alpn_protocols,
+        vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()]
+      );
+    }
+    #[cfg(not(feature = "http3"))]
+    {
+      assert_eq!(
+        server_crypto.aggregated_config_no_client_auth.alpn_protocols,
+        vec![b"h2".to_vec(), b"http/1.1".to_vec()]
+      );
+    }
+  }
+}