tuxmain/rust-rpxy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wip: integrate certmanager to rpxy-bin along with existing old rustls

Browse source
This commit is contained in:
Jun Kurihara 2024-05-28 17:52:00 +09:00
commit e25c6fa81f
No known key found for this signature in database
GPG key ID: 48ADFD173ED22B03
8 changed files with 171 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

8
rpxy-bin/Cargo.toml
View file

@ -26,6 +26,7 @@ rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [
"sticky-cookie",
] }
mimalloc = { version = "*", default-features = false }
anyhow = "1.0.86"
rustc-hash = "1.1.0"
serde = { version = "1.0.202", default-features = false, features = ["derive"] }
@ -39,7 +40,7 @@ tokio = { version = "1.37.0", default-features = false, features = [
] }
async-trait = "0.1.80"
rustls-pemfile = "1.0.4"
mimalloc = { version = "*", default-features = false }
# config
clap = { version = "4.5.4", features = ["std", "cargo", "wrap_help"] }
@ -50,5 +51,10 @@ hot_reload = "0.1.5"
tracing = { version = "0.1.40" }
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
################################
# cert management
rpxy-certs = { path = "../rpxy-certs/", default-features = false, features = [
"http3",
] }
[dev-dependencies]

2
rpxy-bin/src/config/mod.rs
View file

@ -4,6 +4,6 @@ mod toml;
pub use {
self::toml::ConfigToml,
parse::{build_settings, parse_opts},
parse::{build_cert_manager, build_settings, parse_opts},
service::ConfigTomlReloader,
};

43
rpxy-bin/src/config/parse.rs
View file

@ -4,7 +4,10 @@ use crate::{
error::{anyhow, ensure},
};
use clap::{Arg, ArgAction};
use hot_reload::{ReloaderReceiver, ReloaderService};
use rpxy_certs::{build_cert_reloader, CryptoFileSourceBuilder, CryptoReloader, ServerCryptoBase};
use rpxy_lib::{AppConfig, AppConfigList, ProxyConfig};
use rustc_hash::FxHashMap as HashMap;
/// Parsed options
pub struct Opts {
@ -37,20 +40,13 @@ pub fn parse_opts() -> Result<Opts, anyhow::Error> {
let config_file_path = matches.get_one::<String>("config_file").unwrap().to_owned();
let watch = matches.get_one::<bool>("watch").unwrap().to_owned();
Ok(Opts {
config_file_path,
watch,
})
Ok(Opts { config_file_path, watch })
}
pub fn build_settings(
config: &ConfigToml,
) -> std::result::Result<(ProxyConfig, AppConfigList<CryptoFileSource>), anyhow::Error> {
///////////////////////////////////
pub fn build_settings(config: &ConfigToml) -> std::result::Result<(ProxyConfig, AppConfigList<CryptoFileSource>), anyhow::Error> {
// build proxy config
let proxy_config: ProxyConfig = config.try_into()?;
///////////////////////////////////
// backend_apps
let apps = config.apps.clone().ok_or(anyhow!("Missing application spec"))?;
@ -95,3 +91,32 @@ pub fn build_settings(
Ok((proxy_config, app_config_list))
}
/* ----------------------- */
/// Build cert map
pub async fn build_cert_manager(
config: &ConfigToml,
) -> Result<
(
ReloaderService<CryptoReloader, ServerCryptoBase>,
ReloaderReceiver<ServerCryptoBase>,
),
anyhow::Error,
> {
let apps = config.apps.as_ref().ok_or(anyhow!("No apps"))?;
let mut crypto_source_map = HashMap::default();
for app in apps.0.values() {
if let Some(tls) = app.tls.as_ref() {
ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some());
let server_name = app.server_name.as_ref().ok_or(anyhow!("No server name"))?;
let crypto_file_source = CryptoFileSourceBuilder::default()
.tls_cert_path(tls.tls_cert_path.as_ref().unwrap())
.tls_cert_key_path(tls.tls_cert_key_path.as_ref().unwrap())
.client_ca_cert_path(tls.client_ca_cert_path.as_deref())
.build()?;
crypto_source_map.insert(server_name.to_owned(), crypto_file_source);
}
}
let res = build_cert_reloader(&crypto_source_map, None).await?;
Ok(res)
}

92
rpxy-bin/src/main.rs
View file

@ -8,8 +8,9 @@ mod error;
mod log;
use crate::{
config::{build_settings, parse_opts, ConfigToml, ConfigTomlReloader},
config::{build_cert_manager, build_settings, parse_opts, ConfigToml, ConfigTomlReloader},
constants::CONFIG_WATCH_DELAY_SECS,
error::*,
log::*,
};
use hot_reload::{ReloaderReceiver, ReloaderService};
@ -36,13 +37,10 @@ fn main() {
std::process::exit(1);
}
} else {
let (config_service, config_rx) = ReloaderService::<ConfigTomlReloader, ConfigToml>::new(
&parsed_opts.config_file_path,
CONFIG_WATCH_DELAY_SECS,
false,
)
.await
.unwrap();
let (config_service, config_rx) =
ReloaderService::<ConfigTomlReloader, ConfigToml>::new(&parsed_opts.config_file_path, CONFIG_WATCH_DELAY_SECS, false)
.await
.unwrap();
tokio::select! {
Err(e) = config_service.start() => {
@ -53,6 +51,9 @@ fn main() {
error!("rpxy service existed: {e}");
std::process::exit(1);
}
else => {
std::process::exit(0);
}
}
}
});
@ -63,23 +64,22 @@ async fn rpxy_service_without_watcher(
runtime_handle: tokio::runtime::Handle,
) -> Result<(), anyhow::Error> {
info!("Start rpxy service");
let config_toml = match ConfigToml::new(config_file_path) {
Ok(v) => v,
Err(e) => {
error!("Invalid toml file: {e}");
std::process::exit(1);
}
};
let (proxy_conf, app_conf) = match build_settings(&config_toml) {
Ok(v) => v,
Err(e) => {
error!("Invalid configuration: {e}");
return Err(anyhow::anyhow!(e));
}
};
entrypoint(&proxy_conf, &app_conf, &runtime_handle, None)
let config_toml = ConfigToml::new(config_file_path).map_err(|e| anyhow!("Invalid toml file: {e}"))?;
let (proxy_conf, app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?;
let (cert_service, cert_rx) = build_cert_manager(&config_toml)
.await
.map_err(|e| anyhow::anyhow!(e))
.map_err(|e| anyhow!("Invalid cert configuration: {e}"))?;
tokio::select! {
rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, None) => {
error!("rpxy entrypoint exited");
rpxy_res.map_err(|e| anyhow!(e))
}
cert_res = cert_service.start() => {
error!("cert reloader service exited");
cert_res.map_err(|e| anyhow!(e))
}
}
}
async fn rpxy_service_with_watcher(
@ -89,14 +89,15 @@ async fn rpxy_service_with_watcher(
info!("Start rpxy service with dynamic config reloader");
// Initial loading
config_rx.changed().await?;
let config_toml = config_rx.borrow().clone().unwrap();
let (mut proxy_conf, mut app_conf) = match build_settings(&config_toml) {
Ok(v) => v,
Err(e) => {
error!("Invalid configuration: {e}");
return Err(anyhow::anyhow!(e));
}
};
let config_toml = config_rx
.borrow()
.clone()
.ok_or(anyhow!("Something wrong in config reloader receiver"))?;
let (mut proxy_conf, mut app_conf) = build_settings(&config_toml).map_err(|e| anyhow!("Invalid configuration: {e}"))?;
let (mut cert_service, mut cert_rx) = build_cert_manager(&config_toml)
.await
.map_err(|e| anyhow!("Invalid cert configuration: {e}"))?;
// Notifier for proxy service termination
let term_notify = std::sync::Arc::new(tokio::sync::Notify::new());
@ -104,16 +105,15 @@ async fn rpxy_service_with_watcher(
// Continuous monitoring
loop {
tokio::select! {
_ = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => {
rpxy_res = entrypoint(&proxy_conf, &app_conf, &runtime_handle, Some(term_notify.clone())) => {
error!("rpxy entrypoint exited");
break;
return rpxy_res.map_err(|e| anyhow!(e));
}
_ = config_rx.changed() => {
if config_rx.borrow().is_none() {
let Some(config_toml) = config_rx.borrow().clone() else {
error!("Something wrong in config reloader receiver");
break;
}
let config_toml = config_rx.borrow().clone().unwrap();
return Err(anyhow!("Something wrong in config reloader receiver"));
};
match build_settings(&config_toml) {
Ok((p, a)) => {
(proxy_conf, app_conf) = (p, a)
@ -123,13 +123,27 @@ async fn rpxy_service_with_watcher(
continue;
}
};
match build_cert_manager(&config_toml).await {
Ok((c, r)) => {
(cert_service, cert_rx) = (c, r)
},
Err(e) => {
error!("Invalid cert configuration. Configuration does not updated: {e}");
continue;
}
};
info!("Configuration updated. Terminate all spawned proxy services and force to re-bind TCP/UDP sockets");
term_notify.notify_waiters();
// tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
}
cert_res = cert_service.start() => {
error!("cert reloader service exited");
return cert_res.map_err(|e| anyhow!(e));
}
else => break
}
}
Err(anyhow::anyhow!("rpxy or continuous monitoring service exited"))
Ok(())
}

4
rpxy-certs/Cargo.toml
View file

@ -22,6 +22,7 @@ thiserror = { version = "1.0.61" }
hot_reload = { version = "0.1.5" }
async-trait = { version = "0.1.80" }
rustls = { version = "0.23.8", default-features = false, features = [
"std",
"aws_lc_rs",
] }
rustls-pemfile = { version = "2.1.2" }
@ -33,9 +34,6 @@ x509-parser = { version = "0.16.0" }
[dev-dependencies]
tokio = { version = "1.37.0", default-features = false, features = [
# "net",
"rt-multi-thread",
# "time",
# "sync",
"macros",
] }

6
rpxy-certs/src/error.rs
View file

@ -18,4 +18,10 @@ pub enum RpxyCertError {
/// Error when converting server name bytes to string
#[error("Failed to convert server name bytes to string: {0}")]
ServerNameBytesToString(#[from] std::string::FromUtf8Error),
/// Rustls error
#[error("Rustls error: {0}")]
RustlsError(#[from] rustls::Error),
/// Rustls CryptoProvider error
#[error("Rustls No default CryptoProvider error")]
NoDefaultCryptoProvider,
}

11
rpxy-certs/src/lib.rs
View file

@ -9,18 +9,17 @@ mod log {
pub(super) use tracing::{debug, error, info, warn};
}
use crate::{
error::*,
reloader_service::{CryptoReloader, DynCryptoSource},
};
use crate::{error::*, log::*, reloader_service::DynCryptoSource};
use hot_reload::{ReloaderReceiver, ReloaderService};
use rustc_hash::FxHashMap as HashMap;
use rustls::crypto::{aws_lc_rs, CryptoProvider};
use std::sync::Arc;
/* ------------------------------------------------ */
pub use crate::{
certs::SingleServerCertsKeys,
crypto_source::{CryptoFileSource, CryptoFileSourceBuilder, CryptoFileSourceBuilderError, CryptoSource},
reloader_service::CryptoReloader,
server_crypto::{ServerCrypto, ServerCryptoBase},
};
@ -44,6 +43,10 @@ pub async fn build_cert_reloader<T>(
where
T: CryptoSource<Error = RpxyCertError> + Send + Sync + Clone + 'static,
{
info!("Building certificate reloader service");
// Install aws_lc_rs as default crypto provider for rustls
let _ = CryptoProvider::install_default(aws_lc_rs::default_provider());
let source = crypto_source_map
.iter()
.map(|(k, v)| {

67
rpxy-certs/src/server_crypto.rs
View file

@ -1,6 +1,7 @@
use crate::{certs::SingleServerCertsKeys, error::*, log::*};
use rustc_hash::FxHashMap as HashMap;
use rustls::{
crypto::CryptoProvider,
server::{ResolvesServerCertUsingSni, WebPkiClientVerifier},
RootCertStore, ServerConfig,
};
@ -40,7 +41,6 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
fn try_into(self) -> Result<Arc<ServerCrypto>, Self::Error> {
let aggregated = self.build_aggrated_server_crypto()?;
let individual = self.build_individual_server_crypto_map()?;
Ok(Arc::new(ServerCrypto {
aggregated_config_no_client_auth: Arc::new(aggregated),
individual_config_map: Arc::new(individual),
@ -53,6 +53,9 @@ impl ServerCryptoBase {
fn build_individual_server_crypto_map(&self) -> Result<ServerNameCryptoMap, RpxyCertError> {
let mut server_crypto_map: ServerNameCryptoMap = HashMap::default();
// AWS LC provider by default
let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?;
for (server_name_bytes, certs_keys) in self.inner.iter() {
let server_name = server_name_bytes_to_string(server_name_bytes)?;
@ -69,9 +72,11 @@ impl ServerCryptoBase {
// With no client authentication case
if !certs_keys.is_mutual_tls() {
let mut server_crypto_local = ServerConfig::builder()
let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone())
.with_safe_default_protocol_versions()?
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver_local));
#[cfg(feature = "http3")]
{
server_crypto_local.alpn_protocols = vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()];
@ -93,11 +98,14 @@ impl ServerCryptoBase {
let trust_anchors_without_skid = trust_anchors.values().map(|ta| ta.to_owned());
client_ca_roots_local.extend(trust_anchors_without_skid);
let Ok(client_cert_verifier) = WebPkiClientVerifier::builder(Arc::new(client_ca_roots_local)).build() else {
let Ok(client_cert_verifier) =
WebPkiClientVerifier::builder_with_provider(Arc::new(client_ca_roots_local), provider.clone()).build()
else {
warn!("Failed to build client CA certificate verifier for {server_name}");
continue;
};
let mut server_crypto_local = ServerConfig::builder()
let mut server_crypto_local = ServerConfig::builder_with_provider(provider.clone())
.with_safe_default_protocol_versions()?
.with_client_cert_verifier(client_cert_verifier)
.with_cert_resolver(Arc::new(resolver_local));
server_crypto_local.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
@ -112,6 +120,9 @@ impl ServerCryptoBase {
fn build_aggrated_server_crypto(&self) -> Result<ServerConfig, RpxyCertError> {
let mut resolver_global = ResolvesServerCertUsingSni::new();
// AWS LC provider by default
let provider = CryptoProvider::get_default().ok_or(RpxyCertError::NoDefaultCryptoProvider)?;
for (server_name_bytes, certs_keys) in self.inner.iter() {
let server_name = server_name_bytes_to_string(server_name_bytes)?;
@ -129,7 +140,8 @@ impl ServerCryptoBase {
}
}
let mut server_crypto_global = ServerConfig::builder()
let mut server_crypto_global = ServerConfig::builder_with_provider(provider.clone())
.with_safe_default_protocol_versions()?
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver_global));
@ -145,3 +157,48 @@ impl ServerCryptoBase {
Ok(server_crypto_global)
}
}
/* ------------------------------------------------ */
#[cfg(test)]
mod tests {
use super::*;
use crate::{CryptoFileSourceBuilder, CryptoSource};
use std::convert::TryInto;
async fn read_file_source() -> SingleServerCertsKeys {
let tls_cert_path = "../example-certs/server.crt";
let tls_cert_key_path = "../example-certs/server.key";
let client_ca_cert_path = Some("../example-certs/client.ca.crt");
let crypto_file_source = CryptoFileSourceBuilder::default()
.tls_cert_key_path(tls_cert_key_path)
.tls_cert_path(tls_cert_path)
.client_ca_cert_path(client_ca_cert_path)
.build();
crypto_file_source.unwrap().read().await.unwrap()
}
#[tokio::test]
async fn test_server_crypto_base_try_into() {
let mut server_crypto_base = ServerCryptoBase::default();
let single_certs_keys = read_file_source().await;
server_crypto_base.inner.insert(b"localhost".to_vec(), single_certs_keys);
let server_crypto: Arc<ServerCrypto> = (&server_crypto_base).try_into().unwrap();
assert_eq!(server_crypto.individual_config_map.len(), 1);
#[cfg(feature = "http3")]
{
assert_eq!(
server_crypto.aggregated_config_no_client_auth.alpn_protocols,
vec![b"h3".to_vec(), b"h2".to_vec(), b"http/1.1".to_vec()]
);
}
#[cfg(not(feature = "http3"))]
{
assert_eq!(
server_crypto.aggregated_config_no_client_auth.alpn_protocols,
vec![b"h2".to_vec(), b"http/1.1".to_vec()]
);
}
}
}