feat: initial support of s2n-quic

rpxy-lib/src/proxy/crypto_service.rs

@@ -22,7 +22,10 @@ where
 pub type SniServerCryptoMap = HashMap<ServerNameBytesExp, Arc<ServerConfig>>;
 pub struct ServerCrypto {
   // For Quic/HTTP3, only servers with no client authentication
+  #[cfg(feature = "http3-quinn")]
   pub inner_global_no_client_auth: Arc<ServerConfig>,
+  #[cfg(feature = "http3-s2n")]
+  pub inner_global_no_client_auth: s2n_quic_rustls::Server,
   // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers
   pub inner_local_map: Arc<SniServerCryptoMap>,
 }
@@ -68,7 +71,22 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
   type Error = anyhow::Error;
 
   fn try_into(self) -> Result<Arc<ServerCrypto>, Self::Error> {
-    let mut resolver_global = ResolvesServerCertUsingSni::new();
+    #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
+    let server_crypto_global = self.build_server_crypto_global()?;
+    let server_crypto_local_map: SniServerCryptoMap = self.build_server_crypto_local_map()?;
+
+    Ok(Arc::new(ServerCrypto {
+      #[cfg(feature = "http3-quinn")]
+      inner_global_no_client_auth: Arc::new(server_crypto_global),
+      #[cfg(feature = "http3-s2n")]
+      inner_global_no_client_auth: server_crypto_global,
+      inner_local_map: Arc::new(server_crypto_local_map),
+    }))
+  }
+}
+
+impl ServerCryptoBase {
+  fn build_server_crypto_local_map(&self) -> Result<SniServerCryptoMap, ReloaderError<ServerCryptoBase>> {
     let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default();
 
     for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
@@ -93,16 +111,7 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
       }
 
       // add client certificate if specified
-      if certs_and_keys.client_ca_certs.is_none() {
-        // aggregated server config for no client auth server for http3
-        if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
-          error!(
-            "{}: Failed to read some certificates and keys {}",
-            server_name.as_str(),
-            e
-          )
-        }
-      } else {
+      if certs_and_keys.client_ca_certs.is_some() {
         // add client certificate if specified
         match certs_and_keys.parse_client_ca_certs() {
           Ok((owned_trust_anchors, _subject_key_ids)) => {
@@ -120,14 +129,14 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
 
       let mut server_config_local = if client_ca_roots_local.is_empty() {
         // with no client auth, enable http1.1 -- 3
-        #[cfg(not(feature = "http3"))]
+        #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))]
         {
           ServerConfig::builder()
             .with_safe_defaults()
             .with_no_client_auth()
             .with_cert_resolver(Arc::new(resolver_local))
         }
-        #[cfg(feature = "http3")]
+        #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
         {
           let mut sc = ServerConfig::builder()
             .with_safe_defaults()
@@ -150,6 +159,33 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
 
       server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local));
     }
+    Ok(server_crypto_local_map)
+  }
+
+  #[cfg(feature = "http3-quinn")]
+  fn build_server_crypto_global(&self) -> Result<ServerConfig, ReloaderError<ServerCryptoBase>> {
+    let mut resolver_global = ResolvesServerCertUsingSni::new();
+
+    for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
+      let server_name: String = server_name_bytes_exp.try_into()?;
+
+      // Parse server certificates and private keys
+      let Ok(certified_key): Result<CertifiedKey, _> = certs_and_keys.parse_server_certs_and_keys() else {
+        warn!("Failed to add certificate for {}", server_name);
+        continue;
+      };
+
+      if certs_and_keys.client_ca_certs.is_none() {
+        // aggregated server config for no client auth server for http3
+        if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
+          error!(
+            "{}: Failed to read some certificates and keys {}",
+            server_name.as_str(),
+            e
+          )
+        }
+      }
+    }
 
     //////////////
     let mut server_crypto_global = ServerConfig::builder()
@@ -159,23 +195,82 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
 
     //////////////////////////////
 
-    #[cfg(feature = "http3")]
-    {
-      server_crypto_global.alpn_protocols = vec![
-        b"h3".to_vec(),
-        b"hq-29".to_vec(), // TODO: remove later?
-        b"h2".to_vec(),
-        b"http/1.1".to_vec(),
-      ];
-    }
-    #[cfg(not(feature = "http3"))]
-    {
-      server_crypto_global.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
-    }
+    server_crypto_global.alpn_protocols = vec![
+      b"h3".to_vec(),
+      b"hq-29".to_vec(), // TODO: remove later?
+      b"h2".to_vec(),
+      b"http/1.1".to_vec(),
+    ];
+    Ok(server_crypto_global)
+  }
 
-    Ok(Arc::new(ServerCrypto {
-      inner_global_no_client_auth: Arc::new(server_crypto_global),
-      inner_local_map: Arc::new(server_crypto_local_map),
-    }))
+  #[cfg(feature = "http3-s2n")]
+  fn build_server_crypto_global(&self) -> Result<s2n_quic_rustls::Server, ReloaderError<ServerCryptoBase>> {
+    let mut resolver_global = s2n_quic_rustls::rustls::server::ResolvesServerCertUsingSni::new();
+
+    for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
+      let server_name: String = server_name_bytes_exp.try_into()?;
+
+      // Parse server certificates and private keys
+      let Ok(certified_key) = parse_server_certs_and_keys_s2n(certs_and_keys) else {
+        warn!("Failed to add certificate for {}", server_name);
+        continue;
+      };
+
+      if certs_and_keys.client_ca_certs.is_none() {
+        // aggregated server config for no client auth server for http3
+        if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
+          error!(
+            "{}: Failed to read some certificates and keys {}",
+            server_name.as_str(),
+            e
+          )
+        }
+      }
+    }
+    let alpn = vec![
+      b"h3".to_vec(),
+      b"hq-29".to_vec(), // TODO: remove later?
+      b"h2".to_vec(),
+      b"http/1.1".to_vec(),
+    ];
+    let server_crypto_global = s2n_quic::provider::tls::rustls::Server::builder()
+      .with_cert_resolver(Arc::new(resolver_global))
+      .map_err(|e| anyhow::anyhow!(e))?
+      .with_application_protocols(alpn.iter())
+      .map_err(|e| anyhow::anyhow!(e))?
+      .build()
+      .map_err(|e| anyhow::anyhow!(e))?;
+    Ok(server_crypto_global)
   }
 }
+
+#[cfg(feature = "http3-s2n")]
+/// This is workaround for the version difference between rustls and s2n-quic-rustls
+fn parse_server_certs_and_keys_s2n(
+  certs_and_keys: &CertsAndKeys,
+) -> Result<s2n_quic_rustls::rustls::sign::CertifiedKey, anyhow::Error> {
+  let signing_key = certs_and_keys
+    .cert_keys
+    .iter()
+    .find_map(|k| {
+      let s2n_private_key = s2n_quic_rustls::PrivateKey(k.0.clone());
+      if let Ok(sk) = s2n_quic_rustls::rustls::sign::any_supported_type(&s2n_private_key) {
+        Some(sk)
+      } else {
+        None
+      }
+    })
+    .ok_or_else(|| {
+      std::io::Error::new(
+        std::io::ErrorKind::InvalidInput,
+        "Unable to find a valid certificate and key",
+      )
+    })?;
+  let certs: Vec<_> = certs_and_keys
+    .certs
+    .iter()
+    .map(|c| s2n_quic_rustls::rustls::Certificate(c.0.clone()))
+    .collect();
+  Ok(s2n_quic_rustls::rustls::sign::CertifiedKey::new(certs, signing_key))
+}

rpxy-lib/src/proxy/mod.rs

@@ -1,10 +1,12 @@
 mod crypto_service;
 mod proxy_client_cert;
-#[cfg(feature = "http3")]
+#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
 mod proxy_h3;
 mod proxy_main;
-#[cfg(feature = "http3")]
-mod proxy_quic;
+#[cfg(feature = "http3-quinn")]
+mod proxy_quic_quinn;
+#[cfg(feature = "http3-s2n")]
+mod proxy_quic_s2n;
 mod proxy_tls;
 mod socket;
 

rpxy-lib/src/proxy/proxy_h3.rs

@@ -1,8 +1,11 @@
 use super::Proxy;
 use crate::{certs::CryptoSource, error::*, log::*, utils::ServerNameBytesExp};
 use bytes::{Buf, Bytes};
+#[cfg(feature = "http3-quinn")]
 use h3::{quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
 use hyper::{client::connect::Connect, Body, Request, Response};
+#[cfg(feature = "http3-s2n")]
+use s2n_quic_h3::h3::{self, quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
 use std::net::SocketAddr;
 use tokio::time::{timeout, Duration};
 

rpxy-lib/src/proxy/proxy_quic_quinn.rs

@@ -19,7 +19,7 @@ where
     &self,
     mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
   ) -> Result<()> {
-    info!("Start UDP proxy serving with HTTP/3 request for configured host names");
+    info!("Start UDP proxy serving with HTTP/3 request for configured host names [quinn]");
     // first set as null config server
     let rustls_server_config = ServerConfig::builder()
       .with_safe_default_cipher_suites()
@@ -30,8 +30,8 @@ where
 
     let mut transport_config_quic = TransportConfig::default();
     transport_config_quic
-      .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream)
-      .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream)
+      .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream.into())
+      .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream.into())
       .max_idle_timeout(
         self
           .globals

rpxy-lib/src/proxy/proxy_quic_s2n.rs

@@ -0,0 +1,135 @@
+use super::{
+  crypto_service::{ServerCrypto, ServerCryptoBase},
+  proxy_main::Proxy,
+};
+use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName};
+use hot_reload::ReloaderReceiver;
+use hyper::client::connect::Connect;
+use s2n_quic::provider;
+use std::sync::Arc;
+
+impl<T, U> Proxy<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone + Sync + Send + 'static,
+{
+  pub(super) async fn listener_service_h3(
+    &self,
+    mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
+  ) -> Result<()> {
+    info!("Start UDP proxy serving with HTTP/3 request for configured host names [s2n-quic]");
+
+    // initially wait for receipt
+    let mut server_crypto: Option<Arc<ServerCrypto>> = {
+      let _ = server_crypto_rx.changed().await;
+      let sc = self.receive_server_crypto(server_crypto_rx.clone())?;
+      Some(sc)
+    };
+
+    // event loop
+    loop {
+      tokio::select! {
+        v = self.serve_connection(&server_crypto) => {
+          if let Err(e) = v {
+            error!("Quic connection event loop illegally shutdown [s2n-quic] {e}");
+            break;
+          }
+        }
+        _ = server_crypto_rx.changed() => {
+          server_crypto = match self.receive_server_crypto(server_crypto_rx.clone()) {
+            Ok(sc) => Some(sc),
+            Err(e) => {
+              error!("{e}");
+              break;
+            }
+          };
+        }
+        else => break
+      }
+    }
+
+    Ok(())
+  }
+
+  fn receive_server_crypto(&self, server_crypto_rx: ReloaderReceiver<ServerCryptoBase>) -> Result<Arc<ServerCrypto>> {
+    let cert_keys_map = server_crypto_rx.borrow().clone().ok_or_else(|| {
+      error!("Reloader is broken");
+      RpxyError::Other(anyhow!("Reloader is broken"))
+    })?;
+
+    let server_crypto: Option<Arc<ServerCrypto>> = (&cert_keys_map).try_into().ok();
+    server_crypto.ok_or_else(|| {
+      error!("Failed to update server crypto for h3 [s2n-quic]");
+      RpxyError::Other(anyhow!("Failed to update server crypto for h3 [s2n-quic]"))
+    })
+  }
+
+  async fn serve_connection(&self, server_crypto: &Option<Arc<ServerCrypto>>) -> Result<()> {
+    // setup UDP socket
+    let io = provider::io::tokio::Builder::default()
+      .with_receive_address(self.listening_on)?
+      .with_reuse_port()?
+      .build()?;
+
+    // setup limits
+    let mut limits = provider::limits::Limits::default()
+      .with_max_open_local_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_open_remote_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_open_local_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_open_remote_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_active_connection_ids(self.globals.proxy_config.h3_max_concurrent_connections as u64)
+      .map_err(|e| anyhow!(e))?;
+    limits = if let Some(v) = self.globals.proxy_config.h3_max_idle_timeout {
+      limits.with_max_idle_timeout(v).map_err(|e| anyhow!(e))?
+    } else {
+      limits
+    };
+
+    // setup tls
+    let Some(server_crypto) = server_crypto else {
+      warn!("No server crypto is given [s2n-quic]");
+      return Err(RpxyError::Other(anyhow!("No server crypto is given [s2n-quic]")));
+    };
+    let tls = server_crypto.inner_global_no_client_auth.clone();
+
+    let mut server = s2n_quic::Server::builder()
+      .with_tls(tls)
+      .map_err(|e| anyhow::anyhow!(e))?
+      .with_io(io)
+      .map_err(|e| anyhow!(e))?
+      .with_limits(limits)
+      .map_err(|e| anyhow!(e))?
+      .start()
+      .map_err(|e| anyhow!(e))?;
+
+    // quic event loop. this immediately cancels when crypto is updated by tokio::select!
+    while let Some(new_conn) = server.accept().await {
+      debug!("New QUIC connection established");
+      let Ok(Some(new_server_name)) = new_conn.server_name() else {
+          warn!("HTTP/3 no SNI is given");
+          continue;
+        };
+      debug!("HTTP/3 connection incoming (SNI {:?})", new_server_name);
+      let self_clone = self.clone();
+
+      self.globals.runtime_handle.spawn(async move {
+        let client_addr = new_conn.remote_addr()?;
+        let quic_connection = s2n_quic_h3::Connection::new(new_conn);
+        // Timeout is based on underlying quic
+        if let Err(e) = self_clone
+          .connection_serve_h3(quic_connection, new_server_name.to_server_name_vec(), client_addr)
+          .await
+        {
+          warn!("QUIC or HTTP/3 connection failed: {}", e);
+        };
+        Ok(()) as Result<()>
+      });
+    }
+
+    Ok(())
+  }
+}

rpxy-lib/src/proxy/proxy_tls.rs

@@ -108,7 +108,7 @@ where
     .await
     .map_err(|e| anyhow::anyhow!(e))?;
 
-    #[cfg(not(feature = "http3"))]
+    #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))]
     {
       tokio::select! {
         _= cert_reloader_service.start() => {
@@ -124,7 +124,7 @@ where
       };
       Ok(())
     }
-    #[cfg(feature = "http3")]
+    #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
     {
       if self.globals.proxy_config.http3 {
         tokio::select! {

rpxy-lib/src/proxy/socket.rs

@@ -1,8 +1,8 @@
 use crate::{error::*, log::*};
-#[cfg(feature = "http3")]
+#[cfg(feature = "http3-quinn")]
 use socket2::{Domain, Protocol, Socket, Type};
 use std::net::SocketAddr;
-#[cfg(feature = "http3")]
+#[cfg(feature = "http3-quinn")]
 use std::net::UdpSocket;
 use tokio::net::TcpSocket;
 
@@ -23,7 +23,7 @@ pub(super) fn bind_tcp_socket(listening_on: &SocketAddr) -> Result<TcpSocket> {
   Ok(tcp_socket)
 }
 
-#[cfg(feature = "http3")]
+#[cfg(feature = "http3-quinn")]
 /// Bind UDP socket to the given `SocketAddr`, and returns the UDP socket with `SO_REUSEADDR` and `SO_REUSEPORT` options.
 /// This option is required to re-bind the socket address when the proxy instance is reconstructed.
 pub(super) fn bind_udp_socket(listening_on: &SocketAddr) -> Result<UdpSocket> {