changed how to support multiple domains and support client authentication directly by rustls using split server_config

README.md

@@ -226,6 +226,33 @@ Other than them, all you need is to mount your `config.toml` as `/etc/rpxy.toml`
 
 [`./bench`](./bench/) directory could be a very simple example of configuration of `rpxy`. This can also be an example of an example of docker use case.
 
+## Experimental Features and Caveats
+
+### HTTP/3
+
+`rpxy` can serves HTTP/3 requests thanks to `quinn` and `hyperium/h3`. To enable this experimental feature, add an entry `experimental.h3` in your `config.toml` like follows. Any values in the entry like `alt_svc_max_age` are optional.
+
+```toml
+[experimental.h3]
+alt_svc_max_age = 3600
+request_max_body_size = 65536
+max_concurrent_connections = 10000
+max_concurrent_bidistream = 100
+max_concurrent_unistream = 100
+```
+
+### Client Authentication via Client Certificates
+
+Client authentication is enabled when `apps."app_name".tls.client_ca_cert_path` is set for the domain specified by `"app_name"` like
+
+```toml
+[apps.localhost]
+server_name = 'localhost' # Domain name
+tls = { https_redirection = true, tls_cert_path = './server.crt', tls_cert_key_path = './server.key', client_ca_cert_path = './client_cert.ca.crt' }
+```
+
+ However, currently we have a limitation on HTTP/3 support for applications that enables client authentication. If an application is set with client authentication, HTTP/3 doesn't work for the application.
+
 ## TIPS
 
 ### Using Private Key Issued by Let's Encrypt
@@ -284,12 +311,14 @@ First, you need to prepare a CA certificate used to verify client certificate. I
   % openssl x509 -req -days 365 -sha256 -in client.csr -CA client.ca.crt -CAkey client.ca.key -CAcreateserial -out client.crt -extfile client.ext
   ```
 
-  Now you have a client key `client.key` and certificate `client.crt` (version 3). `p12` file can be retrieved as
+  Now you have a client key `client.key` and certificate `client.crt` (version 3). `pfx` (`p12`) file can be retrieved as
 
   ```bash
   % openssl pkcs12 -export -inkey client.key -in client.crt -certfile client.ca.crt -out client.pfx
   ```
 
+  Note that on MacOS, a `pfx` generated by `OpenSSL 3.0.6` cannot be imported to MacOS KeyChain Access. We generated the sample `pfx` using `LibreSSL 2.8.3` instead `OpenSSL`.
+
   All of sample certificate files are found in `./example-certs/` directory.
 
 ### (Work Around) Deployment on Ubuntu 22.04LTS using docker behind `ufw`

TODO.md

@@ -9,5 +9,7 @@
 - Documentation
 - Client certificate
   - support intermediate certificate. Currently, only supports client certificates directly signed by root CA.
-  - split rustls::server::ServerConfig for SNIs
+  - Currently, we took the following approach (caveats)
+    - For Http2 and 1.1, prepare `rustls::ServerConfig` for each domain name and hence client CA cert is set for each one.
+    - For Http3, use aggregated `rustls::ServerConfig` for multiple domain names except for ones requiring client-auth. So, if a domain name is set with client authentication, http3 doesn't work for the domain.
 - etc.

src/backend/mod.rs

@@ -6,7 +6,7 @@ use crate::{
   utils::{BytesName, PathNameBytesExp, ServerNameBytesExp},
 };
 use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
-use rustls::OwnedTrustAnchor;
+use rustls::{OwnedTrustAnchor, RootCertStore};
 use std::{
   fs::File,
   io::{self, BufReader, Cursor, Read},
@@ -146,6 +146,7 @@ impl Backend {
       })
       .collect();
 
+    // TODO: SKID is not used currently
     let subject_key_identifiers: HashSet<_> = certs
       .iter()
       .filter_map(|v| {
@@ -176,82 +177,104 @@ pub struct Backends {
   pub default_server_name_bytes: Option<ServerNameBytesExp>, // for plaintext http
 }
 
-pub type SniKeyIdsMap = HashMap<ServerNameBytesExp, HashSet<Vec<u8>>>;
+pub type SniServerCryptoMap = HashMap<ServerNameBytesExp, Arc<ServerConfig>>;
 pub struct ServerCrypto {
-  pub inner: Arc<ServerConfig>,
+  // For Quic/HTTP3, only servers with no client authentication
-  pub server_name_client_ca_keyids_map: Arc<SniKeyIdsMap>,
+  pub inner_global_no_client_auth: Arc<ServerConfig>,
+  // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers
+  pub inner_local_map: Arc<SniServerCryptoMap>,
 }
 
 impl Backends {
-  pub async fn generate_server_crypto_with_cert_resolver(&self) -> Result<ServerCrypto, anyhow::Error> {
+  pub async fn generate_server_crypto(&self) -> Result<ServerCrypto, anyhow::Error> {
-    let mut resolver = ResolvesServerCertUsingSni::new();
+    let mut resolver_global = ResolvesServerCertUsingSni::new();
-    let mut client_ca_roots = rustls::RootCertStore::empty();
+    let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default();
-    let mut client_ca_key_ids: SniKeyIdsMap = HashMap::default();
 
-    // let mut cnt = 0;
     for (server_name_bytes_exp, backend) in self.apps.iter() {
       if backend.tls_cert_key_path.is_some() && backend.tls_cert_path.is_some() {
         match backend.read_certs_and_key() {
           Ok(certified_key) => {
-            if let Err(e) = resolver.add(backend.server_name.as_str(), certified_key) {
+            let mut resolver_local = ResolvesServerCertUsingSni::new();
+            let mut client_ca_roots_local = RootCertStore::empty();
+
+            // add server certificate and key
+            if let Err(e) = resolver_local.add(backend.server_name.as_str(), certified_key.to_owned()) {
               error!(
                 "{}: Failed to read some certificates and keys {}",
                 backend.server_name.as_str(),
                 e
               )
-            } else {
-              // debug!("Add certificate for server_name: {}", backend.server_name.as_str());
-              // cnt += 1;
             }
+
+            if backend.client_ca_cert_path.is_none() {
+              // aggregated server config for no client auth server for http3
+              if let Err(e) = resolver_global.add(backend.server_name.as_str(), certified_key) {
+                error!(
+                  "{}: Failed to read some certificates and keys {}",
+                  backend.server_name.as_str(),
+                  e
+                )
+              }
+            } else {
+              // add client certificate if specified
+              match backend.read_client_ca_certs() {
+                Ok((owned_trust_anchors, _subject_key_ids)) => {
+                  client_ca_roots_local.add_server_trust_anchors(owned_trust_anchors.into_iter());
+                }
+                Err(e) => {
+                  warn!(
+                    "Failed to add client CA certificate for {}: {}",
+                    backend.server_name.as_str(),
+                    e
+                  );
+                }
+              }
+            }
+
+            let mut server_config_local = if client_ca_roots_local.is_empty() {
+              // with no client auth, enable http1.1 -- 3
+              let mut sc = ServerConfig::builder()
+                .with_safe_defaults()
+                .with_no_client_auth()
+                .with_cert_resolver(Arc::new(resolver_local));
+              #[cfg(feature = "http3")]
+              {
+                sc.alpn_protocols = vec![b"h3".to_vec(), b"hq-29".to_vec()]; // TODO: remove hq-29 later?
+              }
+              sc
+            } else {
+              // with client auth, enable only http1.1 and 2
+              // let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots);
+              let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots_local);
+              ServerConfig::builder()
+                .with_safe_defaults()
+                .with_client_cert_verifier(client_certs_verifier)
+                .with_cert_resolver(Arc::new(resolver_local))
+            };
+            server_config_local.alpn_protocols.push(b"h2".to_vec());
+            server_config_local.alpn_protocols.push(b"http/1.1".to_vec());
+
+            server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local));
           }
           Err(e) => {
             warn!("Failed to add certificate for {}: {}", backend.server_name.as_str(), e);
           }
         }
-        // add client certificate if specified
-        if backend.client_ca_cert_path.is_some() {
-          match backend.read_client_ca_certs() {
-            Ok((owned_trust_anchors, subject_key_ids)) => {
-              // TODO: ここでSubject Key ID (CA Key ID)を記録しておく。認証後にpeer certificateのauthority key idとの一貫性をチェック。
-              // v3 x509前提で特定のkey id extが入ってなければ使えない前提
-              client_ca_roots.add_server_trust_anchors(owned_trust_anchors.into_iter());
-              client_ca_key_ids.insert(server_name_bytes_exp.to_owned(), subject_key_ids);
-            }
-            Err(e) => {
-              warn!(
-                "Failed to add client ca certificate for {}: {}",
-                backend.server_name.as_str(),
-                e
-              );
-            }
-          }
-        }
       }
     }
     // debug!("Load certificate chain for {} server_name's", cnt);
 
     //////////////
-    let mut server_config = if client_ca_key_ids.is_empty() {
+    let mut server_crypto_global = ServerConfig::builder()
-      ServerConfig::builder()
+      .with_safe_defaults()
-        .with_safe_defaults()
+      .with_no_client_auth()
-        .with_no_client_auth()
+      .with_cert_resolver(Arc::new(resolver_global));
-        .with_cert_resolver(Arc::new(resolver))
-    } else {
-      // TODO: Client Certs
-      // No ClientCert or WithClientCert
-      // let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots);
-      let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots);
-      ServerConfig::builder()
-        .with_safe_defaults()
-        .with_client_cert_verifier(client_certs_verifier)
-        .with_cert_resolver(Arc::new(resolver))
-    };
 
     //////////////////////////////
 
     #[cfg(feature = "http3")]
     {
-      server_config.alpn_protocols = vec![
+      server_crypto_global.alpn_protocols = vec![
         b"h3".to_vec(),
         b"hq-29".to_vec(), // TODO: remove later?
         b"h2".to_vec(),
@@ -264,8 +287,8 @@ impl Backends {
     }
 
     Ok(ServerCrypto {
-      inner: Arc::new(server_config),
+      inner_global_no_client_auth: Arc::new(server_crypto_global),
-      server_name_client_ca_keyids_map: Arc::new(client_ca_key_ids),
+      inner_local_map: Arc::new(server_crypto_local_map),
     })
   }
 }

src/error.rs

@@ -44,6 +44,7 @@ pub enum RpxyError {
   Other(#[from] anyhow::Error),
 }
 
+#[allow(dead_code)]
 #[derive(Debug, Error, Clone)]
 pub enum ClientCertsError {
   #[error("TLS Client Certificate is Required for Given SNI: {0}")]

src/handler/handler_main.rs

@@ -1,6 +1,12 @@
 // Highly motivated by https://github.com/felipenoris/hyper-reverse-proxy
 use super::{utils_headers::*, utils_request::*, utils_synth_response::*};
-use crate::{backend::UpstreamGroup, error::*, globals::Globals, log::*, utils::ServerNameBytesExp};
+use crate::{
+  backend::{Backend, UpstreamGroup},
+  error::*,
+  globals::Globals,
+  log::*,
+  utils::ServerNameBytesExp,
+};
 use hyper::{
   client::connect::Connect,
   header::{self, HeaderValue},
@@ -35,26 +41,11 @@ where
     listen_addr: SocketAddr,
     tls_enabled: bool,
     tls_server_name: Option<ServerNameBytesExp>,
-    tls_client_auth_result: Option<std::result::Result<(), ClientCertsError>>,
   ) -> Result<Response<Body>> {
     ////////
     let mut log_data = MessageLog::from(&req);
     log_data.client_addr(&client_addr);
     //////
-    // First check client auth result if exist
-    if let Some(res) = tls_client_auth_result {
-      match res {
-        Err(ClientCertsError::ClientCertRequired(_)) => {
-          // Client cert is required for the TLS server name
-          return self.return_with_error_log(StatusCode::FORBIDDEN, &mut log_data);
-        }
-        Err(ClientCertsError::InconsistentClientCert(_)) => {
-          // Client cert provided was inconsistent to the TLS server name
-          return self.return_with_error_log(StatusCode::BAD_REQUEST, &mut log_data);
-        }
-        _ => (),
-      }
-    }
 
     // Here we start to handle with server_name
     let server_name = if let Ok(v) = req.parse_host() {
@@ -133,7 +124,7 @@ where
 
     if res_backend.status() != StatusCode::SWITCHING_PROTOCOLS {
       // Generate response to client
-      if self.generate_response_forwarded(&mut res_backend).is_ok() {
+      if self.generate_response_forwarded(&mut res_backend, backend).is_ok() {
         log_data.status_code(&res_backend.status()).output();
         return Ok(res_backend);
       } else {
@@ -191,7 +182,11 @@ where
   ////////////////////////////////////////////////////
   // Functions to generate messages
 
-  fn generate_response_forwarded<B: core::fmt::Debug>(&self, response: &mut Response<B>) -> Result<()> {
+  fn generate_response_forwarded<B: core::fmt::Debug>(
+    &self,
+    response: &mut Response<B>,
+    chosen_backend: &Backend,
+  ) -> Result<()> {
     let headers = response.headers_mut();
     remove_connection_header(headers);
     remove_hop_header(headers);
@@ -199,7 +194,8 @@ where
 
     #[cfg(feature = "http3")]
     {
-      if self.globals.http3 {
+      // TODO: Workaround for avoid h3 for client authentication
+      if self.globals.http3 && chosen_backend.client_ca_cert_path.is_none() {
         if let Some(port) = self.globals.https_port {
           add_header_entry_overwrite_if_exist(
             headers,
@@ -210,6 +206,15 @@ where
             ),
           )?;
         }
+      } else {
+        // remove alt-svc to disallow requests via http3
+        headers.remove(header::ALT_SVC.as_str());
+      }
+    }
+    #[cfg(not(feature = "http3"))]
+    {
+      if let Some(port) = self.globals.https_port {
+        headers.remove(header::ALT_SVC.as_str());
       }
     }
 

src/proxy/proxy_client_cert.rs

@@ -4,6 +4,7 @@ use rustls::Certificate;
 use x509_parser::extensions::ParsedExtension;
 use x509_parser::prelude::*;
 
+#[allow(dead_code)]
 // TODO: consider move this function to the layer of handle_request (L7) to return 403
 pub(super) fn check_client_authentication(
   client_certs: Option<&[Certificate]>,

src/proxy/proxy_h3.rs

@@ -1,9 +1,9 @@
-use super::{proxy_client_cert::check_client_authentication, Proxy};
+use super::Proxy;
-use crate::{backend::SniKeyIdsMap, error::*, log::*, utils::ServerNameBytesExp};
+use crate::{error::*, log::*, utils::ServerNameBytesExp};
 use bytes::{Buf, Bytes};
 use h3::{quic::BidiStream, server::RequestStream};
 use hyper::{client::connect::Connect, Body, Request, Response};
-use std::{net::SocketAddr, sync::Arc};
+use std::net::SocketAddr;
 use tokio::time::{timeout, Duration};
 
 impl<T> Proxy<T>
@@ -14,28 +14,11 @@ where
     self,
     conn: quinn::Connecting,
     tls_server_name: ServerNameBytesExp,
-    sni_cc_map: Arc<SniKeyIdsMap>,
   ) -> Result<()> {
     let client_addr = conn.remote_address();
 
     match conn.await {
       Ok(new_conn) => {
-        // Check client certificates
-        let cc = {
-          // https://docs.rs/quinn/latest/quinn/struct.Connection.html
-          let client_certs_setting_for_sni = sni_cc_map.get(&tls_server_name);
-          let client_certs = match new_conn.connection.peer_identity() {
-            Some(peer_identity) => peer_identity
-              .downcast::<Vec<rustls::Certificate>>()
-              .ok()
-              .map(|p| p.into_iter().collect::<Vec<_>>()),
-            None => None,
-          };
-          (client_certs, client_certs_setting_for_sni)
-        };
-        // TODO: pass this value to the layer of handle_request (L7) to return 403
-        let tls_client_auth_result = check_client_authentication(cc.0.as_ref().map(AsRef::as_ref), cc.1);
-
         let mut h3_conn = h3::server::Connection::<_, bytes::Bytes>::new(h3_quinn::Connection::new(new_conn)).await?;
         info!(
           "QUIC/HTTP3 connection established from {:?} {:?}",
@@ -43,43 +26,46 @@ where
         );
         // TODO: Is here enough to fetch server_name from NewConnection?
         // to avoid deep nested call from listener_service_h3
-        while let Some((req, stream)) = match h3_conn.accept().await {
+        loop {
-          Ok(opt_req) => opt_req,
+          // this routine follows hyperium/h3 examples https://github.com/hyperium/h3/blob/master/examples/server.rs
-          Err(e) => {
+          match h3_conn.accept().await {
-            warn!("HTTP/3 failed to accept incoming connection: {}", e);
+            Ok(None) => {
-            return Ok(h3_conn.shutdown(0).await?);
+              break;
-          }
-        } {
-          // We consider the connection count separately from the stream count.
-          // Max clients for h1/h2 = max 'stream' for h3.
-          let request_count = self.globals.request_count.clone();
-          if request_count.increment() > self.globals.max_clients {
-            request_count.decrement();
-            return Ok(h3_conn.shutdown(0).await?);
-          }
-          debug!("Request incoming: current # {}", request_count.current());
-
-          let self_inner = self.clone();
-          let tls_server_name_inner = tls_server_name.clone();
-          let tls_client_auth_result_inner = tls_client_auth_result.clone();
-          self.globals.runtime_handle.spawn(async move {
-            if let Err(e) = timeout(
-              self_inner.globals.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
-              self_inner.stream_serve_h3(
-                req,
-                stream,
-                client_addr,
-                tls_server_name_inner,
-                tls_client_auth_result_inner,
-              ),
-            )
-            .await
-            {
-              error!("HTTP/3 failed to process stream: {}", e);
             }
-            request_count.decrement();
+            Err(e) => {
-            debug!("Request processed: current # {}", request_count.current());
+              warn!("HTTP/3 error on accept incoming connection: {}", e);
-          });
+              match e.get_error_level() {
+                h3::error::ErrorLevel::ConnectionError => break,
+                h3::error::ErrorLevel::StreamError => continue,
+              }
+            }
+            Ok(Some((req, stream))) => {
+              // We consider the connection count separately from the stream count.
+              // Max clients for h1/h2 = max 'stream' for h3.
+              let request_count = self.globals.request_count.clone();
+              if request_count.increment() > self.globals.max_clients {
+                request_count.decrement();
+                h3_conn.shutdown(0).await?;
+                break;
+              }
+              debug!("Request incoming: current # {}", request_count.current());
+
+              let self_inner = self.clone();
+              let tls_server_name_inner = tls_server_name.clone();
+              self.globals.runtime_handle.spawn(async move {
+                if let Err(e) = timeout(
+                  self_inner.globals.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
+                  self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner),
+                )
+                .await
+                {
+                  error!("HTTP/3 failed to process stream: {}", e);
+                }
+                request_count.decrement();
+                debug!("Request processed: current # {}", request_count.current());
+              });
+            }
+          }
         }
       }
       Err(err) => {
@@ -97,7 +83,6 @@ where
     stream: RequestStream<S, Bytes>,
     client_addr: SocketAddr,
     tls_server_name: ServerNameBytesExp,
-    tls_client_auth_result: std::result::Result<(), ClientCertsError>,
   ) -> Result<()>
   where
     S: BidiStream<Bytes> + Send + 'static,
@@ -149,7 +134,6 @@ where
         self.listening_on,
         self.tls_enabled,
         Some(tls_server_name),
-        Some(tls_client_auth_result),
       )
       .await?;
 

src/proxy/proxy_main.rs

@@ -51,7 +51,6 @@ where
     server: Http<LocalExecutor>,
     peer_addr: SocketAddr,
     tls_server_name: Option<ServerNameBytesExp>,
-    tls_client_auth_result: Option<std::result::Result<(), ClientCertsError>>,
   ) where
     I: AsyncRead + AsyncWrite + Send + Unpin + 'static,
   {
@@ -75,7 +74,6 @@ where
                 self.listening_on,
                 self.tls_enabled,
                 tls_server_name.clone(),
-                tls_client_auth_result.clone(),
               )
             }),
           )
@@ -94,9 +92,7 @@ where
       let tcp_listener = TcpListener::bind(&self.listening_on).await?;
       info!("Start TCP proxy serving with HTTP request for configured host names");
       while let Ok((stream, _client_addr)) = tcp_listener.accept().await {
-        self
+        self.clone().client_serve(stream, server.clone(), _client_addr, None);
-          .clone()
-          .client_serve(stream, server.clone(), _client_addr, None, None);
       }
       Ok(()) as Result<()>
     };

src/proxy/proxy_tls.rs

@@ -1,9 +1,6 @@
-use super::{
+use super::proxy_main::{LocalExecutor, Proxy};
-  proxy_client_cert::check_client_authentication,
-  proxy_main::{LocalExecutor, Proxy},
-};
 use crate::{
-  backend::{ServerCrypto, SniKeyIdsMap},
+  backend::{ServerCrypto, SniServerCryptoMap},
   constants::*,
   error::*,
   log::*,
@@ -17,7 +14,6 @@ use tokio::{
   sync::watch,
   time::{sleep, timeout, Duration},
 };
-use tokio_rustls::TlsAcceptor;
 
 #[cfg(feature = "http3")]
 use futures::StreamExt;
@@ -31,7 +27,7 @@ where
   async fn cert_service(&self, server_crypto_tx: watch::Sender<Option<Arc<ServerCrypto>>>) {
     info!("Start cert watch service");
     loop {
-      if let Ok(server_crypto) = self.globals.backends.generate_server_crypto_with_cert_resolver().await {
+      if let Ok(server_crypto) = self.globals.backends.generate_server_crypto().await {
         if let Err(_e) = server_crypto_tx.send(Some(Arc::new(server_crypto))) {
           error!("Failed to populate server crypto");
           break;
@@ -52,61 +48,61 @@ where
     let tcp_listener = TcpListener::bind(&self.listening_on).await?;
     info!("Start TCP proxy serving with HTTPS request for configured host names");
 
-    // let mut server_crypto: Option<Arc<ServerConfig>> = None;
+    let mut server_crypto_map: Option<Arc<SniServerCryptoMap>> = None;
-    let mut tls_acceptor: Option<TlsAcceptor> = None;
-    let mut sni_client_ca_keyid_map: Option<Arc<SniKeyIdsMap>> = None;
     loop {
       tokio::select! {
         tcp_cnx = tcp_listener.accept() => {
-          if tls_acceptor.is_none() || tcp_cnx.is_err() || sni_client_ca_keyid_map.is_none() {
+          if tcp_cnx.is_err() || server_crypto_map.is_none() {
             continue;
           }
           let (raw_stream, client_addr) = tcp_cnx.unwrap();
-          let acceptor = tls_acceptor.clone().unwrap();
+          let sc_map_inner = server_crypto_map.clone();
-          let sni_cc_map = sni_client_ca_keyid_map.clone().unwrap();
           let server_clone = server.clone();
           let self_inner = self.clone();
 
           // spawns async handshake to avoid blocking thread by sequential handshake.
           let handshake_fut = async move {
-            // timeout is introduced to avoid get stuck here.
+            let acceptor = tokio_rustls::LazyConfigAcceptor::new(rustls::server::Acceptor::new().unwrap(), raw_stream).await;
-            let accepted = match timeout(Duration::from_secs(TLS_HANDSHAKE_TIMEOUT_SEC), acceptor.accept(raw_stream)).await {
+            if let Err(e) = acceptor {
-              Ok(a) => a,
+              return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {}", e)));
-              Err(e) => {
+            }
-                return Err(RpxyError::Proxy(format!("Timeout to handshake TLS: {}", e)));
+            let start = acceptor.unwrap();
-              }
+            let client_hello = start.client_hello();
-            };
+            let server_name = client_hello.server_name();
-            let stream = match accepted {
+            debug!("HTTP/2 or 1.1: SNI in ClientHello: {:?}", server_name);
+            let server_name = server_name.map_or_else(|| None, |v| Some(v.to_server_name_vec()));
+            if server_name.is_none(){
+              return Err(RpxyError::Proxy("No SNI is given".to_string()));
+            }
+            let server_crypto = sc_map_inner.as_ref().unwrap().get(server_name.as_ref().unwrap());
+            if server_crypto.is_none() {
+              return Err(RpxyError::Proxy(format!("No TLS serving app for {:?}", "xx")));
+            }
+            let stream = match start.into_stream(server_crypto.unwrap().clone()).await {
               Ok(s) => s,
               Err(e) => {
                 return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {}", e)));
               }
             };
-            // Retrieve SNI
+            self_inner.client_serve(stream, server_clone, client_addr, server_name);
-            let (_, conn) = stream.get_ref();
+            Ok(())
-            let server_name = conn.sni_hostname();
-            debug!("HTTP/2 or 1.1: SNI in ClientHello: {:?}", server_name);
-            let server_name = server_name.map_or_else(|| None, |v| Some(v.to_server_name_vec()));
-            if server_name.is_none(){
-              Err(RpxyError::Proxy("No SNI is given".to_string()))
-            } else {
-              //////////////////////////////
-              // Check client certificate
-              let client_certs = conn.peer_certificates();
-              let client_ca_keyids_set_for_sni = sni_cc_map.get(&server_name.clone().unwrap());
-               // TODO: pass this value to the layer of handle_request (L7) to return 403
-              let client_certs_auth_result = check_client_authentication(client_certs, client_ca_keyids_set_for_sni);
-              //////////////////////////////
-              // this immediately spawns another future to actually handle stream. so it is okay to introduce timeout for handshake.
-              // TODO: don't want to pass copied value...
-              self_inner.client_serve(stream, server_clone, client_addr, server_name, Some(client_certs_auth_result));
-              Ok(())
-            }
           };
+
           self.globals.runtime_handle.spawn( async move {
-            if let Err(e) = handshake_fut.await {
+            // timeout is introduced to avoid get stuck here.
-              error!("{}", e);
+            match timeout(
-            }
+              Duration::from_secs(TLS_HANDSHAKE_TIMEOUT_SEC),
+              handshake_fut
+            ).await {
+              Ok(a) => {
+                if let Err(e) = a {
+                  error!("{}", e);
+                }
+              },
+              Err(e) => {
+                error!("Timeout to handshake TLS: {}", e);
+              }
+            };
           });
         }
         _ = server_crypto_rx.changed() => {
@@ -114,8 +110,7 @@ where
             break;
           }
           let server_crypto = server_crypto_rx.borrow().clone().unwrap();
-          tls_acceptor = Some(TlsAcceptor::from(server_crypto.inner.clone()));
+          server_crypto_map = Some(server_crypto.inner_local_map.clone());
-          sni_client_ca_keyid_map = Some(server_crypto.server_name_client_ca_keyids_map.clone());
         }
         else => break
       }
@@ -143,11 +138,10 @@ where
     let (endpoint, mut incoming) = Endpoint::server(server_config_h3, self.listening_on)?;
 
     let mut server_crypto: Option<Arc<ServerCrypto>> = None;
-    let mut sni_client_ca_keyid_map: Option<Arc<SniKeyIdsMap>> = None;
     loop {
       tokio::select! {
         new_conn = incoming.next() => {
-          if server_crypto.is_none() || new_conn.is_none() || sni_client_ca_keyid_map.is_none() {
+          if server_crypto.is_none() || new_conn.is_none() {
             continue;
           }
           let mut conn = new_conn.unwrap();
@@ -173,7 +167,7 @@ where
           );
           // TODO: server_nameをここで出してどんどん深く投げていくのは効率が悪い。connecting -> connectionsの後でいいのでは？
           // TODO: 通常のTLSと同じenumか何かにまとめたい
-          let fut = self.clone().connection_serve_h3(conn, new_server_name, sni_client_ca_keyid_map.clone().unwrap());
+          let fut = self.clone().connection_serve_h3(conn, new_server_name);
           self.globals.runtime_handle.spawn(async move {
             // Timeout is based on underlying quic
             if let Err(e) = fut.await {
@@ -187,8 +181,7 @@ where
           }
           server_crypto = server_crypto_rx.borrow().clone();
           if server_crypto.is_some(){
-            endpoint.set_server_config(Some(QuicServerConfig::with_crypto(server_crypto.clone().unwrap().inner.clone())));
+            endpoint.set_server_config(Some(QuicServerConfig::with_crypto(server_crypto.clone().unwrap().inner_global_no_client_auth.clone())));
-            sni_client_ca_keyid_map = Some(server_crypto.clone().unwrap().server_name_client_ca_keyids_map.clone());
           }
         }
         else => break