refactor: initial implementation of separeted lib and bin

rpxy-lib/Cargo.toml

@@ -0,0 +1,77 @@
+[package]
+name = "rpxy-lib"
+version = "0.4.0"
+authors = ["Jun Kurihara"]
+homepage = "https://github.com/junkurihara/rust-rpxy"
+repository = "https://github.com/junkurihara/rust-rpxy"
+license = "MIT"
+readme = "README.md"
+edition = "2021"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+default = ["http3", "sticky-cookie"]
+http3 = ["quinn", "h3", "h3-quinn"]
+sticky-cookie = ["base64", "sha2", "chrono"]
+
+[dependencies]
+rand = "0.8.5"
+rustc-hash = "1.1.0"
+bytes = "1.4.0"
+derive_builder = "0.12.0"
+futures = { version = "0.3.28", features = ["alloc", "async-await"] }
+tokio = { version = "1.29.1", default-features = false, features = [
+  "net",
+  "rt-multi-thread",
+  "time",
+  "sync",
+  "macros",
+] }
+async-trait = "0.1.72"
+hot_reload = "0.1.2" # reloading certs
+
+# Error handling
+anyhow = "1.0.72"
+thiserror = "1.0.44"
+
+# http and tls
+hyper = { version = "0.14.27", default-features = false, features = [
+  "server",
+  "http1",
+  "http2",
+  "stream",
+] }
+hyper-rustls = { version = "0.24.1", default-features = false, features = [
+  "tokio-runtime",
+  "webpki-tokio",
+  "http1",
+  "http2",
+] }
+tokio-rustls = { version = "0.24.1", features = ["early-data"] }
+rustls = { version = "0.21.5", default-features = false }
+webpki = "0.22.0"
+x509-parser = "0.15.0"
+
+# logging
+tracing = { version = "0.1.37" }
+
+# http/3
+# quinn = { version = "0.9.3", optional = true }
+quinn = { path = "../quinn/quinn", optional = true } # Tentative to support rustls-0.21
+h3 = { path = "../h3/h3/", optional = true }
+# h3-quinn = { path = "./h3/h3-quinn/", optional = true }
+h3-quinn = { path = "../h3-quinn/", optional = true } # Tentative to support rustls-0.21
+
+# cookie handling for sticky cookie
+chrono = { version = "0.4.26", default-features = false, features = [
+  "unstable-locales",
+  "alloc",
+  "clock",
+], optional = true }
+base64 = { version = "0.21.2", optional = true }
+sha2 = { version = "0.10.7", default-features = false, optional = true }
+
+
+[dev-dependencies]

rpxy-lib/src/backend/load_balance.rs

@@ -0,0 +1,135 @@
+#[cfg(feature = "sticky-cookie")]
+pub use super::{
+  load_balance_sticky::{LbStickyRoundRobin, LbStickyRoundRobinBuilder},
+  sticky_cookie::StickyCookie,
+};
+use derive_builder::Builder;
+use rand::Rng;
+use std::sync::{
+  atomic::{AtomicUsize, Ordering},
+  Arc,
+};
+
+/// Constants to specify a load balance option
+pub(super) mod load_balance_options {
+  pub const FIX_TO_FIRST: &str = "none";
+  pub const ROUND_ROBIN: &str = "round_robin";
+  pub const RANDOM: &str = "random";
+  #[cfg(feature = "sticky-cookie")]
+  pub const STICKY_ROUND_ROBIN: &str = "sticky";
+}
+
+#[derive(Debug, Clone)]
+/// Pointer to upstream serving the incoming request.
+/// If 'sticky cookie'-based LB is enabled and cookie must be updated/created, the new cookie is also given.
+pub(super) struct PointerToUpstream {
+  pub ptr: usize,
+  pub context_lb: Option<LbContext>,
+}
+/// Trait for LB
+pub(super) trait LbWithPointer {
+  fn get_ptr(&self, req_info: Option<&LbContext>) -> PointerToUpstream;
+}
+
+#[derive(Debug, Clone, Builder)]
+/// Round Robin LB object as a pointer to the current serving upstream destination
+pub struct LbRoundRobin {
+  #[builder(default)]
+  /// Pointer to the index of the last served upstream destination
+  ptr: Arc<AtomicUsize>,
+  #[builder(setter(custom), default)]
+  /// Number of upstream destinations
+  num_upstreams: usize,
+}
+impl LbRoundRobinBuilder {
+  pub fn num_upstreams(&mut self, v: &usize) -> &mut Self {
+    self.num_upstreams = Some(*v);
+    self
+  }
+}
+impl LbWithPointer for LbRoundRobin {
+  /// Increment the count of upstream served up to the max value
+  fn get_ptr(&self, _info: Option<&LbContext>) -> PointerToUpstream {
+    // Get a current count of upstream served
+    let current_ptr = self.ptr.load(Ordering::Relaxed);
+
+    let ptr = if current_ptr < self.num_upstreams - 1 {
+      self.ptr.fetch_add(1, Ordering::Relaxed)
+    } else {
+      // Clear the counter
+      self.ptr.fetch_and(0, Ordering::Relaxed)
+    };
+    PointerToUpstream { ptr, context_lb: None }
+  }
+}
+
+#[derive(Debug, Clone, Builder)]
+/// Random LB object to keep the object of random pools
+pub struct LbRandom {
+  #[builder(setter(custom), default)]
+  /// Number of upstream destinations
+  num_upstreams: usize,
+}
+impl LbRandomBuilder {
+  pub fn num_upstreams(&mut self, v: &usize) -> &mut Self {
+    self.num_upstreams = Some(*v);
+    self
+  }
+}
+impl LbWithPointer for LbRandom {
+  /// Returns the random index within the range
+  fn get_ptr(&self, _info: Option<&LbContext>) -> PointerToUpstream {
+    let mut rng = rand::thread_rng();
+    let ptr = rng.gen_range(0..self.num_upstreams);
+    PointerToUpstream { ptr, context_lb: None }
+  }
+}
+
+#[derive(Debug, Clone)]
+/// Load Balancing Option
+pub enum LoadBalance {
+  /// Fix to the first upstream. Use if only one upstream destination is specified
+  FixToFirst,
+  /// Randomly chose one upstream server
+  Random(LbRandom),
+  /// Simple round robin without session persistance
+  RoundRobin(LbRoundRobin),
+  #[cfg(feature = "sticky-cookie")]
+  /// Round robin with session persistance using cookie
+  StickyRoundRobin(LbStickyRoundRobin),
+}
+impl Default for LoadBalance {
+  fn default() -> Self {
+    Self::FixToFirst
+  }
+}
+
+impl LoadBalance {
+  /// Get the index of the upstream serving the incoming request
+  pub(super) fn get_context(&self, _context_to_lb: &Option<LbContext>) -> PointerToUpstream {
+    match self {
+      LoadBalance::FixToFirst => PointerToUpstream {
+        ptr: 0usize,
+        context_lb: None,
+      },
+      LoadBalance::RoundRobin(ptr) => ptr.get_ptr(None),
+      LoadBalance::Random(ptr) => ptr.get_ptr(None),
+      #[cfg(feature = "sticky-cookie")]
+      LoadBalance::StickyRoundRobin(ptr) => {
+        // Generate new context if sticky round robin is enabled.
+        ptr.get_ptr(_context_to_lb.as_ref())
+      }
+    }
+  }
+}
+
+#[derive(Debug, Clone)]
+/// Struct to handle the sticky cookie string,
+/// - passed from Rp module (http handler) to LB module, manipulated from req, only StickyCookieValue exists.
+/// - passed from LB module to Rp module (http handler), will be inserted into res, StickyCookieValue and Info exist.
+pub struct LbContext {
+  #[cfg(feature = "sticky-cookie")]
+  pub sticky_cookie: StickyCookie,
+  #[cfg(not(feature = "sticky-cookie"))]
+  pub sticky_cookie: (),
+}

rpxy-lib/src/backend/load_balance_sticky.rs

@@ -0,0 +1,132 @@
+use super::{
+  load_balance::{LbContext, LbWithPointer, PointerToUpstream},
+  sticky_cookie::StickyCookieConfig,
+  Upstream,
+};
+use crate::{constants::STICKY_COOKIE_NAME, log::*};
+use derive_builder::Builder;
+use rustc_hash::FxHashMap as HashMap;
+use std::{
+  borrow::Cow,
+  sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+  },
+};
+
+#[derive(Debug, Clone, Builder)]
+/// Round Robin LB object in the sticky cookie manner
+pub struct LbStickyRoundRobin {
+  #[builder(default)]
+  /// Pointer to the index of the last served upstream destination
+  ptr: Arc<AtomicUsize>,
+  #[builder(setter(custom), default)]
+  /// Number of upstream destinations
+  num_upstreams: usize,
+  #[builder(setter(custom))]
+  /// Information to build the cookie to stick clients to specific backends
+  pub sticky_config: StickyCookieConfig,
+  #[builder(setter(custom))]
+  /// Hashmaps:
+  /// - Hashmap that maps server indices to server id (string)
+  /// - Hashmap that maps server ids (string) to server indices, for fast reverse lookup
+  upstream_maps: UpstreamMap,
+}
+#[derive(Debug, Clone)]
+pub struct UpstreamMap {
+  /// Hashmap that maps server indices to server id (string)
+  upstream_index_map: Vec<String>,
+  /// Hashmap that maps server ids (string) to server indices, for fast reverse lookup
+  upstream_id_map: HashMap<String, usize>,
+}
+impl LbStickyRoundRobinBuilder {
+  pub fn num_upstreams(&mut self, v: &usize) -> &mut Self {
+    self.num_upstreams = Some(*v);
+    self
+  }
+  pub fn sticky_config(&mut self, server_name: &str, path_opt: &Option<String>) -> &mut Self {
+    self.sticky_config = Some(StickyCookieConfig {
+      name: STICKY_COOKIE_NAME.to_string(), // TODO: config等で変更できるように
+      domain: server_name.to_ascii_lowercase(),
+      path: if let Some(v) = path_opt {
+        v.to_ascii_lowercase()
+      } else {
+        "/".to_string()
+      },
+      duration: 300, // TODO: config等で変更できるように
+    });
+    self
+  }
+  pub fn upstream_maps(&mut self, upstream_vec: &[Upstream]) -> &mut Self {
+    let upstream_index_map: Vec<String> = upstream_vec
+      .iter()
+      .enumerate()
+      .map(|(i, v)| v.calculate_id_with_index(i))
+      .collect();
+    let mut upstream_id_map = HashMap::default();
+    for (i, v) in upstream_index_map.iter().enumerate() {
+      upstream_id_map.insert(v.to_string(), i);
+    }
+    self.upstream_maps = Some(UpstreamMap {
+      upstream_index_map,
+      upstream_id_map,
+    });
+    self
+  }
+}
+impl<'a> LbStickyRoundRobin {
+  fn simple_increment_ptr(&self) -> usize {
+    // Get a current count of upstream served
+    let current_ptr = self.ptr.load(Ordering::Relaxed);
+
+    if current_ptr < self.num_upstreams - 1 {
+      self.ptr.fetch_add(1, Ordering::Relaxed)
+    } else {
+      // Clear the counter
+      self.ptr.fetch_and(0, Ordering::Relaxed)
+    }
+  }
+  /// This is always called only internally. So 'unwrap()' is executed.
+  fn get_server_id_from_index(&self, index: usize) -> String {
+    self.upstream_maps.upstream_index_map.get(index).unwrap().to_owned()
+  }
+  /// This function takes value passed from outside. So 'result' is used.
+  fn get_server_index_from_id(&self, id: impl Into<Cow<'a, str>>) -> Option<usize> {
+    let id_str = id.into().to_string();
+    self.upstream_maps.upstream_id_map.get(&id_str).map(|v| v.to_owned())
+  }
+}
+impl LbWithPointer for LbStickyRoundRobin {
+  fn get_ptr(&self, req_info: Option<&LbContext>) -> PointerToUpstream {
+    // If given context is None or invalid (not contained), get_ptr() is invoked to increment the pointer.
+    // Otherwise, get the server index indicated by the server_id inside the cookie
+    let ptr = match req_info {
+      None => {
+        debug!("No sticky cookie");
+        self.simple_increment_ptr()
+      }
+      Some(context) => {
+        let server_id = &context.sticky_cookie.value.value;
+        if let Some(server_index) = self.get_server_index_from_id(server_id) {
+          debug!("Valid sticky cookie: id={}, index={}", server_id, server_index);
+          server_index
+        } else {
+          debug!("Invalid sticky cookie: id={}", server_id);
+          self.simple_increment_ptr()
+        }
+      }
+    };
+
+    // Get the server id from the ptr.
+    // TODO: This should be simplified and optimized if ptr is not changed (id value exists in cookie).
+    let upstream_id = self.get_server_id_from_index(ptr);
+    let new_cookie = self.sticky_config.build_sticky_cookie(upstream_id).unwrap();
+    let new_context = Some(LbContext {
+      sticky_cookie: new_cookie,
+    });
+    PointerToUpstream {
+      ptr,
+      context_lb: new_context,
+    }
+  }
+}

rpxy-lib/src/backend/mod.rs

@@ -0,0 +1,77 @@
+mod load_balance;
+#[cfg(feature = "sticky-cookie")]
+mod load_balance_sticky;
+#[cfg(feature = "sticky-cookie")]
+mod sticky_cookie;
+mod upstream;
+mod upstream_opts;
+
+#[cfg(feature = "sticky-cookie")]
+pub use self::sticky_cookie::{StickyCookie, StickyCookieValue};
+pub use self::{
+  load_balance::{LbContext, LoadBalance},
+  upstream::{ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder},
+  upstream_opts::UpstreamOption,
+};
+use crate::{
+  certs::CryptoSource,
+  utils::{BytesName, PathNameBytesExp, ServerNameBytesExp},
+};
+use derive_builder::Builder;
+use rustc_hash::FxHashMap as HashMap;
+use std::borrow::Cow;
+
+/// Struct serving information to route incoming connections, like server name to be handled and tls certs/keys settings.
+#[derive(Builder)]
+pub struct Backend<T>
+where
+  T: CryptoSource,
+{
+  #[builder(setter(into))]
+  /// backend application name, e.g., app1
+  pub app_name: String,
+  #[builder(setter(custom))]
+  /// server name, e.g., example.com, in String ascii lower case
+  pub server_name: String,
+  /// struct of reverse proxy serving incoming request
+  pub reverse_proxy: ReverseProxy,
+
+  /// tls settings: https redirection with 30x
+  #[builder(default)]
+  pub https_redirection: Option<bool>,
+
+  /// TLS settings: source meta for server cert, key, client ca cert
+  #[builder(default)]
+  pub crypto_source: Option<T>,
+}
+impl<'a, T> BackendBuilder<T>
+where
+  T: CryptoSource,
+{
+  pub fn server_name(&mut self, server_name: impl Into<Cow<'a, str>>) -> &mut Self {
+    self.server_name = Some(server_name.into().to_ascii_lowercase());
+    self
+  }
+}
+
+/// HashMap and some meta information for multiple Backend structs.
+pub struct Backends<T>
+where
+  T: CryptoSource,
+{
+  pub apps: HashMap<ServerNameBytesExp, Backend<T>>, // hyper::uriで抜いたhostで引っ掛ける
+  pub default_server_name_bytes: Option<ServerNameBytesExp>, // for plaintext http
+}
+
+impl<T> Backends<T>
+where
+  T: CryptoSource,
+{
+  #[allow(clippy::new_without_default)]
+  pub fn new() -> Self {
+    Backends {
+      apps: HashMap::<ServerNameBytesExp, Backend<T>>::default(),
+      default_server_name_bytes: None,
+    }
+  }
+}

rpxy-lib/src/backend/sticky_cookie.rs

@@ -0,0 +1,208 @@
+use std::borrow::Cow;
+
+use crate::error::*;
+use chrono::{TimeZone, Utc};
+use derive_builder::Builder;
+
+#[derive(Debug, Clone, Builder)]
+/// Cookie value only, used for COOKIE in req
+pub struct StickyCookieValue {
+  #[builder(setter(custom))]
+  /// Field name indicating sticky cookie
+  pub name: String,
+  #[builder(setter(custom))]
+  /// Upstream server_id
+  pub value: String,
+}
+impl<'a> StickyCookieValueBuilder {
+  pub fn name(&mut self, v: impl Into<Cow<'a, str>>) -> &mut Self {
+    self.name = Some(v.into().to_ascii_lowercase());
+    self
+  }
+  pub fn value(&mut self, v: impl Into<Cow<'a, str>>) -> &mut Self {
+    self.value = Some(v.into().to_string());
+    self
+  }
+}
+impl StickyCookieValue {
+  pub fn try_from(value: &str, expected_name: &str) -> Result<Self> {
+    if !value.starts_with(expected_name) {
+      return Err(RpxyError::LoadBalance(
+        "Failed to cookie conversion from string".to_string(),
+      ));
+    };
+    let kv = value.split('=').map(|v| v.trim()).collect::<Vec<&str>>();
+    if kv.len() != 2 {
+      return Err(RpxyError::LoadBalance("Invalid cookie structure".to_string()));
+    };
+    if kv[1].is_empty() {
+      return Err(RpxyError::LoadBalance("No sticky cookie value".to_string()));
+    }
+    Ok(StickyCookieValue {
+      name: expected_name.to_string(),
+      value: kv[1].to_string(),
+    })
+  }
+}
+
+#[derive(Debug, Clone, Builder)]
+/// Struct describing sticky cookie meta information used for SET-COOKIE in res
+pub struct StickyCookieInfo {
+  #[builder(setter(custom))]
+  /// Unix time
+  pub expires: i64,
+
+  #[builder(setter(custom))]
+  /// Domain
+  pub domain: String,
+
+  #[builder(setter(custom))]
+  /// Path
+  pub path: String,
+}
+impl<'a> StickyCookieInfoBuilder {
+  pub fn domain(&mut self, v: impl Into<Cow<'a, str>>) -> &mut Self {
+    self.domain = Some(v.into().to_ascii_lowercase());
+    self
+  }
+  pub fn path(&mut self, v: impl Into<Cow<'a, str>>) -> &mut Self {
+    self.path = Some(v.into().to_ascii_lowercase());
+    self
+  }
+  pub fn expires(&mut self, duration_secs: i64) -> &mut Self {
+    let current = Utc::now().timestamp();
+    self.expires = Some(current + duration_secs);
+    self
+  }
+}
+
+#[derive(Debug, Clone, Builder)]
+/// Struct describing sticky cookie
+pub struct StickyCookie {
+  #[builder(setter(custom))]
+  /// Upstream server_id
+  pub value: StickyCookieValue,
+  #[builder(setter(custom), default)]
+  /// Upstream server_id
+  pub info: Option<StickyCookieInfo>,
+}
+
+impl<'a> StickyCookieBuilder {
+  pub fn value(&mut self, n: impl Into<Cow<'a, str>>, v: impl Into<Cow<'a, str>>) -> &mut Self {
+    self.value = Some(StickyCookieValueBuilder::default().name(n).value(v).build().unwrap());
+    self
+  }
+  pub fn info(
+    &mut self,
+    domain: impl Into<Cow<'a, str>>,
+    path: impl Into<Cow<'a, str>>,
+    duration_secs: i64,
+  ) -> &mut Self {
+    let info = StickyCookieInfoBuilder::default()
+      .domain(domain)
+      .path(path)
+      .expires(duration_secs)
+      .build()
+      .unwrap();
+    self.info = Some(Some(info));
+    self
+  }
+}
+
+impl TryInto<String> for StickyCookie {
+  type Error = RpxyError;
+
+  fn try_into(self) -> Result<String> {
+    if self.info.is_none() {
+      return Err(RpxyError::LoadBalance(
+        "Failed to cookie conversion into string: no meta information".to_string(),
+      ));
+    }
+    let info = self.info.unwrap();
+    let chrono::LocalResult::Single(expires_timestamp) = Utc.timestamp_opt(info.expires, 0) else {
+      return Err(RpxyError::LoadBalance("Failed to cookie conversion into string".to_string()));
+    };
+    let exp_str = expires_timestamp.format("%a, %d-%b-%Y %T GMT").to_string();
+    let max_age = info.expires - Utc::now().timestamp();
+
+    Ok(format!(
+      "{}={}; expires={}; Max-Age={}; path={}; domain={}",
+      self.value.name, self.value.value, exp_str, max_age, info.path, info.domain
+    ))
+  }
+}
+
+#[derive(Debug, Clone)]
+/// Configuration to serve incoming requests in the manner of "sticky cookie".
+/// Including a dictionary to map Ids included in cookie and upstream destinations,
+/// and expiration of cookie.
+/// "domain" and "path" in the cookie will be the same as the reverse proxy options.
+pub struct StickyCookieConfig {
+  pub name: String,
+  pub domain: String,
+  pub path: String,
+  pub duration: i64,
+}
+impl<'a> StickyCookieConfig {
+  pub fn build_sticky_cookie(&self, v: impl Into<Cow<'a, str>>) -> Result<StickyCookie> {
+    StickyCookieBuilder::default()
+      .value(self.name.clone(), v)
+      .info(&self.domain, &self.path, self.duration)
+      .build()
+      .map_err(|_| RpxyError::LoadBalance("Failed to build sticky cookie from config".to_string()))
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use crate::constants::STICKY_COOKIE_NAME;
+
+  #[test]
+  fn config_works() {
+    let config = StickyCookieConfig {
+      name: STICKY_COOKIE_NAME.to_string(),
+      domain: "example.com".to_string(),
+      path: "/path".to_string(),
+      duration: 100,
+    };
+    let expires_unix = Utc::now().timestamp() + 100;
+    let sc_string: Result<String> = config.build_sticky_cookie("test_value").unwrap().try_into();
+    let expires_date_string = Utc
+      .timestamp_opt(expires_unix, 0)
+      .unwrap()
+      .format("%a, %d-%b-%Y %T GMT")
+      .to_string();
+    assert_eq!(
+      sc_string.unwrap(),
+      format!(
+        "{}=test_value; expires={}; Max-Age={}; path=/path; domain=example.com",
+        STICKY_COOKIE_NAME, expires_date_string, 100
+      )
+    );
+  }
+  #[test]
+  fn to_string_works() {
+    let sc = StickyCookie {
+      value: StickyCookieValue {
+        name: STICKY_COOKIE_NAME.to_string(),
+        value: "test_value".to_string(),
+      },
+      info: Some(StickyCookieInfo {
+        expires: 1686221173i64,
+        domain: "example.com".to_string(),
+        path: "/path".to_string(),
+      }),
+    };
+    let sc_string: Result<String> = sc.try_into();
+    let max_age = 1686221173i64 - Utc::now().timestamp();
+    assert!(sc_string.is_ok());
+    assert_eq!(
+      sc_string.unwrap(),
+      format!(
+        "{}=test_value; expires=Thu, 08-Jun-2023 10:46:13 GMT; Max-Age={}; path=/path; domain=example.com",
+        STICKY_COOKIE_NAME, max_age
+      )
+    );
+  }
+}

rpxy-lib/src/backend/upstream.rs

@@ -0,0 +1,201 @@
+#[cfg(feature = "sticky-cookie")]
+use super::load_balance::LbStickyRoundRobinBuilder;
+use super::load_balance::{load_balance_options as lb_opts, LbRandomBuilder, LbRoundRobinBuilder, LoadBalance};
+use super::{BytesName, LbContext, PathNameBytesExp, UpstreamOption};
+use crate::log::*;
+#[cfg(feature = "sticky-cookie")]
+use base64::{engine::general_purpose, Engine as _};
+use derive_builder::Builder;
+use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
+#[cfg(feature = "sticky-cookie")]
+use sha2::{Digest, Sha256};
+use std::borrow::Cow;
+#[derive(Debug, Clone)]
+pub struct ReverseProxy {
+  pub upstream: HashMap<PathNameBytesExp, UpstreamGroup>, // TODO: HashMapでいいのかは疑問。max_by_keyでlongest prefix matchしてるのも無駄っぽいが。。。
+}
+
+impl ReverseProxy {
+  /// Get an appropriate upstream destination for given path string.
+  pub fn get<'a>(&self, path_str: impl Into<Cow<'a, str>>) -> Option<&UpstreamGroup> {
+    // trie使ってlongest prefix match させてもいいけどルート記述は少ないと思われるので、
+    // コスト的にこの程度で十分
+    let path_bytes = &path_str.to_path_name_vec();
+
+    let matched_upstream = self
+      .upstream
+      .iter()
+      .filter(|(route_bytes, _)| {
+        match path_bytes.starts_with(route_bytes) {
+          true => {
+            route_bytes.len() == 1 // route = '/', i.e., default
+            || match path_bytes.get(route_bytes.len()) {
+              None => true, // exact case
+              Some(p) => p == &b'/', // sub-path case
+            }
+          }
+          _ => false,
+        }
+      })
+      .max_by_key(|(route_bytes, _)| route_bytes.len());
+    if let Some((_path, u)) = matched_upstream {
+      debug!(
+        "Found upstream: {:?}",
+        String::from_utf8(_path.0.clone()).unwrap_or_else(|_| "<none>".to_string())
+      );
+      Some(u)
+    } else {
+      None
+    }
+  }
+}
+
+#[derive(Debug, Clone)]
+/// Upstream struct just containing uri without path
+pub struct Upstream {
+  /// Base uri without specific path
+  pub uri: hyper::Uri,
+}
+impl Upstream {
+  #[cfg(feature = "sticky-cookie")]
+  /// Hashing uri with index to avoid collision
+  pub fn calculate_id_with_index(&self, index: usize) -> String {
+    let mut hasher = Sha256::new();
+    let uri_string = format!("{}&index={}", self.uri.clone(), index);
+    hasher.update(uri_string.as_bytes());
+    let digest = hasher.finalize();
+    general_purpose::URL_SAFE_NO_PAD.encode(digest)
+  }
+}
+#[derive(Debug, Clone, Builder)]
+/// Struct serving multiple upstream servers for, e.g., load balancing.
+pub struct UpstreamGroup {
+  #[builder(setter(custom))]
+  /// Upstream server(s)
+  pub upstream: Vec<Upstream>,
+  #[builder(setter(custom), default)]
+  /// Path like "/path" in [[PathNameBytesExp]] associated with the upstream server(s)
+  pub path: PathNameBytesExp,
+  #[builder(setter(custom), default)]
+  /// Path in [[PathNameBytesExp]] that will be used to replace the "path" part of incoming url
+  pub replace_path: Option<PathNameBytesExp>,
+
+  #[builder(setter(custom), default)]
+  /// Load balancing option
+  pub lb: LoadBalance,
+  #[builder(setter(custom), default)]
+  /// Activated upstream options defined in [[UpstreamOption]]
+  pub opts: HashSet<UpstreamOption>,
+}
+
+impl UpstreamGroupBuilder {
+  pub fn upstream(&mut self, upstream_vec: &[Upstream]) -> &mut Self {
+    self.upstream = Some(upstream_vec.to_vec());
+    self
+  }
+  pub fn path(&mut self, v: &Option<String>) -> &mut Self {
+    let path = match v {
+      Some(p) => p.to_path_name_vec(),
+      None => "/".to_path_name_vec(),
+    };
+    self.path = Some(path);
+    self
+  }
+  pub fn replace_path(&mut self, v: &Option<String>) -> &mut Self {
+    self.replace_path = Some(
+      v.to_owned()
+        .as_ref()
+        .map_or_else(|| None, |v| Some(v.to_path_name_vec())),
+    );
+    self
+  }
+  pub fn lb(
+    &mut self,
+    v: &Option<String>,
+    // upstream_num: &usize,
+    upstream_vec: &Vec<Upstream>,
+    _server_name: &str,
+    _path_opt: &Option<String>,
+  ) -> &mut Self {
+    let upstream_num = &upstream_vec.len();
+    let lb = if let Some(x) = v {
+      match x.as_str() {
+        lb_opts::FIX_TO_FIRST => LoadBalance::FixToFirst,
+        lb_opts::RANDOM => LoadBalance::Random(LbRandomBuilder::default().num_upstreams(upstream_num).build().unwrap()),
+        lb_opts::ROUND_ROBIN => LoadBalance::RoundRobin(
+          LbRoundRobinBuilder::default()
+            .num_upstreams(upstream_num)
+            .build()
+            .unwrap(),
+        ),
+        #[cfg(feature = "sticky-cookie")]
+        lb_opts::STICKY_ROUND_ROBIN => LoadBalance::StickyRoundRobin(
+          LbStickyRoundRobinBuilder::default()
+            .num_upstreams(upstream_num)
+            .sticky_config(_server_name, _path_opt)
+            .upstream_maps(upstream_vec) // TODO:
+            .build()
+            .unwrap(),
+        ),
+        _ => {
+          error!("Specified load balancing option is invalid.");
+          LoadBalance::default()
+        }
+      }
+    } else {
+      LoadBalance::default()
+    };
+    self.lb = Some(lb);
+    self
+  }
+  pub fn opts(&mut self, v: &Option<Vec<String>>) -> &mut Self {
+    let opts = if let Some(opts) = v {
+      opts
+        .iter()
+        .filter_map(|str| UpstreamOption::try_from(str.as_str()).ok())
+        .collect::<HashSet<UpstreamOption>>()
+    } else {
+      Default::default()
+    };
+    self.opts = Some(opts);
+    self
+  }
+}
+
+impl UpstreamGroup {
+  /// Get an enabled option of load balancing [[LoadBalance]]
+  pub fn get(&self, context_to_lb: &Option<LbContext>) -> (Option<&Upstream>, Option<LbContext>) {
+    let pointer_to_upstream = self.lb.get_context(context_to_lb);
+    debug!("Upstream of index {} is chosen.", pointer_to_upstream.ptr);
+    debug!("Context to LB (Cookie in Req): {:?}", context_to_lb);
+    debug!(
+      "Context from LB (Set-Cookie in Res): {:?}",
+      pointer_to_upstream.context_lb
+    );
+    (
+      self.upstream.get(pointer_to_upstream.ptr),
+      pointer_to_upstream.context_lb,
+    )
+  }
+}
+
+#[cfg(test)]
+mod test {
+  #[allow(unused)]
+  use super::*;
+
+  #[cfg(feature = "sticky-cookie")]
+  #[test]
+  fn calc_id_works() {
+    let uri = "https://www.rust-lang.org".parse::<hyper::Uri>().unwrap();
+    let upstream = Upstream { uri };
+    assert_eq!(
+      "eGsjoPbactQ1eUJjafYjPT3ekYZQkaqJnHdA_FMSkgM",
+      upstream.calculate_id_with_index(0)
+    );
+    assert_eq!(
+      "tNVXFJ9eNCT2mFgKbYq35XgH5q93QZtfU8piUiiDxVA",
+      upstream.calculate_id_with_index(1)
+    );
+  }
+}

rpxy-lib/src/backend/upstream_opts.rs

@@ -0,0 +1,22 @@
+use crate::error::*;
+
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub enum UpstreamOption {
+  OverrideHost,
+  UpgradeInsecureRequests,
+  ConvertHttpsTo11,
+  ConvertHttpsTo2,
+  // TODO: Adds more options for heder override
+}
+impl TryFrom<&str> for UpstreamOption {
+  type Error = RpxyError;
+  fn try_from(val: &str) -> Result<Self> {
+    match val {
+      "override_host" => Ok(Self::OverrideHost),
+      "upgrade_insecure_requests" => Ok(Self::UpgradeInsecureRequests),
+      "convert_https_to_11" => Ok(Self::ConvertHttpsTo11),
+      "convert_https_to_2" => Ok(Self::ConvertHttpsTo2),
+      _ => Err(RpxyError::Other(anyhow!("Unsupported header option"))),
+    }
+  }
+}

rpxy-lib/src/certs.rs

@@ -0,0 +1,91 @@
+use async_trait::async_trait;
+use rustc_hash::FxHashSet as HashSet;
+use rustls::{
+  sign::{any_supported_type, CertifiedKey},
+  Certificate, OwnedTrustAnchor, PrivateKey,
+};
+use std::io;
+use x509_parser::prelude::*;
+
+#[async_trait]
+// Trait to read certs and keys anywhere from KVS, file, sqlite, etc.
+pub trait CryptoSource {
+  type Error;
+
+  /// read crypto materials from source
+  async fn read(&self) -> Result<CertsAndKeys, Self::Error>;
+
+  /// Returns true when mutual tls is enabled
+  fn is_mutual_tls(&self) -> bool;
+}
+
+/// Certificates and private keys in rustls loaded from files
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct CertsAndKeys {
+  pub certs: Vec<Certificate>,
+  pub cert_keys: Vec<PrivateKey>,
+  pub client_ca_certs: Option<Vec<Certificate>>,
+}
+
+impl CertsAndKeys {
+  pub fn parse_server_certs_and_keys(&self) -> Result<CertifiedKey, anyhow::Error> {
+    // for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
+    let signing_key = self
+      .cert_keys
+      .iter()
+      .find_map(|k| {
+        if let Ok(sk) = any_supported_type(k) {
+          Some(sk)
+        } else {
+          None
+        }
+      })
+      .ok_or_else(|| {
+        io::Error::new(
+          io::ErrorKind::InvalidInput,
+          "Unable to find a valid certificate and key",
+        )
+      })?;
+    Ok(CertifiedKey::new(self.certs.clone(), signing_key))
+  }
+
+  pub fn parse_client_ca_certs(&self) -> Result<(Vec<OwnedTrustAnchor>, HashSet<Vec<u8>>), anyhow::Error> {
+    let certs = self.client_ca_certs.as_ref().ok_or(anyhow::anyhow!("No client cert"))?;
+
+    let owned_trust_anchors: Vec<_> = certs
+      .iter()
+      .map(|v| {
+        // let trust_anchor = tokio_rustls::webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap();
+        let trust_anchor = webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap();
+        rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(
+          trust_anchor.subject,
+          trust_anchor.spki,
+          trust_anchor.name_constraints,
+        )
+      })
+      .collect();
+
+    // TODO: SKID is not used currently
+    let subject_key_identifiers: HashSet<_> = certs
+      .iter()
+      .filter_map(|v| {
+        // retrieve ca key id (subject key id)
+        let cert = parse_x509_certificate(&v.0).unwrap().1;
+        let subject_key_ids = cert
+          .iter_extensions()
+          .filter_map(|ext| match ext.parsed_extension() {
+            ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid),
+            _ => None,
+          })
+          .collect::<Vec<_>>();
+        if !subject_key_ids.is_empty() {
+          Some(subject_key_ids[0].0.to_owned())
+        } else {
+          None
+        }
+      })
+      .collect();
+
+    Ok((owned_trust_anchors, subject_key_identifiers))
+  }
+}

rpxy-lib/src/constants.rs

@@ -0,0 +1,31 @@
+// pub const LISTEN_ADDRESSES_V4: &[&str] = &["0.0.0.0"];
+// pub const LISTEN_ADDRESSES_V6: &[&str] = &["[::]"];
+// pub const HTTP_LISTEN_PORT: u16 = 8080;
+// pub const HTTPS_LISTEN_PORT: u16 = 8443;
+pub const PROXY_TIMEOUT_SEC: u64 = 60;
+pub const UPSTREAM_TIMEOUT_SEC: u64 = 60;
+pub const TLS_HANDSHAKE_TIMEOUT_SEC: u64 = 15; // default as with firefox browser
+pub const MAX_CLIENTS: usize = 512;
+pub const MAX_CONCURRENT_STREAMS: u32 = 64;
+pub const CERTS_WATCH_DELAY_SECS: u32 = 60;
+pub const LOAD_CERTS_ONLY_WHEN_UPDATED: bool = true;
+
+// #[cfg(feature = "http3")]
+// pub const H3_RESPONSE_BUF_SIZE: usize = 65_536; // 64KB
+// #[cfg(feature = "http3")]
+// pub const H3_REQUEST_BUF_SIZE: usize = 65_536; // 64KB // handled by quinn
+
+#[allow(non_snake_case)]
+#[cfg(feature = "http3")]
+pub mod H3 {
+  pub const ALT_SVC_MAX_AGE: u32 = 3600;
+  pub const REQUEST_MAX_BODY_SIZE: usize = 268_435_456; // 256MB
+  pub const MAX_CONCURRENT_CONNECTIONS: u32 = 4096;
+  pub const MAX_CONCURRENT_BIDISTREAM: u32 = 64;
+  pub const MAX_CONCURRENT_UNISTREAM: u32 = 64;
+  pub const MAX_IDLE_TIMEOUT: u64 = 10; // secs
+}
+
+#[cfg(feature = "sticky-cookie")]
+/// For load-balancing with sticky cookie
+pub const STICKY_COOKIE_NAME: &str = "rpxy_srv_id";

rpxy-lib/src/error.rs

@@ -0,0 +1,69 @@
+pub use anyhow::{anyhow, bail, ensure, Context};
+use std::io;
+use thiserror::Error;
+
+pub type Result<T> = std::result::Result<T, RpxyError>;
+
+/// Describes things that can go wrong in the Rpxy
+#[derive(Debug, Error)]
+pub enum RpxyError {
+  #[error("Proxy build error")]
+  ProxyBuild(#[from] crate::proxy::ProxyBuilderError),
+
+  #[error("MessageHandler build error")]
+  HandlerBuild(#[from] crate::handler::HttpMessageHandlerBuilderError),
+
+  #[error("Http Message Handler Error: {0}")]
+  Handler(&'static str),
+
+  #[error("Http Request Message Error: {0}")]
+  Request(&'static str),
+
+  #[error("TCP/UDP Proxy Layer Error: {0}")]
+  Proxy(String),
+
+  #[allow(unused)]
+  #[error("LoadBalance Layer Error: {0}")]
+  LoadBalance(String),
+
+  #[error("I/O Error")]
+  Io(#[from] io::Error),
+
+  // #[error("Toml Deserialization Error")]
+  // TomlDe(#[from] toml::de::Error),
+  #[cfg(feature = "http3")]
+  #[error("Quic Connection Error")]
+  QuicConn(#[from] quinn::ConnectionError),
+
+  #[cfg(feature = "http3")]
+  #[error("H3 Error")]
+  H3(#[from] h3::Error),
+
+  #[error("rustls Connection Error")]
+  Rustls(#[from] rustls::Error),
+
+  #[error("Hyper Error")]
+  Hyper(#[from] hyper::Error),
+
+  #[error("Hyper Http Error")]
+  HyperHttp(#[from] hyper::http::Error),
+
+  #[error("Hyper Http HeaderValue Error")]
+  HyperHeaderValue(#[from] hyper::header::InvalidHeaderValue),
+
+  #[error("Hyper Http HeaderName Error")]
+  HyperHeaderName(#[from] hyper::header::InvalidHeaderName),
+
+  #[error(transparent)]
+  Other(#[from] anyhow::Error),
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Error, Clone)]
+pub enum ClientCertsError {
+  #[error("TLS Client Certificate is Required for Given SNI: {0}")]
+  ClientCertRequired(String),
+
+  #[error("Inconsistent TLS Client Certificate for Given SNI: {0}")]
+  InconsistentClientCert(String),
+}

rpxy-lib/src/globals.rs

@@ -0,0 +1,120 @@
+use crate::{backend::Backends, certs::CryptoSource, constants::*};
+use std::net::SocketAddr;
+use std::sync::{
+  atomic::{AtomicUsize, Ordering},
+  Arc,
+};
+use tokio::time::Duration;
+
+/// Global object containing proxy configurations and shared object like counters.
+/// But note that in Globals, we do not have Mutex and RwLock. It is indeed, the context shared among async tasks.
+pub struct Globals<T>
+where
+  T: CryptoSource,
+{
+  /// Configuration parameters for proxy transport and request handlers
+  pub proxy_config: ProxyConfig, // TODO: proxy configはarcに包んでこいつだけ使いまわせばいいように変えていく。backendsも？
+
+  /// Backend application objects to which http request handler forward incoming requests
+  pub backends: Backends<T>,
+
+  /// Shared context - Counter for serving requests
+  pub request_count: RequestCount,
+
+  /// Shared context - Async task runtime handler
+  pub runtime_handle: tokio::runtime::Handle,
+}
+
+/// Configuration parameters for proxy transport and request handlers
+pub struct ProxyConfig {
+  pub listen_sockets: Vec<SocketAddr>, // when instantiate server
+  pub http_port: Option<u16>,          // when instantiate server
+  pub https_port: Option<u16>,         // when instantiate server
+
+  pub proxy_timeout: Duration,    // when serving requests at Proxy
+  pub upstream_timeout: Duration, // when serving requests at Handler
+
+  pub max_clients: usize,          // when serving requests
+  pub max_concurrent_streams: u32, // when instantiate server
+  pub keepalive: bool,             // when instantiate server
+
+  // experimentals
+  pub sni_consistency: bool, // Handler
+  // All need to make packet acceptor
+  #[cfg(feature = "http3")]
+  pub http3: bool,
+  #[cfg(feature = "http3")]
+  pub h3_alt_svc_max_age: u32,
+  #[cfg(feature = "http3")]
+  pub h3_request_max_body_size: usize,
+  #[cfg(feature = "http3")]
+  pub h3_max_concurrent_bidistream: quinn::VarInt,
+  #[cfg(feature = "http3")]
+  pub h3_max_concurrent_unistream: quinn::VarInt,
+  #[cfg(feature = "http3")]
+  pub h3_max_concurrent_connections: u32,
+  #[cfg(feature = "http3")]
+  pub h3_max_idle_timeout: Option<quinn::IdleTimeout>,
+}
+
+impl Default for ProxyConfig {
+  fn default() -> Self {
+    Self {
+      listen_sockets: Vec::new(),
+      http_port: None,
+      https_port: None,
+
+      // TODO: Reconsider each timeout values
+      proxy_timeout: Duration::from_secs(PROXY_TIMEOUT_SEC),
+      upstream_timeout: Duration::from_secs(UPSTREAM_TIMEOUT_SEC),
+
+      max_clients: MAX_CLIENTS,
+      max_concurrent_streams: MAX_CONCURRENT_STREAMS,
+      keepalive: true,
+
+      sni_consistency: true,
+
+      #[cfg(feature = "http3")]
+      http3: false,
+      #[cfg(feature = "http3")]
+      h3_alt_svc_max_age: H3::ALT_SVC_MAX_AGE,
+      #[cfg(feature = "http3")]
+      h3_request_max_body_size: H3::REQUEST_MAX_BODY_SIZE,
+      #[cfg(feature = "http3")]
+      h3_max_concurrent_connections: H3::MAX_CONCURRENT_CONNECTIONS,
+      #[cfg(feature = "http3")]
+      h3_max_concurrent_bidistream: H3::MAX_CONCURRENT_BIDISTREAM.into(),
+      #[cfg(feature = "http3")]
+      h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM.into(),
+      #[cfg(feature = "http3")]
+      h3_max_idle_timeout: Some(quinn::IdleTimeout::try_from(Duration::from_secs(H3::MAX_IDLE_TIMEOUT)).unwrap()),
+    }
+  }
+}
+
+#[derive(Debug, Clone, Default)]
+/// Counter for serving requests
+pub struct RequestCount(Arc<AtomicUsize>);
+
+impl RequestCount {
+  pub fn current(&self) -> usize {
+    self.0.load(Ordering::Relaxed)
+  }
+
+  pub fn increment(&self) -> usize {
+    self.0.fetch_add(1, Ordering::Relaxed)
+  }
+
+  pub fn decrement(&self) -> usize {
+    let mut count;
+    while {
+      count = self.0.load(Ordering::Relaxed);
+      count > 0
+        && self
+          .0
+          .compare_exchange(count, count - 1, Ordering::Relaxed, Ordering::Relaxed)
+          != Ok(count)
+    } {}
+    count
+  }
+}

rpxy-lib/src/handler/handler_main.rs

@@ -0,0 +1,364 @@
+// Highly motivated by https://github.com/felipenoris/hyper-reverse-proxy
+use super::{utils_headers::*, utils_request::*, utils_synth_response::*, HandlerContext};
+use crate::{
+  backend::{Backend, UpstreamGroup},
+  certs::CryptoSource,
+  error::*,
+  globals::Globals,
+  log::*,
+  utils::ServerNameBytesExp,
+};
+use derive_builder::Builder;
+use hyper::{
+  client::connect::Connect,
+  header::{self, HeaderValue},
+  http::uri::Scheme,
+  Body, Client, Request, Response, StatusCode, Uri, Version,
+};
+use std::{env, net::SocketAddr, sync::Arc};
+use tokio::{io::copy_bidirectional, time::timeout};
+
+#[derive(Clone, Builder)]
+pub struct HttpMessageHandler<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone,
+{
+  forwarder: Arc<Client<T>>,
+  globals: Arc<Globals<U>>,
+}
+
+impl<T, U> HttpMessageHandler<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone,
+{
+  fn return_with_error_log(&self, status_code: StatusCode, log_data: &mut MessageLog) -> Result<Response<Body>> {
+    log_data.status_code(&status_code).output();
+    http_error(status_code)
+  }
+
+  pub async fn handle_request(
+    self,
+    mut req: Request<Body>,
+    client_addr: SocketAddr, // アクセス制御用
+    listen_addr: SocketAddr,
+    tls_enabled: bool,
+    tls_server_name: Option<ServerNameBytesExp>,
+  ) -> Result<Response<Body>> {
+    ////////
+    let mut log_data = MessageLog::from(&req);
+    log_data.client_addr(&client_addr);
+    //////
+
+    // Here we start to handle with server_name
+    let server_name = if let Ok(v) = req.parse_host() {
+      ServerNameBytesExp::from(v)
+    } else {
+      return self.return_with_error_log(StatusCode::BAD_REQUEST, &mut log_data);
+    };
+    // check consistency of between TLS SNI and HOST/Request URI Line.
+    #[allow(clippy::collapsible_if)]
+    if tls_enabled && self.globals.proxy_config.sni_consistency {
+      if server_name != tls_server_name.unwrap_or_default() {
+        return self.return_with_error_log(StatusCode::MISDIRECTED_REQUEST, &mut log_data);
+      }
+    }
+    // Find backend application for given server_name, and drop if incoming request is invalid as request.
+    let backend = if let Some(be) = self.globals.backends.apps.get(&server_name) {
+      be
+    } else if let Some(default_server_name) = &self.globals.backends.default_server_name_bytes {
+      debug!("Serving by default app");
+      self.globals.backends.apps.get(default_server_name).unwrap()
+    } else {
+      return self.return_with_error_log(StatusCode::SERVICE_UNAVAILABLE, &mut log_data);
+    };
+
+    // Redirect to https if !tls_enabled and redirect_to_https is true
+    if !tls_enabled && backend.https_redirection.unwrap_or(false) {
+      debug!("Redirect to secure connection: {}", &backend.server_name);
+      log_data.status_code(&StatusCode::PERMANENT_REDIRECT).output();
+      return secure_redirection(&backend.server_name, self.globals.proxy_config.https_port, &req);
+    }
+
+    // Find reverse proxy for given path and choose one of upstream host
+    // Longest prefix match
+    let path = req.uri().path();
+    let upstream_group = match backend.reverse_proxy.get(path) {
+      Some(ug) => ug,
+      None => return self.return_with_error_log(StatusCode::NOT_FOUND, &mut log_data),
+    };
+
+    // Upgrade in request header
+    let upgrade_in_request = extract_upgrade(req.headers());
+    let request_upgraded = req.extensions_mut().remove::<hyper::upgrade::OnUpgrade>();
+
+    // Build request from destination information
+    let _context = match self.generate_request_forwarded(
+      &client_addr,
+      &listen_addr,
+      &mut req,
+      &upgrade_in_request,
+      upstream_group,
+      tls_enabled,
+    ) {
+      Err(e) => {
+        error!("Failed to generate destination uri for reverse proxy: {}", e);
+        return self.return_with_error_log(StatusCode::SERVICE_UNAVAILABLE, &mut log_data);
+      }
+      Ok(v) => v,
+    };
+    debug!("Request to be forwarded: {:?}", req);
+    log_data.xff(&req.headers().get("x-forwarded-for"));
+    log_data.upstream(req.uri());
+    //////
+
+    // Forward request to
+    let mut res_backend = {
+      match timeout(self.globals.proxy_config.upstream_timeout, self.forwarder.request(req)).await {
+        Err(_) => {
+          return self.return_with_error_log(StatusCode::GATEWAY_TIMEOUT, &mut log_data);
+        }
+        Ok(x) => match x {
+          Ok(res) => res,
+          Err(e) => {
+            error!("Failed to get response from backend: {}", e);
+            return self.return_with_error_log(StatusCode::SERVICE_UNAVAILABLE, &mut log_data);
+          }
+        },
+      }
+    };
+
+    // Process reverse proxy context generated during the forwarding request generation.
+    #[cfg(feature = "sticky-cookie")]
+    if let Some(context_from_lb) = _context.context_lb {
+      let res_headers = res_backend.headers_mut();
+      if let Err(e) = set_sticky_cookie_lb_context(res_headers, &context_from_lb) {
+        error!("Failed to append context to the response given from backend: {}", e);
+        return self.return_with_error_log(StatusCode::BAD_GATEWAY, &mut log_data);
+      }
+    }
+
+    if res_backend.status() != StatusCode::SWITCHING_PROTOCOLS {
+      // Generate response to client
+      if self.generate_response_forwarded(&mut res_backend, backend).is_ok() {
+        log_data.status_code(&res_backend.status()).output();
+        return Ok(res_backend);
+      } else {
+        return self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data);
+      }
+    }
+
+    // Handle StatusCode::SWITCHING_PROTOCOLS in response
+    let upgrade_in_response = extract_upgrade(res_backend.headers());
+    if if let (Some(u_req), Some(u_res)) = (upgrade_in_request.as_ref(), upgrade_in_response.as_ref()) {
+      u_req.to_ascii_lowercase() == u_res.to_ascii_lowercase()
+    } else {
+      false
+    } {
+      if let Some(request_upgraded) = request_upgraded {
+        let Some(onupgrade) = res_backend.extensions_mut().remove::<hyper::upgrade::OnUpgrade>() else {
+          error!("Response does not have an upgrade extension");
+          return self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data);
+        };
+
+        self.globals.runtime_handle.spawn(async move {
+          let mut response_upgraded = onupgrade.await.map_err(|e| {
+            error!("Failed to upgrade response: {}", e);
+            RpxyError::Hyper(e)
+          })?;
+          let mut request_upgraded = request_upgraded.await.map_err(|e| {
+            error!("Failed to upgrade request: {}", e);
+            RpxyError::Hyper(e)
+          })?;
+          copy_bidirectional(&mut response_upgraded, &mut request_upgraded)
+            .await
+            .map_err(|e| {
+              error!("Coping between upgraded connections failed: {}", e);
+              RpxyError::Io(e)
+            })?;
+          Ok(()) as Result<()>
+        });
+        log_data.status_code(&res_backend.status()).output();
+        Ok(res_backend)
+      } else {
+        error!("Request does not have an upgrade extension");
+        self.return_with_error_log(StatusCode::BAD_REQUEST, &mut log_data)
+      }
+    } else {
+      error!(
+        "Backend tried to switch to protocol {:?} when {:?} was requested",
+        upgrade_in_response, upgrade_in_request
+      );
+      self.return_with_error_log(StatusCode::INTERNAL_SERVER_ERROR, &mut log_data)
+    }
+  }
+
+  ////////////////////////////////////////////////////
+  // Functions to generate messages
+
+  fn generate_response_forwarded<B>(&self, response: &mut Response<B>, chosen_backend: &Backend<U>) -> Result<()>
+  where
+    B: core::fmt::Debug,
+  {
+    let headers = response.headers_mut();
+    remove_connection_header(headers);
+    remove_hop_header(headers);
+    add_header_entry_overwrite_if_exist(headers, "server", env!("CARGO_PKG_NAME"))?;
+
+    #[cfg(feature = "http3")]
+    {
+      // TODO: Workaround for avoid h3 for client authentication
+      if self.globals.proxy_config.http3
+        && chosen_backend
+          .crypto_source
+          .as_ref()
+          .is_some_and(|v| !v.is_mutual_tls())
+      {
+        if let Some(port) = self.globals.proxy_config.https_port {
+          add_header_entry_overwrite_if_exist(
+            headers,
+            header::ALT_SVC.as_str(),
+            format!(
+              "h3=\":{}\"; ma={}, h3-29=\":{}\"; ma={}",
+              port, self.globals.proxy_config.h3_alt_svc_max_age, port, self.globals.proxy_config.h3_alt_svc_max_age
+            ),
+          )?;
+        }
+      } else {
+        // remove alt-svc to disallow requests via http3
+        headers.remove(header::ALT_SVC.as_str());
+      }
+    }
+    #[cfg(not(feature = "http3"))]
+    {
+      if let Some(port) = self.globals.proxy_config.https_port {
+        headers.remove(header::ALT_SVC.as_str());
+      }
+    }
+
+    Ok(())
+  }
+
+  #[allow(clippy::too_many_arguments)]
+  fn generate_request_forwarded<B>(
+    &self,
+    client_addr: &SocketAddr,
+    listen_addr: &SocketAddr,
+    req: &mut Request<B>,
+    upgrade: &Option<String>,
+    upstream_group: &UpstreamGroup,
+    tls_enabled: bool,
+  ) -> Result<HandlerContext> {
+    debug!("Generate request to be forwarded");
+
+    // Add te: trailer if contained in original request
+    let contains_te_trailers = {
+      if let Some(te) = req.headers().get(header::TE) {
+        te.as_bytes()
+          .split(|v| v == &b',' || v == &b' ')
+          .any(|x| x == "trailers".as_bytes())
+      } else {
+        false
+      }
+    };
+
+    let uri = req.uri().to_string();
+    let headers = req.headers_mut();
+    // delete headers specified in header.connection
+    remove_connection_header(headers);
+    // delete hop headers including header.connection
+    remove_hop_header(headers);
+    // X-Forwarded-For
+    add_forwarding_header(headers, client_addr, listen_addr, tls_enabled, &uri)?;
+
+    // Add te: trailer if te_trailer
+    if contains_te_trailers {
+      headers.insert(header::TE, HeaderValue::from_bytes("trailers".as_bytes()).unwrap());
+    }
+
+    // add "host" header of original server_name if not exist (default)
+    if req.headers().get(header::HOST).is_none() {
+      let org_host = req.uri().host().ok_or_else(|| anyhow!("Invalid request"))?.to_owned();
+      req
+        .headers_mut()
+        .insert(header::HOST, HeaderValue::from_str(&org_host)?);
+    };
+
+    /////////////////////////////////////////////
+    // Fix unique upstream destination since there could be multiple ones.
+    #[cfg(feature = "sticky-cookie")]
+    let (upstream_chosen_opt, context_from_lb) = {
+      let context_to_lb = if let crate::backend::LoadBalance::StickyRoundRobin(lb) = &upstream_group.lb {
+        takeout_sticky_cookie_lb_context(req.headers_mut(), &lb.sticky_config.name)?
+      } else {
+        None
+      };
+      upstream_group.get(&context_to_lb)
+    };
+    #[cfg(not(feature = "sticky-cookie"))]
+    let (upstream_chosen_opt, _) = upstream_group.get(&None);
+
+    let upstream_chosen = upstream_chosen_opt.ok_or_else(|| anyhow!("Failed to get upstream"))?;
+    let context = HandlerContext {
+      #[cfg(feature = "sticky-cookie")]
+      context_lb: context_from_lb,
+      #[cfg(not(feature = "sticky-cookie"))]
+      context_lb: None,
+    };
+    /////////////////////////////////////////////
+
+    // apply upstream-specific headers given in upstream_option
+    let headers = req.headers_mut();
+    apply_upstream_options_to_header(headers, client_addr, upstream_group, &upstream_chosen.uri)?;
+
+    // update uri in request
+    if !(upstream_chosen.uri.authority().is_some() && upstream_chosen.uri.scheme().is_some()) {
+      return Err(RpxyError::Handler("Upstream uri `scheme` and `authority` is broken"));
+    };
+    let new_uri = Uri::builder()
+      .scheme(upstream_chosen.uri.scheme().unwrap().as_str())
+      .authority(upstream_chosen.uri.authority().unwrap().as_str());
+    let org_pq = match req.uri().path_and_query() {
+      Some(pq) => pq.to_string(),
+      None => "/".to_string(),
+    }
+    .into_bytes();
+
+    // replace some parts of path if opt_replace_path is enabled for chosen upstream
+    let new_pq = match &upstream_group.replace_path {
+      Some(new_path) => {
+        let matched_path: &[u8] = upstream_group.path.as_ref();
+        if matched_path.is_empty() || org_pq.len() < matched_path.len() {
+          return Err(RpxyError::Handler("Upstream uri `path and query` is broken"));
+        };
+        let mut new_pq = Vec::<u8>::with_capacity(org_pq.len() - matched_path.len() + new_path.len());
+        new_pq.extend_from_slice(new_path.as_ref());
+        new_pq.extend_from_slice(&org_pq[matched_path.len()..]);
+        new_pq
+      }
+      None => org_pq,
+    };
+    *req.uri_mut() = new_uri.path_and_query(new_pq).build()?;
+
+    // upgrade
+    if let Some(v) = upgrade {
+      req.headers_mut().insert(header::UPGRADE, v.parse()?);
+      req
+        .headers_mut()
+        .insert(header::CONNECTION, HeaderValue::from_str("upgrade")?);
+    }
+
+    // If not specified (force_httpXX_upstream) and https, version is preserved except for http/3
+    apply_upstream_options_to_request_line(req, upstream_group)?;
+    // Maybe workaround: Change version to http/1.1 when destination scheme is http
+    if req.version() != Version::HTTP_11 && upstream_chosen.uri.scheme() == Some(&Scheme::HTTP) {
+      *req.version_mut() = Version::HTTP_11;
+    } else if req.version() == Version::HTTP_3 {
+      debug!("HTTP/3 is currently unsupported for request to upstream. Use HTTP/2.");
+      *req.version_mut() = Version::HTTP_2;
+    }
+
+    Ok(context)
+  }
+}

rpxy-lib/src/handler/mod.rs

@@ -0,0 +1,17 @@
+mod handler_main;
+mod utils_headers;
+mod utils_request;
+mod utils_synth_response;
+
+#[cfg(feature = "sticky-cookie")]
+use crate::backend::LbContext;
+pub use handler_main::{HttpMessageHandler, HttpMessageHandlerBuilder, HttpMessageHandlerBuilderError};
+
+#[allow(dead_code)]
+#[derive(Debug)]
+struct HandlerContext {
+  #[cfg(feature = "sticky-cookie")]
+  context_lb: Option<LbContext>,
+  #[cfg(not(feature = "sticky-cookie"))]
+  context_lb: Option<()>,
+}

rpxy-lib/src/handler/utils_headers.rs

@@ -0,0 +1,269 @@
+#[cfg(feature = "sticky-cookie")]
+use crate::backend::{LbContext, StickyCookie, StickyCookieValue};
+use crate::backend::{UpstreamGroup, UpstreamOption};
+
+use crate::{error::*, log::*, utils::*};
+use bytes::BufMut;
+use hyper::{
+  header::{self, HeaderMap, HeaderName, HeaderValue},
+  Uri,
+};
+use std::net::SocketAddr;
+
+////////////////////////////////////////////////////
+// Functions to manipulate headers
+
+#[cfg(feature = "sticky-cookie")]
+/// Take sticky cookie header value from request header,
+/// and returns LbContext to be forwarded to LB if exist and if needed.
+/// Removing sticky cookie is needed and it must not be passed to the upstream.
+pub(super) fn takeout_sticky_cookie_lb_context(
+  headers: &mut HeaderMap,
+  expected_cookie_name: &str,
+) -> Result<Option<LbContext>> {
+  let mut headers_clone = headers.clone();
+
+  match headers_clone.entry(hyper::header::COOKIE) {
+    header::Entry::Vacant(_) => Ok(None),
+    header::Entry::Occupied(entry) => {
+      let cookies_iter = entry
+        .iter()
+        .flat_map(|v| v.to_str().unwrap_or("").split(';').map(|v| v.trim()));
+      let (sticky_cookies, without_sticky_cookies): (Vec<_>, Vec<_>) = cookies_iter
+        .into_iter()
+        .partition(|v| v.starts_with(expected_cookie_name));
+      if sticky_cookies.is_empty() {
+        return Ok(None);
+      }
+      if sticky_cookies.len() > 1 {
+        error!("Multiple sticky cookie values in request");
+        return Err(RpxyError::Other(anyhow!(
+          "Invalid cookie: Multiple sticky cookie values"
+        )));
+      }
+      let cookies_passed_to_upstream = without_sticky_cookies.join("; ");
+      let cookie_passed_to_lb = sticky_cookies.first().unwrap();
+      headers.remove(hyper::header::COOKIE);
+      headers.insert(hyper::header::COOKIE, cookies_passed_to_upstream.parse()?);
+
+      let sticky_cookie = StickyCookie {
+        value: StickyCookieValue::try_from(cookie_passed_to_lb, expected_cookie_name)?,
+        info: None,
+      };
+      Ok(Some(LbContext { sticky_cookie }))
+    }
+  }
+}
+
+#[cfg(feature = "sticky-cookie")]
+/// Set-Cookie if LB Sticky is enabled and if cookie is newly created/updated.
+/// Set-Cookie response header could be in multiple lines.
+/// https://developer.mozilla.org/ja/docs/Web/HTTP/Headers/Set-Cookie
+pub(super) fn set_sticky_cookie_lb_context(headers: &mut HeaderMap, context_from_lb: &LbContext) -> Result<()> {
+  let sticky_cookie_string: String = context_from_lb.sticky_cookie.clone().try_into()?;
+  let new_header_val: HeaderValue = sticky_cookie_string.parse()?;
+  let expected_cookie_name = &context_from_lb.sticky_cookie.value.name;
+  match headers.entry(hyper::header::SET_COOKIE) {
+    header::Entry::Vacant(entry) => {
+      entry.insert(new_header_val);
+    }
+    header::Entry::Occupied(mut entry) => {
+      let mut flag = false;
+      for e in entry.iter_mut() {
+        if e.to_str().unwrap_or("").starts_with(expected_cookie_name) {
+          *e = new_header_val.clone();
+          flag = true;
+        }
+      }
+      if !flag {
+        entry.append(new_header_val);
+      }
+    }
+  };
+  Ok(())
+}
+
+pub(super) fn apply_upstream_options_to_header(
+  headers: &mut HeaderMap,
+  _client_addr: &SocketAddr,
+  upstream: &UpstreamGroup,
+  upstream_base_uri: &Uri,
+) -> Result<()> {
+  for opt in upstream.opts.iter() {
+    match opt {
+      UpstreamOption::OverrideHost => {
+        // overwrite HOST value with upstream hostname (like 192.168.xx.x seen from rpxy)
+        let upstream_host = upstream_base_uri
+          .host()
+          .ok_or_else(|| anyhow!("No hostname is given in override_host option"))?;
+        headers
+          .insert(header::HOST, HeaderValue::from_str(upstream_host)?)
+          .ok_or_else(|| anyhow!("Failed to insert host header in override_host option"))?;
+      }
+      UpstreamOption::UpgradeInsecureRequests => {
+        // add upgrade-insecure-requests in request header if not exist
+        headers
+          .entry(header::UPGRADE_INSECURE_REQUESTS)
+          .or_insert(HeaderValue::from_bytes(&[b'1']).unwrap());
+      }
+      _ => (),
+    }
+  }
+
+  Ok(())
+}
+
+// https://datatracker.ietf.org/doc/html/rfc9110
+pub(super) fn append_header_entry_with_comma(headers: &mut HeaderMap, key: &str, value: &str) -> Result<()> {
+  match headers.entry(HeaderName::from_bytes(key.as_bytes())?) {
+    header::Entry::Vacant(entry) => {
+      entry.insert(value.parse::<HeaderValue>()?);
+    }
+    header::Entry::Occupied(mut entry) => {
+      // entry.append(value.parse::<HeaderValue>()?);
+      let mut new_value = Vec::<u8>::with_capacity(entry.get().as_bytes().len() + 2 + value.len());
+      new_value.put_slice(entry.get().as_bytes());
+      new_value.put_slice(&[b',', b' ']);
+      new_value.put_slice(value.as_bytes());
+      entry.insert(HeaderValue::from_bytes(&new_value)?);
+    }
+  }
+
+  Ok(())
+}
+
+pub(super) fn add_header_entry_if_not_exist(
+  headers: &mut HeaderMap,
+  key: impl Into<std::borrow::Cow<'static, str>>,
+  value: impl Into<std::borrow::Cow<'static, str>>,
+) -> Result<()> {
+  match headers.entry(HeaderName::from_bytes(key.into().as_bytes())?) {
+    header::Entry::Vacant(entry) => {
+      entry.insert(value.into().parse::<HeaderValue>()?);
+    }
+    header::Entry::Occupied(_) => (),
+  };
+
+  Ok(())
+}
+
+pub(super) fn add_header_entry_overwrite_if_exist(
+  headers: &mut HeaderMap,
+  key: impl Into<std::borrow::Cow<'static, str>>,
+  value: impl Into<std::borrow::Cow<'static, str>>,
+) -> Result<()> {
+  match headers.entry(HeaderName::from_bytes(key.into().as_bytes())?) {
+    header::Entry::Vacant(entry) => {
+      entry.insert(value.into().parse::<HeaderValue>()?);
+    }
+    header::Entry::Occupied(mut entry) => {
+      entry.insert(HeaderValue::from_bytes(value.into().as_bytes())?);
+    }
+  }
+
+  Ok(())
+}
+
+pub(super) fn make_cookie_single_line(headers: &mut HeaderMap) -> Result<()> {
+  // Sometimes violates RFC6265 (for http/1.1).
+  // https://www.rfc-editor.org/rfc/rfc6265#section-5.4
+  // This is allowed in RFC7540 (for http/2).
+  // https://stackoverflow.com/questions/4843556/in-http-specification-what-is-the-string-that-separates-cookies
+  let cookies = headers
+    .iter()
+    .filter(|(k, _)| **k == hyper::header::COOKIE)
+    .map(|(_, v)| v.to_str().unwrap_or(""))
+    .collect::<Vec<_>>()
+    .join("; ");
+  if !cookies.is_empty() {
+    headers.remove(hyper::header::COOKIE);
+    headers.insert(hyper::header::COOKIE, HeaderValue::from_bytes(cookies.as_bytes())?);
+  }
+  Ok(())
+}
+
+pub(super) fn add_forwarding_header(
+  headers: &mut HeaderMap,
+  client_addr: &SocketAddr,
+  listen_addr: &SocketAddr,
+  tls: bool,
+  uri_str: &str,
+) -> Result<()> {
+  // default process
+  // optional process defined by upstream_option is applied in fn apply_upstream_options
+  let canonical_client_addr = client_addr.to_canonical().ip().to_string();
+  append_header_entry_with_comma(headers, "x-forwarded-for", &canonical_client_addr)?;
+
+  // Single line cookie header
+  // TODO: This should be only for HTTP/1.1. For 2+, this can be multi-lined.
+  make_cookie_single_line(headers)?;
+
+  /////////// As Nginx
+  // If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the
+  // scheme used to connect to this server
+  add_header_entry_if_not_exist(headers, "x-forwarded-proto", if tls { "https" } else { "http" })?;
+  // If we receive X-Forwarded-Port, pass it through; otherwise, pass along the
+  // server port the client connected to
+  add_header_entry_if_not_exist(headers, "x-forwarded-port", listen_addr.port().to_string())?;
+
+  /////////// As Nginx-Proxy
+  // x-real-ip
+  add_header_entry_overwrite_if_exist(headers, "x-real-ip", canonical_client_addr)?;
+  // x-forwarded-ssl
+  add_header_entry_overwrite_if_exist(headers, "x-forwarded-ssl", if tls { "on" } else { "off" })?;
+  // x-original-uri
+  add_header_entry_overwrite_if_exist(headers, "x-original-uri", uri_str.to_string())?;
+  // proxy
+  add_header_entry_overwrite_if_exist(headers, "proxy", "")?;
+
+  Ok(())
+}
+
+pub(super) fn remove_connection_header(headers: &mut HeaderMap) {
+  if let Some(values) = headers.get(header::CONNECTION) {
+    if let Ok(v) = values.clone().to_str() {
+      for m in v.split(',') {
+        if !m.is_empty() {
+          headers.remove(m.trim());
+        }
+      }
+    }
+  }
+}
+
+const HOP_HEADERS: &[&str] = &[
+  "connection",
+  "te",
+  "trailer",
+  "keep-alive",
+  "proxy-connection",
+  "proxy-authenticate",
+  "proxy-authorization",
+  "transfer-encoding",
+  "upgrade",
+];
+
+pub(super) fn remove_hop_header(headers: &mut HeaderMap) {
+  HOP_HEADERS.iter().for_each(|key| {
+    headers.remove(*key);
+  });
+}
+
+pub(super) fn extract_upgrade(headers: &HeaderMap) -> Option<String> {
+  if let Some(c) = headers.get(header::CONNECTION) {
+    if c
+      .to_str()
+      .unwrap_or("")
+      .split(',')
+      .any(|w| w.trim().to_ascii_lowercase() == header::UPGRADE.as_str().to_ascii_lowercase())
+    {
+      if let Some(u) = headers.get(header::UPGRADE) {
+        if let Ok(m) = u.to_str() {
+          debug!("Upgrade in request header: {}", m);
+          return Some(m.to_owned());
+        }
+      }
+    }
+  }
+  None
+}

rpxy-lib/src/handler/utils_request.rs

@@ -0,0 +1,57 @@
+use crate::{
+  backend::{UpstreamGroup, UpstreamOption},
+  error::*,
+};
+use hyper::{header, Request};
+
+////////////////////////////////////////////////////
+// Functions to manipulate request line
+
+pub(super) fn apply_upstream_options_to_request_line<B>(req: &mut Request<B>, upstream: &UpstreamGroup) -> Result<()> {
+  for opt in upstream.opts.iter() {
+    match opt {
+      UpstreamOption::ConvertHttpsTo11 => *req.version_mut() = hyper::Version::HTTP_11,
+      UpstreamOption::ConvertHttpsTo2 => *req.version_mut() = hyper::Version::HTTP_2,
+      _ => (),
+    }
+  }
+
+  Ok(())
+}
+
+pub trait ParseHost {
+  fn parse_host(&self) -> Result<&[u8]>;
+}
+impl<B> ParseHost for Request<B> {
+  fn parse_host(&self) -> Result<&[u8]> {
+    let headers_host = self.headers().get(header::HOST);
+    let uri_host = self.uri().host();
+    // let uri_port = self.uri().port_u16();
+
+    if !(!(headers_host.is_none() && uri_host.is_none())) {
+      return Err(RpxyError::Request("No host in request header"));
+    }
+
+    // prioritize server_name in uri
+    uri_host.map_or_else(
+      || {
+        let m = headers_host.unwrap().as_bytes();
+        if m.starts_with(&[b'[']) {
+          // v6 address with bracket case. if port is specified, always it is in this case.
+          let mut iter = m.split(|ptr| ptr == &b'[' || ptr == &b']');
+          iter.next().ok_or(RpxyError::Request("Invalid Host"))?; // first item is always blank
+          iter.next().ok_or(RpxyError::Request("Invalid Host"))
+        } else if m.len() - m.split(|v| v == &b':').fold(0, |acc, s| acc + s.len()) >= 2 {
+          // v6 address case, if 2 or more ':' is contained
+          Ok(m)
+        } else {
+          // v4 address or hostname
+          m.split(|colon| colon == &b':')
+            .next()
+            .ok_or(RpxyError::Request("Invalid Host"))
+        }
+      },
+      |v| Ok(v.as_bytes()),
+    )
+  }
+}

rpxy-lib/src/handler/utils_synth_response.rs

@@ -0,0 +1,33 @@
+// Highly motivated by https://github.com/felipenoris/hyper-reverse-proxy
+use crate::error::*;
+use hyper::{Body, Request, Response, StatusCode, Uri};
+
+////////////////////////////////////////////////////
+// Functions to create response (error or redirect)
+
+pub(super) fn http_error(status_code: StatusCode) -> Result<Response<Body>> {
+  let response = Response::builder().status(status_code).body(Body::empty())?;
+  Ok(response)
+}
+
+pub(super) fn secure_redirection<B>(
+  server_name: &str,
+  tls_port: Option<u16>,
+  req: &Request<B>,
+) -> Result<Response<Body>> {
+  let pq = match req.uri().path_and_query() {
+    Some(x) => x.as_str(),
+    _ => "",
+  };
+  let new_uri = Uri::builder().scheme("https").path_and_query(pq);
+  let dest_uri = match tls_port {
+    Some(443) | None => new_uri.authority(server_name),
+    Some(p) => new_uri.authority(format!("{server_name}:{p}")),
+  }
+  .build()?;
+  let response = Response::builder()
+    .status(StatusCode::MOVED_PERMANENTLY)
+    .header("Location", dest_uri.to_string())
+    .body(Body::empty())?;
+  Ok(response)
+}

rpxy-lib/src/lib.rs

@@ -0,0 +1,72 @@
+mod backend;
+mod certs;
+mod constants;
+mod error;
+mod globals;
+mod handler;
+mod log;
+mod proxy;
+mod utils;
+
+use crate::{error::*, handler::HttpMessageHandlerBuilder, log::*, proxy::ProxyBuilder};
+use futures::future::select_all;
+use hyper::Client;
+// use hyper_trust_dns::TrustDnsResolver;
+use std::sync::Arc;
+
+pub use crate::{
+  backend::{
+    Backend, BackendBuilder, Backends, ReverseProxy, Upstream, UpstreamGroup, UpstreamGroupBuilder, UpstreamOption,
+  },
+  certs::{CertsAndKeys, CryptoSource},
+  globals::{Globals, ProxyConfig}, // TODO: BackendConfigに変える
+  utils::{BytesName, PathNameBytesExp},
+};
+pub mod reexports {
+  pub use hyper::Uri;
+  pub use rustls::{Certificate, PrivateKey};
+}
+
+/// Entrypoint that creates and spawns tasks of reverse proxy services
+pub async fn entrypoint<T>(globals: Arc<Globals<T>>) -> Result<()>
+where
+  T: CryptoSource + Clone + Send + Sync + 'static,
+{
+  // let connector = TrustDnsResolver::default().into_rustls_webpki_https_connector();
+  let connector = hyper_rustls::HttpsConnectorBuilder::new()
+    .with_webpki_roots()
+    .https_or_http()
+    .enable_http1()
+    .enable_http2()
+    .build();
+
+  let msg_handler = HttpMessageHandlerBuilder::default()
+    .forwarder(Arc::new(Client::builder().build::<_, hyper::Body>(connector)))
+    .globals(globals.clone())
+    .build()?;
+
+  let addresses = globals.proxy_config.listen_sockets.clone();
+  let futures = select_all(addresses.into_iter().map(|addr| {
+    let mut tls_enabled = false;
+    if let Some(https_port) = globals.proxy_config.https_port {
+      tls_enabled = https_port == addr.port()
+    }
+
+    let proxy = ProxyBuilder::default()
+      .globals(globals.clone())
+      .listening_on(addr)
+      .tls_enabled(tls_enabled)
+      .msg_handler(msg_handler.clone())
+      .build()
+      .unwrap();
+
+    globals.runtime_handle.spawn(proxy.start())
+  }));
+
+  // wait for all future
+  if let (Ok(_), _, _) = futures.await {
+    error!("Some proxy services are down");
+  };
+
+  Ok(())
+}

rpxy-lib/src/log.rs

@@ -0,0 +1,97 @@
+use crate::utils::ToCanonical;
+use std::net::SocketAddr;
+pub use tracing::{debug, error, info, warn};
+
+#[derive(Debug, Clone)]
+pub struct MessageLog {
+  // pub tls_server_name: String,
+  pub client_addr: String,
+  pub method: String,
+  pub host: String,
+  pub p_and_q: String,
+  pub version: hyper::Version,
+  pub uri_scheme: String,
+  pub uri_host: String,
+  pub ua: String,
+  pub xff: String,
+  pub status: String,
+  pub upstream: String,
+}
+
+impl<T> From<&hyper::Request<T>> for MessageLog {
+  fn from(req: &hyper::Request<T>) -> Self {
+    let header_mapper = |v: hyper::header::HeaderName| {
+      req
+        .headers()
+        .get(v)
+        .map_or_else(|| "", |s| s.to_str().unwrap_or(""))
+        .to_string()
+    };
+    Self {
+      // tls_server_name: "".to_string(),
+      client_addr: "".to_string(),
+      method: req.method().to_string(),
+      host: header_mapper(hyper::header::HOST),
+      p_and_q: req
+        .uri()
+        .path_and_query()
+        .map_or_else(|| "", |v| v.as_str())
+        .to_string(),
+      version: req.version(),
+      uri_scheme: req.uri().scheme_str().unwrap_or("").to_string(),
+      uri_host: req.uri().host().unwrap_or("").to_string(),
+      ua: header_mapper(hyper::header::USER_AGENT),
+      xff: header_mapper(hyper::header::HeaderName::from_static("x-forwarded-for")),
+      status: "".to_string(),
+      upstream: "".to_string(),
+    }
+  }
+}
+
+impl MessageLog {
+  pub fn client_addr(&mut self, client_addr: &SocketAddr) -> &mut Self {
+    self.client_addr = client_addr.to_canonical().to_string();
+    self
+  }
+  // pub fn tls_server_name(&mut self, tls_server_name: &str) -> &mut Self {
+  //   self.tls_server_name = tls_server_name.to_string();
+  //   self
+  // }
+  pub fn status_code(&mut self, status_code: &hyper::StatusCode) -> &mut Self {
+    self.status = status_code.to_string();
+    self
+  }
+  pub fn xff(&mut self, xff: &Option<&hyper::header::HeaderValue>) -> &mut Self {
+    self.xff = xff.map_or_else(|| "", |v| v.to_str().unwrap_or("")).to_string();
+    self
+  }
+  pub fn upstream(&mut self, upstream: &hyper::Uri) -> &mut Self {
+    self.upstream = upstream.to_string();
+    self
+  }
+
+  pub fn output(&self) {
+    info!(
+      "{} <- {} -- {} {} {:?} -- {} -- {} \"{}\", \"{}\" \"{}\"",
+      if !self.host.is_empty() {
+        self.host.as_str()
+      } else {
+        self.uri_host.as_str()
+      },
+      self.client_addr,
+      self.method,
+      self.p_and_q,
+      self.version,
+      self.status,
+      if !self.uri_scheme.is_empty() && !self.uri_host.is_empty() {
+        format!("{}://{}", self.uri_scheme, self.uri_host)
+      } else {
+        "".to_string()
+      },
+      self.ua,
+      self.xff,
+      self.upstream,
+      // self.tls_server_name
+    );
+  }
+}

rpxy-lib/src/proxy/crypto_service.rs

@@ -0,0 +1,181 @@
+use crate::{
+  certs::{CertsAndKeys, CryptoSource},
+  globals::Globals,
+  log::*,
+  utils::ServerNameBytesExp,
+};
+use async_trait::async_trait;
+use hot_reload::*;
+use rustc_hash::FxHashMap as HashMap;
+use rustls::{server::ResolvesServerCertUsingSni, sign::CertifiedKey, RootCertStore, ServerConfig};
+use std::sync::Arc;
+
+#[derive(Clone)]
+/// Reloader service for certificates and keys for TLS
+pub struct CryptoReloader<T>
+where
+  T: CryptoSource,
+{
+  globals: Arc<Globals<T>>,
+}
+
+pub type SniServerCryptoMap = HashMap<ServerNameBytesExp, Arc<ServerConfig>>;
+pub struct ServerCrypto {
+  // For Quic/HTTP3, only servers with no client authentication
+  pub inner_global_no_client_auth: Arc<ServerConfig>,
+  // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers
+  pub inner_local_map: Arc<SniServerCryptoMap>,
+}
+
+/// Reloader target for the certificate reloader service
+#[derive(Debug, PartialEq, Eq, Clone, Default)]
+pub struct ServerCryptoBase {
+  inner: HashMap<ServerNameBytesExp, CertsAndKeys>,
+}
+
+#[async_trait]
+impl<T> Reload<ServerCryptoBase> for CryptoReloader<T>
+where
+  T: CryptoSource + Sync + Send,
+{
+  type Source = Arc<Globals<T>>;
+  async fn new(source: &Self::Source) -> Result<Self, ReloaderError<ServerCryptoBase>> {
+    Ok(Self {
+      globals: source.clone(),
+    })
+  }
+
+  async fn reload(&self) -> Result<Option<ServerCryptoBase>, ReloaderError<ServerCryptoBase>> {
+    let mut certs_and_keys_map = ServerCryptoBase::default();
+
+    for (server_name_bytes_exp, backend) in self.globals.backends.apps.iter() {
+      if let Some(crypto_source) = &backend.crypto_source {
+        let certs_and_keys = crypto_source
+          .read()
+          .await
+          .map_err(|_e| ReloaderError::<ServerCryptoBase>::Reload("Failed to reload cert, key or ca cert"))?;
+        certs_and_keys_map
+          .inner
+          .insert(server_name_bytes_exp.to_owned(), certs_and_keys);
+      }
+    }
+
+    Ok(Some(certs_and_keys_map))
+  }
+}
+
+impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
+  type Error = anyhow::Error;
+
+  fn try_into(self) -> Result<Arc<ServerCrypto>, Self::Error> {
+    let mut resolver_global = ResolvesServerCertUsingSni::new();
+    let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default();
+
+    for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
+      let server_name: String = server_name_bytes_exp.try_into()?;
+
+      // Parse server certificates and private keys
+      let Ok(certified_key): Result<CertifiedKey, _> = certs_and_keys.parse_server_certs_and_keys() else {
+        warn!("Failed to add certificate for {}", server_name);
+        continue;
+      };
+
+      let mut resolver_local = ResolvesServerCertUsingSni::new();
+      let mut client_ca_roots_local = RootCertStore::empty();
+
+      // add server certificate and key
+      if let Err(e) = resolver_local.add(server_name.as_str(), certified_key.to_owned()) {
+        error!(
+          "{}: Failed to read some certificates and keys {}",
+          server_name.as_str(),
+          e
+        )
+      }
+
+      // add client certificate if specified
+      if certs_and_keys.client_ca_certs.is_none() {
+        // aggregated server config for no client auth server for http3
+        if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
+          error!(
+            "{}: Failed to read some certificates and keys {}",
+            server_name.as_str(),
+            e
+          )
+        }
+      } else {
+        // add client certificate if specified
+        match certs_and_keys.parse_client_ca_certs() {
+          Ok((owned_trust_anchors, _subject_key_ids)) => {
+            client_ca_roots_local.add_server_trust_anchors(owned_trust_anchors.into_iter());
+          }
+          Err(e) => {
+            warn!(
+              "Failed to add client CA certificate for {}: {}",
+              server_name.as_str(),
+              e
+            );
+          }
+        }
+      }
+
+      let mut server_config_local = if client_ca_roots_local.is_empty() {
+        // with no client auth, enable http1.1 -- 3
+        #[cfg(not(feature = "http3"))]
+        {
+          ServerConfig::builder()
+            .with_safe_defaults()
+            .with_no_client_auth()
+            .with_cert_resolver(Arc::new(resolver_local))
+        }
+        #[cfg(feature = "http3")]
+        {
+          let mut sc = ServerConfig::builder()
+            .with_safe_defaults()
+            .with_no_client_auth()
+            .with_cert_resolver(Arc::new(resolver_local));
+          sc.alpn_protocols = vec![b"h3".to_vec(), b"hq-29".to_vec()]; // TODO: remove hq-29 later?
+          sc
+        }
+      } else {
+        // with client auth, enable only http1.1 and 2
+        // let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots);
+        let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots_local);
+        ServerConfig::builder()
+          .with_safe_defaults()
+          .with_client_cert_verifier(Arc::new(client_certs_verifier))
+          .with_cert_resolver(Arc::new(resolver_local))
+      };
+      server_config_local.alpn_protocols.push(b"h2".to_vec());
+      server_config_local.alpn_protocols.push(b"http/1.1".to_vec());
+
+      server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local));
+    }
+
+    //////////////
+    let mut server_crypto_global = ServerConfig::builder()
+      .with_safe_defaults()
+      .with_no_client_auth()
+      .with_cert_resolver(Arc::new(resolver_global));
+
+    //////////////////////////////
+
+    #[cfg(feature = "http3")]
+    {
+      server_crypto_global.alpn_protocols = vec![
+        b"h3".to_vec(),
+        b"hq-29".to_vec(), // TODO: remove later?
+        b"h2".to_vec(),
+        b"http/1.1".to_vec(),
+      ];
+    }
+    #[cfg(not(feature = "http3"))]
+    {
+      server_crypto_global.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
+    }
+
+    Ok(Arc::new(ServerCrypto {
+      inner_global_no_client_auth: Arc::new(server_crypto_global),
+      inner_local_map: Arc::new(server_crypto_local_map),
+    }))
+  }
+}

rpxy-lib/src/proxy/mod.rs

@@ -0,0 +1,8 @@
+mod crypto_service;
+mod proxy_client_cert;
+#[cfg(feature = "http3")]
+mod proxy_h3;
+mod proxy_main;
+mod proxy_tls;
+
+pub use proxy_main::{Proxy, ProxyBuilder, ProxyBuilderError};

rpxy-lib/src/proxy/proxy_client_cert.rs

@@ -0,0 +1,47 @@
+use crate::{error::*, log::*};
+use rustc_hash::FxHashSet as HashSet;
+use rustls::Certificate;
+use x509_parser::extensions::ParsedExtension;
+use x509_parser::prelude::*;
+
+#[allow(dead_code)]
+// TODO: consider move this function to the layer of handle_request (L7) to return 403
+pub(super) fn check_client_authentication(
+  client_certs: Option<&[Certificate]>,
+  client_ca_keyids_set_for_sni: Option<&HashSet<Vec<u8>>>,
+) -> std::result::Result<(), ClientCertsError> {
+  let Some(client_ca_keyids_set) = client_ca_keyids_set_for_sni else {
+    // No client cert settings for given server name
+    return Ok(());
+  };
+
+  let Some(client_certs) = client_certs else {
+    error!("Client certificate is needed for given server name");
+    return Err(ClientCertsError::ClientCertRequired(
+      "Client certificate is needed for given server name".to_string(),
+    ));
+  };
+  debug!("Incoming TLS client is (temporarily) authenticated via client cert");
+
+  // Check client certificate key ids
+  let mut client_certs_parsed_iter = client_certs.iter().filter_map(|d| parse_x509_certificate(&d.0).ok());
+  let match_server_crypto_and_client_cert = client_certs_parsed_iter.any(|c| {
+    let mut filtered = c.1.iter_extensions().filter_map(|e| {
+      if let ParsedExtension::AuthorityKeyIdentifier(key_id) = e.parsed_extension() {
+        key_id.key_identifier.as_ref()
+      } else {
+        None
+      }
+    });
+    filtered.any(|id| client_ca_keyids_set.contains(id.0))
+  });
+
+  if !match_server_crypto_and_client_cert {
+    error!("Inconsistent client certificate was provided for SNI");
+    return Err(ClientCertsError::InconsistentClientCert(
+      "Inconsistent client certificate was provided for SNI".to_string(),
+    ));
+  }
+
+  Ok(())
+}

rpxy-lib/src/proxy/proxy_h3.rs

@@ -0,0 +1,163 @@
+use super::Proxy;
+use crate::{certs::CryptoSource, error::*, log::*, utils::ServerNameBytesExp};
+use bytes::{Buf, Bytes};
+use h3::{quic::BidiStream, server::RequestStream};
+use hyper::{client::connect::Connect, Body, Request, Response};
+use std::net::SocketAddr;
+use tokio::time::{timeout, Duration};
+
+impl<T, U> Proxy<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone + Sync + Send + 'static,
+{
+  pub(super) async fn connection_serve_h3(
+    self,
+    conn: quinn::Connecting,
+    tls_server_name: ServerNameBytesExp,
+  ) -> Result<()> {
+    let client_addr = conn.remote_address();
+
+    match conn.await {
+      Ok(new_conn) => {
+        let mut h3_conn = h3::server::Connection::<_, bytes::Bytes>::new(h3_quinn::Connection::new(new_conn)).await?;
+        info!(
+          "QUIC/HTTP3 connection established from {:?} {:?}",
+          client_addr, tls_server_name
+        );
+        // TODO: Is here enough to fetch server_name from NewConnection?
+        // to avoid deep nested call from listener_service_h3
+        loop {
+          // this routine follows hyperium/h3 examples https://github.com/hyperium/h3/blob/master/examples/server.rs
+          match h3_conn.accept().await {
+            Ok(None) => {
+              break;
+            }
+            Err(e) => {
+              warn!("HTTP/3 error on accept incoming connection: {}", e);
+              match e.get_error_level() {
+                h3::error::ErrorLevel::ConnectionError => break,
+                h3::error::ErrorLevel::StreamError => continue,
+              }
+            }
+            Ok(Some((req, stream))) => {
+              // We consider the connection count separately from the stream count.
+              // Max clients for h1/h2 = max 'stream' for h3.
+              let request_count = self.globals.request_count.clone();
+              if request_count.increment() > self.globals.proxy_config.max_clients {
+                request_count.decrement();
+                h3_conn.shutdown(0).await?;
+                break;
+              }
+              debug!("Request incoming: current # {}", request_count.current());
+
+              let self_inner = self.clone();
+              let tls_server_name_inner = tls_server_name.clone();
+              self.globals.runtime_handle.spawn(async move {
+                if let Err(e) = timeout(
+                  self_inner.globals.proxy_config.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
+                  self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner),
+                )
+                .await
+                {
+                  error!("HTTP/3 failed to process stream: {}", e);
+                }
+                request_count.decrement();
+                debug!("Request processed: current # {}", request_count.current());
+              });
+            }
+          }
+        }
+      }
+      Err(err) => {
+        warn!("QUIC accepting connection failed: {:?}", err);
+        return Err(RpxyError::QuicConn(err));
+      }
+    }
+
+    Ok(())
+  }
+
+  async fn stream_serve_h3<S>(
+    self,
+    req: Request<()>,
+    stream: RequestStream<S, Bytes>,
+    client_addr: SocketAddr,
+    tls_server_name: ServerNameBytesExp,
+  ) -> Result<()>
+  where
+    S: BidiStream<Bytes> + Send + 'static,
+    <S as BidiStream<Bytes>>::RecvStream: Send,
+  {
+    let (req_parts, _) = req.into_parts();
+    // split stream and async body handling
+    let (mut send_stream, mut recv_stream) = stream.split();
+
+    // generate streamed body with trailers using channel
+    let (body_sender, req_body) = Body::channel();
+
+    // Buffering and sending body through channel for protocol conversion like h3 -> h2/http1.1
+    // The underling buffering, i.e., buffer given by the API recv_data.await?, is handled by quinn.
+    let max_body_size = self.globals.proxy_config.h3_request_max_body_size;
+    self.globals.runtime_handle.spawn(async move {
+      let mut sender = body_sender;
+      let mut size = 0usize;
+      while let Some(mut body) = recv_stream.recv_data().await? {
+        debug!("HTTP/3 incoming request body: remaining {}", body.remaining());
+        size += body.remaining();
+        if size > max_body_size {
+          error!(
+            "Exceeds max request body size for HTTP/3: received {}, maximum_allowd {}",
+            size, max_body_size
+          );
+          return Err(RpxyError::Proxy("Exceeds max request body size for HTTP/3".to_string()));
+        }
+        // create stream body to save memory, shallow copy (increment of ref-count) to Bytes using copy_to_bytes
+        sender.send_data(body.copy_to_bytes(body.remaining())).await?;
+      }
+
+      // trailers: use inner for work around. (directly get trailer)
+      let trailers = recv_stream.as_mut().recv_trailers().await?;
+      if trailers.is_some() {
+        debug!("HTTP/3 incoming request trailers");
+        sender.send_trailers(trailers.unwrap()).await?;
+      }
+      Ok(())
+    });
+
+    let new_req: Request<Body> = Request::from_parts(req_parts, req_body);
+    let res = self
+      .msg_handler
+      .clone()
+      .handle_request(
+        new_req,
+        client_addr,
+        self.listening_on,
+        self.tls_enabled,
+        Some(tls_server_name),
+      )
+      .await?;
+
+    let (new_res_parts, new_body) = res.into_parts();
+    let new_res = Response::from_parts(new_res_parts, ());
+
+    match send_stream.send_response(new_res).await {
+      Ok(_) => {
+        debug!("HTTP/3 response to connection successful");
+        // aggregate body without copying
+        let mut body_data = hyper::body::aggregate(new_body).await?;
+
+        // create stream body to save memory, shallow copy (increment of ref-count) to Bytes using copy_to_bytes
+        send_stream
+          .send_data(body_data.copy_to_bytes(body_data.remaining()))
+          .await?;
+
+        // TODO: needs handling trailer? should be included in body from handler.
+      }
+      Err(err) => {
+        error!("Unable to send response to connection peer: {:?}", err);
+      }
+    }
+    Ok(send_stream.finish().await?)
+  }
+}

rpxy-lib/src/proxy/proxy_main.rs

@@ -0,0 +1,124 @@
+// use super::proxy_handler::handle_request;
+use crate::{
+  certs::CryptoSource, error::*, globals::Globals, handler::HttpMessageHandler, log::*, utils::ServerNameBytesExp,
+};
+use derive_builder::{self, Builder};
+use hyper::{client::connect::Connect, server::conn::Http, service::service_fn, Body, Request};
+use std::{net::SocketAddr, sync::Arc};
+use tokio::{
+  io::{AsyncRead, AsyncWrite},
+  net::TcpListener,
+  runtime::Handle,
+  time::{timeout, Duration},
+};
+
+#[derive(Clone)]
+pub struct LocalExecutor {
+  runtime_handle: Handle,
+}
+
+impl LocalExecutor {
+  fn new(runtime_handle: Handle) -> Self {
+    LocalExecutor { runtime_handle }
+  }
+}
+
+impl<F> hyper::rt::Executor<F> for LocalExecutor
+where
+  F: std::future::Future + Send + 'static,
+  F::Output: Send,
+{
+  fn execute(&self, fut: F) {
+    self.runtime_handle.spawn(fut);
+  }
+}
+
+#[derive(Clone, Builder)]
+pub struct Proxy<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone + Sync + Send + 'static,
+{
+  pub listening_on: SocketAddr,
+  pub tls_enabled: bool, // TCP待受がTLSかどうか
+  pub msg_handler: HttpMessageHandler<T, U>,
+  pub globals: Arc<Globals<U>>,
+}
+
+impl<T, U> Proxy<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone + Sync + Send,
+{
+  pub(super) fn client_serve<I>(
+    self,
+    stream: I,
+    server: Http<LocalExecutor>,
+    peer_addr: SocketAddr,
+    tls_server_name: Option<ServerNameBytesExp>,
+  ) where
+    I: AsyncRead + AsyncWrite + Send + Unpin + 'static,
+  {
+    let request_count = self.globals.request_count.clone();
+    if request_count.increment() > self.globals.proxy_config.max_clients {
+      request_count.decrement();
+      return;
+    }
+    debug!("Request incoming: current # {}", request_count.current());
+
+    self.globals.runtime_handle.clone().spawn(async move {
+      timeout(
+        self.globals.proxy_config.proxy_timeout + Duration::from_secs(1),
+        server
+          .serve_connection(
+            stream,
+            service_fn(move |req: Request<Body>| {
+              self.msg_handler.clone().handle_request(
+                req,
+                peer_addr,
+                self.listening_on,
+                self.tls_enabled,
+                tls_server_name.clone(),
+              )
+            }),
+          )
+          .with_upgrades(),
+      )
+      .await
+      .ok();
+
+      request_count.decrement();
+      debug!("Request processed: current # {}", request_count.current());
+    });
+  }
+
+  async fn start_without_tls(self, server: Http<LocalExecutor>) -> Result<()> {
+    let listener_service = async {
+      let tcp_listener = TcpListener::bind(&self.listening_on).await?;
+      info!("Start TCP proxy serving with HTTP request for configured host names");
+      while let Ok((stream, _client_addr)) = tcp_listener.accept().await {
+        self.clone().client_serve(stream, server.clone(), _client_addr, None);
+      }
+      Ok(()) as Result<()>
+    };
+    listener_service.await?;
+    Ok(())
+  }
+
+  pub async fn start(self) -> Result<()> {
+    let mut server = Http::new();
+    server.http1_keep_alive(self.globals.proxy_config.keepalive);
+    server.http2_max_concurrent_streams(self.globals.proxy_config.max_concurrent_streams);
+    server.pipeline_flush(true);
+    let executor = LocalExecutor::new(self.globals.runtime_handle.clone());
+    let server = server.with_executor(executor);
+
+    if self.tls_enabled {
+      self.start_with_tls(server).await?;
+    } else {
+      self.start_without_tls(server).await?;
+    }
+
+    Ok(())
+  }
+}

rpxy-lib/src/proxy/proxy_tls.rs

@@ -0,0 +1,245 @@
+use super::{
+  crypto_service::{CryptoReloader, ServerCrypto, ServerCryptoBase, SniServerCryptoMap},
+  proxy_main::{LocalExecutor, Proxy},
+};
+use crate::{certs::CryptoSource, constants::*, error::*, log::*, utils::BytesName};
+use hot_reload::{ReloaderReceiver, ReloaderService};
+use hyper::{client::connect::Connect, server::conn::Http};
+#[cfg(feature = "http3")]
+use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerConfig, TransportConfig};
+#[cfg(feature = "http3")]
+use rustls::ServerConfig;
+use std::sync::Arc;
+use tokio::{
+  net::TcpListener,
+  time::{timeout, Duration},
+};
+
+impl<T, U> Proxy<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone + Sync + Send + 'static,
+{
+  // TCP Listener Service, i.e., http/2 and http/1.1
+  async fn listener_service(
+    &self,
+    server: Http<LocalExecutor>,
+    mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
+  ) -> Result<()> {
+    let tcp_listener = TcpListener::bind(&self.listening_on).await?;
+    info!("Start TCP proxy serving with HTTPS request for configured host names");
+
+    let mut server_crypto_map: Option<Arc<SniServerCryptoMap>> = None;
+    loop {
+      tokio::select! {
+        tcp_cnx = tcp_listener.accept() => {
+          if tcp_cnx.is_err() || server_crypto_map.is_none() {
+            continue;
+          }
+          let (raw_stream, client_addr) = tcp_cnx.unwrap();
+          let sc_map_inner = server_crypto_map.clone();
+          let server_clone = server.clone();
+          let self_inner = self.clone();
+
+          // spawns async handshake to avoid blocking thread by sequential handshake.
+          let handshake_fut = async move {
+            let acceptor = tokio_rustls::LazyConfigAcceptor::new(tokio_rustls::rustls::server::Acceptor::default(), raw_stream).await;
+            if let Err(e) = acceptor {
+              return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {e}")));
+            }
+            let start = acceptor.unwrap();
+            let client_hello = start.client_hello();
+            let server_name = client_hello.server_name();
+            debug!("HTTP/2 or 1.1: SNI in ClientHello: {:?}", server_name);
+            let server_name = server_name.map_or_else(|| None, |v| Some(v.to_server_name_vec()));
+            if server_name.is_none(){
+              return Err(RpxyError::Proxy("No SNI is given".to_string()));
+            }
+            let server_crypto = sc_map_inner.as_ref().unwrap().get(server_name.as_ref().unwrap());
+            if server_crypto.is_none() {
+              return Err(RpxyError::Proxy(format!("No TLS serving app for {:?}", "xx")));
+            }
+            let stream = match start.into_stream(server_crypto.unwrap().clone()).await {
+              Ok(s) => s,
+              Err(e) => {
+                return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {e}")));
+              }
+            };
+            self_inner.client_serve(stream, server_clone, client_addr, server_name);
+            Ok(())
+          };
+
+          self.globals.runtime_handle.spawn( async move {
+            // timeout is introduced to avoid get stuck here.
+            match timeout(
+              Duration::from_secs(TLS_HANDSHAKE_TIMEOUT_SEC),
+              handshake_fut
+            ).await {
+              Ok(a) => {
+                if let Err(e) = a {
+                  error!("{}", e);
+                }
+              },
+              Err(e) => {
+                error!("Timeout to handshake TLS: {}", e);
+              }
+            };
+          });
+        }
+        _ = server_crypto_rx.changed() => {
+          if server_crypto_rx.borrow().is_none() {
+            error!("Reloader is broken");
+            break;
+          }
+          let cert_keys_map = server_crypto_rx.borrow().clone().unwrap();
+          let Some(server_crypto): Option<Arc<ServerCrypto>> = (&cert_keys_map).try_into().ok() else {
+            error!("Failed to update server crypto");
+            break;
+          };
+          server_crypto_map = Some(server_crypto.inner_local_map.clone());
+        }
+        else => break
+      }
+    }
+    Ok(()) as Result<()>
+  }
+
+  #[cfg(feature = "http3")]
+  async fn listener_service_h3(&self, mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>) -> Result<()> {
+    info!("Start UDP proxy serving with HTTP/3 request for configured host names");
+    // first set as null config server
+    let rustls_server_config = ServerConfig::builder()
+      .with_safe_default_cipher_suites()
+      .with_safe_default_kx_groups()
+      .with_protocol_versions(&[&rustls::version::TLS13])?
+      .with_no_client_auth()
+      .with_cert_resolver(Arc::new(rustls::server::ResolvesServerCertUsingSni::new()));
+
+    let mut transport_config_quic = TransportConfig::default();
+    transport_config_quic
+      .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream)
+      .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream)
+      .max_idle_timeout(self.globals.proxy_config.h3_max_idle_timeout);
+
+    let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config));
+    server_config_h3.transport = Arc::new(transport_config_quic);
+    server_config_h3.concurrent_connections(self.globals.proxy_config.h3_max_concurrent_connections);
+    let endpoint = Endpoint::server(server_config_h3, self.listening_on)?;
+
+    let mut server_crypto: Option<Arc<ServerCrypto>> = None;
+    loop {
+      tokio::select! {
+        new_conn = endpoint.accept() => {
+          if server_crypto.is_none() || new_conn.is_none() {
+            continue;
+          }
+          let mut conn: quinn::Connecting = new_conn.unwrap();
+          let Ok(hsd) = conn.handshake_data().await else {
+            continue
+          };
+
+          let Ok(hsd_downcast) = hsd.downcast::<HandshakeData>() else {
+            continue
+          };
+          let Some(new_server_name) = hsd_downcast.server_name else {
+            warn!("HTTP/3 no SNI is given");
+            continue;
+          };
+          debug!(
+            "HTTP/3 connection incoming (SNI {:?})",
+            new_server_name
+          );
+          // TODO: server_nameをここで出してどんどん深く投げていくのは効率が悪い。connecting -> connectionsの後でいいのでは？
+          // TODO: 通常のTLSと同じenumか何かにまとめたい
+          let fut = self.clone().connection_serve_h3(conn, new_server_name.to_server_name_vec());
+          self.globals.runtime_handle.spawn(async move {
+            // Timeout is based on underlying quic
+            if let Err(e) = fut.await {
+              warn!("QUIC or HTTP/3 connection failed: {}", e)
+            }
+          });
+        }
+        _ = server_crypto_rx.changed() => {
+          if server_crypto_rx.borrow().is_none() {
+            error!("Reloader is broken");
+            break;
+          }
+          let cert_keys_map = server_crypto_rx.borrow().clone().unwrap();
+
+          server_crypto = (&cert_keys_map).try_into().ok();
+          let Some(inner) = server_crypto.clone() else {
+            error!("Failed to update server crypto for h3");
+            break;
+          };
+          endpoint.set_server_config(Some(QuicServerConfig::with_crypto(inner.clone().inner_global_no_client_auth.clone())));
+
+        }
+        else => break
+      }
+    }
+    endpoint.wait_idle().await;
+    Ok(()) as Result<()>
+  }
+
+  pub async fn start_with_tls(self, server: Http<LocalExecutor>) -> Result<()> {
+    let (cert_reloader_service, cert_reloader_rx) = ReloaderService::<CryptoReloader<U>, ServerCryptoBase>::new(
+      &self.globals.clone(),
+      CERTS_WATCH_DELAY_SECS,
+      !LOAD_CERTS_ONLY_WHEN_UPDATED,
+    )
+    .await
+    .map_err(|e| anyhow::anyhow!(e))?;
+
+    #[cfg(not(feature = "http3"))]
+    {
+      tokio::select! {
+        _= self.cert_service(tx) => {
+          error!("Cert service for TLS exited");
+        },
+        _ = self.listener_service(server, rx) => {
+          error!("TCP proxy service for TLS exited");
+        },
+        else => {
+          error!("Something went wrong");
+          return Ok(())
+        }
+      };
+      Ok(())
+    }
+    #[cfg(feature = "http3")]
+    {
+      if self.globals.proxy_config.http3 {
+        tokio::select! {
+          _= cert_reloader_service.start() => {
+            error!("Cert service for TLS exited");
+          },
+          _ = self.listener_service(server, cert_reloader_rx.clone()) => {
+            error!("TCP proxy service for TLS exited");
+          },
+          _= self.listener_service_h3(cert_reloader_rx) => {
+            error!("UDP proxy service for QUIC exited");
+          },
+          else => {
+            error!("Something went wrong");
+            return Ok(())
+          }
+        };
+        Ok(())
+      } else {
+        tokio::select! {
+          _= cert_reloader_service.start() => {
+            error!("Cert service for TLS exited");
+          },
+          _ = self.listener_service(server, cert_reloader_rx) => {
+            error!("TCP proxy service for TLS exited");
+          },
+          else => {
+            error!("Something went wrong");
+            return Ok(())
+          }
+        };
+        Ok(())
+      }
+    }
+  }
+}

rpxy-lib/src/utils/bytes_name.rs

@@ -0,0 +1,123 @@
+/// Server name (hostname or ip address) representation in bytes-based struct
+/// for searching hashmap or key list by exact or longest-prefix matching
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)]
+pub struct ServerNameBytesExp(pub Vec<u8>); // lowercase ascii bytes
+impl From<&[u8]> for ServerNameBytesExp {
+  fn from(b: &[u8]) -> Self {
+    Self(b.to_ascii_lowercase())
+  }
+}
+impl TryInto<String> for &ServerNameBytesExp {
+  type Error = anyhow::Error;
+  fn try_into(self) -> Result<String, Self::Error> {
+    let s = std::str::from_utf8(&self.0)?;
+    Ok(s.to_string())
+  }
+}
+
+/// Path name, like "/path/ok", represented in bytes-based struct
+/// for searching hashmap or key list by exact or longest-prefix matching
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)]
+pub struct PathNameBytesExp(pub Vec<u8>); // lowercase ascii bytes
+impl PathNameBytesExp {
+  pub fn len(&self) -> usize {
+    self.0.len()
+  }
+  pub fn is_empty(&self) -> bool {
+    self.0.len() == 0
+  }
+  pub fn get<I>(&self, index: I) -> Option<&I::Output>
+  where
+    I: std::slice::SliceIndex<[u8]>,
+  {
+    self.0.get(index)
+  }
+  pub fn starts_with(&self, needle: &Self) -> bool {
+    self.0.starts_with(&needle.0)
+  }
+}
+impl AsRef<[u8]> for PathNameBytesExp {
+  fn as_ref(&self) -> &[u8] {
+    self.0.as_ref()
+  }
+}
+
+/// Trait to express names in ascii-lowercased bytes
+pub trait BytesName {
+  type OutputSv: Send + Sync + 'static;
+  type OutputPath;
+  fn to_server_name_vec(self) -> Self::OutputSv;
+  fn to_path_name_vec(self) -> Self::OutputPath;
+}
+
+impl<'a, T: Into<std::borrow::Cow<'a, str>>> BytesName for T {
+  type OutputSv = ServerNameBytesExp;
+  type OutputPath = PathNameBytesExp;
+
+  fn to_server_name_vec(self) -> Self::OutputSv {
+    let name = self.into().bytes().collect::<Vec<u8>>().to_ascii_lowercase();
+    ServerNameBytesExp(name)
+  }
+
+  fn to_path_name_vec(self) -> Self::OutputPath {
+    let name = self.into().bytes().collect::<Vec<u8>>().to_ascii_lowercase();
+    PathNameBytesExp(name)
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  #[test]
+  fn bytes_name_str_works() {
+    let s = "OK_string";
+    let bn = s.to_path_name_vec();
+    let bn_lc = s.to_server_name_vec();
+
+    assert_eq!(Vec::from("ok_string".as_bytes()), bn.0);
+    assert_eq!(Vec::from("ok_string".as_bytes()), bn_lc.0);
+  }
+
+  #[test]
+  fn from_works() {
+    let s = "OK_string".to_server_name_vec();
+    let m = ServerNameBytesExp::from("OK_strinG".as_bytes());
+    assert_eq!(s, m);
+    assert_eq!(s.0, "ok_string".as_bytes().to_vec());
+    assert_eq!(m.0, "ok_string".as_bytes().to_vec());
+  }
+
+  #[test]
+  fn get_works() {
+    let s = "OK_str".to_path_name_vec();
+    let i = s.get(0);
+    assert_eq!(Some(&"o".as_bytes()[0]), i);
+    let i = s.get(1);
+    assert_eq!(Some(&"k".as_bytes()[0]), i);
+    let i = s.get(2);
+    assert_eq!(Some(&"_".as_bytes()[0]), i);
+    let i = s.get(3);
+    assert_eq!(Some(&"s".as_bytes()[0]), i);
+    let i = s.get(4);
+    assert_eq!(Some(&"t".as_bytes()[0]), i);
+    let i = s.get(5);
+    assert_eq!(Some(&"r".as_bytes()[0]), i);
+    let i = s.get(6);
+    assert_eq!(None, i);
+  }
+
+  #[test]
+  fn start_with_works() {
+    let s = "OK_str".to_path_name_vec();
+    let correct = "OK".to_path_name_vec();
+    let incorrect = "KO".to_path_name_vec();
+    assert!(s.starts_with(&correct));
+    assert!(!s.starts_with(&incorrect));
+  }
+
+  #[test]
+  fn as_ref_works() {
+    let s = "OK_str".to_path_name_vec();
+    assert_eq!(s.as_ref(), "ok_str".as_bytes());
+  }
+}

rpxy-lib/src/utils/mod.rs

@@ -0,0 +1,5 @@
+mod bytes_name;
+mod socket_addr;
+
+pub use bytes_name::{BytesName, PathNameBytesExp, ServerNameBytesExp};
+pub use socket_addr::ToCanonical;

rpxy-lib/src/utils/socket_addr.rs

@@ -0,0 +1,60 @@
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+pub trait ToCanonical {
+  fn to_canonical(&self) -> Self;
+}
+
+impl ToCanonical for SocketAddr {
+  fn to_canonical(&self) -> Self {
+    match self {
+      SocketAddr::V4(_) => *self,
+      SocketAddr::V6(v6) => match v6.ip().to_ipv4() {
+        Some(mapped) => {
+          if mapped == Ipv4Addr::new(0, 0, 0, 1) {
+            *self
+          } else {
+            SocketAddr::new(IpAddr::V4(mapped), self.port())
+          }
+        }
+        None => *self,
+      },
+    }
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use std::net::Ipv6Addr;
+  #[test]
+  fn ipv4_loopback_to_canonical() {
+    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080);
+    assert_eq!(socket.to_canonical(), socket);
+  }
+  #[test]
+  fn ipv6_loopback_to_canonical() {
+    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), 8080);
+    assert_eq!(socket.to_canonical(), socket);
+  }
+  #[test]
+  fn ipv4_to_canonical() {
+    let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), 8080);
+    assert_eq!(socket.to_canonical(), socket);
+  }
+  #[test]
+  fn ipv6_to_canonical() {
+    let socket = SocketAddr::new(
+      IpAddr::V6(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0xdead, 0xbeef)),
+      8080,
+    );
+    assert_eq!(socket.to_canonical(), socket);
+  }
+  #[test]
+  fn ipv4_mapped_to_ipv6_to_canonical() {
+    let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff)), 8080);
+    assert_eq!(
+      socket.to_canonical(),
+      SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 10, 2, 255)), 8080)
+    );
+  }
+}