tuxmain/rust-rpxy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wip: refactor whole module in lib

Browse source
This commit is contained in:
Jun Kurihara 2023-11-21 22:46:52 +09:00
commit f98c778a0c
No known key found for this signature in database
GPG key ID: 48ADFD173ED22B03
42 changed files with 943 additions and 531 deletions
Download patch file Download diff file Expand all files Collapse all files

276
legacy-lib/src/proxy/crypto_service.rs Normal file
View file

@ -0,0 +1,276 @@
use crate::{
certs::{CertsAndKeys, CryptoSource},
globals::Globals,
log::*,
utils::ServerNameBytesExp,
};
use async_trait::async_trait;
use hot_reload::*;
use rustc_hash::FxHashMap as HashMap;
use rustls::{server::ResolvesServerCertUsingSni, sign::CertifiedKey, RootCertStore, ServerConfig};
use std::sync::Arc;
#[derive(Clone)]
/// Reloader service for certificates and keys for TLS
pub struct CryptoReloader<T>
where
T: CryptoSource,
{
globals: Arc<Globals<T>>,
}
pub type SniServerCryptoMap = HashMap<ServerNameBytesExp, Arc<ServerConfig>>;
pub struct ServerCrypto {
// For Quic/HTTP3, only servers with no client authentication
#[cfg(feature = "http3-quinn")]
pub inner_global_no_client_auth: Arc<ServerConfig>,
#[cfg(feature = "http3-s2n")]
pub inner_global_no_client_auth: s2n_quic_rustls::Server,
// For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers
pub inner_local_map: Arc<SniServerCryptoMap>,
}
/// Reloader target for the certificate reloader service
#[derive(Debug, PartialEq, Eq, Clone, Default)]
pub struct ServerCryptoBase {
inner: HashMap<ServerNameBytesExp, CertsAndKeys>,
}
#[async_trait]
impl<T> Reload<ServerCryptoBase> for CryptoReloader<T>
where
T: CryptoSource + Sync + Send,
{
type Source = Arc<Globals<T>>;
async fn new(source: &Self::Source) -> Result<Self, ReloaderError<ServerCryptoBase>> {
Ok(Self {
globals: source.clone(),
})
}
async fn reload(&self) -> Result<Option<ServerCryptoBase>, ReloaderError<ServerCryptoBase>> {
let mut certs_and_keys_map = ServerCryptoBase::default();
for (server_name_bytes_exp, backend) in self.globals.backends.apps.iter() {
if let Some(crypto_source) = &backend.crypto_source {
let certs_and_keys = crypto_source
.read()
.await
.map_err(|_e| ReloaderError::<ServerCryptoBase>::Reload("Failed to reload cert, key or ca cert"))?;
certs_and_keys_map
.inner
.insert(server_name_bytes_exp.to_owned(), certs_and_keys);
}
}
Ok(Some(certs_and_keys_map))
}
}
impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
type Error = anyhow::Error;
fn try_into(self) -> Result<Arc<ServerCrypto>, Self::Error> {
#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
let server_crypto_global = self.build_server_crypto_global()?;
let server_crypto_local_map: SniServerCryptoMap = self.build_server_crypto_local_map()?;
Ok(Arc::new(ServerCrypto {
#[cfg(feature = "http3-quinn")]
inner_global_no_client_auth: Arc::new(server_crypto_global),
#[cfg(feature = "http3-s2n")]
inner_global_no_client_auth: server_crypto_global,
inner_local_map: Arc::new(server_crypto_local_map),
}))
}
}
impl ServerCryptoBase {
fn build_server_crypto_local_map(&self) -> Result<SniServerCryptoMap, ReloaderError<ServerCryptoBase>> {
let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default();
for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
let server_name: String = server_name_bytes_exp.try_into()?;
// Parse server certificates and private keys
let Ok(certified_key): Result<CertifiedKey, _> = certs_and_keys.parse_server_certs_and_keys() else {
warn!("Failed to add certificate for {}", server_name);
continue;
};
let mut resolver_local = ResolvesServerCertUsingSni::new();
let mut client_ca_roots_local = RootCertStore::empty();
// add server certificate and key
if let Err(e) = resolver_local.add(server_name.as_str(), certified_key.to_owned()) {
error!(
"{}: Failed to read some certificates and keys {}",
server_name.as_str(),
e
)
}
// add client certificate if specified
if certs_and_keys.client_ca_certs.is_some() {
// add client certificate if specified
match certs_and_keys.parse_client_ca_certs() {
Ok((owned_trust_anchors, _subject_key_ids)) => {
client_ca_roots_local.add_trust_anchors(owned_trust_anchors.into_iter());
}
Err(e) => {
warn!(
"Failed to add client CA certificate for {}: {}",
server_name.as_str(),
e
);
}
}
}
let mut server_config_local = if client_ca_roots_local.is_empty() {
// with no client auth, enable http1.1 -- 3
#[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))]
{
ServerConfig::builder()
.with_safe_defaults()
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver_local))
}
#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
{
let mut sc = ServerConfig::builder()
.with_safe_defaults()
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver_local));
sc.alpn_protocols = vec![b"h3".to_vec(), b"hq-29".to_vec()]; // TODO: remove hq-29 later?
sc
}
} else {
// with client auth, enable only http1.1 and 2
// let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots);
let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots_local);
ServerConfig::builder()
.with_safe_defaults()
.with_client_cert_verifier(Arc::new(client_certs_verifier))
.with_cert_resolver(Arc::new(resolver_local))
};
server_config_local.alpn_protocols.push(b"h2".to_vec());
server_config_local.alpn_protocols.push(b"http/1.1".to_vec());
server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local));
}
Ok(server_crypto_local_map)
}
#[cfg(feature = "http3-quinn")]
fn build_server_crypto_global(&self) -> Result<ServerConfig, ReloaderError<ServerCryptoBase>> {
let mut resolver_global = ResolvesServerCertUsingSni::new();
for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
let server_name: String = server_name_bytes_exp.try_into()?;
// Parse server certificates and private keys
let Ok(certified_key): Result<CertifiedKey, _> = certs_and_keys.parse_server_certs_and_keys() else {
warn!("Failed to add certificate for {}", server_name);
continue;
};
if certs_and_keys.client_ca_certs.is_none() {
// aggregated server config for no client auth server for http3
if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
error!(
"{}: Failed to read some certificates and keys {}",
server_name.as_str(),
e
)
}
}
}
//////////////
let mut server_crypto_global = ServerConfig::builder()
.with_safe_defaults()
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver_global));
//////////////////////////////
server_crypto_global.alpn_protocols = vec![
b"h3".to_vec(),
b"hq-29".to_vec(), // TODO: remove later?
b"h2".to_vec(),
b"http/1.1".to_vec(),
];
Ok(server_crypto_global)
}
#[cfg(feature = "http3-s2n")]
fn build_server_crypto_global(&self) -> Result<s2n_quic_rustls::Server, ReloaderError<ServerCryptoBase>> {
let mut resolver_global = s2n_quic_rustls::rustls::server::ResolvesServerCertUsingSni::new();
for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
let server_name: String = server_name_bytes_exp.try_into()?;
// Parse server certificates and private keys
let Ok(certified_key) = parse_server_certs_and_keys_s2n(certs_and_keys) else {
warn!("Failed to add certificate for {}", server_name);
continue;
};
if certs_and_keys.client_ca_certs.is_none() {
// aggregated server config for no client auth server for http3
if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
error!(
"{}: Failed to read some certificates and keys {}",
server_name.as_str(),
e
)
}
}
}
let alpn = vec![
b"h3".to_vec(),
b"hq-29".to_vec(), // TODO: remove later?
b"h2".to_vec(),
b"http/1.1".to_vec(),
];
let server_crypto_global = s2n_quic::provider::tls::rustls::Server::builder()
.with_cert_resolver(Arc::new(resolver_global))
.map_err(|e| anyhow::anyhow!(e))?
.with_application_protocols(alpn.iter())
.map_err(|e| anyhow::anyhow!(e))?
.build()
.map_err(|e| anyhow::anyhow!(e))?;
Ok(server_crypto_global)
}
}
#[cfg(feature = "http3-s2n")]
/// This is workaround for the version difference between rustls and s2n-quic-rustls
fn parse_server_certs_and_keys_s2n(
certs_and_keys: &CertsAndKeys,
) -> Result<s2n_quic_rustls::rustls::sign::CertifiedKey, anyhow::Error> {
let signing_key = certs_and_keys
.cert_keys
.iter()
.find_map(|k| {
let s2n_private_key = s2n_quic_rustls::PrivateKey(k.0.clone());
if let Ok(sk) = s2n_quic_rustls::rustls::sign::any_supported_type(&s2n_private_key) {
Some(sk)
} else {
None
}
})
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"Unable to find a valid certificate and key",
)
})?;
let certs: Vec<_> = certs_and_keys
.certs
.iter()
.map(|c| s2n_quic_rustls::rustls::Certificate(c.0.clone()))
.collect();
Ok(s2n_quic_rustls::rustls::sign::CertifiedKey::new(certs, signing_key))
}

42
legacy-lib/src/proxy/mod.rs Normal file
View file

@ -0,0 +1,42 @@
mod crypto_service;
mod proxy_client_cert;
#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
mod proxy_h3;
mod proxy_main;
#[cfg(feature = "http3-quinn")]
mod proxy_quic_quinn;
#[cfg(feature = "http3-s2n")]
mod proxy_quic_s2n;
mod proxy_tls;
mod socket;
use crate::error::*;
use http::{Response, StatusCode};
use http_body_util::{combinators, BodyExt, Either, Empty};
use hyper::body::{Bytes, Incoming};
pub use proxy_main::{Proxy, ProxyBuilder, ProxyBuilderError};
/// Type for synthetic boxed body
type BoxBody = combinators::BoxBody<Bytes, hyper::Error>;
/// Type for either passthrough body or synthetic body
type EitherBody = Either<Incoming, BoxBody>;
/// helper function to build http response with passthrough body
fn passthrough_response(response: Response<Incoming>) -> Result<Response<EitherBody>> {
Ok(response.map(EitherBody::Left))
}
/// build http response with status code of 4xx and 5xx
fn synthetic_error_response(status_code: StatusCode) -> Result<Response<EitherBody>> {
let res = Response::builder()
.status(status_code)
.body(EitherBody::Right(BoxBody::new(empty())))
.unwrap();
Ok(res)
}
/// helper function to build a empty body
fn empty() -> BoxBody {
Empty::<Bytes>::new().map_err(|never| match never {}).boxed()
}

47
legacy-lib/src/proxy/proxy_client_cert.rs Normal file
View file

@ -0,0 +1,47 @@
use crate::{error::*, log::*};
use rustc_hash::FxHashSet as HashSet;
use rustls::Certificate;
use x509_parser::extensions::ParsedExtension;
use x509_parser::prelude::*;
#[allow(dead_code)]
// TODO: consider move this function to the layer of handle_request (L7) to return 403
pub(super) fn check_client_authentication(
client_certs: Option<&[Certificate]>,
client_ca_keyids_set_for_sni: Option<&HashSet<Vec<u8>>>,
) -> std::result::Result<(), ClientCertsError> {
let Some(client_ca_keyids_set) = client_ca_keyids_set_for_sni else {
// No client cert settings for given server name
return Ok(());
};
let Some(client_certs) = client_certs else {
error!("Client certificate is needed for given server name");
return Err(ClientCertsError::ClientCertRequired(
"Client certificate is needed for given server name".to_string(),
));
};
debug!("Incoming TLS client is (temporarily) authenticated via client cert");
// Check client certificate key ids
let mut client_certs_parsed_iter = client_certs.iter().filter_map(|d| parse_x509_certificate(&d.0).ok());
let match_server_crypto_and_client_cert = client_certs_parsed_iter.any(|c| {
let mut filtered = c.1.iter_extensions().filter_map(|e| {
if let ParsedExtension::AuthorityKeyIdentifier(key_id) = e.parsed_extension() {
key_id.key_identifier.as_ref()
} else {
None
}
});
filtered.any(|id| client_ca_keyids_set.contains(id.0))
});
if !match_server_crypto_and_client_cert {
error!("Inconsistent client certificate was provided for SNI");
return Err(ClientCertsError::InconsistentClientCert(
"Inconsistent client certificate was provided for SNI".to_string(),
));
}
Ok(())
}

186
legacy-lib/src/proxy/proxy_h3.rs Normal file
View file

@ -0,0 +1,186 @@
use super::Proxy;
use crate::{certs::CryptoSource, error::*, log::*, utils::ServerNameBytesExp};
use bytes::{Buf, Bytes};
use futures::Stream;
#[cfg(feature = "http3-quinn")]
use h3::{quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
use http::{Request, Response};
use http_body_util::{BodyExt, BodyStream, StreamBody};
use hyper::body::{Body, Incoming};
use hyper_util::client::legacy::connect::Connect;
#[cfg(feature = "http3-s2n")]
use s2n_quic_h3::h3::{self, quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
use std::net::SocketAddr;
use tokio::time::{timeout, Duration};
impl<U> Proxy<U>
where
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub(super) async fn connection_serve_h3<C>(
&self,
quic_connection: C,
tls_server_name: ServerNameBytesExp,
client_addr: SocketAddr,
) -> Result<()>
where
C: ConnectionQuic<Bytes>,
<C as ConnectionQuic<Bytes>>::BidiStream: BidiStream<Bytes> + Send + 'static,
<<C as ConnectionQuic<Bytes>>::BidiStream as BidiStream<Bytes>>::RecvStream: Send,
<<C as ConnectionQuic<Bytes>>::BidiStream as BidiStream<Bytes>>::SendStream: Send,
{
let mut h3_conn = h3::server::Connection::<_, Bytes>::new(quic_connection).await?;
info!(
"QUIC/HTTP3 connection established from {:?} {:?}",
client_addr, tls_server_name
);
// TODO: Is here enough to fetch server_name from NewConnection?
// to avoid deep nested call from listener_service_h3
loop {
// this routine follows hyperium/h3 examples https://github.com/hyperium/h3/blob/master/examples/server.rs
match h3_conn.accept().await {
Ok(None) => {
break;
}
Err(e) => {
warn!("HTTP/3 error on accept incoming connection: {}", e);
match e.get_error_level() {
h3::error::ErrorLevel::ConnectionError => break,
h3::error::ErrorLevel::StreamError => continue,
}
}
Ok(Some((req, stream))) => {
// We consider the connection count separately from the stream count.
// Max clients for h1/h2 = max 'stream' for h3.
let request_count = self.globals.request_count.clone();
if request_count.increment() > self.globals.proxy_config.max_clients {
request_count.decrement();
h3_conn.shutdown(0).await?;
break;
}
debug!("Request incoming: current # {}", request_count.current());
let self_inner = self.clone();
let tls_server_name_inner = tls_server_name.clone();
self.globals.runtime_handle.spawn(async move {
if let Err(e) = timeout(
self_inner.globals.proxy_config.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner),
)
.await
{
error!("HTTP/3 failed to process stream: {}", e);
}
request_count.decrement();
debug!("Request processed: current # {}", request_count.current());
});
}
}
}
Ok(())
}
async fn stream_serve_h3<S>(
&self,
req: Request<()>,
stream: RequestStream<S, Bytes>,
client_addr: SocketAddr,
tls_server_name: ServerNameBytesExp,
) -> Result<()>
where
S: BidiStream<Bytes> + Send + 'static,
<S as BidiStream<Bytes>>::RecvStream: Send,
{
println!("stream_serve_h3");
let (req_parts, _) = req.into_parts();
// split stream and async body handling
let (mut send_stream, mut recv_stream) = stream.split();
// let max_body_size = self.globals.proxy_config.h3_request_max_body_size;
// // let max = body_stream.size_hint().upper().unwrap_or(u64::MAX);
// // if max > max_body_size as u64 {
// // return Err(HttpError::TooLargeRequestBody);
// // }
// let new_req = Request::from_parts(req_parts, body_stream);
////////////////////
// TODO: TODO: TODO: TODO:
// TODO: Body in hyper-0.14 was changed to Incoming in hyper-1.0, and it is not accessible from outside.
// Thus, we need to implement IncomingLike trait using channel. Also, the backend handler must feed the body in the form of
// Either<Incoming, IncomingLike> as body.
// Also, the downstream from the backend handler could be Incoming, but will be wrapped as Either<Incoming, ()/Empty> as well due to H3.
// Result<Either<_,_>, E> type includes E as HttpError to generate the status code and related Response<BoxBody>.
// Thus to handle synthetic error messages in BoxBody, the serve() function outputs Response<Either<Either<Incoming, ()/Empty>, BoxBody>>>.
////////////////////
// // generate streamed body with trailers using channel
// let (body_sender, req_body) = Incoming::channel();
// Buffering and sending body through channel for protocol conversion like h3 -> h2/http1.1
// The underling buffering, i.e., buffer given by the API recv_data.await?, is handled by quinn.
let max_body_size = self.globals.proxy_config.h3_request_max_body_size;
self.globals.runtime_handle.spawn(async move {
// let mut sender = body_sender;
let mut size = 0usize;
while let Some(mut body) = recv_stream.recv_data().await? {
debug!("HTTP/3 incoming request body: remaining {}", body.remaining());
size += body.remaining();
if size > max_body_size {
error!(
"Exceeds max request body size for HTTP/3: received {}, maximum_allowd {}",
size, max_body_size
);
return Err(RpxyError::Proxy("Exceeds max request body size for HTTP/3".to_string()));
}
// create stream body to save memory, shallow copy (increment of ref-count) to Bytes using copy_to_bytes
// sender.send_data(body.copy_to_bytes(body.remaining())).await?;
}
// trailers: use inner for work around. (directly get trailer)
let trailers = recv_stream.as_mut().recv_trailers().await?;
if trailers.is_some() {
debug!("HTTP/3 incoming request trailers");
// sender.send_trailers(trailers.unwrap()).await?;
}
Ok(())
});
// let new_req: Request<Incoming> = Request::from_parts(req_parts, req_body);
// let res = self
// .msg_handler
// .clone()
// .handle_request(
// new_req,
// client_addr,
// self.listening_on,
// self.tls_enabled,
// Some(tls_server_name),
// )
// .await?;
// let (new_res_parts, new_body) = res.into_parts();
// let new_res = Response::from_parts(new_res_parts, ());
// match send_stream.send_response(new_res).await {
// Ok(_) => {
// debug!("HTTP/3 response to connection successful");
// // aggregate body without copying
// let body_data = new_body.collect().await?.aggregate();
// // create stream body to save memory, shallow copy (increment of ref-count) to Bytes using copy_to_bytes
// send_stream
// .send_data(body_data.copy_to_bytes(body_data.remaining()))
// .await?;
// // TODO: needs handling trailer? should be included in body from handler.
// }
// Err(err) => {
// error!("Unable to send response to connection peer: {:?}", err);
// }
// }
// Ok(send_stream.finish().await?)
todo!()
}
}

150
legacy-lib/src/proxy/proxy_main.rs Normal file
View file

@ -0,0 +1,150 @@
use super::{passthrough_response, socket::bind_tcp_socket, synthetic_error_response, EitherBody};
use crate::{
certs::CryptoSource, error::*, globals::Globals, handler::HttpMessageHandler, hyper_executor::LocalExecutor, log::*,
utils::ServerNameBytesExp,
};
use derive_builder::{self, Builder};
use http::{Request, StatusCode};
use hyper::{
body::Incoming,
rt::{Read, Write},
service::service_fn,
};
use hyper_util::{client::legacy::connect::Connect, rt::TokioIo, server::conn::auto::Builder as ConnectionBuilder};
use std::{net::SocketAddr, sync::Arc};
use tokio::time::{timeout, Duration};
#[derive(Clone, Builder)]
/// Proxy main object
pub struct Proxy<U>
where
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub listening_on: SocketAddr,
pub tls_enabled: bool, // TCP待受がTLSかどうか
/// hyper server receiving http request
pub http_server: Arc<ConnectionBuilder<LocalExecutor>>,
// pub msg_handler: Arc<HttpMessageHandler<U>>,
pub msg_handler: Arc<HttpMessageHandler<U>>,
pub globals: Arc<Globals<U>>,
}
/// Wrapper function to handle request
async fn serve_request<U>(
req: Request<Incoming>,
// handler: Arc<HttpMessageHandler<T, U>>,
handler: Arc<HttpMessageHandler<U>>,
client_addr: SocketAddr,
listen_addr: SocketAddr,
tls_enabled: bool,
tls_server_name: Option<ServerNameBytesExp>,
) -> Result<hyper::Response<EitherBody>>
where
U: CryptoSource + Clone + Sync + Send + 'static,
{
match handler
.handle_request(req, client_addr, listen_addr, tls_enabled, tls_server_name)
.await?
{
Ok(res) => passthrough_response(res),
Err(e) => synthetic_error_response(StatusCode::from(e)),
}
}
impl<U> Proxy<U>
where
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send,
{
/// Serves requests from clients
pub(super) fn serve_connection<I>(
&self,
stream: I,
peer_addr: SocketAddr,
tls_server_name: Option<ServerNameBytesExp>,
) where
I: Read + Write + Send + Unpin + 'static,
{
let request_count = self.globals.request_count.clone();
if request_count.increment() > self.globals.proxy_config.max_clients {
request_count.decrement();
return;
}
debug!("Request incoming: current # {}", request_count.current());
let server_clone = self.http_server.clone();
let msg_handler_clone = self.msg_handler.clone();
let timeout_sec = self.globals.proxy_config.proxy_timeout;
let tls_enabled = self.tls_enabled;
let listening_on = self.listening_on;
self.globals.runtime_handle.clone().spawn(async move {
timeout(
timeout_sec + Duration::from_secs(1),
server_clone.serve_connection_with_upgrades(
stream,
service_fn(move |req: Request<Incoming>| {
serve_request(
req,
msg_handler_clone.clone(),
peer_addr,
listening_on,
tls_enabled,
tls_server_name.clone(),
)
}),
),
)
.await
.ok();
request_count.decrement();
debug!("Request processed: current # {}", request_count.current());
});
}
/// Start without TLS (HTTP cleartext)
async fn start_without_tls(&self) -> Result<()> {
let listener_service = async {
let tcp_socket = bind_tcp_socket(&self.listening_on)?;
let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?;
info!("Start TCP proxy serving with HTTP request for configured host names");
while let Ok((stream, client_addr)) = tcp_listener.accept().await {
self.serve_connection(TokioIo::new(stream), client_addr, None);
}
Ok(()) as Result<()>
};
listener_service.await?;
Ok(())
}
/// Entrypoint for HTTP/1.1 and HTTP/2 servers
pub async fn start(&self) -> Result<()> {
let proxy_service = async {
if self.tls_enabled {
self.start_with_tls().await
} else {
self.start_without_tls().await
}
};
match &self.globals.term_notify {
Some(term) => {
tokio::select! {
_ = proxy_service => {
warn!("Proxy service got down");
}
_ = term.notified() => {
info!("Proxy service listening on {} receives term signal", self.listening_on);
}
}
}
None => {
proxy_service.await?;
warn!("Proxy service got down");
}
}
Ok(())
}
}

124
legacy-lib/src/proxy/proxy_quic_quinn.rs Normal file
View file

@ -0,0 +1,124 @@
use super::socket::bind_udp_socket;
use super::{
crypto_service::{ServerCrypto, ServerCryptoBase},
proxy_main::Proxy,
};
use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName};
use hot_reload::ReloaderReceiver;
use hyper_util::client::legacy::connect::Connect;
use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerConfig, TransportConfig};
use rustls::ServerConfig;
use std::sync::Arc;
impl<U> Proxy<U>
where
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub(super) async fn listener_service_h3(
&self,
mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
) -> Result<()> {
info!("Start UDP proxy serving with HTTP/3 request for configured host names [quinn]");
// first set as null config server
let rustls_server_config = ServerConfig::builder()
.with_safe_default_cipher_suites()
.with_safe_default_kx_groups()
.with_protocol_versions(&[&rustls::version::TLS13])?
.with_no_client_auth()
.with_cert_resolver(Arc::new(rustls::server::ResolvesServerCertUsingSni::new()));
let mut transport_config_quic = TransportConfig::default();
transport_config_quic
.max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream.into())
.max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream.into())
.max_idle_timeout(
self
.globals
.proxy_config
.h3_max_idle_timeout
.map(|v| quinn::IdleTimeout::try_from(v).unwrap()),
);
let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config));
server_config_h3.transport = Arc::new(transport_config_quic);
server_config_h3.concurrent_connections(self.globals.proxy_config.h3_max_concurrent_connections);
// To reuse address
let udp_socket = bind_udp_socket(&self.listening_on)?;
let runtime = quinn::default_runtime()
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No async runtime found"))?;
let endpoint = Endpoint::new(
quinn::EndpointConfig::default(),
Some(server_config_h3),
udp_socket,
runtime,
)?;
let mut server_crypto: Option<Arc<ServerCrypto>> = None;
loop {
tokio::select! {
new_conn = endpoint.accept() => {
if server_crypto.is_none() || new_conn.is_none() {
continue;
}
let mut conn: quinn::Connecting = new_conn.unwrap();
let Ok(hsd) = conn.handshake_data().await else {
continue
};
let Ok(hsd_downcast) = hsd.downcast::<HandshakeData>() else {
continue
};
let Some(new_server_name) = hsd_downcast.server_name else {
warn!("HTTP/3 no SNI is given");
continue;
};
debug!(
"HTTP/3 connection incoming (SNI {:?})",
new_server_name
);
// TODO: server_nameをここで出してどんどん深く投げていくのは効率が悪い。connecting -> connectionsの後でいいのでは？
// TODO: 通常のTLSと同じenumか何かにまとめたい
let self_clone = self.clone();
self.globals.runtime_handle.spawn(async move {
let client_addr = conn.remote_address();
let quic_connection = match conn.await {
Ok(new_conn) => {
info!("New connection established");
h3_quinn::Connection::new(new_conn)
},
Err(e) => {
warn!("QUIC accepting connection failed: {:?}", e);
return Err(RpxyError::QuicConn(e));
}
};
// Timeout is based on underlying quic
if let Err(e) = self_clone.connection_serve_h3(quic_connection, new_server_name.to_server_name_vec(), client_addr).await {
warn!("QUIC or HTTP/3 connection failed: {}", e);
};
Ok(())
});
}
_ = server_crypto_rx.changed() => {
if server_crypto_rx.borrow().is_none() {
error!("Reloader is broken");
break;
}
let cert_keys_map = server_crypto_rx.borrow().clone().unwrap();
server_crypto = (&cert_keys_map).try_into().ok();
let Some(inner) = server_crypto.clone() else {
error!("Failed to update server crypto for h3");
break;
};
endpoint.set_server_config(Some(QuicServerConfig::with_crypto(inner.clone().inner_global_no_client_auth.clone())));
}
else => break
}
}
endpoint.wait_idle().await;
Ok(()) as Result<()>
}
}

135
legacy-lib/src/proxy/proxy_quic_s2n.rs Normal file
View file

@ -0,0 +1,135 @@
use super::{
crypto_service::{ServerCrypto, ServerCryptoBase},
proxy_main::Proxy,
};
use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName};
use hot_reload::ReloaderReceiver;
use hyper_util::client::legacy::connect::Connect;
use s2n_quic::provider;
use std::sync::Arc;
impl<U> Proxy<U>
where
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub(super) async fn listener_service_h3(
&self,
mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
) -> Result<()> {
info!("Start UDP proxy serving with HTTP/3 request for configured host names [s2n-quic]");
// initially wait for receipt
let mut server_crypto: Option<Arc<ServerCrypto>> = {
let _ = server_crypto_rx.changed().await;
let sc = self.receive_server_crypto(server_crypto_rx.clone())?;
Some(sc)
};
// event loop
loop {
tokio::select! {
v = self.listener_service_h3_inner(&server_crypto) => {
if let Err(e) = v {
error!("Quic connection event loop illegally shutdown [s2n-quic] {e}");
break;
}
}
_ = server_crypto_rx.changed() => {
server_crypto = match self.receive_server_crypto(server_crypto_rx.clone()) {
Ok(sc) => Some(sc),
Err(e) => {
error!("{e}");
break;
}
};
}
else => break
}
}
Ok(())
}
fn receive_server_crypto(&self, server_crypto_rx: ReloaderReceiver<ServerCryptoBase>) -> Result<Arc<ServerCrypto>> {
let cert_keys_map = server_crypto_rx.borrow().clone().ok_or_else(|| {
error!("Reloader is broken");
RpxyError::Other(anyhow!("Reloader is broken"))
})?;
let server_crypto: Option<Arc<ServerCrypto>> = (&cert_keys_map).try_into().ok();
server_crypto.ok_or_else(|| {
error!("Failed to update server crypto for h3 [s2n-quic]");
RpxyError::Other(anyhow!("Failed to update server crypto for h3 [s2n-quic]"))
})
}
async fn listener_service_h3_inner(&self, server_crypto: &Option<Arc<ServerCrypto>>) -> Result<()> {
// setup UDP socket
let io = provider::io::tokio::Builder::default()
.with_receive_address(self.listening_on)?
.with_reuse_port()?
.build()?;
// setup limits
let mut limits = provider::limits::Limits::default()
.with_max_open_local_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64)
.map_err(|e| anyhow!(e))?
.with_max_open_remote_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64)
.map_err(|e| anyhow!(e))?
.with_max_open_local_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64)
.map_err(|e| anyhow!(e))?
.with_max_open_remote_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64)
.map_err(|e| anyhow!(e))?
.with_max_active_connection_ids(self.globals.proxy_config.h3_max_concurrent_connections as u64)
.map_err(|e| anyhow!(e))?;
limits = if let Some(v) = self.globals.proxy_config.h3_max_idle_timeout {
limits.with_max_idle_timeout(v).map_err(|e| anyhow!(e))?
} else {
limits
};
// setup tls
let Some(server_crypto) = server_crypto else {
warn!("No server crypto is given [s2n-quic]");
return Err(RpxyError::Other(anyhow!("No server crypto is given [s2n-quic]")));
};
let tls = server_crypto.inner_global_no_client_auth.clone();
let mut server = s2n_quic::Server::builder()
.with_tls(tls)
.map_err(|e| anyhow::anyhow!(e))?
.with_io(io)
.map_err(|e| anyhow!(e))?
.with_limits(limits)
.map_err(|e| anyhow!(e))?
.start()
.map_err(|e| anyhow!(e))?;
// quic event loop. this immediately cancels when crypto is updated by tokio::select!
while let Some(new_conn) = server.accept().await {
debug!("New QUIC connection established");
let Ok(Some(new_server_name)) = new_conn.server_name() else {
warn!("HTTP/3 no SNI is given");
continue;
};
debug!("HTTP/3 connection incoming (SNI {:?})", new_server_name);
let self_clone = self.clone();
self.globals.runtime_handle.spawn(async move {
let client_addr = new_conn.remote_addr()?;
let quic_connection = s2n_quic_h3::Connection::new(new_conn);
// Timeout is based on underlying quic
if let Err(e) = self_clone
.connection_serve_h3(quic_connection, new_server_name.to_server_name_vec(), client_addr)
.await
{
warn!("QUIC or HTTP/3 connection failed: {}", e);
};
Ok(()) as Result<()>
});
}
Ok(())
}
}

155
legacy-lib/src/proxy/proxy_tls.rs Normal file
View file

@ -0,0 +1,155 @@
use super::{
crypto_service::{CryptoReloader, ServerCrypto, ServerCryptoBase, SniServerCryptoMap},
proxy_main::Proxy,
socket::bind_tcp_socket,
};
use crate::{certs::CryptoSource, constants::*, error::*, log::*, utils::BytesName};
use hot_reload::{ReloaderReceiver, ReloaderService};
use hyper_util::{client::legacy::connect::Connect, rt::TokioIo, server::conn::auto::Builder as ConnectionBuilder};
use std::sync::Arc;
use tokio::time::{timeout, Duration};
impl<U> Proxy<U>
where
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
// TCP Listener Service, i.e., http/2 and http/1.1
async fn listener_service(&self, mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>) -> Result<()> {
let tcp_socket = bind_tcp_socket(&self.listening_on)?;
let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?;
info!("Start TCP proxy serving with HTTPS request for configured host names");
let mut server_crypto_map: Option<Arc<SniServerCryptoMap>> = None;
loop {
tokio::select! {
tcp_cnx = tcp_listener.accept() => {
if tcp_cnx.is_err() || server_crypto_map.is_none() {
continue;
}
let (raw_stream, client_addr) = tcp_cnx.unwrap();
let sc_map_inner = server_crypto_map.clone();
let self_inner = self.clone();
// spawns async handshake to avoid blocking thread by sequential handshake.
let handshake_fut = async move {
let acceptor = tokio_rustls::LazyConfigAcceptor::new(tokio_rustls::rustls::server::Acceptor::default(), raw_stream).await;
if let Err(e) = acceptor {
return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {e}")));
}
let start = acceptor.unwrap();
let client_hello = start.client_hello();
let server_name = client_hello.server_name();
debug!("HTTP/2 or 1.1: SNI in ClientHello: {:?}", server_name);
let server_name_in_bytes = server_name.map_or_else(|| None, |v| Some(v.to_server_name_vec()));
if server_name_in_bytes.is_none(){
return Err(RpxyError::Proxy("No SNI is given".to_string()));
}
let server_crypto = sc_map_inner.as_ref().unwrap().get(server_name_in_bytes.as_ref().unwrap());
if server_crypto.is_none() {
return Err(RpxyError::Proxy(format!("No TLS serving app for {:?}", server_name.unwrap())));
}
let stream = match start.into_stream(server_crypto.unwrap().clone()).await {
Ok(s) => TokioIo::new(s),
Err(e) => {
return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {e}")));
}
};
self_inner.serve_connection(stream, client_addr, server_name_in_bytes);
Ok(())
};
self.globals.runtime_handle.spawn( async move {
// timeout is introduced to avoid get stuck here.
let Ok(v) = timeout(
Duration::from_secs(TLS_HANDSHAKE_TIMEOUT_SEC),
handshake_fut
).await else {
error!("Timeout to handshake TLS");
return;
};
if let Err(e) = v {
error!("{}", e);
}
});
}
_ = server_crypto_rx.changed() => {
if server_crypto_rx.borrow().is_none() {
error!("Reloader is broken");
break;
}
let cert_keys_map = server_crypto_rx.borrow().clone().unwrap();
let Some(server_crypto): Option<Arc<ServerCrypto>> = (&cert_keys_map).try_into().ok() else {
error!("Failed to update server crypto");
break;
};
server_crypto_map = Some(server_crypto.inner_local_map.clone());
}
else => break
}
}
Ok(()) as Result<()>
}
pub async fn start_with_tls(&self) -> Result<()> {
let (cert_reloader_service, cert_reloader_rx) = ReloaderService::<CryptoReloader<U>, ServerCryptoBase>::new(
&self.globals.clone(),
CERTS_WATCH_DELAY_SECS,
!LOAD_CERTS_ONLY_WHEN_UPDATED,
)
.await
.map_err(|e| anyhow::anyhow!(e))?;
#[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))]
{
tokio::select! {
_= cert_reloader_service.start() => {
error!("Cert service for TLS exited");
},
_ = self.listener_service(cert_reloader_rx) => {
error!("TCP proxy service for TLS exited");
},
else => {
error!("Something went wrong");
return Ok(())
}
};
Ok(())
}
#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
{
if self.globals.proxy_config.http3 {
tokio::select! {
_= cert_reloader_service.start() => {
error!("Cert service for TLS exited");
},
_ = self.listener_service(cert_reloader_rx.clone()) => {
error!("TCP proxy service for TLS exited");
},
_= self.listener_service_h3(cert_reloader_rx) => {
error!("UDP proxy service for QUIC exited");
},
else => {
error!("Something went wrong");
return Ok(())
}
};
Ok(())
} else {
tokio::select! {
_= cert_reloader_service.start() => {
error!("Cert service for TLS exited");
},
_ = self.listener_service(cert_reloader_rx) => {
error!("TCP proxy service for TLS exited");
},
else => {
error!("Something went wrong");
return Ok(())
}
};
Ok(())
}
}
}
}

46
legacy-lib/src/proxy/socket.rs Normal file
View file

@ -0,0 +1,46 @@
use crate::{error::*, log::*};
#[cfg(feature = "http3-quinn")]
use socket2::{Domain, Protocol, Socket, Type};
use std::net::SocketAddr;
#[cfg(feature = "http3-quinn")]
use std::net::UdpSocket;
use tokio::net::TcpSocket;
/// Bind TCP socket to the given `SocketAddr`, and returns the TCP socket with `SO_REUSEADDR` and `SO_REUSEPORT` options.
/// This option is required to re-bind the socket address when the proxy instance is reconstructed.
pub(super) fn bind_tcp_socket(listening_on: &SocketAddr) -> Result<TcpSocket> {
let tcp_socket = if listening_on.is_ipv6() {
TcpSocket::new_v6()
} else {
TcpSocket::new_v4()
}?;
tcp_socket.set_reuseaddr(true)?;
tcp_socket.set_reuseport(true)?;
if let Err(e) = tcp_socket.bind(*listening_on) {
error!("Failed to bind TCP socket: {}", e);
return Err(RpxyError::Io(e));
};
Ok(tcp_socket)
}
#[cfg(feature = "http3-quinn")]
/// Bind UDP socket to the given `SocketAddr`, and returns the UDP socket with `SO_REUSEADDR` and `SO_REUSEPORT` options.
/// This option is required to re-bind the socket address when the proxy instance is reconstructed.
pub(super) fn bind_udp_socket(listening_on: &SocketAddr) -> Result<UdpSocket> {
let socket = if listening_on.is_ipv6() {
Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP))
} else {
Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP))
}?;
socket.set_reuse_address(true)?; // This isn't necessary?
socket.set_reuse_port(true)?;
socket.set_nonblocking(true)?; // This was made true inside quinn. so this line isn't necessary here. but just in case.
if let Err(e) = socket.bind(&(*listening_on).into()) {
error!("Failed to bind UDP socket: {}", e);
return Err(RpxyError::Io(e));
};
let udp_socket: UdpSocket = socket.into();
Ok(udp_socket)
}