tuxmain/rust-rpxy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wip: implemented hyper-1.0 for http/1.1 and http/2. todo: http/3 and backend handler

Browse source
This commit is contained in:
Jun Kurihara 2023-11-18 14:42:13 +09:00
commit b639e79b4d
No known key found for this signature in database
GPG key ID: 48ADFD173ED22B03
24 changed files with 1134 additions and 1275 deletions
Download patch file Download diff file Expand all files Collapse all files

29
rpxy-lib/src/proxy/mod.rs
View file

@ -10,4 +10,33 @@ mod proxy_quic_s2n;
mod proxy_tls;
mod socket;
use crate::error::*;
use http::{Response, StatusCode};
use http_body_util::{combinators, BodyExt, Either, Empty};
use hyper::body::{Bytes, Incoming};
pub use proxy_main::{Proxy, ProxyBuilder, ProxyBuilderError};
/// Type for synthetic boxed body
type BoxBody = combinators::BoxBody<Bytes, hyper::Error>;
/// Type for either passthrough body or synthetic body
type EitherBody = Either<Incoming, BoxBody>;
/// helper function to build http response with passthrough body
fn passthrough_response(response: Response<Incoming>) -> Result<Response<EitherBody>> {
Ok(response.map(EitherBody::Left))
}
/// build http response with status code of 4xx and 5xx
fn synthetic_error_response(status_code: StatusCode) -> Result<Response<EitherBody>> {
let res = Response::builder()
.status(status_code)
.body(EitherBody::Right(BoxBody::new(empty())))
.unwrap();
Ok(res)
}
/// helper function to build a empty body
fn empty() -> BoxBody {
Empty::<Bytes>::new().map_err(|never| match never {}).boxed()
}

99
rpxy-lib/src/proxy/proxy_h3.rs
View file

@ -1,17 +1,21 @@
use super::Proxy;
use crate::{certs::CryptoSource, error::*, log::*, utils::ServerNameBytesExp};
use bytes::{Buf, Bytes};
use futures::Stream;
#[cfg(feature = "http3-quinn")]
use h3::{quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
use hyper::{client::connect::Connect, Body, Request, Response};
use http::{Request, Response};
use http_body_util::{BodyExt, BodyStream, StreamBody};
use hyper::body::{Body, Incoming};
use hyper_util::client::legacy::connect::Connect;
#[cfg(feature = "http3-s2n")]
use s2n_quic_h3::h3::{self, quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
use std::net::SocketAddr;
use tokio::time::{timeout, Duration};
impl<T, U> Proxy<T, U>
impl<U> Proxy<U>
where
T: Connect + Clone + Sync + Send + 'static,
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub(super) async fn connection_serve_h3<C>(
@ -89,18 +93,36 @@ where
S: BidiStream<Bytes> + Send + 'static,
<S as BidiStream<Bytes>>::RecvStream: Send,
{
println!("stream_serve_h3");
let (req_parts, _) = req.into_parts();
// split stream and async body handling
let (mut send_stream, mut recv_stream) = stream.split();
// generate streamed body with trailers using channel
let (body_sender, req_body) = Body::channel();
// let max_body_size = self.globals.proxy_config.h3_request_max_body_size;
// // let max = body_stream.size_hint().upper().unwrap_or(u64::MAX);
// // if max > max_body_size as u64 {
// // return Err(HttpError::TooLargeRequestBody);
// // }
// let new_req = Request::from_parts(req_parts, body_stream);
////////////////////
// TODO: TODO: TODO: TODO:
// TODO: Body in hyper-0.14 was changed to Incoming in hyper-1.0, and it is not accessible from outside.
// Thus, we need to implement IncomingLike trait using channel. Also, the backend handler must feed the body in the form of
// Either<Incoming, IncomingLike> as body.
// Also, the downstream from the backend handler could be Incoming, but will be wrapped as Either<Incoming, ()/Empty> as well due to H3.
// Result<Either<_,_>, E> type includes E as HttpError to generate the status code and related Response<BoxBody>.
// Thus to handle synthetic error messages in BoxBody, the serve() function outputs Response<Either<Either<Incoming, ()/Empty>, BoxBody>>>.
////////////////////
// // generate streamed body with trailers using channel
// let (body_sender, req_body) = Incoming::channel();
// Buffering and sending body through channel for protocol conversion like h3 -> h2/http1.1
// The underling buffering, i.e., buffer given by the API recv_data.await?, is handled by quinn.
let max_body_size = self.globals.proxy_config.h3_request_max_body_size;
self.globals.runtime_handle.spawn(async move {
let mut sender = body_sender;
// let mut sender = body_sender;
let mut size = 0usize;
while let Some(mut body) = recv_stream.recv_data().await? {
debug!("HTTP/3 incoming request body: remaining {}", body.remaining());
@ -113,51 +135,52 @@ where
return Err(RpxyError::Proxy("Exceeds max request body size for HTTP/3".to_string()));
}
// create stream body to save memory, shallow copy (increment of ref-count) to Bytes using copy_to_bytes
sender.send_data(body.copy_to_bytes(body.remaining())).await?;
// sender.send_data(body.copy_to_bytes(body.remaining())).await?;
}
// trailers: use inner for work around. (directly get trailer)
let trailers = recv_stream.as_mut().recv_trailers().await?;
if trailers.is_some() {
debug!("HTTP/3 incoming request trailers");
sender.send_trailers(trailers.unwrap()).await?;
// sender.send_trailers(trailers.unwrap()).await?;
}
Ok(())
});
let new_req: Request<Body> = Request::from_parts(req_parts, req_body);
let res = self
.msg_handler
.clone()
.handle_request(
new_req,
client_addr,
self.listening_on,
self.tls_enabled,
Some(tls_server_name),
)
.await?;
// let new_req: Request<Incoming> = Request::from_parts(req_parts, req_body);
// let res = self
// .msg_handler
// .clone()
// .handle_request(
// new_req,
// client_addr,
// self.listening_on,
// self.tls_enabled,
// Some(tls_server_name),
// )
// .await?;
let (new_res_parts, new_body) = res.into_parts();
let new_res = Response::from_parts(new_res_parts, ());
// let (new_res_parts, new_body) = res.into_parts();
// let new_res = Response::from_parts(new_res_parts, ());
match send_stream.send_response(new_res).await {
Ok(_) => {
debug!("HTTP/3 response to connection successful");
// aggregate body without copying
let mut body_data = hyper::body::aggregate(new_body).await?;
// match send_stream.send_response(new_res).await {
// Ok(_) => {
// debug!("HTTP/3 response to connection successful");
// // aggregate body without copying
// let body_data = new_body.collect().await?.aggregate();
// create stream body to save memory, shallow copy (increment of ref-count) to Bytes using copy_to_bytes
send_stream
.send_data(body_data.copy_to_bytes(body_data.remaining()))
.await?;
// // create stream body to save memory, shallow copy (increment of ref-count) to Bytes using copy_to_bytes
// send_stream
// .send_data(body_data.copy_to_bytes(body_data.remaining()))
// .await?;
// TODO: needs handling trailer? should be included in body from handler.
}
Err(err) => {
error!("Unable to send response to connection peer: {:?}", err);
}
}
Ok(send_stream.finish().await?)
// // TODO: needs handling trailer? should be included in body from handler.
// }
// Err(err) => {
// error!("Unable to send response to connection peer: {:?}", err);
// }
// }
// Ok(send_stream.finish().await?)
todo!()
}
}

160
rpxy-lib/src/proxy/proxy_main.rs
View file

@ -1,78 +1,70 @@
use super::socket::bind_tcp_socket;
use super::{passthrough_response, socket::bind_tcp_socket, synthetic_error_response, EitherBody};
use crate::{
certs::CryptoSource, error::*, globals::Globals, handler::HttpMessageHandler, log::*, utils::ServerNameBytesExp,
certs::CryptoSource, error::*, globals::Globals, handler::HttpMessageHandler, hyper_executor::LocalExecutor, log::*,
utils::ServerNameBytesExp,
};
use derive_builder::{self, Builder};
use hyper::{client::connect::Connect, server::conn::Http, service::service_fn, Body, Request};
use std::{net::SocketAddr, sync::Arc};
use tokio::{
io::{AsyncRead, AsyncWrite},
runtime::Handle,
sync::Notify,
time::{timeout, Duration},
use http::{Request, StatusCode};
use hyper::{
body::Incoming,
rt::{Read, Write},
service::service_fn,
};
#[derive(Clone)]
pub struct LocalExecutor {
runtime_handle: Handle,
}
impl LocalExecutor {
fn new(runtime_handle: Handle) -> Self {
LocalExecutor { runtime_handle }
}
}
impl<F> hyper::rt::Executor<F> for LocalExecutor
where
F: std::future::Future + Send + 'static,
F::Output: Send,
{
fn execute(&self, fut: F) {
self.runtime_handle.spawn(fut);
}
}
use hyper_util::{client::legacy::connect::Connect, rt::TokioIo, server::conn::auto::Builder as ConnectionBuilder};
use std::{net::SocketAddr, sync::Arc};
use tokio::time::{timeout, Duration};
#[derive(Clone, Builder)]
pub struct Proxy<T, U>
/// Proxy main object
pub struct Proxy<U>
where
T: Connect + Clone + Sync + Send + 'static,
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub listening_on: SocketAddr,
pub tls_enabled: bool, // TCP待受がTLSかどうか
pub msg_handler: Arc<HttpMessageHandler<T, U>>,
/// hyper server receiving http request
pub http_server: Arc<ConnectionBuilder<LocalExecutor>>,
// pub msg_handler: Arc<HttpMessageHandler<U>>,
pub msg_handler: Arc<HttpMessageHandler<U>>,
pub globals: Arc<Globals<U>>,
}
impl<T, U> Proxy<T, U>
/// Wrapper function to handle request
async fn serve_request<U>(
req: Request<Incoming>,
// handler: Arc<HttpMessageHandler<T, U>>,
handler: Arc<HttpMessageHandler<U>>,
client_addr: SocketAddr,
listen_addr: SocketAddr,
tls_enabled: bool,
tls_server_name: Option<ServerNameBytesExp>,
) -> Result<hyper::Response<EitherBody>>
where
T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
match handler
.handle_request(req, client_addr, listen_addr, tls_enabled, tls_server_name)
.await?
{
Ok(res) => passthrough_response(res),
Err(e) => synthetic_error_response(StatusCode::from(e)),
}
}
impl<U> Proxy<U>
where
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send,
{
/// Wrapper function to handle request
async fn serve(
handler: Arc<HttpMessageHandler<T, U>>,
req: Request<Body>,
client_addr: SocketAddr,
listen_addr: SocketAddr,
tls_enabled: bool,
tls_server_name: Option<ServerNameBytesExp>,
) -> Result<hyper::Response<Body>> {
handler
.handle_request(req, client_addr, listen_addr, tls_enabled, tls_server_name)
.await
}
/// Serves requests from clients
pub(super) fn client_serve<I>(
self,
pub(super) fn serve_connection<I>(
&self,
stream: I,
server: Http<LocalExecutor>,
peer_addr: SocketAddr,
tls_server_name: Option<ServerNameBytesExp>,
) where
I: AsyncRead + AsyncWrite + Send + Unpin + 'static,
I: Read + Write + Send + Unpin + 'static,
{
let request_count = self.globals.request_count.clone();
if request_count.increment() > self.globals.proxy_config.max_clients {
@ -81,24 +73,27 @@ where
}
debug!("Request incoming: current # {}", request_count.current());
let server_clone = self.http_server.clone();
let msg_handler_clone = self.msg_handler.clone();
let timeout_sec = self.globals.proxy_config.proxy_timeout;
let tls_enabled = self.tls_enabled;
let listening_on = self.listening_on;
self.globals.runtime_handle.clone().spawn(async move {
timeout(
self.globals.proxy_config.proxy_timeout + Duration::from_secs(1),
server
.serve_connection(
stream,
service_fn(move |req: Request<Body>| {
Self::serve(
self.msg_handler.clone(),
req,
peer_addr,
self.listening_on,
self.tls_enabled,
tls_server_name.clone(),
)
}),
)
.with_upgrades(),
timeout_sec + Duration::from_secs(1),
server_clone.serve_connection_with_upgrades(
stream,
service_fn(move |req: Request<Incoming>| {
serve_request(
req,
msg_handler_clone.clone(),
peer_addr,
listening_on,
tls_enabled,
tls_server_name.clone(),
)
}),
),
)
.await
.ok();
@ -109,13 +104,13 @@ where
}
/// Start without TLS (HTTP cleartext)
async fn start_without_tls(self, server: Http<LocalExecutor>) -> Result<()> {
async fn start_without_tls(&self) -> Result<()> {
let listener_service = async {
let tcp_socket = bind_tcp_socket(&self.listening_on)?;
let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?;
info!("Start TCP proxy serving with HTTP request for configured host names");
while let Ok((stream, _client_addr)) = tcp_listener.accept().await {
self.clone().client_serve(stream, server.clone(), _client_addr, None);
while let Ok((stream, client_addr)) = tcp_listener.accept().await {
self.serve_connection(TokioIo::new(stream), client_addr, None);
}
Ok(()) as Result<()>
};
@ -124,32 +119,23 @@ where
}
/// Entrypoint for HTTP/1.1 and HTTP/2 servers
pub async fn start(self, term_notify: Option<Arc<Notify>>) -> Result<()> {
let mut server = Http::new();
server.http1_keep_alive(self.globals.proxy_config.keepalive);
server.http2_max_concurrent_streams(self.globals.proxy_config.max_concurrent_streams);
server.pipeline_flush(true);
let executor = LocalExecutor::new(self.globals.runtime_handle.clone());
let server = server.with_executor(executor);
let listening_on = self.listening_on;
pub async fn start(&self) -> Result<()> {
let proxy_service = async {
if self.tls_enabled {
self.start_with_tls(server).await
self.start_with_tls().await
} else {
self.start_without_tls(server).await
self.start_without_tls().await
}
};
match term_notify {
match &self.globals.term_notify {
Some(term) => {
tokio::select! {
_ = proxy_service => {
warn!("Proxy service got down");
}
_ = term.notified() => {
info!("Proxy service listening on {} receives term signal", listening_on);
info!("Proxy service listening on {} receives term signal", self.listening_on);
}
}
}
@ -159,8 +145,6 @@ where
}
}
// proxy_service.await?;
Ok(())
}
}

6
rpxy-lib/src/proxy/proxy_quic_quinn.rs
View file

@ -5,14 +5,14 @@ use super::{
};
use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName};
use hot_reload::ReloaderReceiver;
use hyper::client::connect::Connect;
use hyper_util::client::legacy::connect::Connect;
use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerConfig, TransportConfig};
use rustls::ServerConfig;
use std::sync::Arc;
impl<T, U> Proxy<T, U>
impl<U> Proxy<U>
where
T: Connect + Clone + Sync + Send + 'static,
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub(super) async fn listener_service_h3(

16
rpxy-lib/src/proxy/proxy_quic_s2n.rs
View file

@ -4,13 +4,13 @@ use super::{
};
use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName};
use hot_reload::ReloaderReceiver;
use hyper::client::connect::Connect;
use hyper_util::client::legacy::connect::Connect;
use s2n_quic::provider;
use std::sync::Arc;
impl<T, U> Proxy<T, U>
impl<U> Proxy<U>
where
T: Connect + Clone + Sync + Send + 'static,
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
pub(super) async fn listener_service_h3(
@ -29,7 +29,7 @@ where
// event loop
loop {
tokio::select! {
v = self.serve_connection(&server_crypto) => {
v = self.listener_service_h3_inner(&server_crypto) => {
if let Err(e) = v {
error!("Quic connection event loop illegally shutdown [s2n-quic] {e}");
break;
@ -64,7 +64,7 @@ where
})
}
async fn serve_connection(&self, server_crypto: &Option<Arc<ServerCrypto>>) -> Result<()> {
async fn listener_service_h3_inner(&self, server_crypto: &Option<Arc<ServerCrypto>>) -> Result<()> {
// setup UDP socket
let io = provider::io::tokio::Builder::default()
.with_receive_address(self.listening_on)?
@ -110,9 +110,9 @@ where
while let Some(new_conn) = server.accept().await {
debug!("New QUIC connection established");
let Ok(Some(new_server_name)) = new_conn.server_name() else {
warn!("HTTP/3 no SNI is given");
continue;
};
warn!("HTTP/3 no SNI is given");
continue;
};
debug!("HTTP/3 connection incoming (SNI {:?})", new_server_name);
let self_clone = self.clone();

44
rpxy-lib/src/proxy/proxy_tls.rs
View file

@ -1,25 +1,21 @@
use super::{
crypto_service::{CryptoReloader, ServerCrypto, ServerCryptoBase, SniServerCryptoMap},
proxy_main::{LocalExecutor, Proxy},
proxy_main::Proxy,
socket::bind_tcp_socket,
};
use crate::{certs::CryptoSource, constants::*, error::*, log::*, utils::BytesName};
use hot_reload::{ReloaderReceiver, ReloaderService};
use hyper::{client::connect::Connect, server::conn::Http};
use hyper_util::{client::legacy::connect::Connect, rt::TokioIo, server::conn::auto::Builder as ConnectionBuilder};
use std::sync::Arc;
use tokio::time::{timeout, Duration};
impl<T, U> Proxy<T, U>
impl<U> Proxy<U>
where
T: Connect + Clone + Sync + Send + 'static,
// T: Connect + Clone + Sync + Send + 'static,
U: CryptoSource + Clone + Sync + Send + 'static,
{
// TCP Listener Service, i.e., http/2 and http/1.1
async fn listener_service(
&self,
server: Http<LocalExecutor>,
mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
) -> Result<()> {
async fn listener_service(&self, mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>) -> Result<()> {
let tcp_socket = bind_tcp_socket(&self.listening_on)?;
let tcp_listener = tcp_socket.listen(self.globals.proxy_config.tcp_listen_backlog)?;
info!("Start TCP proxy serving with HTTPS request for configured host names");
@ -33,7 +29,6 @@ where
}
let (raw_stream, client_addr) = tcp_cnx.unwrap();
let sc_map_inner = server_crypto_map.clone();
let server_clone = server.clone();
let self_inner = self.clone();
// spawns async handshake to avoid blocking thread by sequential handshake.
@ -55,30 +50,27 @@ where
return Err(RpxyError::Proxy(format!("No TLS serving app for {:?}", server_name.unwrap())));
}
let stream = match start.into_stream(server_crypto.unwrap().clone()).await {
Ok(s) => s,
Ok(s) => TokioIo::new(s),
Err(e) => {
return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {e}")));
}
};
self_inner.client_serve(stream, server_clone, client_addr, server_name_in_bytes);
self_inner.serve_connection(stream, client_addr, server_name_in_bytes);
Ok(())
};
self.globals.runtime_handle.spawn( async move {
// timeout is introduced to avoid get stuck here.
match timeout(
let Ok(v) = timeout(
Duration::from_secs(TLS_HANDSHAKE_TIMEOUT_SEC),
handshake_fut
).await {
Ok(a) => {
if let Err(e) = a {
error!("{}", e);
}
},
Err(e) => {
error!("Timeout to handshake TLS: {}", e);
}
).await else {
error!("Timeout to handshake TLS");
return;
};
if let Err(e) = v {
error!("{}", e);
}
});
}
_ = server_crypto_rx.changed() => {
@ -99,7 +91,7 @@ where
Ok(()) as Result<()>
}
pub async fn start_with_tls(self, server: Http<LocalExecutor>) -> Result<()> {
pub async fn start_with_tls(&self) -> Result<()> {
let (cert_reloader_service, cert_reloader_rx) = ReloaderService::<CryptoReloader<U>, ServerCryptoBase>::new(
&self.globals.clone(),
CERTS_WATCH_DELAY_SECS,
@ -114,7 +106,7 @@ where
_= cert_reloader_service.start() => {
error!("Cert service for TLS exited");
},
_ = self.listener_service(server, cert_reloader_rx) => {
_ = self.listener_service(cert_reloader_rx) => {
error!("TCP proxy service for TLS exited");
},
else => {
@ -131,7 +123,7 @@ where
_= cert_reloader_service.start() => {
error!("Cert service for TLS exited");
},
_ = self.listener_service(server, cert_reloader_rx.clone()) => {
_ = self.listener_service(cert_reloader_rx.clone()) => {
error!("TCP proxy service for TLS exited");
},
_= self.listener_service_h3(cert_reloader_rx) => {
@ -148,7 +140,7 @@ where
_= cert_reloader_service.start() => {
error!("Cert service for TLS exited");
},
_ = self.listener_service(server, cert_reloader_rx) => {
_ = self.listener_service(cert_reloader_rx) => {
error!("TCP proxy service for TLS exited");
},
else => {