diff --git a/bench/bench.sh b/bench/bench.sh index b64ee60..70a8afc 100644 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -18,11 +18,17 @@ echo "Benchmark on rpxy" #wrk -t8 -c100 -d30s http://127.0.0.1:8080/index.html rewrk -c 256 -t 4 -d 10s -h http://localhost:8080 --pct +echo "sleep 3 secs" +sleep 3 + echo "----------------------------" echo "Benchmark on nginx" # wrk -t8 -c100 -d30s http://127.0.0.1:8090/index.html rewrk -c 256 -t 4 -d 10s -h http://localhost:8090 --pct +echo "sleep 3 secs" +sleep 3 + echo "----------------------------" echo "Benchmark on caddy" # wrk -t8 -c100 -d30s http://127.0.0.1:8100/index.html diff --git a/bench/docker-compose.yml b/bench/docker-compose.yml index e7bf65d..391b539 100644 --- a/bench/docker-compose.yml +++ b/bench/docker-compose.yml @@ -22,6 +22,7 @@ services: rpxy-rp: image: jqtype/rpxy container_name: proxy-rpxy + pull_policy: never build: context: ../ restart: unless-stopped @@ -45,8 +46,8 @@ services: tty: false privileged: true volumes: - - ./nginx_data/vhost:/etc/nginx/vhost.d - - ./nginx_data/conf:/etc/nginx/conf.d/ + - ./nginx_data/vhost:/etc/nginx/vhost.d:ro + - ./nginx_data/conf:/etc/nginx/conf.d/:ro - /var/run/docker.sock:/tmp/docker.sock:ro logging: options: diff --git a/bench/nginx_data/conf/default.conf b/bench/nginx_data/conf/default.conf index bea7360..0359ac6 100644 --- a/bench/nginx_data/conf/default.conf +++ b/bench/nginx_data/conf/default.conf @@ -1,63 +1,63 @@ -# nginx-proxy version : 1.0.1-6-gc4ad18f -# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the -# scheme used to connect to this server -map $http_x_forwarded_proto $proxy_x_forwarded_proto { - default $http_x_forwarded_proto; - '' $scheme; -} -# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the -# server port the client connected to -map $http_x_forwarded_port $proxy_x_forwarded_port { - default $http_x_forwarded_port; - '' $server_port; -} -# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any -# Connection header that may have been passed to this server -map $http_upgrade $proxy_connection { - default upgrade; - '' close; -} -# Apply fix for very long server names -server_names_hash_bucket_size 128; -# Default dhparam -ssl_dhparam /etc/nginx/dhparam/dhparam.pem; -# Set appropriate X-Forwarded-Ssl header based on $proxy_x_forwarded_proto -map $proxy_x_forwarded_proto $proxy_x_forwarded_ssl { - default off; - https on; -} -gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; -log_format vhost '$host $remote_addr - $remote_user [$time_local] ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent" ' - '"$upstream_addr"'; -access_log off; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384'; - ssl_prefer_server_ciphers off; -error_log /dev/stderr; -resolver 127.0.0.11; -# HTTP 1.1 support -proxy_http_version 1.1; -proxy_buffering off; -proxy_set_header Host $http_host; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection $proxy_connection; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; -proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; -proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; -proxy_set_header X-Original-URI $request_uri; +# # nginx-proxy version : 1.0.1-6-gc4ad18f +# # If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the +# # scheme used to connect to this server +# map $http_x_forwarded_proto $proxy_x_forwarded_proto { +# default $http_x_forwarded_proto; +# '' $scheme; +# } +# # If we receive X-Forwarded-Port, pass it through; otherwise, pass along the +# # server port the client connected to +# map $http_x_forwarded_port $proxy_x_forwarded_port { +# default $http_x_forwarded_port; +# '' $server_port; +# } +# # If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any +# # Connection header that may have been passed to this server +# map $http_upgrade $proxy_connection { +# default upgrade; +# '' close; +# } +# # Apply fix for very long server names +# server_names_hash_bucket_size 128; +# # Default dhparam +# ssl_dhparam /etc/nginx/dhparam/dhparam.pem; +# # Set appropriate X-Forwarded-Ssl header based on $proxy_x_forwarded_proto +# map $proxy_x_forwarded_proto $proxy_x_forwarded_ssl { +# default off; +# https on; +# } +# gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; +# log_format vhost '$host $remote_addr - $remote_user [$time_local] ' +# '"$request" $status $body_bytes_sent ' +# '"$http_referer" "$http_user_agent" ' +# '"$upstream_addr"'; +# access_log off; +# ssl_protocols TLSv1.2 TLSv1.3; +# ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384'; +# ssl_prefer_server_ciphers off; +# error_log /dev/stderr; +# resolver 127.0.0.11; +# # HTTP 1.1 support +# proxy_http_version 1.1; +# proxy_buffering off; +# proxy_set_header Host $http_host; +# proxy_set_header Upgrade $http_upgrade; +# proxy_set_header Connection $proxy_connection; +# proxy_set_header X-Real-IP $remote_addr; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; +# proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; +# proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; +# proxy_set_header X-Original-URI $request_uri; # Mitigate httpoxy attack (see README for details) -proxy_set_header Proxy ""; -server { - server_name _; # This is just an invalid value which will never trigger on a real hostname. - server_tokens off; - listen 80; - access_log /var/log/nginx/access.log vhost; - return 503; -} +# proxy_set_header Proxy ""; +# server { +# server_name _; # This is just an invalid value which will never trigger on a real hostname. +# server_tokens off; +# listen 80; +# # access_log /var/log/nginx/access.log vhost; +# return 503; +# } # localhost upstream localhost { ## Can be connected with "bench-nw" network @@ -67,7 +67,7 @@ upstream localhost { server { server_name localhost; listen 80 ; - access_log /var/log/nginx/access.log vhost; + # access_log /var/log/nginx/access.log vhost; location / { proxy_pass http://localhost; } diff --git a/src/constants.rs b/src/constants.rs index 1861150..64a9b0e 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -11,3 +11,9 @@ pub const CERTS_WATCH_DELAY_SECS: u32 = 30; #[cfg(feature = "h3")] pub const H3_ALT_SVC_MAX_AGE: u32 = 3600; + +#[cfg(feature = "h3")] +pub const H3_RESPONSE_BUF_SIZE: usize = 65_536; // 64KB + +#[cfg(feature = "h3")] +pub const H3_REQUEST_MAX_BODY_SIZE: usize = 268_435_456; // 256MB diff --git a/src/proxy/proxy_h3.rs b/src/proxy/proxy_h3.rs index 087a69c..794aede 100644 --- a/src/proxy/proxy_h3.rs +++ b/src/proxy/proxy_h3.rs @@ -1,9 +1,9 @@ use super::Proxy; -use crate::{backend::ServerNameLC, error::*, log::*}; -use bytes::{Buf, Bytes}; +use crate::{backend::ServerNameLC, constants::*, error::*, log::*}; +use bytes::{Buf, BufMut, Bytes, BytesMut}; use h3::{quic::BidiStream, server::RequestStream}; use hyper::{client::connect::Connect, Body, Request, Response}; -use std::net::SocketAddr; +use std::{io::Read, net::SocketAddr}; use tokio::time::{timeout, Duration}; impl Proxy @@ -103,20 +103,24 @@ where { let (req_parts, _) = req.into_parts(); - // TODO: h3 -> h2/http1.1等のプロトコル変換のため、一旦バッファリング。 - // 本来はbodyは直でstreamでcopy_bidirectionalしてして転送した方がいい。やむなし。 - let mut body_chunk: Vec = Vec::new(); - while let Some(request_body) = stream.recv_data().await? { + // TODO: h3 -> h2/http1.1等のプロトコル変換のため、一旦全部バッファリングしないと無理そう。H3->H3ならBytesを直に流し込めるのだが。 + let mut body_buf = BytesMut::new(); + while let Some(chunk) = stream.recv_data().await? { debug!("HTTP/3 request body"); - body_chunk.extend_from_slice(request_body.chunk()); + if body_buf.len() + chunk.remaining() > H3_REQUEST_MAX_BODY_SIZE { + error!("Exceeds max request body size for HTTP/3"); + return Err(anyhow!("Exceeds max request body size for HTTP/3")); + } + body_buf.put(chunk); } // trailers let trailers = stream.recv_trailers().await?; + // generate streamed body with trailers using channel let (body_sender, req_body) = Body::channel(); self.globals.runtime_handle.spawn(async move { let mut sender = body_sender; - sender.send_data(Bytes::from(body_chunk)).await?; + sender.send_data(body_buf.freeze()).await?; if trailers.is_some() { debug!("HTTP/3 request with trailers"); sender.send_trailers(trailers.unwrap()).await?; @@ -143,15 +147,19 @@ where match stream.send_response(new_res).await { Ok(_) => { debug!("HTTP/3 response to connection successful"); - // loop { - // let mut buf = BytesMut::with_capacity(4096 * 10); - // if file.read_buf(&mut buf).await? == 0 { - // break; - // } - // stream.send_data(buf.freeze()).await?; - // } - let data = hyper::body::to_bytes(new_body).await?; - stream.send_data(data).await?; + let body_data = hyper::body::aggregate(new_body).await?; // aggregate body without copying + let mut reader = body_data.reader(); + let mut buf = [0u8; H3_RESPONSE_BUF_SIZE]; + loop { + let num = reader.read(&mut buf)?; + if num == 0 { + break; + } + stream + .send_data(Bytes::copy_from_slice(&buf[..num])) + .await?; + } + // TODO: needs handling trailer? should be included in body from handler. } Err(err) => { error!("Unable to send response to connection peer: {:?}", err);