update

bench/bench.sh

@@ -18,11 +18,17 @@ echo "Benchmark on rpxy"
 #wrk -t8 -c100 -d30s http://127.0.0.1:8080/index.html
 rewrk -c 256 -t 4 -d 10s -h http://localhost:8080 --pct
 
+echo "sleep 3 secs"
+sleep 3
+
 echo "----------------------------"
 echo "Benchmark on nginx"
 # wrk -t8 -c100 -d30s http://127.0.0.1:8090/index.html
 rewrk -c 256 -t 4 -d 10s -h http://localhost:8090 --pct
 
+echo "sleep 3 secs"
+sleep 3
+
 echo "----------------------------"
 echo "Benchmark on caddy"
 # wrk -t8 -c100 -d30s http://127.0.0.1:8100/index.html

bench/docker-compose.yml

@@ -22,6 +22,7 @@ services:
   rpxy-rp:
     image: jqtype/rpxy
     container_name: proxy-rpxy
+    pull_policy: never
     build:
       context: ../
     restart: unless-stopped
@@ -45,8 +46,8 @@ services:
     tty: false
     privileged: true
     volumes:
-      - ./nginx_data/vhost:/etc/nginx/vhost.d
+      - ./nginx_data/vhost:/etc/nginx/vhost.d:ro
-      - ./nginx_data/conf:/etc/nginx/conf.d/
+      - ./nginx_data/conf:/etc/nginx/conf.d/:ro
       - /var/run/docker.sock:/tmp/docker.sock:ro
     logging:
       options:

bench/nginx_data/conf/default.conf

@@ -1,63 +1,63 @@
-# nginx-proxy version : 1.0.1-6-gc4ad18f
+# # nginx-proxy version : 1.0.1-6-gc4ad18f
-# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the
+# # If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the
-# scheme used to connect to this server
+# # scheme used to connect to this server
-map $http_x_forwarded_proto $proxy_x_forwarded_proto {
+# map $http_x_forwarded_proto $proxy_x_forwarded_proto {
-  default $http_x_forwarded_proto;
+#   default $http_x_forwarded_proto;
-  ''      $scheme;
+#   ''      $scheme;
-}
+# }
-# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the
+# # If we receive X-Forwarded-Port, pass it through; otherwise, pass along the
-# server port the client connected to
+# # server port the client connected to
-map $http_x_forwarded_port $proxy_x_forwarded_port {
+# map $http_x_forwarded_port $proxy_x_forwarded_port {
-  default $http_x_forwarded_port;
+#   default $http_x_forwarded_port;
-  ''      $server_port;
+#   ''      $server_port;
-}
+# }
-# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any
+# # If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any
-# Connection header that may have been passed to this server
+# # Connection header that may have been passed to this server
-map $http_upgrade $proxy_connection {
+# map $http_upgrade $proxy_connection {
-  default upgrade;
+#   default upgrade;
-  '' close;
+#   '' close;
-}
+# }
-# Apply fix for very long server names
+# # Apply fix for very long server names
-server_names_hash_bucket_size 128;
+# server_names_hash_bucket_size 128;
-# Default dhparam
+# # Default dhparam
-ssl_dhparam /etc/nginx/dhparam/dhparam.pem;
+# ssl_dhparam /etc/nginx/dhparam/dhparam.pem;
-# Set appropriate X-Forwarded-Ssl header based on $proxy_x_forwarded_proto
+# # Set appropriate X-Forwarded-Ssl header based on $proxy_x_forwarded_proto
-map $proxy_x_forwarded_proto $proxy_x_forwarded_ssl {
+# map $proxy_x_forwarded_proto $proxy_x_forwarded_ssl {
-  default off;
+#   default off;
-  https on;
+#   https on;
-}
+# }
-gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+# gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
-log_format vhost '$host $remote_addr - $remote_user [$time_local] '
+# log_format vhost '$host $remote_addr - $remote_user [$time_local] '
-                 '"$request" $status $body_bytes_sent '
+#                  '"$request" $status $body_bytes_sent '
-                 '"$http_referer" "$http_user_agent" '
+#                  '"$http_referer" "$http_user_agent" '
-                 '"$upstream_addr"';
+#                  '"$upstream_addr"';
-access_log off;
+# access_log off;
-		ssl_protocols TLSv1.2 TLSv1.3;
+# 		ssl_protocols TLSv1.2 TLSv1.3;
-		ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384';
+# 		ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384';
-		ssl_prefer_server_ciphers off;
+# 		ssl_prefer_server_ciphers off;
-error_log /dev/stderr;
+# error_log /dev/stderr;
-resolver 127.0.0.11;
+# resolver 127.0.0.11;
-# HTTP 1.1 support
+# # HTTP 1.1 support
-proxy_http_version 1.1;
+# proxy_http_version 1.1;
-proxy_buffering off;
+# proxy_buffering off;
-proxy_set_header Host $http_host;
+# proxy_set_header Host $http_host;
-proxy_set_header Upgrade $http_upgrade;
+# proxy_set_header Upgrade $http_upgrade;
-proxy_set_header Connection $proxy_connection;
+# proxy_set_header Connection $proxy_connection;
-proxy_set_header X-Real-IP $remote_addr;
+# proxy_set_header X-Real-IP $remote_addr;
-proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
+# proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
-proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
+# proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
-proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
+# proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
-proxy_set_header X-Original-URI $request_uri;
+# proxy_set_header X-Original-URI $request_uri;
 # Mitigate httpoxy attack (see README for details)
-proxy_set_header Proxy "";
+# proxy_set_header Proxy "";
-server {
+# server {
-	server_name _; # This is just an invalid value which will never trigger on a real hostname.
+# 	server_name _; # This is just an invalid value which will never trigger on a real hostname.
-	server_tokens off;
+# 	server_tokens off;
-	listen 80;
+# 	listen 80;
-	access_log /var/log/nginx/access.log vhost;
+# 	# access_log /var/log/nginx/access.log vhost;
-	return 503;
+# 	return 503;
-}
+# }
 	# localhost
 upstream localhost {
         ## Can be connected with "bench-nw" network
@@ -67,7 +67,7 @@ upstream localhost {
 server {
 	server_name localhost;
 	listen 80 ;
-	access_log /var/log/nginx/access.log vhost;
+	# access_log /var/log/nginx/access.log vhost;
 	location / {
 		proxy_pass http://localhost;
 }

src/constants.rs

@@ -11,3 +11,9 @@ pub const CERTS_WATCH_DELAY_SECS: u32 = 30;
 
 #[cfg(feature = "h3")]
 pub const H3_ALT_SVC_MAX_AGE: u32 = 3600;
+
+#[cfg(feature = "h3")]
+pub const H3_RESPONSE_BUF_SIZE: usize = 65_536; // 64KB
+
+#[cfg(feature = "h3")]
+pub const H3_REQUEST_MAX_BODY_SIZE: usize = 268_435_456; // 256MB

src/proxy/proxy_h3.rs

@@ -1,9 +1,9 @@
 use super::Proxy;
-use crate::{backend::ServerNameLC, error::*, log::*};
+use crate::{backend::ServerNameLC, constants::*, error::*, log::*};
-use bytes::{Buf, Bytes};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
 use h3::{quic::BidiStream, server::RequestStream};
 use hyper::{client::connect::Connect, Body, Request, Response};
-use std::net::SocketAddr;
+use std::{io::Read, net::SocketAddr};
 use tokio::time::{timeout, Duration};
 
 impl<T> Proxy<T>
@@ -103,20 +103,24 @@ where
   {
     let (req_parts, _) = req.into_parts();
 
-    // TODO: h3 -> h2/http1.1等のプロトコル変換のため、一旦バッファリング。
+    // TODO: h3 -> h2/http1.1等のプロトコル変換のため、一旦全部バッファリングしないと無理そう。H3->H3ならBytesを直に流し込めるのだが。
-    // 本来はbodyは直でstreamでcopy_bidirectionalしてして転送した方がいい。やむなし。
+    let mut body_buf = BytesMut::new();
-    let mut body_chunk: Vec<u8> = Vec::new();
+    while let Some(chunk) = stream.recv_data().await? {
-    while let Some(request_body) = stream.recv_data().await? {
       debug!("HTTP/3 request body");
-      body_chunk.extend_from_slice(request_body.chunk());
+      if body_buf.len() + chunk.remaining() > H3_REQUEST_MAX_BODY_SIZE {
+        error!("Exceeds max request body size for HTTP/3");
+        return Err(anyhow!("Exceeds max request body size for HTTP/3"));
+      }
+      body_buf.put(chunk);
     }
     // trailers
     let trailers = stream.recv_trailers().await?;
+
     // generate streamed body with trailers using channel
     let (body_sender, req_body) = Body::channel();
     self.globals.runtime_handle.spawn(async move {
       let mut sender = body_sender;
-      sender.send_data(Bytes::from(body_chunk)).await?;
+      sender.send_data(body_buf.freeze()).await?;
       if trailers.is_some() {
         debug!("HTTP/3 request with trailers");
         sender.send_trailers(trailers.unwrap()).await?;
@@ -143,15 +147,19 @@ where
     match stream.send_response(new_res).await {
       Ok(_) => {
         debug!("HTTP/3 response to connection successful");
-        // loop {
+        let body_data = hyper::body::aggregate(new_body).await?; // aggregate body without copying
-        //   let mut buf = BytesMut::with_capacity(4096 * 10);
+        let mut reader = body_data.reader();
-        //   if file.read_buf(&mut buf).await? == 0 {
+        let mut buf = [0u8; H3_RESPONSE_BUF_SIZE];
-        //     break;
+        loop {
-        //   }
+          let num = reader.read(&mut buf)?;
-        //   stream.send_data(buf.freeze()).await?;
+          if num == 0 {
-        // }
+            break;
-        let data = hyper::body::to_bytes(new_body).await?;
+          }
-        stream.send_data(data).await?;
+          stream
+            .send_data(Bytes::copy_from_slice(&buf[..num]))
+            .await?;
+        }
+        // TODO: needs handling trailer? should be included in body from handler.
       }
       Err(err) => {
         error!("Unable to send response to connection peer: {:?}", err);