Merge pull request #66 from junkurihara/develop

Closes #57
Release 0.5.0

.github/workflows/docker_build_push.yml

@@ -1,17 +1,49 @@
 name: Build and Publish Docker
-
 on:
   push:
     branches:
-      - main
+      - "develop"
-      - develop
+      - "main"
+  pull_request:
+    types: [synchronize, opened]
 
 env:
-  REGISTRY_IMAGE: jqtype/rpxy
+  GHCR: ghcr.io
+  GHCR_IMAGE_NAME: ${{ github.repository }}
+  DH_REGISTRY_NAME: jqtype/rpxy
 
 jobs:
   build_and_push:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - target: "default"
+            dockerfile: ./docker/Dockerfile
+            platforms: linux/amd64,linux/arm64
+
+          - target: "default-slim"
+            dockerfile: ./docker/Dockerfile-slim
+            build-contexts: |
+              messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl
+              messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl
+            platforms: linux/amd64,linux/arm64
+            tags-suffix: "-slim"
+            # Aliases must be used only for release builds
+            aliases: |
+              "slim"
+
+          - target: "s2n"
+            dockerfile: ./docker/Dockerfile
+            build-args: |
+              "CARGO_FEATURES=--no-default-features --features http3-s2n"
+              "ADDITIONAL_DEPS=pkg-config libssl-dev cmake libclang1 gcc g++"
+            platforms: linux/amd64,linux/arm64
+            tags-suffix: "-s2n"
+            # Aliases must be used only for release builds
+            aliases: |
+              "s2n"
 
     steps:
       - name: Checkout
@@ -19,14 +51,11 @@ jobs:
         with:
           submodules: recursive
 
-      - name: GitHub Environment
-        run: echo "BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
-
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.REGISTRY_IMAGE }}
+          images: ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -34,70 +63,65 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.GHCR }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Release build and push
+      - name: Nightly build test on amd64 for pull requests
-        if: ${{ env.BRANCH == 'main' }}
+        if: ${{ github.event_name == 'pull_request' }}
         uses: docker/build-push-action@v4
         with:
           context: .
-          push: true
+          build-args: ${{ matrix.build-args }}
-          tags: |
+          push: false
-            ${{ env.REGISTRY_IMAGE }}:latest
+          build-contexts: ${{ matrix.build-contexts }}
-          file: ./docker/Dockerfile
+          file: ${{ matrix.dockerfile }}
-          cache-from: type=gha
+          cache-from: type=gha,scope=rpxy-nightly-${{ matrix.target }}
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=max,scope=rpxy-nightly-${{ matrix.target }}
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           labels: ${{ steps.meta.outputs.labels }}
 
-      - name: Release build and push slim
+      - name: Nightly build and push from develop branch
-        if: ${{ env.BRANCH == 'main' }}
+        if: ${{ (github.ref_name == 'develop') && (github.event_name == 'push') }}
         uses: docker/build-push-action@v4
         with:
           context: .
+          build-args: ${{ matrix.build-args }}
           push: true
           tags: |
-            ${{ env.REGISTRY_IMAGE }}:slim, ${{ env.REGISTRY_IMAGE }}:latest-slim
+            ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}:nightly${{ matrix.tags-suffix }}
-          build-contexts: |
+            ${{ env.DH_REGISTRY_NAME }}:nightly${{ matrix.tags-suffix }}
-            messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl
+          build-contexts: ${{ matrix.build-contexts }}
-            messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl
+          file: ${{ matrix.dockerfile }}
-          file: ./docker/Dockerfile.slim
+          cache-from: type=gha,scope=rpxy-nightly-${{ matrix.target }}
-          cache-from: type=gha
+          cache-to: type=gha,mode=max,scope=rpxy-nightly-${{ matrix.target }}
-          cache-to: type=gha,mode=max
+          platforms: ${{ matrix.platforms }}
-          platforms: linux/amd64,linux/arm64
           labels: ${{ steps.meta.outputs.labels }}
 
-      - name: Nightly build and push
+      - name: Release build and push from main branch
-        if: ${{ env.BRANCH == 'develop' }}
+        if: ${{ (github.ref_name == 'main') && (github.event_name == 'push') }}
         uses: docker/build-push-action@v4
         with:
           context: .
+          build-args: ${{ matrix.build-args }}
           push: true
           tags: |
-            ${{ env.REGISTRY_IMAGE }}:nightly
+            ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}:latest${{ matrix.tags-suffix }}
-          file: ./docker/Dockerfile
+            ${{ env.DH_REGISTRY_NAME }}:latest${{ matrix.tags-suffix }}
-          cache-from: type=gha
+            ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}:${{ matrix.aliases }}
-          cache-to: type=gha,mode=max
+            ${{ env.DH_REGISTRY_NAME }}:${{ matrix.aliases }}
-          platforms: linux/amd64,linux/arm64
+          build-contexts: ${{ matrix.build-contexts }}
-          labels: ${{ steps.meta.outputs.labels }}
+          file: ${{ matrix.dockerfile }}
-
+          cache-from: type=gha,scope=rpxy-latest-${{ matrix.target }}
-      - name: Nightly build and push slim
+          cache-to: type=gha,mode=max,scope=rpxy-latest-${{ matrix.target }}
-        if: ${{ env.BRANCH == 'develop' }}
+          platforms: ${{ matrix.platforms }}
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          push: true
-          tags: |
-            ${{ env.REGISTRY_IMAGE }}:nightly-slim
-          build-contexts: |
-            messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl
-            messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl
-          file: ./docker/Dockerfile.slim
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          platforms: linux/amd64,linux/arm64
           labels: ${{ steps.meta.outputs.labels }}

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "quinn"]
 	path = quinn
 	url = git@github.com:junkurihara/quinn.git
+[submodule "s2n-quic"]
+	path = s2n-quic
+	url = git@github.com:junkurihara/s2n-quic.git

CHANGELOG.md

@@ -1,6 +1,17 @@
 # CHANGELOG
 
-## 0.4.0 (unreleased)
+## 0.6.0  (unreleased)
+
+## 0.5.0
+
+### Improvement
+
+- Feat: `s2n-quic` with `s2n-quic-h3` is supported as QUIC and HTTP/3 library in addition to `quinn` with `h3-quinn`, related to #57.
+- Feat: Publish dockerfile for `rpxy` with `s2n-quic` on both `amd64` and `arm64`.
+- Feat: Start to publish docker images on `ghcr.io`
+- Refactor: logs of minor improvements
+
+## 0.4.0
 
 ### Improvement
 

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace]
 
 members = ["rpxy-bin", "rpxy-lib"]
-exclude = ["quinn", "h3-quinn", "h3"]
+exclude = ["quinn", "h3-quinn", "h3", "s2n-quic"]
 
 [profile.release]
 codegen-units = 1

README.md

@@ -12,7 +12,9 @@
 
 `rpxy` [ahr-pik-see] is an implementation of simple and lightweight reverse-proxy with some additional features. The implementation is based on [`hyper`](https://github.com/hyperium/hyper), [`rustls`](https://github.com/rustls/rustls) and [`tokio`](https://github.com/tokio-rs/tokio), i.e., written in pure Rust. Our `rpxy` routes multiple host names to appropriate backend application servers while serving TLS connections.
 
- As default, `rpxy` provides the *TLS connection sanitization* by correctly binding a certificate used to establish a secure channel with the backend application. Specifically, it always keeps the consistency between the given SNI (server name indication) in `ClientHello` of the underlying TLS and the domain name given by the overlaid HTTP HOST header (or URL in Request line) [^1]. Additionally, as a somewhat unstable feature, our `rpxy` can handle the brand-new HTTP/3 connection thanks to [`quinn`](https://github.com/quinn-rs/quinn) and [`hyperium/h3`](https://github.com/hyperium/h3).
+ As default, `rpxy` provides the *TLS connection sanitization* by correctly binding a certificate used to establish a secure channel with the backend application. Specifically, it always keeps the consistency between the given SNI (server name indication) in `ClientHello` of the underlying TLS and the domain name given by the overlaid HTTP HOST header (or URL in Request line) [^1]. Additionally, as a somewhat unstable feature, our `rpxy` can handle the brand-new HTTP/3 connection thanks to [`quinn`](https://github.com/quinn-rs/quinn), [`s2n-quic`](https://github.com/aws/s2n-quic) and [`hyperium/h3`](https://github.com/hyperium/h3).[^h3lib]
+
+ [^h3lib]: HTTP/3 libraries are mutually exclusive. You need to explicitly specify `s2n-quic` with `--no-default-features` flag. Also note that if you build `rpxy` with `s2n-quic`, then it requires `openssl` just for building the package.
 
  This project is still *work-in-progress*. But it is already working in some production environments and serves a number of domain names. Furthermore it *significantly outperforms* NGINX and Caddy, e.g., *1.5x faster than NGINX*, in the setting of a very simple HTTP reverse-proxy scenario (See [`bench`](./bench/) directory).
 
@@ -27,11 +29,14 @@ You can build an executable binary yourself by checking out this Git repository.
 % git clone https://github.com/junkurihara/rust-rpxy
 % cd rust-rpxy
 
-# Update submodule hyperium/h3
+# Update submodules
 % git submodule update --init
 
-# Build
+# Build (default: QUIC and HTTP/3 is enabled using `quinn`)
 % cargo build --release
+
+# If you want to use `s2n-quic`, build as follows. You may need several additional dependencies.
+% cargo build --no-default-features --features http3-s2n --release
 ```
 
 Then you have an executive binary `rust-rpxy/target/release/rpxy`.
@@ -231,7 +236,9 @@ Since it is currently a work-in-progress project, we are frequently adding new o
 
 ## Using Docker Image
 
-You can also use [docker image](https://hub.docker.com/r/jqtype/rpxy) instead of directly executing the binary. There are only several docker-specific environment variables.
+You can also use `docker` image hosted on [Docker Hub](https://hub.docker.com/r/jqtype/rpxy) and [GitHub Container Registry](https://github.com/junkurihara/rust-rpxy/pkgs/container/rust-rpxy) instead of directly executing the binary. See [`./docker/README.md`](./docker/README.md) for the differences on image tags.
+
+There are only several docker-specific environment variables.
 
 - `HOST_USER` (default: `user`): User name executing `rpxy` inside the container.
 - `HOST_UID` (default: `900`): `UID` of `HOST_USER`.

TODO.md

@@ -1,5 +1,6 @@
 # TODO List
 
+- [Try in v0.6.0] **Cache option for the response with `Cache-Control: public` header directive ([#55](https://github.com/junkurihara/rust-rpxy/issues/55))**
 - Improvement of path matcher
 - More flexible option for rewriting path
 - Refactoring
@@ -10,14 +11,6 @@
     - upstream/upstream group: information on targeted destinations for each set of (a domain + a path)
     - load-balance: load balancing mod for a domain + path
 
-  - Done in v0.4.0:
-    ~~Split `rpxy` source codes into `rpxy-lib` and `rpxy-bin` to make the core part (reverse proxy) isolated from the misc part like toml file loader. This is in order to make the configuration-related part more flexible (related to [#33](https://github.com/junkurihara/rust-rpxy/issues/33))~~
-
-- Cache option for the response with `Cache-Control: public` header directive ([#55](https://github.com/junkurihara/rust-rpxy/issues/55))
-- Consideration on migrating from `quinn` and `h3-quinn` to other QUIC implementations ([#57](https://github.com/junkurihara/rust-rpxy/issues/57))
-- Done in v0.4.0:
-  ~~Benchmark with other reverse proxy implementations like Sozu ([#58](https://github.com/junkurihara/rust-rpxy/issues/58)) Currently, Sozu can work only on `amd64` format due to its HTTP message parser limitation... Since the main developer have only `arm64` (Apple M1) laptops, so we should do that on VPS?~~
-
 - Unit tests
 - Options to serve custom http_error page.
 - Prometheus metrics
@@ -30,4 +23,13 @@
 - Make the session-persistance option for load-balancing sophisticated. (mostly done in v0.3.0)
   - add option for sticky cookie name
   - add option for sticky cookie duration
+
+- Done in v0.5.0 ~~**Use `gchr.io`**~~
+- Done in v0.5.0:
+  ~~Consideration on migrating from `quinn` and `h3-quinn` to other QUIC implementations ([#57](https://github.com/junkurihara/rust-rpxy/issues/57))~~
+- Done in v0.4.0:
+  ~~Benchmark with other reverse proxy implementations like Sozu ([#58](https://github.com/junkurihara/rust-rpxy/issues/58)) Currently, Sozu can work only on `amd64` format due to its HTTP message parser limitation... Since the main developer have only `arm64` (Apple M1) laptops, so we should do that on VPS?~~
+- Done in v0.4.0:
+  ~~Split `rpxy` source codes into `rpxy-lib` and `rpxy-bin` to make the core part (reverse proxy) isolated from the misc part like toml file loader. This is in order to make the configuration-related part more flexible (related to [#33](https://github.com/junkurihara/rust-rpxy/issues/33))~~
+
 - etc.

docker/Dockerfile

@@ -9,12 +9,17 @@ FROM --platform=$BUILDPLATFORM base AS builder
 
 ENV CFLAGS=-Ofast
 ENV BUILD_DEPS curl make ca-certificates build-essential
+ENV TARGET_SUFFIX=unknown-linux-gnu
 
 WORKDIR /tmp
 
 COPY . /tmp/
 
 ARG TARGETARCH
+ARG CARGO_FEATURES
+ENV CARGO_FEATURES ${CARGO_FEATURES}
+ARG ADDITIONAL_DEPS
+ENV ADDITIONAL_DEPS ${ADDITIONAL_DEPS}
 
 RUN if [ $TARGETARCH = "amd64" ]; then \
   echo "x86_64" > /arch; \
@@ -29,15 +34,15 @@ ENV RUSTFLAGS "-C link-arg=-s"
 
 RUN update-ca-certificates 2> /dev/null || true
 
-RUN apt-get update && apt-get install -qy --no-install-recommends $BUILD_DEPS && \
+RUN apt-get update && apt-get install -qy --no-install-recommends $BUILD_DEPS ${ADDITIONAL_DEPS} && \
   curl -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain stable && \
   export PATH="$HOME/.cargo/bin:$PATH" && \
   echo "Install toolchain" && \
-  rustup target add $(cat /arch)-unknown-linux-gnu &&\
+  rustup target add $(cat /arch)-${TARGET_SUFFIX} && \
   echo "Building rpxy from source" && \
-  cargo build --release --target=$(cat /arch)-unknown-linux-gnu && \
+  cargo build --release --target=$(cat /arch)-${TARGET_SUFFIX} ${CARGO_FEATURES} && \
-  strip --strip-all /tmp/target/$(cat /arch)-unknown-linux-gnu/release/rpxy &&\
+  strip --strip-all /tmp/target/$(cat /arch)-${TARGET_SUFFIX}/release/rpxy &&\
-  cp /tmp/target/$(cat /arch)-unknown-linux-gnu/release/rpxy /tmp/target/release/rpxy
+  cp /tmp/target/$(cat /arch)-${TARGET_SUFFIX}/release/rpxy /tmp/target/release/rpxy
 
 ########################################
 FROM --platform=$TARGETPLATFORM base AS runner

docker/README.md

@@ -0,0 +1,19 @@
+# Docker Images of `rpxy`
+
+The `rpxy` docker images are hosted both on [Docker Hub](https://hub.docker.com/r/jqtype/rpxy) and [GitHub Container Registry](https://github.com/junkurihara/rust-rpxy/pkgs/container/rust-rpxy). Differences among tags are summarized as follows.
+
+## Latest Builds
+
+- `latest`: Built from the `main` branch with default features, running on Ubuntu.
+- `latest-slim`, `slim`: Built by `musl` from the `main` branch with default features, running on Alpine.
+- `latest-s2n`, `s2n`: Built from the `main` branch with the `http3-s2n` feature, running on Ubuntu.
+
+## Nightly Builds
+
+- `nightly`: Built from the `develop` branch with default features, running on Ubuntu.
+- `nightly-slim`: Built by `musl` from the `develop` branch with default features, running on Alpine.
+- `nightly-s2n`: Built from the `develop` branch with the `http3-s2n` feature, running on Ubuntu.
+
+## Caveats
+
+Due to some compile errors of `s2n-quic` subpackages with `musl`, `nightly-s2n-slim` or `latest-s2n-slim` are not yet provided.

docker/docker-compose-slim.yml

@@ -0,0 +1,34 @@
+version: "3"
+services:
+  rpxy-rp:
+    image: jqtype/rpxy:slim # ghcr.io/junkurihara/rust-rpxy:slim also works
+    container_name: rpxy
+    init: true
+    restart: unless-stopped
+    ports:
+      - 127.0.0.1:8080:8080/tcp
+      - 127.0.0.1:8443:8443/udp
+      - 127.0.0.1:8443:8443/tcp
+    # build: # Uncomment if you build yourself
+    #   context: ../
+    #   additional_contexts:
+    #     - messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl
+    #     - messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl
+    #   dockerfile: ./docker/Dockerfile-slim # based on alpine and build x86_64-unknown-linux-musl
+    #   platforms: # Choose your platforms
+    #     - "linux/amd64"
+    #     # - "linux/arm64"
+    environment:
+      - LOG_LEVEL=debug
+      - LOG_TO_FILE=true
+      - HOST_USER=jun
+      - HOST_UID=501
+      - HOST_GID=501
+      # - WATCH=true
+    tty: false
+    privileged: true
+    volumes:
+      - ./log:/rpxy/log
+      - ../example-certs/server.crt:/certs/server.crt:ro
+      - ../example-certs/server.key:/certs/server.key:ro
+      - ../config-example.toml:/etc/rpxy.toml:ro

docker/docker-compose.yml

@@ -1,19 +1,23 @@
 version: "3"
 services:
   rpxy-rp:
-    image: jqtype/rpxy
+    image: jqtype/rpxy:latest # ghcr.io/junkurihara/rust-rpxy:latest also works
     container_name: rpxy
     init: true
     restart: unless-stopped
     ports:
-      - 127.0.0.1:8080:8080
+      - 127.0.0.1:8080:8080/tcp
-      - 127.0.0.1:8443:8443
+      - 127.0.0.1:8443:8443/udp
-    build:
+      - 127.0.0.1:8443:8443/tcp
-      context: ../
+    # build: # Uncomment if you build yourself
-      dockerfile: ./docker/Dockerfile
+    #   context: ../
-      platforms: # Choose your platforms
+    #   args: # Uncomment when build quic-s2n version
-        - "linux/amd64"
+    #     - "CARGO_FEATURES=--no-default-features --features http3-s2n"
-        # - "linux/arm64"
+    #     - "ADDITIONAL_DEPS=pkg-config libssl-dev cmake libclang1 gcc g++"
+    #   dockerfile: ./docker/Dockerfile # based on ubuntu 22.04 and build x86_64-unknown-linux-gnu
+    #   platforms: # Choose your platforms
+    #     - "linux/amd64"
+    #     # - "linux/arm64"
     environment:
       - LOG_LEVEL=debug
       - LOG_TO_FILE=true

quinn

@@ -1 +1 @@
-Subproject commit 532ba7d80405ad083fd05546fa71becbe5eff1a4
+Subproject commit 70e14b5c26b45ee1e3d5dd64b2a184e2d6376880

rpxy-bin/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rpxy"
-version = "0.4.0"
+version = "0.5.0"
 authors = ["Jun Kurihara"]
 homepage = "https://github.com/junkurihara/rust-rpxy"
 repository = "https://github.com/junkurihara/rust-rpxy"
@@ -12,15 +12,18 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [features]
-default = ["http3"]
+default = ["http3-quinn"]
-http3 = []
+http3-quinn = ["rpxy-lib/http3-quinn"]
+http3-s2n = ["rpxy-lib/http3-s2n"]
 
 [dependencies]
-rpxy-lib = { path = "../rpxy-lib/", features = ["http3", "sticky-cookie"] }
+rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [
+  "sticky-cookie",
+] }
 
 anyhow = "1.0.72"
 rustc-hash = "1.1.0"
-serde = { version = "1.0.178", default-features = false, features = ["derive"] }
+serde = { version = "1.0.180", default-features = false, features = ["derive"] }
 derive_builder = "0.12.0"
 tokio = { version = "1.29.1", default-features = false, features = [
   "net",

rpxy-bin/src/config/toml.rs

@@ -21,7 +21,7 @@ pub struct ConfigToml {
   pub experimental: Option<Experimental>,
 }
 
-#[cfg(feature = "http3")]
+#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
 #[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)]
 pub struct Http3Option {
   pub alt_svc_max_age: Option<u32>,
@@ -34,7 +34,7 @@ pub struct Http3Option {
 
 #[derive(Deserialize, Debug, Default, PartialEq, Eq, Clone)]
 pub struct Experimental {
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
   pub h3: Option<Http3Option>,
   pub ignore_sni_consistency: Option<bool>,
 }
@@ -128,7 +128,7 @@ impl TryInto<ProxyConfig> for &ConfigToml {
 
     // experimental
     if let Some(exp) = &self.experimental {
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
       {
         if let Some(h3option) = &exp.h3 {
           proxy_config.http3 = true;
@@ -142,10 +142,10 @@ impl TryInto<ProxyConfig> for &ConfigToml {
             proxy_config.h3_max_concurrent_connections = x;
           }
           if let Some(x) = h3option.max_concurrent_bidistream {
-            proxy_config.h3_max_concurrent_bidistream = x.into();
+            proxy_config.h3_max_concurrent_bidistream = x;
           }
           if let Some(x) = h3option.max_concurrent_unistream {
-            proxy_config.h3_max_concurrent_unistream = x.into();
+            proxy_config.h3_max_concurrent_unistream = x;
           }
           if let Some(x) = h3option.max_idle_timeout {
             if x == 0u64 {

rpxy-bin/src/main.rs

@@ -19,6 +19,9 @@ use crate::{
 use hot_reload::{ReloaderReceiver, ReloaderService};
 use rpxy_lib::entrypoint;
 
+#[cfg(all(feature = "http3-quinn", feature = "http3-s2n"))]
+compile_error!("feature \"http3-quinn\" and feature \"http3-s2n\" cannot be enabled at the same time");
+
 fn main() {
   init_logger();
 

rpxy-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rpxy-lib"
-version = "0.4.0"
+version = "0.5.0"
 authors = ["Jun Kurihara"]
 homepage = "https://github.com/junkurihara/rust-rpxy"
 repository = "https://github.com/junkurihara/rust-rpxy"
@@ -12,8 +12,9 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [features]
-default = ["http3", "sticky-cookie"]
+default = ["http3-quinn", "sticky-cookie"]
-http3 = ["quinn", "h3", "h3-quinn"]
+http3-quinn = ["quinn", "h3", "h3-quinn", "socket2"]
+http3-s2n = ["h3", "s2n-quic", "s2n-quic-rustls", "s2n-quic-h3"]
 sticky-cookie = ["base64", "sha2", "chrono"]
 
 [dependencies]
@@ -63,8 +64,13 @@ quinn = { path = "../quinn/quinn", optional = true } # Tentative to support rust
 h3 = { path = "../h3/h3/", optional = true }
 # h3-quinn = { path = "./h3/h3-quinn/", optional = true }
 h3-quinn = { path = "../h3-quinn/", optional = true } # Tentative to support rustls-0.21
-# for UDP socket wit SO_REUSEADDR
+# for UDP socket wit SO_REUSEADDR when h3 with quinn
-socket2 = { version = "0.5.3", features = ["all"] }
+socket2 = { version = "0.5.3", features = ["all"], optional = true }
+s2n-quic = { path = "../s2n-quic/quic/s2n-quic/", default-features = false, features = [
+  "provider-tls-rustls",
+], optional = true }
+s2n-quic-h3 = { path = "../s2n-quic/quic/s2n-quic-h3/", optional = true }
+s2n-quic-rustls = { path = "../s2n-quic/quic/s2n-quic-rustls/", optional = true }
 
 # cookie handling for sticky cookie
 chrono = { version = "0.4.26", default-features = false, features = [

rpxy-lib/src/constants.rs

@@ -17,7 +17,7 @@ pub const LOAD_CERTS_ONLY_WHEN_UPDATED: bool = true;
 // pub const H3_REQUEST_BUF_SIZE: usize = 65_536; // 64KB // handled by quinn
 
 #[allow(non_snake_case)]
-#[cfg(feature = "http3")]
+#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
 pub mod H3 {
   pub const ALT_SVC_MAX_AGE: u32 = 3600;
   pub const REQUEST_MAX_BODY_SIZE: usize = 268_435_456; // 256MB

rpxy-lib/src/error.rs

@@ -37,14 +37,22 @@ pub enum RpxyError {
 
   // #[error("Toml Deserialization Error")]
   // TomlDe(#[from] toml::de::Error),
-  #[cfg(feature = "http3")]
+  #[cfg(feature = "http3-quinn")]
   #[error("Quic Connection Error")]
   QuicConn(#[from] quinn::ConnectionError),
 
-  #[cfg(feature = "http3")]
+  #[cfg(feature = "http3-s2n")]
+  #[error("Quic Connection Error [s2n-quic]")]
+  QUicConn(#[from] s2n_quic::connection::Error),
+
+  #[cfg(feature = "http3-quinn")]
   #[error("H3 Error")]
   H3(#[from] h3::Error),
 
+  #[cfg(feature = "http3-s2n")]
+  #[error("H3 Error [s2n-quic]")]
+  H3(#[from] s2n_quic_h3::h3::Error),
+
   #[error("rustls Connection Error")]
   Rustls(#[from] rustls::Error),
 

rpxy-lib/src/globals.rs

@@ -53,19 +53,19 @@ pub struct ProxyConfig {
   // experimentals
   pub sni_consistency: bool, // Handler
   // All need to make packet acceptor
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
   pub http3: bool,
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
   pub h3_alt_svc_max_age: u32,
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
   pub h3_request_max_body_size: usize,
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
-  pub h3_max_concurrent_bidistream: quinn::VarInt,
+  pub h3_max_concurrent_bidistream: u32,
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
-  pub h3_max_concurrent_unistream: quinn::VarInt,
+  pub h3_max_concurrent_unistream: u32,
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
   pub h3_max_concurrent_connections: u32,
-  #[cfg(feature = "http3")]
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
   pub h3_max_idle_timeout: Option<Duration>,
 }
 
@@ -87,19 +87,19 @@ impl Default for ProxyConfig {
 
       sni_consistency: true,
 
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
       http3: false,
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
       h3_alt_svc_max_age: H3::ALT_SVC_MAX_AGE,
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
       h3_request_max_body_size: H3::REQUEST_MAX_BODY_SIZE,
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
       h3_max_concurrent_connections: H3::MAX_CONCURRENT_CONNECTIONS,
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
-      h3_max_concurrent_bidistream: H3::MAX_CONCURRENT_BIDISTREAM.into(),
+      h3_max_concurrent_bidistream: H3::MAX_CONCURRENT_BIDISTREAM,
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
-      h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM.into(),
+      h3_max_concurrent_unistream: H3::MAX_CONCURRENT_UNISTREAM,
-      #[cfg(feature = "http3")]
+      #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
       h3_max_idle_timeout: Some(Duration::from_secs(H3::MAX_IDLE_TIMEOUT)),
     }
   }

rpxy-lib/src/handler/handler_main.rs

@@ -210,7 +210,7 @@ where
     remove_hop_header(headers);
     add_header_entry_overwrite_if_exist(headers, "server", env!("CARGO_PKG_NAME"))?;
 
-    #[cfg(feature = "http3")]
+    #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
     {
       // Manipulate ALT_SVC allowing h3 in response message only when mutual TLS is not enabled
       // TODO: This is a workaround for avoiding a client authentication in HTTP/3
@@ -235,7 +235,7 @@ where
         headers.remove(header::ALT_SVC.as_str());
       }
     }
-    #[cfg(not(feature = "http3"))]
+    #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))]
     {
       if let Some(port) = self.globals.proxy_config.https_port {
         headers.remove(header::ALT_SVC.as_str());

rpxy-lib/src/lib.rs

@@ -23,6 +23,9 @@ pub mod reexports {
   pub use rustls::{Certificate, PrivateKey};
 }
 
+#[cfg(all(feature = "http3-quinn", feature = "http3-s2n"))]
+compile_error!("feature \"http3-quinn\" and feature \"http3-s2n\" cannot be enabled at the same time");
+
 /// Entrypoint that creates and spawns tasks of reverse proxy services
 pub async fn entrypoint<T>(
   proxy_config: &ProxyConfig,
@@ -44,6 +47,7 @@ where
   if proxy_config.https_port.is_some() {
     info!("Listen port: {} (for TLS)", proxy_config.https_port.unwrap());
   }
+  #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
   if proxy_config.http3 {
     info!("Experimental HTTP/3.0 is enabled. Note it is still very unstable.");
   }

rpxy-lib/src/proxy/crypto_service.rs

@@ -22,7 +22,10 @@ where
 pub type SniServerCryptoMap = HashMap<ServerNameBytesExp, Arc<ServerConfig>>;
 pub struct ServerCrypto {
   // For Quic/HTTP3, only servers with no client authentication
+  #[cfg(feature = "http3-quinn")]
   pub inner_global_no_client_auth: Arc<ServerConfig>,
+  #[cfg(feature = "http3-s2n")]
+  pub inner_global_no_client_auth: s2n_quic_rustls::Server,
   // For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers
   pub inner_local_map: Arc<SniServerCryptoMap>,
 }
@@ -68,7 +71,22 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
   type Error = anyhow::Error;
 
   fn try_into(self) -> Result<Arc<ServerCrypto>, Self::Error> {
-    let mut resolver_global = ResolvesServerCertUsingSni::new();
+    #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
+    let server_crypto_global = self.build_server_crypto_global()?;
+    let server_crypto_local_map: SniServerCryptoMap = self.build_server_crypto_local_map()?;
+
+    Ok(Arc::new(ServerCrypto {
+      #[cfg(feature = "http3-quinn")]
+      inner_global_no_client_auth: Arc::new(server_crypto_global),
+      #[cfg(feature = "http3-s2n")]
+      inner_global_no_client_auth: server_crypto_global,
+      inner_local_map: Arc::new(server_crypto_local_map),
+    }))
+  }
+}
+
+impl ServerCryptoBase {
+  fn build_server_crypto_local_map(&self) -> Result<SniServerCryptoMap, ReloaderError<ServerCryptoBase>> {
     let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default();
 
     for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
@@ -93,16 +111,7 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
       }
 
       // add client certificate if specified
-      if certs_and_keys.client_ca_certs.is_none() {
+      if certs_and_keys.client_ca_certs.is_some() {
-        // aggregated server config for no client auth server for http3
-        if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
-          error!(
-            "{}: Failed to read some certificates and keys {}",
-            server_name.as_str(),
-            e
-          )
-        }
-      } else {
         // add client certificate if specified
         match certs_and_keys.parse_client_ca_certs() {
           Ok((owned_trust_anchors, _subject_key_ids)) => {
@@ -120,14 +129,14 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
 
       let mut server_config_local = if client_ca_roots_local.is_empty() {
         // with no client auth, enable http1.1 -- 3
-        #[cfg(not(feature = "http3"))]
+        #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))]
         {
           ServerConfig::builder()
             .with_safe_defaults()
             .with_no_client_auth()
             .with_cert_resolver(Arc::new(resolver_local))
         }
-        #[cfg(feature = "http3")]
+        #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
         {
           let mut sc = ServerConfig::builder()
             .with_safe_defaults()
@@ -150,6 +159,33 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
 
       server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local));
     }
+    Ok(server_crypto_local_map)
+  }
+
+  #[cfg(feature = "http3-quinn")]
+  fn build_server_crypto_global(&self) -> Result<ServerConfig, ReloaderError<ServerCryptoBase>> {
+    let mut resolver_global = ResolvesServerCertUsingSni::new();
+
+    for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
+      let server_name: String = server_name_bytes_exp.try_into()?;
+
+      // Parse server certificates and private keys
+      let Ok(certified_key): Result<CertifiedKey, _> = certs_and_keys.parse_server_certs_and_keys() else {
+        warn!("Failed to add certificate for {}", server_name);
+        continue;
+      };
+
+      if certs_and_keys.client_ca_certs.is_none() {
+        // aggregated server config for no client auth server for http3
+        if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
+          error!(
+            "{}: Failed to read some certificates and keys {}",
+            server_name.as_str(),
+            e
+          )
+        }
+      }
+    }
 
     //////////////
     let mut server_crypto_global = ServerConfig::builder()
@@ -159,23 +195,82 @@ impl TryInto<Arc<ServerCrypto>> for &ServerCryptoBase {
 
     //////////////////////////////
 
-    #[cfg(feature = "http3")]
+    server_crypto_global.alpn_protocols = vec![
-    {
+      b"h3".to_vec(),
-      server_crypto_global.alpn_protocols = vec![
+      b"hq-29".to_vec(), // TODO: remove later?
-        b"h3".to_vec(),
+      b"h2".to_vec(),
-        b"hq-29".to_vec(), // TODO: remove later?
+      b"http/1.1".to_vec(),
-        b"h2".to_vec(),
+    ];
-        b"http/1.1".to_vec(),
+    Ok(server_crypto_global)
-      ];
+  }
-    }
-    #[cfg(not(feature = "http3"))]
-    {
-      server_crypto_global.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
-    }
 
-    Ok(Arc::new(ServerCrypto {
+  #[cfg(feature = "http3-s2n")]
-      inner_global_no_client_auth: Arc::new(server_crypto_global),
+  fn build_server_crypto_global(&self) -> Result<s2n_quic_rustls::Server, ReloaderError<ServerCryptoBase>> {
-      inner_local_map: Arc::new(server_crypto_local_map),
+    let mut resolver_global = s2n_quic_rustls::rustls::server::ResolvesServerCertUsingSni::new();
-    }))
+
+    for (server_name_bytes_exp, certs_and_keys) in self.inner.iter() {
+      let server_name: String = server_name_bytes_exp.try_into()?;
+
+      // Parse server certificates and private keys
+      let Ok(certified_key) = parse_server_certs_and_keys_s2n(certs_and_keys) else {
+        warn!("Failed to add certificate for {}", server_name);
+        continue;
+      };
+
+      if certs_and_keys.client_ca_certs.is_none() {
+        // aggregated server config for no client auth server for http3
+        if let Err(e) = resolver_global.add(server_name.as_str(), certified_key) {
+          error!(
+            "{}: Failed to read some certificates and keys {}",
+            server_name.as_str(),
+            e
+          )
+        }
+      }
+    }
+    let alpn = vec![
+      b"h3".to_vec(),
+      b"hq-29".to_vec(), // TODO: remove later?
+      b"h2".to_vec(),
+      b"http/1.1".to_vec(),
+    ];
+    let server_crypto_global = s2n_quic::provider::tls::rustls::Server::builder()
+      .with_cert_resolver(Arc::new(resolver_global))
+      .map_err(|e| anyhow::anyhow!(e))?
+      .with_application_protocols(alpn.iter())
+      .map_err(|e| anyhow::anyhow!(e))?
+      .build()
+      .map_err(|e| anyhow::anyhow!(e))?;
+    Ok(server_crypto_global)
   }
 }
+
+#[cfg(feature = "http3-s2n")]
+/// This is workaround for the version difference between rustls and s2n-quic-rustls
+fn parse_server_certs_and_keys_s2n(
+  certs_and_keys: &CertsAndKeys,
+) -> Result<s2n_quic_rustls::rustls::sign::CertifiedKey, anyhow::Error> {
+  let signing_key = certs_and_keys
+    .cert_keys
+    .iter()
+    .find_map(|k| {
+      let s2n_private_key = s2n_quic_rustls::PrivateKey(k.0.clone());
+      if let Ok(sk) = s2n_quic_rustls::rustls::sign::any_supported_type(&s2n_private_key) {
+        Some(sk)
+      } else {
+        None
+      }
+    })
+    .ok_or_else(|| {
+      std::io::Error::new(
+        std::io::ErrorKind::InvalidInput,
+        "Unable to find a valid certificate and key",
+      )
+    })?;
+  let certs: Vec<_> = certs_and_keys
+    .certs
+    .iter()
+    .map(|c| s2n_quic_rustls::rustls::Certificate(c.0.clone()))
+    .collect();
+  Ok(s2n_quic_rustls::rustls::sign::CertifiedKey::new(certs, signing_key))
+}

rpxy-lib/src/proxy/mod.rs

@@ -1,8 +1,12 @@
 mod crypto_service;
 mod proxy_client_cert;
-#[cfg(feature = "http3")]
+#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
 mod proxy_h3;
 mod proxy_main;
+#[cfg(feature = "http3-quinn")]
+mod proxy_quic_quinn;
+#[cfg(feature = "http3-s2n")]
+mod proxy_quic_s2n;
 mod proxy_tls;
 mod socket;
 

rpxy-lib/src/proxy/proxy_h3.rs

@@ -1,8 +1,11 @@
 use super::Proxy;
 use crate::{certs::CryptoSource, error::*, log::*, utils::ServerNameBytesExp};
 use bytes::{Buf, Bytes};
-use h3::{quic::BidiStream, server::RequestStream};
+#[cfg(feature = "http3-quinn")]
+use h3::{quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
 use hyper::{client::connect::Connect, Body, Request, Response};
+#[cfg(feature = "http3-s2n")]
+use s2n_quic_h3::h3::{self, quic::BidiStream, quic::Connection as ConnectionQuic, server::RequestStream};
 use std::net::SocketAddr;
 use tokio::time::{timeout, Duration};
 
@@ -11,67 +14,64 @@ where
   T: Connect + Clone + Sync + Send + 'static,
   U: CryptoSource + Clone + Sync + Send + 'static,
 {
-  pub(super) async fn connection_serve_h3(
+  pub(super) async fn connection_serve_h3<C>(
     self,
-    conn: quinn::Connecting,
+    quic_connection: C,
     tls_server_name: ServerNameBytesExp,
-  ) -> Result<()> {
+    client_addr: SocketAddr,
-    let client_addr = conn.remote_address();
+  ) -> Result<()>
-
+  where
-    match conn.await {
+    C: ConnectionQuic<Bytes>,
-      Ok(new_conn) => {
+    <C as ConnectionQuic<Bytes>>::BidiStream: BidiStream<Bytes> + Send + 'static,
-        let mut h3_conn = h3::server::Connection::<_, bytes::Bytes>::new(h3_quinn::Connection::new(new_conn)).await?;
+    <<C as ConnectionQuic<Bytes>>::BidiStream as BidiStream<Bytes>>::RecvStream: Send,
-        info!(
+    <<C as ConnectionQuic<Bytes>>::BidiStream as BidiStream<Bytes>>::SendStream: Send,
-          "QUIC/HTTP3 connection established from {:?} {:?}",
+  {
-          client_addr, tls_server_name
+    let mut h3_conn = h3::server::Connection::<_, Bytes>::new(quic_connection).await?;
-        );
+    info!(
-        // TODO: Is here enough to fetch server_name from NewConnection?
+      "QUIC/HTTP3 connection established from {:?} {:?}",
-        // to avoid deep nested call from listener_service_h3
+      client_addr, tls_server_name
-        loop {
+    );
-          // this routine follows hyperium/h3 examples https://github.com/hyperium/h3/blob/master/examples/server.rs
+    // TODO: Is here enough to fetch server_name from NewConnection?
-          match h3_conn.accept().await {
+    // to avoid deep nested call from listener_service_h3
-            Ok(None) => {
+    loop {
-              break;
+      // this routine follows hyperium/h3 examples https://github.com/hyperium/h3/blob/master/examples/server.rs
-            }
+      match h3_conn.accept().await {
-            Err(e) => {
+        Ok(None) => {
-              warn!("HTTP/3 error on accept incoming connection: {}", e);
+          break;
-              match e.get_error_level() {
+        }
-                h3::error::ErrorLevel::ConnectionError => break,
+        Err(e) => {
-                h3::error::ErrorLevel::StreamError => continue,
+          warn!("HTTP/3 error on accept incoming connection: {}", e);
-              }
+          match e.get_error_level() {
-            }
+            h3::error::ErrorLevel::ConnectionError => break,
-            Ok(Some((req, stream))) => {
+            h3::error::ErrorLevel::StreamError => continue,
-              // We consider the connection count separately from the stream count.
-              // Max clients for h1/h2 = max 'stream' for h3.
-              let request_count = self.globals.request_count.clone();
-              if request_count.increment() > self.globals.proxy_config.max_clients {
-                request_count.decrement();
-                h3_conn.shutdown(0).await?;
-                break;
-              }
-              debug!("Request incoming: current # {}", request_count.current());
-
-              let self_inner = self.clone();
-              let tls_server_name_inner = tls_server_name.clone();
-              self.globals.runtime_handle.spawn(async move {
-                if let Err(e) = timeout(
-                  self_inner.globals.proxy_config.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
-                  self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner),
-                )
-                .await
-                {
-                  error!("HTTP/3 failed to process stream: {}", e);
-                }
-                request_count.decrement();
-                debug!("Request processed: current # {}", request_count.current());
-              });
-            }
           }
         }
-      }
+        Ok(Some((req, stream))) => {
-      Err(err) => {
+          // We consider the connection count separately from the stream count.
-        warn!("QUIC accepting connection failed: {:?}", err);
+          // Max clients for h1/h2 = max 'stream' for h3.
-        return Err(RpxyError::QuicConn(err));
+          let request_count = self.globals.request_count.clone();
+          if request_count.increment() > self.globals.proxy_config.max_clients {
+            request_count.decrement();
+            h3_conn.shutdown(0).await?;
+            break;
+          }
+          debug!("Request incoming: current # {}", request_count.current());
+
+          let self_inner = self.clone();
+          let tls_server_name_inner = tls_server_name.clone();
+          self.globals.runtime_handle.spawn(async move {
+            if let Err(e) = timeout(
+              self_inner.globals.proxy_config.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
+              self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner),
+            )
+            .await
+            {
+              error!("HTTP/3 failed to process stream: {}", e);
+            }
+            request_count.decrement();
+            debug!("Request processed: current # {}", request_count.current());
+          });
+        }
       }
     }
 

rpxy-lib/src/proxy/proxy_quic_quinn.rs

@@ -0,0 +1,124 @@
+use super::socket::bind_udp_socket;
+use super::{
+  crypto_service::{ServerCrypto, ServerCryptoBase},
+  proxy_main::Proxy,
+};
+use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName};
+use hot_reload::ReloaderReceiver;
+use hyper::client::connect::Connect;
+use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerConfig, TransportConfig};
+use rustls::ServerConfig;
+use std::sync::Arc;
+
+impl<T, U> Proxy<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone + Sync + Send + 'static,
+{
+  pub(super) async fn listener_service_h3(
+    &self,
+    mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
+  ) -> Result<()> {
+    info!("Start UDP proxy serving with HTTP/3 request for configured host names [quinn]");
+    // first set as null config server
+    let rustls_server_config = ServerConfig::builder()
+      .with_safe_default_cipher_suites()
+      .with_safe_default_kx_groups()
+      .with_protocol_versions(&[&rustls::version::TLS13])?
+      .with_no_client_auth()
+      .with_cert_resolver(Arc::new(rustls::server::ResolvesServerCertUsingSni::new()));
+
+    let mut transport_config_quic = TransportConfig::default();
+    transport_config_quic
+      .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream.into())
+      .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream.into())
+      .max_idle_timeout(
+        self
+          .globals
+          .proxy_config
+          .h3_max_idle_timeout
+          .map(|v| quinn::IdleTimeout::try_from(v).unwrap()),
+      );
+
+    let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config));
+    server_config_h3.transport = Arc::new(transport_config_quic);
+    server_config_h3.concurrent_connections(self.globals.proxy_config.h3_max_concurrent_connections);
+
+    // To reuse address
+    let udp_socket = bind_udp_socket(&self.listening_on)?;
+    let runtime = quinn::default_runtime()
+      .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No async runtime found"))?;
+    let endpoint = Endpoint::new(
+      quinn::EndpointConfig::default(),
+      Some(server_config_h3),
+      udp_socket,
+      runtime,
+    )?;
+
+    let mut server_crypto: Option<Arc<ServerCrypto>> = None;
+    loop {
+      tokio::select! {
+        new_conn = endpoint.accept() => {
+          if server_crypto.is_none() || new_conn.is_none() {
+            continue;
+          }
+          let mut conn: quinn::Connecting = new_conn.unwrap();
+          let Ok(hsd) = conn.handshake_data().await else {
+            continue
+          };
+
+          let Ok(hsd_downcast) = hsd.downcast::<HandshakeData>() else {
+            continue
+          };
+          let Some(new_server_name) = hsd_downcast.server_name else {
+            warn!("HTTP/3 no SNI is given");
+            continue;
+          };
+          debug!(
+            "HTTP/3 connection incoming (SNI {:?})",
+            new_server_name
+          );
+          // TODO: server_nameをここで出してどんどん深く投げていくのは効率が悪い。connecting -> connectionsの後でいいのでは？
+          // TODO: 通常のTLSと同じenumか何かにまとめたい
+          let self_clone = self.clone();
+          self.globals.runtime_handle.spawn(async move {
+            let client_addr = conn.remote_address();
+            let quic_connection = match conn.await {
+              Ok(new_conn) => {
+                info!("New connection established");
+                h3_quinn::Connection::new(new_conn)
+              },
+              Err(e) => {
+                warn!("QUIC accepting connection failed: {:?}", e);
+                return Err(RpxyError::QuicConn(e));
+              }
+            };
+            // Timeout is based on underlying quic
+            if let Err(e) = self_clone.connection_serve_h3(quic_connection, new_server_name.to_server_name_vec(), client_addr).await {
+              warn!("QUIC or HTTP/3 connection failed: {}", e);
+            };
+            Ok(())
+          });
+        }
+        _ = server_crypto_rx.changed() => {
+          if server_crypto_rx.borrow().is_none() {
+            error!("Reloader is broken");
+            break;
+          }
+          let cert_keys_map = server_crypto_rx.borrow().clone().unwrap();
+
+          server_crypto = (&cert_keys_map).try_into().ok();
+          let Some(inner) = server_crypto.clone() else {
+            error!("Failed to update server crypto for h3");
+            break;
+          };
+          endpoint.set_server_config(Some(QuicServerConfig::with_crypto(inner.clone().inner_global_no_client_auth.clone())));
+
+        }
+        else => break
+      }
+    }
+    endpoint.wait_idle().await;
+    Ok(()) as Result<()>
+  }
+}

rpxy-lib/src/proxy/proxy_quic_s2n.rs

@@ -0,0 +1,135 @@
+use super::{
+  crypto_service::{ServerCrypto, ServerCryptoBase},
+  proxy_main::Proxy,
+};
+use crate::{certs::CryptoSource, error::*, log::*, utils::BytesName};
+use hot_reload::ReloaderReceiver;
+use hyper::client::connect::Connect;
+use s2n_quic::provider;
+use std::sync::Arc;
+
+impl<T, U> Proxy<T, U>
+where
+  T: Connect + Clone + Sync + Send + 'static,
+  U: CryptoSource + Clone + Sync + Send + 'static,
+{
+  pub(super) async fn listener_service_h3(
+    &self,
+    mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>,
+  ) -> Result<()> {
+    info!("Start UDP proxy serving with HTTP/3 request for configured host names [s2n-quic]");
+
+    // initially wait for receipt
+    let mut server_crypto: Option<Arc<ServerCrypto>> = {
+      let _ = server_crypto_rx.changed().await;
+      let sc = self.receive_server_crypto(server_crypto_rx.clone())?;
+      Some(sc)
+    };
+
+    // event loop
+    loop {
+      tokio::select! {
+        v = self.serve_connection(&server_crypto) => {
+          if let Err(e) = v {
+            error!("Quic connection event loop illegally shutdown [s2n-quic] {e}");
+            break;
+          }
+        }
+        _ = server_crypto_rx.changed() => {
+          server_crypto = match self.receive_server_crypto(server_crypto_rx.clone()) {
+            Ok(sc) => Some(sc),
+            Err(e) => {
+              error!("{e}");
+              break;
+            }
+          };
+        }
+        else => break
+      }
+    }
+
+    Ok(())
+  }
+
+  fn receive_server_crypto(&self, server_crypto_rx: ReloaderReceiver<ServerCryptoBase>) -> Result<Arc<ServerCrypto>> {
+    let cert_keys_map = server_crypto_rx.borrow().clone().ok_or_else(|| {
+      error!("Reloader is broken");
+      RpxyError::Other(anyhow!("Reloader is broken"))
+    })?;
+
+    let server_crypto: Option<Arc<ServerCrypto>> = (&cert_keys_map).try_into().ok();
+    server_crypto.ok_or_else(|| {
+      error!("Failed to update server crypto for h3 [s2n-quic]");
+      RpxyError::Other(anyhow!("Failed to update server crypto for h3 [s2n-quic]"))
+    })
+  }
+
+  async fn serve_connection(&self, server_crypto: &Option<Arc<ServerCrypto>>) -> Result<()> {
+    // setup UDP socket
+    let io = provider::io::tokio::Builder::default()
+      .with_receive_address(self.listening_on)?
+      .with_reuse_port()?
+      .build()?;
+
+    // setup limits
+    let mut limits = provider::limits::Limits::default()
+      .with_max_open_local_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_open_remote_bidirectional_streams(self.globals.proxy_config.h3_max_concurrent_bidistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_open_local_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_open_remote_unidirectional_streams(self.globals.proxy_config.h3_max_concurrent_unistream as u64)
+      .map_err(|e| anyhow!(e))?
+      .with_max_active_connection_ids(self.globals.proxy_config.h3_max_concurrent_connections as u64)
+      .map_err(|e| anyhow!(e))?;
+    limits = if let Some(v) = self.globals.proxy_config.h3_max_idle_timeout {
+      limits.with_max_idle_timeout(v).map_err(|e| anyhow!(e))?
+    } else {
+      limits
+    };
+
+    // setup tls
+    let Some(server_crypto) = server_crypto else {
+      warn!("No server crypto is given [s2n-quic]");
+      return Err(RpxyError::Other(anyhow!("No server crypto is given [s2n-quic]")));
+    };
+    let tls = server_crypto.inner_global_no_client_auth.clone();
+
+    let mut server = s2n_quic::Server::builder()
+      .with_tls(tls)
+      .map_err(|e| anyhow::anyhow!(e))?
+      .with_io(io)
+      .map_err(|e| anyhow!(e))?
+      .with_limits(limits)
+      .map_err(|e| anyhow!(e))?
+      .start()
+      .map_err(|e| anyhow!(e))?;
+
+    // quic event loop. this immediately cancels when crypto is updated by tokio::select!
+    while let Some(new_conn) = server.accept().await {
+      debug!("New QUIC connection established");
+      let Ok(Some(new_server_name)) = new_conn.server_name() else {
+          warn!("HTTP/3 no SNI is given");
+          continue;
+        };
+      debug!("HTTP/3 connection incoming (SNI {:?})", new_server_name);
+      let self_clone = self.clone();
+
+      self.globals.runtime_handle.spawn(async move {
+        let client_addr = new_conn.remote_addr()?;
+        let quic_connection = s2n_quic_h3::Connection::new(new_conn);
+        // Timeout is based on underlying quic
+        if let Err(e) = self_clone
+          .connection_serve_h3(quic_connection, new_server_name.to_server_name_vec(), client_addr)
+          .await
+        {
+          warn!("QUIC or HTTP/3 connection failed: {}", e);
+        };
+        Ok(()) as Result<()>
+      });
+    }
+
+    Ok(())
+  }
+}

rpxy-lib/src/proxy/proxy_tls.rs

@@ -1,5 +1,3 @@
-#[cfg(feature = "http3")]
-use super::socket::bind_udp_socket;
 use super::{
   crypto_service::{CryptoReloader, ServerCrypto, ServerCryptoBase, SniServerCryptoMap},
   proxy_main::{LocalExecutor, Proxy},
@@ -8,10 +6,6 @@ use super::{
 use crate::{certs::CryptoSource, constants::*, error::*, log::*, utils::BytesName};
 use hot_reload::{ReloaderReceiver, ReloaderService};
 use hyper::{client::connect::Connect, server::conn::Http};
-#[cfg(feature = "http3")]
-use quinn::{crypto::rustls::HandshakeData, Endpoint, ServerConfig as QuicServerConfig, TransportConfig};
-#[cfg(feature = "http3")]
-use rustls::ServerConfig;
 use std::sync::Arc;
 use tokio::time::{timeout, Duration};
 
@@ -105,99 +99,6 @@ where
     Ok(()) as Result<()>
   }
 
-  #[cfg(feature = "http3")]
-  async fn listener_service_h3(&self, mut server_crypto_rx: ReloaderReceiver<ServerCryptoBase>) -> Result<()> {
-    info!("Start UDP proxy serving with HTTP/3 request for configured host names");
-    // first set as null config server
-    let rustls_server_config = ServerConfig::builder()
-      .with_safe_default_cipher_suites()
-      .with_safe_default_kx_groups()
-      .with_protocol_versions(&[&rustls::version::TLS13])?
-      .with_no_client_auth()
-      .with_cert_resolver(Arc::new(rustls::server::ResolvesServerCertUsingSni::new()));
-
-    let mut transport_config_quic = TransportConfig::default();
-    transport_config_quic
-      .max_concurrent_bidi_streams(self.globals.proxy_config.h3_max_concurrent_bidistream)
-      .max_concurrent_uni_streams(self.globals.proxy_config.h3_max_concurrent_unistream)
-      .max_idle_timeout(
-        self
-          .globals
-          .proxy_config
-          .h3_max_idle_timeout
-          .map(|v| quinn::IdleTimeout::try_from(v).unwrap()),
-      );
-
-    let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config));
-    server_config_h3.transport = Arc::new(transport_config_quic);
-    server_config_h3.concurrent_connections(self.globals.proxy_config.h3_max_concurrent_connections);
-
-    // To reuse address
-    let udp_socket = bind_udp_socket(&self.listening_on)?;
-    let runtime = quinn::default_runtime()
-      .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No async runtime found"))?;
-    let endpoint = Endpoint::new(
-      quinn::EndpointConfig::default(),
-      Some(server_config_h3),
-      udp_socket,
-      runtime,
-    )?;
-
-    let mut server_crypto: Option<Arc<ServerCrypto>> = None;
-    loop {
-      tokio::select! {
-        new_conn = endpoint.accept() => {
-          if server_crypto.is_none() || new_conn.is_none() {
-            continue;
-          }
-          let mut conn: quinn::Connecting = new_conn.unwrap();
-          let Ok(hsd) = conn.handshake_data().await else {
-            continue
-          };
-
-          let Ok(hsd_downcast) = hsd.downcast::<HandshakeData>() else {
-            continue
-          };
-          let Some(new_server_name) = hsd_downcast.server_name else {
-            warn!("HTTP/3 no SNI is given");
-            continue;
-          };
-          debug!(
-            "HTTP/3 connection incoming (SNI {:?})",
-            new_server_name
-          );
-          // TODO: server_nameをここで出してどんどん深く投げていくのは効率が悪い。connecting -> connectionsの後でいいのでは？
-          // TODO: 通常のTLSと同じenumか何かにまとめたい
-          let fut = self.clone().connection_serve_h3(conn, new_server_name.to_server_name_vec());
-          self.globals.runtime_handle.spawn(async move {
-            // Timeout is based on underlying quic
-            if let Err(e) = fut.await {
-              warn!("QUIC or HTTP/3 connection failed: {}", e)
-            }
-          });
-        }
-        _ = server_crypto_rx.changed() => {
-          if server_crypto_rx.borrow().is_none() {
-            error!("Reloader is broken");
-            break;
-          }
-          let cert_keys_map = server_crypto_rx.borrow().clone().unwrap();
-
-          server_crypto = (&cert_keys_map).try_into().ok();
-          let Some(inner) = server_crypto.clone() else {
-            error!("Failed to update server crypto for h3");
-            break;
-          };
-          endpoint.set_server_config(Some(QuicServerConfig::with_crypto(inner.clone().inner_global_no_client_auth.clone())));
-
-        }
-        else => break
-      }
-    }
-    endpoint.wait_idle().await;
-    Ok(()) as Result<()>
-  }
-
   pub async fn start_with_tls(self, server: Http<LocalExecutor>) -> Result<()> {
     let (cert_reloader_service, cert_reloader_rx) = ReloaderService::<CryptoReloader<U>, ServerCryptoBase>::new(
       &self.globals.clone(),
@@ -207,7 +108,7 @@ where
     .await
     .map_err(|e| anyhow::anyhow!(e))?;
 
-    #[cfg(not(feature = "http3"))]
+    #[cfg(not(any(feature = "http3-quinn", feature = "http3-s2n")))]
     {
       tokio::select! {
         _= cert_reloader_service.start() => {
@@ -223,7 +124,7 @@ where
       };
       Ok(())
     }
-    #[cfg(feature = "http3")]
+    #[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))]
     {
       if self.globals.proxy_config.http3 {
         tokio::select! {

rpxy-lib/src/proxy/socket.rs

@@ -1,8 +1,8 @@
 use crate::{error::*, log::*};
-#[cfg(feature = "http3")]
+#[cfg(feature = "http3-quinn")]
 use socket2::{Domain, Protocol, Socket, Type};
 use std::net::SocketAddr;
-#[cfg(feature = "http3")]
+#[cfg(feature = "http3-quinn")]
 use std::net::UdpSocket;
 use tokio::net::TcpSocket;
 
@@ -23,7 +23,7 @@ pub(super) fn bind_tcp_socket(listening_on: &SocketAddr) -> Result<TcpSocket> {
   Ok(tcp_socket)
 }
 
-#[cfg(feature = "http3")]
+#[cfg(feature = "http3-quinn")]
 /// Bind UDP socket to the given `SocketAddr`, and returns the UDP socket with `SO_REUSEADDR` and `SO_REUSEPORT` options.
 /// This option is required to re-bind the socket address when the proxy instance is reconstructed.
 pub(super) fn bind_udp_socket(listening_on: &SocketAddr) -> Result<UdpSocket> {

s2n-quic

@@ -0,0 +1 @@
+Subproject commit 8ef0a6b66a856dc9f34ce18159c617ac29154cc7