From 7c0945a5124418aa9a1024568c1989bb77cf312f Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 24 Jul 2023 15:52:54 +0900 Subject: [PATCH 1/2] submodule --- h3 | 2 +- quinn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/h3 b/h3 index dccb3cd..a57ed22 160000 --- a/h3 +++ b/h3 @@ -1 +1 @@ -Subproject commit dccb3cdae9d5a9d720fae5f774b53f0bd8a16019 +Subproject commit a57ed224ac5d17a635eb71eb6f83c1196f581a51 diff --git a/quinn b/quinn index e652b6d..0ae7c60 160000 --- a/quinn +++ b/quinn @@ -1 +1 @@ -Subproject commit e652b6d999f053ffe21eeea247854882ae480281 +Subproject commit 0ae7c60b15637d7343410ba1e5cc3151e3814557 From 7f52dce23d4985246908628240bd584ae20ffb00 Mon Sep 17 00:00:00 2001 From: Jun Kurihara Date: Mon, 24 Jul 2023 20:08:18 +0900 Subject: [PATCH 2/2] add benchmark result and fix Dockerfile names since it does not depends on architecture --- .github/workflows/docker_build_push.yml | 11 +- bench/README.md | 183 ++++++++++++++++++++++-- bench/bench.amd64.sh | 53 +++++++ bench/bench.sh | 2 + bench/docker-compose.amd64.yml | 96 +++++++++++++ bench/docker-compose.yml | 4 +- bench/nginx.conf | 4 +- bench/sozu-config.toml | 16 +++ docker/{Dockerfile.amd64 => Dockerfile} | 1 - docker/Dockerfile.amd64-slim | 1 - docker/docker-compose.yml | 2 +- 11 files changed, 354 insertions(+), 19 deletions(-) create mode 100644 bench/bench.amd64.sh create mode 100644 bench/docker-compose.amd64.yml create mode 100644 bench/sozu-config.toml rename docker/{Dockerfile.amd64 => Dockerfile} (98%) diff --git a/.github/workflows/docker_build_push.yml b/.github/workflows/docker_build_push.yml index 1dfd260..305aa45 100644 --- a/.github/workflows/docker_build_push.yml +++ b/.github/workflows/docker_build_push.yml @@ -30,7 +30,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Release build and push x86_64 + - name: Release build and push if: ${{ env.BRANCH == 'main' }} uses: docker/build-push-action@v4 with: @@ -38,7 +38,8 @@ jobs: push: true tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest - file: ./docker/Dockerfile.amd64 + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 - name: Release build and push x86_64-slim if: ${{ env.BRANCH == 'main' }} @@ -49,8 +50,9 @@ jobs: tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim file: ./docker/Dockerfile.amd64-slim + platforms: linux/amd64 - - name: Nightly build and push x86_64 + - name: Nightly build and push if: ${{ env.BRANCH == 'develop' }} uses: docker/build-push-action@v4 with: @@ -58,4 +60,5 @@ jobs: push: true tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly - file: ./docker/Dockerfile.amd64 + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 diff --git a/bench/README.md b/bench/README.md index 7a98c81..55ca4dd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,23 +1,25 @@ -# Sample Benchmark Result - -Done at Jul. 15, 2023 +# Sample Benchmark Results This test simply measures the performance of several reverse proxy through HTTP/1.1 by the following command using [`rewrk`](https://github.com/lnx-search/rewrk). -```bash +```sh: $ rewrk -c 512 -t 4 -d 15s -h http://localhost:8080 --pct ``` -## Environment +## Tests on `linux/arm64/v8` + +Done at Jul. 15, 2023 + +### Environment - `rpxy` commit id: `1da7e5bfb77d1ce4ee8d6cfc59b1c725556fc192` - Docker Desktop 4.21.1 (114176) -- ReWrk 0.3.1 +- ReWrk 0.3.2 - Macbook Pro '14 (2021, M1 Max, 64GB RAM) +The docker images of `nginx` and `caddy` for `linux/arm64/v8` are pulled from the official registry. - -## Result +### Result for `rpxy`, `nginx` and `caddy` ``` ---------------------------- @@ -94,3 +96,168 @@ Benchmarking 512 connections @ http://localhost:8100 for 15 second(s) 708 Errors: error shutting down connection: Socket is not connected (os error 57) ``` + +## Results on `linux/amd64` + +Done at Jul. 24, 2023 + +### Environment + +- `rpxy` commit id: `7c0945a5124418aa9a1024568c1989bb77cf312f` +- Docker Desktop 4.21.1 (114176) +- ReWrk 0.3.2 and Wrk 0.4.2 +- iMac '27 (2020, 10-Core Intel Core i9, 128GB RAM) + +The docker images of `nginx` and `caddy` for `linux/amd64` were pulled from the official registry. For `Sozu`, the official docker image from its developers was still version 0.11.0 (currently the latest version is 0.15.2). So we built it by ourselves locally using the `Sozu`'s official [`Dockerfile`](https://github.com/sozu-proxy/sozu/blob/main/Dockerfile). + +Also, when `Sozu` is configured as an HTTP reverse proxy, it cannot handle HTTP request messages emit from `ReWrk` due to hostname parsing errors though it can correctly handle messages dispatched from `curl` and browsers. So, we additionally test using [`Wrk`](https://github.com/wg/wrk) to examine `Sozu` with the following command. + +```sh: +$ wrk -c 512 -t 4 -d 15s http://localhost:8110 +``` + + + +### Result + +#### With ReWrk for `rpxy`, `nginx` and `caddy` + +``` +---------------------------- +Benchmark [x86_64] with ReWrk +---------------------------- +Benchmark on rpxy +Beginning round 1... +Benchmarking 512 connections @ http://localhost:8080 for 15 second(s) + Latencies: + Avg Stdev Min Max + 20.37ms 8.95ms 1.63ms 160.27ms + Requests: + Total: 376345 Req/Sec: 25095.19 + Transfer: + Total: 295.61 MB Transfer Rate: 19.71 MB/Sec ++ --------------- + --------------- + +| Percentile | Avg Latency | ++ --------------- + --------------- + +| 99.9% | 112.50ms | +| 99% | 61.33ms | +| 95% | 44.26ms | +| 90% | 38.74ms | +| 75% | 32.00ms | +| 50% | 26.82ms | ++ --------------- + --------------- + + +626 Errors: error shutting down connection: Socket is not connected (os error 57) + +sleep 3 secs +---------------------------- +Benchmark on nginx +Beginning round 1... +Benchmarking 512 connections @ http://localhost:8090 for 15 second(s) + Latencies: + Avg Stdev Min Max + 23.45ms 12.42ms 1.18ms 154.44ms + Requests: + Total: 326685 Req/Sec: 21784.73 + Transfer: + Total: 265.22 MB Transfer Rate: 17.69 MB/Sec ++ --------------- + --------------- + +| Percentile | Avg Latency | ++ --------------- + --------------- + +| 99.9% | 96.85ms | +| 99% | 73.93ms | +| 95% | 57.57ms | +| 90% | 50.36ms | +| 75% | 40.57ms | +| 50% | 32.70ms | ++ --------------- + --------------- + + +657 Errors: error shutting down connection: Socket is not connected (os error 57) + +sleep 3 secs +---------------------------- +Benchmark on caddy +Beginning round 1... +Benchmarking 512 connections @ http://localhost:8100 for 15 second(s) + Latencies: + Avg Stdev Min Max + 45.71ms 50.47ms 0.88ms 908.49ms + Requests: + Total: 166917 Req/Sec: 11129.80 + Transfer: + Total: 133.77 MB Transfer Rate: 8.92 MB/Sec ++ --------------- + --------------- + +| Percentile | Avg Latency | ++ --------------- + --------------- + +| 99.9% | 608.92ms | +| 99% | 351.18ms | +| 95% | 210.56ms | +| 90% | 162.68ms | +| 75% | 106.97ms | +| 50% | 73.90ms | ++ --------------- + --------------- + + +646 Errors: error shutting down connection: Socket is not connected (os error 57) + +sleep 3 secs +``` + +#### With Wrk for `rpxy`, `nginx`, `caddy` and `sozu` + +``` +---------------------------- +Benchmark [x86_64] with Wrk +---------------------------- +Benchmark on rpxy +Running 15s test @ http://localhost:8080 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 18.68ms 8.09ms 122.64ms 74.03% + Req/Sec 6.95k 815.23 8.45k 83.83% + 414819 requests in 15.01s, 326.37MB read + Socket errors: connect 0, read 608, write 0, timeout 0 +Requests/sec: 27627.79 +Transfer/sec: 21.74MB + +sleep 3 secs +---------------------------- +Benchmark on nginx +Running 15s test @ http://localhost:8090 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 23.34ms 13.80ms 126.06ms 74.66% + Req/Sec 5.71k 607.41 7.07k 73.17% + 341127 requests in 15.03s, 277.50MB read + Socket errors: connect 0, read 641, write 0, timeout 0 +Requests/sec: 22701.54 +Transfer/sec: 18.47MB + +sleep 3 secs +---------------------------- +Benchmark on caddy +Running 15s test @ http://localhost:8100 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 54.19ms 55.63ms 674.53ms 88.55% + Req/Sec 2.92k 1.40k 5.57k 56.17% + 174748 requests in 15.03s, 140.61MB read + Socket errors: connect 0, read 660, write 0, timeout 0 + Non-2xx or 3xx responses: 70 +Requests/sec: 11624.63 +Transfer/sec: 9.35MB + +sleep 3 secs +---------------------------- +Benchmark on sozu +Running 15s test @ http://localhost:8110 + 4 threads and 512 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 19.78ms 4.89ms 98.09ms 76.88% + Req/Sec 6.49k 824.75 8.11k 76.17% + 387744 requests in 15.02s, 329.11MB read + Socket errors: connect 0, read 647, write 0, timeout 0 +Requests/sec: 25821.93 +Transfer/sec: 21.92MB +``` diff --git a/bench/bench.amd64.sh b/bench/bench.amd64.sh new file mode 100644 index 0000000..de73016 --- /dev/null +++ b/bench/bench.amd64.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +echo "----------------------------" +echo "Benchmark [x86_64] with ReWrk" + +echo "----------------------------" +echo "Benchmark on rpxy" +rewrk -c 512 -t 4 -d 15s -h http://localhost:8080 --pct + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on nginx" +rewrk -c 512 -t 4 -d 15s -h http://localhost:8090 --pct + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on caddy" +rewrk -c 512 -t 4 -d 15s -h http://localhost:8100 --pct + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark [x86_64] with Wrk" + +echo "----------------------------" +echo "Benchmark on rpxy" +wrk -c 512 -t 4 -d 15s http://localhost:8080 + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on nginx" +wrk -c 512 -t 4 -d 15s http://localhost:8090 + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on caddy" +wrk -c 512 -t 4 -d 15s http://localhost:8100 + +echo "sleep 3 secs" +sleep 3 + +echo "----------------------------" +echo "Benchmark on sozu" +wrk -c 512 -t 4 -d 15s http://localhost:8110 diff --git a/bench/bench.sh b/bench/bench.sh index 0d5477f..912f5fa 100644 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -12,6 +12,8 @@ # echo "Benchmark on caddy" # ab -c 100 -n 10000 http://127.0.0.1:8100/index.html +echo "----------------------------" +echo "Benchmark [Any Arch]" echo "----------------------------" echo "Benchmark on rpxy" diff --git a/bench/docker-compose.amd64.yml b/bench/docker-compose.amd64.yml new file mode 100644 index 0000000..a440138 --- /dev/null +++ b/bench/docker-compose.amd64.yml @@ -0,0 +1,96 @@ +version: "3" +services: + nginx: + image: nginx:alpine + container_name: backend-nginx + restart: unless-stopped + environment: + - VIRTUAL_HOST=localhost + - VIRTUAL_PORT=80 + expose: + - 80 + # ports: + # - 127.0.0.1:8888:80 + logging: + options: + max-size: "10m" + max-file: "3" + networks: + bench-nw: + ipv4_address: 192.168.100.100 + + rpxy-rp: + image: jqtype/rpxy + container_name: proxy-rpxy + pull_policy: never + build: + context: ../ + dockerfile: docker/Dockerfile + restart: unless-stopped + environment: + - LOG_LEVEL=info + - LOG_TO_FILE=false + ports: + - 127.0.0.1:8080:8080 + tty: false + volumes: + - ./rpxy.toml:/etc/rpxy.toml:ro + networks: + bench-nw: + + nginx-rp: + image: nginx:alpine + container_name: proxy-nginx + ports: + - 127.0.0.1:8090:80 + restart: unless-stopped + tty: false + privileged: true + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - /var/run/docker.sock:/tmp/docker.sock:ro + logging: + options: + max-size: "10m" + max-file: "3" + networks: + bench-nw: + + caddy-rp: + image: caddy:2 + container_name: proxy-caddy + ports: + - 127.0.0.1:8100:80 + restart: unless-stopped + tty: false + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + networks: + bench-nw: + + # Sozu wokrs only in X86_64 (amd64) environment + # Official image from sozu developers is still version 0.11.0. + # So we built it by ourselves locally. + sozu-rp: + image: jqtype/sozu + container_name: proxy-sozu + restart: unless-stopped + ports: + - 127.0.0.1:8110:80 + logging: + options: + max-size: "10m" + max-file: "3" + volumes: + - ./sozu-config.toml:/etc/sozu/config.toml + networks: + bench-nw: + +networks: + bench-nw: + name: bench-nw + driver: bridge + ipam: + driver: default + config: + - subnet: 192.168.100.0/24 diff --git a/bench/docker-compose.yml b/bench/docker-compose.yml index 23308aa..7b2b043 100644 --- a/bench/docker-compose.yml +++ b/bench/docker-compose.yml @@ -10,7 +10,7 @@ services: expose: - 80 # ports: - # - 127.0.0.1:8888:80 + # - 127.0.0.1:8888:80 logging: options: max-size: "10m" @@ -25,7 +25,7 @@ services: pull_policy: never build: context: ../ - dockerfile: docker/Dockerfile.amd64 + dockerfile: docker/Dockerfile restart: unless-stopped environment: - LOG_LEVEL=info diff --git a/bench/nginx.conf b/bench/nginx.conf index 0359ac6..0ef89e3 100644 --- a/bench/nginx.conf +++ b/bench/nginx.conf @@ -59,7 +59,7 @@ # return 503; # } # localhost -upstream localhost { +upstream backend { ## Can be connected with "bench-nw" network # backend-nginx server 192.168.100.100:80; @@ -69,6 +69,6 @@ server { listen 80 ; # access_log /var/log/nginx/access.log vhost; location / { - proxy_pass http://localhost; + proxy_pass http://backend; } } diff --git a/bench/sozu-config.toml b/bench/sozu-config.toml new file mode 100644 index 0000000..ab76515 --- /dev/null +++ b/bench/sozu-config.toml @@ -0,0 +1,16 @@ +log_level = "info" +log_target = "stdout" +max_connections = 512 +activate_listeners = true + +[[listeners]] +protocol = "http" +# listening address +address = "0.0.0.0:80" + +[clusters] + +[clusters.backend] +protocol = "http" +frontends = [{ address = "0.0.0.0:80", hostname = "localhost" }] +backends = [{ address = "192.168.100.100:80" }] diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile similarity index 98% rename from docker/Dockerfile.amd64 rename to docker/Dockerfile index da27439..fd6d510 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile @@ -29,7 +29,6 @@ RUN apt-get update && apt-get install -qy --no-install-recommends $BUILD_DEPS && ######################################## FROM base AS runner -ENV TAG_NAME=amd64 ENV RUNTIME_DEPS logrotate ca-certificates gosu RUN apt-get update && \ diff --git a/docker/Dockerfile.amd64-slim b/docker/Dockerfile.amd64-slim index fb0246e..12cd261 100644 --- a/docker/Dockerfile.amd64-slim +++ b/docker/Dockerfile.amd64-slim @@ -18,7 +18,6 @@ RUN echo "Building rpxy from source" && \ FROM alpine:latest as runner LABEL maintainer="Jun Kurihara" -ENV TAG_NAME=amd64-slim ENV TARGET_DIR=x86_64-unknown-linux-musl ENV RUNTIME_DEPS logrotate ca-certificates su-exec diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 716d0de..d817918 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,7 +10,7 @@ services: - 127.0.0.1:8443:8443 build: context: ../ - dockerfile: ./docker/Dockerfile.amd64 + dockerfile: ./docker/Dockerfile environment: - LOG_LEVEL=debug - LOG_TO_FILE=true