add benchmark result and fix Dockerfile names since it does not depends on architecture

.github/workflows/docker_build_push.yml

@@ -30,7 +30,7 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Release build and push x86_64
+      - name: Release build and push
         if: ${{ env.BRANCH == 'main' }}
         uses: docker/build-push-action@v4
         with:
@@ -38,7 +38,8 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest
-          file: ./docker/Dockerfile.amd64
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
 
       - name: Release build and push x86_64-slim
         if: ${{ env.BRANCH == 'main' }}
@@ -49,8 +50,9 @@ jobs:
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:slim, ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:latest-slim
           file: ./docker/Dockerfile.amd64-slim
+          platforms: linux/amd64
 
-      - name: Nightly build and push x86_64
+      - name: Nightly build and push
         if: ${{ env.BRANCH == 'develop' }}
         uses: docker/build-push-action@v4
         with:
@@ -58,4 +60,5 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.IMAGE_NAME }}:nightly
-          file: ./docker/Dockerfile.amd64
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64

bench/README.md

@@ -1,23 +1,25 @@
-# Sample Benchmark Result
-
-Done at Jul. 15, 2023
+# Sample Benchmark Results
 
 This test simply measures the performance of several reverse proxy through HTTP/1.1 by the following command using [`rewrk`](https://github.com/lnx-search/rewrk).
 
-```bash
+```sh:
 $ rewrk -c 512 -t 4 -d 15s -h http://localhost:8080 --pct
 ```
 
-## Environment
+## Tests on `linux/arm64/v8`
+
+Done at Jul. 15, 2023
+
+### Environment
 
 - `rpxy` commit id: `1da7e5bfb77d1ce4ee8d6cfc59b1c725556fc192`
 - Docker Desktop 4.21.1 (114176)
-- ReWrk 0.3.1
+- ReWrk 0.3.2
 - Macbook Pro '14 (2021, M1 Max, 64GB RAM)
 
+The docker images of `nginx` and `caddy` for `linux/arm64/v8` are pulled from the official registry.
 
-
-## Result
+### Result for `rpxy`, `nginx` and `caddy`
 
 ```
 ----------------------------
@@ -94,3 +96,168 @@ Benchmarking 512 connections @ http://localhost:8100 for 15 second(s)
 
 708 Errors: error shutting down connection: Socket is not connected (os error 57)
 ```
+
+## Results on `linux/amd64`
+
+Done at Jul. 24, 2023
+
+### Environment
+
+- `rpxy` commit id: `7c0945a5124418aa9a1024568c1989bb77cf312f`
+- Docker Desktop 4.21.1 (114176)
+- ReWrk 0.3.2 and Wrk 0.4.2
+- iMac '27 (2020, 10-Core Intel Core i9, 128GB RAM)
+
+The docker images of `nginx` and `caddy` for `linux/amd64` were pulled from the official registry. For `Sozu`, the official docker image from its developers was still version 0.11.0 (currently the latest version is 0.15.2). So we built it by ourselves locally using the `Sozu`'s official [`Dockerfile`](https://github.com/sozu-proxy/sozu/blob/main/Dockerfile).
+
+Also, when `Sozu` is configured as an HTTP reverse proxy, it cannot handle HTTP request messages emit from `ReWrk` due to hostname parsing errors though it can correctly handle messages dispatched from `curl` and browsers. So, we additionally test using [`Wrk`](https://github.com/wg/wrk) to examine `Sozu` with the following command.
+
+```sh:
+$ wrk -c 512 -t 4 -d 15s http://localhost:8110
+```
+
+<!-- ```
+ERROR  Error connecting to backend: Could not get cluster id from request: Host not found: http://localhost:8110/: Hostname parsing failed for host http://localhost:8110/: Parsing Error: Error { input: [58, 47, 47, 108, 111, 99, 97, 108, 104, 111, 115, 116, 58, 56, 49, 49, 48, 47], code: Eof }
+``` -->
+
+### Result
+
+#### With ReWrk for `rpxy`, `nginx` and `caddy`
+
+```
+----------------------------
+Benchmark [x86_64] with ReWrk
+----------------------------
+Benchmark on rpxy
+Beginning round 1...
+Benchmarking 512 connections @ http://localhost:8080 for 15 second(s)
+  Latencies:
+    Avg      Stdev    Min      Max
+    20.37ms  8.95ms   1.63ms   160.27ms
+  Requests:
+    Total: 376345  Req/Sec: 25095.19
+  Transfer:
+    Total: 295.61 MB Transfer Rate: 19.71 MB/Sec
++ --------------- + --------------- +
+|   Percentile    |   Avg Latency   |
++ --------------- + --------------- +
+|      99.9%      |    112.50ms     |
+|       99%       |     61.33ms     |
+|       95%       |     44.26ms     |
+|       90%       |     38.74ms     |
+|       75%       |     32.00ms     |
+|       50%       |     26.82ms     |
++ --------------- + --------------- +
+
+626 Errors: error shutting down connection: Socket is not connected (os error 57)
+
+sleep 3 secs
+----------------------------
+Benchmark on nginx
+Beginning round 1...
+Benchmarking 512 connections @ http://localhost:8090 for 15 second(s)
+  Latencies:
+    Avg      Stdev    Min      Max
+    23.45ms  12.42ms  1.18ms   154.44ms
+  Requests:
+    Total: 326685  Req/Sec: 21784.73
+  Transfer:
+    Total: 265.22 MB Transfer Rate: 17.69 MB/Sec
++ --------------- + --------------- +
+|   Percentile    |   Avg Latency   |
++ --------------- + --------------- +
+|      99.9%      |     96.85ms     |
+|       99%       |     73.93ms     |
+|       95%       |     57.57ms     |
+|       90%       |     50.36ms     |
+|       75%       |     40.57ms     |
+|       50%       |     32.70ms     |
++ --------------- + --------------- +
+
+657 Errors: error shutting down connection: Socket is not connected (os error 57)
+
+sleep 3 secs
+----------------------------
+Benchmark on caddy
+Beginning round 1...
+Benchmarking 512 connections @ http://localhost:8100 for 15 second(s)
+  Latencies:
+    Avg      Stdev    Min      Max
+    45.71ms  50.47ms  0.88ms   908.49ms
+  Requests:
+    Total: 166917  Req/Sec: 11129.80
+  Transfer:
+    Total: 133.77 MB Transfer Rate: 8.92 MB/Sec
++ --------------- + --------------- +
+|   Percentile    |   Avg Latency   |
++ --------------- + --------------- +
+|      99.9%      |    608.92ms     |
+|       99%       |    351.18ms     |
+|       95%       |    210.56ms     |
+|       90%       |    162.68ms     |
+|       75%       |    106.97ms     |
+|       50%       |     73.90ms     |
++ --------------- + --------------- +
+
+646 Errors: error shutting down connection: Socket is not connected (os error 57)
+
+sleep 3 secs
+```
+
+#### With Wrk for `rpxy`, `nginx`, `caddy` and `sozu`
+
+```
+----------------------------
+Benchmark [x86_64] with Wrk
+----------------------------
+Benchmark on rpxy
+Running 15s test @ http://localhost:8080
+  4 threads and 512 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    18.68ms    8.09ms 122.64ms   74.03%
+    Req/Sec     6.95k   815.23     8.45k    83.83%
+  414819 requests in 15.01s, 326.37MB read
+  Socket errors: connect 0, read 608, write 0, timeout 0
+Requests/sec:  27627.79
+Transfer/sec:     21.74MB
+
+sleep 3 secs
+----------------------------
+Benchmark on nginx
+Running 15s test @ http://localhost:8090
+  4 threads and 512 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    23.34ms   13.80ms 126.06ms   74.66%
+    Req/Sec     5.71k   607.41     7.07k    73.17%
+  341127 requests in 15.03s, 277.50MB read
+  Socket errors: connect 0, read 641, write 0, timeout 0
+Requests/sec:  22701.54
+Transfer/sec:     18.47MB
+
+sleep 3 secs
+----------------------------
+Benchmark on caddy
+Running 15s test @ http://localhost:8100
+  4 threads and 512 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    54.19ms   55.63ms 674.53ms   88.55%
+    Req/Sec     2.92k     1.40k    5.57k    56.17%
+  174748 requests in 15.03s, 140.61MB read
+  Socket errors: connect 0, read 660, write 0, timeout 0
+  Non-2xx or 3xx responses: 70
+Requests/sec:  11624.63
+Transfer/sec:      9.35MB
+
+sleep 3 secs
+----------------------------
+Benchmark on sozu
+Running 15s test @ http://localhost:8110
+  4 threads and 512 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    19.78ms    4.89ms  98.09ms   76.88%
+    Req/Sec     6.49k   824.75     8.11k    76.17%
+  387744 requests in 15.02s, 329.11MB read
+  Socket errors: connect 0, read 647, write 0, timeout 0
+Requests/sec:  25821.93
+Transfer/sec:     21.92MB
+```

bench/bench.amd64.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+echo "----------------------------"
+echo "Benchmark [x86_64] with ReWrk"
+
+echo "----------------------------"
+echo "Benchmark on rpxy"
+rewrk -c 512 -t 4 -d 15s -h http://localhost:8080 --pct
+
+echo "sleep 3 secs"
+sleep 3
+
+echo "----------------------------"
+echo "Benchmark on nginx"
+rewrk -c 512 -t 4 -d 15s -h http://localhost:8090 --pct
+
+echo "sleep 3 secs"
+sleep 3
+
+echo "----------------------------"
+echo "Benchmark on caddy"
+rewrk -c 512 -t 4 -d 15s -h http://localhost:8100 --pct
+
+echo "sleep 3 secs"
+sleep 3
+
+echo "----------------------------"
+echo "Benchmark [x86_64] with Wrk"
+
+echo "----------------------------"
+echo "Benchmark on rpxy"
+wrk -c 512 -t 4 -d 15s http://localhost:8080
+
+echo "sleep 3 secs"
+sleep 3
+
+echo "----------------------------"
+echo "Benchmark on nginx"
+wrk -c 512 -t 4 -d 15s http://localhost:8090
+
+echo "sleep 3 secs"
+sleep 3
+
+echo "----------------------------"
+echo "Benchmark on caddy"
+wrk -c 512 -t 4 -d 15s http://localhost:8100
+
+echo "sleep 3 secs"
+sleep 3
+
+echo "----------------------------"
+echo "Benchmark on sozu"
+wrk -c 512 -t 4 -d 15s http://localhost:8110

bench/bench.sh

@@ -12,6 +12,8 @@
 # echo "Benchmark on caddy"
 # ab -c 100 -n 10000  http://127.0.0.1:8100/index.html
 
+echo "----------------------------"
+echo "Benchmark [Any Arch]"
 
 echo "----------------------------"
 echo "Benchmark on rpxy"

bench/docker-compose.amd64.yml

@@ -0,0 +1,96 @@
+version: "3"
+services:
+  nginx:
+    image: nginx:alpine
+    container_name: backend-nginx
+    restart: unless-stopped
+    environment:
+      - VIRTUAL_HOST=localhost
+      - VIRTUAL_PORT=80
+    expose:
+      - 80
+    # ports:
+    # - 127.0.0.1:8888:80
+    logging:
+      options:
+        max-size: "10m"
+        max-file: "3"
+    networks:
+      bench-nw:
+        ipv4_address: 192.168.100.100
+
+  rpxy-rp:
+    image: jqtype/rpxy
+    container_name: proxy-rpxy
+    pull_policy: never
+    build:
+      context: ../
+      dockerfile: docker/Dockerfile
+    restart: unless-stopped
+    environment:
+      - LOG_LEVEL=info
+      - LOG_TO_FILE=false
+    ports:
+      - 127.0.0.1:8080:8080
+    tty: false
+    volumes:
+      - ./rpxy.toml:/etc/rpxy.toml:ro
+    networks:
+      bench-nw:
+
+  nginx-rp:
+    image: nginx:alpine
+    container_name: proxy-nginx
+    ports:
+      - 127.0.0.1:8090:80
+    restart: unless-stopped
+    tty: false
+    privileged: true
+    volumes:
+      - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    logging:
+      options:
+        max-size: "10m"
+        max-file: "3"
+    networks:
+      bench-nw:
+
+  caddy-rp:
+    image: caddy:2
+    container_name: proxy-caddy
+    ports:
+      - 127.0.0.1:8100:80
+    restart: unless-stopped
+    tty: false
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+    networks:
+      bench-nw:
+
+  # Sozu wokrs only in X86_64 (amd64) environment
+  # Official image from sozu developers is still version 0.11.0.
+  # So we built it by ourselves locally.
+  sozu-rp:
+    image: jqtype/sozu
+    container_name: proxy-sozu
+    restart: unless-stopped
+    ports:
+      - 127.0.0.1:8110:80
+    logging:
+      options:
+        max-size: "10m"
+        max-file: "3"
+    volumes:
+      - ./sozu-config.toml:/etc/sozu/config.toml
+    networks:
+      bench-nw:
+
+networks:
+  bench-nw:
+    name: bench-nw
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 192.168.100.0/24

bench/docker-compose.yml

@@ -10,7 +10,7 @@ services:
     expose:
       - 80
     # ports:
-      # - 127.0.0.1:8888:80
+    # - 127.0.0.1:8888:80
     logging:
       options:
         max-size: "10m"
@@ -25,7 +25,7 @@ services:
     pull_policy: never
     build:
       context: ../
-      dockerfile: docker/Dockerfile.amd64
+      dockerfile: docker/Dockerfile
     restart: unless-stopped
     environment:
       - LOG_LEVEL=info

bench/nginx.conf

@@ -59,7 +59,7 @@
 # 	return 503;
 # }
 	# localhost
-upstream localhost {
+upstream backend {
         ## Can be connected with "bench-nw" network
         # backend-nginx
         server 192.168.100.100:80;
@@ -69,6 +69,6 @@ server {
 	listen 80 ;
 	# access_log /var/log/nginx/access.log vhost;
 	location / {
-		proxy_pass http://localhost;
+		proxy_pass http://backend;
 }
 }

bench/sozu-config.toml

@@ -0,0 +1,16 @@
+log_level = "info"
+log_target = "stdout"
+max_connections = 512
+activate_listeners = true
+
+[[listeners]]
+protocol = "http"
+# listening address
+address = "0.0.0.0:80"
+
+[clusters]
+
+[clusters.backend]
+protocol = "http"
+frontends = [{ address = "0.0.0.0:80", hostname = "localhost" }]
+backends = [{ address = "192.168.100.100:80" }]

docker/Dockerfile

@@ -29,7 +29,6 @@ RUN apt-get update && apt-get install -qy --no-install-recommends $BUILD_DEPS &&
 ########################################
 FROM base AS runner
 
-ENV TAG_NAME=amd64
 ENV RUNTIME_DEPS logrotate ca-certificates gosu
 
 RUN apt-get update && \

docker/Dockerfile.amd64-slim

@@ -18,7 +18,6 @@ RUN echo "Building rpxy from source" && \
 FROM alpine:latest as runner
 LABEL maintainer="Jun Kurihara"
 
-ENV TAG_NAME=amd64-slim
 ENV TARGET_DIR=x86_64-unknown-linux-musl
 ENV RUNTIME_DEPS logrotate ca-certificates su-exec
 

docker/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - 127.0.0.1:8443:8443
     build:
       context: ../
-      dockerfile: ./docker/Dockerfile.amd64
+      dockerfile: ./docker/Dockerfile
     environment:
       - LOG_LEVEL=debug
       - LOG_TO_FILE=true