diff --git a/.dockerignore b/.dockerignore index 3538235..4294ee6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ bench/ .private/ .github/ example-certs/ +legacy-lib/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b04184..cee7007 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,35 +44,35 @@ jobs: platform: linux/arm64 tags-suffix: "-s2n" - - target: "gnu" - build-feature: "-native-roots" - platform: linux/amd64 - tags-suffix: "-native-roots" + # - target: "gnu" + # build-feature: "-native-roots" + # platform: linux/amd64 + # tags-suffix: "-native-roots" - - target: "gnu" - build-feature: "-native-roots" - platform: linux/arm64 - tags-suffix: "-native-roots" + # - target: "gnu" + # build-feature: "-native-roots" + # platform: linux/arm64 + # tags-suffix: "-native-roots" - - target: "musl" - build-feature: "-native-roots" - platform: linux/amd64 - tags-suffix: "-slim-native-roots" + # - target: "musl" + # build-feature: "-native-roots" + # platform: linux/amd64 + # tags-suffix: "-slim-native-roots" - - target: "musl" - build-feature: "-native-roots" - platform: linux/arm64 - tags-suffix: "-slim-native-roots" + # - target: "musl" + # build-feature: "-native-roots" + # platform: linux/arm64 + # tags-suffix: "-slim-native-roots" - - target: "gnu" - build-feature: "-s2n-native-roots" - platform: linux/amd64 - tags-suffix: "-s2n-native-roots" + # - target: "gnu" + # build-feature: "-s2n-native-roots" + # platform: linux/amd64 + # tags-suffix: "-s2n-native-roots" - - target: "gnu" - build-feature: "-s2n-native-roots" - platform: linux/arm64 - tags-suffix: "-s2n-native-roots" + # - target: "gnu" + # build-feature: "-s2n-native-roots" + # platform: linux/arm64 + # tags-suffix: "-s2n-native-roots" steps: - run: "echo 'The relese triggering workflows passed'" @@ -81,8 +81,8 @@ jobs: id: "set-env" run: | if [ ${{ matrix.platform }} == 'linux/amd64' ]; then PLATFORM_MAP="x86_64"; else PLATFORM_MAP="aarch64"; fi - if [ ${{ github.ref_name }} == 'develop' ]; then BUILD_NAME="-nightly"; else BUILD_NAME=""; fi - if [ ${{ github.ref_name }} == 'develop' ]; then BUILD_IMG="nightly"; else BUILD_IMG="latest"; fi + if [ ${{ github.ref_name == 'develop' && github.event.client_payload.pull_request.head == 'develop' && github.event.client_payload.pull_request.base == 'main' }} || ${{ github.ref_name == 'main' }}]; then BUILD_NAME=""; else BUILD_NAME="-nightly"; fi + if [ ${{ github.ref_name }} == 'main' ]; then BUILD_IMG="latest"; else BUILD_IMG="nightly"; fi echo "build_img=${BUILD_IMG}" >> $GITHUB_OUTPUT echo "target_name=rpxy${BUILD_NAME}-${PLATFORM_MAP}-unknown-linux-${{ matrix.target }}${{ matrix.build-feature }}" >> $GITHUB_OUTPUT @@ -93,7 +93,7 @@ jobs: docker cp ${CONTAINER_ID}:/rpxy/bin/rpxy /tmp/${{ steps.set-env.outputs.target_name }} - name: "upload artifacts" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ steps.set-env.outputs.target_name }} path: "/tmp/${{ steps.set-env.outputs.target_name }}" @@ -122,7 +122,7 @@ jobs: - name: download artifacts if: ${{ steps.regex-match.outputs.match != ''}} - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: /tmp/rpxy diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 60dd7ff..391d7cd 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -44,7 +44,7 @@ jobs: - target: "s2n" dockerfile: ./docker/Dockerfile build-args: | - "CARGO_FEATURES=--no-default-features --features=http3-s2n,cache" + "CARGO_FEATURES=--no-default-features --features=http3-s2n,cache,native-tls-backend" "ADDITIONAL_DEPS=pkg-config libssl-dev cmake libclang1 gcc g++" platforms: linux/amd64,linux/arm64 tags-suffix: "-s2n" @@ -53,42 +53,42 @@ jobs: jqtype/rpxy:s2n ghcr.io/junkurihara/rust-rpxy:s2n - - target: "native-roots" - dockerfile: ./docker/Dockerfile - platforms: linux/amd64,linux/arm64 - build-args: | - "CARGO_FEATURES=--no-default-features --features=http3-quinn,cache,native-roots" - tags-suffix: "-native-roots" - # Aliases must be used only for release builds - aliases: | - jqtype/rpxy:native-roots - ghcr.io/junkurihara/rust-rpxy:native-roots + # - target: "native-roots" + # dockerfile: ./docker/Dockerfile + # platforms: linux/amd64,linux/arm64 + # build-args: | + # "CARGO_FEATURES=--no-default-features --features=http3-quinn,cache,native-roots" + # tags-suffix: "-native-roots" + # # Aliases must be used only for release builds + # aliases: | + # jqtype/rpxy:native-roots + # ghcr.io/junkurihara/rust-rpxy:native-roots - - target: "slim-native-roots" - dockerfile: ./docker/Dockerfile-slim - build-args: | - "CARGO_FEATURES=--no-default-features --features=http3-quinn,cache,native-roots" - build-contexts: | - messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl - messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl - platforms: linux/amd64,linux/arm64 - tags-suffix: "-slim-native-roots" - # Aliases must be used only for release builds - aliases: | - jqtype/rpxy:slim-native-roots - ghcr.io/junkurihara/rust-rpxy:slim-native-roots + # - target: "slim-native-roots" + # dockerfile: ./docker/Dockerfile-slim + # build-args: | + # "CARGO_FEATURES=--no-default-features --features=http3-quinn,cache,native-roots" + # build-contexts: | + # messense/rust-musl-cross:amd64-musl=docker-image://messense/rust-musl-cross:x86_64-musl + # messense/rust-musl-cross:arm64-musl=docker-image://messense/rust-musl-cross:aarch64-musl + # platforms: linux/amd64,linux/arm64 + # tags-suffix: "-slim-native-roots" + # # Aliases must be used only for release builds + # aliases: | + # jqtype/rpxy:slim-native-roots + # ghcr.io/junkurihara/rust-rpxy:slim-native-roots - - target: "s2n-native-roots" - dockerfile: ./docker/Dockerfile - build-args: | - "CARGO_FEATURES=--no-default-features --features=http3-s2n,cache,native-roots" - "ADDITIONAL_DEPS=pkg-config libssl-dev cmake libclang1 gcc g++" - platforms: linux/amd64,linux/arm64 - tags-suffix: "-s2n-native-roots" - # Aliases must be used only for release builds - aliases: | - jqtype/rpxy:s2n-native-roots - ghcr.io/junkurihara/rust-rpxy:s2n-native-roots + # - target: "s2n-native-roots" + # dockerfile: ./docker/Dockerfile + # build-args: | + # "CARGO_FEATURES=--no-default-features --features=http3-s2n,cache,native-roots" + # "ADDITIONAL_DEPS=pkg-config libssl-dev cmake libclang1 gcc g++" + # platforms: linux/amd64,linux/arm64 + # tags-suffix: "-s2n-native-roots" + # # Aliases must be used only for release builds + # aliases: | + # jqtype/rpxy:s2n-native-roots + # ghcr.io/junkurihara/rust-rpxy:s2n-native-roots steps: - name: Checkout diff --git a/Cargo.toml b/Cargo.toml index 7868088..c512b18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -members = ["rpxy-bin", "rpxy-lib", "legacy-lib"] +members = ["rpxy-bin", "rpxy-lib"] exclude = ["submodules"] resolver = "2" diff --git a/rpxy-bin/Cargo.toml b/rpxy-bin/Cargo.toml index a6e5720..6ec94f1 100644 --- a/rpxy-bin/Cargo.toml +++ b/rpxy-bin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rpxy" -version = "0.7.0" +version = "0.7.0-alpha.0" authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" @@ -12,23 +12,25 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["http3-quinn", "cache"] +default = ["http3-quinn", "cache", "native-tls-backend"] http3-quinn = ["rpxy-lib/http3-quinn"] http3-s2n = ["rpxy-lib/http3-s2n"] +native-tls-backend = ["rpxy-lib/native-tls-backend"] +# Not yet implemented +rustls-backend = ["rpxy-lib/rustls-backend"] cache = ["rpxy-lib/cache"] native-roots = ["rpxy-lib/native-roots"] [dependencies] rpxy-lib = { path = "../rpxy-lib/", default-features = false, features = [ "sticky-cookie", - "native-tls-backend", ] } anyhow = "1.0.75" rustc-hash = "1.1.0" serde = { version = "1.0.193", default-features = false, features = ["derive"] } derive_builder = "0.12.0" -tokio = { version = "1.34.0", default-features = false, features = [ +tokio = { version = "1.35.0", default-features = false, features = [ "net", "rt-multi-thread", "time", diff --git a/rpxy-lib/Cargo.toml b/rpxy-lib/Cargo.toml index f30f4bb..22e091f 100644 --- a/rpxy-lib/Cargo.toml +++ b/rpxy-lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rpxy-lib" -version = "0.7.0" +version = "0.7.0-alpha.0" authors = ["Jun Kurihara"] homepage = "https://github.com/junkurihara/rust-rpxy" repository = "https://github.com/junkurihara/rust-rpxy" @@ -21,11 +21,11 @@ http3-s2n = [ "s2n-quic-rustls", "s2n-quic-h3", ] +cache = ["http-cache-semantics", "lru", "sha2", "base64"] sticky-cookie = ["base64", "sha2", "chrono"] native-tls-backend = ["hyper-tls"] -rustls-backend = [] -cache = ["http-cache-semantics", "lru", "sha2", "base64"] -native-roots = [] #"hyper-rustls/native-tokio"] +rustls-backend = [] # not implemented yet +native-roots = [] #"hyper-rustls/native-tokio"] # not implemented yet [dependencies] rand = "0.8.5" @@ -33,7 +33,7 @@ rustc-hash = "1.1.0" bytes = "1.5.0" derive_builder = "0.12.0" futures = { version = "0.3.29", features = ["alloc", "async-await"] } -tokio = { version = "1.34.0", default-features = false, features = [ +tokio = { version = "1.35.0", default-features = false, features = [ "net", "rt-multi-thread", "time", @@ -57,7 +57,10 @@ futures-util = { version = "0.3.29", default-features = false } futures-channel = { version = "0.3.29", default-features = false } # http client for upstream -hyper-tls = { version = "0.6.0", features = ["alpn"], optional = true } +hyper-tls = { version = "0.6.0", features = [ + "alpn", + "vendored", +], optional = true } # hyper-rustls = { version = "0.24.2", default-features = false, features = [ # "tokio-runtime", # "webpki-tokio", diff --git a/rpxy-lib/src/forwarder/cache/cache_error.rs b/rpxy-lib/src/forwarder/cache/cache_error.rs index 35eae83..341c928 100644 --- a/rpxy-lib/src/forwarder/cache/cache_error.rs +++ b/rpxy-lib/src/forwarder/cache/cache_error.rs @@ -1,6 +1,6 @@ use thiserror::Error; -pub type CacheResult = std::result::Result; +pub(crate) type CacheResult = std::result::Result; /// Describes things that can go wrong in the Rpxy #[derive(Debug, Error)] @@ -9,9 +9,6 @@ pub enum CacheError { #[error("Invalid null request and/or response")] NullRequestOrResponse, - #[error("Failed to write byte buffer")] - FailedToWriteByteBufferForCache, - #[error("Failed to acquire mutex lock for cache")] FailedToAcquiredMutexLockForCache, diff --git a/rpxy-lib/src/forwarder/cache/cache_main.rs b/rpxy-lib/src/forwarder/cache/cache_main.rs index 3c85d0c..f3fc463 100644 --- a/rpxy-lib/src/forwarder/cache/cache_main.rs +++ b/rpxy-lib/src/forwarder/cache/cache_main.rs @@ -30,7 +30,7 @@ use tokio::{ /* ---------------------------------------------- */ #[derive(Clone, Debug)] /// Cache main manager -pub struct RpxyCache { +pub(crate) struct RpxyCache { /// Inner lru cache manager storing http message caching policy inner: LruCacheManager, /// Managing cache file objects through RwLock's lock mechanism for file lock @@ -47,7 +47,7 @@ pub struct RpxyCache { impl RpxyCache { /// Generate cache storage - pub async fn new(globals: &Globals) -> Option { + pub(crate) async fn new(globals: &Globals) -> Option { if !globals.proxy_config.cache_enabled { return None; } @@ -80,7 +80,7 @@ impl RpxyCache { } /// Count cache entries - pub async fn count(&self) -> (usize, usize, usize) { + pub(crate) async fn count(&self) -> (usize, usize, usize) { let total = self.inner.count(); let file = self.file_store.count().await; let on_memory = total - file; @@ -88,7 +88,7 @@ impl RpxyCache { } /// Put response into the cache - pub async fn put( + pub(crate) async fn put( &self, uri: &hyper::Uri, mut body: Incoming, @@ -186,7 +186,7 @@ impl RpxyCache { } /// Get cached response - pub async fn get(&self, req: &Request) -> Option> { + pub(crate) async fn get(&self, req: &Request) -> Option> { debug!( "Current cache status: (total, on-memory, file) = {:?}", self.count().await @@ -394,7 +394,7 @@ impl FileStoreInner { #[derive(Clone, Debug)] /// Cache target in hybrid manner of on-memory and file system -pub enum CacheFileOrOnMemory { +pub(crate) enum CacheFileOrOnMemory { /// Pointer to the temporary cache file File(PathBuf), /// Cached body itself @@ -418,11 +418,11 @@ impl CacheFileOrOnMemory { /// Cache object definition struct CacheObject { /// Cache policy to determine if the stored cache can be used as a response to a new incoming request - pub policy: CachePolicy, + policy: CachePolicy, /// Cache target: on-memory object or temporary file - pub target: CacheFileOrOnMemory, + target: CacheFileOrOnMemory, /// SHA256 hash of target to strongly bind the cache metadata (this object) and file target - pub hash: Bytes, + hash: Bytes, } /* ---------------------------------------------- */ @@ -490,7 +490,7 @@ impl LruCacheManager { /* ---------------------------------------------- */ /// Generate cache policy if the response is cacheable -pub fn get_policy_if_cacheable( +pub(crate) fn get_policy_if_cacheable( req: Option<&Request>, res: Option<&Response>, ) -> CacheResult> diff --git a/rpxy-lib/src/forwarder/cache/mod.rs b/rpxy-lib/src/forwarder/cache/mod.rs index cfe5a1b..076eaa3 100644 --- a/rpxy-lib/src/forwarder/cache/mod.rs +++ b/rpxy-lib/src/forwarder/cache/mod.rs @@ -2,4 +2,4 @@ mod cache_error; mod cache_main; pub use cache_error::CacheError; -pub use cache_main::{get_policy_if_cacheable, CacheFileOrOnMemory, RpxyCache}; +pub(crate) use cache_main::{get_policy_if_cacheable, RpxyCache}; diff --git a/rpxy-lib/src/forwarder/client.rs b/rpxy-lib/src/forwarder/client.rs index c8a8ec7..26c2276 100644 --- a/rpxy-lib/src/forwarder/client.rs +++ b/rpxy-lib/src/forwarder/client.rs @@ -121,7 +121,7 @@ where ::Error: Into>, { /// Build inner client with http - pub fn try_new(_globals: &Arc) -> RpxyResult { + pub async fn try_new(_globals: &Arc) -> RpxyResult { warn!( " -------------------------------------------------------------------------------------------------- @@ -134,6 +134,7 @@ Please enable native-tls-backend or rustls-backend feature to enable TLS support let mut http = HttpConnector::new(); http.set_reuse_address(true); let inner = Client::builder(executor).build::<_, B>(http); + let inner_h2 = inner.clone(); Ok(Self { inner, @@ -191,7 +192,7 @@ where #[cfg(feature = "rustls-backend")] /// Build forwarder with hyper-rustls (rustls) -impl Forwarder, B1> +impl Forwarder where B1: Body + Send + Unpin + 'static, ::Data: Send, diff --git a/rpxy-lib/src/hyper_ext/body_type.rs b/rpxy-lib/src/hyper_ext/body_type.rs index a143eac..ca44756 100644 --- a/rpxy-lib/src/hyper_ext/body_type.rs +++ b/rpxy-lib/src/hyper_ext/body_type.rs @@ -1,11 +1,12 @@ use super::body::IncomingLike; use crate::error::RpxyError; -use http_body_util::{combinators, BodyExt, Empty, Full}; -use hyper::body::{Body, Bytes, Incoming}; +use futures::channel::mpsc::UnboundedReceiver; +use http_body_util::{combinators, BodyExt, Empty, Full, StreamBody}; +use hyper::body::{Body, Bytes, Frame, Incoming}; use std::pin::Pin; /// Type for synthetic boxed body -pub(crate) type BoxBody = combinators::BoxBody; +pub type BoxBody = combinators::BoxBody; /// helper function to build a empty body pub(crate) fn empty() -> BoxBody { @@ -17,11 +18,12 @@ pub(crate) fn full(body: Bytes) -> BoxBody { Full::new(body).map_err(|never| match never {}).boxed() } +#[allow(unused)] /* ------------------------------------ */ /// Request body used in this project /// - Incoming: just a type that only forwards the downstream request body to upstream. /// - IncomingLike: a Incoming-like type in which channel is used -pub(crate) enum RequestBody { +pub enum RequestBody { Incoming(Incoming), IncomingLike(IncomingLike), } @@ -42,24 +44,16 @@ impl Body for RequestBody { } /* ------------------------------------ */ -#[cfg(feature = "cache")] -use futures::channel::mpsc::UnboundedReceiver; -#[cfg(feature = "cache")] -use http_body_util::StreamBody; -#[cfg(feature = "cache")] -use hyper::body::Frame; - -#[cfg(feature = "cache")] -pub(crate) type UnboundedStreamBody = StreamBody, hyper::Error>>>; +pub type UnboundedStreamBody = StreamBody, hyper::Error>>>; +#[allow(unused)] /// Response body use in this project /// - Incoming: just a type that only forwards the upstream response body to downstream. /// - Boxed: a type that is generated from cache or synthetic response body, e.g.,, small byte object. /// - Streamed: another type that is generated from stream, e.g., large byte object. -pub(crate) enum ResponseBody { +pub enum ResponseBody { Incoming(Incoming), Boxed(BoxBody), - #[cfg(feature = "cache")] Streamed(UnboundedStreamBody), } @@ -73,9 +67,7 @@ impl Body for ResponseBody { ) -> std::task::Poll, Self::Error>>> { match self.get_mut() { ResponseBody::Incoming(incoming) => Pin::new(incoming).poll_frame(cx), - #[cfg(feature = "cache")] ResponseBody::Boxed(boxed) => Pin::new(boxed).poll_frame(cx), - #[cfg(feature = "cache")] ResponseBody::Streamed(streamed) => Pin::new(streamed).poll_frame(cx), } .map_err(RpxyError::HyperBodyError) diff --git a/rpxy-lib/src/message_handler/handler_manipulate_messages.rs b/rpxy-lib/src/message_handler/handler_manipulate_messages.rs index 46e572c..143b3e8 100644 --- a/rpxy-lib/src/message_handler/handler_manipulate_messages.rs +++ b/rpxy-lib/src/message_handler/handler_manipulate_messages.rs @@ -19,6 +19,7 @@ where // Functions to generate messages //////////////////////////////////////////////////// + #[allow(unused_variables)] /// Manipulate a response message sent from a backend application to forward downstream to a client. pub(super) fn generate_response_forwarded( &self, diff --git a/rpxy-lib/src/proxy/mod.rs b/rpxy-lib/src/proxy/mod.rs index a7c1ec8..3f19059 100644 --- a/rpxy-lib/src/proxy/mod.rs +++ b/rpxy-lib/src/proxy/mod.rs @@ -1,10 +1,12 @@ -mod proxy_h3; mod proxy_main; +mod socket; + +#[cfg(any(feature = "http3-quinn", feature = "http3-s2n"))] +mod proxy_h3; #[cfg(feature = "http3-quinn")] mod proxy_quic_quinn; #[cfg(all(feature = "http3-s2n", not(feature = "http3-quinn")))] mod proxy_quic_s2n; -mod socket; use crate::{ globals::Globals,