tuxmain/rust-rpxy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #9 from junkurihara/client_cert

Browse source 
Experimental client certificate support
This commit is contained in:
Jun Kurihara 2022-10-14 23:13:48 +09:00 • committed by GitHub
commit d4bc842963
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 476 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "rpxy"
version = "0.1.0"
version = "0.1.1"
authors = ["Jun Kurihara"]
homepage = "https://github.com/junkurihara/rust-rpxy"
repository = "https://github.com/junkurihara/rust-rpxy"
@ -18,7 +18,7 @@ http3 = ["quinn", "h3", "h3-quinn"]
[dependencies]
env_logger = "0.9.1"
anyhow = "1.0.65"
clap = { version = "4.0.14", features = ["std", "cargo", "wrap_help"] }
clap = { version = "4.0.15", features = ["std", "cargo", "wrap_help"] }
futures = { version = "0.3.24", features = ["alloc", "async-await"] }
hyper = { version = "0.14.20", default-features = false, features = [
"server",
@ -53,6 +53,7 @@ quinn = { version = "0.8.5", optional = true }
h3 = { path = "./h3/h3/", optional = true }
h3-quinn = { path = "./h3/h3-quinn/", optional = true }
thiserror = "1.0.37"
x509-parser = "0.14.0"
[target.'cfg(not(target_env = "msvc"))'.dependencies]

87
README.md
View file

@ -128,7 +128,7 @@ listen_port_tls = 443
[apps."app_name"]
server_name = 'app1.example.com'
tls = { tls_cert_path = 'localhost.crt', tls_cert_key_path = 'localhost.key' }
tls = { tls_cert_path = 'server.crt', tls_cert_key_path = 'server.key' }
reverse_proxy = [{ upstream = [{ location = 'app1.local:8080' }] }]
```
@ -141,7 +141,7 @@ We should note that the private key specified by `tls_cert_key_path` must be *in
In the current Web, we believe it is common to serve everything through HTTPS rather than HTTP, and hence *https redirection* is often used for HTTP requests. When you specify both `listen_port` and `listen_port_tls`, you can enable an option of such redirection by making `https_redirection` true.
```toml
tls = { https_redirection = true, tls_cert_path = 'localhost.crt', tls_cert_key_path = 'localhost.key' }
tls = { https_redirection = true, tls_cert_path = 'server.crt', tls_cert_key_path = 'server.key' }
```
If it is true, `rpxy` returns the status code `301` to the cleartext request with new location `https://<requested_host>/<requested_query_and_path>` served over TLS.
@ -155,7 +155,7 @@ listen_port_tls = 443
[apps.app1]
server_name = 'app1.example.com'
tls = { https_redirection = true, tls_cert_path = 'localhost.crt', tls_cert_key_path = 'localhost.key' }
tls = { https_redirection = true, tls_cert_path = 'server.crt', tls_cert_key_path = 'server.key' }
[[apps.app1.reverse_proxy]]
upstream = [
@ -226,6 +226,33 @@ Other than them, all you need is to mount your `config.toml` as `/etc/rpxy.toml`
[`./bench`](./bench/) directory could be a very simple example of configuration of `rpxy`. This can also be an example of an example of docker use case.
## Experimental Features and Caveats
### HTTP/3
`rpxy` can serves HTTP/3 requests thanks to `quinn` and `hyperium/h3`. To enable this experimental feature, add an entry `experimental.h3` in your `config.toml` like follows. Any values in the entry like `alt_svc_max_age` are optional.
```toml
[experimental.h3]
alt_svc_max_age = 3600
request_max_body_size = 65536
max_concurrent_connections = 10000
max_concurrent_bidistream = 100
max_concurrent_unistream = 100
```
### Client Authentication via Client Certificates
Client authentication is enabled when `apps."app_name".tls.client_ca_cert_path` is set for the domain specified by `"app_name"` like
```toml
[apps.localhost]
server_name = 'localhost' # Domain name
tls = { https_redirection = true, tls_cert_path = './server.crt', tls_cert_key_path = './server.key', client_ca_cert_path = './client_cert.ca.crt' }
```
However, currently we have a limitation on HTTP/3 support for applications that enables client authentication. If an application is set with client authentication, HTTP/3 doesn't work for the application.
## TIPS
### Using Private Key Issued by Let's Encrypt
@ -235,13 +262,65 @@ If you obtain certificates and private keys from [Let's Encrypt](https://letsenc
The easiest way is to use `openssl` by
```bash
openssl pkcs8 -topk8 -nocrypt \
% openssl pkcs8 -topk8 -nocrypt \
-in yoru_domain_from_le.key \
-inform PEM \
-out your_domain_pkcs8.key.pem \
-outform PEM
```
### Client Authentication using Client Certificate Signed by Your Own Root CA
First, you need to prepare a CA certificate used to verify client certificate. If you do not have one, you can generate CA key and certificate by OpenSSL commands as follows. *Note that `rustls` accepts X509v3 certificates and reject SHA-1, and that `rpxy` relys on Version 3 extension fields of `KeyID`s of `Subject Key Identifier` and `Authority Key Identifier`.*
1. Generate CA key of `secp256v1`, CSR, and then generate CA certificate that will be set for `tls.client_ca_cert_path` for each server app in `config.toml`.
```bash
% openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:prime256v1 -out client.ca.key
% openssl req -new -key client.ca.key -out client.ca.csr
...
-----
Country Name (2 letter code) []: ...
State or Province Name (full name) []: ...
Locality Name (eg, city) []: ...
Organization Name (eg, company) []: ...
Organizational Unit Name (eg, section) []: ...
Common Name (eg, fully qualified host name) []: <Should not input CN>
Email Address []: ...
% openssl x509 -req -days 3650 -sha256 -in client.ca.csr -signkey client.ca.key -out client.ca.crt -extfile client.ca.ext
```
2. Generate a client key of `secp256v1` and certificate signed by CA key.
```bash
% openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:prime256v1 -out client.key
% openssl req -new -key client.key -out client.csr
...
-----
Country Name (2 letter code) []:
State or Province Name (full name) []:
Locality Name (eg, city) []:
Organization Name (eg, company) []:
Organizational Unit Name (eg, section) []:
Common Name (eg, fully qualified host name) []: <Should not input CN>
Email Address []:
% openssl x509 -req -days 365 -sha256 -in client.csr -CA client.ca.crt -CAkey client.ca.key -CAcreateserial -out client.crt -extfile client.ext
```
Now you have a client key `client.key` and certificate `client.crt` (version 3). `pfx` (`p12`) file can be retrieved as
```bash
% openssl pkcs12 -export -inkey client.key -in client.crt -certfile client.ca.crt -out client.pfx
```
Note that on MacOS, a `pfx` generated by `OpenSSL 3.0.6` cannot be imported to MacOS KeyChain Access. We generated the sample `pfx` using `LibreSSL 2.8.3` instead `OpenSSL`.
All of sample certificate files are found in `./example-certs/` directory.
### (Work Around) Deployment on Ubuntu 22.04LTS using docker behind `ufw`
Basically, docker automatically manage your iptables if you use the port-mapping option, i.e., `--publish` for `docker run` or `ports` in `docker-compose.yml`. This means you do not need to manually expose your port, e.g., 443 TCP/UDP for HTTPS, using `ufw`-like management command.

4
TODO.md
View file

@ -8,4 +8,8 @@
- Prometheus metrics
- Documentation
- Client certificate
- support intermediate certificate. Currently, only supports client certificates directly signed by root CA.
- Currently, we took the following approach (caveats)
- For Http2 and 1.1, prepare `rustls::ServerConfig` for each domain name and hence client CA cert is set for each one.
- For Http3, use aggregated `rustls::ServerConfig` for multiple domain names except for ones requiring client-auth. So, if a domain name is set with client authentication, http3 doesn't work for the domain.
- etc.

5
config-example.toml
View file

@ -37,8 +37,9 @@ default_app = 'another_localhost'
server_name = 'localhost' # Domain name
# Optional: TLS setting. if https_port is specified and tls is true above, this must be given.
tls = { https_redirection = true, tls_cert_path = '/certs/localhost.crt', tls_cert_key_path = '/certs/localhost.key' } # for docker volume mounted certs
#tls = { https_redirection = true, tls_cert_path = './localhost.crt', tls_cert_key_path = './localhost.key' } # for local
tls = { https_redirection = true, tls_cert_path = '/certs/server.crt', tls_cert_key_path = '/certs/server.key' } # for docker volume mounted certs
#tls = { https_redirection = true, tls_cert_path = './server.crt', tls_cert_key_path = './server.key' } # for local
#tls = { https_redirection = true, tls_cert_path = './server.crt', tls_cert_key_path = './server.key', client_ca_cert_path = './client_cert.ca.crt' } # for local with client_cert
## TODO
# allowhosts = ['127.0.0.1', '::1', '192.168.10.0/24'] # TODO

12
example-certs/client.ca.crt Normal file
View file

@ -0,0 +1,12 @@
-----BEGIN CERTIFICATE-----
MIIB0DCCAXegAwIBAgIUeW2Vdqq6y9H0TFvClTW9YkwNlcMwCgYIKoZIzj0EAwIw
PjELMAkGA1UEBhMCSlAxDjAMBgNVBAgMBVRva3lvMQ0wCwYDVQQHDARDaHVvMRAw
DgYDVQQKDAdaZXR0YW50MB4XDTIyMTAwMzE0MTAxM1oXDTMyMDkzMDE0MTAxM1ow
PjELMAkGA1UEBhMCSlAxDjAMBgNVBAgMBVRva3lvMQ0wCwYDVQQHDARDaHVvMRAw
DgYDVQQKDAdaZXR0YW50MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuLuDStaw
D2jTIDUUFDnT2X01kNPAv2UK5QAzwjQPqu61koNHsRSe1GuhkC2jFolCaapOTWnJ
E7EeesOXtihI4KNTMFEwHQYDVR0OBBYEFBHfSGDdI6/YEwGrfPUuFAFxO5ejMB8G
A1UdIwQYMBaAFBHfSGDdI6/YEwGrfPUuFAFxO5ejMA8GA1UdEwEB/wQFMAMBAf8w
CgYIKoZIzj0EAwIDRwAwRAIgBfmM5qivBXQbLOH9+XI4D8ah0nrNjZvTYMS0V32d
888CIF33NCYYf+LB/edqkQeyU/Xuw4pOx72MD3GPJG1lYWkW
-----END CERTIFICATE-----

3
example-certs/client.ca.ext Normal file
View file

@ -0,0 +1,3 @@
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer
basicConstraints=critical,CA:TRUE

5
example-certs/client.ca.key Normal file
View file

@ -0,0 +1,5 @@
-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgj6mxDmE5gPgJ5yQY
pJByP2UL67EwcBHJEVed77CHmRuhRANCAAS4u4NK1rAPaNMgNRQUOdPZfTWQ08C/
ZQrlADPCNA+q7rWSg0exFJ7Ua6GQLaMWiUJpqk5NackTsR56w5e2KEjg
-----END PRIVATE KEY-----

13
example-certs/client.crt Normal file
View file

@ -0,0 +1,13 @@
-----BEGIN CERTIFICATE-----
MIIB3jCCAYSgAwIBAgIUJg74LEgATwFv6xAvbcILjHAx2k4wCgYIKoZIzj0EAwIw
PjELMAkGA1UEBhMCSlAxDjAMBgNVBAgMBVRva3lvMQ0wCwYDVQQHDARDaHVvMRAw
DgYDVQQKDAdaZXR0YW50MB4XDTIyMTAwMzE0MTEwM1oXDTIzMTAwMjE0MTEwM1ow
RDELMAkGA1UEBhMCSlAxDjAMBgNVBAgMBVRva3lvMQ8wDQYDVQQHDAZOZXJpbWEx
FDASBgNVBAoMC1pldHRhbnQgRGV2MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE
C8CxHow7SH0ZuRzFmquRl8lvwlwAuARZBPnT3u44BPwqDerI97555JkKquk35F6g
mFLPB28ljIvMicBDqjzS56NaMFgwHwYDVR0jBBgwFoAUEd9IYN0jr9gTAat89S4U
AXE7l6MwCQYDVR0TBAIwADALBgNVHQ8EBAMCBPAwHQYDVR0OBBYEFBuXDF84bxZt
qK7Y/ngSgm5JHgC2MAoGCCqGSM49BAMCA0gAMEUCIQD2yl6pYXuPnOSne4+yHOw3
PdhPlyARxQqhrWM2LITP4AIgMv+exuURpaVj4ykhmlGS7ut05qZBpVgH4E+gamn2
ZW8=
-----END CERTIFICATE-----

8
example-certs/client.csr Normal file
View file

@ -0,0 +1,8 @@
-----BEGIN CERTIFICATE REQUEST-----
MIIBADCBpgIBADBEMQswCQYDVQQGEwJKUDEOMAwGA1UECAwFVG9reW8xDzANBgNV
BAcMBk5lcmltYTEUMBIGA1UECgwLWmV0dGFudCBEZXYwWTATBgcqhkjOPQIBBggq
hkjOPQMBBwNCAAQLwLEejDtIfRm5HMWaq5GXyW/CXAC4BFkE+dPe7jgE/CoN6sj3
vnnkmQqq6TfkXqCYUs8HbyWMi8yJwEOqPNLnoAAwCgYIKoZIzj0EAwIDSQAwRgIh
AJ0KUTO7x6YvavdLHllW9HWiSyeztquAQrqqHzO7sAHmAiEAitDM1Jv3xHbeK83R
ihWMGj/8y+QMeaL7cPBY/dfwIis=
-----END CERTIFICATE REQUEST-----

4
example-certs/client.ext Normal file
View file

@ -0,0 +1,4 @@
authorityKeyIdentifier=keyid:always
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment

5
example-certs/client.key Normal file
View file

@ -0,0 +1,5 @@
-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgQ70UolUBJK41lMWU
4fid9INB08+kWF5NgXmLr3VknvahRANCAAQLwLEejDtIfRm5HMWaq5GXyW/CXAC4
BFkE+dPe7jgE/CoN6sj3vnnkmQqq6TfkXqCYUs8HbyWMi8yJwEOqPNLn
-----END PRIVATE KEY-----

BIN
example-certs/client_pass=foobar.pfx Normal file
View file

Binary file not shown.

0
localhost.crt → example-certs/server.crt
View file

0
localhost.key → example-certs/server.key
View file

157
src/backend/mod.rs
View file

@ -5,7 +5,8 @@ use crate::{
log::*,
utils::{BytesName, PathNameBytesExp, ServerNameBytesExp},
};
use rustc_hash::FxHashMap as HashMap;
use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
use rustls::{OwnedTrustAnchor, RootCertStore};
use std::{
fs::File,
io::{self, BufReader, Cursor, Read},
@ -19,6 +20,7 @@ use tokio_rustls::rustls::{
};
pub use upstream::{ReverseProxy, Upstream, UpstreamGroup};
pub use upstream_opts::UpstreamOption;
use x509_parser::prelude::*;
/// Struct serving information to route incoming connections, like server name to be handled and tls certs/keys settings.
pub struct Backend {
@ -30,6 +32,7 @@ pub struct Backend {
pub tls_cert_path: Option<PathBuf>,
pub tls_cert_key_path: Option<PathBuf>,
pub https_redirection: Option<bool>,
pub client_ca_cert_path: Option<PathBuf>,
}
impl Backend {
@ -105,6 +108,67 @@ impl Backend {
})?;
Ok(CertifiedKey::new(certs, signing_key))
}
fn read_client_ca_certs(&self) -> io::Result<(Vec<OwnedTrustAnchor>, HashSet<Vec<u8>>)> {
debug!("Read CA certificates for client authentication");
// Reads client certificate and returns client
let client_ca_cert_path = {
if let Some(c) = self.client_ca_cert_path.as_ref() {
c
} else {
return Err(io::Error::new(io::ErrorKind::Other, "Invalid certs and keys paths"));
}
};
let certs: Vec<_> = {
let certs_path_str = client_ca_cert_path.display().to_string();
let mut reader = BufReader::new(File::open(client_ca_cert_path).map_err(|e| {
io::Error::new(
e.kind(),
format!("Unable to load the client certificates [{}]: {}", certs_path_str, e),
)
})?);
rustls_pemfile::certs(&mut reader)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Unable to parse the client certificates"))?
}
.drain(..)
.map(Certificate)
.collect();
let owned_trust_anchors: Vec<_> = certs
.iter()
.map(|v| {
let trust_anchor = tokio_rustls::webpki::TrustAnchor::try_from_cert_der(&v.0).unwrap();
rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(
trust_anchor.subject,
trust_anchor.spki,
trust_anchor.name_constraints,
)
})
.collect();
// TODO: SKID is not used currently
let subject_key_identifiers: HashSet<_> = certs
.iter()
.filter_map(|v| {
// retrieve ca key id (subject key id)
let cert = parse_x509_certificate(&v.0).unwrap().1;
let subject_key_ids = cert
.iter_extensions()
.filter_map(|ext| match ext.parsed_extension() {
ParsedExtension::SubjectKeyIdentifier(skid) => Some(skid),
_ => None,
})
.collect::<Vec<_>>();
if !subject_key_ids.is_empty() {
Some(subject_key_ids[0].0.to_owned())
} else {
None
}
})
.collect();
Ok((owned_trust_anchors, subject_key_identifiers))
}
}
/// HashMap and some meta information for multiple Backend structs.
@ -113,25 +177,84 @@ pub struct Backends {
pub default_server_name_bytes: Option<ServerNameBytesExp>, // for plaintext http
}
impl Backends {
pub async fn generate_server_crypto_with_cert_resolver(&self) -> Result<ServerConfig, anyhow::Error> {
let mut resolver = ResolvesServerCertUsingSni::new();
pub type SniServerCryptoMap = HashMap<ServerNameBytesExp, Arc<ServerConfig>>;
pub struct ServerCrypto {
// For Quic/HTTP3, only servers with no client authentication
pub inner_global_no_client_auth: Arc<ServerConfig>,
// For TLS over TCP/HTTP2 and 1.1, map of SNI to server_crypto for all given servers
pub inner_local_map: Arc<SniServerCryptoMap>,
}
// let mut cnt = 0;
for (_, backend) in self.apps.iter() {
impl Backends {
pub async fn generate_server_crypto(&self) -> Result<ServerCrypto, anyhow::Error> {
let mut resolver_global = ResolvesServerCertUsingSni::new();
let mut server_crypto_local_map: SniServerCryptoMap = HashMap::default();
for (server_name_bytes_exp, backend) in self.apps.iter() {
if backend.tls_cert_key_path.is_some() && backend.tls_cert_path.is_some() {
match backend.read_certs_and_key() {
Ok(certified_key) => {
if let Err(e) = resolver.add(backend.server_name.as_str(), certified_key) {
let mut resolver_local = ResolvesServerCertUsingSni::new();
let mut client_ca_roots_local = RootCertStore::empty();
// add server certificate and key
if let Err(e) = resolver_local.add(backend.server_name.as_str(), certified_key.to_owned()) {
error!(
"{}: Failed to read some certificates and keys {}",
backend.server_name.as_str(),
e
)
} else {
// debug!("Add certificate for server_name: {}", backend.server_name.as_str());
// cnt += 1;
}
if backend.client_ca_cert_path.is_none() {
// aggregated server config for no client auth server for http3
if let Err(e) = resolver_global.add(backend.server_name.as_str(), certified_key) {
error!(
"{}: Failed to read some certificates and keys {}",
backend.server_name.as_str(),
e
)
}
} else {
// add client certificate if specified
match backend.read_client_ca_certs() {
Ok((owned_trust_anchors, _subject_key_ids)) => {
client_ca_roots_local.add_server_trust_anchors(owned_trust_anchors.into_iter());
}
Err(e) => {
warn!(
"Failed to add client CA certificate for {}: {}",
backend.server_name.as_str(),
e
);
}
}
}
let mut server_config_local = if client_ca_roots_local.is_empty() {
// with no client auth, enable http1.1 -- 3
let mut sc = ServerConfig::builder()
.with_safe_defaults()
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver_local));
#[cfg(feature = "http3")]
{
sc.alpn_protocols = vec![b"h3".to_vec(), b"hq-29".to_vec()]; // TODO: remove hq-29 later?
}
sc
} else {
// with client auth, enable only http1.1 and 2
// let client_certs_verifier = rustls::server::AllowAnyAnonymousOrAuthenticatedClient::new(client_ca_roots);
let client_certs_verifier = rustls::server::AllowAnyAuthenticatedClient::new(client_ca_roots_local);
ServerConfig::builder()
.with_safe_defaults()
.with_client_cert_verifier(client_certs_verifier)
.with_cert_resolver(Arc::new(resolver_local))
};
server_config_local.alpn_protocols.push(b"h2".to_vec());
server_config_local.alpn_protocols.push(b"http/1.1".to_vec());
server_crypto_local_map.insert(server_name_bytes_exp.to_owned(), Arc::new(server_config_local));
}
Err(e) => {
warn!("Failed to add certificate for {}: {}", backend.server_name.as_str(), e);
@ -141,14 +264,17 @@ impl Backends {
}
// debug!("Load certificate chain for {} server_name's", cnt);
let mut server_config = ServerConfig::builder()
//////////////
let mut server_crypto_global = ServerConfig::builder()
.with_safe_defaults()
.with_no_client_auth()
.with_cert_resolver(Arc::new(resolver));
.with_cert_resolver(Arc::new(resolver_global));
//////////////////////////////
#[cfg(feature = "http3")]
{
server_config.alpn_protocols = vec![
server_crypto_global.alpn_protocols = vec![
b"h3".to_vec(),
b"hq-29".to_vec(), // TODO: remove later?
b"h2".to_vec(),
@ -160,6 +286,9 @@ impl Backends {
server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
}
Ok(server_config)
Ok(ServerCrypto {
inner_global_no_client_auth: Arc::new(server_crypto_global),
inner_local_map: Arc::new(server_crypto_local_map),
})
}
}

6
src/config/parse.rs
View file

@ -93,9 +93,9 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro
let server_name_string = app.server_name.as_ref().unwrap();
// TLS settings
let (tls_cert_path, tls_cert_key_path, https_redirection) = if app.tls.is_none() {
let (tls_cert_path, tls_cert_key_path, https_redirection, client_ca_cert_path) = if app.tls.is_none() {
ensure!(globals.http_port.is_some(), "Required HTTP port");
(None, None, None)
(None, None, None, None)
} else {
let tls = app.tls.as_ref().unwrap();
ensure!(tls.tls_cert_key_path.is_some() && tls.tls_cert_path.is_some());
@ -109,6 +109,7 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro
ensure!(globals.https_port.is_some()); // only when both https ports are configured.
tls.https_redirection
},
tls.client_ca_cert_path.as_ref().map(PathBuf::from),
)
};
if globals.http_port.is_none() {
@ -130,6 +131,7 @@ pub fn parse_opts(globals: &mut Globals) -> std::result::Result<(), anyhow::Erro
tls_cert_path,
tls_cert_key_path,
https_redirection,
client_ca_cert_path,
},
);
info!("Registering application: {} ({})", app_name, server_name_string);

1
src/config/toml.rs
View file

@ -47,6 +47,7 @@ pub struct TlsOption {
pub tls_cert_path: Option<String>,
pub tls_cert_key_path: Option<String>,
pub https_redirection: Option<bool>,
pub client_ca_cert_path: Option<String>,
}
#[derive(Deserialize, Debug, Default)]

10
src/error.rs
View file

@ -43,3 +43,13 @@ pub enum RpxyError {
#[error(transparent)]
Other(#[from] anyhow::Error),
}
#[allow(dead_code)]
#[derive(Debug, Error, Clone)]
pub enum ClientCertsError {
#[error("TLS Client Certificate is Required for Given SNI: {0}")]
ClientCertRequired(String),
#[error("Inconsistent TLS Client Certificate for Given SNI: {0}")]
InconsistentClientCert(String),
}

28
src/handler/handler_main.rs
View file

@ -1,6 +1,12 @@
// Highly motivated by https://github.com/felipenoris/hyper-reverse-proxy
use super::{utils_headers::*, utils_request::*, utils_synth_response::*};
use crate::{backend::UpstreamGroup, error::*, globals::Globals, log::*, utils::ServerNameBytesExp};
use crate::{
backend::{Backend, UpstreamGroup},
error::*,
globals::Globals,
log::*,
utils::ServerNameBytesExp,
};
use hyper::{
client::connect::Connect,
header::{self, HeaderValue},
@ -118,7 +124,7 @@ where
if res_backend.status() != StatusCode::SWITCHING_PROTOCOLS {
// Generate response to client
if self.generate_response_forwarded(&mut res_backend).is_ok() {
if self.generate_response_forwarded(&mut res_backend, backend).is_ok() {
log_data.status_code(&res_backend.status()).output();
return Ok(res_backend);
} else {
@ -176,7 +182,11 @@ where
////////////////////////////////////////////////////
// Functions to generate messages
fn generate_response_forwarded<B: core::fmt::Debug>(&self, response: &mut Response<B>) -> Result<()> {
fn generate_response_forwarded<B: core::fmt::Debug>(
&self,
response: &mut Response<B>,
chosen_backend: &Backend,
) -> Result<()> {
let headers = response.headers_mut();
remove_connection_header(headers);
remove_hop_header(headers);
@ -184,7 +194,8 @@ where
#[cfg(feature = "http3")]
{
if self.globals.http3 {
// TODO: Workaround for avoid h3 for client authentication
if self.globals.http3 && chosen_backend.client_ca_cert_path.is_none() {
if let Some(port) = self.globals.https_port {
add_header_entry_overwrite_if_exist(
headers,
@ -195,6 +206,15 @@ where
),
)?;
}
} else {
// remove alt-svc to disallow requests via http3
headers.remove(header::ALT_SVC.as_str());
}
}
#[cfg(not(feature = "http3"))]
{
if let Some(port) = self.globals.https_port {
headers.remove(header::ALT_SVC.as_str());
}
}

1
src/proxy/mod.rs
View file

@ -1,3 +1,4 @@
mod proxy_client_cert;
#[cfg(feature = "http3")]
mod proxy_h3;
mod proxy_main;

55
src/proxy/proxy_client_cert.rs Normal file
View file

@ -0,0 +1,55 @@
use crate::{error::*, log::*};
use rustc_hash::FxHashSet as HashSet;
use rustls::Certificate;
use x509_parser::extensions::ParsedExtension;
use x509_parser::prelude::*;
#[allow(dead_code)]
// TODO: consider move this function to the layer of handle_request (L7) to return 403
pub(super) fn check_client_authentication(
client_certs: Option<&[Certificate]>,
client_ca_keyids_set_for_sni: Option<&HashSet<Vec<u8>>>,
) -> std::result::Result<(), ClientCertsError> {
let client_ca_keyids_set = match client_ca_keyids_set_for_sni {
Some(c) => c,
None => {
// No client cert settings for given server name
return Ok(());
}
};
let client_certs = match client_certs {
Some(c) => {
debug!("Incoming TLS client is (temporarily) authenticated via client cert");
c
}
None => {
error!("Client certificate is needed for given server name");
return Err(ClientCertsError::ClientCertRequired(
"Client certificate is needed for given server name".to_string(),
));
}
};
// Check client certificate key ids
let mut client_certs_parsed_iter = client_certs.iter().filter_map(|d| parse_x509_certificate(&d.0).ok());
let match_server_crypto_and_client_cert = client_certs_parsed_iter.any(|c| {
let mut filtered = c.1.iter_extensions().filter_map(|e| {
if let ParsedExtension::AuthorityKeyIdentifier(key_id) = e.parsed_extension() {
key_id.key_identifier.as_ref()
} else {
None
}
});
filtered.any(|id| client_ca_keyids_set.contains(id.0))
});
if !match_server_crypto_and_client_cert {
error!("Inconsistent client certificate was provided for SNI");
return Err(ClientCertsError::InconsistentClientCert(
"Inconsistent client certificate was provided for SNI".to_string(),
));
}
Ok(())
}

68
src/proxy/proxy_h3.rs
View file

@ -26,36 +26,46 @@ where
);
// TODO: Is here enough to fetch server_name from NewConnection?
// to avoid deep nested call from listener_service_h3
while let Some((req, stream)) = match h3_conn.accept().await {
Ok(opt_req) => opt_req,
Err(e) => {
warn!("HTTP/3 failed to accept incoming connection: {}", e);
return Ok(h3_conn.shutdown(0).await?);
}
} {
// We consider the connection count separately from the stream count.
// Max clients for h1/h2 = max 'stream' for h3.
let request_count = self.globals.request_count.clone();
if request_count.increment() > self.globals.max_clients {
request_count.decrement();
return Ok(h3_conn.shutdown(0).await?);
}
debug!("Request incoming: current # {}", request_count.current());
let self_inner = self.clone();
let tls_server_name_inner = tls_server_name.clone();
self.globals.runtime_handle.spawn(async move {
if let Err(e) = timeout(
self_inner.globals.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner),
)
.await
{
error!("HTTP/3 failed to process stream: {}", e);
loop {
// this routine follows hyperium/h3 examples https://github.com/hyperium/h3/blob/master/examples/server.rs
match h3_conn.accept().await {
Ok(None) => {
break;
}
request_count.decrement();
debug!("Request processed: current # {}", request_count.current());
});
Err(e) => {
warn!("HTTP/3 error on accept incoming connection: {}", e);
match e.get_error_level() {
h3::error::ErrorLevel::ConnectionError => break,
h3::error::ErrorLevel::StreamError => continue,
}
}
Ok(Some((req, stream))) => {
// We consider the connection count separately from the stream count.
// Max clients for h1/h2 = max 'stream' for h3.
let request_count = self.globals.request_count.clone();
if request_count.increment() > self.globals.max_clients {
request_count.decrement();
h3_conn.shutdown(0).await?;
break;
}
debug!("Request incoming: current # {}", request_count.current());
let self_inner = self.clone();
let tls_server_name_inner = tls_server_name.clone();
self.globals.runtime_handle.spawn(async move {
if let Err(e) = timeout(
self_inner.globals.proxy_timeout + Duration::from_secs(1), // timeout per stream are considered as same as one in http2
self_inner.stream_serve_h3(req, stream, client_addr, tls_server_name_inner),
)
.await
{
error!("HTTP/3 failed to process stream: {}", e);
}
request_count.decrement();
debug!("Request processed: current # {}", request_count.current());
});
}
}
}
}
Err(err) => {

97
src/proxy/proxy_tls.rs
View file

@ -1,5 +1,11 @@
use super::proxy_main::{LocalExecutor, Proxy};
use crate::{constants::*, error::*, log::*, utils::BytesName};
use crate::{
backend::{ServerCrypto, SniServerCryptoMap},
constants::*,
error::*,
log::*,
utils::BytesName,
};
use hyper::{client::connect::Connect, server::conn::Http};
use rustls::ServerConfig;
use std::sync::Arc;
@ -8,7 +14,6 @@ use tokio::{
sync::watch,
time::{sleep, timeout, Duration},
};
use tokio_rustls::TlsAcceptor;
#[cfg(feature = "http3")]
use futures::StreamExt;
@ -19,10 +24,10 @@ impl<T> Proxy<T>
where
T: Connect + Clone + Sync + Send + 'static,
{
async fn cert_service(&self, server_crypto_tx: watch::Sender<Option<Arc<ServerConfig>>>) {
async fn cert_service(&self, server_crypto_tx: watch::Sender<Option<Arc<ServerCrypto>>>) {
info!("Start cert watch service");
loop {
if let Ok(server_crypto) = self.globals.backends.generate_server_crypto_with_cert_resolver().await {
if let Ok(server_crypto) = self.globals.backends.generate_server_crypto().await {
if let Err(_e) = server_crypto_tx.send(Some(Arc::new(server_crypto))) {
error!("Failed to populate server crypto");
break;
@ -38,56 +43,66 @@ where
async fn listener_service(
&self,
server: Http<LocalExecutor>,
mut server_crypto_rx: watch::Receiver<Option<Arc<ServerConfig>>>,
mut server_crypto_rx: watch::Receiver<Option<Arc<ServerCrypto>>>,
) -> Result<()> {
let tcp_listener = TcpListener::bind(&self.listening_on).await?;
info!("Start TCP proxy serving with HTTPS request for configured host names");
// let mut server_crypto: Option<Arc<ServerConfig>> = None;
let mut tls_acceptor: Option<TlsAcceptor> = None;
let mut server_crypto_map: Option<Arc<SniServerCryptoMap>> = None;
loop {
tokio::select! {
tcp_cnx = tcp_listener.accept() => {
if tls_acceptor.is_none() || tcp_cnx.is_err() {
if tcp_cnx.is_err() || server_crypto_map.is_none() {
continue;
}
let (raw_stream, client_addr) = tcp_cnx.unwrap();
let acceptor = tls_acceptor.clone().unwrap();
let sc_map_inner = server_crypto_map.clone();
let server_clone = server.clone();
let self_inner = self.clone();
// spawns async handshake to avoid blocking thread by sequential handshake.
let handshake_fut = async move {
let acceptor = tokio_rustls::LazyConfigAcceptor::new(rustls::server::Acceptor::new().unwrap(), raw_stream).await;
if let Err(e) = acceptor {
return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {}", e)));
}
let start = acceptor.unwrap();
let client_hello = start.client_hello();
let server_name = client_hello.server_name();
debug!("HTTP/2 or 1.1: SNI in ClientHello: {:?}", server_name);
let server_name = server_name.map_or_else(|| None, |v| Some(v.to_server_name_vec()));
if server_name.is_none(){
return Err(RpxyError::Proxy("No SNI is given".to_string()));
}
let server_crypto = sc_map_inner.as_ref().unwrap().get(server_name.as_ref().unwrap());
if server_crypto.is_none() {
return Err(RpxyError::Proxy(format!("No TLS serving app for {:?}", "xx")));
}
let stream = match start.into_stream(server_crypto.unwrap().clone()).await {
Ok(s) => s,
Err(e) => {
return Err(RpxyError::Proxy(format!("Failed to handshake TLS: {}", e)));
}
};
self_inner.client_serve(stream, server_clone, client_addr, server_name);
Ok(())
};
self.globals.runtime_handle.spawn( async move {
// timeout is introduced to avoid get stuck here.
match timeout(Duration::from_secs(TLS_HANDSHAKE_TIMEOUT_SEC), acceptor.accept(raw_stream)).await {
Ok(x) => match x {
Ok(stream) => {
// Retrieve SNI
let (_, conn) = stream.get_ref();
let server_name = conn.sni_hostname();
debug!("HTTP/2 or 1.1: SNI in ClientHello: {:?}", server_name);
let server_name = server_name.map_or_else(|| None, |v| Some(v.to_server_name_vec()));
if server_name.is_none(){
Err(RpxyError::Proxy("No SNI is given".to_string()))
} else {
// this immediately spawns another future to actually handle stream. so it is okay to introduce timeout for handshake.
self_inner.client_serve(stream, server_clone, client_addr, server_name); // TODO: don't want to pass copied value...
Ok(())
}
},
Err(e) => {
Err(RpxyError::Proxy(format!("Failed to handshake TLS: {}", e)))
match timeout(
Duration::from_secs(TLS_HANDSHAKE_TIMEOUT_SEC),
handshake_fut
).await {
Ok(a) => {
if let Err(e) = a {
error!("{}", e);
}
},
Err(e) => {
Err(RpxyError::Proxy(format!("Timeout to handshake TLS: {}", e)))
error!("Timeout to handshake TLS: {}", e);
}
}
};
self.globals.runtime_handle.spawn( async move {
if let Err(e) = handshake_fut.await {
error!("{}", e);
}
};
});
}
_ = server_crypto_rx.changed() => {
@ -95,7 +110,7 @@ where
break;
}
let server_crypto = server_crypto_rx.borrow().clone().unwrap();
tls_acceptor = Some(TlsAcceptor::from(server_crypto));
server_crypto_map = Some(server_crypto.inner_local_map.clone());
}
else => break
}
@ -104,10 +119,10 @@ where
}
#[cfg(feature = "http3")]
async fn listener_service_h3(&self, mut server_crypto_rx: watch::Receiver<Option<Arc<ServerConfig>>>) -> Result<()> {
async fn listener_service_h3(&self, mut server_crypto_rx: watch::Receiver<Option<Arc<ServerCrypto>>>) -> Result<()> {
info!("Start UDP proxy serving with HTTP/3 request for configured host names");
// first set as null config server
let server_crypto = ServerConfig::builder()
let rustls_server_config = ServerConfig::builder()
.with_safe_defaults()
.with_no_client_auth()
.with_cert_resolver(Arc::new(tokio_rustls::rustls::server::ResolvesServerCertUsingSni::new()));
@ -117,12 +132,12 @@ where
.max_concurrent_bidi_streams(self.globals.h3_max_concurrent_bidistream)
.max_concurrent_uni_streams(self.globals.h3_max_concurrent_unistream);
let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(server_crypto));
let mut server_config_h3 = QuicServerConfig::with_crypto(Arc::new(rustls_server_config));
server_config_h3.transport = Arc::new(transport_config_quic);
server_config_h3.concurrent_connections(self.globals.h3_max_concurrent_connections);
let (endpoint, mut incoming) = Endpoint::server(server_config_h3, self.listening_on)?;
let mut server_crypto: Option<Arc<ServerConfig>> = None;
let mut server_crypto: Option<Arc<ServerCrypto>> = None;
loop {
tokio::select! {
new_conn = incoming.next() => {
@ -166,7 +181,7 @@ where
}
server_crypto = server_crypto_rx.borrow().clone();
if server_crypto.is_some(){
endpoint.set_server_config(Some(QuicServerConfig::with_crypto(server_crypto.clone().unwrap())));
endpoint.set_server_config(Some(QuicServerConfig::with_crypto(server_crypto.clone().unwrap().inner_global_no_client_auth.clone())));
}
}
else => break
@ -177,7 +192,7 @@ where
}
pub async fn start_with_tls(self, server: Http<LocalExecutor>) -> Result<()> {
let (tx, rx) = watch::channel::<Option<Arc<ServerConfig>>>(None);
let (tx, rx) = watch::channel::<Option<Arc<ServerCrypto>>>(None);
#[cfg(not(feature = "http3"))]
{
select! {