use smol for client
This commit is contained in:
parent
2c1793a128
commit
9c70eea4f1
5 changed files with 608 additions and 468 deletions
980
Cargo.lock
generated
980
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -11,7 +11,6 @@ form_urlencoded = "1.2.2"
|
||||||
giallo = { version = "0.3.1", features = ["dump"] }
|
giallo = { version = "0.3.1", features = ["dump"] }
|
||||||
log = "0.4.29"
|
log = "0.4.29"
|
||||||
rand = "0.9.2"
|
rand = "0.9.2"
|
||||||
reqwest = { version = "0.13.1", default-features = false, features = ["json"] }
|
|
||||||
serde = { version = "1.0.228", features = ["derive"] }
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
sha2 = "0.10.9"
|
sha2 = "0.10.9"
|
||||||
simplelog = "0.12.2"
|
simplelog = "0.12.2"
|
||||||
|
|
@ -19,6 +18,8 @@ simplelog = "0.12.2"
|
||||||
trillium = "0.2.20"
|
trillium = "0.2.20"
|
||||||
trillium-askama = "0.3.2"
|
trillium-askama = "0.3.2"
|
||||||
trillium-caching-headers = "0.2.3"
|
trillium-caching-headers = "0.2.3"
|
||||||
|
trillium-client = { version = "0.6.2", features = ["json"] }
|
||||||
|
trillium-native-tls = "0.4.0"
|
||||||
trillium-router = "0.4.1"
|
trillium-router = "0.4.1"
|
||||||
trillium-smol = "0.4.2"
|
trillium-smol = "0.4.2"
|
||||||
#trillium-static-compiled = "0.5.2"
|
#trillium-static-compiled = "0.5.2"
|
||||||
|
|
|
||||||
|
|
@ -3,9 +3,9 @@ use std::{collections::HashSet, io::Write, path::PathBuf};
|
||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
|
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use reqwest::Client;
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use sha2::Digest;
|
use sha2::Digest;
|
||||||
|
use trillium_client::Client;
|
||||||
|
|
||||||
pub const USER_AGENT: &str = "Blindforge";
|
pub const USER_AGENT: &str = "Blindforge";
|
||||||
pub const MAX_PAGE: u32 = 256;
|
pub const MAX_PAGE: u32 = 256;
|
||||||
|
|
@ -19,8 +19,36 @@ struct RepoGetTagResponse {
|
||||||
tarball_url: String,
|
tarball_url: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn make_client() -> Result<Client, reqwest::Error> {
|
#[derive(Debug)]
|
||||||
reqwest::ClientBuilder::new().user_agent(USER_AGENT).build()
|
pub enum ClientError {
|
||||||
|
Client(trillium_client::Error),
|
||||||
|
Http(trillium_client::UnexpectedStatusError),
|
||||||
|
Parse(trillium_client::ClientSerdeError),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<trillium_client::Error> for ClientError {
|
||||||
|
fn from(value: trillium_client::Error) -> Self {
|
||||||
|
Self::Client(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<trillium_client::UnexpectedStatusError> for ClientError {
|
||||||
|
fn from(value: trillium_client::UnexpectedStatusError) -> Self {
|
||||||
|
Self::Http(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<trillium_client::ClientSerdeError> for ClientError {
|
||||||
|
fn from(value: trillium_client::ClientSerdeError) -> Self {
|
||||||
|
Self::Parse(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn make_client() -> Client {
|
||||||
|
Client::new(trillium_native_tls::NativeTlsConfig::<
|
||||||
|
trillium_smol::ClientConfig,
|
||||||
|
>::default())
|
||||||
|
.with_default_header("User-Agent", USER_AGENT)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn fetch_repo_archive_url(
|
pub async fn fetch_repo_archive_url(
|
||||||
|
|
@ -30,12 +58,12 @@ pub async fn fetch_repo_archive_url(
|
||||||
repo: &str,
|
repo: &str,
|
||||||
tag: &str,
|
tag: &str,
|
||||||
token: Option<&str>,
|
token: Option<&str>,
|
||||||
) -> Result<String, reqwest::Error> {
|
) -> Result<String, ClientError> {
|
||||||
let mut req = client.get(&format!("{url}/api/v1/repos/{owner}/{repo}/tags/{tag}"));
|
let mut req = client.get(format!("{url}/api/v1/repos/{owner}/{repo}/tags/{tag}"));
|
||||||
if let Some(token) = token {
|
if let Some(token) = token {
|
||||||
req = req.header("Authorization", format!("token {token}"));
|
req = req.with_request_header("Authorization", format!("token {token}"));
|
||||||
}
|
}
|
||||||
let res: RepoGetTagResponse = req.send().await?.error_for_status()?.json().await?;
|
let res: RepoGetTagResponse = req.await?.success()?.response_json().await?;
|
||||||
Ok(res.tarball_url)
|
Ok(res.tarball_url)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -63,27 +91,45 @@ async fn fetch_repo_tree_at_commit_page(
|
||||||
commit_hash: &str,
|
commit_hash: &str,
|
||||||
token: Option<&str>,
|
token: Option<&str>,
|
||||||
page: u32,
|
page: u32,
|
||||||
) -> Result<GitTreeResponse, reqwest::Error> {
|
) -> Result<GitTreeResponse, ClientError> {
|
||||||
let mut req = client.get(&format!(
|
let mut req = client.get(format!(
|
||||||
"{url}/api/v1/repos/{owner}/{repo}/git/trees/{commit_hash}?recursive=true&page={page}"
|
"{url}/api/v1/repos/{owner}/{repo}/git/trees/{commit_hash}?recursive=true&page={page}"
|
||||||
));
|
));
|
||||||
if let Some(token) = token {
|
if let Some(token) = token {
|
||||||
req = req.header("Authorization", format!("token {token}"));
|
req = req.with_request_header("Authorization", format!("token {token}"));
|
||||||
}
|
}
|
||||||
// TODO ensure we don't use too much memory
|
// TODO ensure we don't use too much memory
|
||||||
req.send().await?.error_for_status()?.json().await
|
Ok(req.await?.success()?.response_json().await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub enum FetchRepoError {
|
pub enum FetchRepoError {
|
||||||
CannotCreateDir(std::io::Error),
|
CannotCreateDir(std::io::Error),
|
||||||
TooManyEntries,
|
TooManyEntries,
|
||||||
Reqwest(reqwest::Error),
|
Client(ClientError),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<reqwest::Error> for FetchRepoError {
|
impl From<ClientError> for FetchRepoError {
|
||||||
fn from(value: reqwest::Error) -> Self {
|
fn from(value: ClientError) -> Self {
|
||||||
Self::Reqwest(value)
|
Self::Client(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<trillium_client::Error> for FetchRepoError {
|
||||||
|
fn from(value: trillium_client::Error) -> Self {
|
||||||
|
Self::Client(value.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<trillium_client::UnexpectedStatusError> for FetchRepoError {
|
||||||
|
fn from(value: trillium_client::UnexpectedStatusError) -> Self {
|
||||||
|
Self::Client(value.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<trillium_client::ClientSerdeError> for FetchRepoError {
|
||||||
|
fn from(value: trillium_client::ClientSerdeError) -> Self {
|
||||||
|
Self::Client(value.into())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -152,7 +198,7 @@ pub async fn fetch_repo_files(
|
||||||
let mut hasher = sha2::Sha256::default();
|
let mut hasher = sha2::Sha256::default();
|
||||||
|
|
||||||
let repo_key: [u8; 32] = rand::rng().random();
|
let repo_key: [u8; 32] = rand::rng().random();
|
||||||
hasher.update(&repo_key);
|
hasher.update(repo_key);
|
||||||
let repo_id: [u8; 32] = hasher.finalize_reset().into();
|
let repo_id: [u8; 32] = hasher.finalize_reset().into();
|
||||||
let mut repo_id_str = [0; 24];
|
let mut repo_id_str = [0; 24];
|
||||||
base64_turbo::URL_SAFE
|
base64_turbo::URL_SAFE
|
||||||
|
|
@ -168,12 +214,12 @@ pub async fn fetch_repo_files(
|
||||||
let mut file_index = HashSet::new();
|
let mut file_index = HashSet::new();
|
||||||
|
|
||||||
for file in repo_index.files.iter() {
|
for file in repo_index.files.iter() {
|
||||||
let mut req = client.get(&file.url);
|
let mut req = client.get(file.url.clone());
|
||||||
if let Some(token) = token {
|
if let Some(token) = token {
|
||||||
req = req.header("Authorization", format!("token {token}"));
|
req = req.with_request_header("Authorization", format!("token {token}"));
|
||||||
}
|
}
|
||||||
// TODO ensure we don't use too much memory
|
// TODO ensure we don't use too much memory
|
||||||
let blob: GitBlob = req.send().await?.error_for_status()?.json().await?;
|
let blob: GitBlob = req.await?.success()?.response_json().await?;
|
||||||
if base64_turbo::STANDARD.estimate_decoded_len(blob.content.len()) as u64 > MAX_FILE_SIZE {
|
if base64_turbo::STANDARD.estimate_decoded_len(blob.content.len()) as u64 > MAX_FILE_SIZE {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -183,7 +229,7 @@ pub async fn fetch_repo_files(
|
||||||
if file_index.insert(file_name) {
|
if file_index.insert(file_name) {
|
||||||
let mut file_name_str = [0; 44];
|
let mut file_name_str = [0; 44];
|
||||||
base64_turbo::URL_SAFE
|
base64_turbo::URL_SAFE
|
||||||
.encode_into(&file_name, &mut file_name_str)
|
.encode_into(file_name, &mut file_name_str)
|
||||||
.expect("unreachable");
|
.expect("unreachable");
|
||||||
let file_name_str = str::from_utf8(&file_name_str).expect("unreachable");
|
let file_name_str = str::from_utf8(&file_name_str).expect("unreachable");
|
||||||
if let Ok(mut file) = std::fs::OpenOptions::new()
|
if let Ok(mut file) = std::fs::OpenOptions::new()
|
||||||
|
|
|
||||||
11
src/cache.rs
11
src/cache.rs
|
|
@ -1,4 +1,9 @@
|
||||||
use std::{borrow::Borrow, collections::{HashMap, VecDeque}, hash::Hash, time::Instant};
|
use std::{
|
||||||
|
borrow::Borrow,
|
||||||
|
collections::{HashMap, VecDeque},
|
||||||
|
hash::Hash,
|
||||||
|
time::Instant,
|
||||||
|
};
|
||||||
|
|
||||||
pub struct CacheEntry<T> {
|
pub struct CacheEntry<T> {
|
||||||
inner: T,
|
inner: T,
|
||||||
|
|
@ -54,3 +59,7 @@ impl<K: Clone + Eq + Hash, V: Clone> Cache<K, V> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*pub struct CacheRef {
|
||||||
|
cache:
|
||||||
|
}*/
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,12 @@ use crate::{cache, config::Config, repo::ReadRepoMetadataError};
|
||||||
|
|
||||||
use askama::Template;
|
use askama::Template;
|
||||||
use log::error;
|
use log::error;
|
||||||
use std::{collections::HashMap, io::ErrorKind, path::PathBuf, sync::{Arc, Mutex}};
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
io::ErrorKind,
|
||||||
|
path::PathBuf,
|
||||||
|
sync::{Arc, Mutex},
|
||||||
|
};
|
||||||
use trillium::{Conn, Handler};
|
use trillium::{Conn, Handler};
|
||||||
use trillium_router::{Router, RouterConnExt};
|
use trillium_router::{Router, RouterConnExt};
|
||||||
|
|
||||||
|
|
@ -15,7 +20,9 @@ pub fn make_router(config: Arc<Config>) -> impl Handler {
|
||||||
hl_registry.link_grammars();
|
hl_registry.link_grammars();
|
||||||
let hl_registry = Arc::new(hl_registry);
|
let hl_registry = Arc::new(hl_registry);
|
||||||
|
|
||||||
let mut metadata_cache = Arc::new(Mutex::new(cache::Cache::<String, HashMap<String, String>>::default()));
|
let mut metadata_cache = Arc::new(Mutex::new(
|
||||||
|
cache::Cache::<String, HashMap<String, String>>::default(),
|
||||||
|
));
|
||||||
|
|
||||||
(
|
(
|
||||||
trillium_caching_headers::CachingHeaders::new(),
|
trillium_caching_headers::CachingHeaders::new(),
|
||||||
|
|
@ -50,10 +57,14 @@ pub fn make_router(config: Arc<Config>) -> impl Handler {
|
||||||
|
|
||||||
let cache_fetch = |key| {
|
let cache_fetch = |key| {
|
||||||
let mut repo_hash = [0; 32];
|
let mut repo_hash = [0; 32];
|
||||||
if base64_turbo::URL_SAFE.decode_into(repo_hash_str, &mut repo_hash) != Ok(32) {
|
if base64_turbo::URL_SAFE.decode_into(repo_hash_str, &mut repo_hash)
|
||||||
|
!= Ok(32)
|
||||||
|
{
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let repo_dir = PathBuf::from(&config.data_dir).join(crate::SUBDIR_REPOS).join(repo_hash_str);
|
let repo_dir = PathBuf::from(&config.data_dir)
|
||||||
|
.join(crate::SUBDIR_REPOS)
|
||||||
|
.join(repo_hash_str);
|
||||||
let repo_metadata =
|
let repo_metadata =
|
||||||
match crate::repo::RepoMetadata::read_from_file(&config, &repo_dir) {
|
match crate::repo::RepoMetadata::read_from_file(&config, &repo_dir) {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
|
|
@ -81,7 +92,10 @@ pub fn make_router(config: Arc<Config>) -> impl Handler {
|
||||||
Some(files)
|
Some(files)
|
||||||
};
|
};
|
||||||
// TODO replace mutex with better thing (less contention or async mutex)
|
// TODO replace mutex with better thing (less contention or async mutex)
|
||||||
metadata_cache.lock().unwrap().fetch(repo_hash_str.to_string(), cache_fetch);
|
metadata_cache
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.fetch(repo_hash_str.to_string(), cache_fetch);
|
||||||
|
|
||||||
let hl_options = giallo::HighlightOptions::new(
|
let hl_options = giallo::HighlightOptions::new(
|
||||||
"py",
|
"py",
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue