use std::{collections::HashSet, io::Write, path::PathBuf}; use crate::{ config::Config, repo::{RepoMetadata, WriteRepoMetadataError}, }; use rand::RngExt; use serde::Deserialize; use sha2::Digest; use trillium_client::Client; pub const USER_AGENT: &str = "Blindforge"; pub const MAX_PAGE: u32 = 256; pub const MAX_ENTRIES: u32 = 256; pub const MAX_FILE_SIZE: u64 = 8 * 1024 * 1024; pub const MAX_PATH_SIZE: usize = 1024; pub const MAX_URL_SIZE: usize = 256; #[derive(Deserialize)] struct RepoGetTagResponse { tarball_url: String, } #[derive(Debug)] pub enum ClientError { Client(trillium_client::Error), Http(trillium_client::UnexpectedStatusError), Parse(trillium_client::ClientSerdeError), } impl From for ClientError { fn from(value: trillium_client::Error) -> Self { Self::Client(value) } } impl From for ClientError { fn from(value: trillium_client::UnexpectedStatusError) -> Self { Self::Http(value) } } impl From for ClientError { fn from(value: trillium_client::ClientSerdeError) -> Self { Self::Parse(value) } } pub fn make_client() -> Client { Client::new(trillium_native_tls::NativeTlsConfig::< trillium_smol::ClientConfig, >::default()) .with_default_header("User-Agent", USER_AGENT) } pub async fn fetch_repo_archive_url( client: &mut Client, url: &str, owner: &str, repo: &str, tag: &str, token: Option<&str>, ) -> Result { let mut req = client.get(format!("{url}/api/v1/repos/{owner}/{repo}/tags/{tag}")); if let Some(token) = token { req = req.with_request_header("Authorization", format!("token {token}")); } let res: RepoGetTagResponse = req.await?.success()?.response_json().await?; Ok(res.tarball_url) } #[derive(Deserialize)] struct GitTreeResponse { tree: Vec, truncated: bool, total_count: u32, } #[derive(Deserialize)] struct GitEntry { path: String, #[serde(rename = "type")] entry_type: String, size: u64, url: String, } async fn fetch_repo_tree_at_commit_page( client: &mut Client, url: &str, owner: &str, repo: &str, commit_hash: &str, token: Option<&str>, page: u32, ) -> Result { let mut req = client.get(format!( "{url}/api/v1/repos/{owner}/{repo}/git/trees/{commit_hash}?recursive=true&page={page}" )); if let Some(token) = token { req = req.with_request_header("Authorization", format!("token {token}")); } // TODO ensure we don't use too much memory Ok(req.await?.success()?.response_json().await?) } #[derive(Debug)] pub enum FetchRepoError { CannotCreateDir(std::io::Error), TooManyEntries, Client(ClientError), UrlParsing, WriteRepoMetadata(WriteRepoMetadataError), } impl From for FetchRepoError { fn from(value: ClientError) -> Self { Self::Client(value) } } impl From for FetchRepoError { fn from(value: trillium_client::Error) -> Self { Self::Client(value.into()) } } impl From for FetchRepoError { fn from(value: trillium_client::UnexpectedStatusError) -> Self { Self::Client(value.into()) } } impl From for FetchRepoError { fn from(value: trillium_client::ClientSerdeError) -> Self { Self::Client(value.into()) } } impl From for FetchRepoError { fn from(_value: url::ParseError) -> Self { Self::UrlParsing } } impl From for FetchRepoError { fn from(value: WriteRepoMetadataError) -> Self { Self::WriteRepoMetadata(value) } } #[derive(Default)] pub struct RepoIndex { pub files: Vec, } pub struct RepoIndexFile { pub path: String, pub url: String, } pub async fn fetch_repo_tree_index_at_commit( client: &mut Client, repo_url: &str, commit_hash: &str, token: Option<&str>, ) -> Result<(RepoIndex, RepoMetadata), FetchRepoError> { let parsed = url::Url::parse(repo_url)?; let mut base = parsed.clone(); base.set_fragment(None); base.set_path(""); let base_url = base.as_str(); // Is the URL always /owner/repo? let mut segments = parsed.path_segments().ok_or(FetchRepoError::UrlParsing)?; let owner = segments.next().ok_or(FetchRepoError::UrlParsing)?; let repo = segments.next().ok_or(FetchRepoError::UrlParsing)?; let repo = repo.strip_suffix(".git").unwrap_or(repo); let mut repo_index = RepoIndex::default(); let mut count: u32 = 0; for page in 1..MAX_PAGE { let res = fetch_repo_tree_at_commit_page(client, base_url, owner, repo, commit_hash, token, page) .await?; count = count.saturating_add(res.total_count); if count > MAX_ENTRIES || res.total_count > MAX_ENTRIES { return Err(FetchRepoError::TooManyEntries); } if res.tree.is_empty() { break; } for entry in res.tree { if entry.entry_type.as_str() == "blob" && entry.size <= MAX_FILE_SIZE && entry.path.len() <= MAX_PATH_SIZE && entry.url.len() <= MAX_URL_SIZE { repo_index.files.push(RepoIndexFile { path: entry.path, url: entry.url, }); } } if !res.truncated || count >= res.total_count { break; } } Ok(( repo_index, RepoMetadata { date: std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(), repo_url: repo_url.to_string(), commit_hash: commit_hash.to_string(), content: Vec::new(), }, )) } #[derive(Deserialize)] struct GitBlob { content: String, } pub async fn fetch_repo_files( config: &Config, client: &mut Client, repo_index: &RepoIndex, repo_metadata: &mut RepoMetadata, token: Option<&str>, ) -> Result<(), FetchRepoError> { let mut hasher = sha2::Sha256::default(); let repo_key: [u8; 32] = rand::rng().random(); hasher.update(repo_key); let repo_id: [u8; 32] = hasher.finalize_reset().into(); let mut repo_id_str = [0; 24]; base64_turbo::URL_SAFE .encode_into(&repo_id[0..16], &mut repo_id_str) .expect("unreachable"); let repo_id_str = str::from_utf8(&repo_id_str).expect("unreachable"); let repo_dir = PathBuf::from(&config.data_dir) .join(crate::SUBDIR_REPOS) .join(repo_id_str); std::fs::create_dir(&repo_dir).map_err(FetchRepoError::CannotCreateDir)?; let mut file_index = HashSet::new(); for file in repo_index.files.iter() { let mut req = client.get(file.url.clone()); if let Some(token) = token { req = req.with_request_header("Authorization", format!("token {token}")); } // TODO ensure we don't use too much memory let blob: GitBlob = req.await?.success()?.response_json().await?; if base64_turbo::STANDARD.estimate_decoded_len(blob.content.len()) as u64 > MAX_FILE_SIZE { continue; } if let Ok(content) = base64_turbo::STANDARD.decode(&blob.content) { hasher.update(&content); let file_name: [u8; 32] = hasher.finalize_reset().into(); if file_index.insert(file_name) { let mut file_name_str = [0; 44]; base64_turbo::URL_SAFE .encode_into(file_name, &mut file_name_str) .expect("unreachable"); repo_metadata .content .extend_from_slice(&(file.path.len() as u32).to_be_bytes()); repo_metadata .content .extend_from_slice(file.path.as_bytes()); repo_metadata.content.extend_from_slice(&file_name_str); let file_name_str = str::from_utf8(&file_name_str).expect("unreachable"); if let Ok(mut file) = std::fs::OpenOptions::new() .create_new(true) .write(true) .open(repo_dir.join(file_name_str)) { file.write_all(&content).unwrap(); } } } } repo_metadata.write_to_file(config, &repo_dir)?; Ok(()) }