diff --git a/Cargo.lock b/Cargo.lock index 94ed307..4da6f9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -244,10 +250,13 @@ dependencies = [ "askama", "base64-turbo", "form_urlencoded", + "giallo", + "log", "rand", "reqwest", "serde", "sha2", + "simplelog", "trillium", "trillium-askama", "trillium-caching-headers", @@ -323,6 +332,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -339,6 +357,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "deranged" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc3dc5ad92c2e2d1c193bbbbdf2ea477cb81331de4f3103f267ca18368b988c4" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -369,6 +396,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -433,6 +466,16 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -532,6 +575,20 @@ dependencies = [ "wasip2", ] +[[package]] +name = "giallo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d312f2ab96db12e6ce9aebc495792774e4d2d759ed124f4d845281816db2bcfa" +dependencies = [ + "flate2", + "onig-regset", + "papaya", + "rmp-serde", + "serde", + "serde_json", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -867,6 +924,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -897,6 +964,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + [[package]] name = "num-traits" version = "0.2.19" @@ -906,12 +979,53 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "onig-regset" +version = "6.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "080c4eeac92e475efa2893e52522ee3de48086cdad5fb5f36631a7fcb5293c81" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "papaya" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f" +dependencies = [ + "equivalent", + "seize", +] + [[package]] name = "parking" version = "2.2.1" @@ -947,6 +1061,12 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "polling" version = "3.11.0" @@ -970,6 +1090,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1089,6 +1215,25 @@ dependencies = [ "libc", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "routefinder" version = "0.5.4" @@ -1125,6 +1270,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "serde" version = "1.0.228" @@ -1214,6 +1369,23 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "simplelog" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" +dependencies = [ + "log", + "termcolor", + "time", +] + [[package]] name = "slab" version = "0.4.11" @@ -1316,6 +1488,15 @@ dependencies = [ "syn", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1345,6 +1526,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -1723,6 +1937,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 85a8074..41a8cca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,13 @@ askama = "0.12.1" base64-turbo = "0.1.3" #flate2 = "1.1.8" form_urlencoded = "1.2.2" +giallo = { version = "0.3.1", features = ["dump"] } +log = "0.4.29" rand = "0.9.2" reqwest = { version = "0.13.1", default-features = false, features = ["json"] } serde = { version = "1.0.228", features = ["derive"] } sha2 = "0.10.9" +simplelog = "0.12.2" #tar = "0.4.44" trillium = "0.2.20" trillium-askama = "0.3.2" diff --git a/src/api_client.rs b/src/api_client.rs index c07929a..a13496f 100644 --- a/src/api_client.rs +++ b/src/api_client.rs @@ -160,7 +160,9 @@ pub async fn fetch_repo_files( .expect("unreachable"); let repo_id_str = str::from_utf8(&repo_id_str).expect("unreachable"); - let repo_dir = PathBuf::from(&config.data_dir).join(repo_id_str); + let repo_dir = PathBuf::from(&config.data_dir) + .join(crate::SUBDIR_REPOS) + .join(repo_id_str); std::fs::create_dir(&repo_dir).map_err(FetchRepoError::CannotCreateDir)?; let mut file_index = HashSet::new(); diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 0000000..ea2ab95 --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,56 @@ +use std::{borrow::Borrow, collections::{HashMap, VecDeque}, hash::Hash, time::Instant}; + +pub struct CacheEntry { + inner: T, + time: u64, +} + +#[derive(Default)] +pub struct Cache { + ttl: u64, + size: usize, + content: HashMap>, +} + +impl Cache { + pub fn sweep(&mut self) { + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + self.content.retain(|_k, entry| entry.time + self.ttl < now); + } + + fn sweep_oldest(&mut self) { + // TODO sweep the N oldest + let mut oldest = None; + let mut time = u64::MAX; + for (key, entry) in self.content.iter() { + if entry.time < time { + time = entry.time; + oldest = Some(key); + } + } + if let Some(oldest) = oldest.cloned() { + self.content.remove(&oldest); + } + } + + pub fn fetch(&mut self, key: impl Borrow, f: impl Fn(K) -> Option) -> Option { + if let Some(entry) = self.content.get_mut(key.borrow()) { + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + entry.time = now; + Some(entry.inner.clone()) + } else if let Some(inner) = (f)(key.borrow().clone()) { + if self.content.len() >= self.size { + self.sweep_oldest(); + } + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + let entry = CacheEntry { + time: now, + inner: inner.clone(), + }; + self.content.insert(key.borrow().clone(), entry); + Some(inner) + } else { + None + } + } +} diff --git a/src/config.rs b/src/config.rs index 6098d09..0422286 100644 --- a/src/config.rs +++ b/src/config.rs @@ -24,7 +24,7 @@ impl Default for Config { Self { listen_host: String::from("127.0.0.1"), listen_port: 44617, - data_dir: String::from("TODO"), + data_dir: String::from("/var/lib/blindforge"), api_client_user_agent: String::from("Blindforge"), api_client_max_page: 32, max_entries: 1024, diff --git a/src/main.rs b/src/main.rs index 53dd655..c78c851 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,13 +1,30 @@ +use std::{path::PathBuf, sync::Arc}; + mod api_client; +mod cache; mod config; +mod queue; mod repo; mod server; mod templates; +const SUBDIR_REPOS: &str = "repos"; + fn main() { - let config = config::Config::default(); + simplelog::TermLogger::init( + simplelog::LevelFilter::Debug, + simplelog::Config::default(), + simplelog::TerminalMode::Stderr, + simplelog::ColorChoice::Auto, + ) + .unwrap(); + let config = Arc::new(config::Config::default()); + std::fs::DirBuilder::new() + .recursive(true) + .create(PathBuf::from(&config.data_dir).join(SUBDIR_REPOS)) + .expect("Cannot create repos dir"); trillium_smol::config() .with_host(&config.listen_host) .with_port(config.listen_port) - .run(server::make_router()); + .run(server::make_router(config)); } diff --git a/src/queue.rs b/src/queue.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/queue.rs @@ -0,0 +1 @@ + diff --git a/src/repo.rs b/src/repo.rs index af7c8d8..e0acc76 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -4,11 +4,12 @@ use crate::{ }; use std::{ - io::{ErrorKind, Read}, + io::{ErrorKind, Read, Write}, path::Path, }; const REPO_METADATA_FILE_NAME: &str = "meta.bin"; +const VERSION: [u8; 1] = [0]; #[derive(Debug)] pub struct RepoMetadata { @@ -17,6 +18,22 @@ pub struct RepoMetadata { content: Vec, } +#[derive(Debug)] +pub enum WriteRepoMetadataError { + AlreadyExists, + CannotOpenFile(std::io::Error), + CannotWriteFile(std::io::Error), +} + +impl From for WriteRepoMetadataError { + fn from(value: std::io::Error) -> Self { + match value.kind() { + ErrorKind::AlreadyExists => Self::AlreadyExists, + _ => Self::CannotWriteFile(value), + } + } +} + #[derive(Debug)] pub enum ReadRepoMetadataError { CannotOpenFile(std::io::Error), @@ -41,10 +58,35 @@ impl From for ReadRepoMetadataError { } impl RepoMetadata { - pub fn new(config: &Config, repo_dir: &Path) -> Result { + pub fn write_to_file( + &self, + config: &Config, + repo_dir: &Path, + ) -> Result<(), WriteRepoMetadataError> { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(repo_dir.join("index")) + .map_err(WriteRepoMetadataError::CannotOpenFile)?; + + file.write_all(&VERSION)?; + + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + file.write_all(&now.to_be_bytes())?; + + file.write_all(&(self.commit_url.len() as u32).to_be_bytes())?; + + file.write_all(self.commit_url.as_bytes())?; + + file.write_all(&self.content)?; + + Ok(()) + } + + pub fn read_from_file(config: &Config, repo_dir: &Path) -> Result { let mut file = std::fs::OpenOptions::new() .read(true) - .open(repo_dir.join("")) + .open(repo_dir.join("index")) .map_err(ReadRepoMetadataError::CannotOpenFile)?; let mut version = [0u8; 1]; @@ -92,8 +134,8 @@ pub struct RepoMetadataIter<'a> { } pub struct RepoMetadataEntry<'a> { - file_path: &'a str, - hash: &'a str, + pub file_path: &'a str, + pub hash: &'a str, } impl<'a> Iterator for RepoMetadataIter<'a> { diff --git a/src/server.rs b/src/server.rs index df61e45..46f5933 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,4 +1,8 @@ +use crate::{cache, config::Config, repo::ReadRepoMetadataError}; + use askama::Template; +use log::error; +use std::{collections::HashMap, io::ErrorKind, path::PathBuf, sync::{Arc, Mutex}}; use trillium::{Conn, Handler}; use trillium_router::{Router, RouterConnExt}; @@ -6,7 +10,13 @@ pub async fn hello_world(conn: Conn) -> Conn { conn.ok("hello world!") } -pub fn make_router() -> impl Handler { +pub fn make_router(config: Arc) -> impl Handler { + let mut hl_registry = giallo::Registry::builtin().unwrap(); + hl_registry.link_grammars(); + let hl_registry = Arc::new(hl_registry); + + let mut metadata_cache = Arc::new(Mutex::new(cache::Cache::>::default())); + ( trillium_caching_headers::CachingHeaders::new(), //trillium_static_compiled::static_compiled!("./static").with_index_file("index.html"), @@ -29,8 +39,70 @@ pub fn make_router() -> impl Handler { //let planet = conn.param("planet").unwrap(); conn.ok(crate::templates::Home {}.render().unwrap()) }) - .get("/r/:hash", |conn: Conn| async move { - conn.ok(crate::templates::Home {}.render().unwrap()) + .get("/r/:hash/*", move |conn: Conn| { + let hl_registry = hl_registry.clone(); + let config = config.clone(); + let metadata_cache = metadata_cache.clone(); + async move { + let Some(repo_hash_str) = conn.param("hash") else { + return conn.with_status(401); + }; + + let cache_fetch = |key| { + let mut repo_hash = [0; 32]; + if base64_turbo::URL_SAFE.decode_into(repo_hash_str, &mut repo_hash) != Ok(32) { + return None; + } + let repo_dir = PathBuf::from(&config.data_dir).join(crate::SUBDIR_REPOS).join(repo_hash_str); + let repo_metadata = + match crate::repo::RepoMetadata::read_from_file(&config, &repo_dir) { + Ok(v) => v, + Err(e) => { + if let ReadRepoMetadataError::CannotOpenFile(e) = &e { + if e.kind() == ErrorKind::NotFound { + return None; + } + } + error!("Reading repo metadata: {e:?}"); + return None; + } + }; + let mut files = HashMap::new(); + for file in repo_metadata.iter_files() { + match file { + Ok(file) => { + files.insert(file.file_path.to_string(), file.hash.to_string()); + } + Err(e) => { + error!("Reading repo metadata file index: {e:?}") + } + } + } + Some(files) + }; + // TODO replace mutex with better thing (less contention or async mutex) + metadata_cache.lock().unwrap().fetch(repo_hash_str.to_string(), cache_fetch); + + let hl_options = giallo::HighlightOptions::new( + "py", + giallo::ThemeVariant::Single("catppuccin-frappe"), + ); + let highlighted = hl_registry + .highlight("def foo():\n\tpass", &hl_options) + .unwrap(); + let html = giallo::HtmlRenderer::default().render( + &highlighted, + &giallo::RenderOptions { + show_line_numbers: true, + ..Default::default() + }, + ); + conn.ok(crate::templates::Repo { + content: html.clone(), + } + .render() + .unwrap()) + } }) .get("/e/:secret", |conn: Conn| async move { conn.ok(crate::templates::Home {}.render().unwrap()) diff --git a/src/templates.rs b/src/templates.rs index f86e350..0432847 100644 --- a/src/templates.rs +++ b/src/templates.rs @@ -3,3 +3,9 @@ use trillium_askama::Template; #[derive(Template)] #[template(path = "home.html")] pub struct Home {} + +#[derive(Template)] +#[template(path = "repo.html")] +pub struct Repo { + pub content: String, +} diff --git a/static/index.html b/static/index.html deleted file mode 100644 index 1e64cab..0000000 --- a/static/index.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - Blindforge - - -
- -
- -
- - diff --git a/templates/repo.html b/templates/repo.html new file mode 100644 index 0000000..95ecdcb --- /dev/null +++ b/templates/repo.html @@ -0,0 +1,62 @@ + + + + + Repo - Blindforge + + + +
+

Titre

+
+
+
    +
  • Fichier
  • +
  • Fichier
  • +
  • +
    + Dossier +
      +
    • Fichier
    • +
    • Fichier
    • +
    • +
      + Dossier + +
      +
    • +
    • Fichier
    • +
    + +
    +
  • +
  • Fichier
  • +
+
+
+

Fichier

+ Root / Dossier / Fichier +
+ {{ content|safe }} +
+
+ +