diff --git a/Cargo.lock b/Cargo.lock index 089a51c..0b08977 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -309,6 +309,7 @@ dependencies = [ "dashmap", "form_urlencoded", "giallo", + "linemd", "log", "rand", "serde", @@ -1048,6 +1049,12 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "linemd" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e8d932ea5e37464982f193ac2c9a7263d9ffaf5a9622688bbf981cc5b0fc92" + [[package]] name = "linux-raw-sys" version = "0.12.1" diff --git a/Cargo.toml b/Cargo.toml index b09bfbc..ea342b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ dashmap = "6.2.1" form_urlencoded = "1.2.2" # Code highlight giallo = { version = "0.5.0", features = ["dump"] } +# Markdown rendering +linemd = "0.5.0" log = "0.4.33" rand = "0.10.2" serde = { version = "1.0.229", features = ["derive"] } diff --git a/FORMATS.md b/FORMATS.md index db8dc0d..a9a984b 100644 --- a/FORMATS.md +++ b/FORMATS.md @@ -9,7 +9,8 @@ This file describes the internal storage formats used for persistent data. File structure: * Version (1 byte): `0x00` -* Time (8 bytes, big endian) +* Creation time (at which the repo was mirrored) (8 bytes, big endian) +* Update time (8 bytes, big endian) * Title length (4 bytes, big endian) (may be zero) * Title * Repository URL length (4 bytes, big endian) diff --git a/favicon.ico b/favicon.ico new file mode 100644 index 0000000..2705937 Binary files /dev/null and b/favicon.ico differ diff --git a/src/api_client.rs b/src/api_client.rs index 62d74e5..44ce89c 100644 --- a/src/api_client.rs +++ b/src/api_client.rs @@ -207,10 +207,13 @@ pub async fn fetch_repo_tree_index_at_commit( } } + let time = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + Ok(( repo_index, RepoMetadata { - date: std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(), + time_created: time, + time_updated: time, title: String::new(), repo_url: repo_url.to_string(), commit_hash: commit_hash.to_string(), @@ -230,7 +233,7 @@ pub async fn fetch_repo_files( repo_index: &RepoIndex, repo_metadata: &mut RepoMetadata, token: Option<&str>, -) -> Result<(), FetchRepoError> { +) -> Result { let mut hasher = sha2::Sha256::default(); let repo_key: [u8; 32] = rand::rng().random(); @@ -288,7 +291,7 @@ pub async fn fetch_repo_files( } } - repo_metadata.write_to_file(config, &repo_dir)?; + repo_metadata.write_to_file(&repo_dir)?; - Ok(()) + Ok(repo_id_str.to_string()) } diff --git a/src/cache.rs b/src/cache.rs index 31eb036..50b977c 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -7,26 +7,38 @@ use std::{ pub struct CacheEntry { inner: T, + /// Latest access time: AtomicU64, } +/// Index values V by key K pub struct Cache { + /// Time to live for entries ttl: u64, + /// Maximum number of entries size: usize, content: DashMap>, } -impl Default for Cache { - fn default() -> Self { +//impl Default for Cache { +// fn default() -> Self { +// Self { +// ttl: 0, +// size: 0, +// content: DashMap::new(), +// } +// } +//} + +impl Cache { + pub fn new(size: usize) -> Self { Self { ttl: 0, - size: 0, + size, content: DashMap::new(), } } -} -impl Cache { pub fn sweep(&self) { let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); self.content @@ -50,13 +62,14 @@ impl Cache { } } + /// Get an existing entry at the given key, or insert a new one pub fn fetch(&self, key: impl Borrow, f: impl Fn(K) -> Option) -> Option { if let Some(entry) = self.content.get(key.borrow()) { let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); entry.time.store(now, Relaxed); Some(entry.inner.clone()) } else if let Some(inner) = (f)(key.borrow().clone()) { - if self.content.len() >= self.size { + while self.content.len() >= self.size { self.sweep_oldest(); } let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); @@ -70,8 +83,67 @@ impl Cache { None } } -} -/*pub struct CacheRef { - cache: -}*/ + /// Same as fetch, but if the entry exists, condition is called on it. + /// If condition returns true, then the entry is kept and returned. + /// Else, the entry is removed and potentially replaced by a fresh one. + pub fn fetch_or_renew( + &self, + key: impl Borrow, + f: impl Fn(K) -> Option, + condition: impl Fn(&V) -> bool, + ) -> Option { + if let Some(entry) = self.content.get(key.borrow()) { + if (condition)(&entry.value().inner) { + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + entry.time.store(now, Relaxed); + return Some(entry.inner.clone()); + } + } + if let Some(inner) = (f)(key.borrow().clone()) { + while self.content.len() >= self.size { + self.sweep_oldest(); + } + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + let entry = CacheEntry { + time: AtomicU64::new(now), + inner: inner.clone(), + }; + self.content.insert(key.borrow().clone(), entry); + Some(inner) + } else { + None + } + } + + pub fn fetch_cond_only( + &self, + key: impl Borrow, + condition: impl Fn(&V) -> bool, + ) -> Option { + if let Some(entry) = self.content.get(key.borrow()) { + if (condition)(&entry.value().inner) { + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + entry.time.store(now, Relaxed); + return Some(entry.inner.clone()); + } + } + None + } + + pub fn update(&self, key: impl Borrow, value: Option) { + if let Some(value) = value { + while self.content.len() >= self.size { + self.sweep_oldest(); + } + let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); + let entry = CacheEntry { + time: AtomicU64::new(now), + inner: value, + }; + self.content.insert(key.borrow().clone(), entry); + } else { + self.content.remove(key.borrow()); + } + } +} diff --git a/src/config.rs b/src/config.rs index 9acaa30..06cbb69 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,6 +17,12 @@ pub struct Config { pub max_file_path_len: usize, /// API URL maximum length pub api_client_max_url_len: usize, + /// Files that are served when showing a directory (separated by colon `:`, in order of decreasing priority) + pub default_files: String, + /// Maximum number of pages to keep in cache + pub page_cache_size: usize, + /// Maximum number of repository trees to keep in cache + pub repo_cache_size: usize, } impl Default for Config { @@ -31,6 +37,11 @@ impl Default for Config { max_file_size: 8 * 1024 * 1024, max_file_path_len: 8192, api_client_max_url_len: 256, + default_files: String::from( + "README.md:README.MD:readme.md:README.txt:README.TXT:readme.txt:README:readme", + ), + page_cache_size: 200, + repo_cache_size: 20, } } } @@ -97,6 +108,23 @@ impl Config { .try_into() .expect("Config: invalid api_client_max_url_len"); } + if let Some(v) = toml.get("default_files") { + config.default_files = v.as_string().expect("Config: invalid default_files").into() + } + if let Some(v) = toml.get("page_cache_size") { + config.page_cache_size = v + .as_integer() + .expect("Config: invalid page_cache_size") + .try_into() + .expect("Config: invalid page_cache_size"); + } + if let Some(v) = toml.get("repo_cache_size") { + config.repo_cache_size = v + .as_integer() + .expect("Config: invalid repo_cache_size") + .try_into() + .expect("Config: invalid repo_cache_size"); + } config } } diff --git a/src/main.rs b/src/main.rs index c74f8d3..1ae9c78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,10 @@ -#![feature(btree_set_entry)] - -use std::{path::PathBuf, sync::Arc}; +use std::path::PathBuf; mod api_client; mod cache; mod config; mod queue; +mod render; mod repo; mod server; mod templates; diff --git a/src/render.rs b/src/render.rs new file mode 100644 index 0000000..20ba175 --- /dev/null +++ b/src/render.rs @@ -0,0 +1,44 @@ +use linemd::Parser; + +pub struct Renderer { + hl_registry: giallo::Registry, +} + +impl Renderer { + pub fn new() -> Self { + let mut hl_registry = giallo::Registry::builtin().unwrap(); + hl_registry.link_grammars(); + Self { hl_registry } + } + pub fn render(&self, filetype: &str, content: &[u8], pretty: bool) -> String { + let Ok(content) = str::from_utf8(content) else { + return String::from("Cannot render file as it is not valid UTF-8."); + }; + if pretty && filetype == "md" { + linemd::render_as_html(content.parse_md()) + } else { + let hl_options = giallo::HighlightOptions::new( + filetype, + giallo::ThemeVariant::Single("catppuccin-frappe"), + ); + let highlighted = self + .hl_registry + .highlight(content, &hl_options) + .unwrap_or_else(|_e| { + // If extension is unknown + let hl_options = giallo::HighlightOptions::new( + giallo::PLAIN_GRAMMAR_NAME, + giallo::ThemeVariant::Single("catppuccin-frappe"), + ); + self.hl_registry.highlight(content, &hl_options).unwrap() + }); + giallo::HtmlRenderer::default().render( + &highlighted, + &giallo::RenderOptions { + show_line_numbers: true, + ..Default::default() + }, + ) + } + } +} diff --git a/src/repo.rs b/src/repo.rs index 0851eac..adfc807 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -1,7 +1,4 @@ -use crate::{ - api_client::{MAX_PATH_SIZE, MAX_URL_SIZE}, - config::Config, -}; +use crate::api_client::{MAX_PATH_SIZE, MAX_URL_SIZE}; use std::{ io::{ErrorKind, Read, Write}, @@ -13,7 +10,8 @@ const VERSION: [u8; 1] = [0]; #[derive(Clone, Debug)] pub struct RepoMetadata { - pub date: u64, + pub time_created: u64, + pub time_updated: u64, pub title: String, pub repo_url: String, pub commit_hash: String, @@ -54,27 +52,24 @@ impl From for ReadRepoMetadataError { } impl From for ReadRepoMetadataError { - fn from(value: std::string::FromUtf8Error) -> Self { + fn from(_value: std::string::FromUtf8Error) -> Self { Self::InvalidFormat } } impl RepoMetadata { - pub fn write_to_file( - &self, - config: &Config, - repo_dir: &Path, - ) -> Result<(), WriteRepoMetadataError> { + pub fn write_to_file(&self, repo_dir: &Path) -> Result<(), WriteRepoMetadataError> { let mut file = std::fs::OpenOptions::new() .write(true) .create_new(true) - .open(repo_dir.join("index")) + .open(repo_dir.join(REPO_METADATA_FILE_NAME)) .map_err(WriteRepoMetadataError::CannotOpenFile)?; file.write_all(&VERSION)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); file.write_all(&now.to_be_bytes())?; + file.write_all(&now.to_be_bytes())?; file.write_all(&(self.title.len() as u32).to_be_bytes())?; @@ -93,10 +88,10 @@ impl RepoMetadata { Ok(()) } - pub fn read_from_file(config: &Config, repo_dir: &Path) -> Result { + pub fn read_from_file(repo_dir: &Path) -> Result { let mut file = std::fs::OpenOptions::new() .read(true) - .open(repo_dir.join("index")) + .open(repo_dir.join(REPO_METADATA_FILE_NAME)) .map_err(ReadRepoMetadataError::CannotOpenFile)?; let mut version = [0u8; 1]; @@ -105,9 +100,13 @@ impl RepoMetadata { return Err(ReadRepoMetadataError::UnsupportedVersion); } - let mut date = [0u8; 8]; - file.read_exact(&mut date)?; - let date = u64::from_be_bytes(date); + let mut time_created = [0u8; 8]; + file.read_exact(&mut time_created)?; + let time_created = u64::from_be_bytes(time_created); + + let mut time_updated = [0u8; 8]; + file.read_exact(&mut time_updated)?; + let time_updated = u64::from_be_bytes(time_updated); let mut title_len = [0u8; 4]; file.read_exact(&mut title_len)?; @@ -146,7 +145,8 @@ impl RepoMetadata { file.read_to_end(&mut content)?; Ok(Self { - date, + time_created, + time_updated, title, repo_url, commit_hash, diff --git a/src/server.rs b/src/server.rs index f5b7f26..efe5c79 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,4 +1,4 @@ -use crate::{cache, config::Config, repo, templates}; +use crate::{cache, config::Config, render, repo, templates}; use askama::Template; use log::error; @@ -9,17 +9,23 @@ use std::{ use trillium::{Conn, Handler}; use trillium_router::{Router, RouterConnExt}; +#[derive(Clone)] +struct PageCacheEntry { + time: u64, + content: String, +} + pub fn make_router(config: &'static Config) -> impl Handler { - let mut hl_registry = giallo::Registry::builtin().unwrap(); - hl_registry.link_grammars(); - let hl_registry: &'static _ = Box::leak(Box::new(hl_registry)); + let renderer: &'static _ = Box::leak(Box::new(render::Renderer::new())); let metadata_cache: &'static _ = Box::leak(Box::new(cache::Cache::< String, (repo::RepoMetadata, templates::Directory), - >::default())); - let file_cache: &'static _ = - Box::leak(Box::new(cache::Cache::<(String, String), String>::default())); + >::new(config.repo_cache_size))); + let page_cache: &'static _ = Box::leak(Box::new(cache::Cache::< + (String, String), + PageCacheEntry, + >::new(config.page_cache_size))); let client: &'static _ = Box::leak(Box::new(async_lock::Mutex::new( crate::api_client::make_client(), ))); @@ -30,6 +36,10 @@ pub fn make_router(config: &'static Config) -> impl Handler { .get("/", |conn: Conn| async move { conn.ok(crate::templates::Home {}.render().unwrap()) }) + .get("/favicon.ico", |conn: Conn| async move { + conn.ok(&include_bytes!("../favicon.ico")[..]) + .with_response_header("Content-type", "image/x-icon") + }) .post("/fetch", move |mut conn: Conn| async move { if let Ok(request_body) = conn.request_body().with_max_len(8192).read_bytes().await { @@ -65,7 +75,7 @@ pub fn make_router(config: &'static Config) -> impl Handler { .await .expect("todo handle error"); repo_metadata.title = title; - crate::api_client::fetch_repo_files( + let repo_id_str = crate::api_client::fetch_repo_files( config, &mut client, &repo_index, @@ -74,6 +84,9 @@ pub fn make_router(config: &'static Config) -> impl Handler { ) .await .expect("todo handle error"); + return conn + .with_status(303) + .with_response_header("Location", format!("/r/{repo_id_str}")); } conn.ok(crate::templates::Home {}.render().unwrap()) }) @@ -88,7 +101,7 @@ pub fn make_router(config: &'static Config) -> impl Handler { .join(crate::SUBDIR_REPOS) .join(key); let mut repo_metadata = - match crate::repo::RepoMetadata::read_from_file(config, &repo_dir) { + match crate::repo::RepoMetadata::read_from_file(&repo_dir) { Ok(v) => v, Err(e) => { if let repo::ReadRepoMetadataError::CannotOpenFile(e) = &e { @@ -113,7 +126,7 @@ pub fn make_router(config: &'static Config) -> impl Handler { root.insert( templates::Entry::File(templates::File { name: name.into(), - path: dbg!(entry.file_path).into(), + path: entry.file_path.into(), hash: entry.hash.into(), link: format!( "/r/{repo_hash_str}/{0}", @@ -138,74 +151,98 @@ pub fn make_router(config: &'static Config) -> impl Handler { return conn.with_status(404); }; - let Some(repo_file) = root.find(conn.path().split('/')) else { - return conn.with_status(404); + let invalidate_page_cache = |entry: &PageCacheEntry| { + // strict cmp because it's better to spend more on useless invalidations + // than to serve outdated pages + // (e.g. privacy issues if the author thinks sensible data are removed but we still serve them) + entry.time > repo_metadata.time_updated }; - - let fetch_file = |(repo_hash, file_hash)| { - let file_path = PathBuf::from(&config.data_dir) - .join(crate::SUBDIR_REPOS) - .join(repo_hash) - .join(file_hash); - - let mut file = match std::fs::OpenOptions::new().read(true).open(&file_path) - { - Ok(file) => file, - Err(e) => { - error!("Cannot open file `{file_path:?}`: {e:?}"); - return None; - } + if let Some(rendered_page) = page_cache.fetch_cond_only( + (repo_hash_str.to_string(), conn.path().to_string()), + invalidate_page_cache, + ) { + conn.ok(rendered_page.content) + } else { + let Some(served_entry) = root.find(conn.path().split('/')) else { + page_cache + .update((repo_hash_str.to_string(), conn.path().to_string()), None); + return conn.with_status(404); }; - let mut buf = String::new(); - if let Err(e) = file.read_to_string(&mut buf) { - error!("Error reading file `{file_path:?}`: {e:?}"); - return None; - } + let rendered_page = match served_entry { + templates::EntryRef::Directory(served_dir) => { + let mut served_file = None; + for default_filename in config.default_files.split(':') { + if let Some(templates::Entry::File(f)) = + served_dir.entries.get(default_filename) + { + served_file = Some(f); + break; + } + } + let mut html = String::new(); + if let Some(served_file) = served_file { + let Some(file_content) = + fetch_file(config, repo_hash_str, &served_file.hash) + else { + return conn.with_status(500); + }; - Some(buf) - }; - let Some(file_content) = file_cache.fetch( - (repo_hash_str.to_string(), repo_file.hash.clone()), - fetch_file, - ) else { - return conn.with_status(500); - }; + let mut file_lang = served_file + .name + .rsplit('.') + .next() + .unwrap_or(giallo::PLAIN_GRAMMAR_NAME) + .to_string(); + file_lang.make_ascii_lowercase(); - let file_lang = conn - .path() - .rsplit('.') - .next() - .unwrap_or(giallo::PLAIN_GRAMMAR_NAME); + html = renderer.render(&file_lang, &file_content, true); + } else { + // TODO something? + } - let hl_options = giallo::HighlightOptions::new( - file_lang, - giallo::ThemeVariant::Single("catppuccin-frappe"), - ); - let highlighted = hl_registry - .highlight(&file_content, &hl_options) - .unwrap_or_else(|_e| { - // If extension is unknown - let hl_options = giallo::HighlightOptions::new( - giallo::PLAIN_GRAMMAR_NAME, - giallo::ThemeVariant::Single("catppuccin-frappe"), - ); - hl_registry.highlight(&file_content, &hl_options).unwrap() - }); - let html = giallo::HtmlRenderer::default().render( - &highlighted, - &giallo::RenderOptions { - show_line_numbers: true, - ..Default::default() - }, - ); - let template = crate::templates::Repo { - content: html.clone(), - root, - path: conn.path().split('/').map(|s| s.to_string()).collect(), - title: repo_metadata.title, - }; - conn.ok(template.render().unwrap()) + let template = crate::templates::Repo { + content: html.clone(), + root, + path: conn.path().split('/').map(|s| s.to_string()).collect(), + title: repo_metadata.title, + }; + template.render().unwrap() + } + templates::EntryRef::File(served_file) => { + let Some(file_content) = + fetch_file(config, repo_hash_str, &served_file.hash) + else { + return conn.with_status(500); + }; + + let mut file_lang = served_file + .name + .rsplit('.') + .next() + .unwrap_or(giallo::PLAIN_GRAMMAR_NAME) + .to_string(); + file_lang.make_ascii_lowercase(); + + let html = renderer.render(&file_lang, &file_content, true); + let template = crate::templates::Repo { + content: html.clone(), + root, + path: conn.path().split('/').map(|s| s.to_string()).collect(), + title: repo_metadata.title, + }; + template.render().unwrap() + } + }; + page_cache.update( + (repo_hash_str.to_string(), conn.path().to_string()), + Some(PageCacheEntry { + time: std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(), + content: rendered_page.clone(), + }), + ); + conn.ok(rendered_page) + } } }) .get("/e/:secret", |conn: Conn| async move { @@ -213,3 +250,26 @@ pub fn make_router(config: &'static Config) -> impl Handler { }), ) } + +fn fetch_file(config: &Config, repo_hash: &str, file_hash: &str) -> Option> { + let file_path = PathBuf::from(&config.data_dir) + .join(crate::SUBDIR_REPOS) + .join(repo_hash) + .join(file_hash); + + let mut file = match std::fs::OpenOptions::new().read(true).open(&file_path) { + Ok(file) => file, + Err(e) => { + error!("Cannot open file `{file_path:?}`: {e:?}"); + return None; + } + }; + + let mut buf = Vec::new(); + if let Err(e) = file.read_to_end(&mut buf) { + error!("Error reading file `{file_path:?}`: {e:?}"); + return None; + } + + Some(buf) +} diff --git a/src/templates.rs b/src/templates.rs index 4233a99..40378ee 100644 --- a/src/templates.rs +++ b/src/templates.rs @@ -48,6 +48,11 @@ pub enum Entry { File(File), } +pub enum EntryRef<'a> { + Directory(&'a Directory), + File(&'a File), +} + #[derive(Clone)] pub struct File { pub name: String, @@ -121,10 +126,16 @@ impl Directory { } } - pub fn find<'a>(&self, mut path: impl Iterator) -> Option<&File> { - match self.entries.get(path.next()?)? { + pub fn find<'a, 'b>(&'a self, mut path: impl Iterator) -> Option> { + let Some(next_path) = path.next() else { + return Some(EntryRef::Directory(self)); + }; + if next_path.is_empty() { + return Some(EntryRef::Directory(self)); + } + match self.entries.get(next_path)? { Entry::Directory(directory) => directory.find(path), - Entry::File(file) => Some(file), + Entry::File(file) => Some(EntryRef::File(file)), } } }