diff --git a/Cargo.lock b/Cargo.lock index 5dca83a9..85bb0a4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,6 +763,7 @@ dependencies = [ "lock_api", "once_cell", "parking_lot_core", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f38981db..e5c205f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,7 +92,7 @@ indoc = "2" base64 = "0.22" console = "0.15" sha2 = "0.10" -dashmap = "6" +dashmap = { version = "6", features = ["serde"] } serde_yaml_ng = "0.10" schemars = { version = "0.8", features = ["chrono"] } pretty_yaml = "0.5" diff --git a/crates/rari-cli/main.rs b/crates/rari-cli/main.rs index a6203975..29caa25d 100644 --- a/crates/rari-cli/main.rs +++ b/crates/rari-cli/main.rs @@ -17,7 +17,7 @@ use rari_doc::build::{ build_generic_pages, build_spas, build_top_level_meta, }; use rari_doc::cached_readers::{read_and_cache_doc_pages, CACHED_DOC_PAGE_FILES}; -use rari_doc::issues::{issues_by, InMemoryLayer}; +use rari_doc::issues::IN_MEMORY; use rari_doc::pages::json::BuiltPage; use rari_doc::pages::types::doc::Doc; use rari_doc::reader::read_docs_parallel; @@ -183,6 +183,8 @@ struct BuildArgs { issues: Option, #[arg(long, help = "Annotate html with 'data-flaw' attributes")] data_issues: bool, + #[arg(long, help = "Add flaws field to index.json for docs")] + json_issues: bool, } #[derive(Debug)] @@ -219,7 +221,7 @@ fn main() -> Result<(), Error> { .with_target("rari_doc", Level::WARN) .with_target("rari", Level::WARN); - let memory_layer = InMemoryLayer::default(); + let memory_layer = IN_MEMORY.clone(); tracing_subscriber::registry() .with( tracing_subscriber::fmt::layer() @@ -235,6 +237,7 @@ fn main() -> Result<(), Error> { settings.deny_warnings = args.deny_warnings; settings.cache_content = !args.no_cache; settings.data_issues = args.data_issues; + settings.json_issues = args.json_issues; let _ = SETTINGS.set(settings); let templ_stats = if args.templ_stats { @@ -383,29 +386,9 @@ fn main() -> Result<(), Error> { if let Some(issues_path) = args.issues { let events = memory_layer.get_events(); - let events = events.lock().unwrap(); - - let issues = issues_by(&events); - - let mut tw = TabWriter::new(vec![]); - writeln!(&mut tw, "--- templ issues ---\t").expect("unable to write"); - for (templ, templ_issues) in &issues.templ { - writeln!(&mut tw, "{}\t{:5}", templ, templ_issues.len()) - .expect("unable to write"); - } - writeln!(&mut tw, "--- other issues ---\t").expect("unable to write"); - for (source, other_issues) in &issues.other { - writeln!(&mut tw, "{}\t{:5}", source, other_issues.len()) - .expect("unable to write"); - } - writeln!(&mut tw, "--- other issues w/o pos ---\t").expect("unable to write"); - for (source, no_pos) in &issues.no_pos { - writeln!(&mut tw, "{}\t{:5}", source, no_pos.len()).expect("unable to write"); - } - print!("{}", String::from_utf8_lossy(&tw.into_inner().unwrap())); let file = File::create(issues_path).unwrap(); let mut buffed = BufWriter::new(file); - serde_json::to_writer_pretty(&mut buffed, &issues).unwrap(); + serde_json::to_writer_pretty(&mut buffed, &*events).unwrap(); } } Commands::Serve(args) => { @@ -413,7 +396,7 @@ fn main() -> Result<(), Error> { settings.cache_content = args.cache; settings.data_issues = true; let _ = SETTINGS.set(settings); - serve::serve(memory_layer.clone())? + serve::serve()? } Commands::GitHistory => { println!("Gathering history 📜"); diff --git a/crates/rari-cli/serve.rs b/crates/rari-cli/serve.rs index c3cdadb3..c4ba74b6 100644 --- a/crates/rari-cli/serve.rs +++ b/crates/rari-cli/serve.rs @@ -1,10 +1,9 @@ use std::cmp::Ordering; use std::str::FromStr; use std::sync::atomic::AtomicU64; -use std::sync::Arc; use axum::body::Body; -use axum::extract::{Path, Request, State}; +use axum::extract::{Path, Request}; use axum::http::{header, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::routing::get; @@ -12,7 +11,7 @@ use axum::{Json, Router}; use rari_doc::cached_readers::wiki_histories; use rari_doc::contributors::contributors_txt; use rari_doc::error::DocError; -use rari_doc::issues::{to_display_issues, InMemoryLayer}; +use rari_doc::issues::{to_display_issues, IN_MEMORY}; use rari_doc::pages::json::BuiltPage; use rari_doc::pages::page::{Page, PageBuilder, PageLike}; use rari_doc::pages::types::doc::Doc; @@ -32,18 +31,15 @@ struct SearchItem { url: String, } -async fn handler(state: State>, req: Request) -> Response { +async fn handler(req: Request) -> Response { if req.uri().path().ends_with("/contributors.txt") { get_contributors_handler(req).await.into_response() } else { - get_json_handler(state, req).await.into_response() + get_json_handler(req).await.into_response() } } -async fn get_json_handler( - State(memory_layer): State>, - req: Request, -) -> Result, AppError> { +async fn get_json_handler(req: Request) -> Result, AppError> { let url = req.uri().path(); let req_id = REQ_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let span = span!(Level::WARN, "serve", req = req_id); @@ -52,22 +48,22 @@ async fn get_json_handler( let _enter1 = span.enter(); let url = url.strip_suffix("/index.json").unwrap_or(url); let page = Page::from_url_with_fallback(url)?; - let slug = &page.slug(); - let locale = page.locale(); - let span = span!(Level::ERROR, "page", "{}:{}", locale, slug); + let file = page.full_path().to_string_lossy(); + let span = span!( + Level::ERROR, + "page", + locale = page.locale().as_url_str(), + slug = page.slug(), + file = file.as_ref() + ); let _enter2 = span.enter(); let mut json = page.build()?; tracing::info!("{url}"); if let BuiltPage::Doc(json_doc) = &mut json { - let m = memory_layer.get_events(); - let mut issues = m.lock().unwrap(); - let req_issues: Vec<_> = issues - .iter() - .filter(|issue| issue.req == req_id) - .cloned() - .collect(); - issues.retain_mut(|i| i.req != req_id); - json_doc.doc.flaws = Some(to_display_issues(req_issues, &page)); + let m = IN_MEMORY.get_events(); + if let Some((_, req_issues)) = m.remove(page.full_path().to_string_lossy().as_ref()) { + json_doc.doc.flaws = Some(to_display_issues(req_issues, &page)); + } } Ok(Json(json)) } @@ -180,7 +176,7 @@ where } } -pub fn serve(memory_layer: InMemoryLayer) -> Result<(), anyhow::Error> { +pub fn serve() -> Result<(), anyhow::Error> { tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -188,8 +184,7 @@ pub fn serve(memory_layer: InMemoryLayer) -> Result<(), anyhow::Error> { .block_on(async { let app = Router::new() .route("/:locale/search-index.json", get(get_search_index_handler)) - .fallback(handler) - .with_state(Arc::new(memory_layer)); + .fallback(handler); let listener = tokio::net::TcpListener::bind("0.0.0.0:8083").await.unwrap(); axum::serve(listener, app).await.unwrap(); diff --git a/crates/rari-doc/src/build.rs b/crates/rari-doc/src/build.rs index f283e794..b96b76bb 100644 --- a/crates/rari-doc/src/build.rs +++ b/crates/rari-doc/src/build.rs @@ -14,7 +14,7 @@ use chrono::NaiveDateTime; use itertools::Itertools; use rari_types::globals::{ base_url, blog_root, build_out_root, contributor_spotlight_root, curriculum_root, - generic_content_root, git_history, + generic_content_root, git_history, settings, }; use rari_types::locale::{default_locale, Locale}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -27,6 +27,7 @@ use crate::cached_readers::{ }; use crate::contributors::contributors_txt; use crate::error::DocError; +use crate::issues::{to_display_issues, IN_MEMORY}; use crate::pages::build::copy_additional_files; use crate::pages::json::{BuiltPage, JsonDocMetadata}; use crate::pages::page::{Page, PageBuilder, PageLike}; @@ -67,7 +68,17 @@ pub fn build_single_page(page: &Page) -> Result<(BuiltPage, String), DocError> { file = file.as_ref() ); let _enter = span.enter(); - let built_page = page.build()?; + let mut built_page = page.build()?; + if settings().json_issues { + if let BuiltPage::Doc(json_doc) = &mut built_page { + if let Some(issues) = IN_MEMORY + .get_events() + .get(page.full_path().to_string_lossy().as_ref()) + { + json_doc.doc.flaws = Some(to_display_issues(issues.value().clone(), page)); + } + } + } let out_path = build_out_root() .expect("No BUILD_OUT_ROOT") .join(url_to_folder_path(page.url().trim_start_matches('/'))); diff --git a/crates/rari-doc/src/issues.rs b/crates/rari-doc/src/issues.rs index d515c7e9..3537d706 100644 --- a/crates/rari-doc/src/issues.rs +++ b/crates/rari-doc/src/issues.rs @@ -1,8 +1,9 @@ use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::sync::atomic::AtomicI64; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, LazyLock}; +use dashmap::DashMap; use itertools::Itertools; use schemars::JsonSchema; use serde::Serialize; @@ -20,12 +21,13 @@ pub(crate) fn get_issue_counter() -> i64 { ISSUE_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed) } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, Serialize)] pub struct Issue { pub req: u64, pub ic: i64, pub col: i64, pub line: i64, + pub file: String, pub fields: Vec<(&'static str, String)>, pub spans: Vec<(&'static str, String)>, } @@ -36,6 +38,7 @@ pub struct IssueEntries { ic: i64, col: i64, line: i64, + file: String, entries: Vec<(&'static str, String)>, } @@ -77,9 +80,6 @@ impl<'a> From<&'a Issue> for TemplIssue<'a> { let mut tissue = DEFAULT_TEMPL_ISSUE.clone(); for (key, value) in value.spans.iter().chain(value.fields.iter()) { match *key { - "file" => { - tissue.file = value.as_str(); - } "slug" => { tissue.slug = value.as_str(); } @@ -95,42 +95,18 @@ impl<'a> From<&'a Issue> for TemplIssue<'a> { } tissue.col = value.col; tissue.line = value.line; + tissue.file = value.file.as_str(); tissue } } -pub fn issues_by(issues: &[Issue]) -> Issues { - let mut templ: BTreeMap<&str, Vec> = BTreeMap::new(); - let mut other: BTreeMap<&str, Vec> = BTreeMap::new(); - let mut no_pos: BTreeMap<&str, Vec> = BTreeMap::new(); - for issue in issues.iter().map(TemplIssue::from) { - if let Some(templ_name) = - issue - .tail - .iter() - .find_map(|(key, value)| if *key == "templ" { Some(value) } else { None }) - { - templ.entry(templ_name).or_default().push(issue); - } else if issue.line != -1 { - other.entry(issue.source).or_default().push(issue) - } else { - no_pos.entry(issue.source).or_default().push(issue); - } - } - Issues { - templ, - other, - no_pos, - } -} - -#[derive(Clone, Default)] +#[derive(Clone, Default, Debug)] pub struct InMemoryLayer { - events: Arc>>, + events: Arc>>, } impl InMemoryLayer { - pub fn get_events(&self) -> Arc>> { + pub fn get_events(&self) -> Arc>> { Arc::clone(&self.events) } } @@ -140,7 +116,11 @@ impl Visit for IssueEntries { self.entries.push((field.name(), format!("{value:?}"))); } fn record_str(&mut self, field: &Field, value: &str) { - self.entries.push((field.name(), value.to_string())); + if field.name() == "file" { + self.file = value.to_string(); + } else { + self.entries.push((field.name(), value.to_string())); + } } fn record_u64(&mut self, field: &Field, value: u64) { if field.name() == "req" { @@ -162,7 +142,11 @@ impl Visit for Issue { self.fields.push((field.name(), format!("{value:?}"))); } fn record_str(&mut self, field: &Field, value: &str) { - self.fields.push((field.name(), value.to_string())); + if field.name() == "file" { + self.file = value.to_string(); + } else { + self.fields.push((field.name(), value.to_string())); + } } fn record_u64(&mut self, field: &Field, value: u64) { if field.name() == "req" { @@ -204,6 +188,7 @@ where ic: 0, col: 0, line: 0, + file: String::default(), fields: vec![], spans: vec![], }; @@ -221,6 +206,9 @@ where if entries.line != 0 { issue.line = entries.line; } + if !entries.file.is_empty() { + issue.file = entries.file.clone(); + } if entries.ic != 0 { issue.ic = entries.ic; } else { @@ -231,8 +219,10 @@ where } event.record(&mut issue); - let mut events = self.events.lock().unwrap(); - events.push(issue); + self.events + .entry(issue.file.clone()) + .or_default() + .push(issue); } } @@ -361,3 +351,5 @@ pub fn to_display_issues(issues: Vec, page: &Page) -> DisplayIssues { } map } + +pub static IN_MEMORY: LazyLock = LazyLock::new(InMemoryLayer::default); diff --git a/crates/rari-types/src/settings.rs b/crates/rari-types/src/settings.rs index ccd8f4f1..0444bce7 100644 --- a/crates/rari-types/src/settings.rs +++ b/crates/rari-types/src/settings.rs @@ -24,6 +24,7 @@ pub struct Settings { pub additional_locales_for_generics_and_spas: Vec, pub reader_ignores_gitignore: bool, pub data_issues: bool, + pub json_issues: bool, } impl Settings {