From 75385e025434879debcf699bd0728853c14caad1 Mon Sep 17 00:00:00 2001 From: Josh Date: Sun, 24 Aug 2025 19:43:51 -0500 Subject: [PATCH 01/56] workspace --- Cargo.lock | 279 ++++++++++++++++++++++++++++ Cargo.toml | 4 + crates/djls-server/Cargo.toml | 4 + crates/djls-workspace/Cargo.toml | 15 ++ crates/djls-workspace/src/bridge.rs | 108 +++++++++++ crates/djls-workspace/src/db.rs | 91 +++++++++ crates/djls-workspace/src/lib.rs | 30 +++ crates/djls-workspace/src/vfs.rs | 245 ++++++++++++++++++++++++ 8 files changed, 776 insertions(+) create mode 100644 crates/djls-workspace/Cargo.toml create mode 100644 crates/djls-workspace/src/bridge.rs create mode 100644 crates/djls-workspace/src/db.rs create mode 100644 crates/djls-workspace/src/lib.rs create mode 100644 crates/djls-workspace/src/vfs.rs diff --git a/Cargo.lock b/Cargo.lock index b48cdd92..95b7a76e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,6 +168,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "camino" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d07aa9a93b00c76f71bc35d598bed923f6d4f3a9ca5c24b7737ae1a292841c0" + [[package]] name = "cfg-if" version = "1.0.1" @@ -403,6 +409,17 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "djls" version = "5.2.0-alpha" @@ -453,10 +470,13 @@ name = "djls-server" version = "0.0.0" dependencies = [ "anyhow", + "camino", + "dashmap", "djls-conf", "djls-dev", "djls-project", "djls-templates", + "djls-workspace", "percent-encoding", "pyo3", "salsa", @@ -467,6 +487,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "url", ] [[package]] @@ -481,6 +502,16 @@ dependencies = [ "toml", ] +[[package]] +name = "djls-workspace" +version = "0.0.0" +dependencies = [ + "camino", + "dashmap", + "salsa", + "url", +] + [[package]] name = "dlv-list" version = "0.5.2" @@ -564,6 +595,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.31" @@ -718,6 +758,113 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.10.0" @@ -817,6 +964,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + [[package]] name = "lock_api" version = "0.4.13" @@ -1056,6 +1209,15 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1465,6 +1627,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" @@ -1613,6 +1781,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tokio" version = "1.47.1" @@ -1859,6 +2037,23 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2111,6 +2306,12 @@ dependencies = [ "bitflags 2.9.2", ] +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + [[package]] name = "yaml-rust2" version = "0.10.3" @@ -2121,3 +2322,81 @@ dependencies = [ "encoding_rs", "hashlink", ] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 2a674a38..8a88a0fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ djls-dev = { path = "crates/djls-dev" } djls-project = { path = "crates/djls-project" } djls-server = { path = "crates/djls-server" } djls-templates = { path = "crates/djls-templates" } +djls-workspace = { path = "crates/djls-workspace" } # core deps, pin exact versions pyo3 = "0.25.0" @@ -17,8 +18,10 @@ salsa = "0.23.0" tower-lsp-server = { version = "0.22.0", features = ["proposed"] } anyhow = "1.0" +camino = "1.1" clap = { version = "4.5", features = ["derive"] } config = { version ="0.15", features = ["toml"] } +dashmap = "6.1" directories = "6.0" percent-encoding = "2.3" serde = { version = "1.0", features = ["derive"] } @@ -29,6 +32,7 @@ toml = "0.9" tracing = "0.1" tracing-appender = "0.2" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "time"] } +url = "2.5" which = "8.0" # testing diff --git a/crates/djls-server/Cargo.toml b/crates/djls-server/Cargo.toml index 396f0f38..e3f27f88 100644 --- a/crates/djls-server/Cargo.toml +++ b/crates/djls-server/Cargo.toml @@ -11,8 +11,11 @@ default = [] djls-conf = { workspace = true } djls-project = { workspace = true } djls-templates = { workspace = true } +djls-workspace = { workspace = true } anyhow = { workspace = true } +camino = { workspace = true } +dashmap = { workspace = true } percent-encoding = { workspace = true } pyo3 = { workspace = true } salsa = { workspace = true } @@ -23,6 +26,7 @@ tower-lsp-server = { workspace = true } tracing = { workspace = true } tracing-appender = { workspace = true } tracing-subscriber = { workspace = true } +url = { workspace = true } [build-dependencies] djls-dev = { workspace = true } diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml new file mode 100644 index 00000000..9f8edcec --- /dev/null +++ b/crates/djls-workspace/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "djls-workspace" +version = "0.0.0" +edition = "2021" + +[dependencies] +camino = { workspace = true } +dashmap = { workspace = true } +salsa = { workspace = true } +url = { workspace = true } + +[dev-dependencies] + +[lints] +workspace = true diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs new file mode 100644 index 00000000..97f894ed --- /dev/null +++ b/crates/djls-workspace/src/bridge.rs @@ -0,0 +1,108 @@ +//! Bridge between VFS snapshots and Salsa inputs. +//! +//! The bridge module isolates Salsa input mutation behind a single, idempotent API. +//! It ensures we only touch Salsa when content or classification changes, maximizing +//! incremental performance. + +use std::{collections::HashMap, sync::Arc}; + +use salsa::Setter; + +use super::{ + db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}, + vfs::{FileKind, VfsSnapshot}, + FileId, +}; + +/// Owner of the Salsa [`Database`] plus the handles for updating inputs. +/// +/// [`FileStore`] serves as the bridge between the VFS (with [`FileId`]s) and Salsa (with entities). +/// It maintains a mapping from [`FileId`]s to [`SourceFile`] entities and manages the global +/// [`TemplateLoaderOrder`] input. The [`FileStore`] ensures that Salsa inputs are only mutated +/// when actual changes occur, preserving incremental computation efficiency. +pub struct FileStore { + /// The Salsa DB instance + pub db: Database, + /// Map from [`FileId`] to its Salsa input entity + files: HashMap, + /// Handle to the global template loader configuration input + template_loader: Option, +} + +impl FileStore { + /// Construct an empty store and DB. + pub fn new() -> Self { + Self { + db: Database::default(), + files: HashMap::new(), + template_loader: None, + } + } + + /// Create or update the global template loader order input. + /// + /// Sets the ordered list of template root directories that Django will search + /// when resolving template names. If the input already exists, it updates the + /// existing value; otherwise, it creates a new [`TemplateLoaderOrder`] input. + pub fn set_template_loader_order(&mut self, ordered_roots: Vec) { + let roots = Arc::from(ordered_roots.into_boxed_slice()); + if let Some(tl) = self.template_loader { + tl.set_roots(&mut self.db).to(roots); + } else { + self.template_loader = Some(TemplateLoaderOrder::new(&self.db, roots)); + } + } + + /// Mirror a VFS snapshot into Salsa inputs. + /// + /// This method is the core synchronization point between the VFS and Salsa. + /// It iterates through all files in the snapshot and: + /// - Creates [`SourceFile`] inputs for new files + /// - Updates `.text` and `.kind` only when changed to preserve incremental reuse + /// + /// The method is idempotent and minimizes Salsa invalidations by checking for + /// actual changes before updating inputs. + pub fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { + for (id, rec) in &snap.files { + let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); + let new_kind = match rec.meta.kind { + FileKind::Python => FileKindMini::Python, + FileKind::Template => FileKindMini::Template, + FileKind::Other => FileKindMini::Other, + }; + + if let Some(sf) = self.files.get(id) { + // Update if changed — avoid touching Salsa when not needed + if sf.kind(&self.db) != new_kind { + sf.set_kind(&mut self.db).to(new_kind.clone()); + } + if sf.text(&self.db).as_ref() != &*new_text { + sf.set_text(&mut self.db).to(new_text.clone()); + } + } else { + let sf = SourceFile::new(&self.db, new_kind, new_text); + self.files.insert(*id, sf); + } + } + } + + /// Get the text content of a file by its [`FileId`]. + /// + /// Returns `None` if the file is not tracked in the [`FileStore`]. + pub fn file_text(&self, id: FileId) -> Option> { + self.files.get(&id).map(|sf| sf.text(&self.db).clone()) + } + + /// Get the file kind classification by its [`FileId`]. + /// + /// Returns `None` if the file is not tracked in the [`FileStore`]. + pub fn file_kind(&self, id: FileId) -> Option { + self.files.get(&id).map(|sf| sf.kind(&self.db)) + } +} + +impl Default for FileStore { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs new file mode 100644 index 00000000..91aa1ccc --- /dev/null +++ b/crates/djls-workspace/src/db.rs @@ -0,0 +1,91 @@ +//! Salsa database and input entities for workspace. +//! +//! This module defines the Salsa world—what can be set and tracked incrementally. +//! Inputs are kept minimal to avoid unnecessary recomputation. + +use std::sync::Arc; +#[cfg(test)] +use std::sync::Mutex; + +/// Salsa database root for workspace +/// +/// The [`Database`] provides default storage and, in tests, captures Salsa events for +/// reuse/diagnostics. It serves as the core incremental computation engine, tracking +/// dependencies and invalidations across all inputs and derived queries. +#[salsa::db] +#[derive(Clone)] +#[cfg_attr(not(test), derive(Default))] +pub struct Database { + storage: salsa::Storage, + + // The logs are only used for testing and demonstrating reuse: + #[cfg(test)] + logs: Arc>>>, +} + +#[cfg(test)] +impl Default for Database { + fn default() -> Self { + let logs = >>>>::default(); + Self { + storage: salsa::Storage::new(Some(Box::new({ + let logs = logs.clone(); + move |event| { + eprintln!("Event: {event:?}"); + // Log interesting events, if logging is enabled + if let Some(logs) = &mut *logs.lock().unwrap() { + // only log interesting events + if let salsa::EventKind::WillExecute { .. } = event.kind { + logs.push(format!("Event: {event:?}")); + } + } + } + }))), + logs, + } + } +} + +#[salsa::db] +impl salsa::Database for Database {} + +/// Minimal classification for analysis routing. +/// +/// [`FileKindMini`] provides a lightweight categorization of files to determine which +/// analysis pipelines should process them. This is the Salsa-side representation +/// of file types, mapped from the VFS layer's `vfs::FileKind`. +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKindMini { + /// Python source file (.py) + Python, + /// Django template file (.html, .jinja, etc.) + Template, + /// Other file types not requiring specialized analysis + Other, +} + +/// Represents a single file's classification and current content. +/// +/// [`SourceFile`] is a Salsa input entity that tracks both the file's type (for routing +/// to appropriate analyzers) and its current text content. The text is stored as +/// `Arc` for efficient sharing across the incremental computation graph. +#[salsa::input] +pub struct SourceFile { + /// The file's classification for analysis routing + pub kind: FileKindMini, + /// The current text content of the file + #[returns(ref)] + pub text: Arc, +} + +/// Global input configuring ordered template loader roots. +/// +/// [`TemplateLoaderOrder`] represents the Django `TEMPLATES[n]['DIRS']` configuration, +/// defining the search order for template resolution. This is a global input that +/// affects template name resolution across the entire project. +#[salsa::input] +pub struct TemplateLoaderOrder { + /// Ordered list of template root directories + #[returns(ref)] + pub roots: Arc<[String]>, +} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs new file mode 100644 index 00000000..aea4a539 --- /dev/null +++ b/crates/djls-workspace/src/lib.rs @@ -0,0 +1,30 @@ +mod bridge; +mod db; +mod vfs; + +// Re-export public API +pub use bridge::FileStore; +pub use db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}; +pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; + +/// Stable, compact identifier for files across the subsystem. +/// +/// [`FileId`] decouples file identity from paths/URIs, providing efficient keys for maps and +/// Salsa inputs. Once assigned to a file (via its URI), a [`FileId`] remains stable for the +/// lifetime of the VFS, even if the file's content or metadata changes. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub struct FileId(u32); + +impl FileId { + /// Create a [`FileId`] from a raw u32 value. + #[must_use] + pub fn from_raw(raw: u32) -> Self { + FileId(raw) + } + + /// Get the underlying u32 index value. + #[must_use] + pub fn index(self) -> u32 { + self.0 + } +} diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs new file mode 100644 index 00000000..65f40084 --- /dev/null +++ b/crates/djls-workspace/src/vfs.rs @@ -0,0 +1,245 @@ +//! Change-tracked, concurrent virtual file system keyed by [`FileId`]. +//! +//! The VFS provides thread-safe, identity-stable storage with cheap change detection +//! and snapshotting. Downstream systems consume snapshots to avoid locking and to +//! batch updates. + +use camino::Utf8PathBuf; +use dashmap::DashMap; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU32, AtomicU64, Ordering}, + Arc, + }, +}; +use url::Url; + +use super::FileId; + +/// Monotonic counter representing global VFS state. +/// +/// [`Revision`] increments whenever file content changes occur in the VFS. +/// This provides a cheap way to detect if any changes have occurred since +/// a previous snapshot was taken. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default, PartialOrd, Ord)] +pub struct Revision(u64); + +impl Revision { + /// Create a [`Revision`] from a raw u64 value. + pub fn from_raw(raw: u64) -> Self { + Revision(raw) + } + + /// Get the underlying u64 value. + pub fn value(self) -> u64 { + self.0 + } +} + +/// File classification at the VFS layer. +/// +/// [`FileKind`] determines how a file should be processed by downstream analyzers. +/// This classification is performed when files are first ingested into the VFS. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKind { + /// Python source file + Python, + /// Django template file + Template, + /// Other file type + Other, +} + +/// Metadata associated with a file in the VFS. +/// +/// [`FileMeta`] contains all non-content information about a file, including its +/// identity (URI), filesystem path, classification, and optional version number +/// from the LSP client. +#[derive(Clone, Debug)] +pub struct FileMeta { + /// The file's URI (typically file:// scheme) + pub uri: Url, + /// The file's path in the filesystem + pub path: Utf8PathBuf, + /// Classification for routing to analyzers + pub kind: FileKind, + /// Optional LSP document version + pub version: Option, +} + +/// Source of text content in the VFS. +/// +/// [`TextSource`] tracks where file content originated from, which is useful for +/// debugging and understanding the current state of the VFS. All variants hold +/// `Arc` for efficient sharing. +#[derive(Clone)] +pub enum TextSource { + /// Content loaded from disk + Disk(Arc), + /// Content from LSP client overlay (in-memory edits) + Overlay(Arc), + /// Content generated programmatically + Generated(Arc), +} + +/// Complete record of a file in the VFS. +/// +/// [`FileRecord`] combines metadata, current text content, and a content hash +/// for efficient change detection. +#[derive(Clone)] +pub struct FileRecord { + /// File metadata (URI, path, kind, version) + pub meta: FileMeta, + /// Current text content and its source + pub text: TextSource, + /// Hash of current content for change detection + pub hash: u64, +} + +/// Thread-safe virtual file system with change tracking. +/// +/// [`Vfs`] provides concurrent access to file content with stable [`FileId`] assignment, +/// content hashing for change detection, and atomic snapshot generation. It uses +/// `DashMap` for lock-free concurrent access and atomic counters for revision tracking. +pub struct Vfs { + /// Atomic counter for generating unique [`FileId`]s + next_file_id: AtomicU32, + /// Map from URI to [`FileId`] for deduplication + by_uri: DashMap, + /// Map from [`FileId`] to [`FileRecord`] for content storage + files: DashMap, + /// Global revision counter, incremented on content changes + head: AtomicU64, +} + +impl Vfs { + /// Construct an empty VFS. + pub fn new() -> Self { + Self { + next_file_id: AtomicU32::new(0), + by_uri: DashMap::new(), + files: DashMap::new(), + head: AtomicU64::new(0), + } + } + + /// Get or create a [`FileId`] for the given URI. + /// + /// Returns the existing [`FileId`] if the URI is already known, or creates a new + /// [`FileRecord`] with the provided metadata and text. This method computes and + /// stores a content hash for change detection. + pub fn intern_file( + &self, + uri: Url, + path: Utf8PathBuf, + kind: FileKind, + text: TextSource, + ) -> FileId { + if let Some(id) = self.by_uri.get(&uri).map(|entry| *entry) { + return id; + } + let id = FileId(self.next_file_id.fetch_add(1, Ordering::SeqCst)); + let meta = FileMeta { + uri: uri.clone(), + path, + kind, + version: None, + }; + let hash = content_hash(&text); + self.by_uri.insert(uri, id); + self.files.insert(id, FileRecord { meta, text, hash }); + id + } + + /// Set overlay text for a file, typically from LSP didChange events. + /// + /// Updates the file's text to an Overlay variant with the new content. + /// Only increments the global revision if the content actually changed + /// (detected via hash comparison). + /// + /// Returns a tuple of (new global revision, whether content changed). + pub fn set_overlay( + &self, + id: FileId, + version: Option, + new_text: Arc, + ) -> (Revision, bool) { + let mut rec = self.files.get_mut(&id).expect("unknown file"); + rec.meta.version = version; + let next = TextSource::Overlay(new_text); + let new_hash = content_hash(&next); + let changed = new_hash != rec.hash; + if changed { + rec.text = next; + rec.hash = new_hash; + self.head.fetch_add(1, Ordering::SeqCst); + } + ( + Revision::from_raw(self.head.load(Ordering::SeqCst)), + changed, + ) + } + + /// Create an immutable snapshot of the current VFS state. + /// + /// Materializes a consistent view of all files for downstream consumers. + /// The snapshot includes the current revision and a clone of all file records. + /// This operation is relatively cheap due to `Arc` sharing of text content. + pub fn snapshot(&self) -> VfsSnapshot { + VfsSnapshot { + revision: Revision::from_raw(self.head.load(Ordering::SeqCst)), + files: self + .files + .iter() + .map(|entry| (*entry.key(), entry.value().clone())) + .collect(), + } + } +} + +/// Compute a stable hash over file content. +/// +/// Used for efficient change detection - if the hash hasn't changed, +/// the content hasn't changed, avoiding unnecessary Salsa invalidations. +fn content_hash(src: &TextSource) -> u64 { + let s: &str = match src { + TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, + }; + let mut h = DefaultHasher::new(); + s.hash(&mut h); + h.finish() +} + +/// Immutable snapshot view of the VFS at a specific revision. +/// +/// [`VfsSnapshot`] provides a consistent view of all files for downstream consumers, +/// avoiding the need for locking during processing. Snapshots are created atomically +/// and can be safely shared across threads. +#[derive(Clone)] +pub struct VfsSnapshot { + /// The global revision at the time of snapshot + pub revision: Revision, + /// All files in the VFS at snapshot time + pub files: HashMap, +} + +impl VfsSnapshot { + /// Get the text content of a file in this snapshot. + /// + /// Returns `None` if the [`FileId`] is not present in the snapshot. + pub fn get_text(&self, id: FileId) -> Option> { + self.files.get(&id).map(|r| match &r.text { + TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s.clone(), + }) + } + + /// Get the metadata for a file in this snapshot. + /// + /// Returns `None` if the [`FileId`] is not present in the snapshot. + pub fn meta(&self, id: FileId) -> Option<&FileMeta> { + self.files.get(&id).map(|r| &r.meta) + } +} From 74b6b5b56daedfdcf79e632194c8512062c3bd50 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 03:47:56 -0500 Subject: [PATCH 02/56] wip --- Cargo.lock | 1 + crates/djls-server/src/server.rs | 10 +- crates/djls-server/src/workspace/document.rs | 154 ++++++--------- crates/djls-server/src/workspace/store.rs | 189 +++++++++++++------ crates/djls-workspace/Cargo.toml | 1 + crates/djls-workspace/src/vfs.rs | 53 +++--- 6 files changed, 227 insertions(+), 181 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95b7a76e..48cf438d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,6 +506,7 @@ dependencies = [ name = "djls-workspace" version = "0.0.0" dependencies = [ + "anyhow", "camino", "dashmap", "salsa", diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index 3977ef60..9df7c634 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -218,8 +218,9 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Opened document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let db = session.db(); - session.documents_mut().handle_did_open(&db, ¶ms); + if let Err(e) = session.documents_mut().handle_did_open(¶ms) { + tracing::error!("Failed to handle did_open: {}", e); + } }) .await; } @@ -228,8 +229,7 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Changed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let db = session.db(); - let _ = session.documents_mut().handle_did_change(&db, ¶ms); + let _ = session.documents_mut().handle_did_change(¶ms); }) .await; } @@ -248,9 +248,7 @@ impl LanguageServer for DjangoLanguageServer { .with_session(|session| { if let Some(project) = session.project() { if let Some(tags) = project.template_tags() { - let db = session.db(); return session.documents().get_completions( - &db, params.text_document_position.text_document.uri.as_str(), params.text_document_position.position, tags, diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs index 4c23f135..30788730 100644 --- a/crates/djls-server/src/workspace/document.rs +++ b/crates/djls-server/src/workspace/document.rs @@ -1,113 +1,66 @@ -use salsa::Database; -use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; -use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; +use std::sync::Arc; +use tower_lsp_server::lsp_types::{Position, Range}; +use djls_workspace::{FileId, VfsSnapshot}; -#[salsa::input(debug)] +/// Document metadata container - no longer a Salsa input, just plain data +#[derive(Clone, Debug)] pub struct TextDocument { - #[returns(ref)] pub uri: String, - #[returns(ref)] - pub contents: String, - #[returns(ref)] - pub index: LineIndex, pub version: i32, pub language_id: LanguageId, + file_id: FileId, } impl TextDocument { - pub fn from_did_open_params(db: &dyn Database, params: &DidOpenTextDocumentParams) -> Self { - let uri = params.text_document.uri.to_string(); - let contents = params.text_document.text.clone(); - let version = params.text_document.version; - let language_id = LanguageId::from(params.text_document.language_id.as_str()); - - let index = LineIndex::new(&contents); - TextDocument::new(db, uri, contents, index, version, language_id) - } - - pub fn with_changes( - self, - db: &dyn Database, - changes: &[TextDocumentContentChangeEvent], - new_version: i32, - ) -> Self { - let mut new_contents = self.contents(db).to_string(); - - for change in changes { - if let Some(range) = change.range { - let index = LineIndex::new(&new_contents); - - if let (Some(start_offset), Some(end_offset)) = ( - index.offset(range.start).map(|o| o as usize), - index.offset(range.end).map(|o| o as usize), - ) { - let mut updated_content = String::with_capacity( - new_contents.len() - (end_offset - start_offset) + change.text.len(), - ); - - updated_content.push_str(&new_contents[..start_offset]); - updated_content.push_str(&change.text); - updated_content.push_str(&new_contents[end_offset..]); - - new_contents = updated_content; - } - } else { - // Full document update - new_contents.clone_from(&change.text); - } + pub fn new(uri: String, version: i32, language_id: LanguageId, file_id: FileId) -> Self { + Self { + uri, + version, + language_id, + file_id, } - - let index = LineIndex::new(&new_contents); - TextDocument::new( - db, - self.uri(db).to_string(), - new_contents, - index, - new_version, - self.language_id(db), - ) } - - #[allow(dead_code)] - pub fn get_text(self, db: &dyn Database) -> String { - self.contents(db).to_string() + + pub fn file_id(&self) -> FileId { + self.file_id } - - #[allow(dead_code)] - pub fn get_text_range(self, db: &dyn Database, range: Range) -> Option { - let index = self.index(db); - let start = index.offset(range.start)? as usize; - let end = index.offset(range.end)? as usize; - let contents = self.contents(db); - Some(contents[start..end].to_string()) + + pub fn get_content(&self, vfs: &VfsSnapshot) -> Option> { + vfs.get_text(self.file_id) } - - pub fn get_line(self, db: &dyn Database, line: u32) -> Option { - let index = self.index(db); - let start = index.line_starts.get(line as usize)?; - let end = index - .line_starts + + pub fn get_line(&self, vfs: &VfsSnapshot, line_index: &LineIndex, line: u32) -> Option { + let content = self.get_content(vfs)?; + + let line_start = *line_index.line_starts.get(line as usize)?; + let line_end = line_index.line_starts .get(line as usize + 1) .copied() - .unwrap_or(index.length); - - let contents = self.contents(db); - Some(contents[*start as usize..end as usize].to_string()) + .unwrap_or(line_index.length); + + Some(content[line_start as usize..line_end as usize].to_string()) } - - #[allow(dead_code)] - pub fn line_count(self, db: &dyn Database) -> usize { - self.index(db).line_starts.len() + + pub fn get_text_range(&self, vfs: &VfsSnapshot, line_index: &LineIndex, range: Range) -> Option { + let content = self.get_content(vfs)?; + + let start_offset = line_index.offset(range.start)? as usize; + let end_offset = line_index.offset(range.end)? as usize; + + Some(content[start_offset..end_offset].to_string()) } + + pub fn get_template_tag_context(&self, vfs: &VfsSnapshot, line_index: &LineIndex, position: Position) -> Option { + let content = self.get_content(vfs)?; + + let start = line_index.line_starts.get(position.line as usize)?; + let end = line_index + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(line_index.length); - pub fn get_template_tag_context( - self, - db: &dyn Database, - position: Position, - ) -> Option { - let line = self.get_line(db, position.line)?; + let line = &content[*start as usize..end as usize]; let char_pos: usize = position.character.try_into().ok()?; let prefix = &line[..char_pos]; let rest_of_line = &line[char_pos..]; @@ -136,8 +89,8 @@ impl TextDocument { #[derive(Clone, Debug)] pub struct LineIndex { - line_starts: Vec, - length: u32, + pub line_starts: Vec, + pub length: u32, } impl LineIndex { @@ -201,6 +154,16 @@ impl From for LanguageId { } } +impl From for djls_workspace::FileKind { + fn from(language_id: LanguageId) -> Self { + match language_id { + LanguageId::Python => Self::Python, + LanguageId::HtmlDjango => Self::Template, + LanguageId::Other => Self::Other, + } + } +} + #[derive(Debug)] pub enum ClosingBrace { None, @@ -214,3 +177,4 @@ pub struct TemplateTagContext { pub closing_brace: ClosingBrace, pub needs_leading_space: bool, } + diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 3ec21097..0026fd0d 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -1,9 +1,11 @@ use std::collections::HashMap; +use std::sync::Arc; use anyhow::anyhow; use anyhow::Result; +use camino::Utf8PathBuf; use djls_project::TemplateTags; -use salsa::Database; +use djls_workspace::{FileId, FileKind, TextSource, Vfs}; use tower_lsp_server::lsp_types::CompletionItem; use tower_lsp_server::lsp_types::CompletionItemKind; use tower_lsp_server::lsp_types::CompletionResponse; @@ -16,83 +18,149 @@ use tower_lsp_server::lsp_types::MarkupContent; use tower_lsp_server::lsp_types::MarkupKind; use tower_lsp_server::lsp_types::Position; -use super::document::ClosingBrace; -use super::document::LanguageId; -use super::document::TextDocument; +use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; -#[derive(Debug, Default)] pub struct Store { - documents: HashMap, + vfs: Arc, + file_ids: HashMap, + line_indices: HashMap, versions: HashMap, + documents: HashMap, +} + +impl Default for Store { + fn default() -> Self { + Self { + vfs: Arc::new(Vfs::default()), + file_ids: HashMap::new(), + line_indices: HashMap::new(), + versions: HashMap::new(), + documents: HashMap::new(), + } + } } impl Store { - pub fn handle_did_open(&mut self, db: &dyn Database, params: &DidOpenTextDocumentParams) { - let uri = params.text_document.uri.to_string(); + pub fn handle_did_open(&mut self, params: &DidOpenTextDocumentParams) -> Result<()> { + let uri_str = params.text_document.uri.to_string(); + let uri = params.text_document.uri.clone(); let version = params.text_document.version; + let content = params.text_document.text.clone(); + let language_id = LanguageId::from(params.text_document.language_id.as_str()); + let kind = FileKind::from(language_id.clone()); + + // Convert URI to Url for VFS + let vfs_url = + url::Url::parse(&uri.to_string()).map_err(|e| anyhow!("Invalid URI: {}", e))?; + + // Convert to path - simplified for now, just use URI string + let path = Utf8PathBuf::from(uri.as_str()); + + // Store content in VFS + let text_source = TextSource::Overlay(Arc::from(content.as_str())); + let file_id = self.vfs.intern_file(vfs_url, path, kind, text_source); - let document = TextDocument::from_did_open_params(db, params); + // Set overlay content in VFS + self.vfs.set_overlay(file_id, Arc::from(content.as_str()))?; - self.add_document(document, uri.clone()); - self.versions.insert(uri, version); + // Create TextDocument metadata + let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); + self.documents.insert(uri_str.clone(), document); + + // Cache mappings and indices + self.file_ids.insert(uri_str.clone(), file_id); + self.line_indices.insert(file_id, LineIndex::new(&content)); + self.versions.insert(uri_str, version); + + Ok(()) } - pub fn handle_did_change( - &mut self, - db: &dyn Database, - params: &DidChangeTextDocumentParams, - ) -> Result<()> { - let uri = params.text_document.uri.as_str().to_string(); + pub fn handle_did_change(&mut self, params: &DidChangeTextDocumentParams) -> Result<()> { + let uri_str = params.text_document.uri.as_str().to_string(); let version = params.text_document.version; - let document = self - .get_document(&uri) - .ok_or_else(|| anyhow!("Document not found: {}", uri))?; + // Look up FileId + let file_id = self + .file_ids + .get(&uri_str) + .copied() + .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; + + // Get current content from VFS + let snapshot = self.vfs.snapshot(); + let current_content = snapshot + .get_text(file_id) + .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; + + // Apply text changes + let mut new_content = current_content.to_string(); + for change in ¶ms.content_changes { + if let Some(range) = change.range { + // Get current line index for position calculations + let line_index = self + .line_indices + .get(&file_id) + .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; + + if let (Some(start_offset), Some(end_offset)) = ( + line_index.offset(range.start).map(|o| o as usize), + line_index.offset(range.end).map(|o| o as usize), + ) { + let mut updated_content = String::with_capacity( + new_content.len() - (end_offset - start_offset) + change.text.len(), + ); + + updated_content.push_str(&new_content[..start_offset]); + updated_content.push_str(&change.text); + updated_content.push_str(&new_content[end_offset..]); + + new_content = updated_content; + } + } else { + // Full document update + new_content.clone_from(&change.text); + } + } + + // Update TextDocument version + if let Some(document) = self.documents.get_mut(&uri_str) { + document.version = version; + } - let new_document = document.with_changes(db, ¶ms.content_changes, version); + // Update VFS with new content + self.vfs + .set_overlay(file_id, Arc::from(new_content.as_str()))?; - self.documents.insert(uri.clone(), new_document); - self.versions.insert(uri, version); + // Update cached line index and version + self.line_indices + .insert(file_id, LineIndex::new(&new_content)); + self.versions.insert(uri_str, version); Ok(()) } pub fn handle_did_close(&mut self, params: &DidCloseTextDocumentParams) { - self.remove_document(params.text_document.uri.as_str()); - } + let uri_str = params.text_document.uri.as_str(); - fn add_document(&mut self, document: TextDocument, uri: String) { - self.documents.insert(uri, document); - } + // Remove TextDocument metadata + self.documents.remove(uri_str); - fn remove_document(&mut self, uri: &str) { - self.documents.remove(uri); - self.versions.remove(uri); - } + // Look up FileId and remove mappings + if let Some(file_id) = self.file_ids.remove(uri_str) { + self.line_indices.remove(&file_id); + } + self.versions.remove(uri_str); - fn get_document(&self, uri: &str) -> Option<&TextDocument> { - self.documents.get(uri) + // Note: We don't remove from VFS as it might be useful for caching + // The VFS will handle cleanup internally } - #[allow(dead_code)] - fn get_document_mut(&mut self, uri: &str) -> Option<&mut TextDocument> { - self.documents.get_mut(uri) + pub fn get_file_id(&self, uri: &str) -> Option { + self.file_ids.get(uri).copied() } - #[allow(dead_code)] - pub fn get_all_documents(&self) -> impl Iterator { - self.documents.values() - } - - #[allow(dead_code)] - pub fn get_documents_by_language<'db>( - &'db self, - db: &'db dyn Database, - language_id: LanguageId, - ) -> impl Iterator + 'db { - self.documents - .values() - .filter(move |doc| doc.language_id(db) == language_id) + pub fn get_line_index(&self, file_id: FileId) -> Option<&LineIndex> { + self.line_indices.get(&file_id) } #[allow(dead_code)] @@ -105,20 +173,31 @@ impl Store { self.get_version(uri) == Some(version) } + // TextDocument helper methods + pub fn get_document(&self, uri: &str) -> Option<&TextDocument> { + self.documents.get(uri) + } + + pub fn get_document_mut(&mut self, uri: &str) -> Option<&mut TextDocument> { + self.documents.get_mut(uri) + } + pub fn get_completions( &self, - db: &dyn Database, uri: &str, position: Position, tags: &TemplateTags, ) -> Option { + // Check if this is a Django template using TextDocument metadata let document = self.get_document(uri)?; - - if document.language_id(db) != LanguageId::HtmlDjango { + if document.language_id != LanguageId::HtmlDjango { return None; } - let context = document.get_template_tag_context(db, position)?; + // Get template tag context from document + let vfs_snapshot = self.vfs.snapshot(); + let line_index = self.get_line_index(document.file_id())?; + let context = document.get_template_tag_context(&vfs_snapshot, line_index, position)?; let mut completions: Vec = tags .iter() diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 9f8edcec..9b5ee6cc 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -4,6 +4,7 @@ version = "0.0.0" edition = "2021" [dependencies] +anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } salsa = { workspace = true } diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs index 65f40084..9ebe6a24 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs.rs @@ -4,6 +4,7 @@ //! and snapshotting. Downstream systems consume snapshots to avoid locking and to //! batch updates. +use anyhow::{anyhow, Result}; use camino::Utf8PathBuf; use dashmap::DashMap; use std::collections::hash_map::DefaultHasher; @@ -29,11 +30,13 @@ pub struct Revision(u64); impl Revision { /// Create a [`Revision`] from a raw u64 value. + #[must_use] pub fn from_raw(raw: u64) -> Self { Revision(raw) } /// Get the underlying u64 value. + #[must_use] pub fn value(self) -> u64 { self.0 } @@ -56,8 +59,7 @@ pub enum FileKind { /// Metadata associated with a file in the VFS. /// /// [`FileMeta`] contains all non-content information about a file, including its -/// identity (URI), filesystem path, classification, and optional version number -/// from the LSP client. +/// identity (URI), filesystem path, and classification. #[derive(Clone, Debug)] pub struct FileMeta { /// The file's URI (typically file:// scheme) @@ -66,8 +68,6 @@ pub struct FileMeta { pub path: Utf8PathBuf, /// Classification for routing to analyzers pub kind: FileKind, - /// Optional LSP document version - pub version: Option, } /// Source of text content in the VFS. @@ -116,16 +116,6 @@ pub struct Vfs { } impl Vfs { - /// Construct an empty VFS. - pub fn new() -> Self { - Self { - next_file_id: AtomicU32::new(0), - by_uri: DashMap::new(), - files: DashMap::new(), - head: AtomicU64::new(0), - } - } - /// Get or create a [`FileId`] for the given URI. /// /// Returns the existing [`FileId`] if the URI is already known, or creates a new @@ -146,7 +136,6 @@ impl Vfs { uri: uri.clone(), path, kind, - version: None, }; let hash = content_hash(&text); self.by_uri.insert(uri, id); @@ -161,14 +150,15 @@ impl Vfs { /// (detected via hash comparison). /// /// Returns a tuple of (new global revision, whether content changed). - pub fn set_overlay( - &self, - id: FileId, - version: Option, - new_text: Arc, - ) -> (Revision, bool) { - let mut rec = self.files.get_mut(&id).expect("unknown file"); - rec.meta.version = version; + /// + /// # Errors + /// + /// Returns an error if the provided `FileId` does not exist in the VFS. + pub fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { + let mut rec = self + .files + .get_mut(&id) + .ok_or_else(|| anyhow!("unknown file: {:?}", id))?; let next = TextSource::Overlay(new_text); let new_hash = content_hash(&next); let changed = new_hash != rec.hash; @@ -177,10 +167,10 @@ impl Vfs { rec.hash = new_hash; self.head.fetch_add(1, Ordering::SeqCst); } - ( + Ok(( Revision::from_raw(self.head.load(Ordering::SeqCst)), changed, - ) + )) } /// Create an immutable snapshot of the current VFS state. @@ -200,6 +190,17 @@ impl Vfs { } } +impl Default for Vfs { + fn default() -> Self { + Self { + next_file_id: AtomicU32::new(0), + by_uri: DashMap::new(), + files: DashMap::new(), + head: AtomicU64::new(0), + } + } +} + /// Compute a stable hash over file content. /// /// Used for efficient change detection - if the hash hasn't changed, @@ -230,6 +231,7 @@ impl VfsSnapshot { /// Get the text content of a file in this snapshot. /// /// Returns `None` if the [`FileId`] is not present in the snapshot. + #[must_use] pub fn get_text(&self, id: FileId) -> Option> { self.files.get(&id).map(|r| match &r.text { TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s.clone(), @@ -239,6 +241,7 @@ impl VfsSnapshot { /// Get the metadata for a file in this snapshot. /// /// Returns `None` if the [`FileId`] is not present in the snapshot. + #[must_use] pub fn meta(&self, id: FileId) -> Option<&FileMeta> { self.files.get(&id).map(|r| &r.meta) } From 48dacb277c9f54d7b72d79732b5bb032aacebc49 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 04:25:35 -0500 Subject: [PATCH 03/56] wip --- crates/djls-server/src/workspace/document.rs | 48 +++- crates/djls-server/src/workspace/store.rs | 276 +++++++++++++++++-- 2 files changed, 291 insertions(+), 33 deletions(-) diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs index 30788730..5a77a65d 100644 --- a/crates/djls-server/src/workspace/document.rs +++ b/crates/djls-server/src/workspace/document.rs @@ -90,24 +90,32 @@ impl TextDocument { #[derive(Clone, Debug)] pub struct LineIndex { pub line_starts: Vec, + pub line_starts_utf16: Vec, pub length: u32, + pub length_utf16: u32, } impl LineIndex { pub fn new(text: &str) -> Self { let mut line_starts = vec![0]; - let mut pos = 0; + let mut line_starts_utf16 = vec![0]; + let mut pos_utf8 = 0; + let mut pos_utf16 = 0; for c in text.chars() { - pos += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); if c == '\n' { - line_starts.push(pos); + line_starts.push(pos_utf8); + line_starts_utf16.push(pos_utf16); } } Self { line_starts, - length: pos, + line_starts_utf16, + length: pos_utf8, + length_utf16: pos_utf16, } } @@ -117,6 +125,38 @@ impl LineIndex { Some(line_start + position.character) } + /// Convert UTF-16 LSP position to UTF-8 byte offset + pub fn offset_utf16(&self, position: Position, text: &str) -> Option { + let line_start_utf8 = self.line_starts.get(position.line as usize)?; + let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; + + // If position is at start of line, return UTF-8 line start + if position.character == 0 { + return Some(*line_start_utf8); + } + + // Find the line text + let next_line_start = self.line_starts.get(position.line as usize + 1) + .copied() + .unwrap_or(self.length); + + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; + + // Convert UTF-16 character offset to UTF-8 byte offset within the line + let mut utf16_pos = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if utf16_pos >= position.character { + break; + } + utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); + } + + Some(line_start_utf8 + utf8_pos) + } + #[allow(dead_code)] pub fn position(&self, offset: u32) -> Position { let line = match self.line_starts.binary_search(&offset) { diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 0026fd0d..1a9efbbb 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -12,6 +12,7 @@ use tower_lsp_server::lsp_types::CompletionResponse; use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; +use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use tower_lsp_server::lsp_types::Documentation; use tower_lsp_server::lsp_types::InsertTextFormat; use tower_lsp_server::lsp_types::MarkupContent; @@ -92,35 +93,14 @@ impl Store { .get_text(file_id) .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - // Apply text changes - let mut new_content = current_content.to_string(); - for change in ¶ms.content_changes { - if let Some(range) = change.range { - // Get current line index for position calculations - let line_index = self - .line_indices - .get(&file_id) - .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; - - if let (Some(start_offset), Some(end_offset)) = ( - line_index.offset(range.start).map(|o| o as usize), - line_index.offset(range.end).map(|o| o as usize), - ) { - let mut updated_content = String::with_capacity( - new_content.len() - (end_offset - start_offset) + change.text.len(), - ); - - updated_content.push_str(&new_content[..start_offset]); - updated_content.push_str(&change.text); - updated_content.push_str(&new_content[end_offset..]); - - new_content = updated_content; - } - } else { - // Full document update - new_content.clone_from(&change.text); - } - } + // Get current line index for position calculations + let line_index = self + .line_indices + .get(&file_id) + .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; + + // Apply text changes using the new function + let new_content = apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; // Update TextDocument version if let Some(document) = self.documents.get_mut(&uri_str) { @@ -235,3 +215,241 @@ impl Store { } } } + +/// Apply text changes to content, handling multiple changes correctly +fn apply_text_changes( + content: &str, + changes: &[TextDocumentContentChangeEvent], + line_index: &LineIndex, +) -> Result { + if changes.is_empty() { + return Ok(content.to_string()); + } + + // Check for full document replacement first + for change in changes { + if change.range.is_none() { + return Ok(change.text.clone()); + } + } + + // Sort changes by start position in reverse order (end to start) + let mut sorted_changes = changes.to_vec(); + sorted_changes.sort_by(|a, b| { + match (a.range, b.range) { + (Some(range_a), Some(range_b)) => { + // Primary sort: by line (reverse) + let line_cmp = range_b.start.line.cmp(&range_a.start.line); + if line_cmp != std::cmp::Ordering::Equal { + line_cmp + } else { + // Secondary sort: by character (reverse) + range_b.start.character.cmp(&range_a.start.character) + } + } + _ => std::cmp::Ordering::Equal, + } + }); + + let mut result = content.to_string(); + + for change in &sorted_changes { + if let Some(range) = change.range { + // Convert UTF-16 positions to UTF-8 offsets + let start_offset = line_index.offset_utf16(range.start, &result) + .ok_or_else(|| anyhow!("Invalid start position: {:?}", range.start))?; + let end_offset = line_index.offset_utf16(range.end, &result) + .ok_or_else(|| anyhow!("Invalid end position: {:?}", range.end))?; + + if start_offset as usize > result.len() || end_offset as usize > result.len() { + return Err(anyhow!("Offset out of bounds: start={}, end={}, len={}", + start_offset, end_offset, result.len())); + } + + // Apply the change + result.replace_range(start_offset as usize..end_offset as usize, &change.text); + } + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use tower_lsp_server::lsp_types::Range; + + #[test] + fn test_apply_single_character_insertion() { + let content = "Hello world"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 6), Position::new(0, 6))), + range_length: None, + text: "beautiful ".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "Hello beautiful world"); + } + + #[test] + fn test_apply_single_character_deletion() { + let content = "Hello world"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 5), Position::new(0, 6))), + range_length: None, + text: "".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "Helloworld"); + } + + #[test] + fn test_apply_multiple_changes_in_reverse_order() { + let content = "line 1\nline 2\nline 3"; + let line_index = LineIndex::new(content); + + // Insert "new " at position (1, 0) and "another " at position (0, 0) + let changes = vec![ + TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), + range_length: None, + text: "another ".to_string(), + }, + TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(1, 0), Position::new(1, 0))), + range_length: None, + text: "new ".to_string(), + }, + ]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "another line 1\nnew line 2\nline 3"); + } + + #[test] + fn test_apply_multiline_replacement() { + let content = "line 1\nline 2\nline 3"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 0), Position::new(2, 6))), + range_length: None, + text: "completely new content".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "completely new content"); + } + + #[test] + fn test_apply_full_document_replacement() { + let content = "old content"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: "brand new content".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "brand new content"); + } + + #[test] + fn test_utf16_line_index_basic() { + let content = "hello world"; + let line_index = LineIndex::new(content); + + // ASCII characters should have 1:1 UTF-8:UTF-16 mapping + let pos = Position::new(0, 6); + let offset = line_index.offset_utf16(pos, content).unwrap(); + assert_eq!(offset, 6); + assert_eq!(&content[6..7], "w"); + } + + #[test] + fn test_utf16_line_index_with_emoji() { + let content = "hello 👋 world"; + let line_index = LineIndex::new(content); + + // 👋 is 2 UTF-16 code units but 4 UTF-8 bytes + let pos_after_emoji = Position::new(0, 8); // UTF-16 position after "hello 👋" + let offset = line_index.offset_utf16(pos_after_emoji, content).unwrap(); + + // Should point to the space before "world" + assert_eq!(offset, 10); // UTF-8 byte offset + assert_eq!(&content[10..11], " "); + } + + #[test] + fn test_utf16_line_index_multiline() { + let content = "first line\nsecond line"; + let line_index = LineIndex::new(content); + + let pos = Position::new(1, 7); // Position at 'l' in "line" on second line + let offset = line_index.offset_utf16(pos, content).unwrap(); + assert_eq!(offset, 18); // 11 (first line + \n) + 7 + assert_eq!(&content[18..19], "l"); + } + + #[test] + fn test_apply_changes_with_emoji() { + let content = "hello 👋 world"; + let line_index = LineIndex::new(content); + + // Insert text after the space following the emoji (UTF-16 position 9) + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 9), Position::new(0, 9))), + range_length: None, + text: "beautiful ".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "hello 👋 beautiful world"); + } + + #[test] + fn test_line_index_utf16_tracking() { + let content = "a👋b"; + let line_index = LineIndex::new(content); + + // Check UTF-16 line starts are tracked correctly + assert_eq!(line_index.line_starts_utf16, vec![0]); + assert_eq!(line_index.length_utf16, 4); // 'a' (1) + 👋 (2) + 'b' (1) = 4 UTF-16 units + assert_eq!(line_index.length, 6); // 'a' (1) + 👋 (4) + 'b' (1) = 6 UTF-8 bytes + } + + #[test] + fn test_edge_case_changes_at_boundaries() { + let content = "abc"; + let line_index = LineIndex::new(content); + + // Insert at beginning + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), + range_length: None, + text: "start".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "startabc"); + + // Insert at end + let line_index = LineIndex::new(content); + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 3), Position::new(0, 3))), + range_length: None, + text: "end".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "abcend"); + } +} From 588b38c8c6d3412d6421a828a4cff594f341029f Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 04:31:21 -0500 Subject: [PATCH 04/56] wip --- crates/djls-server/src/db.rs | 22 ---------------- crates/djls-server/src/lib.rs | 1 - crates/djls-server/src/session.rs | 42 ------------------------------- 3 files changed, 65 deletions(-) delete mode 100644 crates/djls-server/src/db.rs diff --git a/crates/djls-server/src/db.rs b/crates/djls-server/src/db.rs deleted file mode 100644 index f6e4c338..00000000 --- a/crates/djls-server/src/db.rs +++ /dev/null @@ -1,22 +0,0 @@ -use salsa::Database; - -#[salsa::db] -#[derive(Clone, Default)] -pub struct ServerDatabase { - storage: salsa::Storage, -} - -impl ServerDatabase { - /// Create a new database from storage - pub fn new(storage: salsa::Storage) -> Self { - Self { storage } - } -} - -impl std::fmt::Debug for ServerDatabase { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ServerDatabase").finish_non_exhaustive() - } -} - -impl Database for ServerDatabase {} diff --git a/crates/djls-server/src/lib.rs b/crates/djls-server/src/lib.rs index 57c433a2..1ebe6186 100644 --- a/crates/djls-server/src/lib.rs +++ b/crates/djls-server/src/lib.rs @@ -1,5 +1,4 @@ mod client; -mod db; mod logging; mod queue; mod server; diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index d4e8aafa..a5832cb7 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,10 +1,8 @@ use djls_conf::Settings; use djls_project::DjangoProject; -use salsa::StorageHandle; use tower_lsp_server::lsp_types::ClientCapabilities; use tower_lsp_server::lsp_types::InitializeParams; -use crate::db::ServerDatabase; use crate::workspace::Store; #[derive(Default)] @@ -15,36 +13,6 @@ pub struct Session { #[allow(dead_code)] client_capabilities: ClientCapabilities, - - /// A thread-safe Salsa database handle that can be shared between threads. - /// - /// This implements the insight from [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) - /// where we're using the `StorageHandle` to create a thread-safe handle that can be - /// shared between threads. When we need to use it, we clone the handle to get a new reference. - /// - /// This handle allows us to create database instances as needed. - /// Even though we're using a single-threaded runtime, we still need - /// this to be thread-safe because of LSP trait requirements. - /// - /// Usage: - /// ```rust,ignore - /// // Use the StorageHandle in Session - /// let db_handle = StorageHandle::new(None); - /// - /// // Clone it to pass to different threads - /// let db_handle_clone = db_handle.clone(); - /// - /// // Use it in an async context - /// async_fn(move || { - /// // Get a database from the handle - /// let storage = db_handle_clone.into_storage(); - /// let db = ServerDatabase::new(storage); - /// - /// // Use the database - /// db.some_query(args) - /// }); - /// ``` - db_handle: StorageHandle, } impl Session { @@ -67,7 +35,6 @@ impl Session { project, documents: Store::default(), settings, - db_handle: StorageHandle::new(None), } } @@ -94,13 +61,4 @@ impl Session { pub fn set_settings(&mut self, settings: Settings) { self.settings = settings; } - - /// Get a database instance directly from the session - /// - /// This creates a usable database from the handle, which can be used - /// to query and update data in the database. - pub fn db(&self) -> ServerDatabase { - let storage = self.db_handle.clone().into_storage(); - ServerDatabase::new(storage) - } } From b6bc1664ac9ccb895f7fd46dc06a6369e9c77aa6 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 05:54:56 -0500 Subject: [PATCH 05/56] wip --- Cargo.lock | 1 + crates/djls-server/src/workspace/store.rs | 79 +++++++++++- crates/djls-templates/src/ast.rs | 6 +- crates/djls-templates/src/lib.rs | 2 +- crates/djls-workspace/Cargo.toml | 2 + crates/djls-workspace/src/bridge.rs | 129 +++++++++++++++++++- crates/djls-workspace/src/db.rs | 142 ++++++++++++++++++++++ crates/djls-workspace/src/lib.rs | 2 +- 8 files changed, 354 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48cf438d..3665fe4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -509,6 +509,7 @@ dependencies = [ "anyhow", "camino", "dashmap", + "djls-templates", "salsa", "url", ] diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 1a9efbbb..6ecc5374 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -5,24 +5,28 @@ use anyhow::anyhow; use anyhow::Result; use camino::Utf8PathBuf; use djls_project::TemplateTags; -use djls_workspace::{FileId, FileKind, TextSource, Vfs}; +use djls_workspace::{FileId, FileKind, FileStore, TextSource, Vfs}; use tower_lsp_server::lsp_types::CompletionItem; use tower_lsp_server::lsp_types::CompletionItemKind; use tower_lsp_server::lsp_types::CompletionResponse; +use tower_lsp_server::lsp_types::Diagnostic; +use tower_lsp_server::lsp_types::DiagnosticSeverity; use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use tower_lsp_server::lsp_types::Documentation; use tower_lsp_server::lsp_types::InsertTextFormat; use tower_lsp_server::lsp_types::MarkupContent; use tower_lsp_server::lsp_types::MarkupKind; use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; +use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; pub struct Store { vfs: Arc, + file_store: FileStore, file_ids: HashMap, line_indices: HashMap, versions: HashMap, @@ -33,6 +37,7 @@ impl Default for Store { fn default() -> Self { Self { vfs: Arc::new(Vfs::default()), + file_store: FileStore::new(), file_ids: HashMap::new(), line_indices: HashMap::new(), versions: HashMap::new(), @@ -64,6 +69,10 @@ impl Store { // Set overlay content in VFS self.vfs.set_overlay(file_id, Arc::from(content.as_str()))?; + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + self.file_store.apply_vfs_snapshot(&snapshot); + // Create TextDocument metadata let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); self.documents.insert(uri_str.clone(), document); @@ -111,6 +120,10 @@ impl Store { self.vfs .set_overlay(file_id, Arc::from(new_content.as_str()))?; + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + self.file_store.apply_vfs_snapshot(&snapshot); + // Update cached line index and version self.line_indices .insert(file_id, LineIndex::new(&new_content)); @@ -174,9 +187,18 @@ impl Store { return None; } + // Try to get cached AST from FileStore for better context analysis + // This demonstrates using the cached AST, though we still fall back to string parsing + let file_id = document.file_id(); + if let Some(_ast) = self.file_store.get_template_ast(file_id) { + // TODO: In a future enhancement, we could use the AST to provide + // more intelligent completions based on the current node context + // For now, we continue with the existing string-based approach + } + // Get template tag context from document let vfs_snapshot = self.vfs.snapshot(); - let line_index = self.get_line_index(document.file_id())?; + let line_index = self.get_line_index(file_id)?; let context = document.get_template_tag_context(&vfs_snapshot, line_index, position)?; let mut completions: Vec = tags @@ -214,6 +236,57 @@ impl Store { Some(CompletionResponse::Array(completions)) } } + + /// Get template parsing diagnostics for a file. + /// + /// This method uses the cached template errors from Salsa to generate LSP diagnostics. + /// The errors are only re-computed when the file content changes, providing efficient + /// incremental error reporting. + pub fn get_template_diagnostics(&self, uri: &str) -> Vec { + let Some(document) = self.get_document(uri) else { + return vec![]; + }; + + // Only process template files + if document.language_id != LanguageId::HtmlDjango { + return vec![]; + } + + let file_id = document.file_id(); + let Some(_line_index) = self.get_line_index(file_id) else { + return vec![]; + }; + + // Get cached template errors from FileStore + let errors = self.file_store.get_template_errors(file_id); + + // Convert template errors to LSP diagnostics + errors + .iter() + .map(|error| { + // For now, we'll place all errors at the start of the file + // In a future enhancement, we could use error spans for precise locations + let range = Range { + start: Position { + line: 0, + character: 0, + }, + end: Position { + line: 0, + character: 0, + }, + }; + + Diagnostic { + range, + severity: Some(DiagnosticSeverity::ERROR), + source: Some("djls-templates".to_string()), + message: error.clone(), + ..Default::default() + } + }) + .collect() + } } /// Apply text changes to content, handling multiple changes correctly diff --git a/crates/djls-templates/src/ast.rs b/crates/djls-templates/src/ast.rs index f355e703..1b62d4d4 100644 --- a/crates/djls-templates/src/ast.rs +++ b/crates/djls-templates/src/ast.rs @@ -5,7 +5,7 @@ use crate::tokens::Token; use crate::tokens::TokenStream; use crate::tokens::TokenType; -#[derive(Clone, Debug, Default, Serialize)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] pub struct Ast { nodelist: Vec, line_offsets: LineOffsets, @@ -36,7 +36,7 @@ impl Ast { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct LineOffsets(pub Vec); impl LineOffsets { @@ -75,7 +75,7 @@ impl Default for LineOffsets { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub enum Node { Tag { name: String, diff --git a/crates/djls-templates/src/lib.rs b/crates/djls-templates/src/lib.rs index 48350562..7eab1f63 100644 --- a/crates/djls-templates/src/lib.rs +++ b/crates/djls-templates/src/lib.rs @@ -5,7 +5,7 @@ mod parser; mod tagspecs; mod tokens; -use ast::Ast; +pub use ast::Ast; pub use error::QuickFix; pub use error::TemplateError; use lexer::Lexer; diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 9b5ee6cc..3d079cbd 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -4,6 +4,8 @@ version = "0.0.0" edition = "2021" [dependencies] +djls-templates = { workspace = true } + anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 97f894ed..45f92fcb 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -9,7 +9,7 @@ use std::{collections::HashMap, sync::Arc}; use salsa::Setter; use super::{ - db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}, + db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}, vfs::{FileKind, VfsSnapshot}, FileId, }; @@ -99,6 +99,28 @@ impl FileStore { pub fn file_kind(&self, id: FileId) -> Option { self.files.get(&id).map(|sf| sf.kind(&self.db)) } + + /// Get the parsed template AST for a file by its [`FileId`]. + /// + /// This method leverages Salsa's incremental computation to cache parsed ASTs. + /// The AST is only re-parsed when the file's content changes in the VFS. + /// Returns `None` if the file is not tracked or is not a template file. + pub fn get_template_ast(&self, id: FileId) -> Option> { + let source_file = self.files.get(&id)?; + parse_template(&self.db, *source_file) + } + + /// Get template parsing errors for a file by its [`FileId`]. + /// + /// This method provides quick access to template errors without needing the full AST. + /// Useful for diagnostics and error reporting. Returns an empty slice for + /// non-template files or files not tracked in the store. + pub fn get_template_errors(&self, id: FileId) -> Arc<[String]> { + self.files + .get(&id) + .map(|sf| template_errors(&self.db, *sf)) + .unwrap_or_else(|| Arc::from(vec![])) + } } impl Default for FileStore { @@ -106,3 +128,108 @@ impl Default for FileStore { Self::new() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::vfs::{TextSource, Vfs}; + use camino::Utf8PathBuf; + + #[test] + fn test_filestore_template_ast_caching() { + let mut store = FileStore::new(); + let vfs = Vfs::default(); + + // Create a template file in VFS + let url = url::Url::parse("file:///test.html").unwrap(); + let path = Utf8PathBuf::from("/test.html"); + let content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); + let file_id = vfs.intern_file( + url.clone(), + path.clone(), + FileKind::Template, + TextSource::Overlay(content.clone()), + ); + vfs.set_overlay(file_id, content.clone()).unwrap(); + + // Apply VFS snapshot to FileStore + let snapshot = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot); + + // Get template AST - should parse and cache + let ast1 = store.get_template_ast(file_id); + assert!(ast1.is_some()); + + // Get again - should return cached + let ast2 = store.get_template_ast(file_id); + assert!(ast2.is_some()); + assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } + + #[test] + fn test_filestore_template_errors() { + let mut store = FileStore::new(); + let vfs = Vfs::default(); + + // Create a template with an unclosed tag + let url = url::Url::parse("file:///error.html").unwrap(); + let path = Utf8PathBuf::from("/error.html"); + let content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); // Missing closing + let file_id = vfs.intern_file( + url.clone(), + path.clone(), + FileKind::Template, + TextSource::Overlay(content.clone()), + ); + vfs.set_overlay(file_id, content).unwrap(); + + // Apply VFS snapshot + let snapshot = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot); + + // Get errors - should contain parsing errors + let errors = store.get_template_errors(file_id); + // The template has unclosed tags, so there should be errors + // We don't assert on specific error count as the parser may evolve + + // Verify errors are cached + let errors2 = store.get_template_errors(file_id); + assert!(Arc::ptr_eq(&errors, &errors2)); + } + + #[test] + fn test_filestore_invalidation_on_content_change() { + let mut store = FileStore::new(); + let vfs = Vfs::default(); + + // Create initial template + let url = url::Url::parse("file:///change.html").unwrap(); + let path = Utf8PathBuf::from("/change.html"); + let content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); + let file_id = vfs.intern_file( + url.clone(), + path.clone(), + FileKind::Template, + TextSource::Overlay(content1.clone()), + ); + vfs.set_overlay(file_id, content1).unwrap(); + + // Apply snapshot and get AST + let snapshot1 = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot1); + let ast1 = store.get_template_ast(file_id); + + // Change content + let content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); + vfs.set_overlay(file_id, content2).unwrap(); + + // Apply new snapshot + let snapshot2 = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot2); + + // Get AST again - should be different due to content change + let ast2 = store.get_template_ast(file_id); + assert!(ast1.is_some() && ast2.is_some()); + assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } +} diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 91aa1ccc..bbdf0def 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -89,3 +89,145 @@ pub struct TemplateLoaderOrder { #[returns(ref)] pub roots: Arc<[String]>, } + +/// Container for a parsed Django template AST. +/// +/// [`TemplateAst`] wraps the parsed AST from djls-templates along with any parsing errors. +/// This struct is designed to be cached by Salsa and shared across multiple consumers +/// without re-parsing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TemplateAst { + /// The parsed AST from djls-templates + pub ast: djls_templates::Ast, + /// Any errors encountered during parsing (stored as strings for simplicity) + pub errors: Vec, +} + +/// Parse a Django template file into an AST. +/// +/// This Salsa tracked function parses template files on-demand and caches the results. +/// The parse is only re-executed when the file's text content changes, enabling +/// efficient incremental template analysis. +/// +/// Returns `None` for non-template files. +#[salsa::tracked] +pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option> { + // Only parse template files + if file.kind(db) != FileKindMini::Template { + return None; + } + + let text = file.text(db); + + // Call the pure parsing function from djls-templates + match djls_templates::parse_template(&text) { + Ok((ast, errors)) => { + // Convert errors to strings + let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); + Some(Arc::new(TemplateAst { ast, errors: error_strings })) + }, + Err(err) => { + // Even on fatal errors, return an empty AST with the error + Some(Arc::new(TemplateAst { + ast: djls_templates::Ast::default(), + errors: vec![err.to_string()], + })) + } + } +} + +/// Get template parsing errors for a file. +/// +/// This Salsa tracked function extracts just the errors from the parsed template, +/// useful for diagnostics without needing the full AST. +/// +/// Returns an empty vector for non-template files. +#[salsa::tracked] +pub fn template_errors(db: &dyn salsa::Database, file: SourceFile) -> Arc<[String]> { + parse_template(db, file) + .map(|ast| Arc::from(ast.errors.clone())) + .unwrap_or_else(|| Arc::from(vec![])) +} + +#[cfg(test)] +mod tests { + use super::*; + use salsa::Setter; + + #[test] + fn test_template_parsing_caches_result() { + let db = Database::default(); + + // Create a template file + let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); + let file = SourceFile::new(&db, FileKindMini::Template, template_content.clone()); + + // First parse - should execute the parsing + let ast1 = parse_template(&db, file); + assert!(ast1.is_some()); + + // Second parse - should return cached result (same Arc) + let ast2 = parse_template(&db, file); + assert!(ast2.is_some()); + + // Verify they're the same Arc (cached) + assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } + + #[test] + fn test_template_parsing_invalidates_on_change() { + let mut db = Database::default(); + + // Create a template file + let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); + let file = SourceFile::new(&db, FileKindMini::Template, template_content1); + + // First parse + let ast1 = parse_template(&db, file); + assert!(ast1.is_some()); + + // Change the content + let template_content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); + file.set_text(&mut db).to(template_content2); + + // Parse again - should re-execute due to changed content + let ast2 = parse_template(&db, file); + assert!(ast2.is_some()); + + // Verify they're different Arcs (re-parsed) + assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } + + #[test] + fn test_non_template_files_return_none() { + let db = Database::default(); + + // Create a Python file + let python_content: Arc = Arc::from("def hello():\n print('Hello')"); + let file = SourceFile::new(&db, FileKindMini::Python, python_content); + + // Should return None for non-template files + let ast = parse_template(&db, file); + assert!(ast.is_none()); + + // Errors should be empty for non-template files + let errors = template_errors(&db, file); + assert!(errors.is_empty()); + } + + #[test] + fn test_template_errors_tracked_separately() { + let db = Database::default(); + + // Create a template with an error (unclosed tag) + let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); + let file = SourceFile::new(&db, FileKindMini::Template, template_content); + + // Get errors + let errors1 = template_errors(&db, file); + let errors2 = template_errors(&db, file); + + // Should be cached (same Arc) + assert!(Arc::ptr_eq(&errors1, &errors2)); + } +} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index aea4a539..7eb862ad 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -4,7 +4,7 @@ mod vfs; // Re-export public API pub use bridge::FileStore; -pub use db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}; +pub use db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; /// Stable, compact identifier for files across the subsystem. From fb768a86d5f126bd3198fa92eb37d65254677e0e Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 10:14:43 -0500 Subject: [PATCH 06/56] wip --- Cargo.lock | 105 +++++++ Cargo.toml | 1 + crates/djls-server/src/workspace/store.rs | 18 +- crates/djls-workspace/Cargo.toml | 3 + crates/djls-workspace/src/lib.rs | 2 + crates/djls-workspace/src/vfs.rs | 101 ++++++- crates/djls-workspace/src/watcher.rs | 330 ++++++++++++++++++++++ 7 files changed, 551 insertions(+), 9 deletions(-) create mode 100644 crates/djls-workspace/src/watcher.rs diff --git a/Cargo.lock b/Cargo.lock index 3665fe4b..73c6aae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -510,7 +510,10 @@ dependencies = [ "camino", "dashmap", "djls-templates", + "notify", "salsa", + "tempfile", + "tokio", "url", ] @@ -606,6 +609,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures" version = "0.3.31" @@ -883,6 +895,26 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +[[package]] +name = "inotify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" +dependencies = [ + "bitflags 2.9.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "insta" version = "1.43.1" @@ -938,6 +970,26 @@ dependencies = [ "serde", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1041,10 +1093,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", + "log", "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.9.2", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1493,6 +1570,15 @@ dependencies = [ "synstructure", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2074,6 +2160,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2116,6 +2212,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 8a88a0fc..6d18e3b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ clap = { version = "4.5", features = ["derive"] } config = { version ="0.15", features = ["toml"] } dashmap = "6.1" directories = "6.0" +notify = "8.2" percent-encoding = "2.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 6ecc5374..5a38d378 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use anyhow::anyhow; use anyhow::Result; @@ -26,7 +26,7 @@ use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; pub struct Store { vfs: Arc, - file_store: FileStore, + file_store: Arc>, file_ids: HashMap, line_indices: HashMap, versions: HashMap, @@ -37,7 +37,7 @@ impl Default for Store { fn default() -> Self { Self { vfs: Arc::new(Vfs::default()), - file_store: FileStore::new(), + file_store: Arc::new(Mutex::new(FileStore::new())), file_ids: HashMap::new(), line_indices: HashMap::new(), versions: HashMap::new(), @@ -71,7 +71,8 @@ impl Store { // Sync VFS snapshot to FileStore for Salsa tracking let snapshot = self.vfs.snapshot(); - self.file_store.apply_vfs_snapshot(&snapshot); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); // Create TextDocument metadata let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); @@ -122,7 +123,8 @@ impl Store { // Sync VFS snapshot to FileStore for Salsa tracking let snapshot = self.vfs.snapshot(); - self.file_store.apply_vfs_snapshot(&snapshot); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); // Update cached line index and version self.line_indices @@ -190,7 +192,8 @@ impl Store { // Try to get cached AST from FileStore for better context analysis // This demonstrates using the cached AST, though we still fall back to string parsing let file_id = document.file_id(); - if let Some(_ast) = self.file_store.get_template_ast(file_id) { + let file_store = self.file_store.lock().unwrap(); + if let Some(_ast) = file_store.get_template_ast(file_id) { // TODO: In a future enhancement, we could use the AST to provide // more intelligent completions based on the current node context // For now, we continue with the existing string-based approach @@ -258,7 +261,8 @@ impl Store { }; // Get cached template errors from FileStore - let errors = self.file_store.get_template_errors(file_id); + let file_store = self.file_store.lock().unwrap(); + let errors = file_store.get_template_errors(file_id); // Convert template errors to LSP diagnostics errors diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 3d079cbd..34fa01e8 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -9,10 +9,13 @@ djls-templates = { workspace = true } anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } +notify = { workspace = true } salsa = { workspace = true } +tokio = { workspace = true } url = { workspace = true } [dev-dependencies] +tempfile = { workspace = true } [lints] workspace = true diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 7eb862ad..83fc7d05 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,11 +1,13 @@ mod bridge; mod db; mod vfs; +mod watcher; // Re-export public API pub use bridge::FileStore; pub use db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; +pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; /// Stable, compact identifier for files across the subsystem. /// diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs index 9ebe6a24..be3ad189 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs.rs @@ -18,7 +18,7 @@ use std::{ }; use url::Url; -use super::FileId; +use super::{FileId, watcher::{VfsWatcher, WatchConfig, WatchEvent}}; /// Monotonic counter representing global VFS state. /// @@ -113,6 +113,10 @@ pub struct Vfs { files: DashMap, /// Global revision counter, incremented on content changes head: AtomicU64, + /// Optional file system watcher for external change detection + watcher: std::sync::Mutex>, + /// Map from filesystem path to FileId for watcher events + by_path: DashMap, } impl Vfs { @@ -134,11 +138,12 @@ impl Vfs { let id = FileId(self.next_file_id.fetch_add(1, Ordering::SeqCst)); let meta = FileMeta { uri: uri.clone(), - path, + path: path.clone(), kind, }; let hash = content_hash(&text); self.by_uri.insert(uri, id); + self.by_path.insert(path, id); self.files.insert(id, FileRecord { meta, text, hash }); id } @@ -188,6 +193,96 @@ impl Vfs { .collect(), } } + + /// Enable file system watching with the given configuration. + /// + /// This starts monitoring the specified root directories for external changes. + /// Returns an error if file watching is disabled in the config or fails to start. + pub fn enable_file_watching(&self, config: WatchConfig) -> Result<()> { + let watcher = VfsWatcher::new(config)?; + *self.watcher.lock().unwrap() = Some(watcher); + Ok(()) + } + + /// Process pending file system events from the watcher. + /// + /// This should be called periodically to sync external file changes into the VFS. + /// Returns the number of files that were updated. + pub fn process_file_events(&self) -> usize { + // Get events from the watcher + let events = { + let guard = self.watcher.lock().unwrap(); + if let Some(watcher) = guard.as_ref() { + watcher.try_recv_events() + } else { + return 0; + } + }; + + let mut updated_count = 0; + + for event in events { + match event { + WatchEvent::Modified(path) | WatchEvent::Created(path) => { + if let Err(e) = self.load_from_disk(&path) { + eprintln!("Failed to load file from disk: {}: {}", path, e); + } else { + updated_count += 1; + } + } + WatchEvent::Deleted(path) => { + // For now, we don't remove deleted files from VFS + // This maintains stable FileIds for consumers + eprintln!("File deleted (keeping in VFS): {}", path); + } + WatchEvent::Renamed { from, to } => { + // Handle rename by updating the path mapping + if let Some(file_id) = self.by_path.remove(&from).map(|(_, id)| id) { + self.by_path.insert(to.clone(), file_id); + if let Err(e) = self.load_from_disk(&to) { + eprintln!("Failed to load renamed file: {}: {}", to, e); + } else { + updated_count += 1; + } + } + } + } + } + updated_count + } + + /// Load a file's content from disk and update the VFS. + /// + /// This method reads the file from the filesystem and updates the VFS entry + /// if the content has changed. It's used by the file watcher to sync external changes. + fn load_from_disk(&self, path: &Utf8PathBuf) -> Result<()> { + use std::fs; + + // Check if we have this file tracked + if let Some(file_id) = self.by_path.get(path).map(|entry| *entry.value()) { + // Read content from disk + let content = fs::read_to_string(path.as_std_path()) + .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; + + let new_text = TextSource::Disk(Arc::from(content.as_str())); + let new_hash = content_hash(&new_text); + + // Update the file if content changed + if let Some(mut record) = self.files.get_mut(&file_id) { + if record.hash != new_hash { + record.text = new_text; + record.hash = new_hash; + self.head.fetch_add(1, Ordering::SeqCst); + } + } + } + Ok(()) + } + + /// Check if file watching is currently enabled. + pub fn is_file_watching_enabled(&self) -> bool { + self.watcher.lock().unwrap().is_some() + } } impl Default for Vfs { @@ -197,6 +292,8 @@ impl Default for Vfs { by_uri: DashMap::new(), files: DashMap::new(), head: AtomicU64::new(0), + watcher: std::sync::Mutex::new(None), + by_path: DashMap::new(), } } } diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/watcher.rs new file mode 100644 index 00000000..cd6d9583 --- /dev/null +++ b/crates/djls-workspace/src/watcher.rs @@ -0,0 +1,330 @@ +//! File system watching for VFS synchronization. +//! +//! This module provides file system watching capabilities to detect external changes +//! and synchronize them with the VFS. It uses cross-platform file watching with +//! debouncing to handle rapid changes efficiently. + +use anyhow::{anyhow, Result}; +use camino::Utf8PathBuf; +use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; +use std::{ + collections::HashMap, + sync::mpsc, + thread, + time::{Duration, Instant}, +}; + +/// Event types that can occur in the file system. +/// +/// [`WatchEvent`] represents the different types of file system changes that +/// the watcher can detect and process. +#[derive(Clone, Debug, PartialEq)] +pub enum WatchEvent { + /// A file was modified (content changed) + Modified(Utf8PathBuf), + /// A new file was created + Created(Utf8PathBuf), + /// A file was deleted + Deleted(Utf8PathBuf), + /// A file was renamed from one path to another + Renamed { + from: Utf8PathBuf, + to: Utf8PathBuf, + }, +} + +/// Configuration for the file watcher. +/// +/// [`WatchConfig`] controls how the file watcher operates, including what +/// directories to watch and how to filter events. +#[derive(Clone, Debug)] +pub struct WatchConfig { + /// Whether file watching is enabled + pub enabled: bool, + /// Root directories to watch recursively + pub roots: Vec, + /// Debounce time in milliseconds (collect events for this duration before processing) + pub debounce_ms: u64, + /// File patterns to include (e.g., ["*.py", "*.html"]) + pub include_patterns: Vec, + /// File patterns to exclude (e.g., ["__pycache__", ".git", "*.pyc"]) + pub exclude_patterns: Vec, +} + +impl Default for WatchConfig { + fn default() -> Self { + Self { + enabled: true, + roots: Vec::new(), + debounce_ms: 250, + include_patterns: vec!["*.py".to_string(), "*.html".to_string()], + exclude_patterns: vec![ + "__pycache__".to_string(), + ".git".to_string(), + ".pyc".to_string(), + "node_modules".to_string(), + ".venv".to_string(), + "venv".to_string(), + ], + } + } +} + +/// File system watcher for VFS synchronization. +/// +/// [`VfsWatcher`] monitors the file system for changes and provides a channel +/// for consuming batched events. It handles debouncing and filtering internally. +pub struct VfsWatcher { + /// The underlying file system watcher + _watcher: RecommendedWatcher, + /// Receiver for processed watch events + rx: mpsc::Receiver>, + /// Configuration for the watcher + config: WatchConfig, + /// Handle to the background processing thread + _handle: thread::JoinHandle<()>, +} + +impl VfsWatcher { + /// Create a new file watcher with the given configuration. + /// + /// This starts watching the specified root directories and begins processing + /// events in a background thread. + pub fn new(config: WatchConfig) -> Result { + if !config.enabled { + return Err(anyhow!("File watching is disabled")); + } + + let (event_tx, event_rx) = mpsc::channel(); + let (watch_tx, watch_rx) = mpsc::channel(); + + // Create the file system watcher + let mut watcher = RecommendedWatcher::new( + move |res: notify::Result| { + if let Ok(event) = res { + let _ = event_tx.send(event); + } + }, + Config::default(), + )?; + + // Watch all root directories + for root in &config.roots { + let std_path = root.as_std_path(); + if std_path.exists() { + watcher.watch(std_path, RecursiveMode::Recursive)?; + } + } + + // Spawn background thread for event processing + let config_clone = config.clone(); + let handle = thread::spawn(move || { + Self::process_events(event_rx, watch_tx, config_clone); + }); + + Ok(Self { + _watcher: watcher, + rx: watch_rx, + config, + _handle: handle, + }) + } + + /// Get the next batch of processed watch events. + /// + /// This is a non-blocking operation that returns immediately. If no events + /// are available, it returns an empty vector. + pub fn try_recv_events(&self) -> Vec { + match self.rx.try_recv() { + Ok(events) => events, + Err(_) => Vec::new(), + } + } + + + + /// Background thread function for processing raw file system events. + /// + /// This function handles debouncing, filtering, and batching of events before + /// sending them to the main thread for VFS synchronization. + fn process_events( + event_rx: mpsc::Receiver, + watch_tx: mpsc::Sender>, + config: WatchConfig, + ) { + let mut pending_events: HashMap = HashMap::new(); + let mut last_batch_time = Instant::now(); + let debounce_duration = Duration::from_millis(config.debounce_ms); + + loop { + // Try to receive events with a timeout for batching + match event_rx.recv_timeout(Duration::from_millis(50)) { + Ok(event) => { + // Process the raw notify event into our WatchEvent format + if let Some(watch_events) = Self::convert_notify_event(event, &config) { + for watch_event in watch_events { + if let Some(path) = Self::get_event_path(&watch_event) { + // Only keep the latest event for each path + pending_events.insert(path.clone(), watch_event); + } + } + } + } + Err(mpsc::RecvTimeoutError::Timeout) => { + // Timeout - check if we should flush pending events + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + // Channel disconnected, exit the thread + break; + } + } + + // Check if we should flush pending events + if !pending_events.is_empty() + && last_batch_time.elapsed() >= debounce_duration + { + let events: Vec = pending_events.values().cloned().collect(); + if let Err(_) = watch_tx.send(events) { + // Main thread disconnected, exit + break; + } + pending_events.clear(); + last_batch_time = Instant::now(); + } + } + } + + /// Convert a notify Event into our WatchEvent format. + fn convert_notify_event(event: Event, config: &WatchConfig) -> Option> { + let mut watch_events = Vec::new(); + + for path in event.paths { + if let Ok(utf8_path) = Utf8PathBuf::try_from(path) { + if Self::should_include_path_static(&utf8_path, config) { + match event.kind { + EventKind::Create(_) => watch_events.push(WatchEvent::Created(utf8_path)), + EventKind::Modify(_) => watch_events.push(WatchEvent::Modified(utf8_path)), + EventKind::Remove(_) => watch_events.push(WatchEvent::Deleted(utf8_path)), + _ => {} // Ignore other event types for now + } + } + } + } + + if watch_events.is_empty() { + None + } else { + Some(watch_events) + } + } + + /// Static version of should_include_path for use in convert_notify_event. + fn should_include_path_static(path: &Utf8PathBuf, config: &WatchConfig) -> bool { + let path_str = path.as_str(); + + // Check exclude patterns first + for pattern in &config.exclude_patterns { + if path_str.contains(pattern) { + return false; + } + } + + // If no include patterns, include everything (that's not excluded) + if config.include_patterns.is_empty() { + return true; + } + + // Check include patterns + for pattern in &config.include_patterns { + if pattern.starts_with("*.") { + let extension = &pattern[2..]; + if path_str.ends_with(extension) { + return true; + } + } else if path_str.contains(pattern) { + return true; + } + } + + false + } + + /// Extract the path from a WatchEvent. + fn get_event_path(event: &WatchEvent) -> Option<&Utf8PathBuf> { + match event { + WatchEvent::Modified(path) => Some(path), + WatchEvent::Created(path) => Some(path), + WatchEvent::Deleted(path) => Some(path), + WatchEvent::Renamed { to, .. } => Some(to), + } + } +} + +impl Drop for VfsWatcher { + fn drop(&mut self) { + // The background thread will exit when the event channel is dropped + } +} + +#[cfg(test)] +mod tests { + use super::*; + + + #[test] + fn test_watch_config_default() { + let config = WatchConfig::default(); + assert!(config.enabled); + assert_eq!(config.debounce_ms, 250); + assert!(config.include_patterns.contains(&"*.py".to_string())); + assert!(config.exclude_patterns.contains(&".git".to_string())); + } + + #[test] + fn test_should_include_path() { + let config = WatchConfig::default(); + + // Should include Python files + assert!(VfsWatcher::should_include_path_static( + &Utf8PathBuf::from("test.py"), + &config + )); + + // Should include HTML files + assert!(VfsWatcher::should_include_path_static( + &Utf8PathBuf::from("template.html"), + &config + )); + + // Should exclude .git files + assert!(!VfsWatcher::should_include_path_static( + &Utf8PathBuf::from(".git/config"), + &config + )); + + // Should exclude __pycache__ files + assert!(!VfsWatcher::should_include_path_static( + &Utf8PathBuf::from("__pycache__/test.pyc"), + &config + )); + } + + #[test] + fn test_watch_event_types() { + let path1 = Utf8PathBuf::from("test.py"); + let path2 = Utf8PathBuf::from("new.py"); + + let modified = WatchEvent::Modified(path1.clone()); + let created = WatchEvent::Created(path1.clone()); + let deleted = WatchEvent::Deleted(path1.clone()); + let renamed = WatchEvent::Renamed { + from: path1, + to: path2, + }; + + // Test that events can be created and compared + assert_ne!(modified, created); + assert_ne!(created, deleted); + assert_ne!(deleted, renamed); + } +} \ No newline at end of file From 20163b50f8eb0cb1b896e8496884eb756acd27db Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 10:34:57 -0500 Subject: [PATCH 07/56] wip --- crates/djls-workspace/src/bridge.rs | 35 +++++++++-------- crates/djls-workspace/src/db.rs | 48 ++++++++++++----------- crates/djls-workspace/src/lib.rs | 6 ++- crates/djls-workspace/src/vfs.rs | 41 +++++++++++--------- crates/djls-workspace/src/watcher.rs | 58 ++++++++++++---------------- 5 files changed, 94 insertions(+), 94 deletions(-) diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 45f92fcb..78669adf 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -9,7 +9,10 @@ use std::{collections::HashMap, sync::Arc}; use salsa::Setter; use super::{ - db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}, + db::{ + parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, + TemplateLoaderOrder, + }, vfs::{FileKind, VfsSnapshot}, FileId, }; @@ -31,6 +34,7 @@ pub struct FileStore { impl FileStore { /// Construct an empty store and DB. + #[must_use] pub fn new() -> Self { Self { db: Database::default(), @@ -118,8 +122,7 @@ impl FileStore { pub fn get_template_errors(&self, id: FileId) -> Arc<[String]> { self.files .get(&id) - .map(|sf| template_errors(&self.db, *sf)) - .unwrap_or_else(|| Arc::from(vec![])) + .map_or_else(|| Arc::from(vec![]), |sf| template_errors(&self.db, *sf)) } } @@ -139,7 +142,7 @@ mod tests { fn test_filestore_template_ast_caching() { let mut store = FileStore::new(); let vfs = Vfs::default(); - + // Create a template file in VFS let url = url::Url::parse("file:///test.html").unwrap(); let path = Utf8PathBuf::from("/test.html"); @@ -151,15 +154,15 @@ mod tests { TextSource::Overlay(content.clone()), ); vfs.set_overlay(file_id, content.clone()).unwrap(); - + // Apply VFS snapshot to FileStore let snapshot = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot); - + // Get template AST - should parse and cache let ast1 = store.get_template_ast(file_id); assert!(ast1.is_some()); - + // Get again - should return cached let ast2 = store.get_template_ast(file_id); assert!(ast2.is_some()); @@ -170,7 +173,7 @@ mod tests { fn test_filestore_template_errors() { let mut store = FileStore::new(); let vfs = Vfs::default(); - + // Create a template with an unclosed tag let url = url::Url::parse("file:///error.html").unwrap(); let path = Utf8PathBuf::from("/error.html"); @@ -182,16 +185,16 @@ mod tests { TextSource::Overlay(content.clone()), ); vfs.set_overlay(file_id, content).unwrap(); - + // Apply VFS snapshot let snapshot = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot); - + // Get errors - should contain parsing errors let errors = store.get_template_errors(file_id); // The template has unclosed tags, so there should be errors // We don't assert on specific error count as the parser may evolve - + // Verify errors are cached let errors2 = store.get_template_errors(file_id); assert!(Arc::ptr_eq(&errors, &errors2)); @@ -201,7 +204,7 @@ mod tests { fn test_filestore_invalidation_on_content_change() { let mut store = FileStore::new(); let vfs = Vfs::default(); - + // Create initial template let url = url::Url::parse("file:///change.html").unwrap(); let path = Utf8PathBuf::from("/change.html"); @@ -213,20 +216,20 @@ mod tests { TextSource::Overlay(content1.clone()), ); vfs.set_overlay(file_id, content1).unwrap(); - + // Apply snapshot and get AST let snapshot1 = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot1); let ast1 = store.get_template_ast(file_id); - + // Change content let content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); vfs.set_overlay(file_id, content2).unwrap(); - + // Apply new snapshot let snapshot2 = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot2); - + // Get AST again - should be different due to content change let ast2 = store.get_template_ast(file_id); assert!(ast1.is_some() && ast2.is_some()); diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index bbdf0def..a38757dc 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -118,14 +118,17 @@ pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option { // Convert errors to strings let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); - Some(Arc::new(TemplateAst { ast, errors: error_strings })) - }, + Some(Arc::new(TemplateAst { + ast, + errors: error_strings, + })) + } Err(err) => { // Even on fatal errors, return an empty AST with the error Some(Arc::new(TemplateAst { @@ -144,9 +147,7 @@ pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option Arc<[String]> { - parse_template(db, file) - .map(|ast| Arc::from(ast.errors.clone())) - .unwrap_or_else(|| Arc::from(vec![])) + parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) } #[cfg(test)] @@ -157,19 +158,19 @@ mod tests { #[test] fn test_template_parsing_caches_result() { let db = Database::default(); - + // Create a template file let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); let file = SourceFile::new(&db, FileKindMini::Template, template_content.clone()); - + // First parse - should execute the parsing let ast1 = parse_template(&db, file); assert!(ast1.is_some()); - + // Second parse - should return cached result (same Arc) let ast2 = parse_template(&db, file); assert!(ast2.is_some()); - + // Verify they're the same Arc (cached) assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); } @@ -177,23 +178,24 @@ mod tests { #[test] fn test_template_parsing_invalidates_on_change() { let mut db = Database::default(); - + // Create a template file let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); let file = SourceFile::new(&db, FileKindMini::Template, template_content1); - + // First parse let ast1 = parse_template(&db, file); assert!(ast1.is_some()); - + // Change the content - let template_content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); + let template_content2: Arc = + Arc::from("{% for item in items %}{{ item }}{% endfor %}"); file.set_text(&mut db).to(template_content2); - + // Parse again - should re-execute due to changed content let ast2 = parse_template(&db, file); assert!(ast2.is_some()); - + // Verify they're different Arcs (re-parsed) assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); } @@ -201,15 +203,15 @@ mod tests { #[test] fn test_non_template_files_return_none() { let db = Database::default(); - + // Create a Python file let python_content: Arc = Arc::from("def hello():\n print('Hello')"); let file = SourceFile::new(&db, FileKindMini::Python, python_content); - + // Should return None for non-template files let ast = parse_template(&db, file); assert!(ast.is_none()); - + // Errors should be empty for non-template files let errors = template_errors(&db, file); assert!(errors.is_empty()); @@ -218,15 +220,15 @@ mod tests { #[test] fn test_template_errors_tracked_separately() { let db = Database::default(); - + // Create a template with an error (unclosed tag) let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); let file = SourceFile::new(&db, FileKindMini::Template, template_content); - + // Get errors let errors1 = template_errors(&db, file); let errors2 = template_errors(&db, file); - + // Should be cached (same Arc) assert!(Arc::ptr_eq(&errors1, &errors2)); } diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 83fc7d05..30c69a79 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -3,9 +3,11 @@ mod db; mod vfs; mod watcher; -// Re-export public API pub use bridge::FileStore; -pub use db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}; +pub use db::{ + parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, + TemplateLoaderOrder, +}; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs index be3ad189..08eb1827 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs.rs @@ -8,6 +8,7 @@ use anyhow::{anyhow, Result}; use camino::Utf8PathBuf; use dashmap::DashMap; use std::collections::hash_map::DefaultHasher; +use std::fs; use std::hash::{Hash, Hasher}; use std::{ collections::HashMap, @@ -18,7 +19,10 @@ use std::{ }; use url::Url; -use super::{FileId, watcher::{VfsWatcher, WatchConfig, WatchEvent}}; +use super::{ + watcher::{VfsWatcher, WatchConfig, WatchEvent}, + FileId, +}; /// Monotonic counter representing global VFS state. /// @@ -115,7 +119,7 @@ pub struct Vfs { head: AtomicU64, /// Optional file system watcher for external change detection watcher: std::sync::Mutex>, - /// Map from filesystem path to FileId for watcher events + /// Map from filesystem path to [`FileId`] for watcher events by_path: DashMap, } @@ -155,10 +159,6 @@ impl Vfs { /// (detected via hash comparison). /// /// Returns a tuple of (new global revision, whether content changed). - /// - /// # Errors - /// - /// Returns an error if the provided `FileId` does not exist in the VFS. pub fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { let mut rec = self .files @@ -200,7 +200,10 @@ impl Vfs { /// Returns an error if file watching is disabled in the config or fails to start. pub fn enable_file_watching(&self, config: WatchConfig) -> Result<()> { let watcher = VfsWatcher::new(config)?; - *self.watcher.lock().unwrap() = Some(watcher); + *self + .watcher + .lock() + .map_err(|e| anyhow!("Failed to lock watcher mutex: {}", e))? = Some(watcher); Ok(()) } @@ -211,36 +214,38 @@ impl Vfs { pub fn process_file_events(&self) -> usize { // Get events from the watcher let events = { - let guard = self.watcher.lock().unwrap(); + let Ok(guard) = self.watcher.lock() else { + return 0; // Return 0 if mutex is poisoned + }; if let Some(watcher) = guard.as_ref() { watcher.try_recv_events() } else { return 0; } }; - + let mut updated_count = 0; - + for event in events { match event { WatchEvent::Modified(path) | WatchEvent::Created(path) => { if let Err(e) = self.load_from_disk(&path) { - eprintln!("Failed to load file from disk: {}: {}", path, e); + eprintln!("Failed to load file from disk: {path}: {e}"); } else { updated_count += 1; } } WatchEvent::Deleted(path) => { // For now, we don't remove deleted files from VFS - // This maintains stable FileIds for consumers - eprintln!("File deleted (keeping in VFS): {}", path); + // This maintains stable `FileId`s for consumers + eprintln!("File deleted (keeping in VFS): {path}"); } WatchEvent::Renamed { from, to } => { // Handle rename by updating the path mapping if let Some(file_id) = self.by_path.remove(&from).map(|(_, id)| id) { self.by_path.insert(to.clone(), file_id); if let Err(e) = self.load_from_disk(&to) { - eprintln!("Failed to load renamed file: {}: {}", to, e); + eprintln!("Failed to load renamed file: {to}: {e}"); } else { updated_count += 1; } @@ -256,17 +261,15 @@ impl Vfs { /// This method reads the file from the filesystem and updates the VFS entry /// if the content has changed. It's used by the file watcher to sync external changes. fn load_from_disk(&self, path: &Utf8PathBuf) -> Result<()> { - use std::fs; - // Check if we have this file tracked if let Some(file_id) = self.by_path.get(path).map(|entry| *entry.value()) { // Read content from disk let content = fs::read_to_string(path.as_std_path()) .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; - + let new_text = TextSource::Disk(Arc::from(content.as_str())); let new_hash = content_hash(&new_text); - + // Update the file if content changed if let Some(mut record) = self.files.get_mut(&file_id) { if record.hash != new_hash { @@ -281,7 +284,7 @@ impl Vfs { /// Check if file watching is currently enabled. pub fn is_file_watching_enabled(&self) -> bool { - self.watcher.lock().unwrap().is_some() + self.watcher.lock().map(|g| g.is_some()).unwrap_or(false) // Return false if mutex is poisoned } } diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/watcher.rs index cd6d9583..57c798c4 100644 --- a/crates/djls-workspace/src/watcher.rs +++ b/crates/djls-workspace/src/watcher.rs @@ -27,10 +27,7 @@ pub enum WatchEvent { /// A file was deleted Deleted(Utf8PathBuf), /// A file was renamed from one path to another - Renamed { - from: Utf8PathBuf, - to: Utf8PathBuf, - }, + Renamed { from: Utf8PathBuf, to: Utf8PathBuf }, } /// Configuration for the file watcher. @@ -51,6 +48,7 @@ pub struct WatchConfig { pub exclude_patterns: Vec, } +// TODO: Allow for user config instead of hardcoding defaults impl Default for WatchConfig { fn default() -> Self { Self { @@ -119,7 +117,7 @@ impl VfsWatcher { // Spawn background thread for event processing let config_clone = config.clone(); let handle = thread::spawn(move || { - Self::process_events(event_rx, watch_tx, config_clone); + Self::process_events(&event_rx, &watch_tx, &config_clone); }); Ok(Self { @@ -134,23 +132,19 @@ impl VfsWatcher { /// /// This is a non-blocking operation that returns immediately. If no events /// are available, it returns an empty vector. + #[must_use] pub fn try_recv_events(&self) -> Vec { - match self.rx.try_recv() { - Ok(events) => events, - Err(_) => Vec::new(), - } + self.rx.try_recv().unwrap_or_default() } - - /// Background thread function for processing raw file system events. /// /// This function handles debouncing, filtering, and batching of events before /// sending them to the main thread for VFS synchronization. fn process_events( - event_rx: mpsc::Receiver, - watch_tx: mpsc::Sender>, - config: WatchConfig, + event_rx: &mpsc::Receiver, + watch_tx: &mpsc::Sender>, + config: &WatchConfig, ) { let mut pending_events: HashMap = HashMap::new(); let mut last_batch_time = Instant::now(); @@ -161,12 +155,11 @@ impl VfsWatcher { match event_rx.recv_timeout(Duration::from_millis(50)) { Ok(event) => { // Process the raw notify event into our WatchEvent format - if let Some(watch_events) = Self::convert_notify_event(event, &config) { + if let Some(watch_events) = Self::convert_notify_event(event, config) { for watch_event in watch_events { - if let Some(path) = Self::get_event_path(&watch_event) { - // Only keep the latest event for each path - pending_events.insert(path.clone(), watch_event); - } + let path = Self::get_event_path(&watch_event); + // Only keep the latest event for each path + pending_events.insert(path.clone(), watch_event); } } } @@ -180,11 +173,9 @@ impl VfsWatcher { } // Check if we should flush pending events - if !pending_events.is_empty() - && last_batch_time.elapsed() >= debounce_duration - { + if !pending_events.is_empty() && last_batch_time.elapsed() >= debounce_duration { let events: Vec = pending_events.values().cloned().collect(); - if let Err(_) = watch_tx.send(events) { + if watch_tx.send(events).is_err() { // Main thread disconnected, exit break; } @@ -194,7 +185,7 @@ impl VfsWatcher { } } - /// Convert a notify Event into our WatchEvent format. + /// Convert a [`notify::Event`] into our [`WatchEvent`] format. fn convert_notify_event(event: Event, config: &WatchConfig) -> Option> { let mut watch_events = Vec::new(); @@ -236,8 +227,7 @@ impl VfsWatcher { // Check include patterns for pattern in &config.include_patterns { - if pattern.starts_with("*.") { - let extension = &pattern[2..]; + if let Some(extension) = pattern.strip_prefix("*.") { if path_str.ends_with(extension) { return true; } @@ -249,13 +239,13 @@ impl VfsWatcher { false } - /// Extract the path from a WatchEvent. - fn get_event_path(event: &WatchEvent) -> Option<&Utf8PathBuf> { + /// Extract the path from a [`WatchEvent`]. + fn get_event_path(event: &WatchEvent) -> &Utf8PathBuf { match event { - WatchEvent::Modified(path) => Some(path), - WatchEvent::Created(path) => Some(path), - WatchEvent::Deleted(path) => Some(path), - WatchEvent::Renamed { to, .. } => Some(to), + WatchEvent::Modified(path) | WatchEvent::Created(path) | WatchEvent::Deleted(path) => { + path + } + WatchEvent::Renamed { to, .. } => to, } } } @@ -270,7 +260,6 @@ impl Drop for VfsWatcher { mod tests { use super::*; - #[test] fn test_watch_config_default() { let config = WatchConfig::default(); @@ -327,4 +316,5 @@ mod tests { assert_ne!(created, deleted); assert_ne!(deleted, renamed); } -} \ No newline at end of file +} + From 541200cbb13076e0d72eaf41b67a0802848cd5c3 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 10:52:37 -0500 Subject: [PATCH 08/56] wip --- crates/djls-server/src/workspace/document.rs | 53 ++++++++++-------- crates/djls-server/src/workspace/store.rs | 57 +++++++++++--------- crates/djls-workspace/src/bridge.rs | 15 ++---- crates/djls-workspace/src/db.rs | 29 +++------- crates/djls-workspace/src/lib.rs | 3 +- crates/djls-workspace/src/watcher.rs | 1 - 6 files changed, 77 insertions(+), 81 deletions(-) diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs index 5a77a65d..75812b87 100644 --- a/crates/djls-server/src/workspace/document.rs +++ b/crates/djls-server/src/workspace/document.rs @@ -1,8 +1,7 @@ +use djls_workspace::{FileId, VfsSnapshot}; use std::sync::Arc; use tower_lsp_server::lsp_types::{Position, Range}; -use djls_workspace::{FileId, VfsSnapshot}; -/// Document metadata container - no longer a Salsa input, just plain data #[derive(Clone, Debug)] pub struct TextDocument { pub uri: String, @@ -20,39 +19,50 @@ impl TextDocument { file_id, } } - + pub fn file_id(&self) -> FileId { self.file_id } - + pub fn get_content(&self, vfs: &VfsSnapshot) -> Option> { vfs.get_text(self.file_id) } - + pub fn get_line(&self, vfs: &VfsSnapshot, line_index: &LineIndex, line: u32) -> Option { let content = self.get_content(vfs)?; - + let line_start = *line_index.line_starts.get(line as usize)?; - let line_end = line_index.line_starts + let line_end = line_index + .line_starts .get(line as usize + 1) .copied() .unwrap_or(line_index.length); - + Some(content[line_start as usize..line_end as usize].to_string()) } - - pub fn get_text_range(&self, vfs: &VfsSnapshot, line_index: &LineIndex, range: Range) -> Option { + + pub fn get_text_range( + &self, + vfs: &VfsSnapshot, + line_index: &LineIndex, + range: Range, + ) -> Option { let content = self.get_content(vfs)?; - + let start_offset = line_index.offset(range.start)? as usize; let end_offset = line_index.offset(range.end)? as usize; - + Some(content[start_offset..end_offset].to_string()) } - - pub fn get_template_tag_context(&self, vfs: &VfsSnapshot, line_index: &LineIndex, position: Position) -> Option { + + pub fn get_template_tag_context( + &self, + vfs: &VfsSnapshot, + line_index: &LineIndex, + position: Position, + ) -> Option { let content = self.get_content(vfs)?; - + let start = line_index.line_starts.get(position.line as usize)?; let end = line_index .line_starts @@ -136,16 +146,18 @@ impl LineIndex { } // Find the line text - let next_line_start = self.line_starts.get(position.line as usize + 1) + let next_line_start = self + .line_starts + .get(position.line as usize + 1) .copied() .unwrap_or(self.length); - + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; - + // Convert UTF-16 character offset to UTF-8 byte offset within the line let mut utf16_pos = 0; let mut utf8_pos = 0; - + for c in line_text.chars() { if utf16_pos >= position.character { break; @@ -153,7 +165,7 @@ impl LineIndex { utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); } - + Some(line_start_utf8 + utf8_pos) } @@ -217,4 +229,3 @@ pub struct TemplateTagContext { pub closing_brace: ClosingBrace, pub needs_leading_space: bool, } - diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 5a38d378..90962019 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -110,7 +110,8 @@ impl Store { .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; // Apply text changes using the new function - let new_content = apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; + let new_content = + apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; // Update TextDocument version if let Some(document) = self.documents.get_mut(&uri_str) { @@ -317,11 +318,11 @@ fn apply_text_changes( (Some(range_a), Some(range_b)) => { // Primary sort: by line (reverse) let line_cmp = range_b.start.line.cmp(&range_a.start.line); - if line_cmp != std::cmp::Ordering::Equal { - line_cmp - } else { + if line_cmp == std::cmp::Ordering::Equal { // Secondary sort: by character (reverse) range_b.start.character.cmp(&range_a.start.character) + } else { + line_cmp } } _ => std::cmp::Ordering::Equal, @@ -333,14 +334,20 @@ fn apply_text_changes( for change in &sorted_changes { if let Some(range) = change.range { // Convert UTF-16 positions to UTF-8 offsets - let start_offset = line_index.offset_utf16(range.start, &result) + let start_offset = line_index + .offset_utf16(range.start, &result) .ok_or_else(|| anyhow!("Invalid start position: {:?}", range.start))?; - let end_offset = line_index.offset_utf16(range.end, &result) + let end_offset = line_index + .offset_utf16(range.end, &result) .ok_or_else(|| anyhow!("Invalid end position: {:?}", range.end))?; if start_offset as usize > result.len() || end_offset as usize > result.len() { - return Err(anyhow!("Offset out of bounds: start={}, end={}, len={}", - start_offset, end_offset, result.len())); + return Err(anyhow!( + "Offset out of bounds: start={}, end={}, len={}", + start_offset, + end_offset, + result.len() + )); } // Apply the change @@ -360,7 +367,7 @@ mod tests { fn test_apply_single_character_insertion() { let content = "Hello world"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 6), Position::new(0, 6))), range_length: None, @@ -375,11 +382,11 @@ mod tests { fn test_apply_single_character_deletion() { let content = "Hello world"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 5), Position::new(0, 6))), range_length: None, - text: "".to_string(), + text: String::new(), }]; let result = apply_text_changes(content, &changes, &line_index).unwrap(); @@ -390,7 +397,7 @@ mod tests { fn test_apply_multiple_changes_in_reverse_order() { let content = "line 1\nline 2\nline 3"; let line_index = LineIndex::new(content); - + // Insert "new " at position (1, 0) and "another " at position (0, 0) let changes = vec![ TextDocumentContentChangeEvent { @@ -413,7 +420,7 @@ mod tests { fn test_apply_multiline_replacement() { let content = "line 1\nline 2\nline 3"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 0), Position::new(2, 6))), range_length: None, @@ -428,7 +435,7 @@ mod tests { fn test_apply_full_document_replacement() { let content = "old content"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: None, range_length: None, @@ -443,7 +450,7 @@ mod tests { fn test_utf16_line_index_basic() { let content = "hello world"; let line_index = LineIndex::new(content); - + // ASCII characters should have 1:1 UTF-8:UTF-16 mapping let pos = Position::new(0, 6); let offset = line_index.offset_utf16(pos, content).unwrap(); @@ -455,11 +462,11 @@ mod tests { fn test_utf16_line_index_with_emoji() { let content = "hello 👋 world"; let line_index = LineIndex::new(content); - + // 👋 is 2 UTF-16 code units but 4 UTF-8 bytes let pos_after_emoji = Position::new(0, 8); // UTF-16 position after "hello 👋" let offset = line_index.offset_utf16(pos_after_emoji, content).unwrap(); - + // Should point to the space before "world" assert_eq!(offset, 10); // UTF-8 byte offset assert_eq!(&content[10..11], " "); @@ -469,7 +476,7 @@ mod tests { fn test_utf16_line_index_multiline() { let content = "first line\nsecond line"; let line_index = LineIndex::new(content); - + let pos = Position::new(1, 7); // Position at 'l' in "line" on second line let offset = line_index.offset_utf16(pos, content).unwrap(); assert_eq!(offset, 18); // 11 (first line + \n) + 7 @@ -480,7 +487,7 @@ mod tests { fn test_apply_changes_with_emoji() { let content = "hello 👋 world"; let line_index = LineIndex::new(content); - + // Insert text after the space following the emoji (UTF-16 position 9) let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 9), Position::new(0, 9))), @@ -496,28 +503,28 @@ mod tests { fn test_line_index_utf16_tracking() { let content = "a👋b"; let line_index = LineIndex::new(content); - + // Check UTF-16 line starts are tracked correctly assert_eq!(line_index.line_starts_utf16, vec![0]); assert_eq!(line_index.length_utf16, 4); // 'a' (1) + 👋 (2) + 'b' (1) = 4 UTF-16 units assert_eq!(line_index.length, 6); // 'a' (1) + 👋 (4) + 'b' (1) = 6 UTF-8 bytes } - #[test] + #[test] fn test_edge_case_changes_at_boundaries() { let content = "abc"; let line_index = LineIndex::new(content); - + // Insert at beginning let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), range_length: None, text: "start".to_string(), }]; - + let result = apply_text_changes(content, &changes, &line_index).unwrap(); assert_eq!(result, "startabc"); - + // Insert at end let line_index = LineIndex::new(content); let changes = vec![TextDocumentContentChangeEvent { @@ -525,7 +532,7 @@ mod tests { range_length: None, text: "end".to_string(), }]; - + let result = apply_text_changes(content, &changes, &line_index).unwrap(); assert_eq!(result, "abcend"); } diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 78669adf..767afbea 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -9,10 +9,7 @@ use std::{collections::HashMap, sync::Arc}; use salsa::Setter; use super::{ - db::{ - parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, - TemplateLoaderOrder, - }, + db::{parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder}, vfs::{FileKind, VfsSnapshot}, FileId, }; @@ -69,16 +66,12 @@ impl FileStore { pub fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { for (id, rec) in &snap.files { let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); - let new_kind = match rec.meta.kind { - FileKind::Python => FileKindMini::Python, - FileKind::Template => FileKindMini::Template, - FileKind::Other => FileKindMini::Other, - }; + let new_kind = rec.meta.kind; if let Some(sf) = self.files.get(id) { // Update if changed — avoid touching Salsa when not needed if sf.kind(&self.db) != new_kind { - sf.set_kind(&mut self.db).to(new_kind.clone()); + sf.set_kind(&mut self.db).to(new_kind); } if sf.text(&self.db).as_ref() != &*new_text { sf.set_text(&mut self.db).to(new_text.clone()); @@ -100,7 +93,7 @@ impl FileStore { /// Get the file kind classification by its [`FileId`]. /// /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub fn file_kind(&self, id: FileId) -> Option { + pub fn file_kind(&self, id: FileId) -> Option { self.files.get(&id).map(|sf| sf.kind(&self.db)) } diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index a38757dc..1b4ece5c 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -7,6 +7,8 @@ use std::sync::Arc; #[cfg(test)] use std::sync::Mutex; +use crate::vfs::FileKind; + /// Salsa database root for workspace /// /// The [`Database`] provides default storage and, in tests, captures Salsa events for @@ -49,21 +51,6 @@ impl Default for Database { #[salsa::db] impl salsa::Database for Database {} -/// Minimal classification for analysis routing. -/// -/// [`FileKindMini`] provides a lightweight categorization of files to determine which -/// analysis pipelines should process them. This is the Salsa-side representation -/// of file types, mapped from the VFS layer's `vfs::FileKind`. -#[derive(Clone, Eq, PartialEq, Hash, Debug)] -pub enum FileKindMini { - /// Python source file (.py) - Python, - /// Django template file (.html, .jinja, etc.) - Template, - /// Other file types not requiring specialized analysis - Other, -} - /// Represents a single file's classification and current content. /// /// [`SourceFile`] is a Salsa input entity that tracks both the file's type (for routing @@ -72,7 +59,7 @@ pub enum FileKindMini { #[salsa::input] pub struct SourceFile { /// The file's classification for analysis routing - pub kind: FileKindMini, + pub kind: FileKind, /// The current text content of the file #[returns(ref)] pub text: Arc, @@ -113,7 +100,7 @@ pub struct TemplateAst { #[salsa::tracked] pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option> { // Only parse template files - if file.kind(db) != FileKindMini::Template { + if file.kind(db) != FileKind::Template { return None; } @@ -161,7 +148,7 @@ mod tests { // Create a template file let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); - let file = SourceFile::new(&db, FileKindMini::Template, template_content.clone()); + let file = SourceFile::new(&db, FileKind::Template, template_content.clone()); // First parse - should execute the parsing let ast1 = parse_template(&db, file); @@ -181,7 +168,7 @@ mod tests { // Create a template file let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); - let file = SourceFile::new(&db, FileKindMini::Template, template_content1); + let file = SourceFile::new(&db, FileKind::Template, template_content1); // First parse let ast1 = parse_template(&db, file); @@ -206,7 +193,7 @@ mod tests { // Create a Python file let python_content: Arc = Arc::from("def hello():\n print('Hello')"); - let file = SourceFile::new(&db, FileKindMini::Python, python_content); + let file = SourceFile::new(&db, FileKind::Python, python_content); // Should return None for non-template files let ast = parse_template(&db, file); @@ -223,7 +210,7 @@ mod tests { // Create a template with an error (unclosed tag) let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); - let file = SourceFile::new(&db, FileKindMini::Template, template_content); + let file = SourceFile::new(&db, FileKind::Template, template_content); // Get errors let errors1 = template_errors(&db, file); diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 30c69a79..a3db4c34 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -5,8 +5,7 @@ mod watcher; pub use bridge::FileStore; pub use db::{ - parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, - TemplateLoaderOrder, + parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder, }; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/watcher.rs index 57c798c4..55fa9a71 100644 --- a/crates/djls-workspace/src/watcher.rs +++ b/crates/djls-workspace/src/watcher.rs @@ -317,4 +317,3 @@ mod tests { assert_ne!(deleted, renamed); } } - From 3131470cce738d8e2220a4ef66e6d25cf8256e2c Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 12:22:54 -0500 Subject: [PATCH 09/56] wip --- Cargo.lock | 2 + crates/djls-server/src/lib.rs | 1 - crates/djls-server/src/server.rs | 103 ++++++- crates/djls-server/src/session.rs | 55 +++- crates/djls-server/src/workspace/document.rs | 231 -------------- crates/djls-server/src/workspace/mod.rs | 6 - crates/djls-server/src/workspace/utils.rs | 43 --- crates/djls-templates/src/lib.rs | 2 +- crates/djls-workspace/Cargo.toml | 2 + crates/djls-workspace/src/bridge.rs | 33 +- crates/djls-workspace/src/db.rs | 5 +- .../djls-workspace/src/document/language.rs | 34 ++ .../djls-workspace/src/document/line_index.rs | 87 ++++++ crates/djls-workspace/src/document/mod.rs | 132 ++++++++ .../src/document}/store.rs | 291 ++++++++++++++---- .../djls-workspace/src/document/template.rs | 13 + crates/djls-workspace/src/lib.rs | 16 +- .../djls-workspace/src/{vfs.rs => vfs/mod.rs} | 115 ++++--- .../djls-workspace/src/{ => vfs}/watcher.rs | 22 +- 19 files changed, 762 insertions(+), 431 deletions(-) delete mode 100644 crates/djls-server/src/workspace/document.rs delete mode 100644 crates/djls-server/src/workspace/mod.rs delete mode 100644 crates/djls-server/src/workspace/utils.rs create mode 100644 crates/djls-workspace/src/document/language.rs create mode 100644 crates/djls-workspace/src/document/line_index.rs create mode 100644 crates/djls-workspace/src/document/mod.rs rename crates/{djls-server/src/workspace => djls-workspace/src/document}/store.rs (67%) create mode 100644 crates/djls-workspace/src/document/template.rs rename crates/djls-workspace/src/{vfs.rs => vfs/mod.rs} (84%) rename crates/djls-workspace/src/{ => vfs}/watcher.rs (97%) diff --git a/Cargo.lock b/Cargo.lock index 73c6aae4..d739a688 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -509,11 +509,13 @@ dependencies = [ "anyhow", "camino", "dashmap", + "djls-project", "djls-templates", "notify", "salsa", "tempfile", "tokio", + "tower-lsp-server", "url", ] diff --git a/crates/djls-server/src/lib.rs b/crates/djls-server/src/lib.rs index 1ebe6186..b601c7aa 100644 --- a/crates/djls-server/src/lib.rs +++ b/crates/djls-server/src/lib.rs @@ -3,7 +3,6 @@ mod logging; mod queue; mod server; mod session; -mod workspace; use std::io::IsTerminal; diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index 9df7c634..e61bb086 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -218,8 +218,23 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Opened document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - if let Err(e) = session.documents_mut().handle_did_open(¶ms) { - tracing::error!("Failed to handle did_open: {}", e); + let uri = params.text_document.uri.clone(); + let version = params.text_document.version; + let language_id = + djls_workspace::LanguageId::from(params.text_document.language_id.as_str()); + let text = params.text_document.text.clone(); + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + if let Err(e) = + session + .documents_mut() + .open_document(url, version, language_id, text) + { + tracing::error!("Failed to handle did_open: {}", e); + } + } else { + tracing::error!("Invalid URI: {:?}", uri); } }) .await; @@ -229,7 +244,21 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Changed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let _ = session.documents_mut().handle_did_change(¶ms); + let uri = ¶ms.text_document.uri; + let version = params.text_document.version; + let changes = params.content_changes.clone(); + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + if let Err(e) = session + .documents_mut() + .update_document(&url, version, changes) + { + tracing::error!("Failed to handle did_change: {}", e); + } + } else { + tracing::error!("Invalid URI: {:?}", uri); + } }) .await; } @@ -238,7 +267,14 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Closed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - session.documents_mut().handle_did_close(¶ms); + let uri = ¶ms.text_document.uri; + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + session.documents_mut().close_document(&url); + } else { + tracing::error!("Invalid URI: {:?}", uri); + } }) .await; } @@ -248,14 +284,61 @@ impl LanguageServer for DjangoLanguageServer { .with_session(|session| { if let Some(project) = session.project() { if let Some(tags) = project.template_tags() { - return session.documents().get_completions( - params.text_document_position.text_document.uri.as_str(), - params.text_document_position.position, - tags, - ); + let uri = ¶ms.text_document_position.text_document.uri; + let position = params.text_document_position.position; + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + if let Some(context) = session.documents().get_template_context(&url, position) { + // Use the context to generate completions + let mut completions: Vec = tags + .iter() + .filter(|tag| { + context.partial_tag.is_empty() || tag.name().starts_with(&context.partial_tag) + }) + .map(|tag| { + let leading_space = if context.needs_leading_space { " " } else { "" }; + tower_lsp_server::lsp_types::CompletionItem { + label: tag.name().to_string(), + kind: Some(tower_lsp_server::lsp_types::CompletionItemKind::KEYWORD), + detail: Some(format!("Template tag from {}", tag.library())), + documentation: tag.doc().as_ref().map(|doc| { + tower_lsp_server::lsp_types::Documentation::MarkupContent( + tower_lsp_server::lsp_types::MarkupContent { + kind: tower_lsp_server::lsp_types::MarkupKind::Markdown, + value: (*doc).to_string(), + } + ) + }), + insert_text: Some(match context.closing_brace { + djls_workspace::ClosingBrace::None => format!("{}{} %}}", leading_space, tag.name()), + djls_workspace::ClosingBrace::PartialClose => format!("{}{} %", leading_space, tag.name()), + djls_workspace::ClosingBrace::FullClose => format!("{}{} ", leading_space, tag.name()), + }), + insert_text_format: Some(tower_lsp_server::lsp_types::InsertTextFormat::PLAIN_TEXT), + ..Default::default() + } + }) + .collect(); + + if completions.is_empty() { + None + } else { + completions.sort_by(|a, b| a.label.cmp(&b.label)); + Some(tower_lsp_server::lsp_types::CompletionResponse::Array(completions)) + } + } else { + None + } + } else { + None + } + } else { + None } + } else { + None } - None }) .await) } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index a5832cb7..8a584d38 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,14 +1,17 @@ +use std::path::PathBuf; + use djls_conf::Settings; use djls_project::DjangoProject; +use djls_workspace::DocumentStore; +use percent_encoding::percent_decode_str; use tower_lsp_server::lsp_types::ClientCapabilities; use tower_lsp_server::lsp_types::InitializeParams; - -use crate::workspace::Store; +use tower_lsp_server::lsp_types::Uri; #[derive(Default)] pub struct Session { project: Option, - documents: Store, + documents: DocumentStore, settings: Settings, #[allow(dead_code)] @@ -16,8 +19,46 @@ pub struct Session { } impl Session { + /// Determines the project root path from initialization parameters. + /// + /// Tries the current directory first, then falls back to the first workspace folder. + fn get_project_path(params: &InitializeParams) -> Option { + // Try current directory first + std::env::current_dir().ok().or_else(|| { + // Fall back to the first workspace folder URI + params + .workspace_folders + .as_ref() + .and_then(|folders| folders.first()) + .and_then(|folder| Self::uri_to_pathbuf(&folder.uri)) + }) + } + + /// Converts a `file:` URI into an absolute `PathBuf`. + fn uri_to_pathbuf(uri: &Uri) -> Option { + // Check if the scheme is "file" + if uri.scheme().is_none_or(|s| s.as_str() != "file") { + return None; + } + + // Get the path part as a string + let encoded_path_str = uri.path().as_str(); + + // Decode the percent-encoded path string + let decoded_path_cow = percent_decode_str(encoded_path_str).decode_utf8_lossy(); + let path_str = decoded_path_cow.as_ref(); + + #[cfg(windows)] + let path_str = { + // Remove leading '/' for paths like /C:/... + path_str.strip_prefix('/').unwrap_or(path_str) + }; + + Some(PathBuf::from(path_str)) + } + pub fn new(params: &InitializeParams) -> Self { - let project_path = crate::workspace::get_project_path(params); + let project_path = Self::get_project_path(params); let (project, settings) = if let Some(path) = &project_path { let settings = @@ -33,7 +74,7 @@ impl Session { Self { client_capabilities: params.capabilities.clone(), project, - documents: Store::default(), + documents: DocumentStore::new(), settings, } } @@ -46,11 +87,11 @@ impl Session { &mut self.project } - pub fn documents(&self) -> &Store { + pub fn documents(&self) -> &DocumentStore { &self.documents } - pub fn documents_mut(&mut self) -> &mut Store { + pub fn documents_mut(&mut self) -> &mut DocumentStore { &mut self.documents } diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs deleted file mode 100644 index 75812b87..00000000 --- a/crates/djls-server/src/workspace/document.rs +++ /dev/null @@ -1,231 +0,0 @@ -use djls_workspace::{FileId, VfsSnapshot}; -use std::sync::Arc; -use tower_lsp_server::lsp_types::{Position, Range}; - -#[derive(Clone, Debug)] -pub struct TextDocument { - pub uri: String, - pub version: i32, - pub language_id: LanguageId, - file_id: FileId, -} - -impl TextDocument { - pub fn new(uri: String, version: i32, language_id: LanguageId, file_id: FileId) -> Self { - Self { - uri, - version, - language_id, - file_id, - } - } - - pub fn file_id(&self) -> FileId { - self.file_id - } - - pub fn get_content(&self, vfs: &VfsSnapshot) -> Option> { - vfs.get_text(self.file_id) - } - - pub fn get_line(&self, vfs: &VfsSnapshot, line_index: &LineIndex, line: u32) -> Option { - let content = self.get_content(vfs)?; - - let line_start = *line_index.line_starts.get(line as usize)?; - let line_end = line_index - .line_starts - .get(line as usize + 1) - .copied() - .unwrap_or(line_index.length); - - Some(content[line_start as usize..line_end as usize].to_string()) - } - - pub fn get_text_range( - &self, - vfs: &VfsSnapshot, - line_index: &LineIndex, - range: Range, - ) -> Option { - let content = self.get_content(vfs)?; - - let start_offset = line_index.offset(range.start)? as usize; - let end_offset = line_index.offset(range.end)? as usize; - - Some(content[start_offset..end_offset].to_string()) - } - - pub fn get_template_tag_context( - &self, - vfs: &VfsSnapshot, - line_index: &LineIndex, - position: Position, - ) -> Option { - let content = self.get_content(vfs)?; - - let start = line_index.line_starts.get(position.line as usize)?; - let end = line_index - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(line_index.length); - - let line = &content[*start as usize..end as usize]; - let char_pos: usize = position.character.try_into().ok()?; - let prefix = &line[..char_pos]; - let rest_of_line = &line[char_pos..]; - let rest_trimmed = rest_of_line.trim_start(); - - prefix.rfind("{%").map(|tag_start| { - // Check if we're immediately after {% with no space - let needs_leading_space = prefix.ends_with("{%"); - - let closing_brace = if rest_trimmed.starts_with("%}") { - ClosingBrace::FullClose - } else if rest_trimmed.starts_with('}') { - ClosingBrace::PartialClose - } else { - ClosingBrace::None - }; - - TemplateTagContext { - partial_tag: prefix[tag_start + 2..].trim().to_string(), - closing_brace, - needs_leading_space, - } - }) - } -} - -#[derive(Clone, Debug)] -pub struct LineIndex { - pub line_starts: Vec, - pub line_starts_utf16: Vec, - pub length: u32, - pub length_utf16: u32, -} - -impl LineIndex { - pub fn new(text: &str) -> Self { - let mut line_starts = vec![0]; - let mut line_starts_utf16 = vec![0]; - let mut pos_utf8 = 0; - let mut pos_utf16 = 0; - - for c in text.chars() { - pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); - pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); - if c == '\n' { - line_starts.push(pos_utf8); - line_starts_utf16.push(pos_utf16); - } - } - - Self { - line_starts, - line_starts_utf16, - length: pos_utf8, - length_utf16: pos_utf16, - } - } - - pub fn offset(&self, position: Position) -> Option { - let line_start = self.line_starts.get(position.line as usize)?; - - Some(line_start + position.character) - } - - /// Convert UTF-16 LSP position to UTF-8 byte offset - pub fn offset_utf16(&self, position: Position, text: &str) -> Option { - let line_start_utf8 = self.line_starts.get(position.line as usize)?; - let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; - - // If position is at start of line, return UTF-8 line start - if position.character == 0 { - return Some(*line_start_utf8); - } - - // Find the line text - let next_line_start = self - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(self.length); - - let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; - - // Convert UTF-16 character offset to UTF-8 byte offset within the line - let mut utf16_pos = 0; - let mut utf8_pos = 0; - - for c in line_text.chars() { - if utf16_pos >= position.character { - break; - } - utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); - utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); - } - - Some(line_start_utf8 + utf8_pos) - } - - #[allow(dead_code)] - pub fn position(&self, offset: u32) -> Position { - let line = match self.line_starts.binary_search(&offset) { - Ok(line) => line, - Err(line) => line - 1, - }; - - let line_start = self.line_starts[line]; - let character = offset - line_start; - - Position::new(u32::try_from(line).unwrap_or(0), character) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub enum LanguageId { - HtmlDjango, - Other, - Python, -} - -impl From<&str> for LanguageId { - fn from(language_id: &str) -> Self { - match language_id { - "django-html" | "htmldjango" => Self::HtmlDjango, - "python" => Self::Python, - _ => Self::Other, - } - } -} - -impl From for LanguageId { - fn from(language_id: String) -> Self { - Self::from(language_id.as_str()) - } -} - -impl From for djls_workspace::FileKind { - fn from(language_id: LanguageId) -> Self { - match language_id { - LanguageId::Python => Self::Python, - LanguageId::HtmlDjango => Self::Template, - LanguageId::Other => Self::Other, - } - } -} - -#[derive(Debug)] -pub enum ClosingBrace { - None, - PartialClose, // just } - FullClose, // %} -} - -#[derive(Debug)] -pub struct TemplateTagContext { - pub partial_tag: String, - pub closing_brace: ClosingBrace, - pub needs_leading_space: bool, -} diff --git a/crates/djls-server/src/workspace/mod.rs b/crates/djls-server/src/workspace/mod.rs deleted file mode 100644 index fb15df9b..00000000 --- a/crates/djls-server/src/workspace/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod document; -mod store; -mod utils; - -pub use store::Store; -pub use utils::get_project_path; diff --git a/crates/djls-server/src/workspace/utils.rs b/crates/djls-server/src/workspace/utils.rs deleted file mode 100644 index 08a40ba0..00000000 --- a/crates/djls-server/src/workspace/utils.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::path::PathBuf; - -use percent_encoding::percent_decode_str; -use tower_lsp_server::lsp_types::InitializeParams; -use tower_lsp_server::lsp_types::Uri; - -/// Determines the project root path from initialization parameters. -/// -/// Tries the current directory first, then falls back to the first workspace folder. -pub fn get_project_path(params: &InitializeParams) -> Option { - // Try current directory first - std::env::current_dir().ok().or_else(|| { - // Fall back to the first workspace folder URI - params - .workspace_folders - .as_ref() - .and_then(|folders| folders.first()) - .and_then(|folder| uri_to_pathbuf(&folder.uri)) - }) -} - -/// Converts a `file:` URI into an absolute `PathBuf`. -fn uri_to_pathbuf(uri: &Uri) -> Option { - // Check if the scheme is "file" - if uri.scheme().is_none_or(|s| s.as_str() != "file") { - return None; - } - - // Get the path part as a string - let encoded_path_str = uri.path().as_str(); - - // Decode the percent-encoded path string - let decoded_path_cow = percent_decode_str(encoded_path_str).decode_utf8_lossy(); - let path_str = decoded_path_cow.as_ref(); - - #[cfg(windows)] - let path_str = { - // Remove leading '/' for paths like /C:/... - path_str.strip_prefix('/').unwrap_or(path_str) - }; - - Some(PathBuf::from(path_str)) -} diff --git a/crates/djls-templates/src/lib.rs b/crates/djls-templates/src/lib.rs index 7eab1f63..7c2369c5 100644 --- a/crates/djls-templates/src/lib.rs +++ b/crates/djls-templates/src/lib.rs @@ -1,4 +1,4 @@ -mod ast; +pub mod ast; mod error; mod lexer; mod parser; diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 34fa01e8..0a46bd81 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] djls-templates = { workspace = true } +djls-project = { workspace = true } anyhow = { workspace = true } camino = { workspace = true } @@ -12,6 +13,7 @@ dashmap = { workspace = true } notify = { workspace = true } salsa = { workspace = true } tokio = { workspace = true } +tower-lsp-server = { workspace = true } url = { workspace = true } [dev-dependencies] diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 767afbea..03382629 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -4,15 +4,20 @@ //! It ensures we only touch Salsa when content or classification changes, maximizing //! incremental performance. -use std::{collections::HashMap, sync::Arc}; +use std::collections::HashMap; +use std::sync::Arc; use salsa::Setter; -use super::{ - db::{parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder}, - vfs::{FileKind, VfsSnapshot}, - FileId, -}; +use super::db::parse_template; +use super::db::template_errors; +use super::db::Database; +use super::db::SourceFile; +use super::db::TemplateAst; +use super::db::TemplateLoaderOrder; +use super::vfs::FileKind; +use super::vfs::VfsSnapshot; +use super::FileId; /// Owner of the Salsa [`Database`] plus the handles for updating inputs. /// @@ -63,7 +68,7 @@ impl FileStore { /// /// The method is idempotent and minimizes Salsa invalidations by checking for /// actual changes before updating inputs. - pub fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { + pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { for (id, rec) in &snap.files { let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); let new_kind = rec.meta.kind; @@ -86,14 +91,14 @@ impl FileStore { /// Get the text content of a file by its [`FileId`]. /// /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub fn file_text(&self, id: FileId) -> Option> { + pub(crate) fn file_text(&self, id: FileId) -> Option> { self.files.get(&id).map(|sf| sf.text(&self.db).clone()) } /// Get the file kind classification by its [`FileId`]. /// /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub fn file_kind(&self, id: FileId) -> Option { + pub(crate) fn file_kind(&self, id: FileId) -> Option { self.files.get(&id).map(|sf| sf.kind(&self.db)) } @@ -102,7 +107,7 @@ impl FileStore { /// This method leverages Salsa's incremental computation to cache parsed ASTs. /// The AST is only re-parsed when the file's content changes in the VFS. /// Returns `None` if the file is not tracked or is not a template file. - pub fn get_template_ast(&self, id: FileId) -> Option> { + pub(crate) fn get_template_ast(&self, id: FileId) -> Option> { let source_file = self.files.get(&id)?; parse_template(&self.db, *source_file) } @@ -112,7 +117,7 @@ impl FileStore { /// This method provides quick access to template errors without needing the full AST. /// Useful for diagnostics and error reporting. Returns an empty slice for /// non-template files or files not tracked in the store. - pub fn get_template_errors(&self, id: FileId) -> Arc<[String]> { + pub(crate) fn get_template_errors(&self, id: FileId) -> Arc<[String]> { self.files .get(&id) .map_or_else(|| Arc::from(vec![]), |sf| template_errors(&self.db, *sf)) @@ -127,10 +132,12 @@ impl Default for FileStore { #[cfg(test)] mod tests { - use super::*; - use crate::vfs::{TextSource, Vfs}; use camino::Utf8PathBuf; + use super::*; + use crate::vfs::TextSource; + use crate::vfs::Vfs; + #[test] fn test_filestore_template_ast_caching() { let mut store = FileStore::new(); diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 1b4ece5c..2783eae7 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -7,6 +7,8 @@ use std::sync::Arc; #[cfg(test)] use std::sync::Mutex; +use djls_templates::Ast; + use crate::vfs::FileKind; /// Salsa database root for workspace @@ -139,9 +141,10 @@ pub fn template_errors(db: &dyn salsa::Database, file: SourceFile) -> Arc<[Strin #[cfg(test)] mod tests { - use super::*; use salsa::Setter; + use super::*; + #[test] fn test_template_parsing_caches_result() { let db = Database::default(); diff --git a/crates/djls-workspace/src/document/language.rs b/crates/djls-workspace/src/document/language.rs new file mode 100644 index 00000000..09f0bb5d --- /dev/null +++ b/crates/djls-workspace/src/document/language.rs @@ -0,0 +1,34 @@ +use crate::vfs::FileKind; + +#[derive(Clone, Debug, PartialEq)] +pub enum LanguageId { + HtmlDjango, + Other, + Python, +} + +impl From<&str> for LanguageId { + fn from(language_id: &str) -> Self { + match language_id { + "django-html" | "htmldjango" => Self::HtmlDjango, + "python" => Self::Python, + _ => Self::Other, + } + } +} + +impl From for LanguageId { + fn from(language_id: String) -> Self { + Self::from(language_id.as_str()) + } +} + +impl From for FileKind { + fn from(language_id: LanguageId) -> Self { + match language_id { + LanguageId::Python => Self::Python, + LanguageId::HtmlDjango => Self::Template, + LanguageId::Other => Self::Other, + } + } +} diff --git a/crates/djls-workspace/src/document/line_index.rs b/crates/djls-workspace/src/document/line_index.rs new file mode 100644 index 00000000..39f1fde7 --- /dev/null +++ b/crates/djls-workspace/src/document/line_index.rs @@ -0,0 +1,87 @@ +use tower_lsp_server::lsp_types::Position; + +#[derive(Clone, Debug)] +pub struct LineIndex { + pub line_starts: Vec, + pub line_starts_utf16: Vec, + pub length: u32, + pub length_utf16: u32, +} + +impl LineIndex { + pub fn new(text: &str) -> Self { + let mut line_starts = vec![0]; + let mut line_starts_utf16 = vec![0]; + let mut pos_utf8 = 0; + let mut pos_utf16 = 0; + + for c in text.chars() { + pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); + if c == '\n' { + line_starts.push(pos_utf8); + line_starts_utf16.push(pos_utf16); + } + } + + Self { + line_starts, + line_starts_utf16, + length: pos_utf8, + length_utf16: pos_utf16, + } + } + + pub fn offset(&self, position: Position) -> Option { + let line_start = self.line_starts.get(position.line as usize)?; + + Some(line_start + position.character) + } + + /// Convert UTF-16 LSP position to UTF-8 byte offset + pub fn offset_utf16(&self, position: Position, text: &str) -> Option { + let line_start_utf8 = self.line_starts.get(position.line as usize)?; + let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; + + // If position is at start of line, return UTF-8 line start + if position.character == 0 { + return Some(*line_start_utf8); + } + + // Find the line text + let next_line_start = self + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.length); + + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; + + // Convert UTF-16 character offset to UTF-8 byte offset within the line + let mut utf16_pos = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if utf16_pos >= position.character { + break; + } + utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); + } + + Some(line_start_utf8 + utf8_pos) + } + + #[allow(dead_code)] + pub fn position(&self, offset: u32) -> Position { + let line = match self.line_starts.binary_search(&offset) { + Ok(line) => line, + Err(line) => line - 1, + }; + + let line_start = self.line_starts[line]; + let character = offset - line_start; + + Position::new(u32::try_from(line).unwrap_or(0), character) + } +} diff --git a/crates/djls-workspace/src/document/mod.rs b/crates/djls-workspace/src/document/mod.rs new file mode 100644 index 00000000..840cd170 --- /dev/null +++ b/crates/djls-workspace/src/document/mod.rs @@ -0,0 +1,132 @@ +mod language; +mod line_index; +mod store; +mod template; + +pub use language::LanguageId; +pub use line_index::LineIndex; +pub use store::DocumentStore; +pub use template::ClosingBrace; +pub use template::TemplateTagContext; +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; + +use crate::FileId; + +#[derive(Clone, Debug)] +pub struct TextDocument { + pub uri: String, + pub version: i32, + pub language_id: LanguageId, + pub(crate) file_id: FileId, + line_index: LineIndex, +} + +impl TextDocument { + pub(crate) fn new( + uri: String, + version: i32, + language_id: LanguageId, + file_id: FileId, + content: &str, + ) -> Self { + let line_index = LineIndex::new(content); + Self { + uri, + version, + language_id, + file_id, + line_index, + } + } + + pub(crate) fn file_id(&self) -> FileId { + self.file_id + } + + pub fn line_index(&self) -> &LineIndex { + &self.line_index + } + + pub fn get_content<'a>(&self, content: &'a str) -> &'a str { + content + } + + pub fn get_line(&self, content: &str, line: u32) -> Option { + let line_start = *self.line_index.line_starts.get(line as usize)?; + let line_end = self + .line_index + .line_starts + .get(line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + Some(content[line_start as usize..line_end as usize].to_string()) + } + + pub fn get_text_range(&self, content: &str, range: Range) -> Option { + let start_offset = self.line_index.offset(range.start)? as usize; + let end_offset = self.line_index.offset(range.end)? as usize; + + Some(content[start_offset..end_offset].to_string()) + } + + pub fn get_template_tag_context( + &self, + content: &str, + position: Position, + ) -> Option { + let start = self.line_index.line_starts.get(position.line as usize)?; + let end = self + .line_index + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + let line = &content[*start as usize..end as usize]; + let char_pos: usize = position.character.try_into().ok()?; + let prefix = &line[..char_pos]; + let rest_of_line = &line[char_pos..]; + let rest_trimmed = rest_of_line.trim_start(); + + prefix.rfind("{%").map(|tag_start| { + // Check if we're immediately after {% with no space + let needs_leading_space = prefix.ends_with("{%"); + + let closing_brace = if rest_trimmed.starts_with("%}") { + ClosingBrace::FullClose + } else if rest_trimmed.starts_with('}') { + ClosingBrace::PartialClose + } else { + ClosingBrace::None + }; + + TemplateTagContext { + partial_tag: prefix[tag_start + 2..].trim().to_string(), + closing_brace, + needs_leading_space, + } + }) + } + + pub fn position_to_offset(&self, position: Position) -> Option { + self.line_index.offset(position) + } + + pub fn offset_to_position(&self, offset: u32) -> Position { + self.line_index.position(offset) + } + + pub fn update_content(&mut self, content: &str) { + self.line_index = LineIndex::new(content); + } + + pub fn version(&self) -> i32 { + self.version + } + + pub fn language_id(&self) -> LanguageId { + self.language_id.clone() + } +} diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-workspace/src/document/store.rs similarity index 67% rename from crates/djls-server/src/workspace/store.rs rename to crates/djls-workspace/src/document/store.rs index 90962019..82841078 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-workspace/src/document/store.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +use std::sync::Mutex; use anyhow::anyhow; use anyhow::Result; use camino::Utf8PathBuf; use djls_project::TemplateTags; -use djls_workspace::{FileId, FileKind, FileStore, TextSource, Vfs}; use tower_lsp_server::lsp_types::CompletionItem; use tower_lsp_server::lsp_types::CompletionItemKind; use tower_lsp_server::lsp_types::CompletionResponse; @@ -22,31 +22,118 @@ use tower_lsp_server::lsp_types::Position; use tower_lsp_server::lsp_types::Range; use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; -use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; - -pub struct Store { +use crate::bridge::FileStore; +use crate::db::TemplateAst; +use crate::vfs::FileKind; +use crate::vfs::TextSource; +use crate::vfs::Vfs; +use crate::ClosingBrace; +use crate::LanguageId; +use crate::LineIndex; +use crate::TextDocument; + +pub struct DocumentStore { vfs: Arc, file_store: Arc>, - file_ids: HashMap, - line_indices: HashMap, - versions: HashMap, documents: HashMap, } -impl Default for Store { +impl Default for DocumentStore { fn default() -> Self { Self { vfs: Arc::new(Vfs::default()), file_store: Arc::new(Mutex::new(FileStore::new())), - file_ids: HashMap::new(), - line_indices: HashMap::new(), - versions: HashMap::new(), documents: HashMap::new(), } } } -impl Store { +impl DocumentStore { + pub fn new() -> Self { + Self::default() + } + + /// Open a document with the given URI, version, language, and text content. + /// This creates a new TextDocument and stores it internally, hiding VFS details. + pub fn open_document( + &mut self, + uri: url::Url, + version: i32, + language_id: LanguageId, + text: String, + ) -> Result<()> { + let uri_str = uri.to_string(); + let kind = FileKind::from(language_id.clone()); + + // Convert URI to path - simplified for now, just use URI string + let path = Utf8PathBuf::from(uri.as_str()); + + // Store content in VFS + let text_source = TextSource::Overlay(Arc::from(text.as_str())); + let file_id = self.vfs.intern_file(uri, path, kind, text_source); + + // Set overlay content in VFS + self.vfs.set_overlay(file_id, Arc::from(text.as_str()))?; + + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); + + // Create TextDocument with LineIndex + let document = TextDocument::new(uri_str.clone(), version, language_id, file_id, &text); + self.documents.insert(uri_str, document); + + Ok(()) + } + + /// Update a document with the given URI, version, and text changes. + /// This applies changes to the document and updates the VFS accordingly. + pub fn update_document( + &mut self, + uri: &url::Url, + version: i32, + changes: Vec, + ) -> Result<()> { + let uri_str = uri.to_string(); + + // Get document and file_id from the documents HashMap + let document = self + .documents + .get(&uri_str) + .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; + let file_id = document.file_id(); + + // Get current content from VFS + let snapshot = self.vfs.snapshot(); + let current_content = snapshot + .get_text(file_id) + .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; + + // Get line index from the document + let line_index = document.line_index(); + + // Apply text changes using the existing function + let new_content = apply_text_changes(¤t_content, &changes, line_index)?; + + // Update TextDocument version and content + if let Some(document) = self.documents.get_mut(&uri_str) { + document.version = version; + document.update_content(&new_content); + } + + // Update VFS with new content + self.vfs + .set_overlay(file_id, Arc::from(new_content.as_str()))?; + + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); + + Ok(()) + } + pub fn handle_did_open(&mut self, params: &DidOpenTextDocumentParams) -> Result<()> { let uri_str = params.text_document.uri.to_string(); let uri = params.text_document.uri.clone(); @@ -75,13 +162,14 @@ impl Store { file_store.apply_vfs_snapshot(&snapshot); // Create TextDocument metadata - let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); - self.documents.insert(uri_str.clone(), document); - - // Cache mappings and indices - self.file_ids.insert(uri_str.clone(), file_id); - self.line_indices.insert(file_id, LineIndex::new(&content)); - self.versions.insert(uri_str, version); + let document = TextDocument::new( + uri_str.clone(), + version, + language_id.clone(), + file_id, + &content, + ); + self.documents.insert(uri_str, document); Ok(()) } @@ -90,12 +178,12 @@ impl Store { let uri_str = params.text_document.uri.as_str().to_string(); let version = params.text_document.version; - // Look up FileId - let file_id = self - .file_ids + // Get document and file_id from the documents HashMap + let document = self + .documents .get(&uri_str) - .copied() .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; + let file_id = document.file_id(); // Get current content from VFS let snapshot = self.vfs.snapshot(); @@ -103,19 +191,17 @@ impl Store { .get_text(file_id) .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - // Get current line index for position calculations - let line_index = self - .line_indices - .get(&file_id) - .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; + // Get line index from the document (TextDocument now stores its own LineIndex) + let line_index = document.line_index(); // Apply text changes using the new function let new_content = apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; - // Update TextDocument version + // Update TextDocument version and content if let Some(document) = self.documents.get_mut(&uri_str) { document.version = version; + document.update_content(&new_content); } // Update VFS with new content @@ -127,41 +213,38 @@ impl Store { let mut file_store = self.file_store.lock().unwrap(); file_store.apply_vfs_snapshot(&snapshot); - // Update cached line index and version - self.line_indices - .insert(file_id, LineIndex::new(&new_content)); - self.versions.insert(uri_str, version); - Ok(()) } - pub fn handle_did_close(&mut self, params: &DidCloseTextDocumentParams) { - let uri_str = params.text_document.uri.as_str(); + /// Close a document with the given URI. + /// This removes the document from internal storage and cleans up resources. + pub fn close_document(&mut self, uri: &url::Url) { + let uri_str = uri.as_str(); // Remove TextDocument metadata self.documents.remove(uri_str); - // Look up FileId and remove mappings - if let Some(file_id) = self.file_ids.remove(uri_str) { - self.line_indices.remove(&file_id); - } - self.versions.remove(uri_str); - // Note: We don't remove from VFS as it might be useful for caching // The VFS will handle cleanup internally } - pub fn get_file_id(&self, uri: &str) -> Option { - self.file_ids.get(uri).copied() + pub fn handle_did_close(&mut self, params: &DidCloseTextDocumentParams) { + let uri_str = params.text_document.uri.as_str(); + + // Remove TextDocument metadata + self.documents.remove(uri_str); + + // Note: We don't remove from VFS as it might be useful for caching + // The VFS will handle cleanup internally } - pub fn get_line_index(&self, file_id: FileId) -> Option<&LineIndex> { - self.line_indices.get(&file_id) + pub fn get_line_index(&self, uri: &str) -> Option<&LineIndex> { + self.documents.get(uri).map(|doc| doc.line_index()) } #[allow(dead_code)] pub fn get_version(&self, uri: &str) -> Option { - self.versions.get(uri).copied() + self.documents.get(uri).map(|doc| doc.version()) } #[allow(dead_code)] @@ -178,6 +261,99 @@ impl Store { self.documents.get_mut(uri) } + // URI-based query methods (new API) + pub fn get_document_by_url(&self, uri: &url::Url) -> Option<&TextDocument> { + self.get_document(uri.as_str()) + } + + pub fn get_document_text(&self, uri: &url::Url) -> Option> { + let document = self.get_document_by_url(uri)?; + let file_id = document.file_id(); + let snapshot = self.vfs.snapshot(); + snapshot.get_text(file_id) + } + + pub fn get_line_text(&self, uri: &url::Url, line: u32) -> Option { + let document = self.get_document_by_url(uri)?; + let snapshot = self.vfs.snapshot(); + let content = snapshot.get_text(document.file_id())?; + document.get_line(content.as_ref(), line) + } + + pub fn get_word_at_position(&self, uri: &url::Url, position: Position) -> Option { + // This is a simplified implementation - get the line and extract word at position + let line_text = self.get_line_text(uri, position.line)?; + let char_pos: usize = position.character.try_into().ok()?; + + if char_pos >= line_text.len() { + return None; + } + + // Find word boundaries (simplified - considers alphanumeric and underscore as word chars) + let line_bytes = line_text.as_bytes(); + let mut start = char_pos; + let mut end = char_pos; + + // Find start of word + while start > 0 && is_word_char(line_bytes[start - 1]) { + start -= 1; + } + + // Find end of word + while end < line_text.len() && is_word_char(line_bytes[end]) { + end += 1; + } + + if start < end { + Some(line_text[start..end].to_string()) + } else { + None + } + } + + // Position mapping methods + pub fn offset_to_position(&self, uri: &url::Url, offset: usize) -> Option { + let document = self.get_document_by_url(uri)?; + Some(document.offset_to_position(offset as u32)) + } + + pub fn position_to_offset(&self, uri: &url::Url, position: Position) -> Option { + let document = self.get_document_by_url(uri)?; + document + .position_to_offset(position) + .map(|offset| offset as usize) + } + + // Template-specific methods + pub fn get_template_ast(&self, uri: &url::Url) -> Option> { + let document = self.get_document_by_url(uri)?; + let file_id = document.file_id(); + let file_store = self.file_store.lock().unwrap(); + file_store.get_template_ast(file_id) + } + + pub fn get_template_errors(&self, uri: &url::Url) -> Vec { + let document = match self.get_document_by_url(uri) { + Some(doc) => doc, + None => return vec![], + }; + let file_id = document.file_id(); + let file_store = self.file_store.lock().unwrap(); + let errors = file_store.get_template_errors(file_id); + errors.to_vec() + } + + pub fn get_template_context( + &self, + uri: &url::Url, + position: Position, + ) -> Option { + let document = self.get_document_by_url(uri)?; + let snapshot = self.vfs.snapshot(); + let content = snapshot.get_text(document.file_id())?; + document.get_template_tag_context(content.as_ref(), position) + } + pub fn get_completions( &self, uri: &str, @@ -186,7 +362,7 @@ impl Store { ) -> Option { // Check if this is a Django template using TextDocument metadata let document = self.get_document(uri)?; - if document.language_id != LanguageId::HtmlDjango { + if document.language_id() != LanguageId::HtmlDjango { return None; } @@ -202,8 +378,9 @@ impl Store { // Get template tag context from document let vfs_snapshot = self.vfs.snapshot(); - let line_index = self.get_line_index(file_id)?; - let context = document.get_template_tag_context(&vfs_snapshot, line_index, position)?; + let text_content = vfs_snapshot.get_text(file_id)?; + let content = text_content.as_ref(); + let context = document.get_template_tag_context(content, position)?; let mut completions: Vec = tags .iter() @@ -252,12 +429,12 @@ impl Store { }; // Only process template files - if document.language_id != LanguageId::HtmlDjango { + if document.language_id() != LanguageId::HtmlDjango { return vec![]; } let file_id = document.file_id(); - let Some(_line_index) = self.get_line_index(file_id) else { + let Some(_line_index) = self.get_line_index(uri) else { return vec![]; }; @@ -294,6 +471,11 @@ impl Store { } } +/// Check if a byte represents a word character (alphanumeric or underscore) +fn is_word_char(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} + /// Apply text changes to content, handling multiple changes correctly fn apply_text_changes( content: &str, @@ -360,9 +542,10 @@ fn apply_text_changes( #[cfg(test)] mod tests { - use super::*; use tower_lsp_server::lsp_types::Range; + use super::*; + #[test] fn test_apply_single_character_insertion() { let content = "Hello world"; diff --git a/crates/djls-workspace/src/document/template.rs b/crates/djls-workspace/src/document/template.rs new file mode 100644 index 00000000..2a0547c4 --- /dev/null +++ b/crates/djls-workspace/src/document/template.rs @@ -0,0 +1,13 @@ +#[derive(Debug)] +pub enum ClosingBrace { + None, + PartialClose, // just } + FullClose, // %} +} + +#[derive(Debug)] +pub struct TemplateTagContext { + pub partial_tag: String, + pub closing_brace: ClosingBrace, + pub needs_leading_space: bool, +} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index a3db4c34..fb45cc19 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,14 +1,14 @@ mod bridge; mod db; +mod document; mod vfs; -mod watcher; -pub use bridge::FileStore; -pub use db::{ - parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder, -}; -pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; -pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; +pub use document::ClosingBrace; +pub use document::DocumentStore; +pub use document::LanguageId; +pub use document::LineIndex; +pub use document::TemplateTagContext; +pub use document::TextDocument; /// Stable, compact identifier for files across the subsystem. /// @@ -16,7 +16,7 @@ pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; /// Salsa inputs. Once assigned to a file (via its URI), a [`FileId`] remains stable for the /// lifetime of the VFS, even if the file's content or metadata changes. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub struct FileId(u32); +pub(crate) struct FileId(u32); impl FileId { /// Create a [`FileId`] from a raw u32 value. diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs/mod.rs similarity index 84% rename from crates/djls-workspace/src/vfs.rs rename to crates/djls-workspace/src/vfs/mod.rs index 08eb1827..2b4b22f2 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs/mod.rs @@ -4,25 +4,28 @@ //! and snapshotting. Downstream systems consume snapshots to avoid locking and to //! batch updates. -use anyhow::{anyhow, Result}; -use camino::Utf8PathBuf; -use dashmap::DashMap; +mod watcher; + use std::collections::hash_map::DefaultHasher; +use std::collections::HashMap; use std::fs; -use std::hash::{Hash, Hasher}; -use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicU32, AtomicU64, Ordering}, - Arc, - }, -}; +use std::hash::Hash; +use std::hash::Hasher; +use std::sync::atomic::AtomicU32; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use anyhow::anyhow; +use anyhow::Result; +use camino::Utf8PathBuf; +use dashmap::DashMap; use url::Url; +use watcher::VfsWatcher; +use watcher::WatchConfig; +use watcher::WatchEvent; -use super::{ - watcher::{VfsWatcher, WatchConfig, WatchEvent}, - FileId, -}; +use super::FileId; /// Monotonic counter representing global VFS state. /// @@ -30,18 +33,18 @@ use super::{ /// This provides a cheap way to detect if any changes have occurred since /// a previous snapshot was taken. #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default, PartialOrd, Ord)] -pub struct Revision(u64); +pub(crate) struct Revision(u64); impl Revision { /// Create a [`Revision`] from a raw u64 value. #[must_use] - pub fn from_raw(raw: u64) -> Self { + fn from_raw(raw: u64) -> Self { Revision(raw) } /// Get the underlying u64 value. #[must_use] - pub fn value(self) -> u64 { + fn value(self) -> u64 { self.0 } } @@ -65,11 +68,11 @@ pub enum FileKind { /// [`FileMeta`] contains all non-content information about a file, including its /// identity (URI), filesystem path, and classification. #[derive(Clone, Debug)] -pub struct FileMeta { +pub(crate) struct FileMeta { /// The file's URI (typically file:// scheme) - pub uri: Url, + uri: Url, /// The file's path in the filesystem - pub path: Utf8PathBuf, + path: Utf8PathBuf, /// Classification for routing to analyzers pub kind: FileKind, } @@ -80,7 +83,7 @@ pub struct FileMeta { /// debugging and understanding the current state of the VFS. All variants hold /// `Arc` for efficient sharing. #[derive(Clone)] -pub enum TextSource { +pub(crate) enum TextSource { /// Content loaded from disk Disk(Arc), /// Content from LSP client overlay (in-memory edits) @@ -89,18 +92,47 @@ pub enum TextSource { Generated(Arc), } +/// Content hash for efficient change detection. +/// +/// [`FileHash`] encapsulates the hashing logic used to detect when file content +/// has changed, avoiding unnecessary recomputation in downstream systems like Salsa. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +struct FileHash(u64); + +impl FileHash { + /// Compute hash from text source content. + fn from_text_source(src: &TextSource) -> Self { + let s: &str = match src { + TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, + }; + let mut h = DefaultHasher::new(); + s.hash(&mut h); + Self(h.finish()) + } + + /// Check if this hash differs from another, indicating content changed. + fn differs_from(self, other: Self) -> bool { + self.0 != other.0 + } + + /// Get raw hash value (for debugging/logging). + fn raw(self) -> u64 { + self.0 + } +} + /// Complete record of a file in the VFS. /// /// [`FileRecord`] combines metadata, current text content, and a content hash /// for efficient change detection. #[derive(Clone)] -pub struct FileRecord { +pub(crate) struct FileRecord { /// File metadata (URI, path, kind, version) pub meta: FileMeta, /// Current text content and its source - pub text: TextSource, + text: TextSource, /// Hash of current content for change detection - pub hash: u64, + hash: FileHash, } /// Thread-safe virtual file system with change tracking. @@ -129,7 +161,7 @@ impl Vfs { /// Returns the existing [`FileId`] if the URI is already known, or creates a new /// [`FileRecord`] with the provided metadata and text. This method computes and /// stores a content hash for change detection. - pub fn intern_file( + pub(crate) fn intern_file( &self, uri: Url, path: Utf8PathBuf, @@ -145,7 +177,7 @@ impl Vfs { path: path.clone(), kind, }; - let hash = content_hash(&text); + let hash = FileHash::from_text_source(&text); self.by_uri.insert(uri, id); self.by_path.insert(path, id); self.files.insert(id, FileRecord { meta, text, hash }); @@ -159,14 +191,14 @@ impl Vfs { /// (detected via hash comparison). /// /// Returns a tuple of (new global revision, whether content changed). - pub fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { + pub(crate) fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { let mut rec = self .files .get_mut(&id) .ok_or_else(|| anyhow!("unknown file: {:?}", id))?; let next = TextSource::Overlay(new_text); - let new_hash = content_hash(&next); - let changed = new_hash != rec.hash; + let new_hash = FileHash::from_text_source(&next); + let changed = new_hash.differs_from(rec.hash); if changed { rec.text = next; rec.hash = new_hash; @@ -183,7 +215,7 @@ impl Vfs { /// Materializes a consistent view of all files for downstream consumers. /// The snapshot includes the current revision and a clone of all file records. /// This operation is relatively cheap due to `Arc` sharing of text content. - pub fn snapshot(&self) -> VfsSnapshot { + pub(crate) fn snapshot(&self) -> VfsSnapshot { VfsSnapshot { revision: Revision::from_raw(self.head.load(Ordering::SeqCst)), files: self @@ -268,11 +300,11 @@ impl Vfs { .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; let new_text = TextSource::Disk(Arc::from(content.as_str())); - let new_hash = content_hash(&new_text); + let new_hash = FileHash::from_text_source(&new_text); // Update the file if content changed if let Some(mut record) = self.files.get_mut(&file_id) { - if record.hash != new_hash { + if new_hash.differs_from(record.hash) { record.text = new_text; record.hash = new_hash; self.head.fetch_add(1, Ordering::SeqCst); @@ -301,28 +333,15 @@ impl Default for Vfs { } } -/// Compute a stable hash over file content. -/// -/// Used for efficient change detection - if the hash hasn't changed, -/// the content hasn't changed, avoiding unnecessary Salsa invalidations. -fn content_hash(src: &TextSource) -> u64 { - let s: &str = match src { - TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, - }; - let mut h = DefaultHasher::new(); - s.hash(&mut h); - h.finish() -} - /// Immutable snapshot view of the VFS at a specific revision. /// /// [`VfsSnapshot`] provides a consistent view of all files for downstream consumers, /// avoiding the need for locking during processing. Snapshots are created atomically /// and can be safely shared across threads. #[derive(Clone)] -pub struct VfsSnapshot { +pub(crate) struct VfsSnapshot { /// The global revision at the time of snapshot - pub revision: Revision, + revision: Revision, /// All files in the VFS at snapshot time pub files: HashMap, } diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/vfs/watcher.rs similarity index 97% rename from crates/djls-workspace/src/watcher.rs rename to crates/djls-workspace/src/vfs/watcher.rs index 55fa9a71..aee04676 100644 --- a/crates/djls-workspace/src/watcher.rs +++ b/crates/djls-workspace/src/vfs/watcher.rs @@ -4,15 +4,21 @@ //! and synchronize them with the VFS. It uses cross-platform file watching with //! debouncing to handle rapid changes efficiently. -use anyhow::{anyhow, Result}; +use std::collections::HashMap; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; +use std::time::Instant; + +use anyhow::anyhow; +use anyhow::Result; use camino::Utf8PathBuf; -use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; -use std::{ - collections::HashMap, - sync::mpsc, - thread, - time::{Duration, Instant}, -}; +use notify::Config; +use notify::Event; +use notify::EventKind; +use notify::RecommendedWatcher; +use notify::RecursiveMode; +use notify::Watcher; /// Event types that can occur in the file system. /// From 3bf25ac2040fdc52993dad90f02f59aadfe575a2 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 13:11:13 -0500 Subject: [PATCH 10/56] must use! --- .../djls-workspace/src/document/line_index.rs | 3 + crates/djls-workspace/src/document/store.rs | 119 +++--------------- 2 files changed, 23 insertions(+), 99 deletions(-) diff --git a/crates/djls-workspace/src/document/line_index.rs b/crates/djls-workspace/src/document/line_index.rs index 39f1fde7..cbbf6459 100644 --- a/crates/djls-workspace/src/document/line_index.rs +++ b/crates/djls-workspace/src/document/line_index.rs @@ -9,6 +9,7 @@ pub struct LineIndex { } impl LineIndex { + #[must_use] pub fn new(text: &str) -> Self { let mut line_starts = vec![0]; let mut line_starts_utf16 = vec![0]; @@ -32,6 +33,7 @@ impl LineIndex { } } + #[must_use] pub fn offset(&self, position: Position) -> Option { let line_start = self.line_starts.get(position.line as usize)?; @@ -73,6 +75,7 @@ impl LineIndex { } #[allow(dead_code)] + #[must_use] pub fn position(&self, offset: u32) -> Position { let line = match self.line_starts.binary_search(&offset) { Ok(line) => line, diff --git a/crates/djls-workspace/src/document/store.rs b/crates/djls-workspace/src/document/store.rs index 82841078..3f8e77eb 100644 --- a/crates/djls-workspace/src/document/store.rs +++ b/crates/djls-workspace/src/document/store.rs @@ -49,6 +49,7 @@ impl Default for DocumentStore { } impl DocumentStore { + #[must_use] pub fn new() -> Self { Self::default() } @@ -134,88 +135,6 @@ impl DocumentStore { Ok(()) } - pub fn handle_did_open(&mut self, params: &DidOpenTextDocumentParams) -> Result<()> { - let uri_str = params.text_document.uri.to_string(); - let uri = params.text_document.uri.clone(); - let version = params.text_document.version; - let content = params.text_document.text.clone(); - let language_id = LanguageId::from(params.text_document.language_id.as_str()); - let kind = FileKind::from(language_id.clone()); - - // Convert URI to Url for VFS - let vfs_url = - url::Url::parse(&uri.to_string()).map_err(|e| anyhow!("Invalid URI: {}", e))?; - - // Convert to path - simplified for now, just use URI string - let path = Utf8PathBuf::from(uri.as_str()); - - // Store content in VFS - let text_source = TextSource::Overlay(Arc::from(content.as_str())); - let file_id = self.vfs.intern_file(vfs_url, path, kind, text_source); - - // Set overlay content in VFS - self.vfs.set_overlay(file_id, Arc::from(content.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - // Create TextDocument metadata - let document = TextDocument::new( - uri_str.clone(), - version, - language_id.clone(), - file_id, - &content, - ); - self.documents.insert(uri_str, document); - - Ok(()) - } - - pub fn handle_did_change(&mut self, params: &DidChangeTextDocumentParams) -> Result<()> { - let uri_str = params.text_document.uri.as_str().to_string(); - let version = params.text_document.version; - - // Get document and file_id from the documents HashMap - let document = self - .documents - .get(&uri_str) - .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; - let file_id = document.file_id(); - - // Get current content from VFS - let snapshot = self.vfs.snapshot(); - let current_content = snapshot - .get_text(file_id) - .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - - // Get line index from the document (TextDocument now stores its own LineIndex) - let line_index = document.line_index(); - - // Apply text changes using the new function - let new_content = - apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; - - // Update TextDocument version and content - if let Some(document) = self.documents.get_mut(&uri_str) { - document.version = version; - document.update_content(&new_content); - } - - // Update VFS with new content - self.vfs - .set_overlay(file_id, Arc::from(new_content.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - Ok(()) - } - /// Close a document with the given URI. /// This removes the document from internal storage and cleans up resources. pub fn close_document(&mut self, uri: &url::Url) { @@ -228,31 +147,25 @@ impl DocumentStore { // The VFS will handle cleanup internally } - pub fn handle_did_close(&mut self, params: &DidCloseTextDocumentParams) { - let uri_str = params.text_document.uri.as_str(); - - // Remove TextDocument metadata - self.documents.remove(uri_str); - - // Note: We don't remove from VFS as it might be useful for caching - // The VFS will handle cleanup internally - } - + #[must_use] pub fn get_line_index(&self, uri: &str) -> Option<&LineIndex> { - self.documents.get(uri).map(|doc| doc.line_index()) + self.documents.get(uri).map(super::TextDocument::line_index) } #[allow(dead_code)] + #[must_use] pub fn get_version(&self, uri: &str) -> Option { - self.documents.get(uri).map(|doc| doc.version()) + self.documents.get(uri).map(super::TextDocument::version) } #[allow(dead_code)] + #[must_use] pub fn is_version_valid(&self, uri: &str, version: i32) -> bool { self.get_version(uri) == Some(version) } // TextDocument helper methods + #[must_use] pub fn get_document(&self, uri: &str) -> Option<&TextDocument> { self.documents.get(uri) } @@ -262,10 +175,12 @@ impl DocumentStore { } // URI-based query methods (new API) + #[must_use] pub fn get_document_by_url(&self, uri: &url::Url) -> Option<&TextDocument> { self.get_document(uri.as_str()) } + #[must_use] pub fn get_document_text(&self, uri: &url::Url) -> Option> { let document = self.get_document_by_url(uri)?; let file_id = document.file_id(); @@ -273,6 +188,7 @@ impl DocumentStore { snapshot.get_text(file_id) } + #[must_use] pub fn get_line_text(&self, uri: &url::Url, line: u32) -> Option { let document = self.get_document_by_url(uri)?; let snapshot = self.vfs.snapshot(); @@ -280,6 +196,7 @@ impl DocumentStore { document.get_line(content.as_ref(), line) } + #[must_use] pub fn get_word_at_position(&self, uri: &url::Url, position: Position) -> Option { // This is a simplified implementation - get the line and extract word at position let line_text = self.get_line_text(uri, position.line)?; @@ -312,11 +229,13 @@ impl DocumentStore { } // Position mapping methods + #[must_use] pub fn offset_to_position(&self, uri: &url::Url, offset: usize) -> Option { let document = self.get_document_by_url(uri)?; Some(document.offset_to_position(offset as u32)) } + #[must_use] pub fn position_to_offset(&self, uri: &url::Url, position: Position) -> Option { let document = self.get_document_by_url(uri)?; document @@ -325,6 +244,7 @@ impl DocumentStore { } // Template-specific methods + #[must_use] pub fn get_template_ast(&self, uri: &url::Url) -> Option> { let document = self.get_document_by_url(uri)?; let file_id = document.file_id(); @@ -332,10 +252,10 @@ impl DocumentStore { file_store.get_template_ast(file_id) } + #[must_use] pub fn get_template_errors(&self, uri: &url::Url) -> Vec { - let document = match self.get_document_by_url(uri) { - Some(doc) => doc, - None => return vec![], + let Some(document) = self.get_document_by_url(uri) else { + return vec![]; }; let file_id = document.file_id(); let file_store = self.file_store.lock().unwrap(); @@ -343,6 +263,7 @@ impl DocumentStore { errors.to_vec() } + #[must_use] pub fn get_template_context( &self, uri: &url::Url, @@ -354,6 +275,7 @@ impl DocumentStore { document.get_template_tag_context(content.as_ref(), position) } + #[must_use] pub fn get_completions( &self, uri: &str, @@ -379,8 +301,7 @@ impl DocumentStore { // Get template tag context from document let vfs_snapshot = self.vfs.snapshot(); let text_content = vfs_snapshot.get_text(file_id)?; - let content = text_content.as_ref(); - let context = document.get_template_tag_context(content, position)?; + let context = document.get_template_tag_context(text_content.as_ref(), position)?; let mut completions: Vec = tags .iter() From 269d4bceaeb2588ec1ae4c83ad61d9f82185f928 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 13:19:52 -0500 Subject: [PATCH 11/56] lsp types --- crates/djls-server/src/client.rs | 47 ++++++------- crates/djls-server/src/logging.rs | 16 ++--- crates/djls-server/src/server.rs | 109 +++++++++++++++--------------- crates/djls-server/src/session.rs | 12 ++-- 4 files changed, 88 insertions(+), 96 deletions(-) diff --git a/crates/djls-server/src/client.rs b/crates/djls-server/src/client.rs index 35eb8410..11a5f62e 100644 --- a/crates/djls-server/src/client.rs +++ b/crates/djls-server/src/client.rs @@ -123,45 +123,38 @@ macro_rules! request { #[allow(dead_code)] pub mod messages { - use tower_lsp_server::lsp_types::MessageActionItem; - use tower_lsp_server::lsp_types::MessageType; - use tower_lsp_server::lsp_types::ShowDocumentParams; + use tower_lsp_server::lsp_types; use super::get_client; use super::Display; use super::Error; - notify!(log_message, message_type: MessageType, message: impl Display + Send + 'static); - notify!(show_message, message_type: MessageType, message: impl Display + Send + 'static); - request!(show_message_request, message_type: MessageType, message: impl Display + Send + 'static, actions: Option> ; Option); - request!(show_document, params: ShowDocumentParams ; bool); + notify!(log_message, message_type: lsp_types::MessageType, message: impl Display + Send + 'static); + notify!(show_message, message_type: lsp_types::MessageType, message: impl Display + Send + 'static); + request!(show_message_request, message_type: lsp_types::MessageType, message: impl Display + Send + 'static, actions: Option> ; Option); + request!(show_document, params: lsp_types::ShowDocumentParams ; bool); } #[allow(dead_code)] pub mod diagnostics { - use tower_lsp_server::lsp_types::Diagnostic; - use tower_lsp_server::lsp_types::Uri; + use tower_lsp_server::lsp_types; use super::get_client; - notify!(publish_diagnostics, uri: Uri, diagnostics: Vec, version: Option); + notify!(publish_diagnostics, uri: lsp_types::Uri, diagnostics: Vec, version: Option); notify_discard!(workspace_diagnostic_refresh,); } #[allow(dead_code)] pub mod workspace { - use tower_lsp_server::lsp_types::ApplyWorkspaceEditResponse; - use tower_lsp_server::lsp_types::ConfigurationItem; - use tower_lsp_server::lsp_types::LSPAny; - use tower_lsp_server::lsp_types::WorkspaceEdit; - use tower_lsp_server::lsp_types::WorkspaceFolder; + use tower_lsp_server::lsp_types; use super::get_client; use super::Error; - request!(apply_edit, edit: WorkspaceEdit ; ApplyWorkspaceEditResponse); - request!(configuration, items: Vec ; Vec); - request!(workspace_folders, ; Option>); + request!(apply_edit, edit: lsp_types::WorkspaceEdit ; lsp_types::ApplyWorkspaceEditResponse); + request!(configuration, items: Vec ; Vec); + request!(workspace_folders, ; Option>); } #[allow(dead_code)] @@ -176,19 +169,18 @@ pub mod editor { #[allow(dead_code)] pub mod capabilities { - use tower_lsp_server::lsp_types::Registration; - use tower_lsp_server::lsp_types::Unregistration; + use tower_lsp_server::lsp_types; use super::get_client; - notify_discard!(register_capability, registrations: Vec); - notify_discard!(unregister_capability, unregisterations: Vec); + notify_discard!(register_capability, registrations: Vec); + notify_discard!(unregister_capability, unregisterations: Vec); } #[allow(dead_code)] pub mod monitoring { use serde::Serialize; - use tower_lsp_server::lsp_types::ProgressToken; + use tower_lsp_server::lsp_types; use tower_lsp_server::Progress; use super::get_client; @@ -201,22 +193,21 @@ pub mod monitoring { } } - pub fn progress + Send>(token: ProgressToken, title: T) -> Option { + pub fn progress + Send>(token: lsp_types::ProgressToken, title: T) -> Option { get_client().map(|client| client.progress(token, title)) } } #[allow(dead_code)] pub mod protocol { - use tower_lsp_server::lsp_types::notification::Notification; - use tower_lsp_server::lsp_types::request::Request; + use tower_lsp_server::lsp_types; use super::get_client; use super::Error; pub fn send_notification(params: N::Params) where - N: Notification, + N: lsp_types::notification::Notification, N::Params: Send + 'static, { if let Some(client) = get_client() { @@ -228,7 +219,7 @@ pub mod protocol { pub async fn send_request(params: R::Params) -> Result where - R: Request, + R: lsp_types::request::Request, R::Params: Send + 'static, R::Result: Send + 'static, { diff --git a/crates/djls-server/src/logging.rs b/crates/djls-server/src/logging.rs index a540401a..030af946 100644 --- a/crates/djls-server/src/logging.rs +++ b/crates/djls-server/src/logging.rs @@ -15,7 +15,7 @@ use std::sync::Arc; -use tower_lsp_server::lsp_types::MessageType; +use tower_lsp_server::lsp_types; use tracing::field::Visit; use tracing::Level; use tracing_appender::non_blocking::WorkerGuard; @@ -32,13 +32,13 @@ use tracing_subscriber::Registry; /// that are sent to the client. It filters events by level to avoid overwhelming /// the client with verbose trace logs. pub struct LspLayer { - send_message: Arc, + send_message: Arc, } impl LspLayer { pub fn new(send_message: F) -> Self where - F: Fn(MessageType, String) + Send + Sync + 'static, + F: Fn(lsp_types::MessageType, String) + Send + Sync + 'static, { Self { send_message: Arc::new(send_message), @@ -82,10 +82,10 @@ where let metadata = event.metadata(); let message_type = match *metadata.level() { - Level::ERROR => MessageType::ERROR, - Level::WARN => MessageType::WARNING, - Level::INFO => MessageType::INFO, - Level::DEBUG => MessageType::LOG, + Level::ERROR => lsp_types::MessageType::ERROR, + Level::WARN => lsp_types::MessageType::WARNING, + Level::INFO => lsp_types::MessageType::INFO, + Level::DEBUG => lsp_types::MessageType::LOG, Level::TRACE => { // Skip TRACE level - too verbose for LSP client // TODO: Add MessageType::Debug in LSP 3.18.0 @@ -112,7 +112,7 @@ where /// Returns a `WorkerGuard` that must be kept alive for the file logging to work. pub fn init_tracing(send_message: F) -> WorkerGuard where - F: Fn(MessageType, String) + Send + Sync + 'static, + F: Fn(lsp_types::MessageType, String) + Send + Sync + 'static, { let file_appender = tracing_appender::rolling::daily("/tmp", "djls.log"); let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index e61bb086..c5fa5d69 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -3,25 +3,7 @@ use std::sync::Arc; use tokio::sync::RwLock; use tower_lsp_server::jsonrpc::Result as LspResult; -use tower_lsp_server::lsp_types::CompletionOptions; -use tower_lsp_server::lsp_types::CompletionParams; -use tower_lsp_server::lsp_types::CompletionResponse; -use tower_lsp_server::lsp_types::DidChangeConfigurationParams; -use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; -use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; -use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::InitializeParams; -use tower_lsp_server::lsp_types::InitializeResult; -use tower_lsp_server::lsp_types::InitializedParams; -use tower_lsp_server::lsp_types::OneOf; -use tower_lsp_server::lsp_types::SaveOptions; -use tower_lsp_server::lsp_types::ServerCapabilities; -use tower_lsp_server::lsp_types::ServerInfo; -use tower_lsp_server::lsp_types::TextDocumentSyncCapability; -use tower_lsp_server::lsp_types::TextDocumentSyncKind; -use tower_lsp_server::lsp_types::TextDocumentSyncOptions; -use tower_lsp_server::lsp_types::WorkspaceFoldersServerCapabilities; -use tower_lsp_server::lsp_types::WorkspaceServerCapabilities; +use tower_lsp_server::lsp_types; use tower_lsp_server::LanguageServer; use tracing_appender::non_blocking::WorkerGuard; @@ -91,7 +73,10 @@ impl DjangoLanguageServer { } impl LanguageServer for DjangoLanguageServer { - async fn initialize(&self, params: InitializeParams) -> LspResult { + async fn initialize( + &self, + params: lsp_types::InitializeParams, + ) -> LspResult { tracing::info!("Initializing server..."); let session = Session::new(¶ms); @@ -101,9 +86,9 @@ impl LanguageServer for DjangoLanguageServer { *session_lock = Some(session); } - Ok(InitializeResult { - capabilities: ServerCapabilities { - completion_provider: Some(CompletionOptions { + Ok(lsp_types::InitializeResult { + capabilities: lsp_types::ServerCapabilities { + completion_provider: Some(lsp_types::CompletionOptions { resolve_provider: Some(false), trigger_characters: Some(vec![ "{".to_string(), @@ -112,25 +97,25 @@ impl LanguageServer for DjangoLanguageServer { ]), ..Default::default() }), - workspace: Some(WorkspaceServerCapabilities { - workspace_folders: Some(WorkspaceFoldersServerCapabilities { + workspace: Some(lsp_types::WorkspaceServerCapabilities { + workspace_folders: Some(lsp_types::WorkspaceFoldersServerCapabilities { supported: Some(true), - change_notifications: Some(OneOf::Left(true)), + change_notifications: Some(lsp_types::OneOf::Left(true)), }), file_operations: None, }), - text_document_sync: Some(TextDocumentSyncCapability::Options( - TextDocumentSyncOptions { + text_document_sync: Some(lsp_types::TextDocumentSyncCapability::Options( + lsp_types::TextDocumentSyncOptions { open_close: Some(true), - change: Some(TextDocumentSyncKind::INCREMENTAL), + change: Some(lsp_types::TextDocumentSyncKind::INCREMENTAL), will_save: Some(false), will_save_wait_until: Some(false), - save: Some(SaveOptions::default().into()), + save: Some(lsp_types::SaveOptions::default().into()), }, )), ..Default::default() }, - server_info: Some(ServerInfo { + server_info: Some(lsp_types::ServerInfo { name: SERVER_NAME.to_string(), version: Some(SERVER_VERSION.to_string()), }), @@ -139,7 +124,7 @@ impl LanguageServer for DjangoLanguageServer { } #[allow(clippy::too_many_lines)] - async fn initialized(&self, _params: InitializedParams) { + async fn initialized(&self, _params: lsp_types::InitializedParams) { tracing::info!("Server received initialized notification."); self.with_session_task(|session_arc| async move { @@ -214,7 +199,7 @@ impl LanguageServer for DjangoLanguageServer { Ok(()) } - async fn did_open(&self, params: DidOpenTextDocumentParams) { + async fn did_open(&self, params: lsp_types::DidOpenTextDocumentParams) { tracing::info!("Opened document: {:?}", params.text_document.uri); self.with_session_mut(|session| { @@ -240,7 +225,7 @@ impl LanguageServer for DjangoLanguageServer { .await; } - async fn did_change(&self, params: DidChangeTextDocumentParams) { + async fn did_change(&self, params: lsp_types::DidChangeTextDocumentParams) { tracing::info!("Changed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { @@ -263,7 +248,7 @@ impl LanguageServer for DjangoLanguageServer { .await; } - async fn did_close(&self, params: DidCloseTextDocumentParams) { + async fn did_close(&self, params: lsp_types::DidCloseTextDocumentParams) { tracing::info!("Closed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { @@ -279,7 +264,10 @@ impl LanguageServer for DjangoLanguageServer { .await; } - async fn completion(&self, params: CompletionParams) -> LspResult> { + async fn completion( + &self, + params: lsp_types::CompletionParams, + ) -> LspResult> { Ok(self .with_session(|session| { if let Some(project) = session.project() { @@ -289,33 +277,48 @@ impl LanguageServer for DjangoLanguageServer { // Convert LSP Uri to url::Url if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Some(context) = session.documents().get_template_context(&url, position) { + if let Some(context) = + session.documents().get_template_context(&url, position) + { // Use the context to generate completions - let mut completions: Vec = tags + let mut completions: Vec = tags .iter() .filter(|tag| { - context.partial_tag.is_empty() || tag.name().starts_with(&context.partial_tag) + context.partial_tag.is_empty() + || tag.name().starts_with(&context.partial_tag) }) .map(|tag| { - let leading_space = if context.needs_leading_space { " " } else { "" }; - tower_lsp_server::lsp_types::CompletionItem { + let leading_space = + if context.needs_leading_space { " " } else { "" }; + lsp_types::CompletionItem { label: tag.name().to_string(), - kind: Some(tower_lsp_server::lsp_types::CompletionItemKind::KEYWORD), - detail: Some(format!("Template tag from {}", tag.library())), + kind: Some(lsp_types::CompletionItemKind::KEYWORD), + detail: Some(format!( + "Template tag from {}", + tag.library() + )), documentation: tag.doc().as_ref().map(|doc| { - tower_lsp_server::lsp_types::Documentation::MarkupContent( - tower_lsp_server::lsp_types::MarkupContent { - kind: tower_lsp_server::lsp_types::MarkupKind::Markdown, + lsp_types::Documentation::MarkupContent( + lsp_types::MarkupContent { + kind: lsp_types::MarkupKind::Markdown, value: (*doc).to_string(), - } + }, ) }), insert_text: Some(match context.closing_brace { - djls_workspace::ClosingBrace::None => format!("{}{} %}}", leading_space, tag.name()), - djls_workspace::ClosingBrace::PartialClose => format!("{}{} %", leading_space, tag.name()), - djls_workspace::ClosingBrace::FullClose => format!("{}{} ", leading_space, tag.name()), + djls_workspace::ClosingBrace::None => { + format!("{}{} %}}", leading_space, tag.name()) + } + djls_workspace::ClosingBrace::PartialClose => { + format!("{}{} %", leading_space, tag.name()) + } + djls_workspace::ClosingBrace::FullClose => { + format!("{}{} ", leading_space, tag.name()) + } }), - insert_text_format: Some(tower_lsp_server::lsp_types::InsertTextFormat::PLAIN_TEXT), + insert_text_format: Some( + lsp_types::InsertTextFormat::PLAIN_TEXT, + ), ..Default::default() } }) @@ -325,7 +328,7 @@ impl LanguageServer for DjangoLanguageServer { None } else { completions.sort_by(|a, b| a.label.cmp(&b.label)); - Some(tower_lsp_server::lsp_types::CompletionResponse::Array(completions)) + Some(lsp_types::CompletionResponse::Array(completions)) } } else { None @@ -343,7 +346,7 @@ impl LanguageServer for DjangoLanguageServer { .await) } - async fn did_change_configuration(&self, _params: DidChangeConfigurationParams) { + async fn did_change_configuration(&self, _params: lsp_types::DidChangeConfigurationParams) { tracing::info!("Configuration change detected. Reloading settings..."); let project_path = self diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 8a584d38..1f6da451 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -4,9 +4,7 @@ use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::DocumentStore; use percent_encoding::percent_decode_str; -use tower_lsp_server::lsp_types::ClientCapabilities; -use tower_lsp_server::lsp_types::InitializeParams; -use tower_lsp_server::lsp_types::Uri; +use tower_lsp_server::lsp_types; #[derive(Default)] pub struct Session { @@ -15,14 +13,14 @@ pub struct Session { settings: Settings, #[allow(dead_code)] - client_capabilities: ClientCapabilities, + client_capabilities: lsp_types::ClientCapabilities, } impl Session { /// Determines the project root path from initialization parameters. /// /// Tries the current directory first, then falls back to the first workspace folder. - fn get_project_path(params: &InitializeParams) -> Option { + fn get_project_path(params: &lsp_types::InitializeParams) -> Option { // Try current directory first std::env::current_dir().ok().or_else(|| { // Fall back to the first workspace folder URI @@ -35,7 +33,7 @@ impl Session { } /// Converts a `file:` URI into an absolute `PathBuf`. - fn uri_to_pathbuf(uri: &Uri) -> Option { + fn uri_to_pathbuf(uri: &lsp_types::Uri) -> Option { // Check if the scheme is "file" if uri.scheme().is_none_or(|s| s.as_str() != "file") { return None; @@ -57,7 +55,7 @@ impl Session { Some(PathBuf::from(path_str)) } - pub fn new(params: &InitializeParams) -> Self { + pub fn new(params: &lsp_types::InitializeParams) -> Self { let project_path = Self::get_project_path(params); let (project, settings) = if let Some(path) = &project_path { From 4e3446f6ee2bf256f2e1fe42bfcfda0bccbc0e1d Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 27 Aug 2025 15:37:29 -0500 Subject: [PATCH 12/56] wip --- Cargo.lock | 103 +-- crates/djls-server/src/server.rs | 137 +--- crates/djls-server/src/session.rs | 83 ++- crates/djls-workspace/src/bridge.rs | 162 +---- crates/djls-workspace/src/db.rs | 94 ++- .../djls-workspace/src/document/language.rs | 2 +- crates/djls-workspace/src/document/mod.rs | 2 - crates/djls-workspace/src/document/store.rs | 643 ------------------ crates/djls-workspace/src/lib.rs | 32 +- crates/djls-workspace/src/lsp_system.rs | 154 +++++ crates/djls-workspace/src/system.rs | 118 ++++ crates/djls-workspace/src/test_db.rs | 25 + crates/djls-workspace/src/vfs/mod.rs | 367 ---------- crates/djls-workspace/src/vfs/watcher.rs | 325 --------- 14 files changed, 578 insertions(+), 1669 deletions(-) delete mode 100644 crates/djls-workspace/src/document/store.rs create mode 100644 crates/djls-workspace/src/lsp_system.rs create mode 100644 crates/djls-workspace/src/system.rs create mode 100644 crates/djls-workspace/src/test_db.rs delete mode 100644 crates/djls-workspace/src/vfs/mod.rs delete mode 100644 crates/djls-workspace/src/vfs/watcher.rs diff --git a/Cargo.lock b/Cargo.lock index d739a688..d017be40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,9 +140,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.2" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" +checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" dependencies = [ "serde", ] @@ -170,21 +170,21 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "camino" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d07aa9a93b00c76f71bc35d598bed923f6d4f3a9ca5c24b7737ae1a292841c0" +checksum = "dd0b03af37dad7a14518b7691d81acb0f8222604ad3d1b02f6b4bed5188c0cd5" [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "clap" -version = "4.5.45" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" +checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" dependencies = [ "clap_builder", "clap_derive", @@ -192,9 +192,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.44" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" +checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" dependencies = [ "anstream", "anstyle", @@ -442,7 +442,7 @@ dependencies = [ "directories", "serde", "tempfile", - "thiserror 2.0.15", + "thiserror 2.0.16", "toml", ] @@ -498,7 +498,7 @@ dependencies = [ "insta", "serde", "tempfile", - "thiserror 2.0.15", + "thiserror 2.0.16", "toml", ] @@ -883,9 +883,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" dependencies = [ "equivalent", "hashbrown 0.15.5", @@ -903,7 +903,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "inotify-sys", "libc", ] @@ -940,11 +940,11 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "cfg-if", "libc", ] @@ -1010,7 +1010,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "libc", ] @@ -1106,7 +1106,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "fsevent-sys", "inotify", "kqueue", @@ -1224,9 +1224,9 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" @@ -1235,7 +1235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.15", + "thiserror 2.0.16", "ucd-trie", ] @@ -1417,7 +1417,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", ] [[package]] @@ -1428,19 +1428,19 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.15", + "thiserror 2.0.16", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", + "regex-automata 0.4.10", + "regex-syntax 0.8.6", ] [[package]] @@ -1454,13 +1454,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax 0.8.6", ] [[package]] @@ -1471,9 +1471,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" [[package]] name = "ron" @@ -1482,7 +1482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64", - "bitflags 2.9.2", + "bitflags 2.9.3", "serde", "serde_derive", ] @@ -1516,7 +1516,7 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "errno", "libc", "linux-raw-sys", @@ -1630,9 +1630,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -1765,15 +1765,15 @@ checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "tempfile" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -1793,11 +1793,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d76d3f064b981389ecb4b6b7f45a0bf9fdac1d5b9204c7bd6714fecc302850" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.15", + "thiserror-impl 2.0.16", ] [[package]] @@ -1813,9 +1813,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d29feb33e986b6ea906bd9c3559a856983f92371b3eaa5e83782a351623de0" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -2129,13 +2129,14 @@ checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "url" -version = "2.5.4" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2393,9 +2394,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -2412,7 +2413,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", ] [[package]] diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index c5fa5d69..e0026587 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -202,25 +202,13 @@ impl LanguageServer for DjangoLanguageServer { async fn did_open(&self, params: lsp_types::DidOpenTextDocumentParams) { tracing::info!("Opened document: {:?}", params.text_document.uri); - self.with_session_mut(|session| { - let uri = params.text_document.uri.clone(); - let version = params.text_document.version; - let language_id = + self.with_session_mut(|_session| { + // TODO: Handle document open after refactoring + let _uri = params.text_document.uri.clone(); + let _version = params.text_document.version; + let _language_id = djls_workspace::LanguageId::from(params.text_document.language_id.as_str()); - let text = params.text_document.text.clone(); - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Err(e) = - session - .documents_mut() - .open_document(url, version, language_id, text) - { - tracing::error!("Failed to handle did_open: {}", e); - } - } else { - tracing::error!("Invalid URI: {:?}", uri); - } + let _text = params.text_document.text.clone(); }) .await; } @@ -228,22 +216,11 @@ impl LanguageServer for DjangoLanguageServer { async fn did_change(&self, params: lsp_types::DidChangeTextDocumentParams) { tracing::info!("Changed document: {:?}", params.text_document.uri); - self.with_session_mut(|session| { - let uri = ¶ms.text_document.uri; - let version = params.text_document.version; - let changes = params.content_changes.clone(); - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Err(e) = session - .documents_mut() - .update_document(&url, version, changes) - { - tracing::error!("Failed to handle did_change: {}", e); - } - } else { - tracing::error!("Invalid URI: {:?}", uri); - } + self.with_session_mut(|_session| { + // TODO: Handle document change after refactoring + let _uri = ¶ms.text_document.uri; + let _version = params.text_document.version; + let _changes = params.content_changes.clone(); }) .await; } @@ -251,99 +228,19 @@ impl LanguageServer for DjangoLanguageServer { async fn did_close(&self, params: lsp_types::DidCloseTextDocumentParams) { tracing::info!("Closed document: {:?}", params.text_document.uri); - self.with_session_mut(|session| { - let uri = ¶ms.text_document.uri; - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - session.documents_mut().close_document(&url); - } else { - tracing::error!("Invalid URI: {:?}", uri); - } + self.with_session_mut(|_session| { + // TODO: Handle document close after refactoring + let _uri = ¶ms.text_document.uri; }) .await; } async fn completion( &self, - params: lsp_types::CompletionParams, + _params: lsp_types::CompletionParams, ) -> LspResult> { - Ok(self - .with_session(|session| { - if let Some(project) = session.project() { - if let Some(tags) = project.template_tags() { - let uri = ¶ms.text_document_position.text_document.uri; - let position = params.text_document_position.position; - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Some(context) = - session.documents().get_template_context(&url, position) - { - // Use the context to generate completions - let mut completions: Vec = tags - .iter() - .filter(|tag| { - context.partial_tag.is_empty() - || tag.name().starts_with(&context.partial_tag) - }) - .map(|tag| { - let leading_space = - if context.needs_leading_space { " " } else { "" }; - lsp_types::CompletionItem { - label: tag.name().to_string(), - kind: Some(lsp_types::CompletionItemKind::KEYWORD), - detail: Some(format!( - "Template tag from {}", - tag.library() - )), - documentation: tag.doc().as_ref().map(|doc| { - lsp_types::Documentation::MarkupContent( - lsp_types::MarkupContent { - kind: lsp_types::MarkupKind::Markdown, - value: (*doc).to_string(), - }, - ) - }), - insert_text: Some(match context.closing_brace { - djls_workspace::ClosingBrace::None => { - format!("{}{} %}}", leading_space, tag.name()) - } - djls_workspace::ClosingBrace::PartialClose => { - format!("{}{} %", leading_space, tag.name()) - } - djls_workspace::ClosingBrace::FullClose => { - format!("{}{} ", leading_space, tag.name()) - } - }), - insert_text_format: Some( - lsp_types::InsertTextFormat::PLAIN_TEXT, - ), - ..Default::default() - } - }) - .collect(); - - if completions.is_empty() { - None - } else { - completions.sort_by(|a, b| a.label.cmp(&b.label)); - Some(lsp_types::CompletionResponse::Array(completions)) - } - } else { - None - } - } else { - None - } - } else { - None - } - } else { - None - } - }) - .await) + // TODO: Handle completion after refactoring + Ok(None) } async fn did_change_configuration(&self, _params: lsp_types::DidChangeConfigurationParams) { diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 1f6da451..beae4ecb 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,17 +1,59 @@ +use std::collections::HashMap; use std::path::PathBuf; +use std::sync::Arc; use djls_conf::Settings; use djls_project::DjangoProject; -use djls_workspace::DocumentStore; +use djls_workspace::{FileSystem, StdFileSystem, db::Database}; use percent_encoding::percent_decode_str; +use salsa::StorageHandle; use tower_lsp_server::lsp_types; +use url::Url; -#[derive(Default)] pub struct Session { + /// The Django project configuration project: Option, - documents: DocumentStore, + + /// LSP server settings settings: Settings, + /// A thread-safe Salsa database handle that can be shared between threads. + /// + /// This implements the insight from [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) + /// where we're using the `StorageHandle` to create a thread-safe handle that can be + /// shared between threads. When we need to use it, we clone the handle to get a new reference. + /// + /// This handle allows us to create database instances as needed. + /// Even though we're using a single-threaded runtime, we still need + /// this to be thread-safe because of LSP trait requirements. + /// + /// Usage: + /// ```rust,ignore + /// // Clone the StorageHandle for use in an async context + /// let db_handle = session.db_handle.clone(); + /// + /// // Use it in an async context + /// async_fn(move || { + /// // Get a database from the handle + /// let storage = db_handle.into_storage(); + /// let db = Database::from_storage(storage); + /// + /// // Use the database + /// db.some_query(args) + /// }); + /// ``` + db_handle: StorageHandle, + + /// File system abstraction for reading files + file_system: Arc, + + /// Index of open documents with overlays (in-memory changes) + /// Maps document URL to its current content + overlays: HashMap, + + /// Tracks the session revision for change detection + revision: u64, + #[allow(dead_code)] client_capabilities: lsp_types::ClientCapabilities, } @@ -72,8 +114,11 @@ impl Session { Self { client_capabilities: params.capabilities.clone(), project, - documents: DocumentStore::new(), settings, + db_handle: StorageHandle::new(None), + file_system: Arc::new(StdFileSystem), + overlays: HashMap::new(), + revision: 0, } } @@ -85,13 +130,7 @@ impl Session { &mut self.project } - pub fn documents(&self) -> &DocumentStore { - &self.documents - } - pub fn documents_mut(&mut self) -> &mut DocumentStore { - &mut self.documents - } pub fn settings(&self) -> &Settings { &self.settings @@ -100,4 +139,28 @@ impl Session { pub fn set_settings(&mut self, settings: Settings) { self.settings = settings; } + + /// Get a database instance from the session. + /// + /// This creates a usable database from the handle, which can be used + /// to query and update data. The database itself is not Send/Sync, + /// but the StorageHandle is, allowing us to work with tower-lsp. + pub fn db(&self) -> Database { + let storage = self.db_handle.clone().into_storage(); + Database::from_storage(storage) + } +} + +impl Default for Session { + fn default() -> Self { + Self { + project: None, + settings: Settings::default(), + db_handle: StorageHandle::new(None), + file_system: Arc::new(StdFileSystem), + overlays: HashMap::new(), + revision: 0, + client_capabilities: lsp_types::ClientCapabilities::default(), + } + } } diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 03382629..2da695e8 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -15,9 +15,8 @@ use super::db::Database; use super::db::SourceFile; use super::db::TemplateAst; use super::db::TemplateLoaderOrder; -use super::vfs::FileKind; -use super::vfs::VfsSnapshot; use super::FileId; +use super::FileKind; /// Owner of the Salsa [`Database`] plus the handles for updating inputs. /// @@ -39,7 +38,7 @@ impl FileStore { #[must_use] pub fn new() -> Self { Self { - db: Database::default(), + db: Database::new(), files: HashMap::new(), template_loader: None, } @@ -59,34 +58,26 @@ impl FileStore { } } - /// Mirror a VFS snapshot into Salsa inputs. - /// - /// This method is the core synchronization point between the VFS and Salsa. - /// It iterates through all files in the snapshot and: - /// - Creates [`SourceFile`] inputs for new files - /// - Updates `.text` and `.kind` only when changed to preserve incremental reuse - /// - /// The method is idempotent and minimizes Salsa invalidations by checking for - /// actual changes before updating inputs. - pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { - for (id, rec) in &snap.files { - let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); - let new_kind = rec.meta.kind; - - if let Some(sf) = self.files.get(id) { - // Update if changed — avoid touching Salsa when not needed - if sf.kind(&self.db) != new_kind { - sf.set_kind(&mut self.db).to(new_kind); - } - if sf.text(&self.db).as_ref() != &*new_text { - sf.set_text(&mut self.db).to(new_text.clone()); - } - } else { - let sf = SourceFile::new(&self.db, new_kind, new_text); - self.files.insert(*id, sf); - } - } - } + // TODO: This will be replaced with direct file management + // pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { + // for (id, rec) in &snap.files { + // let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); + // let new_kind = rec.meta.kind; + + // if let Some(sf) = self.files.get(id) { + // // Update if changed — avoid touching Salsa when not needed + // if sf.kind(&self.db) != new_kind { + // sf.set_kind(&mut self.db).to(new_kind); + // } + // if sf.text(&self.db).as_ref() != &*new_text { + // sf.set_text(&mut self.db).to(new_text.clone()); + // } + // } else { + // let sf = SourceFile::new(&self.db, new_kind, new_text); + // self.files.insert(*id, sf); + // } + // } + // } /// Get the text content of a file by its [`FileId`]. /// @@ -130,109 +121,6 @@ impl Default for FileStore { } } -#[cfg(test)] -mod tests { - use camino::Utf8PathBuf; - - use super::*; - use crate::vfs::TextSource; - use crate::vfs::Vfs; - - #[test] - fn test_filestore_template_ast_caching() { - let mut store = FileStore::new(); - let vfs = Vfs::default(); - - // Create a template file in VFS - let url = url::Url::parse("file:///test.html").unwrap(); - let path = Utf8PathBuf::from("/test.html"); - let content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); - let file_id = vfs.intern_file( - url.clone(), - path.clone(), - FileKind::Template, - TextSource::Overlay(content.clone()), - ); - vfs.set_overlay(file_id, content.clone()).unwrap(); - - // Apply VFS snapshot to FileStore - let snapshot = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot); - - // Get template AST - should parse and cache - let ast1 = store.get_template_ast(file_id); - assert!(ast1.is_some()); - - // Get again - should return cached - let ast2 = store.get_template_ast(file_id); - assert!(ast2.is_some()); - assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } - - #[test] - fn test_filestore_template_errors() { - let mut store = FileStore::new(); - let vfs = Vfs::default(); - - // Create a template with an unclosed tag - let url = url::Url::parse("file:///error.html").unwrap(); - let path = Utf8PathBuf::from("/error.html"); - let content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); // Missing closing - let file_id = vfs.intern_file( - url.clone(), - path.clone(), - FileKind::Template, - TextSource::Overlay(content.clone()), - ); - vfs.set_overlay(file_id, content).unwrap(); - - // Apply VFS snapshot - let snapshot = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot); - - // Get errors - should contain parsing errors - let errors = store.get_template_errors(file_id); - // The template has unclosed tags, so there should be errors - // We don't assert on specific error count as the parser may evolve - - // Verify errors are cached - let errors2 = store.get_template_errors(file_id); - assert!(Arc::ptr_eq(&errors, &errors2)); - } - - #[test] - fn test_filestore_invalidation_on_content_change() { - let mut store = FileStore::new(); - let vfs = Vfs::default(); - - // Create initial template - let url = url::Url::parse("file:///change.html").unwrap(); - let path = Utf8PathBuf::from("/change.html"); - let content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); - let file_id = vfs.intern_file( - url.clone(), - path.clone(), - FileKind::Template, - TextSource::Overlay(content1.clone()), - ); - vfs.set_overlay(file_id, content1).unwrap(); - - // Apply snapshot and get AST - let snapshot1 = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot1); - let ast1 = store.get_template_ast(file_id); - - // Change content - let content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); - vfs.set_overlay(file_id, content2).unwrap(); - - // Apply new snapshot - let snapshot2 = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot2); - - // Get AST again - should be different due to content change - let ast2 = store.get_template_ast(file_id); - assert!(ast1.is_some() && ast2.is_some()); - assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } -} +// TODO: Re-enable tests after VFS removal is complete +// #[cfg(test)] +// mod tests { diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 2783eae7..c542ea44 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -4,23 +4,36 @@ //! Inputs are kept minimal to avoid unnecessary recomputation. use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; #[cfg(test)] use std::sync::Mutex; -use djls_templates::Ast; +use dashmap::DashMap; +use url::Url; -use crate::vfs::FileKind; +use crate::{FileId, FileKind}; /// Salsa database root for workspace /// /// The [`Database`] provides default storage and, in tests, captures Salsa events for /// reuse/diagnostics. It serves as the core incremental computation engine, tracking /// dependencies and invalidations across all inputs and derived queries. +/// +/// This database also manages the file system overlay for the workspace, +/// mapping URLs to FileIds and storing file content. #[salsa::db] #[derive(Clone)] -#[cfg_attr(not(test), derive(Default))] pub struct Database { storage: salsa::Storage, + + /// Map from file URL to FileId (thread-safe) + files: DashMap, + + /// Map from FileId to file content (thread-safe) + content: DashMap>, + + /// Next FileId to allocate (thread-safe counter) + next_file_id: Arc, // The logs are only used for testing and demonstrating reuse: #[cfg(test)] @@ -45,11 +58,86 @@ impl Default for Database { } } }))), + files: DashMap::new(), + content: DashMap::new(), + next_file_id: Arc::new(AtomicU32::new(0)), logs, } } } +impl Database { + /// Create a new database instance + pub fn new() -> Self { + Self { + storage: salsa::Storage::new(None), + files: DashMap::new(), + content: DashMap::new(), + next_file_id: Arc::new(AtomicU32::new(0)), + #[cfg(test)] + logs: Arc::new(Mutex::new(None)), + } + } + + /// Create a new database instance from a storage handle. + /// This is used by Session::db() to create databases from the StorageHandle. + pub fn from_storage(storage: salsa::Storage) -> Self { + Self { + storage, + files: DashMap::new(), + content: DashMap::new(), + next_file_id: Arc::new(AtomicU32::new(0)), + #[cfg(test)] + logs: Arc::new(Mutex::new(None)), + } + } + + /// Add or update a file in the workspace + pub fn set_file(&mut self, url: Url, content: String, _kind: FileKind) { + let file_id = if let Some(existing_id) = self.files.get(&url) { + *existing_id + } else { + let new_id = FileId::from_raw(self.next_file_id.fetch_add(1, Ordering::SeqCst)); + self.files.insert(url.clone(), new_id); + new_id + }; + + let content = Arc::::from(content); + self.content.insert(file_id, content.clone()); + + // TODO: Update Salsa inputs here when we connect them + } + + /// Remove a file from the workspace + pub fn remove_file(&mut self, url: &Url) { + if let Some((_, file_id)) = self.files.remove(url) { + self.content.remove(&file_id); + // TODO: Remove from Salsa when we connect inputs + } + } + + /// Get the content of a file by URL + pub fn get_file_content(&self, url: &Url) -> Option> { + let file_id = self.files.get(url)?; + self.content.get(&*file_id).map(|content| content.clone()) + } + + /// Get the content of a file by FileId + pub(crate) fn get_content_by_id(&self, file_id: FileId) -> Option> { + self.content.get(&file_id).map(|content| content.clone()) + } + + /// Check if a file exists in the workspace + pub fn has_file(&self, url: &Url) -> bool { + self.files.contains_key(url) + } + + /// Get all file URLs in the workspace + pub fn files(&self) -> impl Iterator + use<'_> { + self.files.iter().map(|entry| entry.key().clone()) + } +} + #[salsa::db] impl salsa::Database for Database {} diff --git a/crates/djls-workspace/src/document/language.rs b/crates/djls-workspace/src/document/language.rs index 09f0bb5d..65c322ab 100644 --- a/crates/djls-workspace/src/document/language.rs +++ b/crates/djls-workspace/src/document/language.rs @@ -1,4 +1,4 @@ -use crate::vfs::FileKind; +use crate::FileKind; #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { diff --git a/crates/djls-workspace/src/document/mod.rs b/crates/djls-workspace/src/document/mod.rs index 840cd170..93d443f2 100644 --- a/crates/djls-workspace/src/document/mod.rs +++ b/crates/djls-workspace/src/document/mod.rs @@ -1,11 +1,9 @@ mod language; mod line_index; -mod store; mod template; pub use language::LanguageId; pub use line_index::LineIndex; -pub use store::DocumentStore; pub use template::ClosingBrace; pub use template::TemplateTagContext; use tower_lsp_server::lsp_types::Position; diff --git a/crates/djls-workspace/src/document/store.rs b/crates/djls-workspace/src/document/store.rs deleted file mode 100644 index 3f8e77eb..00000000 --- a/crates/djls-workspace/src/document/store.rs +++ /dev/null @@ -1,643 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::Mutex; - -use anyhow::anyhow; -use anyhow::Result; -use camino::Utf8PathBuf; -use djls_project::TemplateTags; -use tower_lsp_server::lsp_types::CompletionItem; -use tower_lsp_server::lsp_types::CompletionItemKind; -use tower_lsp_server::lsp_types::CompletionResponse; -use tower_lsp_server::lsp_types::Diagnostic; -use tower_lsp_server::lsp_types::DiagnosticSeverity; -use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; -use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; -use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::Documentation; -use tower_lsp_server::lsp_types::InsertTextFormat; -use tower_lsp_server::lsp_types::MarkupContent; -use tower_lsp_server::lsp_types::MarkupKind; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; -use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; - -use crate::bridge::FileStore; -use crate::db::TemplateAst; -use crate::vfs::FileKind; -use crate::vfs::TextSource; -use crate::vfs::Vfs; -use crate::ClosingBrace; -use crate::LanguageId; -use crate::LineIndex; -use crate::TextDocument; - -pub struct DocumentStore { - vfs: Arc, - file_store: Arc>, - documents: HashMap, -} - -impl Default for DocumentStore { - fn default() -> Self { - Self { - vfs: Arc::new(Vfs::default()), - file_store: Arc::new(Mutex::new(FileStore::new())), - documents: HashMap::new(), - } - } -} - -impl DocumentStore { - #[must_use] - pub fn new() -> Self { - Self::default() - } - - /// Open a document with the given URI, version, language, and text content. - /// This creates a new TextDocument and stores it internally, hiding VFS details. - pub fn open_document( - &mut self, - uri: url::Url, - version: i32, - language_id: LanguageId, - text: String, - ) -> Result<()> { - let uri_str = uri.to_string(); - let kind = FileKind::from(language_id.clone()); - - // Convert URI to path - simplified for now, just use URI string - let path = Utf8PathBuf::from(uri.as_str()); - - // Store content in VFS - let text_source = TextSource::Overlay(Arc::from(text.as_str())); - let file_id = self.vfs.intern_file(uri, path, kind, text_source); - - // Set overlay content in VFS - self.vfs.set_overlay(file_id, Arc::from(text.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - // Create TextDocument with LineIndex - let document = TextDocument::new(uri_str.clone(), version, language_id, file_id, &text); - self.documents.insert(uri_str, document); - - Ok(()) - } - - /// Update a document with the given URI, version, and text changes. - /// This applies changes to the document and updates the VFS accordingly. - pub fn update_document( - &mut self, - uri: &url::Url, - version: i32, - changes: Vec, - ) -> Result<()> { - let uri_str = uri.to_string(); - - // Get document and file_id from the documents HashMap - let document = self - .documents - .get(&uri_str) - .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; - let file_id = document.file_id(); - - // Get current content from VFS - let snapshot = self.vfs.snapshot(); - let current_content = snapshot - .get_text(file_id) - .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - - // Get line index from the document - let line_index = document.line_index(); - - // Apply text changes using the existing function - let new_content = apply_text_changes(¤t_content, &changes, line_index)?; - - // Update TextDocument version and content - if let Some(document) = self.documents.get_mut(&uri_str) { - document.version = version; - document.update_content(&new_content); - } - - // Update VFS with new content - self.vfs - .set_overlay(file_id, Arc::from(new_content.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - Ok(()) - } - - /// Close a document with the given URI. - /// This removes the document from internal storage and cleans up resources. - pub fn close_document(&mut self, uri: &url::Url) { - let uri_str = uri.as_str(); - - // Remove TextDocument metadata - self.documents.remove(uri_str); - - // Note: We don't remove from VFS as it might be useful for caching - // The VFS will handle cleanup internally - } - - #[must_use] - pub fn get_line_index(&self, uri: &str) -> Option<&LineIndex> { - self.documents.get(uri).map(super::TextDocument::line_index) - } - - #[allow(dead_code)] - #[must_use] - pub fn get_version(&self, uri: &str) -> Option { - self.documents.get(uri).map(super::TextDocument::version) - } - - #[allow(dead_code)] - #[must_use] - pub fn is_version_valid(&self, uri: &str, version: i32) -> bool { - self.get_version(uri) == Some(version) - } - - // TextDocument helper methods - #[must_use] - pub fn get_document(&self, uri: &str) -> Option<&TextDocument> { - self.documents.get(uri) - } - - pub fn get_document_mut(&mut self, uri: &str) -> Option<&mut TextDocument> { - self.documents.get_mut(uri) - } - - // URI-based query methods (new API) - #[must_use] - pub fn get_document_by_url(&self, uri: &url::Url) -> Option<&TextDocument> { - self.get_document(uri.as_str()) - } - - #[must_use] - pub fn get_document_text(&self, uri: &url::Url) -> Option> { - let document = self.get_document_by_url(uri)?; - let file_id = document.file_id(); - let snapshot = self.vfs.snapshot(); - snapshot.get_text(file_id) - } - - #[must_use] - pub fn get_line_text(&self, uri: &url::Url, line: u32) -> Option { - let document = self.get_document_by_url(uri)?; - let snapshot = self.vfs.snapshot(); - let content = snapshot.get_text(document.file_id())?; - document.get_line(content.as_ref(), line) - } - - #[must_use] - pub fn get_word_at_position(&self, uri: &url::Url, position: Position) -> Option { - // This is a simplified implementation - get the line and extract word at position - let line_text = self.get_line_text(uri, position.line)?; - let char_pos: usize = position.character.try_into().ok()?; - - if char_pos >= line_text.len() { - return None; - } - - // Find word boundaries (simplified - considers alphanumeric and underscore as word chars) - let line_bytes = line_text.as_bytes(); - let mut start = char_pos; - let mut end = char_pos; - - // Find start of word - while start > 0 && is_word_char(line_bytes[start - 1]) { - start -= 1; - } - - // Find end of word - while end < line_text.len() && is_word_char(line_bytes[end]) { - end += 1; - } - - if start < end { - Some(line_text[start..end].to_string()) - } else { - None - } - } - - // Position mapping methods - #[must_use] - pub fn offset_to_position(&self, uri: &url::Url, offset: usize) -> Option { - let document = self.get_document_by_url(uri)?; - Some(document.offset_to_position(offset as u32)) - } - - #[must_use] - pub fn position_to_offset(&self, uri: &url::Url, position: Position) -> Option { - let document = self.get_document_by_url(uri)?; - document - .position_to_offset(position) - .map(|offset| offset as usize) - } - - // Template-specific methods - #[must_use] - pub fn get_template_ast(&self, uri: &url::Url) -> Option> { - let document = self.get_document_by_url(uri)?; - let file_id = document.file_id(); - let file_store = self.file_store.lock().unwrap(); - file_store.get_template_ast(file_id) - } - - #[must_use] - pub fn get_template_errors(&self, uri: &url::Url) -> Vec { - let Some(document) = self.get_document_by_url(uri) else { - return vec![]; - }; - let file_id = document.file_id(); - let file_store = self.file_store.lock().unwrap(); - let errors = file_store.get_template_errors(file_id); - errors.to_vec() - } - - #[must_use] - pub fn get_template_context( - &self, - uri: &url::Url, - position: Position, - ) -> Option { - let document = self.get_document_by_url(uri)?; - let snapshot = self.vfs.snapshot(); - let content = snapshot.get_text(document.file_id())?; - document.get_template_tag_context(content.as_ref(), position) - } - - #[must_use] - pub fn get_completions( - &self, - uri: &str, - position: Position, - tags: &TemplateTags, - ) -> Option { - // Check if this is a Django template using TextDocument metadata - let document = self.get_document(uri)?; - if document.language_id() != LanguageId::HtmlDjango { - return None; - } - - // Try to get cached AST from FileStore for better context analysis - // This demonstrates using the cached AST, though we still fall back to string parsing - let file_id = document.file_id(); - let file_store = self.file_store.lock().unwrap(); - if let Some(_ast) = file_store.get_template_ast(file_id) { - // TODO: In a future enhancement, we could use the AST to provide - // more intelligent completions based on the current node context - // For now, we continue with the existing string-based approach - } - - // Get template tag context from document - let vfs_snapshot = self.vfs.snapshot(); - let text_content = vfs_snapshot.get_text(file_id)?; - let context = document.get_template_tag_context(text_content.as_ref(), position)?; - - let mut completions: Vec = tags - .iter() - .filter(|tag| { - context.partial_tag.is_empty() || tag.name().starts_with(&context.partial_tag) - }) - .map(|tag| { - let leading_space = if context.needs_leading_space { " " } else { "" }; - CompletionItem { - label: tag.name().to_string(), - kind: Some(CompletionItemKind::KEYWORD), - detail: Some(format!("Template tag from {}", tag.library())), - documentation: tag.doc().as_ref().map(|doc| { - Documentation::MarkupContent(MarkupContent { - kind: MarkupKind::Markdown, - value: (*doc).to_string(), - }) - }), - insert_text: Some(match context.closing_brace { - ClosingBrace::None => format!("{}{} %}}", leading_space, tag.name()), - ClosingBrace::PartialClose => format!("{}{} %", leading_space, tag.name()), - ClosingBrace::FullClose => format!("{}{} ", leading_space, tag.name()), - }), - insert_text_format: Some(InsertTextFormat::PLAIN_TEXT), - ..Default::default() - } - }) - .collect(); - - if completions.is_empty() { - None - } else { - completions.sort_by(|a, b| a.label.cmp(&b.label)); - Some(CompletionResponse::Array(completions)) - } - } - - /// Get template parsing diagnostics for a file. - /// - /// This method uses the cached template errors from Salsa to generate LSP diagnostics. - /// The errors are only re-computed when the file content changes, providing efficient - /// incremental error reporting. - pub fn get_template_diagnostics(&self, uri: &str) -> Vec { - let Some(document) = self.get_document(uri) else { - return vec![]; - }; - - // Only process template files - if document.language_id() != LanguageId::HtmlDjango { - return vec![]; - } - - let file_id = document.file_id(); - let Some(_line_index) = self.get_line_index(uri) else { - return vec![]; - }; - - // Get cached template errors from FileStore - let file_store = self.file_store.lock().unwrap(); - let errors = file_store.get_template_errors(file_id); - - // Convert template errors to LSP diagnostics - errors - .iter() - .map(|error| { - // For now, we'll place all errors at the start of the file - // In a future enhancement, we could use error spans for precise locations - let range = Range { - start: Position { - line: 0, - character: 0, - }, - end: Position { - line: 0, - character: 0, - }, - }; - - Diagnostic { - range, - severity: Some(DiagnosticSeverity::ERROR), - source: Some("djls-templates".to_string()), - message: error.clone(), - ..Default::default() - } - }) - .collect() - } -} - -/// Check if a byte represents a word character (alphanumeric or underscore) -fn is_word_char(byte: u8) -> bool { - byte.is_ascii_alphanumeric() || byte == b'_' -} - -/// Apply text changes to content, handling multiple changes correctly -fn apply_text_changes( - content: &str, - changes: &[TextDocumentContentChangeEvent], - line_index: &LineIndex, -) -> Result { - if changes.is_empty() { - return Ok(content.to_string()); - } - - // Check for full document replacement first - for change in changes { - if change.range.is_none() { - return Ok(change.text.clone()); - } - } - - // Sort changes by start position in reverse order (end to start) - let mut sorted_changes = changes.to_vec(); - sorted_changes.sort_by(|a, b| { - match (a.range, b.range) { - (Some(range_a), Some(range_b)) => { - // Primary sort: by line (reverse) - let line_cmp = range_b.start.line.cmp(&range_a.start.line); - if line_cmp == std::cmp::Ordering::Equal { - // Secondary sort: by character (reverse) - range_b.start.character.cmp(&range_a.start.character) - } else { - line_cmp - } - } - _ => std::cmp::Ordering::Equal, - } - }); - - let mut result = content.to_string(); - - for change in &sorted_changes { - if let Some(range) = change.range { - // Convert UTF-16 positions to UTF-8 offsets - let start_offset = line_index - .offset_utf16(range.start, &result) - .ok_or_else(|| anyhow!("Invalid start position: {:?}", range.start))?; - let end_offset = line_index - .offset_utf16(range.end, &result) - .ok_or_else(|| anyhow!("Invalid end position: {:?}", range.end))?; - - if start_offset as usize > result.len() || end_offset as usize > result.len() { - return Err(anyhow!( - "Offset out of bounds: start={}, end={}, len={}", - start_offset, - end_offset, - result.len() - )); - } - - // Apply the change - result.replace_range(start_offset as usize..end_offset as usize, &change.text); - } - } - - Ok(result) -} - -#[cfg(test)] -mod tests { - use tower_lsp_server::lsp_types::Range; - - use super::*; - - #[test] - fn test_apply_single_character_insertion() { - let content = "Hello world"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 6), Position::new(0, 6))), - range_length: None, - text: "beautiful ".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "Hello beautiful world"); - } - - #[test] - fn test_apply_single_character_deletion() { - let content = "Hello world"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 5), Position::new(0, 6))), - range_length: None, - text: String::new(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "Helloworld"); - } - - #[test] - fn test_apply_multiple_changes_in_reverse_order() { - let content = "line 1\nline 2\nline 3"; - let line_index = LineIndex::new(content); - - // Insert "new " at position (1, 0) and "another " at position (0, 0) - let changes = vec![ - TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), - range_length: None, - text: "another ".to_string(), - }, - TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(1, 0), Position::new(1, 0))), - range_length: None, - text: "new ".to_string(), - }, - ]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "another line 1\nnew line 2\nline 3"); - } - - #[test] - fn test_apply_multiline_replacement() { - let content = "line 1\nline 2\nline 3"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 0), Position::new(2, 6))), - range_length: None, - text: "completely new content".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "completely new content"); - } - - #[test] - fn test_apply_full_document_replacement() { - let content = "old content"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: None, - range_length: None, - text: "brand new content".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "brand new content"); - } - - #[test] - fn test_utf16_line_index_basic() { - let content = "hello world"; - let line_index = LineIndex::new(content); - - // ASCII characters should have 1:1 UTF-8:UTF-16 mapping - let pos = Position::new(0, 6); - let offset = line_index.offset_utf16(pos, content).unwrap(); - assert_eq!(offset, 6); - assert_eq!(&content[6..7], "w"); - } - - #[test] - fn test_utf16_line_index_with_emoji() { - let content = "hello 👋 world"; - let line_index = LineIndex::new(content); - - // 👋 is 2 UTF-16 code units but 4 UTF-8 bytes - let pos_after_emoji = Position::new(0, 8); // UTF-16 position after "hello 👋" - let offset = line_index.offset_utf16(pos_after_emoji, content).unwrap(); - - // Should point to the space before "world" - assert_eq!(offset, 10); // UTF-8 byte offset - assert_eq!(&content[10..11], " "); - } - - #[test] - fn test_utf16_line_index_multiline() { - let content = "first line\nsecond line"; - let line_index = LineIndex::new(content); - - let pos = Position::new(1, 7); // Position at 'l' in "line" on second line - let offset = line_index.offset_utf16(pos, content).unwrap(); - assert_eq!(offset, 18); // 11 (first line + \n) + 7 - assert_eq!(&content[18..19], "l"); - } - - #[test] - fn test_apply_changes_with_emoji() { - let content = "hello 👋 world"; - let line_index = LineIndex::new(content); - - // Insert text after the space following the emoji (UTF-16 position 9) - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 9), Position::new(0, 9))), - range_length: None, - text: "beautiful ".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "hello 👋 beautiful world"); - } - - #[test] - fn test_line_index_utf16_tracking() { - let content = "a👋b"; - let line_index = LineIndex::new(content); - - // Check UTF-16 line starts are tracked correctly - assert_eq!(line_index.line_starts_utf16, vec![0]); - assert_eq!(line_index.length_utf16, 4); // 'a' (1) + 👋 (2) + 'b' (1) = 4 UTF-16 units - assert_eq!(line_index.length, 6); // 'a' (1) + 👋 (4) + 'b' (1) = 6 UTF-8 bytes - } - - #[test] - fn test_edge_case_changes_at_boundaries() { - let content = "abc"; - let line_index = LineIndex::new(content); - - // Insert at beginning - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), - range_length: None, - text: "start".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "startabc"); - - // Insert at end - let line_index = LineIndex::new(content); - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 3), Position::new(0, 3))), - range_length: None, - text: "end".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "abcend"); - } -} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index fb45cc19..9fbb34fc 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,22 +1,33 @@ mod bridge; -mod db; +pub mod db; mod document; -mod vfs; +mod lsp_system; +mod system; -pub use document::ClosingBrace; -pub use document::DocumentStore; -pub use document::LanguageId; -pub use document::LineIndex; -pub use document::TemplateTagContext; -pub use document::TextDocument; +pub use db::Database; +pub use document::{TextDocument, LanguageId}; +pub use system::{FileSystem, StdFileSystem}; + +/// File classification for routing to analyzers. +/// +/// [`FileKind`] determines how a file should be processed by downstream analyzers. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKind { + /// Python source file + Python, + /// Django template file + Template, + /// Other file type + Other, +} /// Stable, compact identifier for files across the subsystem. /// /// [`FileId`] decouples file identity from paths/URIs, providing efficient keys for maps and /// Salsa inputs. Once assigned to a file (via its URI), a [`FileId`] remains stable for the -/// lifetime of the VFS, even if the file's content or metadata changes. +/// lifetime of the system. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub(crate) struct FileId(u32); +pub struct FileId(u32); impl FileId { /// Create a [`FileId`] from a raw u32 value. @@ -27,6 +38,7 @@ impl FileId { /// Get the underlying u32 index value. #[must_use] + #[allow(dead_code)] pub fn index(self) -> u32 { self.0 } diff --git a/crates/djls-workspace/src/lsp_system.rs b/crates/djls-workspace/src/lsp_system.rs new file mode 100644 index 00000000..b03c8e82 --- /dev/null +++ b/crates/djls-workspace/src/lsp_system.rs @@ -0,0 +1,154 @@ +//! LSP-aware file system wrapper that handles overlays +//! +//! This is the KEY pattern from Ruff - the LspSystem wraps a FileSystem +//! and intercepts reads to check for overlays first. This allows unsaved +//! changes to be used without going through Salsa. + +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use url::Url; + +use crate::system::FileSystem; + +/// LSP-aware file system that checks overlays before disk +/// +/// This is the critical piece that makes overlays work efficiently in Ruff's +/// architecture. Instead of updating Salsa for every keystroke, we intercept +/// file reads here and return overlay content when available. +pub struct LspSystem { + /// The underlying file system (usually StdFileSystem) + inner: Arc, + + /// Map of open document URLs to their overlay content + overlays: HashMap, +} + +impl LspSystem { + /// Create a new LspSystem wrapping the given file system + pub fn new(file_system: Arc) -> Self { + Self { + inner: file_system, + overlays: HashMap::new(), + } + } + + /// Set overlay content for a document + pub fn set_overlay(&mut self, url: Url, content: String) { + self.overlays.insert(url, content); + } + + /// Remove overlay content for a document + pub fn remove_overlay(&mut self, url: &Url) { + self.overlays.remove(url); + } + + /// Check if a document has an overlay + pub fn has_overlay(&self, url: &Url) -> bool { + self.overlays.contains_key(url) + } + + /// Get overlay content if it exists + pub fn get_overlay(&self, url: &Url) -> Option<&String> { + self.overlays.get(url) + } + + /// Convert a URL to a file path + fn url_to_path(url: &Url) -> Option { + if url.scheme() == "file" { + url.to_file_path().ok().or_else(|| { + // Fallback for simple conversion + Some(PathBuf::from(url.path())) + }) + } else { + None + } + } +} + +impl FileSystem for LspSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + // First check if we have an overlay for this path + // Convert path to URL for lookup + let url = Url::from_file_path(path) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid path"))?; + + if let Some(content) = self.overlays.get(&url) { + // Return overlay content instead of reading from disk + return Ok(content.clone()); + } + + // No overlay, read from underlying file system + self.inner.read_to_string(path) + } + + fn exists(&self, path: &Path) -> bool { + // Check overlays first + if let Ok(url) = Url::from_file_path(path) { + if self.overlays.contains_key(&url) { + return true; + } + } + + self.inner.exists(path) + } + + fn is_file(&self, path: &Path) -> bool { + // Overlays are always files + if let Ok(url) = Url::from_file_path(path) { + if self.overlays.contains_key(&url) { + return true; + } + } + + self.inner.is_file(path) + } + + fn is_directory(&self, path: &Path) -> bool { + // Overlays are never directories + if let Ok(url) = Url::from_file_path(path) { + if self.overlays.contains_key(&url) { + return false; + } + } + + self.inner.is_directory(path) + } + + fn read_directory(&self, path: &Path) -> io::Result> { + // Overlays don't affect directory listings + self.inner.read_directory(path) + } + + fn metadata(&self, path: &Path) -> io::Result { + // Can't provide metadata for overlays + self.inner.metadata(path) + } +} + +/// Extension trait for working with URL-based overlays +pub trait LspSystemExt { + /// Read file content by URL, checking overlays first + fn read_url(&self, url: &Url) -> io::Result; +} + +impl LspSystemExt for LspSystem { + fn read_url(&self, url: &Url) -> io::Result { + // Check overlays first + if let Some(content) = self.overlays.get(url) { + return Ok(content.clone()); + } + + // Convert URL to path and read from file system + if let Some(path_buf) = Self::url_to_path(url) { + self.inner.read_to_string(&path_buf) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("Cannot convert URL to path: {}", url), + )) + } + } +} \ No newline at end of file diff --git a/crates/djls-workspace/src/system.rs b/crates/djls-workspace/src/system.rs new file mode 100644 index 00000000..04a1b8a2 --- /dev/null +++ b/crates/djls-workspace/src/system.rs @@ -0,0 +1,118 @@ +//! File system abstraction following Ruff's pattern +//! +//! This module provides the FileSystem trait that abstracts file I/O operations. +//! This allows the LSP to work with both real files and in-memory overlays. + +use std::io; +use std::path::Path; + +/// Trait for file system operations +/// +/// This follows Ruff's pattern of abstracting file system operations behind a trait, +/// allowing different implementations for testing, in-memory operation, and real file access. +pub trait FileSystem: Send + Sync { + /// Read the entire contents of a file + fn read_to_string(&self, path: &Path) -> io::Result; + + /// Check if a path exists + fn exists(&self, path: &Path) -> bool; + + /// Check if a path is a file + fn is_file(&self, path: &Path) -> bool; + + /// Check if a path is a directory + fn is_directory(&self, path: &Path) -> bool; + + /// List directory contents + fn read_directory(&self, path: &Path) -> io::Result>; + + /// Get file metadata (size, modified time, etc.) + fn metadata(&self, path: &Path) -> io::Result; +} + +/// Standard file system implementation that uses std::fs +pub struct StdFileSystem; + +impl FileSystem for StdFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + std::fs::read_to_string(path) + } + + fn exists(&self, path: &Path) -> bool { + path.exists() + } + + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + fn is_directory(&self, path: &Path) -> bool { + path.is_dir() + } + + fn read_directory(&self, path: &Path) -> io::Result> { + let mut entries = Vec::new(); + for entry in std::fs::read_dir(path)? { + entries.push(entry?.path()); + } + Ok(entries) + } + + fn metadata(&self, path: &Path) -> io::Result { + std::fs::metadata(path) + } +} + +/// In-memory file system for testing +#[cfg(test)] +pub struct MemoryFileSystem { + files: std::collections::HashMap, +} + +#[cfg(test)] +impl MemoryFileSystem { + pub fn new() -> Self { + Self { + files: std::collections::HashMap::new(), + } + } + + pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { + self.files.insert(path, content); + } +} + +#[cfg(test)] +impl FileSystem for MemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + // Simplified for testing - no directories in memory filesystem + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + // Simplified for testing + Ok(Vec::new()) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Metadata not supported in memory filesystem", + )) + } +} \ No newline at end of file diff --git a/crates/djls-workspace/src/test_db.rs b/crates/djls-workspace/src/test_db.rs new file mode 100644 index 00000000..92683d5f --- /dev/null +++ b/crates/djls-workspace/src/test_db.rs @@ -0,0 +1,25 @@ +//! Test module to explore Salsa thread safety + +#[cfg(test)] +mod tests { + use crate::db::Database; + use std::thread; + + #[test] + fn test_database_clone() { + let db = Database::new(); + let _db2 = db.clone(); + println!("✅ Database can be cloned"); + } + + #[test] + #[ignore] // This will fail + fn test_database_send() { + let db = Database::new(); + let db2 = db.clone(); + + thread::spawn(move || { + let _ = db2; + }).join().unwrap(); + } +} diff --git a/crates/djls-workspace/src/vfs/mod.rs b/crates/djls-workspace/src/vfs/mod.rs deleted file mode 100644 index 2b4b22f2..00000000 --- a/crates/djls-workspace/src/vfs/mod.rs +++ /dev/null @@ -1,367 +0,0 @@ -//! Change-tracked, concurrent virtual file system keyed by [`FileId`]. -//! -//! The VFS provides thread-safe, identity-stable storage with cheap change detection -//! and snapshotting. Downstream systems consume snapshots to avoid locking and to -//! batch updates. - -mod watcher; - -use std::collections::hash_map::DefaultHasher; -use std::collections::HashMap; -use std::fs; -use std::hash::Hash; -use std::hash::Hasher; -use std::sync::atomic::AtomicU32; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; -use std::sync::Arc; - -use anyhow::anyhow; -use anyhow::Result; -use camino::Utf8PathBuf; -use dashmap::DashMap; -use url::Url; -use watcher::VfsWatcher; -use watcher::WatchConfig; -use watcher::WatchEvent; - -use super::FileId; - -/// Monotonic counter representing global VFS state. -/// -/// [`Revision`] increments whenever file content changes occur in the VFS. -/// This provides a cheap way to detect if any changes have occurred since -/// a previous snapshot was taken. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default, PartialOrd, Ord)] -pub(crate) struct Revision(u64); - -impl Revision { - /// Create a [`Revision`] from a raw u64 value. - #[must_use] - fn from_raw(raw: u64) -> Self { - Revision(raw) - } - - /// Get the underlying u64 value. - #[must_use] - fn value(self) -> u64 { - self.0 - } -} - -/// File classification at the VFS layer. -/// -/// [`FileKind`] determines how a file should be processed by downstream analyzers. -/// This classification is performed when files are first ingested into the VFS. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub enum FileKind { - /// Python source file - Python, - /// Django template file - Template, - /// Other file type - Other, -} - -/// Metadata associated with a file in the VFS. -/// -/// [`FileMeta`] contains all non-content information about a file, including its -/// identity (URI), filesystem path, and classification. -#[derive(Clone, Debug)] -pub(crate) struct FileMeta { - /// The file's URI (typically file:// scheme) - uri: Url, - /// The file's path in the filesystem - path: Utf8PathBuf, - /// Classification for routing to analyzers - pub kind: FileKind, -} - -/// Source of text content in the VFS. -/// -/// [`TextSource`] tracks where file content originated from, which is useful for -/// debugging and understanding the current state of the VFS. All variants hold -/// `Arc` for efficient sharing. -#[derive(Clone)] -pub(crate) enum TextSource { - /// Content loaded from disk - Disk(Arc), - /// Content from LSP client overlay (in-memory edits) - Overlay(Arc), - /// Content generated programmatically - Generated(Arc), -} - -/// Content hash for efficient change detection. -/// -/// [`FileHash`] encapsulates the hashing logic used to detect when file content -/// has changed, avoiding unnecessary recomputation in downstream systems like Salsa. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -struct FileHash(u64); - -impl FileHash { - /// Compute hash from text source content. - fn from_text_source(src: &TextSource) -> Self { - let s: &str = match src { - TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, - }; - let mut h = DefaultHasher::new(); - s.hash(&mut h); - Self(h.finish()) - } - - /// Check if this hash differs from another, indicating content changed. - fn differs_from(self, other: Self) -> bool { - self.0 != other.0 - } - - /// Get raw hash value (for debugging/logging). - fn raw(self) -> u64 { - self.0 - } -} - -/// Complete record of a file in the VFS. -/// -/// [`FileRecord`] combines metadata, current text content, and a content hash -/// for efficient change detection. -#[derive(Clone)] -pub(crate) struct FileRecord { - /// File metadata (URI, path, kind, version) - pub meta: FileMeta, - /// Current text content and its source - text: TextSource, - /// Hash of current content for change detection - hash: FileHash, -} - -/// Thread-safe virtual file system with change tracking. -/// -/// [`Vfs`] provides concurrent access to file content with stable [`FileId`] assignment, -/// content hashing for change detection, and atomic snapshot generation. It uses -/// `DashMap` for lock-free concurrent access and atomic counters for revision tracking. -pub struct Vfs { - /// Atomic counter for generating unique [`FileId`]s - next_file_id: AtomicU32, - /// Map from URI to [`FileId`] for deduplication - by_uri: DashMap, - /// Map from [`FileId`] to [`FileRecord`] for content storage - files: DashMap, - /// Global revision counter, incremented on content changes - head: AtomicU64, - /// Optional file system watcher for external change detection - watcher: std::sync::Mutex>, - /// Map from filesystem path to [`FileId`] for watcher events - by_path: DashMap, -} - -impl Vfs { - /// Get or create a [`FileId`] for the given URI. - /// - /// Returns the existing [`FileId`] if the URI is already known, or creates a new - /// [`FileRecord`] with the provided metadata and text. This method computes and - /// stores a content hash for change detection. - pub(crate) fn intern_file( - &self, - uri: Url, - path: Utf8PathBuf, - kind: FileKind, - text: TextSource, - ) -> FileId { - if let Some(id) = self.by_uri.get(&uri).map(|entry| *entry) { - return id; - } - let id = FileId(self.next_file_id.fetch_add(1, Ordering::SeqCst)); - let meta = FileMeta { - uri: uri.clone(), - path: path.clone(), - kind, - }; - let hash = FileHash::from_text_source(&text); - self.by_uri.insert(uri, id); - self.by_path.insert(path, id); - self.files.insert(id, FileRecord { meta, text, hash }); - id - } - - /// Set overlay text for a file, typically from LSP didChange events. - /// - /// Updates the file's text to an Overlay variant with the new content. - /// Only increments the global revision if the content actually changed - /// (detected via hash comparison). - /// - /// Returns a tuple of (new global revision, whether content changed). - pub(crate) fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { - let mut rec = self - .files - .get_mut(&id) - .ok_or_else(|| anyhow!("unknown file: {:?}", id))?; - let next = TextSource::Overlay(new_text); - let new_hash = FileHash::from_text_source(&next); - let changed = new_hash.differs_from(rec.hash); - if changed { - rec.text = next; - rec.hash = new_hash; - self.head.fetch_add(1, Ordering::SeqCst); - } - Ok(( - Revision::from_raw(self.head.load(Ordering::SeqCst)), - changed, - )) - } - - /// Create an immutable snapshot of the current VFS state. - /// - /// Materializes a consistent view of all files for downstream consumers. - /// The snapshot includes the current revision and a clone of all file records. - /// This operation is relatively cheap due to `Arc` sharing of text content. - pub(crate) fn snapshot(&self) -> VfsSnapshot { - VfsSnapshot { - revision: Revision::from_raw(self.head.load(Ordering::SeqCst)), - files: self - .files - .iter() - .map(|entry| (*entry.key(), entry.value().clone())) - .collect(), - } - } - - /// Enable file system watching with the given configuration. - /// - /// This starts monitoring the specified root directories for external changes. - /// Returns an error if file watching is disabled in the config or fails to start. - pub fn enable_file_watching(&self, config: WatchConfig) -> Result<()> { - let watcher = VfsWatcher::new(config)?; - *self - .watcher - .lock() - .map_err(|e| anyhow!("Failed to lock watcher mutex: {}", e))? = Some(watcher); - Ok(()) - } - - /// Process pending file system events from the watcher. - /// - /// This should be called periodically to sync external file changes into the VFS. - /// Returns the number of files that were updated. - pub fn process_file_events(&self) -> usize { - // Get events from the watcher - let events = { - let Ok(guard) = self.watcher.lock() else { - return 0; // Return 0 if mutex is poisoned - }; - if let Some(watcher) = guard.as_ref() { - watcher.try_recv_events() - } else { - return 0; - } - }; - - let mut updated_count = 0; - - for event in events { - match event { - WatchEvent::Modified(path) | WatchEvent::Created(path) => { - if let Err(e) = self.load_from_disk(&path) { - eprintln!("Failed to load file from disk: {path}: {e}"); - } else { - updated_count += 1; - } - } - WatchEvent::Deleted(path) => { - // For now, we don't remove deleted files from VFS - // This maintains stable `FileId`s for consumers - eprintln!("File deleted (keeping in VFS): {path}"); - } - WatchEvent::Renamed { from, to } => { - // Handle rename by updating the path mapping - if let Some(file_id) = self.by_path.remove(&from).map(|(_, id)| id) { - self.by_path.insert(to.clone(), file_id); - if let Err(e) = self.load_from_disk(&to) { - eprintln!("Failed to load renamed file: {to}: {e}"); - } else { - updated_count += 1; - } - } - } - } - } - updated_count - } - - /// Load a file's content from disk and update the VFS. - /// - /// This method reads the file from the filesystem and updates the VFS entry - /// if the content has changed. It's used by the file watcher to sync external changes. - fn load_from_disk(&self, path: &Utf8PathBuf) -> Result<()> { - // Check if we have this file tracked - if let Some(file_id) = self.by_path.get(path).map(|entry| *entry.value()) { - // Read content from disk - let content = fs::read_to_string(path.as_std_path()) - .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; - - let new_text = TextSource::Disk(Arc::from(content.as_str())); - let new_hash = FileHash::from_text_source(&new_text); - - // Update the file if content changed - if let Some(mut record) = self.files.get_mut(&file_id) { - if new_hash.differs_from(record.hash) { - record.text = new_text; - record.hash = new_hash; - self.head.fetch_add(1, Ordering::SeqCst); - } - } - } - Ok(()) - } - - /// Check if file watching is currently enabled. - pub fn is_file_watching_enabled(&self) -> bool { - self.watcher.lock().map(|g| g.is_some()).unwrap_or(false) // Return false if mutex is poisoned - } -} - -impl Default for Vfs { - fn default() -> Self { - Self { - next_file_id: AtomicU32::new(0), - by_uri: DashMap::new(), - files: DashMap::new(), - head: AtomicU64::new(0), - watcher: std::sync::Mutex::new(None), - by_path: DashMap::new(), - } - } -} - -/// Immutable snapshot view of the VFS at a specific revision. -/// -/// [`VfsSnapshot`] provides a consistent view of all files for downstream consumers, -/// avoiding the need for locking during processing. Snapshots are created atomically -/// and can be safely shared across threads. -#[derive(Clone)] -pub(crate) struct VfsSnapshot { - /// The global revision at the time of snapshot - revision: Revision, - /// All files in the VFS at snapshot time - pub files: HashMap, -} - -impl VfsSnapshot { - /// Get the text content of a file in this snapshot. - /// - /// Returns `None` if the [`FileId`] is not present in the snapshot. - #[must_use] - pub fn get_text(&self, id: FileId) -> Option> { - self.files.get(&id).map(|r| match &r.text { - TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s.clone(), - }) - } - - /// Get the metadata for a file in this snapshot. - /// - /// Returns `None` if the [`FileId`] is not present in the snapshot. - #[must_use] - pub fn meta(&self, id: FileId) -> Option<&FileMeta> { - self.files.get(&id).map(|r| &r.meta) - } -} diff --git a/crates/djls-workspace/src/vfs/watcher.rs b/crates/djls-workspace/src/vfs/watcher.rs deleted file mode 100644 index aee04676..00000000 --- a/crates/djls-workspace/src/vfs/watcher.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! File system watching for VFS synchronization. -//! -//! This module provides file system watching capabilities to detect external changes -//! and synchronize them with the VFS. It uses cross-platform file watching with -//! debouncing to handle rapid changes efficiently. - -use std::collections::HashMap; -use std::sync::mpsc; -use std::thread; -use std::time::Duration; -use std::time::Instant; - -use anyhow::anyhow; -use anyhow::Result; -use camino::Utf8PathBuf; -use notify::Config; -use notify::Event; -use notify::EventKind; -use notify::RecommendedWatcher; -use notify::RecursiveMode; -use notify::Watcher; - -/// Event types that can occur in the file system. -/// -/// [`WatchEvent`] represents the different types of file system changes that -/// the watcher can detect and process. -#[derive(Clone, Debug, PartialEq)] -pub enum WatchEvent { - /// A file was modified (content changed) - Modified(Utf8PathBuf), - /// A new file was created - Created(Utf8PathBuf), - /// A file was deleted - Deleted(Utf8PathBuf), - /// A file was renamed from one path to another - Renamed { from: Utf8PathBuf, to: Utf8PathBuf }, -} - -/// Configuration for the file watcher. -/// -/// [`WatchConfig`] controls how the file watcher operates, including what -/// directories to watch and how to filter events. -#[derive(Clone, Debug)] -pub struct WatchConfig { - /// Whether file watching is enabled - pub enabled: bool, - /// Root directories to watch recursively - pub roots: Vec, - /// Debounce time in milliseconds (collect events for this duration before processing) - pub debounce_ms: u64, - /// File patterns to include (e.g., ["*.py", "*.html"]) - pub include_patterns: Vec, - /// File patterns to exclude (e.g., ["__pycache__", ".git", "*.pyc"]) - pub exclude_patterns: Vec, -} - -// TODO: Allow for user config instead of hardcoding defaults -impl Default for WatchConfig { - fn default() -> Self { - Self { - enabled: true, - roots: Vec::new(), - debounce_ms: 250, - include_patterns: vec!["*.py".to_string(), "*.html".to_string()], - exclude_patterns: vec![ - "__pycache__".to_string(), - ".git".to_string(), - ".pyc".to_string(), - "node_modules".to_string(), - ".venv".to_string(), - "venv".to_string(), - ], - } - } -} - -/// File system watcher for VFS synchronization. -/// -/// [`VfsWatcher`] monitors the file system for changes and provides a channel -/// for consuming batched events. It handles debouncing and filtering internally. -pub struct VfsWatcher { - /// The underlying file system watcher - _watcher: RecommendedWatcher, - /// Receiver for processed watch events - rx: mpsc::Receiver>, - /// Configuration for the watcher - config: WatchConfig, - /// Handle to the background processing thread - _handle: thread::JoinHandle<()>, -} - -impl VfsWatcher { - /// Create a new file watcher with the given configuration. - /// - /// This starts watching the specified root directories and begins processing - /// events in a background thread. - pub fn new(config: WatchConfig) -> Result { - if !config.enabled { - return Err(anyhow!("File watching is disabled")); - } - - let (event_tx, event_rx) = mpsc::channel(); - let (watch_tx, watch_rx) = mpsc::channel(); - - // Create the file system watcher - let mut watcher = RecommendedWatcher::new( - move |res: notify::Result| { - if let Ok(event) = res { - let _ = event_tx.send(event); - } - }, - Config::default(), - )?; - - // Watch all root directories - for root in &config.roots { - let std_path = root.as_std_path(); - if std_path.exists() { - watcher.watch(std_path, RecursiveMode::Recursive)?; - } - } - - // Spawn background thread for event processing - let config_clone = config.clone(); - let handle = thread::spawn(move || { - Self::process_events(&event_rx, &watch_tx, &config_clone); - }); - - Ok(Self { - _watcher: watcher, - rx: watch_rx, - config, - _handle: handle, - }) - } - - /// Get the next batch of processed watch events. - /// - /// This is a non-blocking operation that returns immediately. If no events - /// are available, it returns an empty vector. - #[must_use] - pub fn try_recv_events(&self) -> Vec { - self.rx.try_recv().unwrap_or_default() - } - - /// Background thread function for processing raw file system events. - /// - /// This function handles debouncing, filtering, and batching of events before - /// sending them to the main thread for VFS synchronization. - fn process_events( - event_rx: &mpsc::Receiver, - watch_tx: &mpsc::Sender>, - config: &WatchConfig, - ) { - let mut pending_events: HashMap = HashMap::new(); - let mut last_batch_time = Instant::now(); - let debounce_duration = Duration::from_millis(config.debounce_ms); - - loop { - // Try to receive events with a timeout for batching - match event_rx.recv_timeout(Duration::from_millis(50)) { - Ok(event) => { - // Process the raw notify event into our WatchEvent format - if let Some(watch_events) = Self::convert_notify_event(event, config) { - for watch_event in watch_events { - let path = Self::get_event_path(&watch_event); - // Only keep the latest event for each path - pending_events.insert(path.clone(), watch_event); - } - } - } - Err(mpsc::RecvTimeoutError::Timeout) => { - // Timeout - check if we should flush pending events - } - Err(mpsc::RecvTimeoutError::Disconnected) => { - // Channel disconnected, exit the thread - break; - } - } - - // Check if we should flush pending events - if !pending_events.is_empty() && last_batch_time.elapsed() >= debounce_duration { - let events: Vec = pending_events.values().cloned().collect(); - if watch_tx.send(events).is_err() { - // Main thread disconnected, exit - break; - } - pending_events.clear(); - last_batch_time = Instant::now(); - } - } - } - - /// Convert a [`notify::Event`] into our [`WatchEvent`] format. - fn convert_notify_event(event: Event, config: &WatchConfig) -> Option> { - let mut watch_events = Vec::new(); - - for path in event.paths { - if let Ok(utf8_path) = Utf8PathBuf::try_from(path) { - if Self::should_include_path_static(&utf8_path, config) { - match event.kind { - EventKind::Create(_) => watch_events.push(WatchEvent::Created(utf8_path)), - EventKind::Modify(_) => watch_events.push(WatchEvent::Modified(utf8_path)), - EventKind::Remove(_) => watch_events.push(WatchEvent::Deleted(utf8_path)), - _ => {} // Ignore other event types for now - } - } - } - } - - if watch_events.is_empty() { - None - } else { - Some(watch_events) - } - } - - /// Static version of should_include_path for use in convert_notify_event. - fn should_include_path_static(path: &Utf8PathBuf, config: &WatchConfig) -> bool { - let path_str = path.as_str(); - - // Check exclude patterns first - for pattern in &config.exclude_patterns { - if path_str.contains(pattern) { - return false; - } - } - - // If no include patterns, include everything (that's not excluded) - if config.include_patterns.is_empty() { - return true; - } - - // Check include patterns - for pattern in &config.include_patterns { - if let Some(extension) = pattern.strip_prefix("*.") { - if path_str.ends_with(extension) { - return true; - } - } else if path_str.contains(pattern) { - return true; - } - } - - false - } - - /// Extract the path from a [`WatchEvent`]. - fn get_event_path(event: &WatchEvent) -> &Utf8PathBuf { - match event { - WatchEvent::Modified(path) | WatchEvent::Created(path) | WatchEvent::Deleted(path) => { - path - } - WatchEvent::Renamed { to, .. } => to, - } - } -} - -impl Drop for VfsWatcher { - fn drop(&mut self) { - // The background thread will exit when the event channel is dropped - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_watch_config_default() { - let config = WatchConfig::default(); - assert!(config.enabled); - assert_eq!(config.debounce_ms, 250); - assert!(config.include_patterns.contains(&"*.py".to_string())); - assert!(config.exclude_patterns.contains(&".git".to_string())); - } - - #[test] - fn test_should_include_path() { - let config = WatchConfig::default(); - - // Should include Python files - assert!(VfsWatcher::should_include_path_static( - &Utf8PathBuf::from("test.py"), - &config - )); - - // Should include HTML files - assert!(VfsWatcher::should_include_path_static( - &Utf8PathBuf::from("template.html"), - &config - )); - - // Should exclude .git files - assert!(!VfsWatcher::should_include_path_static( - &Utf8PathBuf::from(".git/config"), - &config - )); - - // Should exclude __pycache__ files - assert!(!VfsWatcher::should_include_path_static( - &Utf8PathBuf::from("__pycache__/test.pyc"), - &config - )); - } - - #[test] - fn test_watch_event_types() { - let path1 = Utf8PathBuf::from("test.py"); - let path2 = Utf8PathBuf::from("new.py"); - - let modified = WatchEvent::Modified(path1.clone()); - let created = WatchEvent::Created(path1.clone()); - let deleted = WatchEvent::Deleted(path1.clone()); - let renamed = WatchEvent::Renamed { - from: path1, - to: path2, - }; - - // Test that events can be created and compared - assert_ne!(modified, created); - assert_ne!(created, deleted); - assert_ne!(deleted, renamed); - } -} From 96e0b814179cfa6793d42cd38043e31bc6986b9e Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 28 Aug 2025 22:54:41 -0500 Subject: [PATCH 13/56] wowza --- ARCHITECTURE_INSIGHTS.md | 96 +++ REVISION_TRACKING_ARCHITECTURE.md | 341 ++++++++++ RUFF_ARCHITECTURE_INSIGHTS.md | 77 +++ check_ruff_pattern.md | 94 +++ crates/djls-server/src/server.rs | 100 ++- crates/djls-server/src/session.rs | 637 ++++++++++++++++-- crates/djls-workspace/src/bridge.rs | 126 ---- crates/djls-workspace/src/db.rs | 415 +++++++----- crates/djls-workspace/src/document.rs | 216 ++++++ .../djls-workspace/src/document/line_index.rs | 90 --- crates/djls-workspace/src/document/mod.rs | 130 ---- crates/djls-workspace/src/fs.rs | 269 ++++++++ .../src/{document => }/language.rs | 6 +- crates/djls-workspace/src/lib.rs | 49 +- crates/djls-workspace/src/lsp_system.rs | 154 ----- crates/djls-workspace/src/system.rs | 118 ---- .../src/{document => }/template.rs | 0 crates/djls-workspace/src/test_db.rs | 25 - task_order.md | 61 ++ 19 files changed, 2091 insertions(+), 913 deletions(-) create mode 100644 ARCHITECTURE_INSIGHTS.md create mode 100644 REVISION_TRACKING_ARCHITECTURE.md create mode 100644 RUFF_ARCHITECTURE_INSIGHTS.md create mode 100644 check_ruff_pattern.md delete mode 100644 crates/djls-workspace/src/bridge.rs create mode 100644 crates/djls-workspace/src/document.rs delete mode 100644 crates/djls-workspace/src/document/line_index.rs delete mode 100644 crates/djls-workspace/src/document/mod.rs create mode 100644 crates/djls-workspace/src/fs.rs rename crates/djls-workspace/src/{document => }/language.rs (79%) delete mode 100644 crates/djls-workspace/src/lsp_system.rs delete mode 100644 crates/djls-workspace/src/system.rs rename crates/djls-workspace/src/{document => }/template.rs (100%) delete mode 100644 crates/djls-workspace/src/test_db.rs create mode 100644 task_order.md diff --git a/ARCHITECTURE_INSIGHTS.md b/ARCHITECTURE_INSIGHTS.md new file mode 100644 index 00000000..b3dcdf0b --- /dev/null +++ b/ARCHITECTURE_INSIGHTS.md @@ -0,0 +1,96 @@ +# Architecture Insights from Ruff Investigation + +## Key Discovery: Two-Layer Architecture + +### The Problem +- LSP documents change frequently (every keystroke) +- Salsa invalidation is expensive +- Tower-lsp requires Send+Sync, but Salsa Database contains RefCell/UnsafeCell + +### The Solution (Ruff Pattern) + +#### Layer 1: LSP Document Management (Outside Salsa) +- Store overlays in `Session` using `Arc>` +- TextDocument contains actual content, version, language_id +- Changes are immediate, no Salsa invalidation + +#### Layer 2: Salsa Incremental Computation +- Database is pure Salsa, no file storage +- Queries read through FileSystem trait +- LspFileSystem intercepts reads, returns overlay or disk content + +### Critical Insights + +1. **Overlays NEVER become Salsa inputs directly** + - They're intercepted at FileSystem::read_to_string() time + - Salsa only knows "something changed", reads content lazily + +2. **StorageHandle Pattern (for tower-lsp)** + - Session stores `StorageHandle` not Database directly + - StorageHandle IS Send+Sync even though Database isn't + - Create Database instances on-demand: `session.db()` + +3. **File Management Location** + - WRONG: Store files in Database (what we initially did) + - RIGHT: Store overlays in Session, Database is pure Salsa + +4. **The Bridge** + - LspFileSystem has Arc to same overlays as Session + - When Salsa queries need content, they call FileSystem + - FileSystem checks overlays first, falls back to disk + +### Implementation Flow + +1. **did_open/did_change/did_close** → Update overlays in Session +2. **notify_file_changed()** → Tell Salsa something changed +3. **Salsa query executes** → Calls FileSystem::read_to_string() +4. **LspFileSystem intercepts** → Returns overlay if exists, else disk +5. **Query gets content** → Without knowing about LSP/overlays + +### Why This Works + +- Fast: Overlay updates don't trigger Salsa invalidation cascade +- Thread-safe: DashMap for overlays, StorageHandle for Database +- Clean separation: LSP concerns vs computation concerns +- Efficient: Salsa caching still works, just reads through FileSystem + +### Tower-lsp vs lsp-server + +- **Ruff uses lsp-server**: No Send+Sync requirement, can store Database directly +- **We use tower-lsp**: Requires Send+Sync, must use StorageHandle pattern +- Both achieve same result, different mechanisms + +## Critical Implementation Details (From Ruff Expert) + +### The Revision Dependency Trick + +**THE MOST CRITICAL INSIGHT**: In the `source_text` tracked function, calling `file.revision(db)` is what creates the Salsa dependency chain: + +```rust +#[salsa::tracked] +pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { + // THIS LINE IS CRITICAL - Creates Salsa dependency on revision! + let _ = file.revision(db); + + // Now read from FileSystem (checks overlays first) + db.read_file_content(file.path(db)) +} +``` + +Without that `file.revision(db)` call, revision changes won't trigger invalidation! + +### Key Implementation Points + +1. **Files have no text**: SourceFile inputs only have `path` and `revision`, never `text` +2. **Revision bumping triggers invalidation**: Change revision → source_text invalidated → dependent queries invalidated +3. **Files created lazily**: Don't pre-create, let them be created on first access +4. **Simple counters work**: Revision can be a simple u64 counter, doesn't need timestamps +5. **StorageHandle update required**: After DB modifications in LSP handlers, must update the handle + +### Common Pitfalls + +- **Forgetting the revision dependency** - Without `file.revision(db)`, nothing invalidates +- **Storing text in Salsa inputs** - Breaks the entire pattern +- **Not bumping revision on overlay changes** - Queries won't see new content +- **Creating files eagerly** - Unnecessary and inefficient + diff --git a/REVISION_TRACKING_ARCHITECTURE.md b/REVISION_TRACKING_ARCHITECTURE.md new file mode 100644 index 00000000..202c1186 --- /dev/null +++ b/REVISION_TRACKING_ARCHITECTURE.md @@ -0,0 +1,341 @@ +# Revision Tracking Architecture for Django Language Server + +## Overview + +This document captures the complete understanding of how to implement revision tracking for task-112, based on extensive discussions with a Ruff architecture expert. The goal is to connect the Session's overlay system with Salsa's query invalidation mechanism through per-file revision tracking. + +## The Critical Breakthrough: Dual-Layer Architecture + +### The Confusion We Had + +We conflated two different concepts: +1. **Database struct** - The Rust struct that implements the Salsa database trait +2. **Salsa database** - The actual Salsa storage system with inputs/queries + +### The Key Insight + +**Database struct ≠ Salsa database** + +The Database struct can contain: +- Salsa storage (the actual Salsa database) +- Additional non-Salsa data structures (like file tracking) + +## The Architecture Pattern (From Ruff) + +### Ruff's Implementation + +```rust +// Ruff's Database contains BOTH Salsa and non-Salsa data +pub struct ProjectDatabase { + storage: salsa::Storage, // Salsa's data + files: Files, // NOT Salsa data, but in Database struct! +} + +// Files is Arc-wrapped for cheap cloning +#[derive(Clone)] +pub struct Files { + inner: Arc, // Shared across clones +} + +struct FilesInner { + system_by_path: FxDashMap, // Thread-safe +} +``` + +### Our Implementation + +```rust +// Django LS Database structure +#[derive(Clone)] +pub struct Database { + storage: salsa::Storage, + files: Arc>, // Arc makes cloning cheap! +} + +// Session still uses StorageHandle for tower-lsp +pub struct Session { + db_handle: StorageHandle, // Still needed! + overlays: Arc>, // LSP document state +} +``` + +## Why This Works with Send+Sync Requirements + +1. **Arc is Send+Sync** - Thread-safe by design +2. **Cloning is cheap** - Only clones the Arc pointer (8 bytes) +3. **Persistence across clones** - All clones share the same DashMap +4. **StorageHandle compatible** - Database remains clonable and Send+Sync + +## Implementation Details + +### 1. Database Implementation + +```rust +impl Database { + pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { + self.files + .entry(path.clone()) + .or_insert_with(|| { + // Create Salsa input with initial revision 0 + SourceFile::new(self, path, 0) + }) + .clone() + } +} + +impl Clone for Database { + fn clone(&self) -> self { + Self { + storage: self.storage.clone(), // Salsa handles this + files: self.files.clone(), // Just clones Arc! + } + } +} +``` + +### 2. The Critical Pattern for Every Overlay Change + +```rust +pub fn handle_overlay_change(session: &mut Session, url: Url, content: String) { + // 1. Extract database from StorageHandle + let mut db = session.db_handle.get(); + + // 2. Update overlay in Session + session.overlays.insert(url.clone(), TextDocument::new(content)); + + // 3. Get or create file in Database + let path = path_from_url(&url); + let file = db.get_or_create_file(path); + + // 4. Bump revision (simple incrementing counter) + let current_rev = file.revision(&db); + file.set_revision(&mut db).to(current_rev + 1); + + // 5. Update StorageHandle with modified database + session.db_handle.update(db); // CRITICAL! +} +``` + +### 3. LSP Handler Updates + +#### did_open + +```rust +pub fn did_open(&mut self, params: DidOpenTextDocumentParams) { + let mut db = self.session.db_handle.get(); + + // Set overlay + self.session.overlays.insert( + params.text_document.uri.clone(), + TextDocument::new(params.text_document.text) + ); + + // Create file with initial revision 0 + let path = path_from_url(¶ms.text_document.uri); + db.get_or_create_file(path); // Creates with revision 0 + + self.session.db_handle.update(db); +} +``` + +#### did_change + +```rust +pub fn did_change(&mut self, params: DidChangeTextDocumentParams) { + let mut db = self.session.db_handle.get(); + + // Update overlay + let new_content = params.content_changes[0].text.clone(); + self.session.overlays.insert( + params.text_document.uri.clone(), + TextDocument::new(new_content) + ); + + // Bump revision + let path = path_from_url(¶ms.text_document.uri); + let file = db.get_or_create_file(path); + let new_rev = file.revision(&db) + 1; + file.set_revision(&mut db).to(new_rev); + + self.session.db_handle.update(db); +} +``` + +#### did_close + +```rust +pub fn did_close(&mut self, params: DidCloseTextDocumentParams) { + let mut db = self.session.db_handle.get(); + + // Remove overlay + self.session.overlays.remove(¶ms.text_document.uri); + + // Bump revision to trigger re-read from disk + let path = path_from_url(¶ms.text_document.uri); + if let Some(file) = db.files.get(&path) { + let new_rev = file.revision(&db) + 1; + file.set_revision(&mut db).to(new_rev); + } + + self.session.db_handle.update(db); +} +``` + +## Key Implementation Guidelines from Ruff Expert + +### 1. File Tracking Location + +- Store in Database struct (not Session) +- Use Arc for thread-safety and cheap cloning +- This keeps file tracking close to where it's used + +### 2. Revision Management + +- Use simple incrementing counter per file (not timestamps) +- Each file has independent revision tracking +- Revision just needs to change, doesn't need to be monotonic +- Example: `file.set_revision(&mut db).to(current + 1)` + +### 3. Lazy File Creation + +Files should be created: +- On did_open (via get_or_create_file) +- On first query access if needed +- NOT eagerly for all possible files + +### 4. File Lifecycle + +- **On open**: Create file with revision 0 +- **On change**: Bump revision to trigger invalidation +- **On close**: Keep file alive, bump revision for re-read from disk +- **Never remove**: Files stay in tracking even after close + +### 5. Batch Changes for Performance + +When possible, batch multiple changes: + +```rust +pub fn apply_batch_changes(&mut self, changes: Vec) { + let mut db = self.session.db_handle.get(); + + for change in changes { + // Process each change + let file = db.get_or_create_file(change.path); + file.set_revision(&mut db).to(file.revision(&db) + 1); + } + + // Single StorageHandle update at the end + self.session.db_handle.update(db); +} +``` + +### 6. Thread Safety with DashMap + +Use DashMap's atomic entry API: + +```rust +self.files.entry(path.clone()) + .and_modify(|file| { + // Modify existing + file.set_revision(db).to(new_rev); + }) + .or_insert_with(|| { + // Create new + SourceFile::builder(path) + .revision(0) + .new(db) + }); +``` + +## Critical Pitfalls to Avoid + +1. **NOT BUMPING REVISION** - Every overlay change MUST bump revision or Salsa won't invalidate +2. **FORGETTING STORAGEHANDLE UPDATE** - Must call `session.db_handle.update(db)` after changes +3. **CREATING FILES EAGERLY** - Let files be created lazily on first access +4. **USING TIMESTAMPS** - Simple incrementing counter is sufficient +5. **REMOVING FILES** - Keep files alive even after close, just bump revision + +## The Two-Layer Model + +### Layer 1: Non-Salsa (but in Database struct) +- `Arc>` - File tracking +- Thread-safe via Arc+DashMap +- Cheap to clone via Arc +- Acts as a lookup table + +### Layer 2: Salsa Inputs +- `SourceFile` entities created via `SourceFile::new(db)` +- Have revision fields for invalidation +- Tracked by Salsa's dependency system +- Invalidation cascades through dependent queries + +## Complete Architecture Summary + +| Component | Contains | Purpose | +|-----------|----------|---------| +| **Database** | `storage` + `Arc>` | Salsa queries + file tracking | +| **Session** | `StorageHandle` + `Arc>` | LSP state + overlays | +| **StorageHandle** | `Arc>>` | Bridge for tower-lsp lifetime requirements | +| **SourceFile** | Salsa input with path + revision | Triggers query invalidation | + +## The Flow + +1. **LSP request arrives** → tower-lsp handler +2. **Extract database** → `db = session.db_handle.get()` +3. **Update overlay** → `session.overlays.insert(url, content)` +4. **Get/create file** → `db.get_or_create_file(path)` +5. **Bump revision** → `file.set_revision(&mut db).to(current + 1)` +6. **Update handle** → `session.db_handle.update(db)` +7. **Salsa invalidates** → `source_text` query re-executes +8. **Queries see new content** → Through overlay-aware FileSystem + +## Why StorageHandle is Still Essential + +1. **tower-lsp requirement**: Needs 'static lifetime for async handlers +2. **Snapshot management**: Safe extraction and update of database +3. **Thread safety**: Bridges async boundaries safely +4. **Atomic updates**: Ensures consistent state transitions + +## Testing Strategy + +1. **Revision bumping**: Verify each overlay operation bumps revision +2. **Invalidation cascade**: Ensure source_text re-executes after revision bump +3. **Thread safety**: Concurrent overlay updates work correctly +4. **Clone behavior**: Database clones share the same file tracking +5. **Lazy creation**: Files only created when accessed + +## Implementation Checklist + +- [ ] Add `Arc>` to Database struct +- [ ] Implement Clone for Database (clone both storage and Arc) +- [ ] Create `get_or_create_file` method using atomic entry API +- [ ] Update did_open to create files with revision 0 +- [ ] Update did_change to bump revision after overlay update +- [ ] Update did_close to bump revision (keep file alive) +- [ ] Ensure StorageHandle updates after all database modifications +- [ ] Add tests for revision tracking and invalidation + +## Questions That Were Answered + +1. **Q: Files in Database or Session?** + A: In Database, but Arc-wrapped for cheap cloning + +2. **Q: How does this work with Send+Sync?** + A: Arc is Send+Sync, making Database clonable and thread-safe + +3. **Q: Do we still need StorageHandle?** + A: YES! It bridges tower-lsp's lifetime requirements + +4. **Q: Timestamp or counter for revisions?** + A: Simple incrementing counter per file + +5. **Q: Remove files on close?** + A: No, keep them alive and bump revision for re-read + +## The Key Insight + +Database struct is a container that holds BOTH: +- Salsa storage (for queries and inputs) +- Non-Salsa data (file tracking via Arc) + +Arc makes the non-Salsa data cheap to clone while maintaining Send+Sync compatibility. This is the pattern Ruff uses and what we should implement. \ No newline at end of file diff --git a/RUFF_ARCHITECTURE_INSIGHTS.md b/RUFF_ARCHITECTURE_INSIGHTS.md new file mode 100644 index 00000000..f4a0bf1c --- /dev/null +++ b/RUFF_ARCHITECTURE_INSIGHTS.md @@ -0,0 +1,77 @@ +# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution + +## This document is preserved for historical context but is OUTDATED +## We found the StorageHandle solution that solves the Send+Sync issue + +# Critical Discovery: The Tower-LSP vs lsp-server Architectural Mismatch + +## The Real Problem + +Your Ruff expert friend is correct. The fundamental issue is: + +### What We Found: + +1. **Salsa commit a3ffa22 uses `RefCell` and `UnsafeCell`** - These are inherently not `Sync` +2. **Tower-LSP requires `Sync`** - Because handlers take `&self` in async contexts +3. **Ruff uses `lsp-server`** - Which doesn't require `Sync` on the server struct + +### The Mystery: + +Your expert suggests Ruff's database IS `Send + Sync`, but our testing shows that with the same Salsa commit, the database contains: +- `RefCell` (not Sync) +- `UnsafeCell>` (not Sync) + +## Possible Explanations: + +### Theory 1: Ruff Has Custom Patches +Ruff might have additional patches or workarounds not visible in the commit hash. + +### Theory 2: Different Usage Pattern +Ruff might structure their database differently to avoid the Sync requirement entirely. + +### Theory 3: lsp-server Architecture +Since Ruff uses `lsp-server` (not `tower-lsp`), they might never need the database to be Sync: +- They clone the database for background work (requires Send only) +- The main thread owns the database, background threads get clones +- No shared references across threads + +## Verification Needed: + +1. **Check if Ruff's database is actually Sync**: + - Look for unsafe impl Sync in their codebase + - Check if they wrap the database differently + +2. **Understand lsp-server's threading model**: + - How does it handle async without requiring Sync? + - What's the message passing pattern? + +## Solution Decision Matrix (Updated): + +| Solution | Effort | Performance | Risk | Compatibility | +|----------|---------|------------|------|---------------| +| **Switch to lsp-server** | High | High | Medium | Perfect Ruff parity | +| **Actor Pattern** | Medium | Medium | Low | Works with tower-lsp | +| **Arc** | Low | Poor | Low | Works but slow | +| **Unsafe Sync wrapper** | Low | High | Very High | Dangerous | +| **Database per request** | Medium | Poor | Low | Loses memoization | + +## Recommended Action Plan: + +### Immediate (Today): +1. Verify that Salsa a3ffa22 truly has RefCell/UnsafeCell +2. Check if there are any Ruff-specific patches to Salsa +3. Test the actor pattern as a better alternative to Arc + +### Short-term (This Week): +1. Implement actor pattern if Salsa can't be made Sync +2. OR investigate unsafe Sync wrapper with careful single-threaded access guarantees + +### Long-term (This Month): +1. Consider migrating to lsp-server for full Ruff compatibility +2. OR contribute Sync support to Salsa upstream + +## The Key Insight: + +**Tower-LSP's architecture is fundamentally incompatible with Salsa's current design.** + +Ruff avoided this by using `lsp-server`, which has a different threading model that doesn't require Sync on the database. diff --git a/check_ruff_pattern.md b/check_ruff_pattern.md new file mode 100644 index 00000000..5253b69e --- /dev/null +++ b/check_ruff_pattern.md @@ -0,0 +1,94 @@ +# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution + +## This document is preserved for historical context but is OUTDATED +## We found the StorageHandle solution that solves the Send+Sync issue + +# Key Findings from Ruff's Architecture + +Based on the exploration, here's what we discovered: + +## Current Django LS Architecture + +### What We Have: +1. `Database` struct with `#[derive(Clone)]` and Salsa storage +2. `WorkspaceDatabase` that wraps `Database` and uses `DashMap` for thread-safe file storage +3. `Session` that owns `WorkspaceDatabase` directly (not wrapped in Arc) +4. Tower-LSP server that requires `Send + Sync` for async handlers + +### The Problem: +- `Database` is not `Sync` due to `RefCell` and `UnsafeCell` in Salsa's `ZalsaLocal` +- This prevents `Session` from being `Sync`, which breaks tower-lsp async handlers + +## Ruff's Solution (From Analysis) + +### They Don't Make Database Sync! +The key insight is that Ruff **doesn't actually make the database Send + Sync**. Instead: + +1. **Clone for Background Work**: They clone the database for each background task +2. **Move Not Share**: The cloned database is *moved* into the task (requires Send, not Sync) +3. **Message Passing**: Results are sent back via channels + +### Critical Difference: +- Ruff uses a custom LSP implementation that doesn't require `Sync` on the session +- Tower-LSP *does* require `Sync` because handlers take `&self` + +## The Real Problem + +Tower-LSP's `LanguageServer` trait requires: +```rust +async fn initialize(&self, ...) -> ... +// ^^^^^ This requires self to be Sync! +``` + +But with Salsa's current implementation, the Database can never be Sync. + +## Solution Options + +### Option 1: Wrap Database in Arc (Current Workaround) +```rust +pub struct Session { + database: Arc>, + // ... +} +``` +Downsides: Lock contention, defeats purpose of Salsa's internal optimization + +### Option 2: Move Database Out of Session +```rust +pub struct Session { + // Don't store database here + file_index: Arc>, + settings: Settings, +} + +// Create database on demand for each request +impl LanguageServer for Server { + async fn some_handler(&self) { + let db = create_database_from_index(&self.session.file_index); + // Use db for this request + } +} +``` + +### Option 3: Use Actor Pattern +```rust +pub struct DatabaseActor { + database: WorkspaceDatabase, + rx: mpsc::Receiver, +} + +pub struct Session { + db_tx: mpsc::Sender, +} +``` + +### Option 4: Custom unsafe Send/Sync implementation +This is risky but possible if we ensure single-threaded access patterns. + +## The Salsa Version Mystery + +We're using the exact same Salsa commit as Ruff, with the same features. The issue is NOT the Salsa version, but how tower-lsp forces us to use it. + +Ruff likely either: +1. Doesn't use tower-lsp (has custom implementation) +2. Or structures their server differently to avoid needing Sync on the database diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index e0026587..daa64ac0 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -6,6 +6,7 @@ use tower_lsp_server::jsonrpc::Result as LspResult; use tower_lsp_server::lsp_types; use tower_lsp_server::LanguageServer; use tracing_appender::non_blocking::WorkerGuard; +use url::Url; use crate::queue::Queue; use crate::session::Session; @@ -202,13 +203,19 @@ impl LanguageServer for DjangoLanguageServer { async fn did_open(&self, params: lsp_types::DidOpenTextDocumentParams) { tracing::info!("Opened document: {:?}", params.text_document.uri); - self.with_session_mut(|_session| { - // TODO: Handle document open after refactoring - let _uri = params.text_document.uri.clone(); - let _version = params.text_document.version; - let _language_id = + self.with_session_mut(|session| { + // Convert LSP types to our types + let url = + Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + let language_id = djls_workspace::LanguageId::from(params.text_document.language_id.as_str()); - let _text = params.text_document.text.clone(); + let document = djls_workspace::TextDocument::new( + params.text_document.text, + params.text_document.version, + language_id, + ); + + session.open_document(url, document); }) .await; } @@ -216,11 +223,29 @@ impl LanguageServer for DjangoLanguageServer { async fn did_change(&self, params: lsp_types::DidChangeTextDocumentParams) { tracing::info!("Changed document: {:?}", params.text_document.uri); - self.with_session_mut(|_session| { - // TODO: Handle document change after refactoring - let _uri = ¶ms.text_document.uri; - let _version = params.text_document.version; - let _changes = params.content_changes.clone(); + self.with_session_mut(|session| { + let url = + Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + let new_version = params.text_document.version; + let changes = params.content_changes; + + if let Some(mut document) = session.get_overlay(&url) { + document.update(changes, new_version); + session.update_document(url, document); + } else { + // No existing overlay - shouldn't normally happen + tracing::warn!("Received change for document without overlay: {}", url); + + // Handle full content changes only for recovery + if let Some(change) = changes.into_iter().next() { + let document = djls_workspace::TextDocument::new( + change.text, + new_version, + djls_workspace::LanguageId::Other, + ); + session.update_document(url, document); + } + } }) .await; } @@ -228,19 +253,60 @@ impl LanguageServer for DjangoLanguageServer { async fn did_close(&self, params: lsp_types::DidCloseTextDocumentParams) { tracing::info!("Closed document: {:?}", params.text_document.uri); - self.with_session_mut(|_session| { - // TODO: Handle document close after refactoring - let _uri = ¶ms.text_document.uri; + self.with_session_mut(|session| { + let url = + Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + + if session.close_document(&url).is_none() { + tracing::warn!("Attempted to close document without overlay: {}", url); + } }) .await; } async fn completion( &self, - _params: lsp_types::CompletionParams, + params: lsp_types::CompletionParams, ) -> LspResult> { - // TODO: Handle completion after refactoring - Ok(None) + let response = self + .with_session(|session| { + let lsp_uri = ¶ms.text_document_position.text_document.uri; + let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); + let position = params.text_document_position.position; + + tracing::debug!("Completion requested for {} at {:?}", url, position); + + // Check if we have an overlay for this document + if let Some(document) = session.get_overlay(&url) { + tracing::debug!("Using overlay content for completion in {}", url); + + // Use the overlay content for completion + // For now, we'll return None, but this is where completion logic would go + // The key point is that we're using overlay content, not disk content + let _content = document.content(); + let _version = document.version(); + + // TODO: Implement actual completion logic using overlay content + // This would involve: + // 1. Getting context around the cursor position + // 2. Analyzing the Django template or Python content + // 3. Returning appropriate completions + + None + } else { + tracing::debug!("No overlay found for {}, using disk content", url); + + // No overlay - would use disk content via the file system + // The LspFileSystem will automatically fall back to disk + // when no overlay is available + + // TODO: Implement completion using file system content + None + } + }) + .await; + + Ok(response) } async fn did_change_configuration(&self, _params: lsp_types::DidChangeConfigurationParams) { diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index beae4ecb..6b4adbb5 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,15 +1,75 @@ -use std::collections::HashMap; -use std::path::PathBuf; +//! # Salsa StorageHandle Pattern for LSP +//! +//! This module implements a thread-safe Salsa database wrapper for use with +//! tower-lsp's async runtime. The key challenge is that tower-lsp requires +//! `Send + Sync + 'static` bounds, but Salsa's `Storage` contains thread-local +//! state and is not `Send`. +//! +//! ## The Solution: StorageHandle +//! +//! Salsa provides `StorageHandle` which IS `Send + Sync` because it contains +//! no thread-local state. We store the handle and create `Storage`/`Database` +//! instances on-demand. +//! +//! ## The Mutation Challenge +//! +//! When mutating Salsa inputs (e.g., updating file revisions), Salsa must +//! ensure exclusive access to prevent race conditions. It does this via +//! `cancel_others()` which: +//! +//! 1. Sets a cancellation flag (causes other threads to panic with `Cancelled`) +//! 2. Waits for all `StorageHandle` clones to drop +//! 3. Proceeds with the mutation +//! +//! If we accidentally clone the handle instead of taking ownership, step 2 +//! never completes → deadlock! +//! +//! ## The Pattern +//! +//! - **Reads**: Clone the handle freely (`with_db`) +//! - **Mutations**: Take exclusive ownership (`with_db_mut` via `take_db_handle_for_mutation`) +//! +//! The explicit method names make the intent clear and prevent accidental misuse. + +use std::path::{Path, PathBuf}; use std::sync::Arc; +use dashmap::DashMap; use djls_conf::Settings; use djls_project::DjangoProject; -use djls_workspace::{FileSystem, StdFileSystem, db::Database}; +use djls_workspace::{ + db::{Database, SourceFile}, + FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, +}; use percent_encoding::percent_decode_str; -use salsa::StorageHandle; +use salsa::{Setter, StorageHandle}; use tower_lsp_server::lsp_types; use url::Url; +/// LSP Session with thread-safe Salsa database access. +/// +/// Uses Salsa's `StorageHandle` pattern to maintain `Send + Sync + 'static` +/// compatibility required by tower-lsp. The handle can be safely shared +/// across threads and async boundaries. +/// +/// See [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) +/// for more information about `StorageHandle`. +/// +/// ## Architecture +/// +/// Two-layer system inspired by Ruff/Ty: +/// - **Layer 1**: In-memory overlays (LSP document edits) +/// - **Layer 2**: Salsa database (incremental computation cache) +/// +/// ## Salsa Mutation Protocol +/// +/// When mutating Salsa inputs (like changing file revisions), we must ensure +/// exclusive access to prevent race conditions. Salsa enforces this through +/// its `cancel_others()` mechanism, which waits for all `StorageHandle` clones +/// to drop before allowing mutations. +/// +/// We use explicit methods (`take_db_handle_for_mutation`/`restore_db_handle`) +/// to make this ownership transfer clear and prevent accidental deadlocks. pub struct Session { /// The Django project configuration project: Option, @@ -17,48 +77,82 @@ pub struct Session { /// LSP server settings settings: Settings, - /// A thread-safe Salsa database handle that can be shared between threads. - /// - /// This implements the insight from [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) - /// where we're using the `StorageHandle` to create a thread-safe handle that can be - /// shared between threads. When we need to use it, we clone the handle to get a new reference. - /// - /// This handle allows us to create database instances as needed. - /// Even though we're using a single-threaded runtime, we still need - /// this to be thread-safe because of LSP trait requirements. - /// - /// Usage: - /// ```rust,ignore - /// // Clone the StorageHandle for use in an async context - /// let db_handle = session.db_handle.clone(); + /// Layer 1: Thread-safe overlay storage (Arc>) /// - /// // Use it in an async context - /// async_fn(move || { - /// // Get a database from the handle - /// let storage = db_handle.into_storage(); - /// let db = Database::from_storage(storage); + /// This implements Ruff's two-layer architecture where Layer 1 contains + /// LSP overlays that take precedence over disk files. The overlays map + /// document URLs to TextDocuments containing current in-memory content. /// - /// // Use the database - /// db.some_query(args) - /// }); - /// ``` - db_handle: StorageHandle, + /// Key properties: + /// - Thread-safe via Arc for Send+Sync requirements + /// - Contains full TextDocument with content, version, and metadata + /// - Never becomes Salsa inputs - only intercepted at read time + overlays: Arc>, - /// File system abstraction for reading files + /// File system abstraction with overlay interception + /// + /// This LspFileSystem bridges Layer 1 (overlays) and Layer 2 (Salsa). + /// It intercepts FileSystem::read_to_string() calls to return overlay + /// content when available, falling back to disk otherwise. file_system: Arc, - /// Index of open documents with overlays (in-memory changes) - /// Maps document URL to its current content - overlays: HashMap, - - /// Tracks the session revision for change detection - revision: u64, + /// Shared file tracking across all Database instances + /// + /// This is the canonical Salsa pattern from the lazy-input example. + /// The DashMap provides O(1) lookups and is shared via Arc across + /// all Database instances created from StorageHandle. + files: Arc>, #[allow(dead_code)] client_capabilities: lsp_types::ClientCapabilities, + + /// Layer 2: Thread-safe Salsa database handle for pure computation + /// + /// where we're using the `StorageHandle` to create a thread-safe handle that can be + /// shared between threads. + /// + /// The database receives file content via the FileSystem trait, which + /// is intercepted by our LspFileSystem to provide overlay content. + /// This maintains proper separation between Layer 1 and Layer 2. + db_handle: StorageHandle, } impl Session { + pub fn new(params: &lsp_types::InitializeParams) -> Self { + let project_path = Self::get_project_path(params); + + let (project, settings) = if let Some(path) = &project_path { + let settings = + djls_conf::Settings::new(path).unwrap_or_else(|_| djls_conf::Settings::default()); + + let project = Some(djls_project::DjangoProject::new(path.clone())); + + (project, settings) + } else { + (None, Settings::default()) + }; + + let overlays = Arc::new(DashMap::new()); + let files = Arc::new(DashMap::new()); + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(OsFileSystem), + )); + let db_handle = Database::new(file_system.clone(), files.clone()) + .storage() + .clone() + .into_zalsa_handle(); + + Self { + project, + settings, + overlays, + file_system, + files, + client_capabilities: params.capabilities.clone(), + db_handle, + } + } /// Determines the project root path from initialization parameters. /// /// Tries the current directory first, then falls back to the first workspace folder. @@ -97,31 +191,6 @@ impl Session { Some(PathBuf::from(path_str)) } - pub fn new(params: &lsp_types::InitializeParams) -> Self { - let project_path = Self::get_project_path(params); - - let (project, settings) = if let Some(path) = &project_path { - let settings = - djls_conf::Settings::new(path).unwrap_or_else(|_| djls_conf::Settings::default()); - - let project = Some(djls_project::DjangoProject::new(path.clone())); - - (project, settings) - } else { - (None, Settings::default()) - }; - - Self { - client_capabilities: params.capabilities.clone(), - project, - settings, - db_handle: StorageHandle::new(None), - file_system: Arc::new(StdFileSystem), - overlays: HashMap::new(), - revision: 0, - } - } - pub fn project(&self) -> Option<&DjangoProject> { self.project.as_ref() } @@ -130,8 +199,6 @@ impl Session { &mut self.project } - - pub fn settings(&self) -> &Settings { &self.settings } @@ -144,23 +211,457 @@ impl Session { /// /// This creates a usable database from the handle, which can be used /// to query and update data. The database itself is not Send/Sync, - /// but the StorageHandle is, allowing us to work with tower-lsp. + /// but the `StorageHandle` is, allowing us to work with tower-lsp-server. + /// + /// The database will read files through the LspFileSystem, which + /// automatically returns overlay content when available. + /// + /// CRITICAL: We pass the shared files Arc to preserve file tracking + /// across Database reconstructions from StorageHandle. + #[allow(dead_code)] pub fn db(&self) -> Database { let storage = self.db_handle.clone().into_storage(); - Database::from_storage(storage) + Database::from_storage(storage, self.file_system.clone(), self.files.clone()) + } + + /// Get access to the file system (for Salsa integration) + #[allow(dead_code)] + pub fn file_system(&self) -> Arc { + self.file_system.clone() + } + + /// Set or update an overlay for the given document URL + /// + /// This implements Layer 1 of Ruff's architecture - storing in-memory + /// document changes that take precedence over disk content. + #[allow(dead_code)] // Used in tests + pub fn set_overlay(&self, url: Url, document: TextDocument) { + self.overlays.insert(url, document); + } + + /// Remove an overlay for the given document URL + /// + /// After removal, file reads will fall back to disk content. + #[allow(dead_code)] // Used in tests + pub fn remove_overlay(&self, url: &Url) -> Option { + self.overlays.remove(url).map(|(_, doc)| doc) + } + + /// Check if an overlay exists for the given URL + #[allow(dead_code)] + pub fn has_overlay(&self, url: &Url) -> bool { + self.overlays.contains_key(url) + } + + /// Get a copy of an overlay document + pub fn get_overlay(&self, url: &Url) -> Option { + self.overlays.get(url).map(|doc| doc.clone()) + } + + /// Takes exclusive ownership of the database handle for mutation operations. + /// + /// This method extracts the `StorageHandle` from the session, replacing it + /// with a temporary placeholder. This ensures there's exactly one handle + /// active during mutations, preventing deadlocks in Salsa's `cancel_others()`. + /// + /// # Why Not Clone? + /// + /// Cloning would create multiple handles. When Salsa needs to mutate inputs, + /// it calls `cancel_others()` which waits for all handles to drop. With + /// multiple handles, this wait would never complete → deadlock. + /// + /// # Panics + /// + /// This is an internal method that should only be called by `with_db_mut`. + /// Multiple concurrent calls would panic when trying to take an already-taken handle. + fn take_db_handle_for_mutation(&mut self) -> StorageHandle { + std::mem::replace(&mut self.db_handle, StorageHandle::new(None)) + } + + /// Restores the database handle after a mutation operation completes. + /// + /// This should be called with the handle extracted from the database + /// after mutations are complete. It updates the session's handle to + /// reflect any changes made during the mutation. + fn restore_db_handle(&mut self, handle: StorageHandle) { + self.db_handle = handle; + } + + /// Execute a closure with mutable access to the database. + /// + /// This method implements Salsa's required protocol for mutations: + /// 1. Takes exclusive ownership of the StorageHandle (no clones exist) + /// 2. Creates a temporary Database for the operation + /// 3. Executes your closure with `&mut Database` + /// 4. Extracts and restores the updated handle + /// + /// # Example + /// + /// ```rust,ignore + /// session.with_db_mut(|db| { + /// let file = db.get_or_create_file(path); + /// file.set_revision(db).to(new_revision); // Mutation requires exclusive access + /// }); + /// ``` + /// + /// # Why This Pattern? + /// + /// This ensures that when Salsa needs to modify inputs (via setters like + /// `set_revision`), it has exclusive access. The internal `cancel_others()` + /// call will succeed because we guarantee only one handle exists. + pub fn with_db_mut(&mut self, f: F) -> R + where + F: FnOnce(&mut Database) -> R, + { + let handle = self.take_db_handle_for_mutation(); + + let storage = handle.into_storage(); + let mut db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); + + let result = f(&mut db); + + // The database may have changed during mutations, so we need + // to extract its current handle state + let new_handle = db.storage().clone().into_zalsa_handle(); + self.restore_db_handle(new_handle); + + result + } + + /// Execute a closure with read-only access to the database. + /// + /// For read-only operations, we can safely clone the `StorageHandle` + /// since Salsa allows multiple concurrent readers. This is more + /// efficient than taking exclusive ownership. + /// + /// # Example + /// + /// ```rust,ignore + /// let content = session.with_db(|db| { + /// let file = db.get_file(path)?; + /// source_text(db, file).to_string() // Read-only query + /// }); + /// ``` + pub fn with_db(&self, f: F) -> R + where + F: FnOnce(&Database) -> R, + { + // For reads, cloning is safe and efficient + let storage = self.db_handle.clone().into_storage(); + let db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); + f(&db) + } + + /// Convert a URL to a PathBuf for file operations. + /// + /// This is needed to convert between LSP URLs and file paths for + /// SourceFile creation and tracking. + pub fn url_to_path(&self, url: &Url) -> Option { + // Only handle file:// URLs + if url.scheme() != "file" { + return None; + } + + // Decode and convert to PathBuf + let path = percent_decode_str(url.path()).decode_utf8().ok()?; + + #[cfg(windows)] + let path = path.strip_prefix('/').unwrap_or(&path); + + Some(PathBuf::from(path.as_ref())) + } + + // ===== Document Lifecycle Management ===== + // These methods encapsulate the two-layer architecture coordination: + // Layer 1 (overlays) and Layer 2 (Salsa revision tracking) + + /// Handle opening a document - sets overlay and creates file. + /// + /// This method coordinates both layers: + /// - Layer 1: Stores the document content in overlays + /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) + pub fn open_document(&mut self, url: Url, document: TextDocument) { + tracing::debug!("Opening document: {}", url); + + // Layer 1: Set overlay + self.overlays.insert(url.clone(), document); + + // Layer 2: Create file if needed (starts at revision 0) + if let Some(path) = self.url_to_path(&url) { + self.with_db_mut(|db| { + let file = db.get_or_create_file(path.clone()); + tracing::debug!( + "Created/retrieved SourceFile for {}: revision {}", + path.display(), + file.revision(db) + ); + }); + } + } + + /// Handle document changes - updates overlay and bumps revision. + /// + /// This method coordinates both layers: + /// - Layer 1: Updates the document content in overlays + /// - Layer 2: Bumps the file revision to trigger Salsa invalidation + pub fn update_document(&mut self, url: Url, document: TextDocument) { + let version = document.version(); + tracing::debug!("Updating document: {} (version {})", url, version); + + // Layer 1: Update overlay + self.overlays.insert(url.clone(), document); + + // Layer 2: Bump revision to trigger invalidation + if let Some(path) = self.url_to_path(&url) { + self.notify_file_changed(path); + } + } + + /// Handle closing a document - removes overlay and bumps revision. + /// + /// This method coordinates both layers: + /// - Layer 1: Removes the overlay (falls back to disk) + /// - Layer 2: Bumps revision to trigger re-read from disk + /// + /// Returns the removed document if it existed. + pub fn close_document(&mut self, url: &Url) -> Option { + tracing::debug!("Closing document: {}", url); + + // Layer 1: Remove overlay + let removed = self.overlays.remove(url).map(|(_, doc)| { + tracing::debug!( + "Removed overlay for closed document: {} (was version {})", + url, + doc.version() + ); + doc + }); + + // Layer 2: Bump revision to trigger re-read from disk + // We keep the file alive for potential re-opening + if let Some(path) = self.url_to_path(url) { + self.notify_file_changed(path); + } + + removed + } + + /// Internal: Notify that a file's content has changed. + /// + /// This bumps the file's revision number in Salsa, which triggers + /// invalidation of any queries that depend on the file's content. + fn notify_file_changed(&mut self, path: PathBuf) { + self.with_db_mut(|db| { + // Only bump revision if file is already being tracked + // We don't create files just for notifications + if db.has_file(&path) { + let file = db.get_or_create_file(path.clone()); + let current_rev = file.revision(db); + let new_rev = current_rev + 1; + file.set_revision(db).to(new_rev); + tracing::debug!( + "Bumped revision for {}: {} -> {}", + path.display(), + current_rev, + new_rev + ); + } else { + tracing::debug!( + "File {} not tracked, skipping revision bump", + path.display() + ); + } + }); + } + + // ===== Safe Query API ===== + // These methods encapsulate all Salsa interactions, preventing the + // "mixed database instance" bug by never exposing SourceFile or Database. + + /// Get the current content of a file (from overlay or disk). + /// + /// This is the safe way to read file content through the system. + /// The file is created if it doesn't exist, and content is read + /// through the FileSystem abstraction (overlay first, then disk). + pub fn file_content(&mut self, path: PathBuf) -> String { + use djls_workspace::db::source_text; + + self.with_db_mut(|db| { + let file = db.get_or_create_file(path); + source_text(db, file).to_string() + }) + } + + /// Get the current revision of a file, if it's being tracked. + /// + /// Returns None if the file hasn't been created yet. + pub fn file_revision(&mut self, path: &Path) -> Option { + self.with_db_mut(|db| { + db.has_file(path).then(|| { + let file = db.get_or_create_file(path.to_path_buf()); + file.revision(db) + }) + }) + } + + /// Check if a file is currently being tracked in Salsa. + pub fn has_file(&mut self, path: &Path) -> bool { + self.with_db(|db| db.has_file(path)) } } impl Default for Session { fn default() -> Self { + let overlays = Arc::new(DashMap::new()); + let files = Arc::new(DashMap::new()); + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(OsFileSystem), + )); + let db_handle = Database::new(file_system.clone(), files.clone()) + .storage() + .clone() + .into_zalsa_handle(); + Self { project: None, settings: Settings::default(), - db_handle: StorageHandle::new(None), - file_system: Arc::new(StdFileSystem), - overlays: HashMap::new(), - revision: 0, + db_handle, + file_system, + files, + overlays, client_capabilities: lsp_types::ClientCapabilities::default(), } } } + +#[cfg(test)] +mod tests { + use super::*; + use djls_workspace::LanguageId; + + #[test] + fn test_session_overlay_management() { + let session = Session::default(); + + let url = Url::parse("file:///test/file.py").unwrap(); + let document = TextDocument::new("print('hello')".to_string(), 1, LanguageId::Python); + + // Initially no overlay + assert!(!session.has_overlay(&url)); + assert!(session.get_overlay(&url).is_none()); + + // Set overlay + session.set_overlay(url.clone(), document.clone()); + assert!(session.has_overlay(&url)); + + let retrieved = session.get_overlay(&url).unwrap(); + assert_eq!(retrieved.content(), document.content()); + assert_eq!(retrieved.version(), document.version()); + + // Remove overlay + let removed = session.remove_overlay(&url).unwrap(); + assert_eq!(removed.content(), document.content()); + assert!(!session.has_overlay(&url)); + } + + #[test] + fn test_session_two_layer_architecture() { + let session = Session::default(); + + // Verify we have both layers + let _filesystem = session.file_system(); // Layer 2: FileSystem bridge + let _db = session.db(); // Layer 2: Salsa database + + // Verify overlay operations work (Layer 1) + let url = Url::parse("file:///test/integration.py").unwrap(); + let document = TextDocument::new("# Layer 1 content".to_string(), 1, LanguageId::Python); + + session.set_overlay(url.clone(), document); + assert!(session.has_overlay(&url)); + + // FileSystem should now return overlay content through LspFileSystem + // (This would be tested more thoroughly in integration tests) + } + + #[test] + fn test_revision_invalidation_chain() { + use std::path::PathBuf; + + let mut session = Session::default(); + + // Create a test file path + let path = PathBuf::from("/test/template.html"); + let url = Url::parse("file:///test/template.html").unwrap(); + + // Open document with initial content + println!("**[test]** open document with initial content"); + let document = TextDocument::new( + "

Original Content

".to_string(), + 1, + LanguageId::Other, + ); + session.open_document(url.clone(), document); + + // Try to read content - this might be where it hangs + println!("**[test]** try to read content - this might be where it hangs"); + let content1 = session.file_content(path.clone()); + assert_eq!(content1, "

Original Content

"); + + // Update document with new content + println!("**[test]** Update document with new content"); + let updated_document = + TextDocument::new("

Updated Content

".to_string(), 2, LanguageId::Other); + session.update_document(url.clone(), updated_document); + + // Read content again (should get new overlay content due to invalidation) + println!( + "**[test]** Read content again (should get new overlay content due to invalidation)" + ); + let content2 = session.file_content(path.clone()); + assert_eq!(content2, "

Updated Content

"); + assert_ne!(content1, content2); + + // Close document (removes overlay, bumps revision) + println!("**[test]** Close document (removes overlay, bumps revision)"); + session.close_document(&url); + + // Read content again (should now read from disk, which returns empty for missing files) + println!( + "**[test]** Read content again (should now read from disk, which returns empty for missing files)" + ); + let content3 = session.file_content(path.clone()); + assert_eq!(content3, ""); // No file on disk, returns empty + } + + #[test] + fn test_with_db_mut_preserves_files() { + use std::path::PathBuf; + + let mut session = Session::default(); + + // Create multiple files + let path1 = PathBuf::from("/test/file1.py"); + let path2 = PathBuf::from("/test/file2.py"); + + // Create files through safe API + session.file_content(path1.clone()); // Creates file1 + session.file_content(path2.clone()); // Creates file2 + + // Verify files are preserved across operations + assert!(session.has_file(&path1)); + assert!(session.has_file(&path2)); + + // Files should persist even after multiple operations + let content1 = session.file_content(path1.clone()); + let content2 = session.file_content(path2.clone()); + + // Both should return empty (no disk content) + assert_eq!(content1, ""); + assert_eq!(content2, ""); + + // One more verification + assert!(session.has_file(&path1)); + assert!(session.has_file(&path2)); + } +} diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs deleted file mode 100644 index 2da695e8..00000000 --- a/crates/djls-workspace/src/bridge.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! Bridge between VFS snapshots and Salsa inputs. -//! -//! The bridge module isolates Salsa input mutation behind a single, idempotent API. -//! It ensures we only touch Salsa when content or classification changes, maximizing -//! incremental performance. - -use std::collections::HashMap; -use std::sync::Arc; - -use salsa::Setter; - -use super::db::parse_template; -use super::db::template_errors; -use super::db::Database; -use super::db::SourceFile; -use super::db::TemplateAst; -use super::db::TemplateLoaderOrder; -use super::FileId; -use super::FileKind; - -/// Owner of the Salsa [`Database`] plus the handles for updating inputs. -/// -/// [`FileStore`] serves as the bridge between the VFS (with [`FileId`]s) and Salsa (with entities). -/// It maintains a mapping from [`FileId`]s to [`SourceFile`] entities and manages the global -/// [`TemplateLoaderOrder`] input. The [`FileStore`] ensures that Salsa inputs are only mutated -/// when actual changes occur, preserving incremental computation efficiency. -pub struct FileStore { - /// The Salsa DB instance - pub db: Database, - /// Map from [`FileId`] to its Salsa input entity - files: HashMap, - /// Handle to the global template loader configuration input - template_loader: Option, -} - -impl FileStore { - /// Construct an empty store and DB. - #[must_use] - pub fn new() -> Self { - Self { - db: Database::new(), - files: HashMap::new(), - template_loader: None, - } - } - - /// Create or update the global template loader order input. - /// - /// Sets the ordered list of template root directories that Django will search - /// when resolving template names. If the input already exists, it updates the - /// existing value; otherwise, it creates a new [`TemplateLoaderOrder`] input. - pub fn set_template_loader_order(&mut self, ordered_roots: Vec) { - let roots = Arc::from(ordered_roots.into_boxed_slice()); - if let Some(tl) = self.template_loader { - tl.set_roots(&mut self.db).to(roots); - } else { - self.template_loader = Some(TemplateLoaderOrder::new(&self.db, roots)); - } - } - - // TODO: This will be replaced with direct file management - // pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { - // for (id, rec) in &snap.files { - // let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); - // let new_kind = rec.meta.kind; - - // if let Some(sf) = self.files.get(id) { - // // Update if changed — avoid touching Salsa when not needed - // if sf.kind(&self.db) != new_kind { - // sf.set_kind(&mut self.db).to(new_kind); - // } - // if sf.text(&self.db).as_ref() != &*new_text { - // sf.set_text(&mut self.db).to(new_text.clone()); - // } - // } else { - // let sf = SourceFile::new(&self.db, new_kind, new_text); - // self.files.insert(*id, sf); - // } - // } - // } - - /// Get the text content of a file by its [`FileId`]. - /// - /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub(crate) fn file_text(&self, id: FileId) -> Option> { - self.files.get(&id).map(|sf| sf.text(&self.db).clone()) - } - - /// Get the file kind classification by its [`FileId`]. - /// - /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub(crate) fn file_kind(&self, id: FileId) -> Option { - self.files.get(&id).map(|sf| sf.kind(&self.db)) - } - - /// Get the parsed template AST for a file by its [`FileId`]. - /// - /// This method leverages Salsa's incremental computation to cache parsed ASTs. - /// The AST is only re-parsed when the file's content changes in the VFS. - /// Returns `None` if the file is not tracked or is not a template file. - pub(crate) fn get_template_ast(&self, id: FileId) -> Option> { - let source_file = self.files.get(&id)?; - parse_template(&self.db, *source_file) - } - - /// Get template parsing errors for a file by its [`FileId`]. - /// - /// This method provides quick access to template errors without needing the full AST. - /// Useful for diagnostics and error reporting. Returns an empty slice for - /// non-template files or files not tracked in the store. - pub(crate) fn get_template_errors(&self, id: FileId) -> Arc<[String]> { - self.files - .get(&id) - .map_or_else(|| Arc::from(vec![]), |sf| template_errors(&self.db, *sf)) - } -} - -impl Default for FileStore { - fn default() -> Self { - Self::new() - } -} - -// TODO: Re-enable tests after VFS removal is complete -// #[cfg(test)] -// mod tests { diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index c542ea44..0105616c 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -1,39 +1,100 @@ //! Salsa database and input entities for workspace. //! -//! This module defines the Salsa world—what can be set and tracked incrementally. -//! Inputs are kept minimal to avoid unnecessary recomputation. +//! This module implements a two-layer architecture inspired by Ruff's design pattern +//! for efficient LSP document management with Salsa incremental computation. +//! +//! # Two-Layer Architecture +//! +//! ## Layer 1: LSP Document Management (in Session) +//! - Stores overlays in `Session` using `Arc>` +//! - TextDocument contains actual content, version, language_id +//! - Changes are immediate, no Salsa invalidation on every keystroke +//! - Thread-safe via DashMap for tower-lsp's Send+Sync requirements +//! +//! ## Layer 2: Salsa Incremental Computation (in Database) +//! - Database is pure Salsa, no file content storage +//! - Files tracked via `Arc>` for O(1) lookups +//! - SourceFile inputs only have path and revision (no text) +//! - Content read lazily through FileSystem trait +//! - LspFileSystem intercepts reads, returns overlay or disk content +//! +//! # Critical Implementation Details +//! +//! ## The Revision Dependency Trick +//! The `source_text` tracked function MUST call `file.revision(db)` to create +//! the Salsa dependency chain. Without this, revision changes won't trigger +//! invalidation of dependent queries. +//! +//! ## StorageHandle Pattern (for tower-lsp) +//! - Database itself is NOT Send+Sync (due to RefCell in Salsa's Storage) +//! - StorageHandle IS Send+Sync, enabling use across threads +//! - Session stores StorageHandle, creates Database instances on-demand +//! +//! ## Why Files are in Database, Overlays in Session +//! - Files need persistent tracking across all queries (thus in Database) +//! - Overlays are LSP-specific and change frequently (thus in Session) +//! - This separation prevents Salsa invalidation cascades on every keystroke +//! - Both are accessed via Arc for thread safety and cheap cloning +//! +//! # Data Flow +//! +//! 1. **did_open/did_change** → Update overlays in Session +//! 2. **notify_file_changed()** → Bump revision, tell Salsa something changed +//! 3. **Salsa query executes** → Calls source_text() +//! 4. **source_text() calls file.revision(db)** → Creates dependency +//! 5. **source_text() calls db.read_file_content()** → Goes through FileSystem +//! 6. **LspFileSystem intercepts** → Returns overlay if exists, else disk +//! 7. **Query gets content** → Without knowing about LSP/overlays +//! +//! This design achieves: +//! - Fast overlay updates (no Salsa invalidation) +//! - Proper incremental computation (via revision tracking) +//! - Thread safety (via Arc and StorageHandle) +//! - Clean separation of concerns (LSP vs computation) +use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; #[cfg(test)] use std::sync::Mutex; use dashmap::DashMap; -use url::Url; -use crate::{FileId, FileKind}; +use crate::{FileKind, FileSystem}; + +/// Database trait that provides file system access for Salsa queries +#[salsa::db] +pub trait Db: salsa::Database { + /// Get the file system for reading files (with overlay support) + fn fs(&self) -> Option>; + + /// Read file content through the file system + /// This is the primary way Salsa queries should read files, as it + /// automatically checks overlays before falling back to disk. + fn read_file_content(&self, path: &Path) -> std::io::Result; +} /// Salsa database root for workspace /// /// The [`Database`] provides default storage and, in tests, captures Salsa events for /// reuse/diagnostics. It serves as the core incremental computation engine, tracking /// dependencies and invalidations across all inputs and derived queries. -/// -/// This database also manages the file system overlay for the workspace, -/// mapping URLs to FileIds and storing file content. +/// +/// The database integrates with the FileSystem abstraction to read files through +/// the LspFileSystem, which automatically checks overlays before falling back to disk. #[salsa::db] #[derive(Clone)] pub struct Database { storage: salsa::Storage, - - /// Map from file URL to FileId (thread-safe) - files: DashMap, - - /// Map from FileId to file content (thread-safe) - content: DashMap>, - - /// Next FileId to allocate (thread-safe counter) - next_file_id: Arc, + + /// FileSystem integration for reading files (with overlay support) + /// This allows the database to read files through LspFileSystem, which + /// automatically checks for overlays before falling back to disk files. + fs: Option>, + + /// File tracking outside of Salsa but within Database (Arc for cheap cloning). + /// This follows Ruff's pattern where files are tracked in the Database struct + /// but not as part of Salsa's storage, enabling cheap clones via Arc. + files: Arc>, // The logs are only used for testing and demonstrating reuse: #[cfg(test)] @@ -58,101 +119,148 @@ impl Default for Database { } } }))), - files: DashMap::new(), - content: DashMap::new(), - next_file_id: Arc::new(AtomicU32::new(0)), + fs: None, + files: Arc::new(DashMap::new()), logs, } } } impl Database { - /// Create a new database instance - pub fn new() -> Self { + /// Create a new database with fresh storage. + pub fn new(file_system: Arc, files: Arc>) -> Self { Self { storage: salsa::Storage::new(None), - files: DashMap::new(), - content: DashMap::new(), - next_file_id: Arc::new(AtomicU32::new(0)), + fs: Some(file_system), + files, #[cfg(test)] logs: Arc::new(Mutex::new(None)), } } - - /// Create a new database instance from a storage handle. - /// This is used by Session::db() to create databases from the StorageHandle. - pub fn from_storage(storage: salsa::Storage) -> Self { + + /// Create a database instance from an existing storage. + /// This preserves both the file system and files Arc across database operations. + pub fn from_storage( + storage: salsa::Storage, + file_system: Arc, + files: Arc>, + ) -> Self { Self { storage, - files: DashMap::new(), - content: DashMap::new(), - next_file_id: Arc::new(AtomicU32::new(0)), + fs: Some(file_system), + files, #[cfg(test)] logs: Arc::new(Mutex::new(None)), } } - - /// Add or update a file in the workspace - pub fn set_file(&mut self, url: Url, content: String, _kind: FileKind) { - let file_id = if let Some(existing_id) = self.files.get(&url) { - *existing_id - } else { - let new_id = FileId::from_raw(self.next_file_id.fetch_add(1, Ordering::SeqCst)); - self.files.insert(url.clone(), new_id); - new_id - }; - let content = Arc::::from(content); - self.content.insert(file_id, content.clone()); - - // TODO: Update Salsa inputs here when we connect them - } - - /// Remove a file from the workspace - pub fn remove_file(&mut self, url: &Url) { - if let Some((_, file_id)) = self.files.remove(url) { - self.content.remove(&file_id); - // TODO: Remove from Salsa when we connect inputs + /// Read file content through the file system + /// This is the primary way Salsa queries should read files, as it + /// automatically checks overlays before falling back to disk. + pub fn read_file_content(&self, path: &Path) -> std::io::Result { + if let Some(fs) = &self.fs { + fs.read_to_string(path) + } else { + std::fs::read_to_string(path) } } - - /// Get the content of a file by URL - pub fn get_file_content(&self, url: &Url) -> Option> { - let file_id = self.files.get(url)?; - self.content.get(&*file_id).map(|content| content.clone()) + + /// Get or create a SourceFile for the given path. + /// + /// This method implements Ruff's pattern for lazy file creation. Files are created + /// with an initial revision of 0 and tracked in the Database's DashMap. The Arc + /// ensures cheap cloning while maintaining thread safety. + pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { + if let Some(file_ref) = self.files.get(&path) { + // Copy the value (SourceFile is Copy) and drop the guard immediately + let file = *file_ref; + drop(file_ref); // Explicitly drop the guard to release the lock + return file; + } + + // File doesn't exist, so we need to create it + let kind = FileKind::from_path(&path); + let file = SourceFile::new(self, kind, Arc::from(path.to_string_lossy().as_ref()), 0); + + self.files.insert(path.clone(), file); + file } - - /// Get the content of a file by FileId - pub(crate) fn get_content_by_id(&self, file_id: FileId) -> Option> { - self.content.get(&file_id).map(|content| content.clone()) + + /// Check if a file is being tracked without creating it. + /// + /// This is primarily used for testing to verify that files have been + /// created without affecting the database state. + pub fn has_file(&self, path: &Path) -> bool { + self.files.contains_key(path) } - - /// Check if a file exists in the workspace - pub fn has_file(&self, url: &Url) -> bool { - self.files.contains_key(url) + + /// Get a reference to the storage for handle extraction. + /// + /// This is used by Session to extract the StorageHandle after mutations. + pub fn storage(&self) -> &salsa::Storage { + &self.storage } - - /// Get all file URLs in the workspace - pub fn files(&self) -> impl Iterator + use<'_> { - self.files.iter().map(|entry| entry.key().clone()) + + /// Consume the database and return its storage. + /// + /// This is used when you need to take ownership of the storage. + pub fn into_storage(self) -> salsa::Storage { + self.storage } } #[salsa::db] impl salsa::Database for Database {} -/// Represents a single file's classification and current content. +#[salsa::db] +impl Db for Database { + fn fs(&self) -> Option> { + self.fs.clone() + } + + fn read_file_content(&self, path: &Path) -> std::io::Result { + match &self.fs { + Some(fs) => fs.read_to_string(path), + None => std::fs::read_to_string(path), // Fallback to direct disk access + } + } +} + +/// Represents a single file without storing its content. /// -/// [`SourceFile`] is a Salsa input entity that tracks both the file's type (for routing -/// to appropriate analyzers) and its current text content. The text is stored as -/// `Arc` for efficient sharing across the incremental computation graph. +/// [`SourceFile`] is a Salsa input entity that tracks a file's path, revision, and +/// classification for analysis routing. Following Ruff's pattern, content is NOT +/// stored here but read on-demand through the `source_text` tracked function. #[salsa::input] pub struct SourceFile { /// The file's classification for analysis routing pub kind: FileKind, - /// The current text content of the file + /// The file path #[returns(ref)] - pub text: Arc, + pub path: Arc, + /// The revision number for invalidation tracking + pub revision: u64, +} + +/// Read file content through the FileSystem, creating proper Salsa dependencies. +/// +/// This is the CRITICAL function that implements Ruff's two-layer architecture. +/// The call to `file.revision(db)` creates a Salsa dependency, ensuring that +/// when the revision changes, this function (and all dependent queries) are +/// invalidated and re-executed. +#[salsa::tracked] +pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { + // This line creates the Salsa dependency on revision! Without this call, + // revision changes won't trigger invalidation + let _ = file.revision(db); + + let path = Path::new(file.path(db).as_ref()); + match db.read_file_content(path) { + Ok(content) => Arc::from(content), + Err(_) => { + Arc::from("") // Return empty string for missing files + } + } } /// Global input configuring ordered template loader roots. @@ -167,6 +275,18 @@ pub struct TemplateLoaderOrder { pub roots: Arc<[String]>, } +/// Represents a file path for Salsa tracking. +/// +/// [`FilePath`] is a Salsa input entity that tracks a file path for use in +/// path-based queries. This allows Salsa to properly track dependencies +/// on files identified by path rather than by SourceFile input. +#[salsa::input] +pub struct FilePath { + /// The file path as a string + #[returns(ref)] + pub path: Arc, +} + /// Container for a parsed Django template AST. /// /// [`TemplateAst`] wraps the parsed AST from djls-templates along with any parsing errors. @@ -183,18 +303,18 @@ pub struct TemplateAst { /// Parse a Django template file into an AST. /// /// This Salsa tracked function parses template files on-demand and caches the results. -/// The parse is only re-executed when the file's text content changes, enabling -/// efficient incremental template analysis. +/// The parse is only re-executed when the file's content changes (detected via content changes). /// /// Returns `None` for non-template files. #[salsa::tracked] -pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option> { +pub fn parse_template(db: &dyn Db, file: SourceFile) -> Option> { // Only parse template files if file.kind(db) != FileKind::Template { return None; } - let text = file.text(db); + let text_arc = source_text(db, file); + let text = text_arc.as_ref(); // Call the pure parsing function from djls-templates match djls_templates::parse_template(text) { @@ -216,98 +336,61 @@ pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option Option> { + // Read file content through the FileSystem (checks overlays first) + let path = Path::new(file_path.path(db).as_ref()); + let Ok(text) = db.read_file_content(path) else { + return None; + }; + + // Call the pure parsing function from djls-templates + match djls_templates::parse_template(&text) { + Ok((ast, errors)) => { + // Convert errors to strings + let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); + Some(Arc::new(TemplateAst { + ast, + errors: error_strings, + })) + } + Err(err) => { + // Even on fatal errors, return an empty AST with the error + Some(Arc::new(TemplateAst { + ast: djls_templates::Ast::default(), + errors: vec![err.to_string()], + })) + } + } +} + +/// Get template parsing errors for a file by path. /// /// This Salsa tracked function extracts just the errors from the parsed template, /// useful for diagnostics without needing the full AST. /// +/// Reads files through the FileSystem for overlay support. +/// /// Returns an empty vector for non-template files. #[salsa::tracked] -pub fn template_errors(db: &dyn salsa::Database, file: SourceFile) -> Arc<[String]> { - parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) +pub fn template_errors_by_path(db: &dyn Db, file_path: FilePath) -> Arc<[String]> { + parse_template_by_path(db, file_path) + .map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) } -#[cfg(test)] -mod tests { - use salsa::Setter; - - use super::*; - - #[test] - fn test_template_parsing_caches_result() { - let db = Database::default(); - - // Create a template file - let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); - let file = SourceFile::new(&db, FileKind::Template, template_content.clone()); - - // First parse - should execute the parsing - let ast1 = parse_template(&db, file); - assert!(ast1.is_some()); - - // Second parse - should return cached result (same Arc) - let ast2 = parse_template(&db, file); - assert!(ast2.is_some()); - - // Verify they're the same Arc (cached) - assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } - - #[test] - fn test_template_parsing_invalidates_on_change() { - let mut db = Database::default(); - - // Create a template file - let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); - let file = SourceFile::new(&db, FileKind::Template, template_content1); - - // First parse - let ast1 = parse_template(&db, file); - assert!(ast1.is_some()); - - // Change the content - let template_content2: Arc = - Arc::from("{% for item in items %}{{ item }}{% endfor %}"); - file.set_text(&mut db).to(template_content2); - - // Parse again - should re-execute due to changed content - let ast2 = parse_template(&db, file); - assert!(ast2.is_some()); - - // Verify they're different Arcs (re-parsed) - assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } - - #[test] - fn test_non_template_files_return_none() { - let db = Database::default(); - - // Create a Python file - let python_content: Arc = Arc::from("def hello():\n print('Hello')"); - let file = SourceFile::new(&db, FileKind::Python, python_content); - - // Should return None for non-template files - let ast = parse_template(&db, file); - assert!(ast.is_none()); - - // Errors should be empty for non-template files - let errors = template_errors(&db, file); - assert!(errors.is_empty()); - } - - #[test] - fn test_template_errors_tracked_separately() { - let db = Database::default(); - - // Create a template with an error (unclosed tag) - let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); - let file = SourceFile::new(&db, FileKind::Template, template_content); - - // Get errors - let errors1 = template_errors(&db, file); - let errors2 = template_errors(&db, file); - - // Should be cached (same Arc) - assert!(Arc::ptr_eq(&errors1, &errors2)); - } +/// Get template parsing errors for a file. +/// +/// This Salsa tracked function extracts just the errors from the parsed template, +/// useful for diagnostics without needing the full AST. +/// +/// Returns an empty vector for non-template files. +#[salsa::tracked] +pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { + parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) } diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs new file mode 100644 index 00000000..c1447afa --- /dev/null +++ b/crates/djls-workspace/src/document.rs @@ -0,0 +1,216 @@ +use crate::language::LanguageId; +use crate::template::ClosingBrace; +use crate::template::TemplateTagContext; +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; + +#[derive(Clone, Debug)] +pub struct TextDocument { + /// The document's content + content: String, + /// The version number of this document (from LSP) + version: i32, + /// The language identifier (python, htmldjango, etc.) + language_id: LanguageId, + /// Line index for efficient position lookups + line_index: LineIndex, +} + +impl TextDocument { + /// Create a new TextDocument with the given content + pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { + let line_index = LineIndex::new(&content); + Self { + content, + version, + language_id, + line_index, + } + } + + /// Get the document's content + pub fn content(&self) -> &str { + &self.content + } + + /// Get the version number + pub fn version(&self) -> i32 { + self.version + } + + /// Get the language identifier + pub fn language_id(&self) -> LanguageId { + self.language_id.clone() + } + + pub fn line_index(&self) -> &LineIndex { + &self.line_index + } + + pub fn get_line(&self, line: u32) -> Option { + let line_start = *self.line_index.line_starts.get(line as usize)?; + let line_end = self + .line_index + .line_starts + .get(line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + Some(self.content[line_start as usize..line_end as usize].to_string()) + } + + pub fn get_text_range(&self, range: Range) -> Option { + let start_offset = self.line_index.offset(range.start)? as usize; + let end_offset = self.line_index.offset(range.end)? as usize; + + Some(self.content[start_offset..end_offset].to_string()) + } + + /// Update the document content with LSP text changes + pub fn update( + &mut self, + changes: Vec, + version: i32, + ) { + // For now, we'll just handle full document updates + // TODO: Handle incremental updates + for change in changes { + // TextDocumentContentChangeEvent has a `text` field that's a String, not Option + self.content = change.text; + self.line_index = LineIndex::new(&self.content); + } + self.version = version; + } + + pub fn get_template_tag_context(&self, position: Position) -> Option { + let start = self.line_index.line_starts.get(position.line as usize)?; + let end = self + .line_index + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + let line = &self.content[*start as usize..end as usize]; + let char_pos: usize = position.character.try_into().ok()?; + let prefix = &line[..char_pos]; + let rest_of_line = &line[char_pos..]; + let rest_trimmed = rest_of_line.trim_start(); + + prefix.rfind("{%").map(|tag_start| { + // Check if we're immediately after {% with no space + let needs_leading_space = prefix.ends_with("{%"); + + let closing_brace = if rest_trimmed.starts_with("%}") { + ClosingBrace::FullClose + } else if rest_trimmed.starts_with('}') { + ClosingBrace::PartialClose + } else { + ClosingBrace::None + }; + + TemplateTagContext { + partial_tag: prefix[tag_start + 2..].trim().to_string(), + needs_leading_space, + closing_brace, + } + }) + } + + pub fn position_to_offset(&self, position: Position) -> Option { + self.line_index.offset(position) + } + + pub fn offset_to_position(&self, offset: u32) -> Position { + self.line_index.position(offset) + } +} + +#[derive(Clone, Debug)] +pub struct LineIndex { + pub line_starts: Vec, + pub line_starts_utf16: Vec, + pub length: u32, + pub length_utf16: u32, +} + +impl LineIndex { + #[must_use] + pub fn new(text: &str) -> Self { + let mut line_starts = vec![0]; + let mut line_starts_utf16 = vec![0]; + let mut pos_utf8 = 0; + let mut pos_utf16 = 0; + + for c in text.chars() { + pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); + if c == '\n' { + line_starts.push(pos_utf8); + line_starts_utf16.push(pos_utf16); + } + } + + Self { + line_starts, + line_starts_utf16, + length: pos_utf8, + length_utf16: pos_utf16, + } + } + + #[must_use] + pub fn offset(&self, position: Position) -> Option { + let line_start = self.line_starts.get(position.line as usize)?; + + Some(line_start + position.character) + } + + /// Convert UTF-16 LSP position to UTF-8 byte offset + pub fn offset_utf16(&self, position: Position, text: &str) -> Option { + let line_start_utf8 = self.line_starts.get(position.line as usize)?; + let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; + + // If position is at start of line, return UTF-8 line start + if position.character == 0 { + return Some(*line_start_utf8); + } + + // Find the line text + let next_line_start = self + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.length); + + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; + + // Convert UTF-16 character offset to UTF-8 byte offset within the line + let mut utf16_pos = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if utf16_pos >= position.character { + break; + } + utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); + } + + Some(line_start_utf8 + utf8_pos) + } + + #[allow(dead_code)] + #[must_use] + pub fn position(&self, offset: u32) -> Position { + let line = match self.line_starts.binary_search(&offset) { + Ok(line) => line, + Err(line) => line - 1, + }; + + let line_start = self.line_starts[line]; + let character = offset - line_start; + + Position::new(u32::try_from(line).unwrap_or(0), character) + } +} diff --git a/crates/djls-workspace/src/document/line_index.rs b/crates/djls-workspace/src/document/line_index.rs deleted file mode 100644 index cbbf6459..00000000 --- a/crates/djls-workspace/src/document/line_index.rs +++ /dev/null @@ -1,90 +0,0 @@ -use tower_lsp_server::lsp_types::Position; - -#[derive(Clone, Debug)] -pub struct LineIndex { - pub line_starts: Vec, - pub line_starts_utf16: Vec, - pub length: u32, - pub length_utf16: u32, -} - -impl LineIndex { - #[must_use] - pub fn new(text: &str) -> Self { - let mut line_starts = vec![0]; - let mut line_starts_utf16 = vec![0]; - let mut pos_utf8 = 0; - let mut pos_utf16 = 0; - - for c in text.chars() { - pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); - pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); - if c == '\n' { - line_starts.push(pos_utf8); - line_starts_utf16.push(pos_utf16); - } - } - - Self { - line_starts, - line_starts_utf16, - length: pos_utf8, - length_utf16: pos_utf16, - } - } - - #[must_use] - pub fn offset(&self, position: Position) -> Option { - let line_start = self.line_starts.get(position.line as usize)?; - - Some(line_start + position.character) - } - - /// Convert UTF-16 LSP position to UTF-8 byte offset - pub fn offset_utf16(&self, position: Position, text: &str) -> Option { - let line_start_utf8 = self.line_starts.get(position.line as usize)?; - let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; - - // If position is at start of line, return UTF-8 line start - if position.character == 0 { - return Some(*line_start_utf8); - } - - // Find the line text - let next_line_start = self - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(self.length); - - let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; - - // Convert UTF-16 character offset to UTF-8 byte offset within the line - let mut utf16_pos = 0; - let mut utf8_pos = 0; - - for c in line_text.chars() { - if utf16_pos >= position.character { - break; - } - utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); - utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); - } - - Some(line_start_utf8 + utf8_pos) - } - - #[allow(dead_code)] - #[must_use] - pub fn position(&self, offset: u32) -> Position { - let line = match self.line_starts.binary_search(&offset) { - Ok(line) => line, - Err(line) => line - 1, - }; - - let line_start = self.line_starts[line]; - let character = offset - line_start; - - Position::new(u32::try_from(line).unwrap_or(0), character) - } -} diff --git a/crates/djls-workspace/src/document/mod.rs b/crates/djls-workspace/src/document/mod.rs deleted file mode 100644 index 93d443f2..00000000 --- a/crates/djls-workspace/src/document/mod.rs +++ /dev/null @@ -1,130 +0,0 @@ -mod language; -mod line_index; -mod template; - -pub use language::LanguageId; -pub use line_index::LineIndex; -pub use template::ClosingBrace; -pub use template::TemplateTagContext; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; - -use crate::FileId; - -#[derive(Clone, Debug)] -pub struct TextDocument { - pub uri: String, - pub version: i32, - pub language_id: LanguageId, - pub(crate) file_id: FileId, - line_index: LineIndex, -} - -impl TextDocument { - pub(crate) fn new( - uri: String, - version: i32, - language_id: LanguageId, - file_id: FileId, - content: &str, - ) -> Self { - let line_index = LineIndex::new(content); - Self { - uri, - version, - language_id, - file_id, - line_index, - } - } - - pub(crate) fn file_id(&self) -> FileId { - self.file_id - } - - pub fn line_index(&self) -> &LineIndex { - &self.line_index - } - - pub fn get_content<'a>(&self, content: &'a str) -> &'a str { - content - } - - pub fn get_line(&self, content: &str, line: u32) -> Option { - let line_start = *self.line_index.line_starts.get(line as usize)?; - let line_end = self - .line_index - .line_starts - .get(line as usize + 1) - .copied() - .unwrap_or(self.line_index.length); - - Some(content[line_start as usize..line_end as usize].to_string()) - } - - pub fn get_text_range(&self, content: &str, range: Range) -> Option { - let start_offset = self.line_index.offset(range.start)? as usize; - let end_offset = self.line_index.offset(range.end)? as usize; - - Some(content[start_offset..end_offset].to_string()) - } - - pub fn get_template_tag_context( - &self, - content: &str, - position: Position, - ) -> Option { - let start = self.line_index.line_starts.get(position.line as usize)?; - let end = self - .line_index - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(self.line_index.length); - - let line = &content[*start as usize..end as usize]; - let char_pos: usize = position.character.try_into().ok()?; - let prefix = &line[..char_pos]; - let rest_of_line = &line[char_pos..]; - let rest_trimmed = rest_of_line.trim_start(); - - prefix.rfind("{%").map(|tag_start| { - // Check if we're immediately after {% with no space - let needs_leading_space = prefix.ends_with("{%"); - - let closing_brace = if rest_trimmed.starts_with("%}") { - ClosingBrace::FullClose - } else if rest_trimmed.starts_with('}') { - ClosingBrace::PartialClose - } else { - ClosingBrace::None - }; - - TemplateTagContext { - partial_tag: prefix[tag_start + 2..].trim().to_string(), - closing_brace, - needs_leading_space, - } - }) - } - - pub fn position_to_offset(&self, position: Position) -> Option { - self.line_index.offset(position) - } - - pub fn offset_to_position(&self, offset: u32) -> Position { - self.line_index.position(offset) - } - - pub fn update_content(&mut self, content: &str) { - self.line_index = LineIndex::new(content); - } - - pub fn version(&self) -> i32 { - self.version - } - - pub fn language_id(&self) -> LanguageId { - self.language_id.clone() - } -} diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs new file mode 100644 index 00000000..26603d4f --- /dev/null +++ b/crates/djls-workspace/src/fs.rs @@ -0,0 +1,269 @@ +//! File system abstraction following Ruff's pattern +//! +//! This module provides the `FileSystem` trait that abstracts file I/O operations. +//! This allows the LSP to work with both real files and in-memory overlays. + +use dashmap::DashMap; +use std::io; +use std::path::Path; +use std::sync::Arc; +use url::Url; + +use crate::document::TextDocument; + +/// Trait for file system operations +/// +/// This follows Ruff's pattern of abstracting file system operations behind a trait, +/// allowing different implementations for testing, in-memory operation, and real file access. +pub trait FileSystem: Send + Sync { + /// Read the entire contents of a file + fn read_to_string(&self, path: &Path) -> io::Result; + + /// Check if a path exists + fn exists(&self, path: &Path) -> bool; + + /// Check if a path is a file + fn is_file(&self, path: &Path) -> bool; + + /// Check if a path is a directory + fn is_directory(&self, path: &Path) -> bool; + + /// List directory contents + fn read_directory(&self, path: &Path) -> io::Result>; + + /// Get file metadata (size, modified time, etc.) + fn metadata(&self, path: &Path) -> io::Result; +} + +/// Standard file system implementation that uses `std::fs` +pub struct OsFileSystem; + +impl FileSystem for OsFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + std::fs::read_to_string(path) + } + + fn exists(&self, path: &Path) -> bool { + path.exists() + } + + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + fn is_directory(&self, path: &Path) -> bool { + path.is_dir() + } + + fn read_directory(&self, path: &Path) -> io::Result> { + std::fs::read_dir(path)? + .map(|entry| entry.map(|e| e.path())) + .collect() + } + + fn metadata(&self, path: &Path) -> io::Result { + std::fs::metadata(path) + } +} + +/// LSP file system that intercepts reads for overlay files +/// +/// This implements Ruff's two-layer architecture where Layer 1 (LSP overlays) +/// takes precedence over Layer 2 (Salsa database). When a file is read, +/// this system first checks for an overlay (in-memory changes) and returns +/// that content. If no overlay exists, it falls back to reading from disk. +pub struct WorkspaceFileSystem { + /// In-memory overlays that take precedence over disk files + /// Maps URL to `TextDocument` containing current content + buffers: Arc>, + /// Fallback file system for disk operations + disk: Arc, +} + +impl WorkspaceFileSystem { + /// Create a new [`LspFileSystem`] with the given overlay storage and fallback + pub fn new(buffers: Arc>, disk: Arc) -> Self { + Self { buffers, disk } + } +} + +impl FileSystem for WorkspaceFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + if let Some(document) = path_to_url(path).and_then(|url| self.buffers.get(&url)) { + Ok(document.content().to_string()) + } else { + self.disk.read_to_string(path) + } + } + + fn exists(&self, path: &Path) -> bool { + path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + || self.disk.exists(path) + } + + fn is_file(&self, path: &Path) -> bool { + path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + || self.disk.is_file(path) + } + + fn is_directory(&self, path: &Path) -> bool { + // Overlays are never directories, so just delegate + self.disk.is_directory(path) + } + + fn read_directory(&self, path: &Path) -> io::Result> { + // Overlays are never directories, so just delegate + self.disk.read_directory(path) + } + + fn metadata(&self, path: &Path) -> io::Result { + // For overlays, we could synthesize metadata, but for simplicity, + // fall back to disk. This might need refinement for edge cases. + self.disk.metadata(path) + } +} + +/// Convert a file path to URL for overlay lookup +/// +/// This is a simplified conversion - in a full implementation, +/// you might want more robust path-to-URL conversion +fn path_to_url(path: &Path) -> Option { + if let Ok(absolute_path) = std::fs::canonicalize(path) { + return Url::from_file_path(absolute_path).ok(); + } + + // For test scenarios where the file doesn't exist on disk, + // try to create URL from the path directly if it's absolute + if path.is_absolute() { + return Url::from_file_path(path).ok(); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::TextDocument; + use crate::language::LanguageId; + + /// In-memory file system for testing + pub struct InMemoryFileSystem { + files: std::collections::HashMap, + } + + impl InMemoryFileSystem { + pub fn new() -> Self { + Self { + files: std::collections::HashMap::new(), + } + } + + pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { + self.files.insert(path, content); + } + } + + impl FileSystem for InMemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + // Simplified for testing - no directories in memory filesystem + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + // Simplified for testing + Ok(Vec::new()) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Metadata not supported in memory filesystem", + )) + } + } + + #[test] + fn test_lsp_filesystem_overlay_precedence() { + // Create a memory filesystem with some content + let mut memory_fs = InMemoryFileSystem::new(); + memory_fs.add_file( + std::path::PathBuf::from("/test/file.py"), + "original content".to_string(), + ); + + // Create overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create LspFileSystem with memory fallback + let lsp_fs = WorkspaceFileSystem::new(overlays.clone(), Arc::new(memory_fs)); + + // Before adding overlay, should read from fallback + let path = std::path::Path::new("/test/file.py"); + assert_eq!(lsp_fs.read_to_string(path).unwrap(), "original content"); + + // Add overlay - this simulates having an open document with changes + let url = Url::from_file_path("/test/file.py").unwrap(); + let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); + overlays.insert(url, document); + + // Now should read from overlay + assert_eq!(lsp_fs.read_to_string(path).unwrap(), "overlay content"); + } + + #[test] + fn test_lsp_filesystem_fallback_when_no_overlay() { + // Create memory filesystem + let mut memory_fs = InMemoryFileSystem::new(); + memory_fs.add_file( + std::path::PathBuf::from("/test/file.py"), + "disk content".to_string(), + ); + + // Create empty overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create LspFileSystem + let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + + // Should fall back to disk when no overlay exists + let path = std::path::Path::new("/test/file.py"); + assert_eq!(lsp_fs.read_to_string(path).unwrap(), "disk content"); + } + + #[test] + fn test_lsp_filesystem_other_operations_delegate() { + // Create memory filesystem + let mut memory_fs = InMemoryFileSystem::new(); + memory_fs.add_file( + std::path::PathBuf::from("/test/file.py"), + "content".to_string(), + ); + + // Create LspFileSystem + let overlays = Arc::new(DashMap::new()); + let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + + let path = std::path::Path::new("/test/file.py"); + + // These should delegate to the fallback filesystem + assert!(lsp_fs.exists(path)); + assert!(lsp_fs.is_file(path)); + assert!(!lsp_fs.is_directory(path)); + } +} diff --git a/crates/djls-workspace/src/document/language.rs b/crates/djls-workspace/src/language.rs similarity index 79% rename from crates/djls-workspace/src/document/language.rs rename to crates/djls-workspace/src/language.rs index 65c322ab..8db778f1 100644 --- a/crates/djls-workspace/src/document/language.rs +++ b/crates/djls-workspace/src/language.rs @@ -2,8 +2,10 @@ use crate::FileKind; #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { + Html, HtmlDjango, Other, + PlainText, Python, } @@ -11,6 +13,8 @@ impl From<&str> for LanguageId { fn from(language_id: &str) -> Self { match language_id { "django-html" | "htmldjango" => Self::HtmlDjango, + "html" => Self::Html, + "plaintext" => Self::PlainText, "python" => Self::Python, _ => Self::Other, } @@ -28,7 +32,7 @@ impl From for FileKind { match language_id { LanguageId::Python => Self::Python, LanguageId::HtmlDjango => Self::Template, - LanguageId::Other => Self::Other, + LanguageId::Html | LanguageId::PlainText | LanguageId::Other => Self::Other, } } } diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 9fbb34fc..22e0faf1 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,25 +1,13 @@ -mod bridge; pub mod db; mod document; -mod lsp_system; -mod system; +mod fs; +mod language; +mod template; pub use db::Database; -pub use document::{TextDocument, LanguageId}; -pub use system::{FileSystem, StdFileSystem}; - -/// File classification for routing to analyzers. -/// -/// [`FileKind`] determines how a file should be processed by downstream analyzers. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub enum FileKind { - /// Python source file - Python, - /// Django template file - Template, - /// Other file type - Other, -} +pub use document::TextDocument; +pub use fs::{FileSystem, OsFileSystem, WorkspaceFileSystem}; +pub use language::LanguageId; /// Stable, compact identifier for files across the subsystem. /// @@ -43,3 +31,28 @@ impl FileId { self.0 } } + +/// File classification for routing to analyzers. +/// +/// [`FileKind`] determines how a file should be processed by downstream analyzers. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKind { + /// Python source file + Python, + /// Django template file + Template, + /// Other file type + Other, +} + +impl FileKind { + /// Determine `FileKind` from a file path extension. + #[must_use] + pub fn from_path(path: &std::path::Path) -> Self { + match path.extension().and_then(|s| s.to_str()) { + Some("py") => FileKind::Python, + Some("html" | "htm") => FileKind::Template, + _ => FileKind::Other, + } + } +} diff --git a/crates/djls-workspace/src/lsp_system.rs b/crates/djls-workspace/src/lsp_system.rs deleted file mode 100644 index b03c8e82..00000000 --- a/crates/djls-workspace/src/lsp_system.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! LSP-aware file system wrapper that handles overlays -//! -//! This is the KEY pattern from Ruff - the LspSystem wraps a FileSystem -//! and intercepts reads to check for overlays first. This allows unsaved -//! changes to be used without going through Salsa. - -use std::collections::HashMap; -use std::io; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use url::Url; - -use crate::system::FileSystem; - -/// LSP-aware file system that checks overlays before disk -/// -/// This is the critical piece that makes overlays work efficiently in Ruff's -/// architecture. Instead of updating Salsa for every keystroke, we intercept -/// file reads here and return overlay content when available. -pub struct LspSystem { - /// The underlying file system (usually StdFileSystem) - inner: Arc, - - /// Map of open document URLs to their overlay content - overlays: HashMap, -} - -impl LspSystem { - /// Create a new LspSystem wrapping the given file system - pub fn new(file_system: Arc) -> Self { - Self { - inner: file_system, - overlays: HashMap::new(), - } - } - - /// Set overlay content for a document - pub fn set_overlay(&mut self, url: Url, content: String) { - self.overlays.insert(url, content); - } - - /// Remove overlay content for a document - pub fn remove_overlay(&mut self, url: &Url) { - self.overlays.remove(url); - } - - /// Check if a document has an overlay - pub fn has_overlay(&self, url: &Url) -> bool { - self.overlays.contains_key(url) - } - - /// Get overlay content if it exists - pub fn get_overlay(&self, url: &Url) -> Option<&String> { - self.overlays.get(url) - } - - /// Convert a URL to a file path - fn url_to_path(url: &Url) -> Option { - if url.scheme() == "file" { - url.to_file_path().ok().or_else(|| { - // Fallback for simple conversion - Some(PathBuf::from(url.path())) - }) - } else { - None - } - } -} - -impl FileSystem for LspSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - // First check if we have an overlay for this path - // Convert path to URL for lookup - let url = Url::from_file_path(path) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid path"))?; - - if let Some(content) = self.overlays.get(&url) { - // Return overlay content instead of reading from disk - return Ok(content.clone()); - } - - // No overlay, read from underlying file system - self.inner.read_to_string(path) - } - - fn exists(&self, path: &Path) -> bool { - // Check overlays first - if let Ok(url) = Url::from_file_path(path) { - if self.overlays.contains_key(&url) { - return true; - } - } - - self.inner.exists(path) - } - - fn is_file(&self, path: &Path) -> bool { - // Overlays are always files - if let Ok(url) = Url::from_file_path(path) { - if self.overlays.contains_key(&url) { - return true; - } - } - - self.inner.is_file(path) - } - - fn is_directory(&self, path: &Path) -> bool { - // Overlays are never directories - if let Ok(url) = Url::from_file_path(path) { - if self.overlays.contains_key(&url) { - return false; - } - } - - self.inner.is_directory(path) - } - - fn read_directory(&self, path: &Path) -> io::Result> { - // Overlays don't affect directory listings - self.inner.read_directory(path) - } - - fn metadata(&self, path: &Path) -> io::Result { - // Can't provide metadata for overlays - self.inner.metadata(path) - } -} - -/// Extension trait for working with URL-based overlays -pub trait LspSystemExt { - /// Read file content by URL, checking overlays first - fn read_url(&self, url: &Url) -> io::Result; -} - -impl LspSystemExt for LspSystem { - fn read_url(&self, url: &Url) -> io::Result { - // Check overlays first - if let Some(content) = self.overlays.get(url) { - return Ok(content.clone()); - } - - // Convert URL to path and read from file system - if let Some(path_buf) = Self::url_to_path(url) { - self.inner.read_to_string(&path_buf) - } else { - Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("Cannot convert URL to path: {}", url), - )) - } - } -} \ No newline at end of file diff --git a/crates/djls-workspace/src/system.rs b/crates/djls-workspace/src/system.rs deleted file mode 100644 index 04a1b8a2..00000000 --- a/crates/djls-workspace/src/system.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! File system abstraction following Ruff's pattern -//! -//! This module provides the FileSystem trait that abstracts file I/O operations. -//! This allows the LSP to work with both real files and in-memory overlays. - -use std::io; -use std::path::Path; - -/// Trait for file system operations -/// -/// This follows Ruff's pattern of abstracting file system operations behind a trait, -/// allowing different implementations for testing, in-memory operation, and real file access. -pub trait FileSystem: Send + Sync { - /// Read the entire contents of a file - fn read_to_string(&self, path: &Path) -> io::Result; - - /// Check if a path exists - fn exists(&self, path: &Path) -> bool; - - /// Check if a path is a file - fn is_file(&self, path: &Path) -> bool; - - /// Check if a path is a directory - fn is_directory(&self, path: &Path) -> bool; - - /// List directory contents - fn read_directory(&self, path: &Path) -> io::Result>; - - /// Get file metadata (size, modified time, etc.) - fn metadata(&self, path: &Path) -> io::Result; -} - -/// Standard file system implementation that uses std::fs -pub struct StdFileSystem; - -impl FileSystem for StdFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - std::fs::read_to_string(path) - } - - fn exists(&self, path: &Path) -> bool { - path.exists() - } - - fn is_file(&self, path: &Path) -> bool { - path.is_file() - } - - fn is_directory(&self, path: &Path) -> bool { - path.is_dir() - } - - fn read_directory(&self, path: &Path) -> io::Result> { - let mut entries = Vec::new(); - for entry in std::fs::read_dir(path)? { - entries.push(entry?.path()); - } - Ok(entries) - } - - fn metadata(&self, path: &Path) -> io::Result { - std::fs::metadata(path) - } -} - -/// In-memory file system for testing -#[cfg(test)] -pub struct MemoryFileSystem { - files: std::collections::HashMap, -} - -#[cfg(test)] -impl MemoryFileSystem { - pub fn new() -> Self { - Self { - files: std::collections::HashMap::new(), - } - } - - pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { - self.files.insert(path, content); - } -} - -#[cfg(test)] -impl FileSystem for MemoryFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - self.files - .get(path) - .cloned() - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) - } - - fn exists(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_file(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_directory(&self, _path: &Path) -> bool { - // Simplified for testing - no directories in memory filesystem - false - } - - fn read_directory(&self, _path: &Path) -> io::Result> { - // Simplified for testing - Ok(Vec::new()) - } - - fn metadata(&self, _path: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "Metadata not supported in memory filesystem", - )) - } -} \ No newline at end of file diff --git a/crates/djls-workspace/src/document/template.rs b/crates/djls-workspace/src/template.rs similarity index 100% rename from crates/djls-workspace/src/document/template.rs rename to crates/djls-workspace/src/template.rs diff --git a/crates/djls-workspace/src/test_db.rs b/crates/djls-workspace/src/test_db.rs deleted file mode 100644 index 92683d5f..00000000 --- a/crates/djls-workspace/src/test_db.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Test module to explore Salsa thread safety - -#[cfg(test)] -mod tests { - use crate::db::Database; - use std::thread; - - #[test] - fn test_database_clone() { - let db = Database::new(); - let _db2 = db.clone(); - println!("✅ Database can be cloned"); - } - - #[test] - #[ignore] // This will fail - fn test_database_send() { - let db = Database::new(); - let db2 = db.clone(); - - thread::spawn(move || { - let _ = db2; - }).join().unwrap(); - } -} diff --git a/task_order.md b/task_order.md new file mode 100644 index 00000000..137a4023 --- /dev/null +++ b/task_order.md @@ -0,0 +1,61 @@ +# Revised Task Order for Ruff Pattern Implementation + +## The Correct Architecture Understanding + +Based on Ruff expert clarification: +- **SourceFile should NOT store text content** (our current implementation is wrong) +- **File content is read on-demand** through a `source_text` tracked function +- **Overlays are never Salsa inputs**, they're read through FileSystem +- **File revision triggers invalidation**, not content changes + +## Implementation Order + +### Phase 1: Database Foundation +1. **task-129** - Complete Database FileSystem integration + - Database needs access to LspFileSystem to read files + - This enables tracked functions to read through FileSystem + +### Phase 2: Salsa Input Restructuring +2. **task-126** - Bridge Salsa queries to LspFileSystem + - Remove `text` field from SourceFile + - Add `path` and `revision` fields + - Create `source_text` tracked function + +### Phase 3: Query Updates +3. **task-95** - Update template parsing to use source_text query + - Update all queries to use `source_text(db, file)` + - Remove direct text access from SourceFile + +### Phase 4: LSP Integration +4. **task-112** - Add file revision tracking + - Bump file revision when overlays change + - This triggers Salsa invalidation + +### Phase 5: Testing +5. **task-127** - Test overlay behavior and Salsa integration + - Verify overlays work correctly + - Test invalidation behavior + +## Key Changes from Current Implementation + +Current (WRONG): +```rust +#[salsa::input] +pub struct SourceFile { + pub text: Arc, // ❌ Storing content in Salsa +} +``` + +Target (RIGHT): +```rust +#[salsa::input] +pub struct SourceFile { + pub path: PathBuf, + pub revision: u32, // ✅ Only track changes +} + +#[salsa::tracked] +pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { + // Read through FileSystem (checks overlays first) +} +``` From 21403df0ba1a50e418021efbb9e3b2a06901e233 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 28 Aug 2025 23:07:44 -0500 Subject: [PATCH 14/56] tests --- crates/djls-workspace/src/db.rs | 175 ++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 0105616c..8a9220f7 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -394,3 +394,178 @@ pub fn template_errors_by_path(db: &dyn Db, file_path: FilePath) -> Arc<[String] pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::TextDocument; + use crate::fs::WorkspaceFileSystem; + use crate::language::LanguageId; + use dashmap::DashMap; + use salsa::Setter; + use std::collections::HashMap; + use std::io; + use url::Url; + + // Simple in-memory filesystem for testing + struct InMemoryFileSystem { + files: HashMap, + } + + impl InMemoryFileSystem { + fn new() -> Self { + Self { + files: HashMap::new(), + } + } + + fn add_file(&mut self, path: PathBuf, content: String) { + self.files.insert(path, content); + } + } + + impl FileSystem for InMemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + Ok(vec![]) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, "Not supported")) + } + } + + #[test] + fn test_parse_template_with_overlay() { + // Create a memory filesystem with initial template content + let mut memory_fs = InMemoryFileSystem::new(); + let template_path = PathBuf::from("/test/template.html"); + memory_fs.add_file( + template_path.clone(), + "{% block content %}Original{% endblock %}".to_string(), + ); + + // Create overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create WorkspaceFileSystem that checks overlays first + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(memory_fs), + )); + + // Create database with the file system + let files = Arc::new(DashMap::new()); + let mut db = Database::new(file_system, files); + + // Create a SourceFile for the template + let file = db.get_or_create_file(template_path.clone()); + + // Parse template - should get original content from disk + let ast1 = parse_template(&db, file).expect("Should parse template"); + assert!(ast1.errors.is_empty(), "Should have no errors"); + + // Add an overlay with updated content + let url = Url::from_file_path(&template_path).unwrap(); + let updated_document = TextDocument::new( + "{% block content %}Updated from overlay{% endblock %}".to_string(), + 2, + LanguageId::Other, + ); + overlays.insert(url, updated_document); + + // Bump the file revision to trigger re-parse + file.set_revision(&mut db).to(1); + + // Parse again - should now get overlay content + let ast2 = parse_template(&db, file).expect("Should parse template"); + assert!(ast2.errors.is_empty(), "Should have no errors"); + + // Verify the content changed (we can't directly check the text, + // but the AST should be different) + // The AST will have different content in the block + assert_ne!( + format!("{:?}", ast1.ast), + format!("{:?}", ast2.ast), + "AST should change when overlay is added" + ); + } + + #[test] + fn test_parse_template_invalidation_on_revision_change() { + // Create a memory filesystem + let mut memory_fs = InMemoryFileSystem::new(); + let template_path = PathBuf::from("/test/template.html"); + memory_fs.add_file( + template_path.clone(), + "{% if true %}Initial{% endif %}".to_string(), + ); + + // Create overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create WorkspaceFileSystem + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(memory_fs), + )); + + // Create database + let files = Arc::new(DashMap::new()); + let mut db = Database::new(file_system, files); + + // Create a SourceFile for the template + let file = db.get_or_create_file(template_path.clone()); + + // Parse template first time + let ast1 = parse_template(&db, file).expect("Should parse"); + + // Parse again without changing revision - should return same Arc (cached) + let ast2 = parse_template(&db, file).expect("Should parse"); + assert!(Arc::ptr_eq(&ast1, &ast2), "Should return cached result"); + + // Update overlay content + let url = Url::from_file_path(&template_path).unwrap(); + let updated_document = TextDocument::new( + "{% if false %}Changed{% endif %}".to_string(), + 2, + LanguageId::Other, + ); + overlays.insert(url, updated_document); + + // Bump revision to trigger invalidation + file.set_revision(&mut db).to(1); + + // Parse again - should get different result due to invalidation + let ast3 = parse_template(&db, file).expect("Should parse"); + assert!( + !Arc::ptr_eq(&ast1, &ast3), + "Should re-execute after revision change" + ); + + // Content should be different + assert_ne!( + format!("{:?}", ast1.ast), + format!("{:?}", ast3.ast), + "AST should be different after content change" + ); + } +} From 2dd779bcdac1a8e4a8a1068aa36a80ff07d4e36e Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 07:55:37 -0500 Subject: [PATCH 15/56] Fix overlay bug: Salsa wasn't re-reading from buffers when files were opened The core issue was that when a file was opened in the LSP, if it had already been read from disk, Salsa would return cached content instead of reading from the overlay system. This happened because opening a file didn't bump its revision, so Salsa had no reason to invalidate its cache. Key changes: - Created Buffers abstraction to encapsulate shared buffer storage - Fixed Session::open_document() to bump revision when file already exists - Added comprehensive integration tests to verify overlay behavior - Refactored WorkspaceFileSystem to use Buffers instead of raw DashMap This ensures that overlays always take precedence over disk content, fixing the issue where LSP edits weren't being reflected in template parsing. --- Cargo.lock | 1 + crates/djls-server/Cargo.toml | 3 + crates/djls-server/src/lib.rs | 7 +- crates/djls-server/src/session.rs | 109 +++-- crates/djls-server/tests/lsp_integration.rs | 463 ++++++++++++++++++++ crates/djls-workspace/src/buffers.rs | 68 +++ crates/djls-workspace/src/db.rs | 13 +- crates/djls-workspace/src/fs.rs | 73 +-- crates/djls-workspace/src/lib.rs | 2 + 9 files changed, 650 insertions(+), 89 deletions(-) create mode 100644 crates/djls-server/tests/lsp_integration.rs create mode 100644 crates/djls-workspace/src/buffers.rs diff --git a/Cargo.lock b/Cargo.lock index d017be40..efc1695b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "salsa", "serde", "serde_json", + "tempfile", "tokio", "tower-lsp-server", "tracing", diff --git a/crates/djls-server/Cargo.toml b/crates/djls-server/Cargo.toml index e3f27f88..7829bf05 100644 --- a/crates/djls-server/Cargo.toml +++ b/crates/djls-server/Cargo.toml @@ -31,5 +31,8 @@ url = { workspace = true } [build-dependencies] djls-dev = { workspace = true } +[dev-dependencies] +tempfile = { workspace = true } + [lints] workspace = true diff --git a/crates/djls-server/src/lib.rs b/crates/djls-server/src/lib.rs index b601c7aa..ba468301 100644 --- a/crates/djls-server/src/lib.rs +++ b/crates/djls-server/src/lib.rs @@ -1,8 +1,8 @@ mod client; mod logging; mod queue; -mod server; -mod session; +pub mod server; +pub mod session; use std::io::IsTerminal; @@ -10,7 +10,8 @@ use anyhow::Result; use tower_lsp_server::LspService; use tower_lsp_server::Server; -use crate::server::DjangoLanguageServer; +pub use crate::server::DjangoLanguageServer; +pub use crate::session::Session; pub fn run() -> Result<()> { if std::io::stdin().is_terminal() { diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 6b4adbb5..189a3331 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -39,7 +39,7 @@ use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::{ db::{Database, SourceFile}, - FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, + Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, }; use percent_encoding::percent_decode_str; use salsa::{Setter, StorageHandle}; @@ -77,22 +77,23 @@ pub struct Session { /// LSP server settings settings: Settings, - /// Layer 1: Thread-safe overlay storage (Arc>) + /// Layer 1: Shared buffer storage for open documents /// /// This implements Ruff's two-layer architecture where Layer 1 contains - /// LSP overlays that take precedence over disk files. The overlays map - /// document URLs to TextDocuments containing current in-memory content. + /// open document buffers that take precedence over disk files. The buffers + /// are shared between Session (which manages them) and WorkspaceFileSystem + /// (which reads from them). /// /// Key properties: - /// - Thread-safe via Arc for Send+Sync requirements + /// - Thread-safe via the Buffers abstraction /// - Contains full TextDocument with content, version, and metadata /// - Never becomes Salsa inputs - only intercepted at read time - overlays: Arc>, + buffers: Buffers, - /// File system abstraction with overlay interception + /// File system abstraction with buffer interception /// - /// This LspFileSystem bridges Layer 1 (overlays) and Layer 2 (Salsa). - /// It intercepts FileSystem::read_to_string() calls to return overlay + /// This WorkspaceFileSystem bridges Layer 1 (buffers) and Layer 2 (Salsa). + /// It intercepts FileSystem::read_to_string() calls to return buffer /// content when available, falling back to disk otherwise. file_system: Arc, @@ -132,10 +133,10 @@ impl Session { (None, Settings::default()) }; - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); let files = Arc::new(DashMap::new()); let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(OsFileSystem), )); let db_handle = Database::new(file_system.clone(), files.clone()) @@ -146,7 +147,7 @@ impl Session { Self { project, settings, - overlays, + buffers, file_system, files, client_capabilities: params.capabilities.clone(), @@ -230,32 +231,32 @@ impl Session { self.file_system.clone() } - /// Set or update an overlay for the given document URL + /// Set or update a buffer for the given document URL /// /// This implements Layer 1 of Ruff's architecture - storing in-memory /// document changes that take precedence over disk content. #[allow(dead_code)] // Used in tests pub fn set_overlay(&self, url: Url, document: TextDocument) { - self.overlays.insert(url, document); + self.buffers.open(url, document); } - /// Remove an overlay for the given document URL + /// Remove a buffer for the given document URL /// /// After removal, file reads will fall back to disk content. #[allow(dead_code)] // Used in tests pub fn remove_overlay(&self, url: &Url) -> Option { - self.overlays.remove(url).map(|(_, doc)| doc) + self.buffers.close(url) } - /// Check if an overlay exists for the given URL + /// Check if a buffer exists for the given URL #[allow(dead_code)] pub fn has_overlay(&self, url: &Url) -> bool { - self.overlays.contains_key(url) + self.buffers.contains(url) } - /// Get a copy of an overlay document + /// Get a copy of a buffered document pub fn get_overlay(&self, url: &Url) -> Option { - self.overlays.get(url).map(|doc| doc.clone()) + self.buffers.get(url) } /// Takes exclusive ownership of the database handle for mutation operations. @@ -375,41 +376,61 @@ impl Session { // These methods encapsulate the two-layer architecture coordination: // Layer 1 (overlays) and Layer 2 (Salsa revision tracking) - /// Handle opening a document - sets overlay and creates file. + /// Handle opening a document - sets buffer and creates file. /// /// This method coordinates both layers: - /// - Layer 1: Stores the document content in overlays + /// - Layer 1: Stores the document content in buffers /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) pub fn open_document(&mut self, url: Url, document: TextDocument) { tracing::debug!("Opening document: {}", url); - // Layer 1: Set overlay - self.overlays.insert(url.clone(), document); + // Layer 1: Set buffer + self.buffers.open(url.clone(), document); - // Layer 2: Create file if needed (starts at revision 0) + // Layer 2: Create file and bump revision if it already exists + // This is crucial: if the file was already read from disk, we need to + // invalidate Salsa's cache so it re-reads through the buffer system if let Some(path) = self.url_to_path(&url) { self.with_db_mut(|db| { + // Check if file already exists (was previously read from disk) + let already_exists = db.has_file(&path); let file = db.get_or_create_file(path.clone()); - tracing::debug!( - "Created/retrieved SourceFile for {}: revision {}", - path.display(), - file.revision(db) - ); + + if already_exists { + // File was already read - bump revision to invalidate cache + let current_rev = file.revision(db); + let new_rev = current_rev + 1; + file.set_revision(db).to(new_rev); + tracing::debug!( + "Bumped revision for {} on open: {} -> {}", + path.display(), + current_rev, + new_rev + ); + + } else { + // New file - starts at revision 0 + tracing::debug!( + "Created new SourceFile for {}: revision {}", + path.display(), + file.revision(db) + ); + } }); } } - /// Handle document changes - updates overlay and bumps revision. + /// Handle document changes - updates buffer and bumps revision. /// /// This method coordinates both layers: - /// - Layer 1: Updates the document content in overlays + /// - Layer 1: Updates the document content in buffers /// - Layer 2: Bumps the file revision to trigger Salsa invalidation pub fn update_document(&mut self, url: Url, document: TextDocument) { let version = document.version(); tracing::debug!("Updating document: {} (version {})", url, version); - // Layer 1: Update overlay - self.overlays.insert(url.clone(), document); + // Layer 1: Update buffer + self.buffers.update(url.clone(), document); // Layer 2: Bump revision to trigger invalidation if let Some(path) = self.url_to_path(&url) { @@ -417,25 +438,25 @@ impl Session { } } - /// Handle closing a document - removes overlay and bumps revision. + /// Handle closing a document - removes buffer and bumps revision. /// /// This method coordinates both layers: - /// - Layer 1: Removes the overlay (falls back to disk) + /// - Layer 1: Removes the buffer (falls back to disk) /// - Layer 2: Bumps revision to trigger re-read from disk /// /// Returns the removed document if it existed. pub fn close_document(&mut self, url: &Url) -> Option { tracing::debug!("Closing document: {}", url); - // Layer 1: Remove overlay - let removed = self.overlays.remove(url).map(|(_, doc)| { + // Layer 1: Remove buffer + let removed = self.buffers.close(url); + if let Some(ref doc) = removed { tracing::debug!( - "Removed overlay for closed document: {} (was version {})", + "Removed buffer for closed document: {} (was version {})", url, doc.version() ); - doc - }); + } // Layer 2: Bump revision to trigger re-read from disk // We keep the file alive for potential re-opening @@ -512,10 +533,10 @@ impl Session { impl Default for Session { fn default() -> Self { - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); let files = Arc::new(DashMap::new()); let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(OsFileSystem), )); let db_handle = Database::new(file_system.clone(), files.clone()) @@ -529,7 +550,7 @@ impl Default for Session { db_handle, file_system, files, - overlays, + buffers, client_capabilities: lsp_types::ClientCapabilities::default(), } } diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs new file mode 100644 index 00000000..5c14607a --- /dev/null +++ b/crates/djls-server/tests/lsp_integration.rs @@ -0,0 +1,463 @@ +//! Integration tests for the LSP server's overlay → revision → invalidation flow +//! +//! These tests verify the complete two-layer architecture: +//! - Layer 1: LSP overlays (in-memory document state) +//! - Layer 2: Salsa database with revision tracking +//! +//! The tests ensure that document changes properly invalidate cached queries +//! and that overlays take precedence over disk content. + +use std::path::PathBuf; +use std::sync::Arc; + +use djls_server::DjangoLanguageServer; +use tempfile::TempDir; +use tower_lsp_server::lsp_types::{ + DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, + InitializeParams, InitializedParams, TextDocumentContentChangeEvent, TextDocumentIdentifier, + TextDocumentItem, Uri, VersionedTextDocumentIdentifier, WorkspaceFolder, +}; +use tower_lsp_server::LanguageServer; +use url::Url; + +/// Test helper that manages an LSP server instance for testing +struct TestServer { + server: DjangoLanguageServer, + _temp_dir: TempDir, + workspace_root: PathBuf, +} + +impl TestServer { + /// Create a new test server with a temporary workspace + async fn new() -> Self { + // Create temporary directory for test workspace + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let workspace_root = temp_dir.path().to_path_buf(); + + // Set up logging + let (non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); + + // Create server (guard is moved into server, so we return it too) + let server = DjangoLanguageServer::new(guard); + + // Initialize the server + let workspace_folder = WorkspaceFolder { + uri: format!("file://{}", workspace_root.display()) + .parse() + .unwrap(), + name: "test_workspace".to_string(), + }; + + let init_params = InitializeParams { + workspace_folders: Some(vec![workspace_folder]), + ..Default::default() + }; + + server + .initialize(init_params) + .await + .expect("Failed to initialize"); + server.initialized(InitializedParams {}).await; + + Self { + server, + _temp_dir: temp_dir, + workspace_root, + } + } + + /// Helper to create a file path in the test workspace + fn workspace_file(&self, name: &str) -> PathBuf { + self.workspace_root.join(name) + } + + /// Helper to create a file URL in the test workspace + fn workspace_url(&self, name: &str) -> Url { + Url::from_file_path(self.workspace_file(name)).unwrap() + } + + /// Open a document in the LSP server + async fn open_document(&self, file_name: &str, content: &str, version: i32) { + let params = DidOpenTextDocumentParams { + text_document: TextDocumentItem { + uri: self.workspace_url(file_name).to_string().parse().unwrap(), + language_id: if file_name.ends_with(".html") { + "html".to_string() + } else if file_name.ends_with(".py") { + "python".to_string() + } else { + "plaintext".to_string() + }, + version, + text: content.to_string(), + }, + }; + + self.server.did_open(params).await; + } + + /// Change a document in the LSP server + async fn change_document(&self, file_name: &str, new_content: &str, version: i32) { + let params = DidChangeTextDocumentParams { + text_document: VersionedTextDocumentIdentifier { + uri: self.workspace_url(file_name).to_string().parse().unwrap(), + version, + }, + content_changes: vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: new_content.to_string(), + }], + }; + + self.server.did_change(params).await; + } + + /// Close a document in the LSP server + async fn close_document(&self, file_name: &str) { + let params = DidCloseTextDocumentParams { + text_document: TextDocumentIdentifier { + uri: self.workspace_url(file_name).to_string().parse().unwrap(), + }, + }; + + self.server.did_close(params).await; + } + + /// Get the content of a file through the session's query system + async fn get_file_content(&self, file_name: &str) -> String { + let path = self.workspace_file(file_name); + self.server + .with_session_mut(|session| session.file_content(path)) + .await + } + + /// Write a file to disk in the test workspace + fn write_file(&self, file_name: &str, content: &str) { + let path = self.workspace_file(file_name); + std::fs::write(path, content).expect("Failed to write test file"); + } + + /// Check if a file has an overlay in the session + async fn has_overlay(&self, file_name: &str) -> bool { + let url = self.workspace_url(file_name); + self.server + .with_session(|session| session.get_overlay(&url).is_some()) + .await + } + + /// Get the revision of a file + async fn get_file_revision(&self, file_name: &str) -> Option { + let path = self.workspace_file(file_name); + self.server + .with_session_mut(|session| session.file_revision(&path)) + .await + } +} + +#[tokio::test] +async fn test_full_lsp_lifecycle() { + let server = TestServer::new().await; + let file_name = "test.html"; + + // Write initial content to disk + server.write_file(file_name, "

Disk Content

"); + + // 1. Test did_open creates overlay and file + server + .open_document(file_name, "

Overlay Content

", 1) + .await; + + // Verify overlay exists + assert!(server.has_overlay(file_name).await); + + // Verify overlay content is returned (not disk content) + let content = server.get_file_content(file_name).await; + assert_eq!(content, "

Overlay Content

"); + + // Verify file was created with revision 0 + let revision = server.get_file_revision(file_name).await; + assert_eq!(revision, Some(0)); + + // 2. Test did_change updates overlay and bumps revision + server + .change_document(file_name, "

Updated Content

", 2) + .await; + + // Verify content changed + let content = server.get_file_content(file_name).await; + assert_eq!(content, "

Updated Content

"); + + // Verify revision was bumped + let revision = server.get_file_revision(file_name).await; + assert_eq!(revision, Some(1)); + + // 3. Test did_close removes overlay and bumps revision + server.close_document(file_name).await; + + // Verify overlay is removed + assert!(!server.has_overlay(file_name).await); + + // Verify content now comes from disk + let content = server.get_file_content(file_name).await; + assert_eq!(content, "

Disk Content

"); + + // Verify revision was bumped again + let revision = server.get_file_revision(file_name).await; + assert_eq!(revision, Some(2)); +} + +#[tokio::test] +async fn test_overlay_precedence() { + let server = TestServer::new().await; + let file_name = "template.html"; + + // Write content to disk + server.write_file(file_name, "{% block content %}Disk{% endblock %}"); + + // Read content before overlay - should get disk content + let content = server.get_file_content(file_name).await; + assert_eq!(content, "{% block content %}Disk{% endblock %}"); + + // Open document with different content + server + .open_document(file_name, "{% block content %}Overlay{% endblock %}", 1) + .await; + + // Verify overlay content takes precedence + let content = server.get_file_content(file_name).await; + assert_eq!(content, "{% block content %}Overlay{% endblock %}"); + + // Close document + server.close_document(file_name).await; + + // Verify we're back to disk content + let content = server.get_file_content(file_name).await; + assert_eq!(content, "{% block content %}Disk{% endblock %}"); +} + +#[tokio::test] +async fn test_template_parsing_with_overlays() { + let server = TestServer::new().await; + let file_name = "template.html"; + + // Write initial template to disk + server.write_file(file_name, "{% if true %}Original{% endif %}"); + + // Open with different template content + server + .open_document( + file_name, + "{% for item in items %}{{ item }}{% endfor %}", + 1, + ) + .await; + use djls_workspace::db::parse_template; + + // Parse template through the session + let workspace_path = server.workspace_file(file_name); + let ast = server + .server + .with_session_mut(|session| { + session.with_db_mut(|db| { + let file = db.get_or_create_file(workspace_path); + parse_template(db, file) + }) + }) + .await; + + // Verify we parsed the overlay content (for loop), not disk content (if statement) + assert!(ast.is_some()); + let ast = ast.unwrap(); + let ast_str = format!("{:?}", ast.ast); + assert!(ast_str.contains("for") || ast_str.contains("For")); + assert!(!ast_str.contains("if") && !ast_str.contains("If")); +} + +#[tokio::test] +async fn test_multiple_documents_independent() { + let server = TestServer::new().await; + + // Open multiple documents + server.open_document("file1.html", "Content 1", 1).await; + server.open_document("file2.html", "Content 2", 1).await; + server.open_document("file3.html", "Content 3", 1).await; + + // Verify all have overlays + assert!(server.has_overlay("file1.html").await); + assert!(server.has_overlay("file2.html").await); + assert!(server.has_overlay("file3.html").await); + + // Change one document + server.change_document("file2.html", "Updated 2", 2).await; + + // Verify only file2 was updated + assert_eq!(server.get_file_content("file1.html").await, "Content 1"); + assert_eq!(server.get_file_content("file2.html").await, "Updated 2"); + assert_eq!(server.get_file_content("file3.html").await, "Content 3"); + + // Verify revision changes + assert_eq!(server.get_file_revision("file1.html").await, Some(0)); + assert_eq!(server.get_file_revision("file2.html").await, Some(1)); + assert_eq!(server.get_file_revision("file3.html").await, Some(0)); +} + +#[tokio::test] +async fn test_concurrent_overlay_updates() { + let server = Arc::new(TestServer::new().await); + + // Open initial documents + for i in 0..5 { + server + .open_document(&format!("file{}.html", i), &format!("Initial {}", i), 1) + .await; + } + + // Spawn concurrent tasks to update different documents + let mut handles = vec![]; + + for i in 0..5 { + let server_clone = Arc::clone(&server); + let handle = tokio::spawn(async move { + // Each task updates its document multiple times + for version in 2..10 { + server_clone + .change_document( + &format!("file{}.html", i), + &format!("Updated {} v{}", i, version), + version, + ) + .await; + + // Small delay to encourage interleaving + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + } + }); + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + handle.await.expect("Task failed"); + } + + // Verify final state of all documents + for i in 0..5 { + let content = server.get_file_content(&format!("file{}.html", i)).await; + assert_eq!(content, format!("Updated {} v9", i)); + + // Each document should have had 8 changes (versions 2-9) + let revision = server.get_file_revision(&format!("file{}.html", i)).await; + assert_eq!(revision, Some(8)); + } +} + +#[tokio::test] +async fn test_caching_behavior() { + let server = TestServer::new().await; + + // Open three template files + server + .open_document("template1.html", "{% block a %}1{% endblock %}", 1) + .await; + server + .open_document("template2.html", "{% block b %}2{% endblock %}", 1) + .await; + server + .open_document("template3.html", "{% block c %}3{% endblock %}", 1) + .await; + + // Parse all templates once to populate cache + for i in 1..=3 { + let _ = server + .get_file_content(&format!("template{}.html", i)) + .await; + } + + // Store initial revisions + let rev1_before = server.get_file_revision("template1.html").await.unwrap(); + let rev2_before = server.get_file_revision("template2.html").await.unwrap(); + let rev3_before = server.get_file_revision("template3.html").await.unwrap(); + + // Change only template2 + server + .change_document("template2.html", "{% block b %}CHANGED{% endblock %}", 2) + .await; + + // Verify only template2's revision changed + let rev1_after = server.get_file_revision("template1.html").await.unwrap(); + let rev2_after = server.get_file_revision("template2.html").await.unwrap(); + let rev3_after = server.get_file_revision("template3.html").await.unwrap(); + + assert_eq!( + rev1_before, rev1_after, + "template1 revision should not change" + ); + assert_eq!( + rev2_before + 1, + rev2_after, + "template2 revision should increment" + ); + assert_eq!( + rev3_before, rev3_after, + "template3 revision should not change" + ); + + // Verify content + assert_eq!( + server.get_file_content("template1.html").await, + "{% block a %}1{% endblock %}" + ); + assert_eq!( + server.get_file_content("template2.html").await, + "{% block b %}CHANGED{% endblock %}" + ); + assert_eq!( + server.get_file_content("template3.html").await, + "{% block c %}3{% endblock %}" + ); +} + +#[tokio::test] +async fn test_revision_tracking_across_lifecycle() { + let server = TestServer::new().await; + let file_name = "tracked.html"; + + // Create file on disk + server.write_file(file_name, "Initial"); + + // Open document - should create file with revision 0 + server.open_document(file_name, "Opened", 1).await; + assert_eq!(server.get_file_revision(file_name).await, Some(0)); + + // Change document multiple times + for i in 2..=5 { + server + .change_document(file_name, &format!("Change {}", i), i) + .await; + assert_eq!( + server.get_file_revision(file_name).await, + Some((i - 1) as u64), + "Revision should be {} after change {}", + i - 1, + i + ); + } + + // Close document - should bump revision one more time + server.close_document(file_name).await; + assert_eq!(server.get_file_revision(file_name).await, Some(5)); + + // Re-open document - file already exists, should bump revision to invalidate cache + server.open_document(file_name, "Reopened", 10).await; + assert_eq!( + server.get_file_revision(file_name).await, + Some(6), + "Revision should bump on re-open to invalidate cache" + ); + + // Change again + server.change_document(file_name, "Final", 11).await; + assert_eq!(server.get_file_revision(file_name).await, Some(7)); +} + diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs new file mode 100644 index 00000000..702220ac --- /dev/null +++ b/crates/djls-workspace/src/buffers.rs @@ -0,0 +1,68 @@ +//! Shared buffer storage for open documents +//! +//! This module provides the `Buffers` type which represents the in-memory +//! content of open files. These buffers are shared between the Session +//! (which manages document lifecycle) and the WorkspaceFileSystem (which +//! reads from them). + +use dashmap::DashMap; +use std::sync::Arc; +use url::Url; + +use crate::document::TextDocument; + +/// Shared buffer storage between Session and FileSystem +/// +/// Buffers represent the in-memory content of open files that takes +/// precedence over disk content when reading through the FileSystem. +/// This is the key abstraction that makes the sharing between Session +/// and WorkspaceFileSystem explicit and type-safe. +#[derive(Clone, Debug)] +pub struct Buffers { + inner: Arc>, +} + +impl Buffers { + /// Create a new empty buffer storage + pub fn new() -> Self { + Self { + inner: Arc::new(DashMap::new()), + } + } + + /// Open a document in the buffers + pub fn open(&self, url: Url, document: TextDocument) { + self.inner.insert(url, document); + } + + /// Update an open document + pub fn update(&self, url: Url, document: TextDocument) { + self.inner.insert(url, document); + } + + /// Close a document and return it if it was open + pub fn close(&self, url: &Url) -> Option { + self.inner.remove(url).map(|(_, doc)| doc) + } + + /// Get a document if it's open + pub fn get(&self, url: &Url) -> Option { + self.inner.get(url).map(|entry| entry.clone()) + } + + /// Check if a document is open + pub fn contains(&self, url: &Url) -> bool { + self.inner.contains_key(url) + } + + /// Iterate over all open buffers (for debugging) + pub fn iter(&self) -> impl Iterator + '_ { + self.inner.iter().map(|entry| (entry.key().clone(), entry.value().clone())) + } +} + +impl Default for Buffers { + fn default() -> Self { + Self::new() + } +} \ No newline at end of file diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 8a9220f7..3fba2173 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -398,6 +398,7 @@ pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { #[cfg(test)] mod tests { use super::*; + use crate::buffers::Buffers; use crate::document::TextDocument; use crate::fs::WorkspaceFileSystem; use crate::language::LanguageId; @@ -464,11 +465,11 @@ mod tests { ); // Create overlay storage - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); // Create WorkspaceFileSystem that checks overlays first let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(memory_fs), )); @@ -490,7 +491,7 @@ mod tests { 2, LanguageId::Other, ); - overlays.insert(url, updated_document); + buffers.open(url, updated_document); // Bump the file revision to trigger re-parse file.set_revision(&mut db).to(1); @@ -520,11 +521,11 @@ mod tests { ); // Create overlay storage - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); // Create WorkspaceFileSystem let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(memory_fs), )); @@ -549,7 +550,7 @@ mod tests { 2, LanguageId::Other, ); - overlays.insert(url, updated_document); + buffers.open(url, updated_document); // Bump revision to trigger invalidation file.set_revision(&mut db).to(1); diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 26603d4f..151c6d65 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -3,13 +3,12 @@ //! This module provides the `FileSystem` trait that abstracts file I/O operations. //! This allows the LSP to work with both real files and in-memory overlays. -use dashmap::DashMap; use std::io; use std::path::Path; use std::sync::Arc; use url::Url; -use crate::document::TextDocument; +use crate::buffers::Buffers; /// Trait for file system operations /// @@ -66,43 +65,43 @@ impl FileSystem for OsFileSystem { } } -/// LSP file system that intercepts reads for overlay files +/// LSP file system that intercepts reads for buffered files /// -/// This implements Ruff's two-layer architecture where Layer 1 (LSP overlays) +/// This implements Ruff's two-layer architecture where Layer 1 (open buffers) /// takes precedence over Layer 2 (Salsa database). When a file is read, -/// this system first checks for an overlay (in-memory changes) and returns -/// that content. If no overlay exists, it falls back to reading from disk. +/// this system first checks for a buffer (in-memory content) and returns +/// that content. If no buffer exists, it falls back to reading from disk. pub struct WorkspaceFileSystem { - /// In-memory overlays that take precedence over disk files - /// Maps URL to `TextDocument` containing current content - buffers: Arc>, + /// In-memory buffers that take precedence over disk files + buffers: Buffers, /// Fallback file system for disk operations disk: Arc, } impl WorkspaceFileSystem { - /// Create a new [`LspFileSystem`] with the given overlay storage and fallback - pub fn new(buffers: Arc>, disk: Arc) -> Self { + /// Create a new [`WorkspaceFileSystem`] with the given buffer storage and fallback + pub fn new(buffers: Buffers, disk: Arc) -> Self { Self { buffers, disk } } } impl FileSystem for WorkspaceFileSystem { fn read_to_string(&self, path: &Path) -> io::Result { - if let Some(document) = path_to_url(path).and_then(|url| self.buffers.get(&url)) { - Ok(document.content().to_string()) - } else { - self.disk.read_to_string(path) + if let Some(url) = path_to_url(path) { + if let Some(document) = self.buffers.get(&url) { + return Ok(document.content().to_string()); + } } + self.disk.read_to_string(path) } fn exists(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.exists(path) } fn is_file(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.is_file(path) } @@ -128,22 +127,24 @@ impl FileSystem for WorkspaceFileSystem { /// This is a simplified conversion - in a full implementation, /// you might want more robust path-to-URL conversion fn path_to_url(path: &Path) -> Option { - if let Ok(absolute_path) = std::fs::canonicalize(path) { - return Url::from_file_path(absolute_path).ok(); - } - - // For test scenarios where the file doesn't exist on disk, - // try to create URL from the path directly if it's absolute + // For absolute paths, use them directly without canonicalization + // This ensures consistency with how URLs are created when storing overlays if path.is_absolute() { return Url::from_file_path(path).ok(); } + // Only try to canonicalize for relative paths + if let Ok(absolute_path) = std::fs::canonicalize(path) { + return Url::from_file_path(absolute_path).ok(); + } + None } #[cfg(test)] mod tests { use super::*; + use crate::buffers::Buffers; use crate::document::TextDocument; use crate::language::LanguageId; @@ -207,22 +208,22 @@ mod tests { "original content".to_string(), ); - // Create overlay storage - let overlays = Arc::new(DashMap::new()); + // Create buffer storage + let buffers = Buffers::new(); // Create LspFileSystem with memory fallback - let lsp_fs = WorkspaceFileSystem::new(overlays.clone(), Arc::new(memory_fs)); + let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); - // Before adding overlay, should read from fallback + // Before adding buffer, should read from fallback let path = std::path::Path::new("/test/file.py"); assert_eq!(lsp_fs.read_to_string(path).unwrap(), "original content"); - // Add overlay - this simulates having an open document with changes + // Add buffer - this simulates having an open document with changes let url = Url::from_file_path("/test/file.py").unwrap(); let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); - overlays.insert(url, document); + buffers.open(url, document); - // Now should read from overlay + // Now should read from buffer assert_eq!(lsp_fs.read_to_string(path).unwrap(), "overlay content"); } @@ -235,13 +236,13 @@ mod tests { "disk content".to_string(), ); - // Create empty overlay storage - let overlays = Arc::new(DashMap::new()); + // Create empty buffer storage + let buffers = Buffers::new(); // Create LspFileSystem - let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); - // Should fall back to disk when no overlay exists + // Should fall back to disk when no buffer exists let path = std::path::Path::new("/test/file.py"); assert_eq!(lsp_fs.read_to_string(path).unwrap(), "disk content"); } @@ -256,8 +257,8 @@ mod tests { ); // Create LspFileSystem - let overlays = Arc::new(DashMap::new()); - let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + let buffers = Buffers::new(); + let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); let path = std::path::Path::new("/test/file.py"); diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 22e0faf1..a3a8d018 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,9 +1,11 @@ +mod buffers; pub mod db; mod document; mod fs; mod language; mod template; +pub use buffers::Buffers; pub use db::Database; pub use document::TextDocument; pub use fs::{FileSystem, OsFileSystem, WorkspaceFileSystem}; From f3fb8e70451e9490cf18f7c906496b14c5e2c064 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 15:35:02 -0500 Subject: [PATCH 16/56] Improve documentation and consolidate path/URL utilities - Added comprehensive module-level documentation to all djls-workspace modules - Consolidated scattered URL/path conversion utilities into paths module - Added documentation explaining the 'why' for key types and abstractions - Added #[must_use] annotations to constructors and getters - Focused on explaining architecture and design decisions rather than obvious behavior --- Cargo.lock | 1 + crates/djls-server/src/session.rs | 55 +----- crates/djls-server/tests/lsp_integration.rs | 6 +- crates/djls-workspace/Cargo.toml | 1 + crates/djls-workspace/src/buffers.rs | 11 +- crates/djls-workspace/src/db.rs | 5 +- crates/djls-workspace/src/document.rs | 22 +++ crates/djls-workspace/src/fs.rs | 30 +-- crates/djls-workspace/src/language.rs | 9 + crates/djls-workspace/src/lib.rs | 15 ++ crates/djls-workspace/src/paths.rs | 200 ++++++++++++++++++++ crates/djls-workspace/src/template.rs | 23 ++- 12 files changed, 295 insertions(+), 83 deletions(-) create mode 100644 crates/djls-workspace/src/paths.rs diff --git a/Cargo.lock b/Cargo.lock index efc1695b..27220220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -513,6 +513,7 @@ dependencies = [ "djls-project", "djls-templates", "notify", + "percent-encoding", "salsa", "tempfile", "tokio", diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 189a3331..d0ddafa1 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -39,9 +39,8 @@ use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::{ db::{Database, SourceFile}, - Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, + paths, Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, }; -use percent_encoding::percent_decode_str; use salsa::{Setter, StorageHandle}; use tower_lsp_server::lsp_types; use url::Url; @@ -171,25 +170,7 @@ impl Session { /// Converts a `file:` URI into an absolute `PathBuf`. fn uri_to_pathbuf(uri: &lsp_types::Uri) -> Option { - // Check if the scheme is "file" - if uri.scheme().is_none_or(|s| s.as_str() != "file") { - return None; - } - - // Get the path part as a string - let encoded_path_str = uri.path().as_str(); - - // Decode the percent-encoded path string - let decoded_path_cow = percent_decode_str(encoded_path_str).decode_utf8_lossy(); - let path_str = decoded_path_cow.as_ref(); - - #[cfg(windows)] - let path_str = { - // Remove leading '/' for paths like /C:/... - path_str.strip_prefix('/').unwrap_or(path_str) - }; - - Some(PathBuf::from(path_str)) + paths::lsp_uri_to_path(uri) } pub fn project(&self) -> Option<&DjangoProject> { @@ -353,29 +334,6 @@ impl Session { f(&db) } - /// Convert a URL to a PathBuf for file operations. - /// - /// This is needed to convert between LSP URLs and file paths for - /// SourceFile creation and tracking. - pub fn url_to_path(&self, url: &Url) -> Option { - // Only handle file:// URLs - if url.scheme() != "file" { - return None; - } - - // Decode and convert to PathBuf - let path = percent_decode_str(url.path()).decode_utf8().ok()?; - - #[cfg(windows)] - let path = path.strip_prefix('/').unwrap_or(&path); - - Some(PathBuf::from(path.as_ref())) - } - - // ===== Document Lifecycle Management ===== - // These methods encapsulate the two-layer architecture coordination: - // Layer 1 (overlays) and Layer 2 (Salsa revision tracking) - /// Handle opening a document - sets buffer and creates file. /// /// This method coordinates both layers: @@ -390,12 +348,12 @@ impl Session { // Layer 2: Create file and bump revision if it already exists // This is crucial: if the file was already read from disk, we need to // invalidate Salsa's cache so it re-reads through the buffer system - if let Some(path) = self.url_to_path(&url) { + if let Some(path) = paths::url_to_path(&url) { self.with_db_mut(|db| { // Check if file already exists (was previously read from disk) let already_exists = db.has_file(&path); let file = db.get_or_create_file(path.clone()); - + if already_exists { // File was already read - bump revision to invalidate cache let current_rev = file.revision(db); @@ -407,7 +365,6 @@ impl Session { current_rev, new_rev ); - } else { // New file - starts at revision 0 tracing::debug!( @@ -433,7 +390,7 @@ impl Session { self.buffers.update(url.clone(), document); // Layer 2: Bump revision to trigger invalidation - if let Some(path) = self.url_to_path(&url) { + if let Some(path) = paths::url_to_path(&url) { self.notify_file_changed(path); } } @@ -460,7 +417,7 @@ impl Session { // Layer 2: Bump revision to trigger re-read from disk // We keep the file alive for potential re-opening - if let Some(path) = self.url_to_path(url) { + if let Some(path) = paths::url_to_path(url) { self.notify_file_changed(path); } diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index 5c14607a..4bfc6660 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -15,7 +15,7 @@ use tempfile::TempDir; use tower_lsp_server::lsp_types::{ DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, InitializeParams, InitializedParams, TextDocumentContentChangeEvent, TextDocumentIdentifier, - TextDocumentItem, Uri, VersionedTextDocumentIdentifier, WorkspaceFolder, + TextDocumentItem, VersionedTextDocumentIdentifier, WorkspaceFolder, }; use tower_lsp_server::LanguageServer; use url::Url; @@ -35,7 +35,7 @@ impl TestServer { let workspace_root = temp_dir.path().to_path_buf(); // Set up logging - let (non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); + let (_non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); // Create server (guard is moved into server, so we return it too) let server = DjangoLanguageServer::new(guard); @@ -73,7 +73,7 @@ impl TestServer { /// Helper to create a file URL in the test workspace fn workspace_url(&self, name: &str) -> Url { - Url::from_file_path(self.workspace_file(name)).unwrap() + djls_workspace::paths::path_to_url(&self.workspace_file(name)).unwrap() } /// Open a document in the LSP server diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 0a46bd81..b1fa2e09 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -11,6 +11,7 @@ anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } notify = { workspace = true } +percent-encoding = { workspace = true } salsa = { workspace = true } tokio = { workspace = true } tower-lsp-server = { workspace = true } diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 702220ac..0d400e9d 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -24,6 +24,7 @@ pub struct Buffers { impl Buffers { /// Create a new empty buffer storage + #[must_use] pub fn new() -> Self { Self { inner: Arc::new(DashMap::new()), @@ -41,23 +42,28 @@ impl Buffers { } /// Close a document and return it if it was open + #[must_use] pub fn close(&self, url: &Url) -> Option { self.inner.remove(url).map(|(_, doc)| doc) } /// Get a document if it's open + #[must_use] pub fn get(&self, url: &Url) -> Option { self.inner.get(url).map(|entry| entry.clone()) } /// Check if a document is open + #[must_use] pub fn contains(&self, url: &Url) -> bool { self.inner.contains_key(url) } /// Iterate over all open buffers (for debugging) pub fn iter(&self) -> impl Iterator + '_ { - self.inner.iter().map(|entry| (entry.key().clone(), entry.value().clone())) + self.inner + .iter() + .map(|entry| (entry.key().clone(), entry.value().clone())) } } @@ -65,4 +71,5 @@ impl Default for Buffers { fn default() -> Self { Self::new() } -} \ No newline at end of file +} + diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 3fba2173..1892e2d8 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -406,7 +406,6 @@ mod tests { use salsa::Setter; use std::collections::HashMap; use std::io; - use url::Url; // Simple in-memory filesystem for testing struct InMemoryFileSystem { @@ -485,7 +484,7 @@ mod tests { assert!(ast1.errors.is_empty(), "Should have no errors"); // Add an overlay with updated content - let url = Url::from_file_path(&template_path).unwrap(); + let url = crate::paths::path_to_url(&template_path).unwrap(); let updated_document = TextDocument::new( "{% block content %}Updated from overlay{% endblock %}".to_string(), 2, @@ -544,7 +543,7 @@ mod tests { assert!(Arc::ptr_eq(&ast1, &ast2), "Should return cached result"); // Update overlay content - let url = Url::from_file_path(&template_path).unwrap(); + let url = crate::paths::path_to_url(&template_path).unwrap(); let updated_document = TextDocument::new( "{% if false %}Changed{% endif %}".to_string(), 2, diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index c1447afa..9def9451 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -1,9 +1,21 @@ +//! LSP text document representation with efficient line indexing +//! +//! [`TextDocument`] stores open file content with version tracking for the LSP protocol. +//! Pre-computed line indices enable O(1) position lookups, which is critical for +//! performance when handling frequent position-based operations like hover, completion, +//! and diagnostics. + use crate::language::LanguageId; use crate::template::ClosingBrace; use crate::template::TemplateTagContext; use tower_lsp_server::lsp_types::Position; use tower_lsp_server::lsp_types::Range; +/// In-memory representation of an open document in the LSP. +/// +/// Combines document content with metadata needed for LSP operations, +/// including version tracking for synchronization and pre-computed line +/// indices for efficient position lookups. #[derive(Clone, Debug)] pub struct TextDocument { /// The document's content @@ -18,6 +30,7 @@ pub struct TextDocument { impl TextDocument { /// Create a new TextDocument with the given content + #[must_use] pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { let line_index = LineIndex::new(&content); Self { @@ -29,20 +42,24 @@ impl TextDocument { } /// Get the document's content + #[must_use] pub fn content(&self) -> &str { &self.content } /// Get the version number + #[must_use] pub fn version(&self) -> i32 { self.version } /// Get the language identifier + #[must_use] pub fn language_id(&self) -> LanguageId { self.language_id.clone() } + #[must_use] pub fn line_index(&self) -> &LineIndex { &self.line_index } @@ -126,6 +143,11 @@ impl TextDocument { } } +/// Pre-computed line start positions for efficient position/offset conversion. +/// +/// Computing line positions on every lookup would be O(n) where n is the document size. +/// By pre-computing during document creation/updates, we get O(1) lookups for line starts +/// and O(log n) for position-to-offset conversions via binary search. #[derive(Clone, Debug)] pub struct LineIndex { pub line_starts: Vec, diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 151c6d65..8ef755f0 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -6,9 +6,8 @@ use std::io; use std::path::Path; use std::sync::Arc; -use url::Url; -use crate::buffers::Buffers; +use crate::{buffers::Buffers, paths}; /// Trait for file system operations /// @@ -80,6 +79,7 @@ pub struct WorkspaceFileSystem { impl WorkspaceFileSystem { /// Create a new [`WorkspaceFileSystem`] with the given buffer storage and fallback + #[must_use] pub fn new(buffers: Buffers, disk: Arc) -> Self { Self { buffers, disk } } @@ -87,7 +87,7 @@ impl WorkspaceFileSystem { impl FileSystem for WorkspaceFileSystem { fn read_to_string(&self, path: &Path) -> io::Result { - if let Some(url) = path_to_url(path) { + if let Some(url) = paths::path_to_url(path) { if let Some(document) = self.buffers.get(&url) { return Ok(document.content().to_string()); } @@ -96,12 +96,12 @@ impl FileSystem for WorkspaceFileSystem { } fn exists(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) + paths::path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.exists(path) } fn is_file(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) + paths::path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.is_file(path) } @@ -122,31 +122,13 @@ impl FileSystem for WorkspaceFileSystem { } } -/// Convert a file path to URL for overlay lookup -/// -/// This is a simplified conversion - in a full implementation, -/// you might want more robust path-to-URL conversion -fn path_to_url(path: &Path) -> Option { - // For absolute paths, use them directly without canonicalization - // This ensures consistency with how URLs are created when storing overlays - if path.is_absolute() { - return Url::from_file_path(path).ok(); - } - - // Only try to canonicalize for relative paths - if let Ok(absolute_path) = std::fs::canonicalize(path) { - return Url::from_file_path(absolute_path).ok(); - } - - None -} - #[cfg(test)] mod tests { use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; use crate::language::LanguageId; + use url::Url; /// In-memory file system for testing pub struct InMemoryFileSystem { diff --git a/crates/djls-workspace/src/language.rs b/crates/djls-workspace/src/language.rs index 8db778f1..f92811c7 100644 --- a/crates/djls-workspace/src/language.rs +++ b/crates/djls-workspace/src/language.rs @@ -1,5 +1,14 @@ +//! Language identification for document routing +//! +//! Maps LSP language identifiers to internal [`FileKind`] for analyzer routing. +//! Language IDs come from the LSP client and determine how files are processed. + use crate::FileKind; +/// Language identifier as reported by the LSP client. +/// +/// These identifiers follow VS Code's language ID conventions and determine +/// which analyzers and features are available for a document. #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { Html, diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index a3a8d018..9e17d154 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,8 +1,23 @@ +//! Workspace management for the Django Language Server +//! +//! This crate provides the core workspace functionality including document management, +//! file system abstractions, and Salsa integration for incremental computation of +//! Django projects. +//! +//! # Key Components +//! +//! - [`Buffers`] - Thread-safe storage for open documents +//! - [`Database`] - Salsa database for incremental computation +//! - [`TextDocument`] - LSP document representation with efficient indexing +//! - [`FileSystem`] - Abstraction layer for file operations with overlay support +//! - [`paths`] - Consistent URL/path conversion utilities + mod buffers; pub mod db; mod document; mod fs; mod language; +pub mod paths; mod template; pub use buffers::Buffers; diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs new file mode 100644 index 00000000..515df66f --- /dev/null +++ b/crates/djls-workspace/src/paths.rs @@ -0,0 +1,200 @@ +//! Path and URL conversion utilities +//! +//! This module provides consistent conversion between file paths and URLs, +//! handling platform-specific differences and encoding issues. + +use std::path::{Path, PathBuf}; +use tower_lsp_server::lsp_types; +use url::Url; + +/// Convert a file:// URL to a `PathBuf` +/// +/// Handles percent-encoding and platform-specific path formats (e.g., Windows drives). +#[must_use] +pub fn url_to_path(url: &Url) -> Option { + // Only handle file:// URLs + if url.scheme() != "file" { + return None; + } + + // Get the path component and decode percent-encoding + let path = percent_encoding::percent_decode_str(url.path()) + .decode_utf8() + .ok()?; + + #[cfg(windows)] + let path = { + // Remove leading '/' for paths like /C:/... + path.strip_prefix('/').unwrap_or(&path) + }; + + Some(PathBuf::from(path.as_ref())) +} + +/// Convert an LSP URI to a `PathBuf` +/// +/// This is a convenience wrapper that parses the LSP URI string and converts it. +pub fn lsp_uri_to_path(lsp_uri: &lsp_types::Uri) -> Option { + // Parse the URI string as a URL + let url = Url::parse(lsp_uri.as_str()).ok()?; + url_to_path(&url) +} + +/// Convert a Path to a file:// URL +/// +/// Handles both absolute and relative paths. Relative paths are resolved +/// to absolute paths before conversion. +#[must_use] +pub fn path_to_url(path: &Path) -> Option { + // For absolute paths, convert directly + if path.is_absolute() { + return Url::from_file_path(path).ok(); + } + + // For relative paths, try to make them absolute first + if let Ok(absolute_path) = std::fs::canonicalize(path) { + return Url::from_file_path(absolute_path).ok(); + } + + // If canonicalization fails, try converting as-is (might fail) + Url::from_file_path(path).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_url_to_path_basic() { + let url = Url::parse("file:///home/user/file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("/home/user/file.txt")); + } + + #[test] + fn test_url_to_path_with_spaces() { + let url = Url::parse("file:///home/user/my%20file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("/home/user/my file.txt")); + } + + #[test] + fn test_url_to_path_non_file_scheme() { + let url = Url::parse("https://example.com/file.txt").unwrap(); + assert!(url_to_path(&url).is_none()); + } + + #[cfg(windows)] + #[test] + fn test_url_to_path_windows() { + let url = Url::parse("file:///C:/Users/user/file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("C:/Users/user/file.txt")); + } + + #[test] + fn test_path_to_url_absolute() { + let path = if cfg!(windows) { + PathBuf::from("C:/Users/user/file.txt") + } else { + PathBuf::from("/home/user/file.txt") + }; + + let url = path_to_url(&path).unwrap(); + assert_eq!(url.scheme(), "file"); + assert!(url.path().contains("file.txt")); + } + + #[test] + fn test_round_trip() { + let original_path = if cfg!(windows) { + PathBuf::from("C:/Users/user/test file.txt") + } else { + PathBuf::from("/home/user/test file.txt") + }; + + let url = path_to_url(&original_path).unwrap(); + let converted_path = url_to_path(&url).unwrap(); + + assert_eq!(original_path, converted_path); + } + + #[test] + fn test_url_with_localhost() { + // Some systems use file://localhost/path format + let url = Url::parse("file://localhost/home/user/file.txt").unwrap(); + let path = url_to_path(&url); + + // Current implementation might not handle this correctly + // since it only checks scheme, not host + if let Some(p) = path { + assert_eq!(p, PathBuf::from("/home/user/file.txt")); + } + } + + #[test] + fn test_url_with_empty_host() { + // Standard file:///path format (three slashes, empty host) + let url = Url::parse("file:///home/user/file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("/home/user/file.txt")); + } + + #[cfg(windows)] + #[test] + fn test_unc_path_to_url() { + // UNC paths like \\server\share\file.txt + let unc_path = PathBuf::from(r"\\server\share\file.txt"); + let url = path_to_url(&unc_path); + + // Check if UNC paths are handled + if let Some(u) = url { + // UNC paths should convert to file://server/share/file.txt + assert!(u.to_string().contains("server")); + assert!(u.to_string().contains("share")); + } + } + + #[test] + fn test_relative_path_with_dotdot() { + // Test relative paths with .. that might not exist + let path = PathBuf::from("../some/nonexistent/path.txt"); + let url = path_to_url(&path); + + // This might fail if the path doesn't exist and can't be canonicalized + // Current implementation falls back to trying direct conversion + assert!(url.is_none() || url.is_some()); + } + + #[test] + fn test_path_with_special_chars() { + // Test paths with special characters that need encoding + let path = PathBuf::from("/home/user/file with spaces & special!.txt"); + let url = path_to_url(&path).unwrap(); + + // Should be properly percent-encoded + assert!(url.as_str().contains("%20") || url.as_str().contains("with%20spaces")); + + // Round-trip should work + let back = url_to_path(&url).unwrap(); + assert_eq!(back, path); + } + + #[test] + fn test_url_with_query_or_fragment() { + // URLs with query parameters or fragments should probably be rejected + let url_with_query = Url::parse("file:///path/file.txt?query=param").unwrap(); + let url_with_fragment = Url::parse("file:///path/file.txt#section").unwrap(); + + // These should still work, extracting just the path part + let path1 = url_to_path(&url_with_query); + let path2 = url_to_path(&url_with_fragment); + + if let Some(p) = path1 { + assert_eq!(p, PathBuf::from("/path/file.txt")); + } + if let Some(p) = path2 { + assert_eq!(p, PathBuf::from("/path/file.txt")); + } + } +} diff --git a/crates/djls-workspace/src/template.rs b/crates/djls-workspace/src/template.rs index 2a0547c4..2ebc3247 100644 --- a/crates/djls-workspace/src/template.rs +++ b/crates/djls-workspace/src/template.rs @@ -1,13 +1,32 @@ +//! Django template context detection for completions +//! +//! Detects cursor position context within Django template tags to provide +//! appropriate completions and auto-closing behavior. + +/// Tracks what closing characters are needed to complete a template tag. +/// +/// Used to determine whether the completion system needs to insert +/// closing braces when completing a Django template tag. #[derive(Debug)] pub enum ClosingBrace { + /// No closing brace present - need to add full `%}` or `}}` None, - PartialClose, // just } - FullClose, // %} + /// Partial close present (just `}`) - need to add `%` or second `}` + PartialClose, + /// Full close present (`%}` or `}}`) - no closing needed + FullClose, } +/// Cursor context within a Django template tag for completion support. +/// +/// Captures the state around the cursor position to provide intelligent +/// completions and determine what text needs to be inserted. #[derive(Debug)] pub struct TemplateTagContext { + /// The partial tag text before the cursor (e.g., "loa" for "{% loa|") pub partial_tag: String, + /// What closing characters are already present after the cursor pub closing_brace: ClosingBrace, + /// Whether a space is needed before the completion (true if cursor is right after `{%`) pub needs_leading_space: bool, } From 89e979ba3f1afefbe5df747694ad398b4d53f9a9 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 15:39:23 -0500 Subject: [PATCH 17/56] Add cross-references and improve documentation consistency - Added [] cross-references between related types - Fixed parameter naming consistency (lsp_uri -> uri) - Added Returns sections to document when functions return None - Added example to url_to_path function - Linked Buffers <-> WorkspaceFileSystem relationship - Linked LanguageId -> FileKind conversion --- crates/djls-workspace/src/buffers.rs | 7 +++++-- crates/djls-workspace/src/fs.rs | 9 ++++++--- crates/djls-workspace/src/language.rs | 3 ++- crates/djls-workspace/src/paths.rs | 23 +++++++++++++++++++++-- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 0d400e9d..ae42916b 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -14,9 +14,12 @@ use crate::document::TextDocument; /// Shared buffer storage between Session and FileSystem /// /// Buffers represent the in-memory content of open files that takes -/// precedence over disk content when reading through the FileSystem. +/// precedence over disk content when reading through the [`FileSystem`]. /// This is the key abstraction that makes the sharing between Session -/// and WorkspaceFileSystem explicit and type-safe. +/// and [`WorkspaceFileSystem`] explicit and type-safe. +/// +/// The [`WorkspaceFileSystem`] holds a clone of this structure and checks +/// it before falling back to disk reads. #[derive(Clone, Debug)] pub struct Buffers { inner: Arc>, diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 8ef755f0..3f457861 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -66,10 +66,13 @@ impl FileSystem for OsFileSystem { /// LSP file system that intercepts reads for buffered files /// -/// This implements Ruff's two-layer architecture where Layer 1 (open buffers) +/// This implements Ruff's two-layer architecture where Layer 1 (open [`Buffers`]) /// takes precedence over Layer 2 (Salsa database). When a file is read, -/// this system first checks for a buffer (in-memory content) and returns -/// that content. If no buffer exists, it falls back to reading from disk. +/// this system first checks for a buffer (in-memory content from [`TextDocument`]) +/// and returns that content. If no buffer exists, it falls back to reading from disk. +/// +/// This type is used by the [`Database`] to ensure all file reads go through +/// the buffer system first. pub struct WorkspaceFileSystem { /// In-memory buffers that take precedence over disk files buffers: Buffers, diff --git a/crates/djls-workspace/src/language.rs b/crates/djls-workspace/src/language.rs index f92811c7..ea9b3834 100644 --- a/crates/djls-workspace/src/language.rs +++ b/crates/djls-workspace/src/language.rs @@ -8,7 +8,8 @@ use crate::FileKind; /// Language identifier as reported by the LSP client. /// /// These identifiers follow VS Code's language ID conventions and determine -/// which analyzers and features are available for a document. +/// which analyzers and features are available for a document. Converts to +/// [`FileKind`] to route files to appropriate analyzers (Python vs Template). #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { Html, diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 515df66f..9d64f69a 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -10,6 +10,20 @@ use url::Url; /// Convert a file:// URL to a `PathBuf` /// /// Handles percent-encoding and platform-specific path formats (e.g., Windows drives). +/// +/// # Returns +/// +/// Returns `None` if the URL scheme is not "file" or if decoding fails. +/// +/// # Examples +/// +/// ``` +/// # use url::Url; +/// # use djls_workspace::paths::url_to_path; +/// let url = Url::parse("file:///home/user/file.txt").unwrap(); +/// let path = url_to_path(&url).unwrap(); +/// assert_eq!(path.to_str().unwrap(), "/home/user/file.txt"); +/// ``` #[must_use] pub fn url_to_path(url: &Url) -> Option { // Only handle file:// URLs @@ -34,9 +48,14 @@ pub fn url_to_path(url: &Url) -> Option { /// Convert an LSP URI to a `PathBuf` /// /// This is a convenience wrapper that parses the LSP URI string and converts it. -pub fn lsp_uri_to_path(lsp_uri: &lsp_types::Uri) -> Option { +/// +/// # Returns +/// +/// Returns `None` if the URI cannot be parsed as a URL or is not a file:// URI. +#[must_use] +pub fn lsp_uri_to_path(uri: &lsp_types::Uri) -> Option { // Parse the URI string as a URL - let url = Url::parse(lsp_uri.as_str()).ok()?; + let url = Url::parse(uri.as_str()).ok()?; url_to_path(&url) } From f6e7f9084e49da9520a587dbed388242f8fcbe4c Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 16:02:39 -0500 Subject: [PATCH 18/56] Fix missing backticks in documentation Fixed unclosed HTML tag warnings by adding backticks around: - Generic types like Arc - Type names in documentation like StorageHandle - The word 'Arc' when referring to the type --- crates/djls-workspace/src/db.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 1892e2d8..03477a55 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -27,14 +27,14 @@ //! //! ## StorageHandle Pattern (for tower-lsp) //! - Database itself is NOT Send+Sync (due to RefCell in Salsa's Storage) -//! - StorageHandle IS Send+Sync, enabling use across threads +//! - `StorageHandle` IS Send+Sync, enabling use across threads //! - Session stores StorageHandle, creates Database instances on-demand //! //! ## Why Files are in Database, Overlays in Session //! - Files need persistent tracking across all queries (thus in Database) //! - Overlays are LSP-specific and change frequently (thus in Session) //! - This separation prevents Salsa invalidation cascades on every keystroke -//! - Both are accessed via Arc for thread safety and cheap cloning +//! - Both are accessed via `Arc` for thread safety and cheap cloning //! //! # Data Flow //! @@ -49,7 +49,7 @@ //! This design achieves: //! - Fast overlay updates (no Salsa invalidation) //! - Proper incremental computation (via revision tracking) -//! - Thread safety (via Arc and StorageHandle) +//! - Thread safety (via `Arc` and StorageHandle) //! - Clean separation of concerns (LSP vs computation) use std::path::{Path, PathBuf}; @@ -168,7 +168,7 @@ impl Database { /// Get or create a SourceFile for the given path. /// /// This method implements Ruff's pattern for lazy file creation. Files are created - /// with an initial revision of 0 and tracked in the Database's DashMap. The Arc + /// with an initial revision of 0 and tracked in the Database's `DashMap`. The `Arc` /// ensures cheap cloning while maintaining thread safety. pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { if let Some(file_ref) = self.files.get(&path) { From af8820b7bcb94660a9b494924e99e15b76abfaca Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 16:57:14 -0500 Subject: [PATCH 19/56] Fix outdated and incorrect db.rs documentation The documentation was completely out of sync with the code: - Referenced 'overlays in Session using Arc' when they're now in Buffers - Mentioned 'LspFileSystem' which was renamed to WorkspaceFileSystem - Was overly verbose without being helpful - 'vomit of words' Rewrote to be concise and accurate: - Correctly describes the current two-layer architecture - Focuses on the critical revision dependency trick - Removes outdated implementation details - Uses proper cross-references --- crates/djls-workspace/src/buffers.rs | 20 ++++----- crates/djls-workspace/src/db.rs | 64 ++++++++-------------------- crates/djls-workspace/src/fs.rs | 3 -- 3 files changed, 26 insertions(+), 61 deletions(-) diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index ae42916b..0365a3a6 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -1,8 +1,8 @@ //! Shared buffer storage for open documents //! -//! This module provides the `Buffers` type which represents the in-memory -//! content of open files. These buffers are shared between the Session -//! (which manages document lifecycle) and the WorkspaceFileSystem (which +//! This module provides the [`Buffers`] type which represents the in-memory +//! content of open files. These buffers are shared between the `Session` +//! (which manages document lifecycle) and the [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem) (which //! reads from them). use dashmap::DashMap; @@ -11,22 +11,24 @@ use url::Url; use crate::document::TextDocument; -/// Shared buffer storage between Session and FileSystem +/// Shared buffer storage between `Session` and [`FileSystem`]. /// /// Buffers represent the in-memory content of open files that takes /// precedence over disk content when reading through the [`FileSystem`]. /// This is the key abstraction that makes the sharing between Session /// and [`WorkspaceFileSystem`] explicit and type-safe. -/// +/// /// The [`WorkspaceFileSystem`] holds a clone of this structure and checks /// it before falling back to disk reads. +/// +/// [`FileSystem`]: crate::fs::FileSystem +/// [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem #[derive(Clone, Debug)] pub struct Buffers { inner: Arc>, } impl Buffers { - /// Create a new empty buffer storage #[must_use] pub fn new() -> Self { Self { @@ -34,23 +36,19 @@ impl Buffers { } } - /// Open a document in the buffers pub fn open(&self, url: Url, document: TextDocument) { self.inner.insert(url, document); } - /// Update an open document pub fn update(&self, url: Url, document: TextDocument) { self.inner.insert(url, document); } - /// Close a document and return it if it was open #[must_use] pub fn close(&self, url: &Url) -> Option { self.inner.remove(url).map(|(_, doc)| doc) } - /// Get a document if it's open #[must_use] pub fn get(&self, url: &Url) -> Option { self.inner.get(url).map(|entry| entry.clone()) @@ -62,7 +60,6 @@ impl Buffers { self.inner.contains_key(url) } - /// Iterate over all open buffers (for debugging) pub fn iter(&self) -> impl Iterator + '_ { self.inner .iter() @@ -75,4 +72,3 @@ impl Default for Buffers { Self::new() } } - diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 03477a55..4af7fefa 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -1,56 +1,28 @@ -//! Salsa database and input entities for workspace. +//! Salsa database for incremental computation. //! -//! This module implements a two-layer architecture inspired by Ruff's design pattern -//! for efficient LSP document management with Salsa incremental computation. +//! This module provides the [`Database`] which integrates with Salsa for +//! incremental computation of Django template parsing and analysis. //! -//! # Two-Layer Architecture +//! ## Architecture //! -//! ## Layer 1: LSP Document Management (in Session) -//! - Stores overlays in `Session` using `Arc>` -//! - TextDocument contains actual content, version, language_id -//! - Changes are immediate, no Salsa invalidation on every keystroke -//! - Thread-safe via DashMap for tower-lsp's Send+Sync requirements +//! The system uses a two-layer approach: +//! 1. **Buffer layer** ([`crate::Buffers`]) - Stores open document content in memory +//! 2. **Salsa layer** ([`Database`]) - Tracks files and computes derived queries //! -//! ## Layer 2: Salsa Incremental Computation (in Database) -//! - Database is pure Salsa, no file content storage -//! - Files tracked via `Arc>` for O(1) lookups -//! - SourceFile inputs only have path and revision (no text) -//! - Content read lazily through FileSystem trait -//! - LspFileSystem intercepts reads, returns overlay or disk content +//! When Salsa needs file content, it calls [`source_text`] which: +//! 1. Creates a dependency on the file's revision (critical!) +//! 2. Reads through [`crate::WorkspaceFileSystem`] which checks buffers first +//! 3. Falls back to disk if no buffer exists //! -//! # Critical Implementation Details +//! ## The Revision Dependency //! -//! ## The Revision Dependency Trick -//! The `source_text` tracked function MUST call `file.revision(db)` to create -//! the Salsa dependency chain. Without this, revision changes won't trigger -//! invalidation of dependent queries. +//! The [`source_text`] function **must** call `file.revision(db)` to create +//! a Salsa dependency. Without this, revision changes won't invalidate queries: //! -//! ## StorageHandle Pattern (for tower-lsp) -//! - Database itself is NOT Send+Sync (due to RefCell in Salsa's Storage) -//! - `StorageHandle` IS Send+Sync, enabling use across threads -//! - Session stores StorageHandle, creates Database instances on-demand -//! -//! ## Why Files are in Database, Overlays in Session -//! - Files need persistent tracking across all queries (thus in Database) -//! - Overlays are LSP-specific and change frequently (thus in Session) -//! - This separation prevents Salsa invalidation cascades on every keystroke -//! - Both are accessed via `Arc` for thread safety and cheap cloning -//! -//! # Data Flow -//! -//! 1. **did_open/did_change** → Update overlays in Session -//! 2. **notify_file_changed()** → Bump revision, tell Salsa something changed -//! 3. **Salsa query executes** → Calls source_text() -//! 4. **source_text() calls file.revision(db)** → Creates dependency -//! 5. **source_text() calls db.read_file_content()** → Goes through FileSystem -//! 6. **LspFileSystem intercepts** → Returns overlay if exists, else disk -//! 7. **Query gets content** → Without knowing about LSP/overlays -//! -//! This design achieves: -//! - Fast overlay updates (no Salsa invalidation) -//! - Proper incremental computation (via revision tracking) -//! - Thread safety (via `Arc` and StorageHandle) -//! - Clean separation of concerns (LSP vs computation) +//! ```ignore +//! let _ = file.revision(db); // Creates the dependency chain! +//! ``` + use std::path::{Path, PathBuf}; use std::sync::Arc; diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 3f457861..c3c9ef92 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -10,9 +10,6 @@ use std::sync::Arc; use crate::{buffers::Buffers, paths}; /// Trait for file system operations -/// -/// This follows Ruff's pattern of abstracting file system operations behind a trait, -/// allowing different implementations for testing, in-memory operation, and real file access. pub trait FileSystem: Send + Sync { /// Read the entire contents of a file fn read_to_string(&self, path: &Path) -> io::Result; From c685f53dec6e922d8521d04e91bd2cc471fdbb96 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 17:05:38 -0500 Subject: [PATCH 20/56] Clean up more outdated documentation in db.rs - Removed references to 'LspFileSystem' (now WorkspaceFileSystem) - Removed verbose 'overlay' explanations (now using Buffers abstraction) - Simplified struct field documentation to be accurate and concise - Removed unnecessary mentions of 'Ruff's pattern' everywhere The documentation now accurately reflects the current implementation without verbose explanations of outdated architecture. --- crates/djls-workspace/src/db.rs | 47 +++++++++++++-------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 4af7fefa..85d571d0 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -6,12 +6,12 @@ //! ## Architecture //! //! The system uses a two-layer approach: -//! 1. **Buffer layer** ([`crate::Buffers`]) - Stores open document content in memory +//! 1. **Buffer layer** ([`Buffers`]) - Stores open document content in memory //! 2. **Salsa layer** ([`Database`]) - Tracks files and computes derived queries //! //! When Salsa needs file content, it calls [`source_text`] which: //! 1. Creates a dependency on the file's revision (critical!) -//! 2. Reads through [`crate::WorkspaceFileSystem`] which checks buffers first +//! 2. Reads through [`WorkspaceFileSystem`] which checks buffers first //! 3. Falls back to disk if no buffer exists //! //! ## The Revision Dependency @@ -22,7 +22,9 @@ //! ```ignore //! let _ = file.revision(db); // Creates the dependency chain! //! ``` - +//! +//! [`Buffers`]: crate::buffers::Buffers +//! [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -36,36 +38,29 @@ use crate::{FileKind, FileSystem}; /// Database trait that provides file system access for Salsa queries #[salsa::db] pub trait Db: salsa::Database { - /// Get the file system for reading files (with overlay support) + /// Get the file system for reading files. fn fs(&self) -> Option>; - /// Read file content through the file system - /// This is the primary way Salsa queries should read files, as it - /// automatically checks overlays before falling back to disk. + /// Read file content through the file system. + /// + /// Checks buffers first via [`crate::WorkspaceFileSystem`], then falls back to disk. fn read_file_content(&self, path: &Path) -> std::io::Result; } -/// Salsa database root for workspace -/// -/// The [`Database`] provides default storage and, in tests, captures Salsa events for -/// reuse/diagnostics. It serves as the core incremental computation engine, tracking -/// dependencies and invalidations across all inputs and derived queries. +/// Salsa database for incremental computation. /// -/// The database integrates with the FileSystem abstraction to read files through -/// the LspFileSystem, which automatically checks overlays before falling back to disk. +/// Tracks files and computes derived queries incrementally. Integrates with +/// [`crate::WorkspaceFileSystem`] to read file content, which checks buffers +/// before falling back to disk. #[salsa::db] #[derive(Clone)] pub struct Database { storage: salsa::Storage, - /// FileSystem integration for reading files (with overlay support) - /// This allows the database to read files through LspFileSystem, which - /// automatically checks for overlays before falling back to disk files. + /// File system for reading file content (checks buffers first, then disk). fs: Option>, - /// File tracking outside of Salsa but within Database (Arc for cheap cloning). - /// This follows Ruff's pattern where files are tracked in the Database struct - /// but not as part of Salsa's storage, enabling cheap clones via Arc. + /// Maps paths to [`SourceFile`] entities for O(1) lookup. files: Arc>, // The logs are only used for testing and demonstrating reuse: @@ -126,9 +121,7 @@ impl Database { } } - /// Read file content through the file system - /// This is the primary way Salsa queries should read files, as it - /// automatically checks overlays before falling back to disk. + /// Read file content through the file system. pub fn read_file_content(&self, path: &Path) -> std::io::Result { if let Some(fs) = &self.fs { fs.read_to_string(path) @@ -214,12 +207,10 @@ pub struct SourceFile { pub revision: u64, } -/// Read file content through the FileSystem, creating proper Salsa dependencies. +/// Read file content, creating a Salsa dependency on the file's revision. /// -/// This is the CRITICAL function that implements Ruff's two-layer architecture. -/// The call to `file.revision(db)` creates a Salsa dependency, ensuring that -/// when the revision changes, this function (and all dependent queries) are -/// invalidated and re-executed. +/// **Critical**: The call to `file.revision(db)` creates the dependency chain. +/// Without it, revision changes won't trigger query invalidation. #[salsa::tracked] pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { // This line creates the Salsa dependency on revision! Without this call, From 361d7e25983e99986624524bd60f7ea3d1945618 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 21:06:52 -0500 Subject: [PATCH 21/56] stuf and thinggs --- crates/djls-server/src/server.rs | 67 ++++----- crates/djls-server/src/session.rs | 156 +++++--------------- crates/djls-server/tests/lsp_integration.rs | 21 +-- crates/djls-workspace/src/db.rs | 64 ++------ crates/djls-workspace/src/document.rs | 2 +- crates/djls-workspace/src/fs.rs | 140 +++++++++--------- crates/djls-workspace/src/lib.rs | 6 +- crates/djls-workspace/src/paths.rs | 28 +--- 8 files changed, 153 insertions(+), 331 deletions(-) diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index daa64ac0..efb3bc80 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -1,6 +1,7 @@ use std::future::Future; use std::sync::Arc; +use djls_workspace::paths; use tokio::sync::RwLock; use tower_lsp_server::jsonrpc::Result as LspResult; use tower_lsp_server::lsp_types; @@ -229,21 +230,19 @@ impl LanguageServer for DjangoLanguageServer { let new_version = params.text_document.version; let changes = params.content_changes; - if let Some(mut document) = session.get_overlay(&url) { - document.update(changes, new_version); - session.update_document(url, document); - } else { - // No existing overlay - shouldn't normally happen - tracing::warn!("Received change for document without overlay: {}", url); - - // Handle full content changes only for recovery - if let Some(change) = changes.into_iter().next() { - let document = djls_workspace::TextDocument::new( - change.text, - new_version, - djls_workspace::LanguageId::Other, - ); - session.update_document(url, document); + match session.apply_document_changes(&url, changes.clone(), new_version) { + Ok(()) => {} + Err(err) => { + tracing::warn!("{}", err); + // Recovery: handle full content changes only + if let Some(change) = changes.into_iter().next() { + let document = djls_workspace::TextDocument::new( + change.text, + new_version, + djls_workspace::LanguageId::Other, + ); + session.update_document(&url, document); + } } } }) @@ -269,40 +268,24 @@ impl LanguageServer for DjangoLanguageServer { params: lsp_types::CompletionParams, ) -> LspResult> { let response = self - .with_session(|session| { + .with_session_mut(|session| { let lsp_uri = ¶ms.text_document_position.text_document.uri; let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); let position = params.text_document_position.position; tracing::debug!("Completion requested for {} at {:?}", url, position); - // Check if we have an overlay for this document - if let Some(document) = session.get_overlay(&url) { - tracing::debug!("Using overlay content for completion in {}", url); - - // Use the overlay content for completion - // For now, we'll return None, but this is where completion logic would go - // The key point is that we're using overlay content, not disk content - let _content = document.content(); - let _version = document.version(); - - // TODO: Implement actual completion logic using overlay content - // This would involve: - // 1. Getting context around the cursor position - // 2. Analyzing the Django template or Python content - // 3. Returning appropriate completions - - None - } else { - tracing::debug!("No overlay found for {}, using disk content", url); - - // No overlay - would use disk content via the file system - // The LspFileSystem will automatically fall back to disk - // when no overlay is available - - // TODO: Implement completion using file system content - None + if let Some(path) = paths::url_to_path(&url) { + let content = session.file_content(path); + if content.is_empty() { + tracing::debug!("File {} has no content", url); + } else { + tracing::debug!("Using content for completion in {}", url); + // TODO: Implement actual completion logic using content + } } + + None }) .await; diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index d0ddafa1..2be2abda 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -164,15 +164,11 @@ impl Session { .workspace_folders .as_ref() .and_then(|folders| folders.first()) - .and_then(|folder| Self::uri_to_pathbuf(&folder.uri)) + .and_then(|folder| paths::lsp_uri_to_path(&folder.uri)) }) } - /// Converts a `file:` URI into an absolute `PathBuf`. - fn uri_to_pathbuf(uri: &lsp_types::Uri) -> Option { - paths::lsp_uri_to_path(uri) - } - + #[must_use] pub fn project(&self) -> Option<&DjangoProject> { self.project.as_ref() } @@ -181,6 +177,7 @@ impl Session { &mut self.project } + #[must_use] pub fn settings(&self) -> &Settings { &self.settings } @@ -189,70 +186,19 @@ impl Session { self.settings = settings; } - /// Get a database instance from the session. - /// - /// This creates a usable database from the handle, which can be used - /// to query and update data. The database itself is not Send/Sync, - /// but the `StorageHandle` is, allowing us to work with tower-lsp-server. - /// - /// The database will read files through the LspFileSystem, which - /// automatically returns overlay content when available. - /// - /// CRITICAL: We pass the shared files Arc to preserve file tracking - /// across Database reconstructions from StorageHandle. - #[allow(dead_code)] - pub fn db(&self) -> Database { - let storage = self.db_handle.clone().into_storage(); - Database::from_storage(storage, self.file_system.clone(), self.files.clone()) - } - - /// Get access to the file system (for Salsa integration) - #[allow(dead_code)] - pub fn file_system(&self) -> Arc { - self.file_system.clone() - } - - /// Set or update a buffer for the given document URL - /// - /// This implements Layer 1 of Ruff's architecture - storing in-memory - /// document changes that take precedence over disk content. - #[allow(dead_code)] // Used in tests - pub fn set_overlay(&self, url: Url, document: TextDocument) { - self.buffers.open(url, document); - } - - /// Remove a buffer for the given document URL - /// - /// After removal, file reads will fall back to disk content. - #[allow(dead_code)] // Used in tests - pub fn remove_overlay(&self, url: &Url) -> Option { - self.buffers.close(url) - } - - /// Check if a buffer exists for the given URL - #[allow(dead_code)] - pub fn has_overlay(&self, url: &Url) -> bool { - self.buffers.contains(url) - } - - /// Get a copy of a buffered document - pub fn get_overlay(&self, url: &Url) -> Option { - self.buffers.get(url) - } - /// Takes exclusive ownership of the database handle for mutation operations. /// /// This method extracts the `StorageHandle` from the session, replacing it /// with a temporary placeholder. This ensures there's exactly one handle /// active during mutations, preventing deadlocks in Salsa's `cancel_others()`. /// - /// # Why Not Clone? + /// ## Why Not Clone? /// /// Cloning would create multiple handles. When Salsa needs to mutate inputs, /// it calls `cancel_others()` which waits for all handles to drop. With /// multiple handles, this wait would never complete → deadlock. /// - /// # Panics + /// ## Panics /// /// This is an internal method that should only be called by `with_db_mut`. /// Multiple concurrent calls would panic when trying to take an already-taken handle. @@ -272,12 +218,13 @@ impl Session { /// Execute a closure with mutable access to the database. /// /// This method implements Salsa's required protocol for mutations: - /// 1. Takes exclusive ownership of the StorageHandle (no clones exist) + /// 1. Takes exclusive ownership of the [`StorageHandle`](salsa::StorageHandle) + /// (no clones exist) /// 2. Creates a temporary Database for the operation /// 3. Executes your closure with `&mut Database` /// 4. Extracts and restores the updated handle /// - /// # Example + /// ## Example /// /// ```rust,ignore /// session.with_db_mut(|db| { @@ -286,7 +233,7 @@ impl Session { /// }); /// ``` /// - /// # Why This Pattern? + /// ## Why This Pattern? /// /// This ensures that when Salsa needs to modify inputs (via setters like /// `set_revision`), it has exclusive access. The internal `cancel_others()` @@ -312,11 +259,11 @@ impl Session { /// Execute a closure with read-only access to the database. /// - /// For read-only operations, we can safely clone the `StorageHandle` + /// For read-only operations, we can safely clone the [`StorageHandle`](salsa::StorageHandle) /// since Salsa allows multiple concurrent readers. This is more /// efficient than taking exclusive ownership. /// - /// # Example + /// ## Example /// /// ```rust,ignore /// let content = session.with_db(|db| { @@ -382,7 +329,7 @@ impl Session { /// This method coordinates both layers: /// - Layer 1: Updates the document content in buffers /// - Layer 2: Bumps the file revision to trigger Salsa invalidation - pub fn update_document(&mut self, url: Url, document: TextDocument) { + pub fn update_document(&mut self, url: &Url, document: TextDocument) { let version = document.version(); tracing::debug!("Updating document: {} (version {})", url, version); @@ -390,8 +337,29 @@ impl Session { self.buffers.update(url.clone(), document); // Layer 2: Bump revision to trigger invalidation - if let Some(path) = paths::url_to_path(&url) { - self.notify_file_changed(path); + if let Some(path) = paths::url_to_path(url) { + self.notify_file_changed(&path); + } + } + + /// Apply incremental changes to an open document. + /// + /// This encapsulates the full update cycle: retrieving the document, + /// applying changes, updating the buffer, and bumping Salsa revision. + /// + /// Returns an error if the document is not currently open. + pub fn apply_document_changes( + &mut self, + url: &Url, + changes: Vec, + new_version: i32, + ) -> Result<(), String> { + if let Some(mut document) = self.buffers.get(url) { + document.update(changes, new_version); + self.update_document(url, document); + Ok(()) + } else { + Err(format!("Document not open: {url}")) } } @@ -418,7 +386,7 @@ impl Session { // Layer 2: Bump revision to trigger re-read from disk // We keep the file alive for potential re-opening if let Some(path) = paths::url_to_path(url) { - self.notify_file_changed(path); + self.notify_file_changed(&path); } removed @@ -428,12 +396,12 @@ impl Session { /// /// This bumps the file's revision number in Salsa, which triggers /// invalidation of any queries that depend on the file's content. - fn notify_file_changed(&mut self, path: PathBuf) { + fn notify_file_changed(&mut self, path: &Path) { self.with_db_mut(|db| { // Only bump revision if file is already being tracked // We don't create files just for notifications - if db.has_file(&path) { - let file = db.get_or_create_file(path.clone()); + if db.has_file(path) { + let file = db.get_or_create_file(path.to_path_buf()); let current_rev = file.revision(db); let new_rev = current_rev + 1; file.set_revision(db).to(new_rev); @@ -518,50 +486,6 @@ mod tests { use super::*; use djls_workspace::LanguageId; - #[test] - fn test_session_overlay_management() { - let session = Session::default(); - - let url = Url::parse("file:///test/file.py").unwrap(); - let document = TextDocument::new("print('hello')".to_string(), 1, LanguageId::Python); - - // Initially no overlay - assert!(!session.has_overlay(&url)); - assert!(session.get_overlay(&url).is_none()); - - // Set overlay - session.set_overlay(url.clone(), document.clone()); - assert!(session.has_overlay(&url)); - - let retrieved = session.get_overlay(&url).unwrap(); - assert_eq!(retrieved.content(), document.content()); - assert_eq!(retrieved.version(), document.version()); - - // Remove overlay - let removed = session.remove_overlay(&url).unwrap(); - assert_eq!(removed.content(), document.content()); - assert!(!session.has_overlay(&url)); - } - - #[test] - fn test_session_two_layer_architecture() { - let session = Session::default(); - - // Verify we have both layers - let _filesystem = session.file_system(); // Layer 2: FileSystem bridge - let _db = session.db(); // Layer 2: Salsa database - - // Verify overlay operations work (Layer 1) - let url = Url::parse("file:///test/integration.py").unwrap(); - let document = TextDocument::new("# Layer 1 content".to_string(), 1, LanguageId::Python); - - session.set_overlay(url.clone(), document); - assert!(session.has_overlay(&url)); - - // FileSystem should now return overlay content through LspFileSystem - // (This would be tested more thoroughly in integration tests) - } - #[test] fn test_revision_invalidation_chain() { use std::path::PathBuf; @@ -590,7 +514,7 @@ mod tests { println!("**[test]** Update document with new content"); let updated_document = TextDocument::new("

Updated Content

".to_string(), 2, LanguageId::Other); - session.update_document(url.clone(), updated_document); + session.update_document(&url, updated_document); // Read content again (should get new overlay content due to invalidation) println!( diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index 4bfc6660..fa5613f5 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -138,14 +138,6 @@ impl TestServer { std::fs::write(path, content).expect("Failed to write test file"); } - /// Check if a file has an overlay in the session - async fn has_overlay(&self, file_name: &str) -> bool { - let url = self.workspace_url(file_name); - self.server - .with_session(|session| session.get_overlay(&url).is_some()) - .await - } - /// Get the revision of a file async fn get_file_revision(&self, file_name: &str) -> Option { let path = self.workspace_file(file_name); @@ -168,9 +160,6 @@ async fn test_full_lsp_lifecycle() { .open_document(file_name, "

Overlay Content

", 1) .await; - // Verify overlay exists - assert!(server.has_overlay(file_name).await); - // Verify overlay content is returned (not disk content) let content = server.get_file_content(file_name).await; assert_eq!(content, "

Overlay Content

"); @@ -195,10 +184,7 @@ async fn test_full_lsp_lifecycle() { // 3. Test did_close removes overlay and bumps revision server.close_document(file_name).await; - // Verify overlay is removed - assert!(!server.has_overlay(file_name).await); - - // Verify content now comes from disk + // Verify content now comes from disk (empty since file doesn't exist) let content = server.get_file_content(file_name).await; assert_eq!(content, "

Disk Content

"); @@ -283,11 +269,6 @@ async fn test_multiple_documents_independent() { server.open_document("file2.html", "Content 2", 1).await; server.open_document("file3.html", "Content 3", 1).await; - // Verify all have overlays - assert!(server.has_overlay("file1.html").await); - assert!(server.has_overlay("file2.html").await); - assert!(server.has_overlay("file3.html").await); - // Change one document server.change_document("file2.html", "Updated 2", 2).await; diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 85d571d0..36c52ce5 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -42,16 +42,17 @@ pub trait Db: salsa::Database { fn fs(&self) -> Option>; /// Read file content through the file system. - /// - /// Checks buffers first via [`crate::WorkspaceFileSystem`], then falls back to disk. + /// + /// Checks buffers first via [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem), + /// then falls back to disk. fn read_file_content(&self, path: &Path) -> std::io::Result; } /// Salsa database for incremental computation. /// /// Tracks files and computes derived queries incrementally. Integrates with -/// [`crate::WorkspaceFileSystem`] to read file content, which checks buffers -/// before falling back to disk. +/// [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem) to read file content, +/// which checks buffers before falling back to disk. #[salsa::db] #[derive(Clone)] pub struct Database { @@ -130,10 +131,10 @@ impl Database { } } - /// Get or create a SourceFile for the given path. + /// Get or create a [`SourceFile`] for the given path. /// /// This method implements Ruff's pattern for lazy file creation. Files are created - /// with an initial revision of 0 and tracked in the Database's `DashMap`. The `Arc` + /// with an initial revision of 0 and tracked in the [`Database`]'s `DashMap`. The `Arc` /// ensures cheap cloning while maintaining thread safety. pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { if let Some(file_ref) = self.files.get(&path) { @@ -161,7 +162,7 @@ impl Database { /// Get a reference to the storage for handle extraction. /// - /// This is used by Session to extract the StorageHandle after mutations. + /// This is used by `Session` to extract the [`StorageHandle`](salsa::StorageHandle) after mutations. pub fn storage(&self) -> &salsa::Storage { &self.storage } @@ -363,58 +364,11 @@ mod tests { use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; + use crate::fs::InMemoryFileSystem; use crate::fs::WorkspaceFileSystem; use crate::language::LanguageId; use dashmap::DashMap; use salsa::Setter; - use std::collections::HashMap; - use std::io; - - // Simple in-memory filesystem for testing - struct InMemoryFileSystem { - files: HashMap, - } - - impl InMemoryFileSystem { - fn new() -> Self { - Self { - files: HashMap::new(), - } - } - - fn add_file(&mut self, path: PathBuf, content: String) { - self.files.insert(path, content); - } - } - - impl FileSystem for InMemoryFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - self.files - .get(path) - .cloned() - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) - } - - fn exists(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_file(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_directory(&self, _path: &Path) -> bool { - false - } - - fn read_directory(&self, _path: &Path) -> io::Result> { - Ok(vec![]) - } - - fn metadata(&self, _path: &Path) -> io::Result { - Err(io::Error::new(io::ErrorKind::Unsupported, "Not supported")) - } - } #[test] fn test_parse_template_with_overlay() { diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 9def9451..3d3b8765 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -29,7 +29,7 @@ pub struct TextDocument { } impl TextDocument { - /// Create a new TextDocument with the given content + /// Create a new [`TextDocument`] with the given content #[must_use] pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { let line_index = LineIndex::new(&content); diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index c3c9ef92..0f600401 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -1,10 +1,12 @@ //! File system abstraction following Ruff's pattern //! -//! This module provides the `FileSystem` trait that abstracts file I/O operations. +//! This module provides the [`FileSystem`] trait that abstracts file I/O operations. //! This allows the LSP to work with both real files and in-memory overlays. +#[cfg(test)] +use std::collections::HashMap; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::{buffers::Buffers, paths}; @@ -24,13 +26,67 @@ pub trait FileSystem: Send + Sync { fn is_directory(&self, path: &Path) -> bool; /// List directory contents - fn read_directory(&self, path: &Path) -> io::Result>; + fn read_directory(&self, path: &Path) -> io::Result>; /// Get file metadata (size, modified time, etc.) fn metadata(&self, path: &Path) -> io::Result; } -/// Standard file system implementation that uses `std::fs` +/// In-memory file system for testing +#[cfg(test)] +pub struct InMemoryFileSystem { + files: HashMap, +} + +#[cfg(test)] +impl InMemoryFileSystem { + pub fn new() -> Self { + Self { + files: HashMap::new(), + } + } + + pub fn add_file(&mut self, path: PathBuf, content: String) { + self.files.insert(path, content); + } +} + +#[cfg(test)] +impl FileSystem for InMemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + // Simplified for testing - no directories in memory filesystem + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + // Simplified for testing + Ok(Vec::new()) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Metadata not supported in memory filesystem", + )) + } +} + +/// Standard file system implementation that uses [`std::fs`]. pub struct OsFileSystem; impl FileSystem for OsFileSystem { @@ -50,7 +106,7 @@ impl FileSystem for OsFileSystem { path.is_dir() } - fn read_directory(&self, path: &Path) -> io::Result> { + fn read_directory(&self, path: &Path) -> io::Result> { std::fs::read_dir(path)? .map(|entry| entry.map(|e| e.path())) .collect() @@ -61,15 +117,16 @@ impl FileSystem for OsFileSystem { } } -/// LSP file system that intercepts reads for buffered files +/// LSP file system that intercepts reads for buffered files. /// /// This implements Ruff's two-layer architecture where Layer 1 (open [`Buffers`]) /// takes precedence over Layer 2 (Salsa database). When a file is read, -/// this system first checks for a buffer (in-memory content from [`TextDocument`]) -/// and returns that content. If no buffer exists, it falls back to reading from disk. +/// this system first checks for a buffer (in-memory content from +/// [`TextDocument`](crate::document::TextDocument)) and returns that content. +/// If no buffer exists, it falls back to reading from disk. /// -/// This type is used by the [`Database`] to ensure all file reads go through -/// the buffer system first. +/// This type is used by the [`Database`](crate::db::Database) to ensure all file reads go +/// through the buffer system first. pub struct WorkspaceFileSystem { /// In-memory buffers that take precedence over disk files buffers: Buffers, @@ -110,7 +167,7 @@ impl FileSystem for WorkspaceFileSystem { self.disk.is_directory(path) } - fn read_directory(&self, path: &Path) -> io::Result> { + fn read_directory(&self, path: &Path) -> io::Result> { // Overlays are never directories, so just delegate self.disk.read_directory(path) } @@ -130,70 +187,15 @@ mod tests { use crate::language::LanguageId; use url::Url; - /// In-memory file system for testing - pub struct InMemoryFileSystem { - files: std::collections::HashMap, - } - - impl InMemoryFileSystem { - pub fn new() -> Self { - Self { - files: std::collections::HashMap::new(), - } - } - - pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { - self.files.insert(path, content); - } - } - - impl FileSystem for InMemoryFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - self.files - .get(path) - .cloned() - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) - } - - fn exists(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_file(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_directory(&self, _path: &Path) -> bool { - // Simplified for testing - no directories in memory filesystem - false - } - - fn read_directory(&self, _path: &Path) -> io::Result> { - // Simplified for testing - Ok(Vec::new()) - } - - fn metadata(&self, _path: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "Metadata not supported in memory filesystem", - )) - } - } - #[test] fn test_lsp_filesystem_overlay_precedence() { - // Create a memory filesystem with some content let mut memory_fs = InMemoryFileSystem::new(); memory_fs.add_file( std::path::PathBuf::from("/test/file.py"), "original content".to_string(), ); - // Create buffer storage let buffers = Buffers::new(); - - // Create LspFileSystem with memory fallback let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); // Before adding buffer, should read from fallback @@ -211,17 +213,13 @@ mod tests { #[test] fn test_lsp_filesystem_fallback_when_no_overlay() { - // Create memory filesystem let mut memory_fs = InMemoryFileSystem::new(); memory_fs.add_file( std::path::PathBuf::from("/test/file.py"), "disk content".to_string(), ); - // Create empty buffer storage let buffers = Buffers::new(); - - // Create LspFileSystem let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); // Should fall back to disk when no buffer exists @@ -231,14 +229,12 @@ mod tests { #[test] fn test_lsp_filesystem_other_operations_delegate() { - // Create memory filesystem let mut memory_fs = InMemoryFileSystem::new(); memory_fs.add_file( std::path::PathBuf::from("/test/file.py"), "content".to_string(), ); - // Create LspFileSystem let buffers = Buffers::new(); let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 9e17d154..861a8ca8 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -20,6 +20,8 @@ mod language; pub mod paths; mod template; +use std::path::Path; + pub use buffers::Buffers; pub use db::Database; pub use document::TextDocument; @@ -63,9 +65,9 @@ pub enum FileKind { } impl FileKind { - /// Determine `FileKind` from a file path extension. + /// Determine [`FileKind`] from a file path extension. #[must_use] - pub fn from_path(path: &std::path::Path) -> Self { + pub fn from_path(path: &Path) -> Self { match path.extension().and_then(|s| s.to_str()) { Some("py") => FileKind::Python, Some("html" | "htm") => FileKind::Template, diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 9d64f69a..df1a0df7 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -7,23 +7,9 @@ use std::path::{Path, PathBuf}; use tower_lsp_server::lsp_types; use url::Url; -/// Convert a file:// URL to a `PathBuf` +/// Convert a `file://` URL to a [`PathBuf`]. /// /// Handles percent-encoding and platform-specific path formats (e.g., Windows drives). -/// -/// # Returns -/// -/// Returns `None` if the URL scheme is not "file" or if decoding fails. -/// -/// # Examples -/// -/// ``` -/// # use url::Url; -/// # use djls_workspace::paths::url_to_path; -/// let url = Url::parse("file:///home/user/file.txt").unwrap(); -/// let path = url_to_path(&url).unwrap(); -/// assert_eq!(path.to_str().unwrap(), "/home/user/file.txt"); -/// ``` #[must_use] pub fn url_to_path(url: &Url) -> Option { // Only handle file:// URLs @@ -45,21 +31,17 @@ pub fn url_to_path(url: &Url) -> Option { Some(PathBuf::from(path.as_ref())) } -/// Convert an LSP URI to a `PathBuf` +/// Convert an LSP URI to a [`PathBuf`]. /// /// This is a convenience wrapper that parses the LSP URI string and converts it. -/// -/// # Returns -/// -/// Returns `None` if the URI cannot be parsed as a URL or is not a file:// URI. #[must_use] -pub fn lsp_uri_to_path(uri: &lsp_types::Uri) -> Option { +pub fn lsp_uri_to_path(lsp_uri: &lsp_types::Uri) -> Option { // Parse the URI string as a URL - let url = Url::parse(uri.as_str()).ok()?; + let url = Url::parse(lsp_uri.as_str()).ok()?; url_to_path(&url) } -/// Convert a Path to a file:// URL +/// Convert a [`Path`] to a `file://` URL /// /// Handles both absolute and relative paths. Relative paths are resolved /// to absolute paths before conversion. From 00fef522ad3aa0eb703d1504f3242d05292c2660 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:09:56 -0500 Subject: [PATCH 22/56] weeeee --- Cargo.lock | 1 + crates/djls-server/src/server.rs | 2 +- crates/djls-server/src/session.rs | 56 ++++++------------------------- crates/djls-workspace/Cargo.toml | 1 + crates/djls-workspace/src/db.rs | 31 +++++++++++++++++ 5 files changed, 44 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27220220..79834e6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,6 +518,7 @@ dependencies = [ "tempfile", "tokio", "tower-lsp-server", + "tracing", "url", ] diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index efb3bc80..ae472a9b 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -216,7 +216,7 @@ impl LanguageServer for DjangoLanguageServer { language_id, ); - session.open_document(url, document); + session.open_document(&url, document); }) .await; } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 2be2abda..8f4d9eac 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -41,7 +41,7 @@ use djls_workspace::{ db::{Database, SourceFile}, paths, Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, }; -use salsa::{Setter, StorageHandle}; +use salsa::StorageHandle; use tower_lsp_server::lsp_types; use url::Url; @@ -286,13 +286,13 @@ impl Session { /// This method coordinates both layers: /// - Layer 1: Stores the document content in buffers /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) - pub fn open_document(&mut self, url: Url, document: TextDocument) { + pub fn open_document(&mut self, url: &Url, document: TextDocument) { tracing::debug!("Opening document: {}", url); // Layer 1: Set buffer self.buffers.open(url.clone(), document); - // Layer 2: Create file and bump revision if it already exists + // Layer 2: Create file and touch if it already exists // This is crucial: if the file was already read from disk, we need to // invalidate Salsa's cache so it re-reads through the buffer system if let Some(path) = paths::url_to_path(&url) { @@ -302,16 +302,8 @@ impl Session { let file = db.get_or_create_file(path.clone()); if already_exists { - // File was already read - bump revision to invalidate cache - let current_rev = file.revision(db); - let new_rev = current_rev + 1; - file.set_revision(db).to(new_rev); - tracing::debug!( - "Bumped revision for {} on open: {} -> {}", - path.display(), - current_rev, - new_rev - ); + // File was already read - touch to invalidate cache + db.touch_file(&path); } else { // New file - starts at revision 0 tracing::debug!( @@ -336,9 +328,9 @@ impl Session { // Layer 1: Update buffer self.buffers.update(url.clone(), document); - // Layer 2: Bump revision to trigger invalidation + // Layer 2: Touch file to trigger invalidation if let Some(path) = paths::url_to_path(url) { - self.notify_file_changed(&path); + self.with_db_mut(|db| db.touch_file(&path)); } } @@ -383,43 +375,15 @@ impl Session { ); } - // Layer 2: Bump revision to trigger re-read from disk + // Layer 2: Touch file to trigger re-read from disk // We keep the file alive for potential re-opening if let Some(path) = paths::url_to_path(url) { - self.notify_file_changed(&path); + self.with_db_mut(|db| db.touch_file(&path)); } removed } - /// Internal: Notify that a file's content has changed. - /// - /// This bumps the file's revision number in Salsa, which triggers - /// invalidation of any queries that depend on the file's content. - fn notify_file_changed(&mut self, path: &Path) { - self.with_db_mut(|db| { - // Only bump revision if file is already being tracked - // We don't create files just for notifications - if db.has_file(path) { - let file = db.get_or_create_file(path.to_path_buf()); - let current_rev = file.revision(db); - let new_rev = current_rev + 1; - file.set_revision(db).to(new_rev); - tracing::debug!( - "Bumped revision for {}: {} -> {}", - path.display(), - current_rev, - new_rev - ); - } else { - tracing::debug!( - "File {} not tracked, skipping revision bump", - path.display() - ); - } - }); - } - // ===== Safe Query API ===== // These methods encapsulate all Salsa interactions, preventing the // "mixed database instance" bug by never exposing SourceFile or Database. @@ -503,7 +467,7 @@ mod tests { 1, LanguageId::Other, ); - session.open_document(url.clone(), document); + session.open_document(&url, document); // Try to read content - this might be where it hangs println!("**[test]** try to read content - this might be where it hangs"); diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index b1fa2e09..e2fb358a 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -15,6 +15,7 @@ percent-encoding = { workspace = true } salsa = { workspace = true } tokio = { workspace = true } tower-lsp-server = { workspace = true } +tracing = { workspace = true } url = { workspace = true } [dev-dependencies] diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 36c52ce5..3bcef5fc 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -32,6 +32,7 @@ use std::sync::Arc; use std::sync::Mutex; use dashmap::DashMap; +use salsa::Setter; use crate::{FileKind, FileSystem}; @@ -160,6 +161,36 @@ impl Database { self.files.contains_key(path) } + /// Touch a file to mark it as modified, triggering re-evaluation of dependent queries. + /// + /// Similar to Unix `touch`, this updates the file's revision number to signal + /// that cached query results depending on this file should be invalidated. + /// + /// This is typically called when: + /// - A file is opened in the editor (if it was previously cached from disk) + /// - A file's content is modified + /// - A file's buffer is closed (reverting to disk content) + pub fn touch_file(&mut self, path: &Path) { + // Get the file if it exists + let Some(file_ref) = self.files.get(path) else { + tracing::debug!("File {} not tracked, skipping touch", path.display()); + return; + }; + let file = *file_ref; + drop(file_ref); // Explicitly drop to release the lock + + let current_rev = file.revision(self); + let new_rev = current_rev + 1; + file.set_revision(self).to(new_rev); + + tracing::debug!( + "Touched {}: revision {} -> {}", + path.display(), + current_rev, + new_rev + ); + } + /// Get a reference to the storage for handle extraction. /// /// This is used by `Session` to extract the [`StorageHandle`](salsa::StorageHandle) after mutations. From f474f55b7a6558994f0519731fc7c0b8fd25f43a Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:11:06 -0500 Subject: [PATCH 23/56] remove --- task_order.md | 61 --------------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 task_order.md diff --git a/task_order.md b/task_order.md deleted file mode 100644 index 137a4023..00000000 --- a/task_order.md +++ /dev/null @@ -1,61 +0,0 @@ -# Revised Task Order for Ruff Pattern Implementation - -## The Correct Architecture Understanding - -Based on Ruff expert clarification: -- **SourceFile should NOT store text content** (our current implementation is wrong) -- **File content is read on-demand** through a `source_text` tracked function -- **Overlays are never Salsa inputs**, they're read through FileSystem -- **File revision triggers invalidation**, not content changes - -## Implementation Order - -### Phase 1: Database Foundation -1. **task-129** - Complete Database FileSystem integration - - Database needs access to LspFileSystem to read files - - This enables tracked functions to read through FileSystem - -### Phase 2: Salsa Input Restructuring -2. **task-126** - Bridge Salsa queries to LspFileSystem - - Remove `text` field from SourceFile - - Add `path` and `revision` fields - - Create `source_text` tracked function - -### Phase 3: Query Updates -3. **task-95** - Update template parsing to use source_text query - - Update all queries to use `source_text(db, file)` - - Remove direct text access from SourceFile - -### Phase 4: LSP Integration -4. **task-112** - Add file revision tracking - - Bump file revision when overlays change - - This triggers Salsa invalidation - -### Phase 5: Testing -5. **task-127** - Test overlay behavior and Salsa integration - - Verify overlays work correctly - - Test invalidation behavior - -## Key Changes from Current Implementation - -Current (WRONG): -```rust -#[salsa::input] -pub struct SourceFile { - pub text: Arc, // ❌ Storing content in Salsa -} -``` - -Target (RIGHT): -```rust -#[salsa::input] -pub struct SourceFile { - pub path: PathBuf, - pub revision: u32, // ✅ Only track changes -} - -#[salsa::tracked] -pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { - // Read through FileSystem (checks overlays first) -} -``` From f7a1816de47fba092098ff22598389ac2c7f145d Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:16:31 -0500 Subject: [PATCH 24/56] remove --- ARCHITECTURE_INSIGHTS.md | 96 ---------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 ARCHITECTURE_INSIGHTS.md diff --git a/ARCHITECTURE_INSIGHTS.md b/ARCHITECTURE_INSIGHTS.md deleted file mode 100644 index b3dcdf0b..00000000 --- a/ARCHITECTURE_INSIGHTS.md +++ /dev/null @@ -1,96 +0,0 @@ -# Architecture Insights from Ruff Investigation - -## Key Discovery: Two-Layer Architecture - -### The Problem -- LSP documents change frequently (every keystroke) -- Salsa invalidation is expensive -- Tower-lsp requires Send+Sync, but Salsa Database contains RefCell/UnsafeCell - -### The Solution (Ruff Pattern) - -#### Layer 1: LSP Document Management (Outside Salsa) -- Store overlays in `Session` using `Arc>` -- TextDocument contains actual content, version, language_id -- Changes are immediate, no Salsa invalidation - -#### Layer 2: Salsa Incremental Computation -- Database is pure Salsa, no file storage -- Queries read through FileSystem trait -- LspFileSystem intercepts reads, returns overlay or disk content - -### Critical Insights - -1. **Overlays NEVER become Salsa inputs directly** - - They're intercepted at FileSystem::read_to_string() time - - Salsa only knows "something changed", reads content lazily - -2. **StorageHandle Pattern (for tower-lsp)** - - Session stores `StorageHandle` not Database directly - - StorageHandle IS Send+Sync even though Database isn't - - Create Database instances on-demand: `session.db()` - -3. **File Management Location** - - WRONG: Store files in Database (what we initially did) - - RIGHT: Store overlays in Session, Database is pure Salsa - -4. **The Bridge** - - LspFileSystem has Arc to same overlays as Session - - When Salsa queries need content, they call FileSystem - - FileSystem checks overlays first, falls back to disk - -### Implementation Flow - -1. **did_open/did_change/did_close** → Update overlays in Session -2. **notify_file_changed()** → Tell Salsa something changed -3. **Salsa query executes** → Calls FileSystem::read_to_string() -4. **LspFileSystem intercepts** → Returns overlay if exists, else disk -5. **Query gets content** → Without knowing about LSP/overlays - -### Why This Works - -- Fast: Overlay updates don't trigger Salsa invalidation cascade -- Thread-safe: DashMap for overlays, StorageHandle for Database -- Clean separation: LSP concerns vs computation concerns -- Efficient: Salsa caching still works, just reads through FileSystem - -### Tower-lsp vs lsp-server - -- **Ruff uses lsp-server**: No Send+Sync requirement, can store Database directly -- **We use tower-lsp**: Requires Send+Sync, must use StorageHandle pattern -- Both achieve same result, different mechanisms - -## Critical Implementation Details (From Ruff Expert) - -### The Revision Dependency Trick - -**THE MOST CRITICAL INSIGHT**: In the `source_text` tracked function, calling `file.revision(db)` is what creates the Salsa dependency chain: - -```rust -#[salsa::tracked] -pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { - // THIS LINE IS CRITICAL - Creates Salsa dependency on revision! - let _ = file.revision(db); - - // Now read from FileSystem (checks overlays first) - db.read_file_content(file.path(db)) -} -``` - -Without that `file.revision(db)` call, revision changes won't trigger invalidation! - -### Key Implementation Points - -1. **Files have no text**: SourceFile inputs only have `path` and `revision`, never `text` -2. **Revision bumping triggers invalidation**: Change revision → source_text invalidated → dependent queries invalidated -3. **Files created lazily**: Don't pre-create, let them be created on first access -4. **Simple counters work**: Revision can be a simple u64 counter, doesn't need timestamps -5. **StorageHandle update required**: After DB modifications in LSP handlers, must update the handle - -### Common Pitfalls - -- **Forgetting the revision dependency** - Without `file.revision(db)`, nothing invalidates -- **Storing text in Salsa inputs** - Breaks the entire pattern -- **Not bumping revision on overlay changes** - Queries won't see new content -- **Creating files eagerly** - Unnecessary and inefficient - From 8a63ebc3d20af240ec78193aa03c8d63c601399c Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:16:51 -0500 Subject: [PATCH 25/56] remove --- REVISION_TRACKING_ARCHITECTURE.md | 341 ------------------------------ 1 file changed, 341 deletions(-) delete mode 100644 REVISION_TRACKING_ARCHITECTURE.md diff --git a/REVISION_TRACKING_ARCHITECTURE.md b/REVISION_TRACKING_ARCHITECTURE.md deleted file mode 100644 index 202c1186..00000000 --- a/REVISION_TRACKING_ARCHITECTURE.md +++ /dev/null @@ -1,341 +0,0 @@ -# Revision Tracking Architecture for Django Language Server - -## Overview - -This document captures the complete understanding of how to implement revision tracking for task-112, based on extensive discussions with a Ruff architecture expert. The goal is to connect the Session's overlay system with Salsa's query invalidation mechanism through per-file revision tracking. - -## The Critical Breakthrough: Dual-Layer Architecture - -### The Confusion We Had - -We conflated two different concepts: -1. **Database struct** - The Rust struct that implements the Salsa database trait -2. **Salsa database** - The actual Salsa storage system with inputs/queries - -### The Key Insight - -**Database struct ≠ Salsa database** - -The Database struct can contain: -- Salsa storage (the actual Salsa database) -- Additional non-Salsa data structures (like file tracking) - -## The Architecture Pattern (From Ruff) - -### Ruff's Implementation - -```rust -// Ruff's Database contains BOTH Salsa and non-Salsa data -pub struct ProjectDatabase { - storage: salsa::Storage, // Salsa's data - files: Files, // NOT Salsa data, but in Database struct! -} - -// Files is Arc-wrapped for cheap cloning -#[derive(Clone)] -pub struct Files { - inner: Arc, // Shared across clones -} - -struct FilesInner { - system_by_path: FxDashMap, // Thread-safe -} -``` - -### Our Implementation - -```rust -// Django LS Database structure -#[derive(Clone)] -pub struct Database { - storage: salsa::Storage, - files: Arc>, // Arc makes cloning cheap! -} - -// Session still uses StorageHandle for tower-lsp -pub struct Session { - db_handle: StorageHandle, // Still needed! - overlays: Arc>, // LSP document state -} -``` - -## Why This Works with Send+Sync Requirements - -1. **Arc is Send+Sync** - Thread-safe by design -2. **Cloning is cheap** - Only clones the Arc pointer (8 bytes) -3. **Persistence across clones** - All clones share the same DashMap -4. **StorageHandle compatible** - Database remains clonable and Send+Sync - -## Implementation Details - -### 1. Database Implementation - -```rust -impl Database { - pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { - self.files - .entry(path.clone()) - .or_insert_with(|| { - // Create Salsa input with initial revision 0 - SourceFile::new(self, path, 0) - }) - .clone() - } -} - -impl Clone for Database { - fn clone(&self) -> self { - Self { - storage: self.storage.clone(), // Salsa handles this - files: self.files.clone(), // Just clones Arc! - } - } -} -``` - -### 2. The Critical Pattern for Every Overlay Change - -```rust -pub fn handle_overlay_change(session: &mut Session, url: Url, content: String) { - // 1. Extract database from StorageHandle - let mut db = session.db_handle.get(); - - // 2. Update overlay in Session - session.overlays.insert(url.clone(), TextDocument::new(content)); - - // 3. Get or create file in Database - let path = path_from_url(&url); - let file = db.get_or_create_file(path); - - // 4. Bump revision (simple incrementing counter) - let current_rev = file.revision(&db); - file.set_revision(&mut db).to(current_rev + 1); - - // 5. Update StorageHandle with modified database - session.db_handle.update(db); // CRITICAL! -} -``` - -### 3. LSP Handler Updates - -#### did_open - -```rust -pub fn did_open(&mut self, params: DidOpenTextDocumentParams) { - let mut db = self.session.db_handle.get(); - - // Set overlay - self.session.overlays.insert( - params.text_document.uri.clone(), - TextDocument::new(params.text_document.text) - ); - - // Create file with initial revision 0 - let path = path_from_url(¶ms.text_document.uri); - db.get_or_create_file(path); // Creates with revision 0 - - self.session.db_handle.update(db); -} -``` - -#### did_change - -```rust -pub fn did_change(&mut self, params: DidChangeTextDocumentParams) { - let mut db = self.session.db_handle.get(); - - // Update overlay - let new_content = params.content_changes[0].text.clone(); - self.session.overlays.insert( - params.text_document.uri.clone(), - TextDocument::new(new_content) - ); - - // Bump revision - let path = path_from_url(¶ms.text_document.uri); - let file = db.get_or_create_file(path); - let new_rev = file.revision(&db) + 1; - file.set_revision(&mut db).to(new_rev); - - self.session.db_handle.update(db); -} -``` - -#### did_close - -```rust -pub fn did_close(&mut self, params: DidCloseTextDocumentParams) { - let mut db = self.session.db_handle.get(); - - // Remove overlay - self.session.overlays.remove(¶ms.text_document.uri); - - // Bump revision to trigger re-read from disk - let path = path_from_url(¶ms.text_document.uri); - if let Some(file) = db.files.get(&path) { - let new_rev = file.revision(&db) + 1; - file.set_revision(&mut db).to(new_rev); - } - - self.session.db_handle.update(db); -} -``` - -## Key Implementation Guidelines from Ruff Expert - -### 1. File Tracking Location - -- Store in Database struct (not Session) -- Use Arc for thread-safety and cheap cloning -- This keeps file tracking close to where it's used - -### 2. Revision Management - -- Use simple incrementing counter per file (not timestamps) -- Each file has independent revision tracking -- Revision just needs to change, doesn't need to be monotonic -- Example: `file.set_revision(&mut db).to(current + 1)` - -### 3. Lazy File Creation - -Files should be created: -- On did_open (via get_or_create_file) -- On first query access if needed -- NOT eagerly for all possible files - -### 4. File Lifecycle - -- **On open**: Create file with revision 0 -- **On change**: Bump revision to trigger invalidation -- **On close**: Keep file alive, bump revision for re-read from disk -- **Never remove**: Files stay in tracking even after close - -### 5. Batch Changes for Performance - -When possible, batch multiple changes: - -```rust -pub fn apply_batch_changes(&mut self, changes: Vec) { - let mut db = self.session.db_handle.get(); - - for change in changes { - // Process each change - let file = db.get_or_create_file(change.path); - file.set_revision(&mut db).to(file.revision(&db) + 1); - } - - // Single StorageHandle update at the end - self.session.db_handle.update(db); -} -``` - -### 6. Thread Safety with DashMap - -Use DashMap's atomic entry API: - -```rust -self.files.entry(path.clone()) - .and_modify(|file| { - // Modify existing - file.set_revision(db).to(new_rev); - }) - .or_insert_with(|| { - // Create new - SourceFile::builder(path) - .revision(0) - .new(db) - }); -``` - -## Critical Pitfalls to Avoid - -1. **NOT BUMPING REVISION** - Every overlay change MUST bump revision or Salsa won't invalidate -2. **FORGETTING STORAGEHANDLE UPDATE** - Must call `session.db_handle.update(db)` after changes -3. **CREATING FILES EAGERLY** - Let files be created lazily on first access -4. **USING TIMESTAMPS** - Simple incrementing counter is sufficient -5. **REMOVING FILES** - Keep files alive even after close, just bump revision - -## The Two-Layer Model - -### Layer 1: Non-Salsa (but in Database struct) -- `Arc>` - File tracking -- Thread-safe via Arc+DashMap -- Cheap to clone via Arc -- Acts as a lookup table - -### Layer 2: Salsa Inputs -- `SourceFile` entities created via `SourceFile::new(db)` -- Have revision fields for invalidation -- Tracked by Salsa's dependency system -- Invalidation cascades through dependent queries - -## Complete Architecture Summary - -| Component | Contains | Purpose | -|-----------|----------|---------| -| **Database** | `storage` + `Arc>` | Salsa queries + file tracking | -| **Session** | `StorageHandle` + `Arc>` | LSP state + overlays | -| **StorageHandle** | `Arc>>` | Bridge for tower-lsp lifetime requirements | -| **SourceFile** | Salsa input with path + revision | Triggers query invalidation | - -## The Flow - -1. **LSP request arrives** → tower-lsp handler -2. **Extract database** → `db = session.db_handle.get()` -3. **Update overlay** → `session.overlays.insert(url, content)` -4. **Get/create file** → `db.get_or_create_file(path)` -5. **Bump revision** → `file.set_revision(&mut db).to(current + 1)` -6. **Update handle** → `session.db_handle.update(db)` -7. **Salsa invalidates** → `source_text` query re-executes -8. **Queries see new content** → Through overlay-aware FileSystem - -## Why StorageHandle is Still Essential - -1. **tower-lsp requirement**: Needs 'static lifetime for async handlers -2. **Snapshot management**: Safe extraction and update of database -3. **Thread safety**: Bridges async boundaries safely -4. **Atomic updates**: Ensures consistent state transitions - -## Testing Strategy - -1. **Revision bumping**: Verify each overlay operation bumps revision -2. **Invalidation cascade**: Ensure source_text re-executes after revision bump -3. **Thread safety**: Concurrent overlay updates work correctly -4. **Clone behavior**: Database clones share the same file tracking -5. **Lazy creation**: Files only created when accessed - -## Implementation Checklist - -- [ ] Add `Arc>` to Database struct -- [ ] Implement Clone for Database (clone both storage and Arc) -- [ ] Create `get_or_create_file` method using atomic entry API -- [ ] Update did_open to create files with revision 0 -- [ ] Update did_change to bump revision after overlay update -- [ ] Update did_close to bump revision (keep file alive) -- [ ] Ensure StorageHandle updates after all database modifications -- [ ] Add tests for revision tracking and invalidation - -## Questions That Were Answered - -1. **Q: Files in Database or Session?** - A: In Database, but Arc-wrapped for cheap cloning - -2. **Q: How does this work with Send+Sync?** - A: Arc is Send+Sync, making Database clonable and thread-safe - -3. **Q: Do we still need StorageHandle?** - A: YES! It bridges tower-lsp's lifetime requirements - -4. **Q: Timestamp or counter for revisions?** - A: Simple incrementing counter per file - -5. **Q: Remove files on close?** - A: No, keep them alive and bump revision for re-read - -## The Key Insight - -Database struct is a container that holds BOTH: -- Salsa storage (for queries and inputs) -- Non-Salsa data (file tracking via Arc) - -Arc makes the non-Salsa data cheap to clone while maintaining Send+Sync compatibility. This is the pattern Ruff uses and what we should implement. \ No newline at end of file From 84f1073a1dc280fc7b1497bfeb40dca50e8d9afc Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:17:04 -0500 Subject: [PATCH 26/56] remove --- RUFF_ARCHITECTURE_INSIGHTS.md | 77 ----------------------------------- 1 file changed, 77 deletions(-) delete mode 100644 RUFF_ARCHITECTURE_INSIGHTS.md diff --git a/RUFF_ARCHITECTURE_INSIGHTS.md b/RUFF_ARCHITECTURE_INSIGHTS.md deleted file mode 100644 index f4a0bf1c..00000000 --- a/RUFF_ARCHITECTURE_INSIGHTS.md +++ /dev/null @@ -1,77 +0,0 @@ -# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution - -## This document is preserved for historical context but is OUTDATED -## We found the StorageHandle solution that solves the Send+Sync issue - -# Critical Discovery: The Tower-LSP vs lsp-server Architectural Mismatch - -## The Real Problem - -Your Ruff expert friend is correct. The fundamental issue is: - -### What We Found: - -1. **Salsa commit a3ffa22 uses `RefCell` and `UnsafeCell`** - These are inherently not `Sync` -2. **Tower-LSP requires `Sync`** - Because handlers take `&self` in async contexts -3. **Ruff uses `lsp-server`** - Which doesn't require `Sync` on the server struct - -### The Mystery: - -Your expert suggests Ruff's database IS `Send + Sync`, but our testing shows that with the same Salsa commit, the database contains: -- `RefCell` (not Sync) -- `UnsafeCell>` (not Sync) - -## Possible Explanations: - -### Theory 1: Ruff Has Custom Patches -Ruff might have additional patches or workarounds not visible in the commit hash. - -### Theory 2: Different Usage Pattern -Ruff might structure their database differently to avoid the Sync requirement entirely. - -### Theory 3: lsp-server Architecture -Since Ruff uses `lsp-server` (not `tower-lsp`), they might never need the database to be Sync: -- They clone the database for background work (requires Send only) -- The main thread owns the database, background threads get clones -- No shared references across threads - -## Verification Needed: - -1. **Check if Ruff's database is actually Sync**: - - Look for unsafe impl Sync in their codebase - - Check if they wrap the database differently - -2. **Understand lsp-server's threading model**: - - How does it handle async without requiring Sync? - - What's the message passing pattern? - -## Solution Decision Matrix (Updated): - -| Solution | Effort | Performance | Risk | Compatibility | -|----------|---------|------------|------|---------------| -| **Switch to lsp-server** | High | High | Medium | Perfect Ruff parity | -| **Actor Pattern** | Medium | Medium | Low | Works with tower-lsp | -| **Arc** | Low | Poor | Low | Works but slow | -| **Unsafe Sync wrapper** | Low | High | Very High | Dangerous | -| **Database per request** | Medium | Poor | Low | Loses memoization | - -## Recommended Action Plan: - -### Immediate (Today): -1. Verify that Salsa a3ffa22 truly has RefCell/UnsafeCell -2. Check if there are any Ruff-specific patches to Salsa -3. Test the actor pattern as a better alternative to Arc - -### Short-term (This Week): -1. Implement actor pattern if Salsa can't be made Sync -2. OR investigate unsafe Sync wrapper with careful single-threaded access guarantees - -### Long-term (This Month): -1. Consider migrating to lsp-server for full Ruff compatibility -2. OR contribute Sync support to Salsa upstream - -## The Key Insight: - -**Tower-LSP's architecture is fundamentally incompatible with Salsa's current design.** - -Ruff avoided this by using `lsp-server`, which has a different threading model that doesn't require Sync on the database. From 43138a9dd3615c9785d18070c7da94653cb83c0c Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:17:16 -0500 Subject: [PATCH 27/56] remove --- check_ruff_pattern.md | 94 ------------------------------------------- 1 file changed, 94 deletions(-) delete mode 100644 check_ruff_pattern.md diff --git a/check_ruff_pattern.md b/check_ruff_pattern.md deleted file mode 100644 index 5253b69e..00000000 --- a/check_ruff_pattern.md +++ /dev/null @@ -1,94 +0,0 @@ -# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution - -## This document is preserved for historical context but is OUTDATED -## We found the StorageHandle solution that solves the Send+Sync issue - -# Key Findings from Ruff's Architecture - -Based on the exploration, here's what we discovered: - -## Current Django LS Architecture - -### What We Have: -1. `Database` struct with `#[derive(Clone)]` and Salsa storage -2. `WorkspaceDatabase` that wraps `Database` and uses `DashMap` for thread-safe file storage -3. `Session` that owns `WorkspaceDatabase` directly (not wrapped in Arc) -4. Tower-LSP server that requires `Send + Sync` for async handlers - -### The Problem: -- `Database` is not `Sync` due to `RefCell` and `UnsafeCell` in Salsa's `ZalsaLocal` -- This prevents `Session` from being `Sync`, which breaks tower-lsp async handlers - -## Ruff's Solution (From Analysis) - -### They Don't Make Database Sync! -The key insight is that Ruff **doesn't actually make the database Send + Sync**. Instead: - -1. **Clone for Background Work**: They clone the database for each background task -2. **Move Not Share**: The cloned database is *moved* into the task (requires Send, not Sync) -3. **Message Passing**: Results are sent back via channels - -### Critical Difference: -- Ruff uses a custom LSP implementation that doesn't require `Sync` on the session -- Tower-LSP *does* require `Sync` because handlers take `&self` - -## The Real Problem - -Tower-LSP's `LanguageServer` trait requires: -```rust -async fn initialize(&self, ...) -> ... -// ^^^^^ This requires self to be Sync! -``` - -But with Salsa's current implementation, the Database can never be Sync. - -## Solution Options - -### Option 1: Wrap Database in Arc (Current Workaround) -```rust -pub struct Session { - database: Arc>, - // ... -} -``` -Downsides: Lock contention, defeats purpose of Salsa's internal optimization - -### Option 2: Move Database Out of Session -```rust -pub struct Session { - // Don't store database here - file_index: Arc>, - settings: Settings, -} - -// Create database on demand for each request -impl LanguageServer for Server { - async fn some_handler(&self) { - let db = create_database_from_index(&self.session.file_index); - // Use db for this request - } -} -``` - -### Option 3: Use Actor Pattern -```rust -pub struct DatabaseActor { - database: WorkspaceDatabase, - rx: mpsc::Receiver, -} - -pub struct Session { - db_tx: mpsc::Sender, -} -``` - -### Option 4: Custom unsafe Send/Sync implementation -This is risky but possible if we ensure single-threaded access patterns. - -## The Salsa Version Mystery - -We're using the exact same Salsa commit as Ruff, with the same features. The issue is NOT the Salsa version, but how tower-lsp forces us to use it. - -Ruff likely either: -1. Doesn't use tower-lsp (has custom implementation) -2. Or structures their server differently to avoid needing Sync on the database From f47d9dfe4dab462e5de2987768af063f8b1db64b Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:18:12 -0500 Subject: [PATCH 28/56] lint --- crates/djls-server/src/client.rs | 5 ++++- crates/djls-server/src/session.rs | 18 ++++++++++++------ crates/djls-server/tests/lsp_integration.rs | 18 +++++++++++------- crates/djls-workspace/src/buffers.rs | 3 ++- crates/djls-workspace/src/db.rs | 11 +++++++---- crates/djls-workspace/src/document.rs | 5 +++-- crates/djls-workspace/src/fs.rs | 9 ++++++--- crates/djls-workspace/src/lib.rs | 4 +++- crates/djls-workspace/src/paths.rs | 4 +++- 9 files changed, 51 insertions(+), 26 deletions(-) diff --git a/crates/djls-server/src/client.rs b/crates/djls-server/src/client.rs index 11a5f62e..35e616fb 100644 --- a/crates/djls-server/src/client.rs +++ b/crates/djls-server/src/client.rs @@ -193,7 +193,10 @@ pub mod monitoring { } } - pub fn progress + Send>(token: lsp_types::ProgressToken, title: T) -> Option { + pub fn progress + Send>( + token: lsp_types::ProgressToken, + title: T, + ) -> Option { get_client().map(|client| client.progress(token, title)) } } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 8f4d9eac..be9224ce 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -31,16 +31,21 @@ //! //! The explicit method names make the intent clear and prevent accidental misuse. -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use dashmap::DashMap; use djls_conf::Settings; use djls_project::DjangoProject; -use djls_workspace::{ - db::{Database, SourceFile}, - paths, Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, -}; +use djls_workspace::db::Database; +use djls_workspace::db::SourceFile; +use djls_workspace::paths; +use djls_workspace::Buffers; +use djls_workspace::FileSystem; +use djls_workspace::OsFileSystem; +use djls_workspace::TextDocument; +use djls_workspace::WorkspaceFileSystem; use salsa::StorageHandle; use tower_lsp_server::lsp_types; use url::Url; @@ -447,9 +452,10 @@ impl Default for Session { #[cfg(test)] mod tests { - use super::*; use djls_workspace::LanguageId; + use super::*; + #[test] fn test_revision_invalidation_chain() { use std::path::PathBuf; diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index fa5613f5..d277275b 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -12,11 +12,16 @@ use std::sync::Arc; use djls_server::DjangoLanguageServer; use tempfile::TempDir; -use tower_lsp_server::lsp_types::{ - DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, - InitializeParams, InitializedParams, TextDocumentContentChangeEvent, TextDocumentIdentifier, - TextDocumentItem, VersionedTextDocumentIdentifier, WorkspaceFolder, -}; +use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; +use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; +use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; +use tower_lsp_server::lsp_types::InitializeParams; +use tower_lsp_server::lsp_types::InitializedParams; +use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; +use tower_lsp_server::lsp_types::TextDocumentIdentifier; +use tower_lsp_server::lsp_types::TextDocumentItem; +use tower_lsp_server::lsp_types::VersionedTextDocumentIdentifier; +use tower_lsp_server::lsp_types::WorkspaceFolder; use tower_lsp_server::LanguageServer; use url::Url; @@ -240,7 +245,7 @@ async fn test_template_parsing_with_overlays() { .await; use djls_workspace::db::parse_template; - // Parse template through the session + // Parse template through the session let workspace_path = server.workspace_file(file_name); let ast = server .server @@ -441,4 +446,3 @@ async fn test_revision_tracking_across_lifecycle() { server.change_document(file_name, "Final", 11).await; assert_eq!(server.get_file_revision(file_name).await, Some(7)); } - diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 0365a3a6..6f26ad16 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -5,8 +5,9 @@ //! (which manages document lifecycle) and the [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem) (which //! reads from them). -use dashmap::DashMap; use std::sync::Arc; + +use dashmap::DashMap; use url::Url; use crate::document::TextDocument; diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 3bcef5fc..deaccc70 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -26,7 +26,8 @@ //! [`Buffers`]: crate::buffers::Buffers //! [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; #[cfg(test)] use std::sync::Mutex; @@ -34,7 +35,8 @@ use std::sync::Mutex; use dashmap::DashMap; use salsa::Setter; -use crate::{FileKind, FileSystem}; +use crate::FileKind; +use crate::FileSystem; /// Database trait that provides file system access for Salsa queries #[salsa::db] @@ -392,14 +394,15 @@ pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { #[cfg(test)] mod tests { + use dashmap::DashMap; + use salsa::Setter; + use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; use crate::fs::InMemoryFileSystem; use crate::fs::WorkspaceFileSystem; use crate::language::LanguageId; - use dashmap::DashMap; - use salsa::Setter; #[test] fn test_parse_template_with_overlay() { diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 3d3b8765..60c70bb1 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -5,11 +5,12 @@ //! performance when handling frequent position-based operations like hover, completion, //! and diagnostics. +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; + use crate::language::LanguageId; use crate::template::ClosingBrace; use crate::template::TemplateTagContext; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; /// In-memory representation of an open document in the LSP. /// diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 0f600401..b0e7ac37 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -6,10 +6,12 @@ #[cfg(test)] use std::collections::HashMap; use std::io; -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; -use crate::{buffers::Buffers, paths}; +use crate::buffers::Buffers; +use crate::paths; /// Trait for file system operations pub trait FileSystem: Send + Sync { @@ -181,11 +183,12 @@ impl FileSystem for WorkspaceFileSystem { #[cfg(test)] mod tests { + use url::Url; + use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; use crate::language::LanguageId; - use url::Url; #[test] fn test_lsp_filesystem_overlay_precedence() { diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 861a8ca8..b8b80e5c 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -25,7 +25,9 @@ use std::path::Path; pub use buffers::Buffers; pub use db::Database; pub use document::TextDocument; -pub use fs::{FileSystem, OsFileSystem, WorkspaceFileSystem}; +pub use fs::FileSystem; +pub use fs::OsFileSystem; +pub use fs::WorkspaceFileSystem; pub use language::LanguageId; /// Stable, compact identifier for files across the subsystem. diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index df1a0df7..2fde6281 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -3,7 +3,9 @@ //! This module provides consistent conversion between file paths and URLs, //! handling platform-specific differences and encoding issues. -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; + use tower_lsp_server::lsp_types; use url::Url; From 196a6344fed35b7a61d5056f18402975561b1358 Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 2 Sep 2025 23:19:24 -0500 Subject: [PATCH 29/56] remove comments and adjust some others --- crates/djls-workspace/src/db.rs | 52 +++------------------------ crates/djls-workspace/src/document.rs | 9 ++--- crates/djls-workspace/src/fs.rs | 5 ++- crates/djls-workspace/src/template.rs | 2 ++ 4 files changed, 14 insertions(+), 54 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index deaccc70..02908970 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -61,6 +61,7 @@ pub trait Db: salsa::Database { pub struct Database { storage: salsa::Storage, + // TODO: does this need to be an Option? /// File system for reading file content (checks buffers first, then disk). fs: Option>, @@ -98,7 +99,6 @@ impl Default for Database { } impl Database { - /// Create a new database with fresh storage. pub fn new(file_system: Arc, files: Arc>) -> Self { Self { storage: salsa::Storage::new(None), @@ -109,8 +109,6 @@ impl Database { } } - /// Create a database instance from an existing storage. - /// This preserves both the file system and files Arc across database operations. pub fn from_storage( storage: salsa::Storage, file_system: Arc, @@ -136,9 +134,8 @@ impl Database { /// Get or create a [`SourceFile`] for the given path. /// - /// This method implements Ruff's pattern for lazy file creation. Files are created - /// with an initial revision of 0 and tracked in the [`Database`]'s `DashMap`. The `Arc` - /// ensures cheap cloning while maintaining thread safety. + /// Files are created with an initial revision of 0 and tracked in the [`Database`]'s + /// `DashMap`. The `Arc` ensures cheap cloning while maintaining thread safety. pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { if let Some(file_ref) = self.files.get(&path) { // Copy the value (SourceFile is Copy) and drop the guard immediately @@ -242,9 +239,6 @@ pub struct SourceFile { } /// Read file content, creating a Salsa dependency on the file's revision. -/// -/// **Critical**: The call to `file.revision(db)` creates the dependency chain. -/// Without it, revision changes won't trigger query invalidation. #[salsa::tracked] pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { // This line creates the Salsa dependency on revision! Without this call, @@ -260,18 +254,6 @@ pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { } } -/// Global input configuring ordered template loader roots. -/// -/// [`TemplateLoaderOrder`] represents the Django `TEMPLATES[n]['DIRS']` configuration, -/// defining the search order for template resolution. This is a global input that -/// affects template name resolution across the entire project. -#[salsa::input] -pub struct TemplateLoaderOrder { - /// Ordered list of template root directories - #[returns(ref)] - pub roots: Arc<[String]>, -} - /// Represents a file path for Salsa tracking. /// /// [`FilePath`] is a Salsa input entity that tracks a file path for use in @@ -347,7 +329,8 @@ pub fn parse_template_by_path(db: &dyn Db, file_path: FilePath) -> Option { // Convert errors to strings @@ -367,31 +350,6 @@ pub fn parse_template_by_path(db: &dyn Db, file_path: FilePath) -> Option Arc<[String]> { - parse_template_by_path(db, file_path) - .map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) -} - -/// Get template parsing errors for a file. -/// -/// This Salsa tracked function extracts just the errors from the parsed template, -/// useful for diagnostics without needing the full AST. -/// -/// Returns an empty vector for non-template files. -#[salsa::tracked] -pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { - parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) -} - #[cfg(test)] mod tests { use dashmap::DashMap; diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 60c70bb1..eb67d479 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -30,7 +30,6 @@ pub struct TextDocument { } impl TextDocument { - /// Create a new [`TextDocument`] with the given content #[must_use] pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { let line_index = LineIndex::new(&content); @@ -42,19 +41,16 @@ impl TextDocument { } } - /// Get the document's content #[must_use] pub fn content(&self) -> &str { &self.content } - /// Get the version number #[must_use] pub fn version(&self) -> i32 { self.version } - /// Get the language identifier #[must_use] pub fn language_id(&self) -> LanguageId { self.language_id.clone() @@ -65,6 +61,7 @@ impl TextDocument { &self.line_index } + #[must_use] pub fn get_line(&self, line: u32) -> Option { let line_start = *self.line_index.line_starts.get(line as usize)?; let line_end = self @@ -77,6 +74,7 @@ impl TextDocument { Some(self.content[line_start as usize..line_end as usize].to_string()) } + #[must_use] pub fn get_text_range(&self, range: Range) -> Option { let start_offset = self.line_index.offset(range.start)? as usize; let end_offset = self.line_index.offset(range.end)? as usize; @@ -100,6 +98,7 @@ impl TextDocument { self.version = version; } + #[must_use] pub fn get_template_tag_context(&self, position: Position) -> Option { let start = self.line_index.line_starts.get(position.line as usize)?; let end = self @@ -135,10 +134,12 @@ impl TextDocument { }) } + #[must_use] pub fn position_to_offset(&self, position: Position) -> Option { self.line_index.offset(position) } + #[must_use] pub fn offset_to_position(&self, offset: u32) -> Position { self.line_index.position(offset) } diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index b0e7ac37..00b5cb1c 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -1,4 +1,4 @@ -//! File system abstraction following Ruff's pattern +//! Virtual file system abstraction //! //! This module provides the [`FileSystem`] trait that abstracts file I/O operations. //! This allows the LSP to work with both real files and in-memory overlays. @@ -121,7 +121,7 @@ impl FileSystem for OsFileSystem { /// LSP file system that intercepts reads for buffered files. /// -/// This implements Ruff's two-layer architecture where Layer 1 (open [`Buffers`]) +/// This implements a two-layer architecture where Layer 1 (open [`Buffers`]) /// takes precedence over Layer 2 (Salsa database). When a file is read, /// this system first checks for a buffer (in-memory content from /// [`TextDocument`](crate::document::TextDocument)) and returns that content. @@ -137,7 +137,6 @@ pub struct WorkspaceFileSystem { } impl WorkspaceFileSystem { - /// Create a new [`WorkspaceFileSystem`] with the given buffer storage and fallback #[must_use] pub fn new(buffers: Buffers, disk: Arc) -> Self { Self { buffers, disk } diff --git a/crates/djls-workspace/src/template.rs b/crates/djls-workspace/src/template.rs index 2ebc3247..b2bd44b8 100644 --- a/crates/djls-workspace/src/template.rs +++ b/crates/djls-workspace/src/template.rs @@ -3,6 +3,8 @@ //! Detects cursor position context within Django template tags to provide //! appropriate completions and auto-closing behavior. +// TODO: is this module in the right spot or even needed? + /// Tracks what closing characters are needed to complete a template tag. /// /// Used to determine whether the completion system needs to insert From d4b0397fd19a0f27de31e438b88ed3a8867cf009 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 10:26:14 -0500 Subject: [PATCH 30/56] fix some documentation --- crates/djls-server/src/session.rs | 87 +++++++++++++------------------ 1 file changed, 36 insertions(+), 51 deletions(-) diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index be9224ce..e0ffc3ca 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,13 +1,13 @@ -//! # Salsa StorageHandle Pattern for LSP +//! # Salsa [`StorageHandle`] Pattern for LSP //! //! This module implements a thread-safe Salsa database wrapper for use with //! tower-lsp's async runtime. The key challenge is that tower-lsp requires //! `Send + Sync + 'static` bounds, but Salsa's `Storage` contains thread-local //! state and is not `Send`. //! -//! ## The Solution: StorageHandle +//! ## The Solution: [`StorageHandle`] //! -//! Salsa provides `StorageHandle` which IS `Send + Sync` because it contains +//! Salsa provides [`StorageHandle`] which IS `Send + Sync` because it contains //! no thread-local state. We store the handle and create `Storage`/`Database` //! instances on-demand. //! @@ -26,10 +26,12 @@ //! //! ## The Pattern //! -//! - **Reads**: Clone the handle freely (`with_db`) -//! - **Mutations**: Take exclusive ownership (`with_db_mut` via `take_db_handle_for_mutation`) +//! - **Reads**: Clone the handle freely ([`with_db`](Session::with_db)) +//! - **Mutations**: Take exclusive ownership ([`with_db_mut`](Session::with_db_mut) via [`take_db_handle_for_mutation`](Session::take_db_handle_for_mutation)) //! //! The explicit method names make the intent clear and prevent accidental misuse. +//! +//! [`StorageHandle`]: salsa::StorageHandle use std::path::Path; use std::path::PathBuf; @@ -52,12 +54,12 @@ use url::Url; /// LSP Session with thread-safe Salsa database access. /// -/// Uses Salsa's `StorageHandle` pattern to maintain `Send + Sync + 'static` +/// Uses Salsa's [`StorageHandle`] pattern to maintain `Send + Sync + 'static` /// compatibility required by tower-lsp. The handle can be safely shared /// across threads and async boundaries. /// /// See [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) -/// for more information about `StorageHandle`. +/// for more information about [`StorageHandle`]. /// /// ## Architecture /// @@ -69,11 +71,13 @@ use url::Url; /// /// When mutating Salsa inputs (like changing file revisions), we must ensure /// exclusive access to prevent race conditions. Salsa enforces this through -/// its `cancel_others()` mechanism, which waits for all `StorageHandle` clones +/// its `cancel_others()` mechanism, which waits for all [`StorageHandle`] clones /// to drop before allowing mutations. /// /// We use explicit methods (`take_db_handle_for_mutation`/`restore_db_handle`) /// to make this ownership transfer clear and prevent accidental deadlocks. +/// +/// [`StorageHandle`]: salsa::StorageHandle pub struct Session { /// The Django project configuration project: Option, @@ -85,27 +89,27 @@ pub struct Session { /// /// This implements Ruff's two-layer architecture where Layer 1 contains /// open document buffers that take precedence over disk files. The buffers - /// are shared between Session (which manages them) and WorkspaceFileSystem - /// (which reads from them). + /// are shared between Session (which manages them) and + /// [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) (which reads from them). /// /// Key properties: /// - Thread-safe via the Buffers abstraction - /// - Contains full TextDocument with content, version, and metadata + /// - Contains full [`TextDocument`](djls_workspace::TextDocument) with content, version, and metadata /// - Never becomes Salsa inputs - only intercepted at read time buffers: Buffers, /// File system abstraction with buffer interception /// - /// This WorkspaceFileSystem bridges Layer 1 (buffers) and Layer 2 (Salsa). - /// It intercepts FileSystem::read_to_string() calls to return buffer + /// This [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) bridges Layer 1 (buffers) and Layer 2 (Salsa). + /// It intercepts [`FileSystem::read_to_string()`](djls_workspace::FileSystem::read_to_string()) calls to return buffer /// content when available, falling back to disk otherwise. file_system: Arc, /// Shared file tracking across all Database instances /// /// This is the canonical Salsa pattern from the lazy-input example. - /// The DashMap provides O(1) lookups and is shared via Arc across - /// all Database instances created from StorageHandle. + /// The [`DashMap`] provides O(1) lookups and is shared via Arc across + /// all Database instances created from [`StorageHandle`](salsa::StorageHandle). files: Arc>, #[allow(dead_code)] @@ -113,11 +117,11 @@ pub struct Session { /// Layer 2: Thread-safe Salsa database handle for pure computation /// - /// where we're using the `StorageHandle` to create a thread-safe handle that can be + /// where we're using the [`StorageHandle`](salsa::StorageHandle) to create a thread-safe handle that can be /// shared between threads. /// - /// The database receives file content via the FileSystem trait, which - /// is intercepted by our LspFileSystem to provide overlay content. + /// The database receives file content via the [`FileSystem`](djls_workspace::FileSystem) trait, which + /// is intercepted by our [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) to provide overlay content. /// This maintains proper separation between Layer 1 and Layer 2. db_handle: StorageHandle, } @@ -191,9 +195,13 @@ impl Session { self.settings = settings; } + // TODO: Explore an abstraction around [`salsa::StorageHandle`] and the following two methods + // to make it easy in the future to avoid deadlocks. For now, this is simpler and TBH may be + // all we ever need, but still.. might be a nice CYA for future me + /// Takes exclusive ownership of the database handle for mutation operations. /// - /// This method extracts the `StorageHandle` from the session, replacing it + /// This method extracts the [`StorageHandle`](salsa::StorageHandle) from the session, replacing it /// with a temporary placeholder. This ensures there's exactly one handle /// active during mutations, preventing deadlocks in Salsa's `cancel_others()`. /// @@ -205,8 +213,9 @@ impl Session { /// /// ## Panics /// - /// This is an internal method that should only be called by `with_db_mut`. - /// Multiple concurrent calls would panic when trying to take an already-taken handle. + /// This is an internal method that should only be called by + /// [`with_db_mut`](Session::with_db_mut). Multiple concurrent calls would panic when trying + /// to take an already-taken handle. fn take_db_handle_for_mutation(&mut self) -> StorageHandle { std::mem::replace(&mut self.db_handle, StorageHandle::new(None)) } @@ -290,7 +299,7 @@ impl Session { /// /// This method coordinates both layers: /// - Layer 1: Stores the document content in buffers - /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) + /// - Layer 2: Creates the [`SourceFile`](djls_workspace::SourceFile) in Salsa (if path is resolvable) pub fn open_document(&mut self, url: &Url, document: TextDocument) { tracing::debug!("Opening document: {}", url); @@ -300,7 +309,7 @@ impl Session { // Layer 2: Create file and touch if it already exists // This is crucial: if the file was already read from disk, we need to // invalidate Salsa's cache so it re-reads through the buffer system - if let Some(path) = paths::url_to_path(&url) { + if let Some(path) = paths::url_to_path(url) { self.with_db_mut(|db| { // Check if file already exists (was previously read from disk) let already_exists = db.has_file(&path); @@ -389,15 +398,11 @@ impl Session { removed } - // ===== Safe Query API ===== - // These methods encapsulate all Salsa interactions, preventing the - // "mixed database instance" bug by never exposing SourceFile or Database. - /// Get the current content of a file (from overlay or disk). /// /// This is the safe way to read file content through the system. /// The file is created if it doesn't exist, and content is read - /// through the FileSystem abstraction (overlay first, then disk). + /// through the `FileSystem` abstraction (overlay first, then disk). pub fn file_content(&mut self, path: PathBuf) -> String { use djls_workspace::db::source_text; @@ -452,22 +457,17 @@ impl Default for Session { #[cfg(test)] mod tests { - use djls_workspace::LanguageId; - use super::*; + use djls_workspace::LanguageId; #[test] fn test_revision_invalidation_chain() { - use std::path::PathBuf; - let mut session = Session::default(); - // Create a test file path let path = PathBuf::from("/test/template.html"); let url = Url::parse("file:///test/template.html").unwrap(); // Open document with initial content - println!("**[test]** open document with initial content"); let document = TextDocument::new( "

Original Content

".to_string(), 1, @@ -475,50 +475,36 @@ mod tests { ); session.open_document(&url, document); - // Try to read content - this might be where it hangs - println!("**[test]** try to read content - this might be where it hangs"); let content1 = session.file_content(path.clone()); assert_eq!(content1, "

Original Content

"); // Update document with new content - println!("**[test]** Update document with new content"); let updated_document = TextDocument::new("

Updated Content

".to_string(), 2, LanguageId::Other); session.update_document(&url, updated_document); // Read content again (should get new overlay content due to invalidation) - println!( - "**[test]** Read content again (should get new overlay content due to invalidation)" - ); let content2 = session.file_content(path.clone()); assert_eq!(content2, "

Updated Content

"); assert_ne!(content1, content2); // Close document (removes overlay, bumps revision) - println!("**[test]** Close document (removes overlay, bumps revision)"); session.close_document(&url); // Read content again (should now read from disk, which returns empty for missing files) - println!( - "**[test]** Read content again (should now read from disk, which returns empty for missing files)" - ); let content3 = session.file_content(path.clone()); assert_eq!(content3, ""); // No file on disk, returns empty } #[test] fn test_with_db_mut_preserves_files() { - use std::path::PathBuf; - let mut session = Session::default(); - // Create multiple files let path1 = PathBuf::from("/test/file1.py"); let path2 = PathBuf::from("/test/file2.py"); - // Create files through safe API - session.file_content(path1.clone()); // Creates file1 - session.file_content(path2.clone()); // Creates file2 + session.file_content(path1.clone()); + session.file_content(path2.clone()); // Verify files are preserved across operations assert!(session.has_file(&path1)); @@ -532,7 +518,6 @@ mod tests { assert_eq!(content1, ""); assert_eq!(content2, ""); - // One more verification assert!(session.has_file(&path1)); assert!(session.has_file(&path2)); } From 4790ee78c547b7d4e539d8bc1449f38188338c98 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 12:15:38 -0500 Subject: [PATCH 31/56] wip --- crates/djls-server/src/server.rs | 9 +- crates/djls-server/src/session.rs | 15 ++ crates/djls-workspace/src/document.rs | 236 +++++++++++++++++++++----- crates/djls-workspace/src/encoding.rs | 148 ++++++++++++++++ crates/djls-workspace/src/lib.rs | 2 + 5 files changed, 368 insertions(+), 42 deletions(-) create mode 100644 crates/djls-workspace/src/encoding.rs diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index ae472a9b..8d1b9f24 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -82,6 +82,7 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Initializing server..."); let session = Session::new(¶ms); + let encoding = session.position_encoding(); { let mut session_lock = self.session.write().await; @@ -115,13 +116,14 @@ impl LanguageServer for DjangoLanguageServer { save: Some(lsp_types::SaveOptions::default().into()), }, )), + position_encoding: Some(encoding.to_lsp_kind()), ..Default::default() }, server_info: Some(lsp_types::ServerInfo { name: SERVER_NAME.to_string(), version: Some(SERVER_VERSION.to_string()), }), - offset_encoding: None, + offset_encoding: Some(encoding.as_str().to_string()), }) } @@ -269,9 +271,10 @@ impl LanguageServer for DjangoLanguageServer { ) -> LspResult> { let response = self .with_session_mut(|session| { - let lsp_uri = ¶ms.text_document_position.text_document.uri; + let lsp_uri = params.text_document_position.text_document.uri; let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); let position = params.text_document_position.position; + let _encoding = session.position_encoding(); tracing::debug!("Completion requested for {} at {:?}", url, position); @@ -281,7 +284,7 @@ impl LanguageServer for DjangoLanguageServer { tracing::debug!("File {} has no content", url); } else { tracing::debug!("Using content for completion in {}", url); - // TODO: Implement actual completion logic using content + // TODO: Implement actual completion logic using content and encoding } } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index e0ffc3ca..722b2e04 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -52,6 +52,8 @@ use salsa::StorageHandle; use tower_lsp_server::lsp_types; use url::Url; +use djls_workspace::PositionEncoding; + /// LSP Session with thread-safe Salsa database access. /// /// Uses Salsa's [`StorageHandle`] pattern to maintain `Send + Sync + 'static` @@ -115,6 +117,9 @@ pub struct Session { #[allow(dead_code)] client_capabilities: lsp_types::ClientCapabilities, + /// Position encoding negotiated with client + position_encoding: PositionEncoding, + /// Layer 2: Thread-safe Salsa database handle for pure computation /// /// where we're using the [`StorageHandle`](salsa::StorageHandle) to create a thread-safe handle that can be @@ -152,6 +157,9 @@ impl Session { .clone() .into_zalsa_handle(); + // Negotiate position encoding with client + let position_encoding = PositionEncoding::negotiate(params); + Self { project, settings, @@ -159,6 +167,7 @@ impl Session { file_system, files, client_capabilities: params.capabilities.clone(), + position_encoding, db_handle, } } @@ -195,6 +204,11 @@ impl Session { self.settings = settings; } + #[must_use] + pub fn position_encoding(&self) -> PositionEncoding { + self.position_encoding + } + // TODO: Explore an abstraction around [`salsa::StorageHandle`] and the following two methods // to make it easy in the future to avoid deadlocks. For now, this is simpler and TBH may be // all we ever need, but still.. might be a nice CYA for future me @@ -451,6 +465,7 @@ impl Default for Session { files, buffers, client_capabilities: lsp_types::ClientCapabilities::default(), + position_encoding: PositionEncoding::default(), } } } diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index eb67d479..1f6fd250 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -8,6 +8,7 @@ use tower_lsp_server::lsp_types::Position; use tower_lsp_server::lsp_types::Range; +use crate::encoding::PositionEncoding; use crate::language::LanguageId; use crate::template::ClosingBrace; use crate::template::TemplateTagContext; @@ -75,9 +76,9 @@ impl TextDocument { } #[must_use] - pub fn get_text_range(&self, range: Range) -> Option { - let start_offset = self.line_index.offset(range.start)? as usize; - let end_offset = self.line_index.offset(range.end)? as usize; + pub fn get_text_range(&self, range: Range, encoding: PositionEncoding) -> Option { + let start_offset = self.line_index.offset(range.start, &self.content, encoding) as usize; + let end_offset = self.line_index.offset(range.end, &self.content, encoding) as usize; Some(self.content[start_offset..end_offset].to_string()) } @@ -99,7 +100,11 @@ impl TextDocument { } #[must_use] - pub fn get_template_tag_context(&self, position: Position) -> Option { + pub fn get_template_tag_context( + &self, + position: Position, + encoding: PositionEncoding, + ) -> Option { let start = self.line_index.line_starts.get(position.line as usize)?; let end = self .line_index @@ -109,7 +114,11 @@ impl TextDocument { .unwrap_or(self.line_index.length); let line = &self.content[*start as usize..end as usize]; - let char_pos: usize = position.character.try_into().ok()?; + + // Use the new offset method with the specified encoding + let char_offset = self.line_index.offset(position, &self.content, encoding) as usize; + let char_pos = char_offset - *start as usize; + let prefix = &line[..char_pos]; let rest_of_line = &line[char_pos..]; let rest_trimmed = rest_of_line.trim_start(); @@ -135,8 +144,12 @@ impl TextDocument { } #[must_use] - pub fn position_to_offset(&self, position: Position) -> Option { - self.line_index.offset(position) + pub fn position_to_offset( + &self, + position: Position, + encoding: PositionEncoding, + ) -> Option { + Some(self.line_index.offset(position, &self.content, encoding)) } #[must_use] @@ -153,51 +166,50 @@ impl TextDocument { #[derive(Clone, Debug)] pub struct LineIndex { pub line_starts: Vec, - pub line_starts_utf16: Vec, pub length: u32, - pub length_utf16: u32, + pub kind: IndexKind, } impl LineIndex { #[must_use] pub fn new(text: &str) -> Self { let mut line_starts = vec![0]; - let mut line_starts_utf16 = vec![0]; let mut pos_utf8 = 0; - let mut pos_utf16 = 0; + + // Check if text is pure ASCII for optimization + let kind = if text.is_ascii() { + IndexKind::Ascii + } else { + IndexKind::Utf8 + }; for c in text.chars() { pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); - pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); if c == '\n' { line_starts.push(pos_utf8); - line_starts_utf16.push(pos_utf16); } } Self { line_starts, - line_starts_utf16, length: pos_utf8, - length_utf16: pos_utf16, + kind, } } - #[must_use] - pub fn offset(&self, position: Position) -> Option { - let line_start = self.line_starts.get(position.line as usize)?; - - Some(line_start + position.character) - } - - /// Convert UTF-16 LSP position to UTF-8 byte offset - pub fn offset_utf16(&self, position: Position, text: &str) -> Option { - let line_start_utf8 = self.line_starts.get(position.line as usize)?; - let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; + /// Convert position to text offset using the specified encoding + /// + /// Returns a valid offset, clamping out-of-bounds positions to document/line boundaries + pub fn offset(&self, position: Position, text: &str, encoding: PositionEncoding) -> u32 { + // Handle line bounds - if line > line_count, return document length + let line_start_utf8 = match self.line_starts.get(position.line as usize) { + Some(start) => *start, + None => return self.length, // Past end of document + }; - // If position is at start of line, return UTF-8 line start + // If position is at start of line, return line start if position.character == 0 { - return Some(*line_start_utf8); + return line_start_utf8; } // Find the line text @@ -207,21 +219,61 @@ impl LineIndex { .copied() .unwrap_or(self.length); - let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; + let Some(line_text) = text.get(line_start_utf8 as usize..next_line_start as usize) else { + return line_start_utf8; + }; - // Convert UTF-16 character offset to UTF-8 byte offset within the line - let mut utf16_pos = 0; - let mut utf8_pos = 0; + // ASCII fast path optimization + if matches!(self.kind, IndexKind::Ascii) { + // For ASCII text, all encodings are equivalent to byte offsets + let char_offset = position.character.min(line_text.len() as u32); + return line_start_utf8 + char_offset; + } - for c in line_text.chars() { - if utf16_pos >= position.character { - break; + // Handle different encodings for non-ASCII text + match encoding { + PositionEncoding::Utf8 => { + // UTF-8: character positions are already byte offsets + let char_offset = position.character.min(line_text.len() as u32); + line_start_utf8 + char_offset + } + PositionEncoding::Utf16 => { + // UTF-16: count UTF-16 code units + let mut utf16_pos = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if utf16_pos >= position.character { + break; + } + utf16_pos += c.len_utf16() as u32; + utf8_pos += c.len_utf8() as u32; + } + + // If character position exceeds line length, clamp to line end + if utf16_pos < position.character && utf8_pos == line_text.len() as u32 { + line_start_utf8 + utf8_pos + } else { + line_start_utf8 + utf8_pos + } + } + PositionEncoding::Utf32 => { + // UTF-32: count Unicode code points (characters) + let mut char_count = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if char_count >= position.character { + break; + } + char_count += 1; + utf8_pos += c.len_utf8() as u32; + } + + // If character position exceeds line length, clamp to line end + line_start_utf8 + utf8_pos } - utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); - utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); } - - Some(line_start_utf8 + utf8_pos) } #[allow(dead_code)] @@ -238,3 +290,109 @@ impl LineIndex { Position::new(u32::try_from(line).unwrap_or(0), character) } } + +/// Index kind for ASCII optimization +#[derive(Clone, Debug)] +pub enum IndexKind { + /// Document contains only ASCII characters - enables fast path optimization + Ascii, + /// Document contains multi-byte UTF-8 characters - requires full UTF-8 processing + Utf8, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::language::LanguageId; + + #[test] + fn test_utf16_position_handling() { + // Test document with emoji and multi-byte characters + let content = "Hello 🌍!\nSecond 行 line"; + let doc = TextDocument::new(content.to_string(), 1, LanguageId::HtmlDjango); + + // Test position after emoji + // "Hello 🌍!" - the 🌍 emoji is 4 UTF-8 bytes but 2 UTF-16 code units + // Position after the emoji should be at UTF-16 position 7 (Hello + space + emoji) + let pos_after_emoji = Position::new(0, 7); + let offset = doc + .position_to_offset(pos_after_emoji, PositionEncoding::Utf16) + .expect("Should get offset"); + + // The UTF-8 byte offset should be at the "!" character + assert_eq!(doc.content().chars().nth(7).unwrap(), '!'); + assert_eq!(&doc.content()[offset as usize..offset as usize + 1], "!"); + + // Test range extraction with non-ASCII characters + let range = Range::new(Position::new(0, 0), Position::new(0, 7)); + let text = doc + .get_text_range(range, PositionEncoding::Utf16) + .expect("Should get text range"); + assert_eq!(text, "Hello 🌍"); + + // Test position on second line with CJK character + // "Second 行 line" - 行 is 3 UTF-8 bytes but 1 UTF-16 code unit + // Position after the CJK character should be at UTF-16 position 8 + let pos_after_cjk = Position::new(1, 8); + let offset_cjk = doc + .position_to_offset(pos_after_cjk, PositionEncoding::Utf16) + .expect("Should get offset"); + + // Find the start of line 2 in UTF-8 bytes + let line2_start = doc.content().find('\n').unwrap() + 1; + let line2_offset = offset_cjk as usize - line2_start; + let line2 = &doc.content()[line2_start..]; + assert_eq!(&line2[line2_offset..line2_offset + 1], " "); + } + + #[test] + fn test_template_tag_context_with_utf16() { + // Test template with non-ASCII characters before template tag + let content = "Título 🌍: {% for"; + let doc = TextDocument::new(content.to_string(), 1, LanguageId::HtmlDjango); + + // Position after "for" - UTF-16 position 17 (after 'r') + let pos = Position::new(0, 17); + let context = doc + .get_template_tag_context(pos, PositionEncoding::Utf16) + .expect("Should get template context"); + + assert_eq!(context.partial_tag, "for"); + assert!(!context.needs_leading_space); + } + + #[test] + fn test_get_text_range_with_emoji() { + let content = "Hello 🌍 world"; + let doc = TextDocument::new(content.to_string(), 1, LanguageId::HtmlDjango); + + // Range that spans across the emoji + // "Hello 🌍 world" + // H(1) e(1) l(1) l(1) o(1) space(1) 🌍(2) space(1) w(1)... + // From position 5 (space before emoji) to position 8 (space after emoji) + let range = Range::new(Position::new(0, 5), Position::new(0, 8)); + let text = doc + .get_text_range(range, PositionEncoding::Utf16) + .expect("Should get text range"); + assert_eq!(text, " 🌍"); + } + + #[test] + fn test_line_index_utf16_conversion() { + let text = "Hello 🌍!\nWorld 行 test"; + let line_index = LineIndex::new(text); + + // Test position conversion with emoji on first line + let pos_emoji = Position::new(0, 7); // After emoji + let offset = line_index.offset(pos_emoji, text, PositionEncoding::Utf16); + assert_eq!(&text[offset as usize..offset as usize + 1], "!"); + + // Test position conversion with CJK on second line + // "World 行 test" + // W(1) o(1) r(1) l(1) d(1) space(1) 行(1) space(1) t(1)... + // Position after CJK character should be at UTF-16 position 7 + let pos_cjk = Position::new(1, 7); + let offset_cjk = line_index.offset(pos_cjk, text, PositionEncoding::Utf16); + assert_eq!(&text[offset_cjk as usize..offset_cjk as usize + 1], " "); + } +} diff --git a/crates/djls-workspace/src/encoding.rs b/crates/djls-workspace/src/encoding.rs new file mode 100644 index 00000000..203f190f --- /dev/null +++ b/crates/djls-workspace/src/encoding.rs @@ -0,0 +1,148 @@ +use tower_lsp_server::lsp_types::{InitializeParams, PositionEncodingKind}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum PositionEncoding { + Utf8, + #[default] + Utf16, + Utf32, +} + +impl PositionEncoding { + /// Get the LSP string representation of this encoding + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + PositionEncoding::Utf8 => "utf-8", + PositionEncoding::Utf16 => "utf-16", + PositionEncoding::Utf32 => "utf-32", + } + } + + /// Convert from LSP [`PositionEncodingKind`](tower_lsp_server::lsp_types::PositionEncodingKind) + #[must_use] + pub fn from_lsp_kind(kind: &PositionEncodingKind) -> Option { + match kind.as_str() { + "utf-8" => Some(PositionEncoding::Utf8), + "utf-16" => Some(PositionEncoding::Utf16), + "utf-32" => Some(PositionEncoding::Utf32), + _ => None, + } + } + + /// Convert to LSP [`PositionEncodingKind`](tower_lsp_server::lsp_types::PositionEncodingKind) + #[must_use] + pub fn to_lsp_kind(&self) -> PositionEncodingKind { + PositionEncodingKind::new(self.as_str()) + } + + /// Negotiate the best encoding with the client based on their capabilities. + /// Prefers UTF-8 > UTF-32 > UTF-16 for performance reasons. + pub fn negotiate(params: &InitializeParams) -> Self { + let client_encodings = params + .capabilities + .general + .as_ref() + .and_then(|general| general.position_encodings.as_ref()) + .map(|encodings| encodings.as_slice()) + .unwrap_or(&[]); + + // Try to find the best encoding in preference order + for preferred in [ + PositionEncoding::Utf8, + PositionEncoding::Utf32, + PositionEncoding::Utf16, + ] { + if client_encodings + .iter() + .any(|kind| Self::from_lsp_kind(kind) == Some(preferred)) + { + return preferred; + } + } + + // Fallback to UTF-16 if client doesn't specify encodings + PositionEncoding::Utf16 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tower_lsp_server::lsp_types::{ClientCapabilities, GeneralClientCapabilities}; + + #[test] + fn test_encoding_str_conversion() { + assert_eq!(PositionEncoding::Utf8.as_str(), "utf-8"); + assert_eq!(PositionEncoding::Utf16.as_str(), "utf-16"); + assert_eq!(PositionEncoding::Utf32.as_str(), "utf-32"); + } + + #[test] + fn test_from_lsp_kind() { + assert_eq!( + PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("utf-8")), + Some(PositionEncoding::Utf8) + ); + assert_eq!( + PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("utf-16")), + Some(PositionEncoding::Utf16) + ); + assert_eq!( + PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("utf-32")), + Some(PositionEncoding::Utf32) + ); + assert_eq!( + PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("unknown")), + None + ); + } + + #[test] + fn test_negotiate_prefers_utf8() { + let mut params = InitializeParams::default(); + params.capabilities = ClientCapabilities { + general: Some(GeneralClientCapabilities { + position_encodings: Some(vec![ + PositionEncodingKind::new("utf-16"), + PositionEncodingKind::new("utf-8"), + PositionEncodingKind::new("utf-32"), + ]), + ..Default::default() + }), + ..Default::default() + }; + + assert_eq!(PositionEncoding::negotiate(¶ms), PositionEncoding::Utf8); + } + + #[test] + fn test_negotiate_fallback_utf16() { + let params = InitializeParams::default(); + assert_eq!( + PositionEncoding::negotiate(¶ms), + PositionEncoding::Utf16 + ); + } + + #[test] + fn test_negotiate_prefers_utf32_over_utf16() { + let mut params = InitializeParams::default(); + params.capabilities = ClientCapabilities { + general: Some(GeneralClientCapabilities { + position_encodings: Some(vec![ + PositionEncodingKind::new("utf-16"), + PositionEncodingKind::new("utf-32"), + ]), + ..Default::default() + }), + ..Default::default() + }; + + assert_eq!( + PositionEncoding::negotiate(¶ms), + PositionEncoding::Utf32 + ); + } +} + diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index b8b80e5c..9b4f251a 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -15,6 +15,7 @@ mod buffers; pub mod db; mod document; +pub mod encoding; mod fs; mod language; pub mod paths; @@ -25,6 +26,7 @@ use std::path::Path; pub use buffers::Buffers; pub use db::Database; pub use document::TextDocument; +pub use encoding::PositionEncoding; pub use fs::FileSystem; pub use fs::OsFileSystem; pub use fs::WorkspaceFileSystem; From fb1f27ed4a11149a64f53980a997a15e5c33c5c0 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 12:33:09 -0500 Subject: [PATCH 32/56] wipp --- crates/djls-server/src/server.rs | 4 +- crates/djls-workspace/src/document.rs | 41 +++--- crates/djls-workspace/src/encoding.rs | 175 +++++++++++++++++--------- 3 files changed, 136 insertions(+), 84 deletions(-) diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index 8d1b9f24..b6f3c74f 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -116,14 +116,14 @@ impl LanguageServer for DjangoLanguageServer { save: Some(lsp_types::SaveOptions::default().into()), }, )), - position_encoding: Some(encoding.to_lsp_kind()), + position_encoding: Some(lsp_types::PositionEncodingKind::from(encoding)), ..Default::default() }, server_info: Some(lsp_types::ServerInfo { name: SERVER_NAME.to_string(), version: Some(SERVER_VERSION.to_string()), }), - offset_encoding: Some(encoding.as_str().to_string()), + offset_encoding: Some(encoding.to_string()), }) } diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 1f6fd250..df3da2ef 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -92,7 +92,6 @@ impl TextDocument { // For now, we'll just handle full document updates // TODO: Handle incremental updates for change in changes { - // TextDocumentContentChangeEvent has a `text` field that's a String, not Option self.content = change.text; self.line_index = LineIndex::new(&self.content); } @@ -226,7 +225,9 @@ impl LineIndex { // ASCII fast path optimization if matches!(self.kind, IndexKind::Ascii) { // For ASCII text, all encodings are equivalent to byte offsets - let char_offset = position.character.min(line_text.len() as u32); + let char_offset = position + .character + .min(u32::try_from(line_text.len()).unwrap_or(u32::MAX)); return line_start_utf8 + char_offset; } @@ -234,7 +235,9 @@ impl LineIndex { match encoding { PositionEncoding::Utf8 => { // UTF-8: character positions are already byte offsets - let char_offset = position.character.min(line_text.len() as u32); + let char_offset = position + .character + .min(u32::try_from(line_text.len()).unwrap_or(u32::MAX)); line_start_utf8 + char_offset } PositionEncoding::Utf16 => { @@ -246,28 +249,22 @@ impl LineIndex { if utf16_pos >= position.character { break; } - utf16_pos += c.len_utf16() as u32; - utf8_pos += c.len_utf8() as u32; + utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); } // If character position exceeds line length, clamp to line end - if utf16_pos < position.character && utf8_pos == line_text.len() as u32 { - line_start_utf8 + utf8_pos - } else { - line_start_utf8 + utf8_pos - } + line_start_utf8 + utf8_pos } PositionEncoding::Utf32 => { // UTF-32: count Unicode code points (characters) - let mut char_count = 0; let mut utf8_pos = 0; - for c in line_text.chars() { - if char_count >= position.character { + for (char_count, c) in line_text.chars().enumerate() { + if char_count >= position.character as usize { break; } - char_count += 1; - utf8_pos += c.len_utf8() as u32; + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); } // If character position exceeds line length, clamp to line end @@ -321,7 +318,7 @@ mod tests { // The UTF-8 byte offset should be at the "!" character assert_eq!(doc.content().chars().nth(7).unwrap(), '!'); - assert_eq!(&doc.content()[offset as usize..offset as usize + 1], "!"); + assert_eq!(&doc.content()[(offset as usize)..=(offset as usize)], "!"); // Test range extraction with non-ASCII characters let range = Range::new(Position::new(0, 0), Position::new(0, 7)); @@ -342,7 +339,7 @@ mod tests { let line2_start = doc.content().find('\n').unwrap() + 1; let line2_offset = offset_cjk as usize - line2_start; let line2 = &doc.content()[line2_start..]; - assert_eq!(&line2[line2_offset..line2_offset + 1], " "); + assert_eq!(&line2[line2_offset..=line2_offset], " "); } #[test] @@ -353,12 +350,12 @@ mod tests { // Position after "for" - UTF-16 position 17 (after 'r') let pos = Position::new(0, 17); - let context = doc + let tag_context = doc .get_template_tag_context(pos, PositionEncoding::Utf16) .expect("Should get template context"); - assert_eq!(context.partial_tag, "for"); - assert!(!context.needs_leading_space); + assert_eq!(tag_context.partial_tag, "for"); + assert!(!tag_context.needs_leading_space); } #[test] @@ -385,7 +382,7 @@ mod tests { // Test position conversion with emoji on first line let pos_emoji = Position::new(0, 7); // After emoji let offset = line_index.offset(pos_emoji, text, PositionEncoding::Utf16); - assert_eq!(&text[offset as usize..offset as usize + 1], "!"); + assert_eq!(&text[(offset as usize)..=(offset as usize)], "!"); // Test position conversion with CJK on second line // "World 行 test" @@ -393,6 +390,6 @@ mod tests { // Position after CJK character should be at UTF-16 position 7 let pos_cjk = Position::new(1, 7); let offset_cjk = line_index.offset(pos_cjk, text, PositionEncoding::Utf16); - assert_eq!(&text[offset_cjk as usize..offset_cjk as usize + 1], " "); + assert_eq!(&text[(offset_cjk as usize)..=(offset_cjk as usize)], " "); } } diff --git a/crates/djls-workspace/src/encoding.rs b/crates/djls-workspace/src/encoding.rs index 203f190f..7de8f18a 100644 --- a/crates/djls-workspace/src/encoding.rs +++ b/crates/djls-workspace/src/encoding.rs @@ -1,3 +1,5 @@ +use std::fmt; +use std::str::FromStr; use tower_lsp_server::lsp_types::{InitializeParams, PositionEncodingKind}; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] @@ -9,43 +11,15 @@ pub enum PositionEncoding { } impl PositionEncoding { - /// Get the LSP string representation of this encoding - #[must_use] - pub fn as_str(&self) -> &'static str { - match self { - PositionEncoding::Utf8 => "utf-8", - PositionEncoding::Utf16 => "utf-16", - PositionEncoding::Utf32 => "utf-32", - } - } - - /// Convert from LSP [`PositionEncodingKind`](tower_lsp_server::lsp_types::PositionEncodingKind) - #[must_use] - pub fn from_lsp_kind(kind: &PositionEncodingKind) -> Option { - match kind.as_str() { - "utf-8" => Some(PositionEncoding::Utf8), - "utf-16" => Some(PositionEncoding::Utf16), - "utf-32" => Some(PositionEncoding::Utf32), - _ => None, - } - } - - /// Convert to LSP [`PositionEncodingKind`](tower_lsp_server::lsp_types::PositionEncodingKind) - #[must_use] - pub fn to_lsp_kind(&self) -> PositionEncodingKind { - PositionEncodingKind::new(self.as_str()) - } - /// Negotiate the best encoding with the client based on their capabilities. /// Prefers UTF-8 > UTF-32 > UTF-16 for performance reasons. pub fn negotiate(params: &InitializeParams) -> Self { - let client_encodings = params + let client_encodings: &[PositionEncodingKind] = params .capabilities .general .as_ref() .and_then(|general| general.position_encodings.as_ref()) - .map(|encodings| encodings.as_slice()) - .unwrap_or(&[]); + .map_or(&[], |encodings| encodings.as_slice()); // Try to find the best encoding in preference order for preferred in [ @@ -55,7 +29,7 @@ impl PositionEncoding { ] { if client_encodings .iter() - .any(|kind| Self::from_lsp_kind(kind) == Some(preferred)) + .any(|kind| PositionEncoding::try_from(kind.clone()).ok() == Some(preferred)) { return preferred; } @@ -66,6 +40,48 @@ impl PositionEncoding { } } +impl FromStr for PositionEncoding { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "utf-8" => Ok(PositionEncoding::Utf8), + "utf-16" => Ok(PositionEncoding::Utf16), + "utf-32" => Ok(PositionEncoding::Utf32), + _ => Err(()), + } + } +} + +impl fmt::Display for PositionEncoding { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + PositionEncoding::Utf8 => "utf-8", + PositionEncoding::Utf16 => "utf-16", + PositionEncoding::Utf32 => "utf-32", + }; + write!(f, "{s}") + } +} + +impl From for PositionEncodingKind { + fn from(encoding: PositionEncoding) -> Self { + match encoding { + PositionEncoding::Utf8 => PositionEncodingKind::new("utf-8"), + PositionEncoding::Utf16 => PositionEncodingKind::new("utf-16"), + PositionEncoding::Utf32 => PositionEncodingKind::new("utf-32"), + } + } +} + +impl TryFrom for PositionEncoding { + type Error = (); + + fn try_from(kind: PositionEncodingKind) -> Result { + kind.as_str().parse() + } +} + #[cfg(test)] mod tests { use super::*; @@ -73,43 +89,81 @@ mod tests { #[test] fn test_encoding_str_conversion() { - assert_eq!(PositionEncoding::Utf8.as_str(), "utf-8"); - assert_eq!(PositionEncoding::Utf16.as_str(), "utf-16"); - assert_eq!(PositionEncoding::Utf32.as_str(), "utf-32"); + // Test FromStr trait + assert_eq!("utf-8".parse(), Ok(PositionEncoding::Utf8)); + assert_eq!("utf-16".parse(), Ok(PositionEncoding::Utf16)); + assert_eq!("utf-32".parse(), Ok(PositionEncoding::Utf32)); + assert!("invalid".parse::().is_err()); + + // Test ToString trait + assert_eq!(PositionEncoding::Utf8.to_string(), "utf-8"); + assert_eq!(PositionEncoding::Utf16.to_string(), "utf-16"); + assert_eq!(PositionEncoding::Utf32.to_string(), "utf-32"); } #[test] fn test_from_lsp_kind() { assert_eq!( - PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("utf-8")), - Some(PositionEncoding::Utf8) + PositionEncoding::try_from(PositionEncodingKind::new("utf-8")), + Ok(PositionEncoding::Utf8) ); assert_eq!( - PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("utf-16")), - Some(PositionEncoding::Utf16) + PositionEncoding::try_from(PositionEncodingKind::new("utf-16")), + Ok(PositionEncoding::Utf16) ); assert_eq!( - PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("utf-32")), - Some(PositionEncoding::Utf32) + PositionEncoding::try_from(PositionEncodingKind::new("utf-32")), + Ok(PositionEncoding::Utf32) ); + assert!(PositionEncoding::try_from(PositionEncodingKind::new("unknown")).is_err()); + } + + #[test] + fn test_trait_conversions() { + // Test TryFrom for PositionEncoding assert_eq!( - PositionEncoding::from_lsp_kind(&PositionEncodingKind::new("unknown")), - None + PositionEncoding::try_from(PositionEncodingKind::new("utf-8")), + Ok(PositionEncoding::Utf8) + ); + assert_eq!( + PositionEncoding::try_from(PositionEncodingKind::new("utf-16")), + Ok(PositionEncoding::Utf16) + ); + assert_eq!( + PositionEncoding::try_from(PositionEncodingKind::new("utf-32")), + Ok(PositionEncoding::Utf32) + ); + assert!(PositionEncoding::try_from(PositionEncodingKind::new("unknown")).is_err()); + + // Test From for PositionEncodingKind + assert_eq!( + PositionEncodingKind::from(PositionEncoding::Utf8).as_str(), + "utf-8" + ); + assert_eq!( + PositionEncodingKind::from(PositionEncoding::Utf16).as_str(), + "utf-16" + ); + assert_eq!( + PositionEncodingKind::from(PositionEncoding::Utf32).as_str(), + "utf-32" ); } #[test] fn test_negotiate_prefers_utf8() { - let mut params = InitializeParams::default(); - params.capabilities = ClientCapabilities { - general: Some(GeneralClientCapabilities { - position_encodings: Some(vec![ - PositionEncodingKind::new("utf-16"), - PositionEncodingKind::new("utf-8"), - PositionEncodingKind::new("utf-32"), - ]), + let params = InitializeParams { + capabilities: ClientCapabilities { + general: Some(GeneralClientCapabilities { + position_encodings: Some(vec![ + PositionEncodingKind::new("utf-16"), + PositionEncodingKind::new("utf-8"), + PositionEncodingKind::new("utf-32"), + ]), + ..Default::default() + }), ..Default::default() - }), + }, ..Default::default() }; @@ -127,15 +181,17 @@ mod tests { #[test] fn test_negotiate_prefers_utf32_over_utf16() { - let mut params = InitializeParams::default(); - params.capabilities = ClientCapabilities { - general: Some(GeneralClientCapabilities { - position_encodings: Some(vec![ - PositionEncodingKind::new("utf-16"), - PositionEncodingKind::new("utf-32"), - ]), + let params = InitializeParams { + capabilities: ClientCapabilities { + general: Some(GeneralClientCapabilities { + position_encodings: Some(vec![ + PositionEncodingKind::new("utf-16"), + PositionEncodingKind::new("utf-32"), + ]), + ..Default::default() + }), ..Default::default() - }), + }, ..Default::default() }; @@ -145,4 +201,3 @@ mod tests { ); } } - From e9e1d77584e068b52ebcf8eab2b48ac2873ea1ee Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 13:48:00 -0500 Subject: [PATCH 33/56] keep on worrrrking --- crates/djls-server/src/server.rs | 20 +- crates/djls-server/src/session.rs | 107 ++++++---- crates/djls-server/tests/lsp_integration.rs | 216 ++++++++++++++++++++ crates/djls-workspace/src/document.rs | 180 +++++++++++++++- 4 files changed, 465 insertions(+), 58 deletions(-) diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index b6f3c74f..5638ce59 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -229,24 +229,8 @@ impl LanguageServer for DjangoLanguageServer { self.with_session_mut(|session| { let url = Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); - let new_version = params.text_document.version; - let changes = params.content_changes; - - match session.apply_document_changes(&url, changes.clone(), new_version) { - Ok(()) => {} - Err(err) => { - tracing::warn!("{}", err); - // Recovery: handle full content changes only - if let Some(change) = changes.into_iter().next() { - let document = djls_workspace::TextDocument::new( - change.text, - new_version, - djls_workspace::LanguageId::Other, - ); - session.update_document(&url, document); - } - } - } + + session.update_document(&url, params.content_changes, params.text_document.version); }) .await; } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 722b2e04..54b8ea72 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -40,6 +40,7 @@ use std::sync::Arc; use dashmap::DashMap; use djls_conf::Settings; use djls_project::DjangoProject; +use djls_workspace::db::source_text; use djls_workspace::db::Database; use djls_workspace::db::SourceFile; use djls_workspace::paths; @@ -344,42 +345,67 @@ impl Session { } } - /// Handle document changes - updates buffer and bumps revision. + /// Update a document with the given changes. /// - /// This method coordinates both layers: + /// This method handles both incremental updates and full document replacements, + /// coordinating both layers of the architecture: /// - Layer 1: Updates the document content in buffers /// - Layer 2: Bumps the file revision to trigger Salsa invalidation - pub fn update_document(&mut self, url: &Url, document: TextDocument) { - let version = document.version(); - tracing::debug!("Updating document: {} (version {})", url, version); - - // Layer 1: Update buffer - self.buffers.update(url.clone(), document); - - // Layer 2: Touch file to trigger invalidation - if let Some(path) = paths::url_to_path(url) { - self.with_db_mut(|db| db.touch_file(&path)); - } - } - - /// Apply incremental changes to an open document. - /// - /// This encapsulates the full update cycle: retrieving the document, - /// applying changes, updating the buffer, and bumping Salsa revision. /// - /// Returns an error if the document is not currently open. - pub fn apply_document_changes( + /// If the document is not currently open (no buffer exists), this method will + /// attempt a fallback recovery by using the first content change as a full + /// document replacement, preserving the existing language_id if possible. + pub fn update_document( &mut self, url: &Url, changes: Vec, new_version: i32, - ) -> Result<(), String> { + ) { + // Try to apply changes to existing document if let Some(mut document) = self.buffers.get(url) { + // Document exists - apply incremental changes document.update(changes, new_version); - self.update_document(url, document); - Ok(()) + + let version = document.version(); + tracing::debug!("Updating document: {} (version {})", url, version); + + // Layer 1: Update buffer + self.buffers.update(url.clone(), document); + + // Layer 2: Touch file to trigger invalidation + if let Some(path) = paths::url_to_path(url) { + self.with_db_mut(|db| db.touch_file(&path)); + } } else { - Err(format!("Document not open: {url}")) + // Document not open - attempt fallback recovery + tracing::warn!("Document not open: {}, attempting fallback recovery", url); + + // Use first change as full content replacement + if let Some(change) = changes.into_iter().next() { + // Preserve existing language_id if document was previously opened + // This handles the case where we get changes for a document that + // somehow lost its buffer but we want to maintain its type + let language_id = self + .get_document(url) + .map_or(djls_workspace::LanguageId::Other, |doc| doc.language_id()); + + let document = + djls_workspace::TextDocument::new(change.text, new_version, language_id); + + tracing::debug!( + "Fallback: creating document {} (version {})", + url, + new_version + ); + + // Layer 1: Update buffer + self.buffers.update(url.clone(), document); + + // Layer 2: Touch file to trigger invalidation + if let Some(path) = paths::url_to_path(url) { + self.with_db_mut(|db| db.touch_file(&path)); + } + } } } @@ -393,9 +419,9 @@ impl Session { pub fn close_document(&mut self, url: &Url) -> Option { tracing::debug!("Closing document: {}", url); - // Layer 1: Remove buffer - let removed = self.buffers.close(url); - if let Some(ref doc) = removed { + // Layer 1: Remove buffer (falls back to disk) + let document = self.buffers.close(url); + if let Some(ref doc) = document { tracing::debug!( "Removed buffer for closed document: {} (was version {})", url, @@ -404,12 +430,20 @@ impl Session { } // Layer 2: Touch file to trigger re-read from disk - // We keep the file alive for potential re-opening if let Some(path) = paths::url_to_path(url) { self.with_db_mut(|db| db.touch_file(&path)); } - removed + document + } + + /// Get an open document from the buffer layer, if it exists. + /// + /// This provides read-only access to Layer 1 (buffer) documents. + /// Returns None if the document is not currently open in the editor. + #[must_use] + pub fn get_document(&self, url: &Url) -> Option { + self.buffers.get(url) } /// Get the current content of a file (from overlay or disk). @@ -418,8 +452,6 @@ impl Session { /// The file is created if it doesn't exist, and content is read /// through the `FileSystem` abstraction (overlay first, then disk). pub fn file_content(&mut self, path: PathBuf) -> String { - use djls_workspace::db::source_text; - self.with_db_mut(|db| { let file = db.get_or_create_file(path); source_text(db, file).to_string() @@ -493,10 +525,13 @@ mod tests { let content1 = session.file_content(path.clone()); assert_eq!(content1, "

Original Content

"); - // Update document with new content - let updated_document = - TextDocument::new("

Updated Content

".to_string(), 2, LanguageId::Other); - session.update_document(&url, updated_document); + // Update document with new content using a full replacement change + let changes = vec![lsp_types::TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: "

Updated Content

".to_string(), + }]; + session.update_document(&url, changes, 2); // Read content again (should get new overlay content due to invalidation) let content2 = session.file_content(path.clone()); diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index d277275b..ccb89be9 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -17,6 +17,8 @@ use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; use tower_lsp_server::lsp_types::InitializeParams; use tower_lsp_server::lsp_types::InitializedParams; +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use tower_lsp_server::lsp_types::TextDocumentIdentifier; use tower_lsp_server::lsp_types::TextDocumentItem; @@ -118,6 +120,24 @@ impl TestServer { self.server.did_change(params).await; } + /// Send incremental changes to a document + async fn change_document_incremental( + &self, + file_name: &str, + changes: Vec, + version: i32, + ) { + let params = DidChangeTextDocumentParams { + text_document: VersionedTextDocumentIdentifier { + uri: self.workspace_url(file_name).to_string().parse().unwrap(), + version, + }, + content_changes: changes, + }; + + self.server.did_change(params).await; + } + /// Close a document in the LSP server async fn close_document(&self, file_name: &str) { let params = DidCloseTextDocumentParams { @@ -265,6 +285,94 @@ async fn test_template_parsing_with_overlays() { assert!(!ast_str.contains("if") && !ast_str.contains("If")); } +#[tokio::test] +async fn test_incremental_sync() { + let server = TestServer::new().await; + let file_name = "test.html"; + + // Open document with initial content + server + .open_document(file_name, "Hello world", 1) + .await; + + // Apply incremental change to replace "world" with "Rust" + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new( + Position::new(0, 6), + Position::new(0, 11), + )), + range_length: None, + text: "Rust".to_string(), + }]; + + server + .change_document_incremental(file_name, changes, 2) + .await; + + // Verify the incremental change was applied correctly + let content = server.get_file_content(file_name).await; + assert_eq!(content, "Hello Rust"); + + // Apply multiple incremental changes + let changes = vec![ + // Insert " programming" after "Rust" + TextDocumentContentChangeEvent { + range: Some(Range::new( + Position::new(0, 10), + Position::new(0, 10), + )), + range_length: None, + text: " programming".to_string(), + }, + // Replace "Hello" with "Learning" + TextDocumentContentChangeEvent { + range: Some(Range::new( + Position::new(0, 0), + Position::new(0, 5), + )), + range_length: None, + text: "Learning".to_string(), + }, + ]; + + server + .change_document_incremental(file_name, changes, 3) + .await; + + // Verify multiple changes were applied in order + let content = server.get_file_content(file_name).await; + assert_eq!(content, "Learning Rust programming"); +} + +#[tokio::test] +async fn test_incremental_sync_with_newlines() { + let server = TestServer::new().await; + let file_name = "multiline.html"; + + // Open document with multiline content + server + .open_document(file_name, "Line 1\nLine 2\nLine 3", 1) + .await; + + // Replace text spanning multiple lines + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new( + Position::new(0, 5), // After "Line " on first line + Position::new(2, 4), // Before " 3" on third line + )), + range_length: None, + text: "A\nB\nC".to_string(), + }]; + + server + .change_document_incremental(file_name, changes, 2) + .await; + + // Verify the change was applied correctly across lines + let content = server.get_file_content(file_name).await; + assert_eq!(content, "Line A\nB\nC 3"); +} + #[tokio::test] async fn test_multiple_documents_independent() { let server = TestServer::new().await; @@ -446,3 +554,111 @@ async fn test_revision_tracking_across_lifecycle() { server.change_document(file_name, "Final", 11).await; assert_eq!(server.get_file_revision(file_name).await, Some(7)); } + +#[tokio::test] +async fn test_language_id_preservation_during_fallback() { + let server = TestServer::new().await; + let file_name = "template.html"; + + // Open document with htmldjango language_id + let url = server.workspace_url(file_name); + let document = TextDocumentItem { + uri: url.to_string().parse().unwrap(), + language_id: "htmldjango".to_string(), + version: 1, + text: "{% block content %}Initial{% endblock %}".to_string(), + }; + + let params = DidOpenTextDocumentParams { text_document: document }; + server.server.did_open(params).await; + + // Verify the document was opened with the correct language_id + let document = server + .server + .with_session_mut(|session| session.get_document(&url)) + .await; + match document.unwrap().language_id() { + djls_workspace::LanguageId::HtmlDjango => {}, // Expected + _ => panic!("Expected HtmlDjango language_id"), + } + + // Simulate a scenario that would trigger the fallback path by sending + // a change with an invalid range that would cause apply_document_changes to fail + let params = DidChangeTextDocumentParams { + text_document: VersionedTextDocumentIdentifier { + uri: url.to_string().parse().unwrap(), + version: 2, + }, + content_changes: vec![TextDocumentContentChangeEvent { + range: Some(Range { + start: Position { line: 100, character: 0 }, // Invalid position + end: Position { line: 100, character: 0 }, + }), + range_length: None, + text: "Fallback content".to_string(), + }], + }; + + server.server.did_change(params).await; + + // Verify the document still has the correct language_id after fallback + let document = server + .server + .with_session_mut(|session| session.get_document(&url)) + .await; + match document.unwrap().language_id() { + djls_workspace::LanguageId::HtmlDjango => {}, // Expected + _ => panic!("Expected HtmlDjango language_id after fallback"), + } + + // Also test with a Python file + let py_file_name = "views.py"; + let py_url = server.workspace_url(py_file_name); + let document = TextDocumentItem { + uri: py_url.to_string().parse().unwrap(), + language_id: "python".to_string(), + version: 1, + text: "def hello():\n return 'world'".to_string(), + }; + + let params = DidOpenTextDocumentParams { text_document: document }; + server.server.did_open(params).await; + + // Verify the Python document was opened with the correct language_id + let document = server + .server + .with_session_mut(|session| session.get_document(&py_url)) + .await; + match document.unwrap().language_id() { + djls_workspace::LanguageId::Python => {}, // Expected + _ => panic!("Expected Python language_id"), + } + + // Trigger fallback for Python file as well + let params = DidChangeTextDocumentParams { + text_document: VersionedTextDocumentIdentifier { + uri: py_url.to_string().parse().unwrap(), + version: 2, + }, + content_changes: vec![TextDocumentContentChangeEvent { + range: Some(Range { + start: Position { line: 100, character: 0 }, // Invalid position + end: Position { line: 100, character: 0 }, + }), + range_length: None, + text: "def fallback():\n pass".to_string(), + }], + }; + + server.server.did_change(params).await; + + // Verify the Python document still has the correct language_id after fallback + let document = server + .server + .with_session_mut(|session| session.get_document(&py_url)) + .await; + match document.unwrap().language_id() { + djls_workspace::LanguageId::Python => {}, // Expected + _ => panic!("Expected Python language_id after fallback"), + } +} diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index df3da2ef..dbb677c6 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -84,17 +84,58 @@ impl TextDocument { } /// Update the document content with LSP text changes + /// + /// Supports both full document replacement and incremental updates. + /// Following ruff's approach: incremental sync is used for network efficiency, + /// but we rebuild the full document text internally. pub fn update( &mut self, changes: Vec, version: i32, ) { - // For now, we'll just handle full document updates - // TODO: Handle incremental updates - for change in changes { - self.content = change.text; + // Fast path: single change without range = full document replacement + if changes.len() == 1 && changes[0].range.is_none() { + self.content = changes[0].text.clone(); self.line_index = LineIndex::new(&self.content); + self.version = version; + return; + } + + // Incremental path: apply changes to rebuild the document + // Clone current content and apply each change + let mut new_content = self.content.clone(); + + for change in changes { + if let Some(range) = change.range { + // Convert LSP range to byte offsets + // Note: We use UTF-16 encoding by default for LSP compatibility + // This will need to use the negotiated encoding in the future + let start_offset = self.line_index.offset( + range.start, + &new_content, + PositionEncoding::Utf16, + ) as usize; + let end_offset = self.line_index.offset( + range.end, + &new_content, + PositionEncoding::Utf16, + ) as usize; + + // Apply the change by replacing the range + new_content.replace_range(start_offset..end_offset, &change.text); + + // Rebuild line index after each change since positions shift + // This is necessary for subsequent changes to have correct offsets + self.line_index = LineIndex::new(&new_content); + } else { + // No range means full replacement + new_content = change.text; + self.line_index = LineIndex::new(&new_content); + } } + + // Store the rebuilt document + self.content = new_content; self.version = version; } @@ -301,6 +342,137 @@ pub enum IndexKind { mod tests { use super::*; use crate::language::LanguageId; + use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; + + #[test] + fn test_incremental_update_single_change() { + let mut doc = TextDocument::new("Hello world".to_string(), 1, LanguageId::Other); + + // Replace "world" with "Rust" + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 6), Position::new(0, 11))), + range_length: None, + text: "Rust".to_string(), + }]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "Hello Rust"); + assert_eq!(doc.version(), 2); + } + + #[test] + fn test_incremental_update_multiple_changes() { + let mut doc = TextDocument::new("First line\nSecond line\nThird line".to_string(), 1, LanguageId::Other); + + // Multiple changes: replace "First" with "1st" and "Third" with "3rd" + let changes = vec![ + TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 0), Position::new(0, 5))), + range_length: None, + text: "1st".to_string(), + }, + TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(2, 0), Position::new(2, 5))), + range_length: None, + text: "3rd".to_string(), + }, + ]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "1st line\nSecond line\n3rd line"); + } + + #[test] + fn test_incremental_update_insertion() { + let mut doc = TextDocument::new("Hello world".to_string(), 1, LanguageId::Other); + + // Insert text at position (empty range) + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 5), Position::new(0, 5))), + range_length: None, + text: " beautiful".to_string(), + }]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "Hello beautiful world"); + } + + #[test] + fn test_incremental_update_deletion() { + let mut doc = TextDocument::new("Hello beautiful world".to_string(), 1, LanguageId::Other); + + // Delete "beautiful " (replace with empty string) + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 6), Position::new(0, 16))), + range_length: None, + text: String::new(), + }]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "Hello world"); + } + + #[test] + fn test_full_document_replacement() { + let mut doc = TextDocument::new("Old content".to_string(), 1, LanguageId::Other); + + // Full document replacement (no range) + let changes = vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: "Completely new content".to_string(), + }]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "Completely new content"); + assert_eq!(doc.version(), 2); + } + + #[test] + fn test_incremental_update_multiline() { + let mut doc = TextDocument::new("Line 1\nLine 2\nLine 3".to_string(), 1, LanguageId::Other); + + // Replace across multiple lines + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 5), Position::new(2, 4))), + range_length: None, + text: "A\nB\nC".to_string(), + }]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "Line A\nB\nC 3"); + } + + #[test] + fn test_incremental_update_with_emoji() { + let mut doc = TextDocument::new("Hello 🌍 world".to_string(), 1, LanguageId::Other); + + // Replace "world" after emoji - must handle UTF-16 positions correctly + // "Hello " = 6 UTF-16 units, "🌍" = 2 UTF-16 units, " " = 1 unit, "world" starts at 9 + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 9), Position::new(0, 14))), + range_length: None, + text: "Rust".to_string(), + }]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "Hello 🌍 Rust"); + } + + #[test] + fn test_incremental_update_newline_at_end() { + let mut doc = TextDocument::new("Hello".to_string(), 1, LanguageId::Other); + + // Add newline and new line at end + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 5), Position::new(0, 5))), + range_length: None, + text: "\nWorld".to_string(), + }]; + + doc.update(changes, 2); + assert_eq!(doc.content(), "Hello\nWorld"); + } #[test] fn test_utf16_position_handling() { From 6d6b1b29a9a28a3d479972b365d8caea2f92d1c4 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 15:07:34 -0500 Subject: [PATCH 34/56] wipwip --- crates/djls-project/src/python.rs | 4 +- crates/djls-project/src/system.rs | 4 +- crates/djls-server/src/session.rs | 35 +++-- crates/djls-server/tests/lsp_integration.rs | 161 ++++++++++++++++---- crates/djls-workspace/src/db.rs | 26 ++-- crates/djls-workspace/src/document.rs | 28 ++-- crates/djls-workspace/src/fs.rs | 91 ++++++++--- crates/djls-workspace/src/paths.rs | 63 ++++++-- 8 files changed, 304 insertions(+), 108 deletions(-) diff --git a/crates/djls-project/src/python.rs b/crates/djls-project/src/python.rs index 69358453..c3678d37 100644 --- a/crates/djls-project/src/python.rs +++ b/crates/djls-project/src/python.rs @@ -208,9 +208,7 @@ mod tests { use super::*; use crate::system::mock::MockGuard; - use crate::system::mock::{ - self as sys_mock, - }; + use crate::system::mock::{self as sys_mock}; #[test] fn test_explicit_venv_path_found() { diff --git a/crates/djls-project/src/system.rs b/crates/djls-project/src/system.rs index 167e51c6..88269e38 100644 --- a/crates/djls-project/src/system.rs +++ b/crates/djls-project/src/system.rs @@ -103,9 +103,7 @@ mod tests { use which::Error as WhichError; use super::mock::MockGuard; - use super::mock::{ - self as sys_mock, - }; + use super::mock::{self as sys_mock}; use super::*; #[test] diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 54b8ea72..126bae64 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -174,17 +174,18 @@ impl Session { } /// Determines the project root path from initialization parameters. /// - /// Tries the current directory first, then falls back to the first workspace folder. + /// Tries workspace folders first (using the first one), then falls back to current directory. fn get_project_path(params: &lsp_types::InitializeParams) -> Option { - // Try current directory first - std::env::current_dir().ok().or_else(|| { - // Fall back to the first workspace folder URI - params - .workspace_folders - .as_ref() - .and_then(|folders| folders.first()) - .and_then(|folder| paths::lsp_uri_to_path(&folder.uri)) - }) + // Try workspace folders first + params + .workspace_folders + .as_ref() + .and_then(|folders| folders.first()) + .and_then(|folder| paths::lsp_uri_to_path(&folder.uri)) + .or_else(|| { + // Fall back to current directory + std::env::current_dir().ok() + }) } #[must_use] @@ -231,6 +232,20 @@ impl Session { /// This is an internal method that should only be called by /// [`with_db_mut`](Session::with_db_mut). Multiple concurrent calls would panic when trying /// to take an already-taken handle. + /// + /// ## Safety Note on Placeholder Handle + /// + /// This method uses `StorageHandle::new(None)` as a temporary placeholder, which + /// creates a new Salsa instance without event callbacks. While this could theoretically + /// lose state if called concurrently, the outer `Arc>>` at the + /// server level ensures this method is only called with exclusive access to the Session. + /// + /// The placeholder is immediately replaced when `restore_db_handle()` is called at the + /// end of the mutation operation, so no actual state is lost. + /// + /// A future improvement (see TODO above) would be to implement a `StorageHandleGuard` + /// abstraction that makes these state transitions more explicit and type-safe. See + /// task-152 for the planned implementation. fn take_db_handle_for_mutation(&mut self) -> StorageHandle { std::mem::replace(&mut self.db_handle, StorageHandle::new(None)) } diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index ccb89be9..a30ec1ec 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -291,16 +291,11 @@ async fn test_incremental_sync() { let file_name = "test.html"; // Open document with initial content - server - .open_document(file_name, "Hello world", 1) - .await; + server.open_document(file_name, "Hello world", 1).await; // Apply incremental change to replace "world" with "Rust" let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new( - Position::new(0, 6), - Position::new(0, 11), - )), + range: Some(Range::new(Position::new(0, 6), Position::new(0, 11))), range_length: None, text: "Rust".to_string(), }]; @@ -317,19 +312,13 @@ async fn test_incremental_sync() { let changes = vec![ // Insert " programming" after "Rust" TextDocumentContentChangeEvent { - range: Some(Range::new( - Position::new(0, 10), - Position::new(0, 10), - )), + range: Some(Range::new(Position::new(0, 10), Position::new(0, 10))), range_length: None, text: " programming".to_string(), }, // Replace "Hello" with "Learning" TextDocumentContentChangeEvent { - range: Some(Range::new( - Position::new(0, 0), - Position::new(0, 5), - )), + range: Some(Range::new(Position::new(0, 0), Position::new(0, 5))), range_length: None, text: "Learning".to_string(), }, @@ -357,8 +346,8 @@ async fn test_incremental_sync_with_newlines() { // Replace text spanning multiple lines let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new( - Position::new(0, 5), // After "Line " on first line - Position::new(2, 4), // Before " 3" on third line + Position::new(0, 5), // After "Line " on first line + Position::new(2, 4), // Before " 3" on third line )), range_length: None, text: "A\nB\nC".to_string(), @@ -555,6 +544,98 @@ async fn test_revision_tracking_across_lifecycle() { assert_eq!(server.get_file_revision(file_name).await, Some(7)); } +#[tokio::test] +async fn test_workspace_folder_priority() { + // Set up logging + let (_non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); + let server = DjangoLanguageServer::new(guard); + + // Test case 1: Workspace folders provided - should use first workspace folder + let workspace_folder1 = WorkspaceFolder { + uri: "file:///workspace/folder1".parse().unwrap(), + name: "workspace1".to_string(), + }; + let workspace_folder2 = WorkspaceFolder { + uri: "file:///workspace/folder2".parse().unwrap(), + name: "workspace2".to_string(), + }; + + let init_params = InitializeParams { + workspace_folders: Some(vec![workspace_folder1.clone(), workspace_folder2.clone()]), + ..Default::default() + }; + + server + .initialize(init_params) + .await + .expect("Failed to initialize"); + server.initialized(InitializedParams {}).await; + + // Check that the session uses the first workspace folder + let project_path = server + .with_session(|session| { + session + .project() + .map(|project| project.path().to_path_buf()) + }) + .await; + + assert_eq!(project_path, Some(PathBuf::from("/workspace/folder1"))); + + // Test case 2: No workspace folders - should fall back to current directory + let (_non_blocking2, guard2) = tracing_appender::non_blocking(std::io::sink()); + let server2 = DjangoLanguageServer::new(guard2); + + let init_params2 = InitializeParams { + workspace_folders: None, + ..Default::default() + }; + + server2 + .initialize(init_params2) + .await + .expect("Failed to initialize"); + server2.initialized(InitializedParams {}).await; + + // Check that the session falls back to current directory + let current_dir = std::env::current_dir().ok(); + let project_path2 = server2 + .with_session(|session| { + session + .project() + .map(|project| project.path().to_path_buf()) + }) + .await; + + assert_eq!(project_path2, current_dir); + + // Test case 3: Empty workspace folders array - should fall back to current directory + let (_non_blocking3, guard3) = tracing_appender::non_blocking(std::io::sink()); + let server3 = DjangoLanguageServer::new(guard3); + + let init_params3 = InitializeParams { + workspace_folders: Some(vec![]), + ..Default::default() + }; + + server3 + .initialize(init_params3) + .await + .expect("Failed to initialize"); + server3.initialized(InitializedParams {}).await; + + // Check that the session falls back to current directory + let project_path3 = server3 + .with_session(|session| { + session + .project() + .map(|project| project.path().to_path_buf()) + }) + .await; + + assert_eq!(project_path3, current_dir); +} + #[tokio::test] async fn test_language_id_preservation_during_fallback() { let server = TestServer::new().await; @@ -568,8 +649,10 @@ async fn test_language_id_preservation_during_fallback() { version: 1, text: "{% block content %}Initial{% endblock %}".to_string(), }; - - let params = DidOpenTextDocumentParams { text_document: document }; + + let params = DidOpenTextDocumentParams { + text_document: document, + }; server.server.did_open(params).await; // Verify the document was opened with the correct language_id @@ -578,7 +661,7 @@ async fn test_language_id_preservation_during_fallback() { .with_session_mut(|session| session.get_document(&url)) .await; match document.unwrap().language_id() { - djls_workspace::LanguageId::HtmlDjango => {}, // Expected + djls_workspace::LanguageId::HtmlDjango => {} // Expected _ => panic!("Expected HtmlDjango language_id"), } @@ -591,14 +674,20 @@ async fn test_language_id_preservation_during_fallback() { }, content_changes: vec![TextDocumentContentChangeEvent { range: Some(Range { - start: Position { line: 100, character: 0 }, // Invalid position - end: Position { line: 100, character: 0 }, + start: Position { + line: 100, + character: 0, + }, // Invalid position + end: Position { + line: 100, + character: 0, + }, }), range_length: None, text: "Fallback content".to_string(), }], }; - + server.server.did_change(params).await; // Verify the document still has the correct language_id after fallback @@ -607,10 +696,10 @@ async fn test_language_id_preservation_during_fallback() { .with_session_mut(|session| session.get_document(&url)) .await; match document.unwrap().language_id() { - djls_workspace::LanguageId::HtmlDjango => {}, // Expected + djls_workspace::LanguageId::HtmlDjango => {} // Expected _ => panic!("Expected HtmlDjango language_id after fallback"), } - + // Also test with a Python file let py_file_name = "views.py"; let py_url = server.workspace_url(py_file_name); @@ -620,8 +709,10 @@ async fn test_language_id_preservation_during_fallback() { version: 1, text: "def hello():\n return 'world'".to_string(), }; - - let params = DidOpenTextDocumentParams { text_document: document }; + + let params = DidOpenTextDocumentParams { + text_document: document, + }; server.server.did_open(params).await; // Verify the Python document was opened with the correct language_id @@ -630,7 +721,7 @@ async fn test_language_id_preservation_during_fallback() { .with_session_mut(|session| session.get_document(&py_url)) .await; match document.unwrap().language_id() { - djls_workspace::LanguageId::Python => {}, // Expected + djls_workspace::LanguageId::Python => {} // Expected _ => panic!("Expected Python language_id"), } @@ -642,14 +733,20 @@ async fn test_language_id_preservation_during_fallback() { }, content_changes: vec![TextDocumentContentChangeEvent { range: Some(Range { - start: Position { line: 100, character: 0 }, // Invalid position - end: Position { line: 100, character: 0 }, + start: Position { + line: 100, + character: 0, + }, // Invalid position + end: Position { + line: 100, + character: 0, + }, }), range_length: None, text: "def fallback():\n pass".to_string(), }], }; - + server.server.did_change(params).await; // Verify the Python document still has the correct language_id after fallback @@ -658,7 +755,7 @@ async fn test_language_id_preservation_during_fallback() { .with_session_mut(|session| session.get_document(&py_url)) .await; match document.unwrap().language_id() { - djls_workspace::LanguageId::Python => {}, // Expected + djls_workspace::LanguageId::Python => {} // Expected _ => panic!("Expected Python language_id after fallback"), } } diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 02908970..cbfdc29d 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -42,7 +42,7 @@ use crate::FileSystem; #[salsa::db] pub trait Db: salsa::Database { /// Get the file system for reading files. - fn fs(&self) -> Option>; + fn fs(&self) -> Arc; /// Read file content through the file system. /// @@ -61,9 +61,8 @@ pub trait Db: salsa::Database { pub struct Database { storage: salsa::Storage, - // TODO: does this need to be an Option? /// File system for reading file content (checks buffers first, then disk). - fs: Option>, + fs: Arc, /// Maps paths to [`SourceFile`] entities for O(1) lookup. files: Arc>, @@ -76,6 +75,8 @@ pub struct Database { #[cfg(test)] impl Default for Database { fn default() -> Self { + use crate::fs::InMemoryFileSystem; + let logs = >>>>::default(); Self { storage: salsa::Storage::new(Some(Box::new({ @@ -91,7 +92,7 @@ impl Default for Database { } } }))), - fs: None, + fs: Arc::new(InMemoryFileSystem::new()), files: Arc::new(DashMap::new()), logs, } @@ -102,7 +103,7 @@ impl Database { pub fn new(file_system: Arc, files: Arc>) -> Self { Self { storage: salsa::Storage::new(None), - fs: Some(file_system), + fs: file_system, files, #[cfg(test)] logs: Arc::new(Mutex::new(None)), @@ -116,7 +117,7 @@ impl Database { ) -> Self { Self { storage, - fs: Some(file_system), + fs: file_system, files, #[cfg(test)] logs: Arc::new(Mutex::new(None)), @@ -125,11 +126,7 @@ impl Database { /// Read file content through the file system. pub fn read_file_content(&self, path: &Path) -> std::io::Result { - if let Some(fs) = &self.fs { - fs.read_to_string(path) - } else { - std::fs::read_to_string(path) - } + self.fs.read_to_string(path) } /// Get or create a [`SourceFile`] for the given path. @@ -210,15 +207,12 @@ impl salsa::Database for Database {} #[salsa::db] impl Db for Database { - fn fs(&self) -> Option> { + fn fs(&self) -> Arc { self.fs.clone() } fn read_file_content(&self, path: &Path) -> std::io::Result { - match &self.fs { - Some(fs) => fs.read_to_string(path), - None => std::fs::read_to_string(path), // Fallback to direct disk access - } + self.fs.read_to_string(path) } } diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index dbb677c6..96ef46f3 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -104,26 +104,24 @@ impl TextDocument { // Incremental path: apply changes to rebuild the document // Clone current content and apply each change let mut new_content = self.content.clone(); - + for change in changes { if let Some(range) = change.range { // Convert LSP range to byte offsets // Note: We use UTF-16 encoding by default for LSP compatibility // This will need to use the negotiated encoding in the future - let start_offset = self.line_index.offset( - range.start, - &new_content, - PositionEncoding::Utf16, - ) as usize; - let end_offset = self.line_index.offset( - range.end, - &new_content, - PositionEncoding::Utf16, - ) as usize; + let start_offset = + self.line_index + .offset(range.start, &new_content, PositionEncoding::Utf16) + as usize; + let end_offset = + self.line_index + .offset(range.end, &new_content, PositionEncoding::Utf16) + as usize; // Apply the change by replacing the range new_content.replace_range(start_offset..end_offset, &change.text); - + // Rebuild line index after each change since positions shift // This is necessary for subsequent changes to have correct offsets self.line_index = LineIndex::new(&new_content); @@ -362,7 +360,11 @@ mod tests { #[test] fn test_incremental_update_multiple_changes() { - let mut doc = TextDocument::new("First line\nSecond line\nThird line".to_string(), 1, LanguageId::Other); + let mut doc = TextDocument::new( + "First line\nSecond line\nThird line".to_string(), + 1, + LanguageId::Other, + ); // Multiple changes: replace "First" with "1st" and "Third" with "3rd" let changes = vec![ diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 00b5cb1c..bc7bea71 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -29,9 +29,6 @@ pub trait FileSystem: Send + Sync { /// List directory contents fn read_directory(&self, path: &Path) -> io::Result>; - - /// Get file metadata (size, modified time, etc.) - fn metadata(&self, path: &Path) -> io::Result; } /// In-memory file system for testing @@ -79,13 +76,6 @@ impl FileSystem for InMemoryFileSystem { // Simplified for testing Ok(Vec::new()) } - - fn metadata(&self, _path: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "Metadata not supported in memory filesystem", - )) - } } /// Standard file system implementation that uses [`std::fs`]. @@ -113,10 +103,6 @@ impl FileSystem for OsFileSystem { .map(|entry| entry.map(|e| e.path())) .collect() } - - fn metadata(&self, path: &Path) -> io::Result { - std::fs::metadata(path) - } } /// LSP file system that intercepts reads for buffered files. @@ -127,6 +113,16 @@ impl FileSystem for OsFileSystem { /// [`TextDocument`](crate::document::TextDocument)) and returns that content. /// If no buffer exists, it falls back to reading from disk. /// +/// ## Overlay Semantics +/// +/// Files in the overlay (buffered files) are treated as first-class files: +/// - `exists()` returns true for overlay files even if they don't exist on disk +/// - `is_file()` returns true for overlay files +/// - `read_to_string()` returns the overlay content +/// +/// This ensures consistent behavior across all filesystem operations for +/// buffered files that may not yet be saved to disk. +/// /// This type is used by the [`Database`](crate::db::Database) to ensure all file reads go /// through the buffer system first. pub struct WorkspaceFileSystem { @@ -172,12 +168,6 @@ impl FileSystem for WorkspaceFileSystem { // Overlays are never directories, so just delegate self.disk.read_directory(path) } - - fn metadata(&self, path: &Path) -> io::Result { - // For overlays, we could synthesize metadata, but for simplicity, - // fall back to disk. This might need refinement for edge cases. - self.disk.metadata(path) - } } #[cfg(test)] @@ -247,4 +237,65 @@ mod tests { assert!(lsp_fs.is_file(path)); assert!(!lsp_fs.is_directory(path)); } + + #[test] + fn test_overlay_consistency() { + // Create an empty filesystem (no files on disk) + let memory_fs = InMemoryFileSystem::new(); + let buffers = Buffers::new(); + let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); + + let path = std::path::Path::new("/test/overlay_only.py"); + + // Before adding to overlay, file doesn't exist + assert!(!lsp_fs.exists(path)); + assert!(!lsp_fs.is_file(path)); + + // Add file to overlay only (not on disk) + let url = Url::from_file_path("/test/overlay_only.py").unwrap(); + let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); + buffers.open(url, document); + + // Now file should exist and be recognized as a file + assert!(lsp_fs.exists(path), "Overlay file should exist"); + assert!(lsp_fs.is_file(path), "Overlay file should be recognized as a file"); + assert!(!lsp_fs.is_directory(path), "Overlay file should not be a directory"); + + // And we should be able to read its content + assert_eq!( + lsp_fs.read_to_string(path).unwrap(), + "overlay content", + "Should read overlay content" + ); + } + + #[test] + fn test_overlay_with_relative_path() { + // Create an empty filesystem (no files on disk) + let memory_fs = InMemoryFileSystem::new(); + let buffers = Buffers::new(); + let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); + + // Use a relative path that doesn't exist on disk + let relative_path = std::path::Path::new("nonexistent/overlay.py"); + + // Convert to absolute URL for the buffer (simulating how LSP would provide it) + let absolute_path = std::env::current_dir() + .unwrap() + .join(relative_path); + let url = Url::from_file_path(&absolute_path).unwrap(); + + // Add to overlay + let document = TextDocument::new("relative overlay".to_string(), 1, LanguageId::Python); + buffers.open(url, document); + + // The relative path should now work through the overlay + assert!(lsp_fs.exists(relative_path), "Relative overlay file should exist"); + assert!(lsp_fs.is_file(relative_path), "Relative overlay file should be a file"); + assert_eq!( + lsp_fs.read_to_string(relative_path).unwrap(), + "relative overlay", + "Should read relative overlay content" + ); + } } diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 2fde6281..6be337c0 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -46,7 +46,9 @@ pub fn lsp_uri_to_path(lsp_uri: &lsp_types::Uri) -> Option { /// Convert a [`Path`] to a `file://` URL /// /// Handles both absolute and relative paths. Relative paths are resolved -/// to absolute paths before conversion. +/// to absolute paths before conversion. This function does not require +/// the path to exist on the filesystem, making it suitable for overlay +/// files and other virtual content. #[must_use] pub fn path_to_url(path: &Path) -> Option { // For absolute paths, convert directly @@ -54,13 +56,17 @@ pub fn path_to_url(path: &Path) -> Option { return Url::from_file_path(path).ok(); } - // For relative paths, try to make them absolute first - if let Ok(absolute_path) = std::fs::canonicalize(path) { - return Url::from_file_path(absolute_path).ok(); - } - - // If canonicalization fails, try converting as-is (might fail) - Url::from_file_path(path).ok() + // For relative paths, make them absolute without requiring existence + // First try to get the current directory + let current_dir = std::env::current_dir().ok()?; + let absolute_path = current_dir.join(path); + + // Try to canonicalize if the file exists (to resolve symlinks, etc.) + // but if it doesn't exist, use the joined path as-is + let final_path = std::fs::canonicalize(&absolute_path) + .unwrap_or(absolute_path); + + Url::from_file_path(final_path).ok() } #[cfg(test)] @@ -164,9 +170,44 @@ mod tests { let path = PathBuf::from("../some/nonexistent/path.txt"); let url = path_to_url(&path); - // This might fail if the path doesn't exist and can't be canonicalized - // Current implementation falls back to trying direct conversion - assert!(url.is_none() || url.is_some()); + // Should now work even for non-existent files + assert!(url.is_some(), "Should handle non-existent relative paths"); + if let Some(u) = url { + assert_eq!(u.scheme(), "file"); + assert!(u.path().ends_with("some/nonexistent/path.txt")); + } + } + + #[test] + fn test_non_existent_absolute_path() { + // Test that absolute paths work even if they don't exist + let path = if cfg!(windows) { + PathBuf::from("C:/NonExistent/Directory/file.txt") + } else { + PathBuf::from("/nonexistent/directory/file.txt") + }; + + let url = path_to_url(&path); + assert!(url.is_some(), "Should handle non-existent absolute paths"); + if let Some(u) = url { + assert_eq!(u.scheme(), "file"); + assert!(u.path().contains("file.txt")); + } + } + + #[test] + fn test_non_existent_relative_path() { + // Test that relative paths work even if they don't exist + let path = PathBuf::from("nonexistent/file.txt"); + let url = path_to_url(&path); + + assert!(url.is_some(), "Should handle non-existent relative paths"); + if let Some(u) = url { + assert_eq!(u.scheme(), "file"); + assert!(u.path().ends_with("nonexistent/file.txt")); + // Should be an absolute URL + assert!(u.path().starts_with('/') || cfg!(windows)); + } } #[test] From fb88e8fa248cbe2ad730af2ca13c082cd57423a2 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 15:36:05 -0500 Subject: [PATCH 35/56] wip --- crates/djls-workspace/src/db.rs | 36 +------------------------------- crates/djls-workspace/src/lib.rs | 21 ------------------- 2 files changed, 1 insertion(+), 56 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index cbfdc29d..db816974 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -290,42 +290,8 @@ pub fn parse_template(db: &dyn Db, file: SourceFile) -> Option> let text = text_arc.as_ref(); // Call the pure parsing function from djls-templates - match djls_templates::parse_template(text) { - Ok((ast, errors)) => { - // Convert errors to strings - let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); - Some(Arc::new(TemplateAst { - ast, - errors: error_strings, - })) - } - Err(err) => { - // Even on fatal errors, return an empty AST with the error - Some(Arc::new(TemplateAst { - ast: djls_templates::Ast::default(), - errors: vec![err.to_string()], - })) - } - } -} - -/// Parse a Django template file by path using the file system. -/// -/// This Salsa tracked function reads file content through the FileSystem, which automatically -/// checks overlays before falling back to disk, implementing Ruff's two-layer architecture. -/// -/// Returns `None` for non-template files or if file cannot be read. -#[salsa::tracked] -pub fn parse_template_by_path(db: &dyn Db, file_path: FilePath) -> Option> { - // Read file content through the FileSystem (checks overlays first) - let path = Path::new(file_path.path(db).as_ref()); - let Ok(text) = db.read_file_content(path) else { - return None; - }; - - // Call the parsing function from djls-templates // TODO: Move this whole function into djls-templates - match djls_templates::parse_template(&text) { + match djls_templates::parse_template(text) { Ok((ast, errors)) => { // Convert errors to strings let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 9b4f251a..3e94aa99 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -32,28 +32,7 @@ pub use fs::OsFileSystem; pub use fs::WorkspaceFileSystem; pub use language::LanguageId; -/// Stable, compact identifier for files across the subsystem. -/// -/// [`FileId`] decouples file identity from paths/URIs, providing efficient keys for maps and -/// Salsa inputs. Once assigned to a file (via its URI), a [`FileId`] remains stable for the -/// lifetime of the system. -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub struct FileId(u32); - -impl FileId { - /// Create a [`FileId`] from a raw u32 value. - #[must_use] - pub fn from_raw(raw: u32) -> Self { - FileId(raw) - } - /// Get the underlying u32 index value. - #[must_use] - #[allow(dead_code)] - pub fn index(self) -> u32 { - self.0 - } -} /// File classification for routing to analyzers. /// From 54577aa839c2d741fd050449b6855603258da187 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 16:42:26 -0500 Subject: [PATCH 36/56] simplify --- crates/djls-server/src/completions.rs | 288 ++++++++++++++++++++++++++ crates/djls-server/src/lib.rs | 1 + crates/djls-server/src/server.rs | 34 ++- crates/djls-workspace/src/document.rs | 67 +----- crates/djls-workspace/src/fs.rs | 91 ++------ crates/djls-workspace/src/lib.rs | 2 +- crates/djls-workspace/src/paths.rs | 15 +- crates/djls-workspace/src/template.rs | 34 --- 8 files changed, 342 insertions(+), 190 deletions(-) create mode 100644 crates/djls-server/src/completions.rs delete mode 100644 crates/djls-workspace/src/template.rs diff --git a/crates/djls-server/src/completions.rs b/crates/djls-server/src/completions.rs new file mode 100644 index 00000000..dbbceeb8 --- /dev/null +++ b/crates/djls-server/src/completions.rs @@ -0,0 +1,288 @@ +//! Completion logic for Django Language Server +//! +//! This module handles all LSP completion requests, analyzing cursor context +//! and generating appropriate completion items for Django templates. + +use djls_project::TemplateTags; +use djls_workspace::{FileKind, PositionEncoding, TextDocument}; +use tower_lsp_server::lsp_types::{ + CompletionItem, CompletionItemKind, Documentation, InsertTextFormat, Position, +}; + +/// Tracks what closing characters are needed to complete a template tag. +/// +/// Used to determine whether the completion system needs to insert +/// closing braces when completing a Django template tag. +#[derive(Debug)] +pub enum ClosingBrace { + /// No closing brace present - need to add full `%}` or `}}` + None, + /// Partial close present (just `}`) - need to add `%` or second `}` + PartialClose, + /// Full close present (`%}` or `}}`) - no closing needed + FullClose, +} + +/// Cursor context within a Django template tag for completion support. +/// +/// Captures the state around the cursor position to provide intelligent +/// completions and determine what text needs to be inserted. +#[derive(Debug)] +pub struct TemplateTagContext { + /// The partial tag text before the cursor (e.g., "loa" for "{% loa|") + pub partial_tag: String, + /// What closing characters are already present after the cursor + pub closing_brace: ClosingBrace, + /// Whether a space is needed before the completion (true if cursor is right after `{%`) + pub needs_leading_space: bool, +} + +/// Information about a line of text and cursor position within it +#[derive(Debug)] +pub struct LineInfo { + /// The complete line text + pub line_text: String, + /// The cursor offset within the line (in characters) + pub cursor_offset_in_line: usize, +} + +/// Main entry point for handling completion requests +pub fn handle_completion( + document: &TextDocument, + position: Position, + encoding: PositionEncoding, + file_kind: FileKind, + template_tags: Option<&TemplateTags>, +) -> Vec { + // Only handle template files + if file_kind != FileKind::Template { + return Vec::new(); + } + + // Get line information from document + let Some(line_info) = get_line_info(document, position, encoding) else { + return Vec::new(); + }; + + // Analyze template context at cursor position + let Some(context) = + analyze_template_context(&line_info.line_text, line_info.cursor_offset_in_line) + else { + return Vec::new(); + }; + + // Generate completions based on available template tags + generate_template_completions(&context, template_tags) +} + +/// Extract line information from document at given position +fn get_line_info( + document: &TextDocument, + position: Position, + encoding: PositionEncoding, +) -> Option { + // Get the line content and calculate cursor position within line + let content = document.content(); + let lines: Vec<&str> = content.lines().collect(); + + let line_index = position.line as usize; + if line_index >= lines.len() { + return None; + } + + let line_text = lines[line_index].to_string(); + + // For UTF-16 encoding, we need to convert the character position + let cursor_offset_in_line = match encoding { + PositionEncoding::Utf16 => { + // Convert UTF-16 position to UTF-8 character offset + let utf16_pos = position.character as usize; + let mut utf8_offset = 0; + let mut utf16_offset = 0; + + for ch in line_text.chars() { + if utf16_offset >= utf16_pos { + break; + } + utf16_offset += ch.len_utf16(); + utf8_offset += 1; + } + utf8_offset + } + _ => position.character as usize, + }; + + Some(LineInfo { + line_text, + cursor_offset_in_line: cursor_offset_in_line.min(lines[line_index].chars().count()), + }) +} + +/// Analyze a line of template text to determine completion context +fn analyze_template_context(line: &str, cursor_offset: usize) -> Option { + if cursor_offset > line.chars().count() { + return None; + } + + let chars: Vec = line.chars().collect(); + let prefix = chars[..cursor_offset].iter().collect::(); + let rest_of_line = chars[cursor_offset..].iter().collect::(); + let rest_trimmed = rest_of_line.trim_start(); + + prefix.rfind("{%").map(|tag_start| { + let closing_brace = if rest_trimmed.starts_with("%}") { + ClosingBrace::FullClose + } else if rest_trimmed.starts_with('}') { + ClosingBrace::PartialClose + } else { + ClosingBrace::None + }; + + let partial_tag_start = tag_start + 2; // Skip "{%" + let content_after_tag = if partial_tag_start < prefix.len() { + &prefix[partial_tag_start..] + } else { + "" + }; + + // Check if we need a leading space - true if there's no space after {% + let needs_leading_space = + !content_after_tag.starts_with(' ') && !content_after_tag.is_empty(); + + let partial_tag = content_after_tag.trim().to_string(); + + TemplateTagContext { + partial_tag, + closing_brace, + needs_leading_space, + } + }) +} + +/// Generate Django template tag completion items based on context +fn generate_template_completions( + context: &TemplateTagContext, + template_tags: Option<&TemplateTags>, +) -> Vec { + let Some(tags) = template_tags else { + return Vec::new(); + }; + + let mut completions = Vec::new(); + + for tag in tags.iter() { + // Filter tags based on partial match + if tag.name().starts_with(&context.partial_tag) { + // Determine insertion text based on context + let mut insert_text = String::new(); + + // Add leading space if needed (cursor right after {%) + if context.needs_leading_space { + insert_text.push(' '); + } + + // Add the tag name + insert_text.push_str(tag.name()); + + // Add closing based on what's already present + match context.closing_brace { + ClosingBrace::None => insert_text.push_str(" %}"), + ClosingBrace::PartialClose => insert_text.push('%'), + ClosingBrace::FullClose => {} // No closing needed + } + + // Create completion item + let completion_item = CompletionItem { + label: tag.name().clone(), + kind: Some(CompletionItemKind::FUNCTION), + detail: Some(format!("from {}", tag.library())), + documentation: tag.doc().map(|doc| Documentation::String(doc.clone())), + insert_text: Some(insert_text), + insert_text_format: Some(InsertTextFormat::PLAIN_TEXT), + filter_text: Some(tag.name().clone()), + ..Default::default() + }; + + completions.push(completion_item); + } + } + + completions +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_analyze_template_context_basic() { + let line = "{% loa"; + let cursor_offset = 6; // After "loa" + + let context = analyze_template_context(line, cursor_offset).expect("Should get context"); + + assert_eq!(context.partial_tag, "loa"); + assert!(!context.needs_leading_space); + assert!(matches!(context.closing_brace, ClosingBrace::None)); + } + + #[test] + fn test_analyze_template_context_needs_leading_space() { + let line = "{%loa"; + let cursor_offset = 5; // After "loa" + + let context = analyze_template_context(line, cursor_offset).expect("Should get context"); + + assert_eq!(context.partial_tag, "loa"); + assert!(context.needs_leading_space); + assert!(matches!(context.closing_brace, ClosingBrace::None)); + } + + #[test] + fn test_analyze_template_context_with_closing() { + let line = "{% load %}"; + let cursor_offset = 7; // After "load" + + let context = analyze_template_context(line, cursor_offset).expect("Should get context"); + + assert_eq!(context.partial_tag, "load"); + assert!(!context.needs_leading_space); + assert!(matches!(context.closing_brace, ClosingBrace::FullClose)); + } + + #[test] + fn test_analyze_template_context_partial_closing() { + let line = "{% load }"; + let cursor_offset = 7; // After "load" + + let context = analyze_template_context(line, cursor_offset).expect("Should get context"); + + assert_eq!(context.partial_tag, "load"); + assert!(!context.needs_leading_space); + assert!(matches!(context.closing_brace, ClosingBrace::PartialClose)); + } + + #[test] + fn test_analyze_template_context_no_template() { + let line = "Just regular HTML"; + let cursor_offset = 5; + + let context = analyze_template_context(line, cursor_offset); + + assert!(context.is_none()); + } + + #[test] + fn test_generate_template_completions_empty_tags() { + let context = TemplateTagContext { + partial_tag: "loa".to_string(), + needs_leading_space: false, + closing_brace: ClosingBrace::None, + }; + + let completions = generate_template_completions(&context, None); + + assert!(completions.is_empty()); + } +} + diff --git a/crates/djls-server/src/lib.rs b/crates/djls-server/src/lib.rs index ba468301..055f9d03 100644 --- a/crates/djls-server/src/lib.rs +++ b/crates/djls-server/src/lib.rs @@ -1,4 +1,5 @@ mod client; +mod completions; mod logging; mod queue; pub mod server; diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index 5638ce59..b7970b61 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -2,6 +2,7 @@ use std::future::Future; use std::sync::Arc; use djls_workspace::paths; +use djls_workspace::FileKind; use tokio::sync::RwLock; use tower_lsp_server::jsonrpc::Result as LspResult; use tower_lsp_server::lsp_types; @@ -258,21 +259,38 @@ impl LanguageServer for DjangoLanguageServer { let lsp_uri = params.text_document_position.text_document.uri; let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); let position = params.text_document_position.position; - let _encoding = session.position_encoding(); + let encoding = session.position_encoding(); tracing::debug!("Completion requested for {} at {:?}", url, position); + // Get file path and determine file type if let Some(path) = paths::url_to_path(&url) { - let content = session.file_content(path); - if content.is_empty() { - tracing::debug!("File {} has no content", url); + let file_kind = FileKind::from_path(&path); + + // Get the document from buffers + let document = session.get_document(&url)?; + + // Get template tags from the project + let template_tags = session.project() + .and_then(|p| p.template_tags()); + + // Generate completions using the new completions module + let completions = crate::completions::handle_completion( + &document, + position, + encoding, + file_kind, + template_tags, + ); + + if completions.is_empty() { + None } else { - tracing::debug!("Using content for completion in {}", url); - // TODO: Implement actual completion logic using content and encoding + Some(lsp_types::CompletionResponse::Array(completions)) } + } else { + None } - - None }) .await; diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 96ef46f3..eb0fae76 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -5,13 +5,10 @@ //! performance when handling frequent position-based operations like hover, completion, //! and diagnostics. -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; +use tower_lsp_server::lsp_types::{Position, Range}; use crate::encoding::PositionEncoding; use crate::language::LanguageId; -use crate::template::ClosingBrace; -use crate::template::TemplateTagContext; /// In-memory representation of an open document in the LSP. /// @@ -95,7 +92,7 @@ impl TextDocument { ) { // Fast path: single change without range = full document replacement if changes.len() == 1 && changes[0].range.is_none() { - self.content = changes[0].text.clone(); + self.content.clone_from(&changes[0].text); self.line_index = LineIndex::new(&self.content); self.version = version; return; @@ -137,50 +134,6 @@ impl TextDocument { self.version = version; } - #[must_use] - pub fn get_template_tag_context( - &self, - position: Position, - encoding: PositionEncoding, - ) -> Option { - let start = self.line_index.line_starts.get(position.line as usize)?; - let end = self - .line_index - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(self.line_index.length); - - let line = &self.content[*start as usize..end as usize]; - - // Use the new offset method with the specified encoding - let char_offset = self.line_index.offset(position, &self.content, encoding) as usize; - let char_pos = char_offset - *start as usize; - - let prefix = &line[..char_pos]; - let rest_of_line = &line[char_pos..]; - let rest_trimmed = rest_of_line.trim_start(); - - prefix.rfind("{%").map(|tag_start| { - // Check if we're immediately after {% with no space - let needs_leading_space = prefix.ends_with("{%"); - - let closing_brace = if rest_trimmed.starts_with("%}") { - ClosingBrace::FullClose - } else if rest_trimmed.starts_with('}') { - ClosingBrace::PartialClose - } else { - ClosingBrace::None - }; - - TemplateTagContext { - partial_tag: prefix[tag_start + 2..].trim().to_string(), - needs_leading_space, - closing_brace, - } - }) - } - #[must_use] pub fn position_to_offset( &self, @@ -516,22 +469,6 @@ mod tests { assert_eq!(&line2[line2_offset..=line2_offset], " "); } - #[test] - fn test_template_tag_context_with_utf16() { - // Test template with non-ASCII characters before template tag - let content = "Título 🌍: {% for"; - let doc = TextDocument::new(content.to_string(), 1, LanguageId::HtmlDjango); - - // Position after "for" - UTF-16 position 17 (after 'r') - let pos = Position::new(0, 17); - let tag_context = doc - .get_template_tag_context(pos, PositionEncoding::Utf16) - .expect("Should get template context"); - - assert_eq!(tag_context.partial_tag, "for"); - assert!(!tag_context.needs_leading_space); - } - #[test] fn test_get_text_range_with_emoji() { let content = "Hello 🌍 world"; diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index bc7bea71..e424ff5f 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use std::io; use std::path::Path; +#[cfg(test)] use std::path::PathBuf; use std::sync::Arc; @@ -20,15 +21,6 @@ pub trait FileSystem: Send + Sync { /// Check if a path exists fn exists(&self, path: &Path) -> bool; - - /// Check if a path is a file - fn is_file(&self, path: &Path) -> bool; - - /// Check if a path is a directory - fn is_directory(&self, path: &Path) -> bool; - - /// List directory contents - fn read_directory(&self, path: &Path) -> io::Result>; } /// In-memory file system for testing @@ -62,20 +54,6 @@ impl FileSystem for InMemoryFileSystem { fn exists(&self, path: &Path) -> bool { self.files.contains_key(path) } - - fn is_file(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_directory(&self, _path: &Path) -> bool { - // Simplified for testing - no directories in memory filesystem - false - } - - fn read_directory(&self, _path: &Path) -> io::Result> { - // Simplified for testing - Ok(Vec::new()) - } } /// Standard file system implementation that uses [`std::fs`]. @@ -89,20 +67,6 @@ impl FileSystem for OsFileSystem { fn exists(&self, path: &Path) -> bool { path.exists() } - - fn is_file(&self, path: &Path) -> bool { - path.is_file() - } - - fn is_directory(&self, path: &Path) -> bool { - path.is_dir() - } - - fn read_directory(&self, path: &Path) -> io::Result> { - std::fs::read_dir(path)? - .map(|entry| entry.map(|e| e.path())) - .collect() - } } /// LSP file system that intercepts reads for buffered files. @@ -117,7 +81,6 @@ impl FileSystem for OsFileSystem { /// /// Files in the overlay (buffered files) are treated as first-class files: /// - `exists()` returns true for overlay files even if they don't exist on disk -/// - `is_file()` returns true for overlay files /// - `read_to_string()` returns the overlay content /// /// This ensures consistent behavior across all filesystem operations for @@ -153,21 +116,6 @@ impl FileSystem for WorkspaceFileSystem { paths::path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.exists(path) } - - fn is_file(&self, path: &Path) -> bool { - paths::path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) - || self.disk.is_file(path) - } - - fn is_directory(&self, path: &Path) -> bool { - // Overlays are never directories, so just delegate - self.disk.is_directory(path) - } - - fn read_directory(&self, path: &Path) -> io::Result> { - // Overlays are never directories, so just delegate - self.disk.read_directory(path) - } } #[cfg(test)] @@ -232,10 +180,8 @@ mod tests { let path = std::path::Path::new("/test/file.py"); - // These should delegate to the fallback filesystem + // This should delegate to the fallback filesystem assert!(lsp_fs.exists(path)); - assert!(lsp_fs.is_file(path)); - assert!(!lsp_fs.is_directory(path)); } #[test] @@ -246,21 +192,18 @@ mod tests { let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); let path = std::path::Path::new("/test/overlay_only.py"); - + // Before adding to overlay, file doesn't exist assert!(!lsp_fs.exists(path)); - assert!(!lsp_fs.is_file(path)); - + // Add file to overlay only (not on disk) let url = Url::from_file_path("/test/overlay_only.py").unwrap(); let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); buffers.open(url, document); - - // Now file should exist and be recognized as a file + + // Now file should exist assert!(lsp_fs.exists(path), "Overlay file should exist"); - assert!(lsp_fs.is_file(path), "Overlay file should be recognized as a file"); - assert!(!lsp_fs.is_directory(path), "Overlay file should not be a directory"); - + // And we should be able to read its content assert_eq!( lsp_fs.read_to_string(path).unwrap(), @@ -268,30 +211,30 @@ mod tests { "Should read overlay content" ); } - + #[test] fn test_overlay_with_relative_path() { // Create an empty filesystem (no files on disk) let memory_fs = InMemoryFileSystem::new(); let buffers = Buffers::new(); let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); - + // Use a relative path that doesn't exist on disk let relative_path = std::path::Path::new("nonexistent/overlay.py"); - + // Convert to absolute URL for the buffer (simulating how LSP would provide it) - let absolute_path = std::env::current_dir() - .unwrap() - .join(relative_path); + let absolute_path = std::env::current_dir().unwrap().join(relative_path); let url = Url::from_file_path(&absolute_path).unwrap(); - + // Add to overlay let document = TextDocument::new("relative overlay".to_string(), 1, LanguageId::Python); buffers.open(url, document); - + // The relative path should now work through the overlay - assert!(lsp_fs.exists(relative_path), "Relative overlay file should exist"); - assert!(lsp_fs.is_file(relative_path), "Relative overlay file should be a file"); + assert!( + lsp_fs.exists(relative_path), + "Relative overlay file should exist" + ); assert_eq!( lsp_fs.read_to_string(relative_path).unwrap(), "relative overlay", diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 3e94aa99..f584e785 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -19,7 +19,7 @@ pub mod encoding; mod fs; mod language; pub mod paths; -mod template; + use std::path::Path; diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 6be337c0..1d2b47f8 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -60,12 +60,11 @@ pub fn path_to_url(path: &Path) -> Option { // First try to get the current directory let current_dir = std::env::current_dir().ok()?; let absolute_path = current_dir.join(path); - + // Try to canonicalize if the file exists (to resolve symlinks, etc.) // but if it doesn't exist, use the joined path as-is - let final_path = std::fs::canonicalize(&absolute_path) - .unwrap_or(absolute_path); - + let final_path = std::fs::canonicalize(&absolute_path).unwrap_or(absolute_path); + Url::from_file_path(final_path).ok() } @@ -177,7 +176,7 @@ mod tests { assert!(u.path().ends_with("some/nonexistent/path.txt")); } } - + #[test] fn test_non_existent_absolute_path() { // Test that absolute paths work even if they don't exist @@ -186,7 +185,7 @@ mod tests { } else { PathBuf::from("/nonexistent/directory/file.txt") }; - + let url = path_to_url(&path); assert!(url.is_some(), "Should handle non-existent absolute paths"); if let Some(u) = url { @@ -194,13 +193,13 @@ mod tests { assert!(u.path().contains("file.txt")); } } - + #[test] fn test_non_existent_relative_path() { // Test that relative paths work even if they don't exist let path = PathBuf::from("nonexistent/file.txt"); let url = path_to_url(&path); - + assert!(url.is_some(), "Should handle non-existent relative paths"); if let Some(u) = url { assert_eq!(u.scheme(), "file"); diff --git a/crates/djls-workspace/src/template.rs b/crates/djls-workspace/src/template.rs deleted file mode 100644 index b2bd44b8..00000000 --- a/crates/djls-workspace/src/template.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Django template context detection for completions -//! -//! Detects cursor position context within Django template tags to provide -//! appropriate completions and auto-closing behavior. - -// TODO: is this module in the right spot or even needed? - -/// Tracks what closing characters are needed to complete a template tag. -/// -/// Used to determine whether the completion system needs to insert -/// closing braces when completing a Django template tag. -#[derive(Debug)] -pub enum ClosingBrace { - /// No closing brace present - need to add full `%}` or `}}` - None, - /// Partial close present (just `}`) - need to add `%` or second `}` - PartialClose, - /// Full close present (`%}` or `}}`) - no closing needed - FullClose, -} - -/// Cursor context within a Django template tag for completion support. -/// -/// Captures the state around the cursor position to provide intelligent -/// completions and determine what text needs to be inserted. -#[derive(Debug)] -pub struct TemplateTagContext { - /// The partial tag text before the cursor (e.g., "loa" for "{% loa|") - pub partial_tag: String, - /// What closing characters are already present after the cursor - pub closing_brace: ClosingBrace, - /// Whether a space is needed before the completion (true if cursor is right after `{%`) - pub needs_leading_space: bool, -} From db1081ef1e59b23d606d555c1ba10f0bec8fb433 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 16:47:42 -0500 Subject: [PATCH 37/56] more clean up --- crates/djls-server/src/completions.rs | 12 +++++------- crates/djls-server/src/server.rs | 21 +++++++++------------ 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/crates/djls-server/src/completions.rs b/crates/djls-server/src/completions.rs index dbbceeb8..46f8d6eb 100644 --- a/crates/djls-server/src/completions.rs +++ b/crates/djls-server/src/completions.rs @@ -41,9 +41,9 @@ pub struct TemplateTagContext { #[derive(Debug)] pub struct LineInfo { /// The complete line text - pub line_text: String, + pub text: String, /// The cursor offset within the line (in characters) - pub cursor_offset_in_line: usize, + pub cursor_offset: usize, } /// Main entry point for handling completion requests @@ -65,9 +65,7 @@ pub fn handle_completion( }; // Analyze template context at cursor position - let Some(context) = - analyze_template_context(&line_info.line_text, line_info.cursor_offset_in_line) - else { + let Some(context) = analyze_template_context(&line_info.text, line_info.cursor_offset) else { return Vec::new(); }; @@ -113,8 +111,8 @@ fn get_line_info( }; Some(LineInfo { - line_text, - cursor_offset_in_line: cursor_offset_in_line.min(lines[line_index].chars().count()), + text: line_text, + cursor_offset: cursor_offset_in_line.min(lines[line_index].chars().count()), }) } diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index b7970b61..8eae018b 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -258,23 +258,20 @@ impl LanguageServer for DjangoLanguageServer { .with_session_mut(|session| { let lsp_uri = params.text_document_position.text_document.uri; let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); - let position = params.text_document_position.position; - let encoding = session.position_encoding(); - tracing::debug!("Completion requested for {} at {:?}", url, position); + tracing::debug!( + "Completion requested for {} at {:?}", + url, + params.text_document_position.position + ); - // Get file path and determine file type if let Some(path) = paths::url_to_path(&url) { - let file_kind = FileKind::from_path(&path); - - // Get the document from buffers let document = session.get_document(&url)?; - - // Get template tags from the project - let template_tags = session.project() - .and_then(|p| p.template_tags()); + let position = params.text_document_position.position; + let encoding = session.position_encoding(); + let file_kind = FileKind::from_path(&path); + let template_tags = session.project().and_then(|p| p.template_tags()); - // Generate completions using the new completions module let completions = crate::completions::handle_completion( &document, position, From 9f70432cc1c8419db1ce366181407579bac443d1 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 11:25:13 -0500 Subject: [PATCH 38/56] wip --- crates/djls-workspace/src/lib.rs | 2 + crates/djls-workspace/src/workspace.rs | 276 +++++++++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 crates/djls-workspace/src/workspace.rs diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index f584e785..2791d875 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -19,6 +19,7 @@ pub mod encoding; mod fs; mod language; pub mod paths; +mod workspace; use std::path::Path; @@ -31,6 +32,7 @@ pub use fs::FileSystem; pub use fs::OsFileSystem; pub use fs::WorkspaceFileSystem; pub use language::LanguageId; +pub use workspace::Workspace; diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs new file mode 100644 index 00000000..a1f6df42 --- /dev/null +++ b/crates/djls-workspace/src/workspace.rs @@ -0,0 +1,276 @@ +//! Workspace facade for managing all workspace components +//! +//! This module provides the [`Workspace`] struct that encapsulates all workspace +//! components including buffers, file system, file tracking, and database handle. +//! This provides a clean API boundary between server and workspace layers. + +use std::mem; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use dashmap::DashMap; +use salsa::StorageHandle; +use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; +use url::Url; + +use crate::buffers::Buffers; +use crate::db::{source_text, Database, SourceFile}; +use crate::document::TextDocument; +use crate::fs::{OsFileSystem, WorkspaceFileSystem}; +use crate::paths::url_to_path; + +/// Workspace facade that encapsulates all workspace components. +/// +/// This struct provides a unified interface for managing workspace state, +/// including in-memory buffers, file system abstraction, file tracking, +/// and the Salsa database handle. It follows the same initialization pattern +/// as the Session but encapsulates it in a reusable component. +/// +/// ## Components +/// +/// - **Buffers**: Thread-safe storage for open document content +/// - **WorkspaceFileSystem**: File system abstraction with buffer interception +/// - **Files**: Shared file tracking across all Database instances +/// - **Database Handle**: Thread-safe Salsa database handle for incremental computation +pub struct Workspace { + /// Layer 1: Shared buffer storage for open documents + buffers: Buffers, + + /// File system abstraction with buffer interception + file_system: Arc, + + /// Shared file tracking across all Database instances + files: Arc>, + + /// Layer 2: Thread-safe Salsa database handle for pure computation + db_handle: StorageHandle, +} + +impl Workspace { + /// Create a new Workspace with all components initialized. + /// + /// This follows the same initialization pattern as Session: + /// 1. Creates Buffers for in-memory document storage + /// 2. Creates shared file tracking DashMap + /// 3. Creates WorkspaceFileSystem with buffer interception + /// 4. Initializes Database and extracts StorageHandle + #[must_use] + pub fn new() -> Self { + let buffers = Buffers::new(); + let files = Arc::new(DashMap::new()); + let file_system = Arc::new(WorkspaceFileSystem::new( + buffers.clone(), + Arc::new(OsFileSystem), + )); + let db_handle = Database::new(file_system.clone(), files.clone()) + .storage() + .clone() + .into_zalsa_handle(); + + Self { + buffers, + file_system, + files, + db_handle, + } + } + + // Database Access Methods (AC #1) + + /// Execute a read-only operation with access to the database. + /// + /// Creates a temporary Database instance from the handle for the closure. + /// This is safe for concurrent read operations. + pub fn with_db(&self, f: F) -> R + where + F: FnOnce(&Database) -> R, + { + let storage = self.db_handle.clone().into_storage(); + let db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); + f(&db) + } + + /// Execute a mutable operation with exclusive access to the database. + /// + /// Takes ownership of the handle, creates a mutable Database, and restores + /// the handle after the operation completes. + pub fn with_db_mut(&mut self, f: F) -> R + where + F: FnOnce(&mut Database) -> R, + { + let handle = self.take_db_handle_for_mutation(); + let storage = handle.into_storage(); + let mut db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); + let result = f(&mut db); + let new_handle = db.storage().clone().into_zalsa_handle(); + self.restore_db_handle(new_handle); + result + } + + /// Private helper: Take the database handle for mutation. + fn take_db_handle_for_mutation(&mut self) -> StorageHandle { + // Create a placeholder handle and swap it with the current one + let placeholder = Database::new(self.file_system.clone(), self.files.clone()) + .storage() + .clone() + .into_zalsa_handle(); + mem::replace(&mut self.db_handle, placeholder) + } + + /// Private helper: Restore the database handle after mutation. + fn restore_db_handle(&mut self, handle: StorageHandle) { + self.db_handle = handle; + } + + // Document Lifecycle Methods (AC #2) + + /// Open a document in the workspace. + /// + /// Updates both the buffer layer and database layer. If the file exists + /// in the database, it's marked as touched to trigger invalidation. + pub fn open_document(&mut self, url: &Url, document: TextDocument) { + // Layer 1: Add to buffers + self.buffers.open(url.clone(), document); + + // Layer 2: Update database if file exists + if let Some(path) = url_to_path(url) { + self.invalidate_file_if_exists(&path); + } + } + + /// Update a document with incremental changes. + /// + /// Applies changes to the existing document and triggers database invalidation. + /// Falls back to full replacement if the document isn't currently open. + pub fn update_document( + &mut self, + url: &Url, + changes: Vec, + version: i32, + ) { + if let Some(mut document) = self.buffers.get(url) { + // Apply incremental changes to existing document + document.update(changes, version); + self.buffers.update(url.clone(), document); + } else if let Some(first_change) = changes.into_iter().next() { + // Fallback: treat first change as full replacement + if first_change.range.is_none() { + let document = TextDocument::new( + first_change.text, + version, + crate::language::LanguageId::Other, + ); + self.buffers.open(url.clone(), document); + } + } + + // Touch file in database to trigger invalidation + if let Some(path) = url_to_path(url) { + self.invalidate_file_if_exists(&path); + } + } + + /// Close a document and return it. + /// + /// Removes from buffers and triggers database invalidation to fall back to disk. + pub fn close_document(&mut self, url: &Url) -> Option { + let document = self.buffers.close(url); + + // Touch file in database to trigger re-read from disk + if let Some(path) = url_to_path(url) { + self.invalidate_file_if_exists(&path); + } + + document + } + + // File Operations (AC #3) + + /// Get file content through the database. + /// + /// Creates or retrieves the file entity and returns its source text. + pub fn file_content(&mut self, path: PathBuf) -> String { + self.with_db_mut(|db| { + let file = db.get_or_create_file(path); + source_text(db, file).to_string() + }) + } + + /// Get the revision number of a file if it exists. + /// + /// Returns None if the file is not being tracked by the database. + pub fn file_revision(&mut self, path: &Path) -> Option { + self.with_db_mut(|db| { + if db.has_file(path) { + let file = db.get_or_create_file(path.to_path_buf()); + Some(file.revision(db)) + } else { + None + } + }) + } + + // Buffer Query Method (AC #4) + + /// Get a document from the buffer if it's open. + /// + /// Returns a cloned TextDocument for the given URL if it exists in buffers. + #[must_use] + pub fn get_document(&self, url: &Url) -> Option { + self.buffers.get(url) + } + + // File Invalidation Helper (AC #5) + + /// Private helper: Invalidate a file if it exists in the database. + /// + /// Used by document lifecycle methods to trigger cache invalidation. + fn invalidate_file_if_exists(&mut self, path: &Path) { + self.with_db_mut(|db| { + if db.has_file(path) { + db.touch_file(path); + } + }); + } + + // Existing methods preserved below + + /// Get a reference to the buffers. + #[must_use] + pub fn buffers(&self) -> &Buffers { + &self.buffers + } + + /// Get a clone of the file system. + /// + /// Returns an Arc-wrapped WorkspaceFileSystem that can be shared + /// across threads and used for file operations. + #[must_use] + pub fn file_system(&self) -> Arc { + self.file_system.clone() + } + + /// Get a clone of the files tracking map. + /// + /// Returns an Arc-wrapped DashMap for O(1) file lookups that + /// can be shared across Database instances. + #[must_use] + pub fn files(&self) -> Arc> { + self.files.clone() + } + + /// Get a reference to the database handle. + /// + /// The StorageHandle can be cloned safely for read operations + /// or moved for mutation operations following Salsa's patterns. + #[must_use] + pub fn db_handle(&self) -> &StorageHandle { + &self.db_handle + } +} + +impl Default for Workspace { + fn default() -> Self { + Self::new() + } +} \ No newline at end of file From 1397513fad988f2f9d390410dcc9ddbda9f3eaae Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 12:26:50 -0500 Subject: [PATCH 39/56] guard --- crates/djls-workspace/src/db.rs | 8 + crates/djls-workspace/src/workspace.rs | 574 +++++++++++++++++++++++-- 2 files changed, 541 insertions(+), 41 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index db816974..db335639 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -129,6 +129,14 @@ impl Database { self.fs.read_to_string(path) } + /// Get an existing [`SourceFile`] for the given path without creating it. + /// + /// Returns `Some(SourceFile)` if the file is already tracked, `None` otherwise. + /// This method uses an immutable reference and doesn't modify the database. + pub fn get_file(&self, path: &Path) -> Option { + self.files.get(path).map(|file_ref| *file_ref) + } + /// Get or create a [`SourceFile`] for the given path. /// /// Files are created with an initial revision of 0 and tracked in the [`Database`]'s diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index a1f6df42..f225ead9 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -4,7 +4,6 @@ //! components including buffers, file system, file tracking, and database handle. //! This provides a clean API boundary between server and workspace layers. -use std::mem; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -19,6 +18,209 @@ use crate::document::TextDocument; use crate::fs::{OsFileSystem, WorkspaceFileSystem}; use crate::paths::url_to_path; +/// Safe wrapper for StorageHandle that prevents misuse through type safety. +/// +/// This enum ensures that database handles can only be in one of two valid states, +/// making invalid states unrepresentable and eliminating the need for placeholder +/// handles during mutations. +enum SafeStorageHandle { + /// Handle is available for use + Available(StorageHandle), + /// Handle has been taken for mutation - no handle available + TakenForMutation, +} + +impl SafeStorageHandle { + /// Create a new SafeStorageHandle in the Available state + fn new(handle: StorageHandle) -> Self { + Self::Available(handle) + } + + /// Take the handle for mutation, leaving the enum in TakenForMutation state. + /// + /// # Panics + /// + /// Panics if the handle has already been taken for mutation. + fn take_for_mutation(&mut self) -> StorageHandle { + match std::mem::replace(self, Self::TakenForMutation) { + Self::Available(handle) => handle, + Self::TakenForMutation => panic!( + "Database handle already taken for mutation. This indicates a programming error - \ + ensure you're not calling multiple mutation operations concurrently or forgetting \ + to restore the handle after a previous mutation." + ), + } + } + + /// Restore the handle after mutation, returning it to Available state. + /// + /// # Panics + /// + /// Panics if the handle is not currently taken for mutation. + fn restore_from_mutation(&mut self, handle: StorageHandle) { + match self { + Self::TakenForMutation => { + *self = Self::Available(handle); + } + Self::Available(_) => panic!( + "Cannot restore database handle - handle is not currently taken for mutation. \ + This indicates a programming error in the StorageHandleGuard implementation." + ), + } + } + + /// Get a clone of the handle for read-only operations. + /// + /// # Panics + /// + /// Panics if the handle is currently taken for mutation. + fn clone_for_read(&self) -> StorageHandle { + match self { + Self::Available(handle) => handle.clone(), + Self::TakenForMutation => panic!( + "Cannot access database handle for read - handle is currently taken for mutation. \ + Wait for the current mutation operation to complete." + ), + } + } +} + +/// State of the StorageHandleGuard during its lifetime. +/// +/// This enum captures the exact state of the guard and the handle it manages, +/// making state transitions explicit and preventing invalid combinations. +/// +/// ## State Machine +/// +/// The valid state transitions are: +/// - `Active` → `Consumed` (via `handle()`) +/// - `Consumed` → `Restored` (via `restore()`) +/// +/// Invalid transitions will panic with specific error messages: +/// - `handle()` on `Consumed` state: "StorageHandle already consumed" +/// - `handle()` on `Restored` state: "Cannot consume handle - guard has already been restored" +/// - `restore()` on `Active` state: "Cannot restore handle - it hasn't been consumed yet" +/// - `restore()` on `Restored` state: "Handle has already been restored" +/// +/// ## Drop Behavior +/// +/// The guard will panic on drop unless it's in the `Restored` state: +/// - Drop in `Active` state: "StorageHandleGuard dropped without using the handle" +/// - Drop in `Consumed` state: "StorageHandleGuard dropped without restoring handle" +/// - Drop in `Restored` state: No panic - proper cleanup +enum GuardState { + /// Guard holds the handle and it's ready to be consumed via `handle()` + Active { handle: StorageHandle }, + + /// Handle has been consumed via `handle()` method, awaiting restoration via `restore()` + Consumed, + + /// Handle has been properly restored to the SafeStorageHandle - guard is complete + Restored, +} + +/// RAII guard for safe StorageHandle management during mutations. +/// +/// This guard ensures that database handles are automatically restored even if +/// panics occur during mutation operations. It prevents double-takes and +/// provides clear error messages for misuse. +/// +/// ## State Machine +/// +/// The guard follows a clear state machine: +/// - `Active` → `Consumed` (via `handle()`) +/// - `Consumed` → `Restored` (via `restore()`) +/// - `Active` → `Restored` (direct restore without consuming - future extension) +#[must_use = "StorageHandleGuard must be used - dropping it immediately defeats the purpose"] +pub struct StorageHandleGuard<'a> { + /// Reference to the workspace's SafeStorageHandle for restoration + safe_handle: &'a mut SafeStorageHandle, + /// Current state of the guard and handle + state: GuardState, +} + +impl<'a> StorageHandleGuard<'a> { + /// Create a new guard by taking the handle from the SafeStorageHandle. + fn new(safe_handle: &'a mut SafeStorageHandle) -> Self { + let handle = safe_handle.take_for_mutation(); + Self { + safe_handle, + state: GuardState::Active { handle }, + } + } + + /// Get the StorageHandle for mutation operations. + /// + /// # Panics + /// + /// Panics if the handle has already been consumed or restored. + pub fn handle(&mut self) -> StorageHandle { + match std::mem::replace(&mut self.state, GuardState::Consumed) { + GuardState::Active { handle } => handle, + GuardState::Consumed => panic!( + "StorageHandle already consumed from guard. Each guard can only provide \ + the handle once - this prevents accidental multiple uses." + ), + GuardState::Restored => panic!( + "Cannot consume handle - guard has already been restored. Once restored, \ + the guard cannot provide the handle again." + ), + } + } + + /// Restore the handle manually before the guard drops. + /// + /// This is useful when you want to restore the handle and continue using + /// the workspace in the same scope. + /// + /// # Panics + /// + /// Panics if the handle hasn't been consumed yet, or if already restored. + pub fn restore(mut self, handle: StorageHandle) { + match self.state { + GuardState::Consumed => { + self.safe_handle.restore_from_mutation(handle); + self.state = GuardState::Restored; + } + GuardState::Active { .. } => panic!( + "Cannot restore handle - it hasn't been consumed yet. Call guard.handle() \ + first to get the handle, then restore the updated handle after mutations." + ), + GuardState::Restored => { + panic!("Handle has already been restored. Each guard can only restore once.") + } + } + } +} + +impl<'a> Drop for StorageHandleGuard<'a> { + fn drop(&mut self) { + // Provide specific error messages based on the exact state + // Avoid double-panic during unwinding + if !std::thread::panicking() { + match &self.state { + GuardState::Active { .. } => { + panic!( + "StorageHandleGuard dropped without using the handle. Either call \ + guard.handle() to consume the handle for mutations, or ensure the \ + guard is properly used in your mutation workflow." + ); + } + GuardState::Consumed => { + panic!( + "StorageHandleGuard dropped without restoring handle. You must call \ + guard.restore(updated_handle) to properly restore the database handle \ + after mutation operations complete." + ); + } + GuardState::Restored => { + // All good - proper cleanup completed + } + } + } + } +} + /// Workspace facade that encapsulates all workspace components. /// /// This struct provides a unified interface for managing workspace state, @@ -42,8 +244,8 @@ pub struct Workspace { /// Shared file tracking across all Database instances files: Arc>, - /// Layer 2: Thread-safe Salsa database handle for pure computation - db_handle: StorageHandle, + /// Layer 2: Thread-safe Salsa database handle with safe mutation management + db_handle: SafeStorageHandle, } impl Workspace { @@ -62,7 +264,7 @@ impl Workspace { buffers.clone(), Arc::new(OsFileSystem), )); - let db_handle = Database::new(file_system.clone(), files.clone()) + let handle = Database::new(file_system.clone(), files.clone()) .storage() .clone() .into_zalsa_handle(); @@ -71,7 +273,7 @@ impl Workspace { buffers, file_system, files, - db_handle, + db_handle: SafeStorageHandle::new(handle), } } @@ -85,43 +287,30 @@ impl Workspace { where F: FnOnce(&Database) -> R, { - let storage = self.db_handle.clone().into_storage(); + let handle = self.db_handle.clone_for_read(); + let storage = handle.into_storage(); let db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); f(&db) } /// Execute a mutable operation with exclusive access to the database. /// - /// Takes ownership of the handle, creates a mutable Database, and restores - /// the handle after the operation completes. + /// Uses the StorageHandleGuard pattern to ensure the handle is safely restored + /// even if the operation panics. This eliminates the need for placeholder handles. pub fn with_db_mut(&mut self, f: F) -> R where F: FnOnce(&mut Database) -> R, { - let handle = self.take_db_handle_for_mutation(); + let mut guard = StorageHandleGuard::new(&mut self.db_handle); + let handle = guard.handle(); let storage = handle.into_storage(); let mut db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); let result = f(&mut db); let new_handle = db.storage().clone().into_zalsa_handle(); - self.restore_db_handle(new_handle); + guard.restore(new_handle); result } - /// Private helper: Take the database handle for mutation. - fn take_db_handle_for_mutation(&mut self) -> StorageHandle { - // Create a placeholder handle and swap it with the current one - let placeholder = Database::new(self.file_system.clone(), self.files.clone()) - .storage() - .clone() - .into_zalsa_handle(); - mem::replace(&mut self.db_handle, placeholder) - } - - /// Private helper: Restore the database handle after mutation. - fn restore_db_handle(&mut self, handle: StorageHandle) { - self.db_handle = handle; - } - // Document Lifecycle Methods (AC #2) /// Open a document in the workspace. @@ -186,10 +375,41 @@ impl Workspace { // File Operations (AC #3) + /// Try to read file content using read-only database access. + /// + /// Returns `Some(content)` if the file exists in the database, `None` otherwise. + /// This avoids write locks for files that are already being tracked. + fn try_read_file(&self, path: &Path) -> Option { + self.with_db(|db| { + if let Some(file) = db.get_file(path) { + tracing::debug!("Using optimized read path for {}", path.display()); + Some(source_text(db, file).to_string()) + } else { + tracing::debug!( + "File {} not in database, requiring creation", + path.display() + ); + None + } + }) + } + /// Get file content through the database. /// - /// Creates or retrieves the file entity and returns its source text. + /// First attempts read-only access for existing files, then escalates to write + /// access only if the file needs to be created. This improves concurrency by + /// avoiding unnecessary write locks. pub fn file_content(&mut self, path: PathBuf) -> String { + // Try read-only access first for existing files + if let Some(content) = self.try_read_file(&path) { + return content; + } + + // File doesn't exist, escalate to write access to create it + tracing::debug!( + "Escalating to write access to create file {}", + path.display() + ); self.with_db_mut(|db| { let file = db.get_or_create_file(path); source_text(db, file).to_string() @@ -199,15 +419,9 @@ impl Workspace { /// Get the revision number of a file if it exists. /// /// Returns None if the file is not being tracked by the database. - pub fn file_revision(&mut self, path: &Path) -> Option { - self.with_db_mut(|db| { - if db.has_file(path) { - let file = db.get_or_create_file(path.to_path_buf()); - Some(file.revision(db)) - } else { - None - } - }) + /// Uses read-only database access since no mutation is needed. + pub fn file_revision(&self, path: &Path) -> Option { + self.with_db(|db| db.get_file(path).map(|file| file.revision(db))) } // Buffer Query Method (AC #4) @@ -259,13 +473,20 @@ impl Workspace { self.files.clone() } - /// Get a reference to the database handle. + /// Get a cloned database handle for read operations. + /// + /// This provides access to a [`StorageHandle`](salsa::StorageHandle) for cases where + /// [`with_db`](Self::with_db) isn't sufficient. The handle is cloned to allow + /// concurrent read operations. + /// + /// For mutation operations, use [`with_db_mut`](Self::with_db_mut) instead. /// - /// The StorageHandle can be cloned safely for read operations - /// or moved for mutation operations following Salsa's patterns. + /// # Panics + /// + /// Panics if the handle is currently taken for mutation. #[must_use] - pub fn db_handle(&self) -> &StorageHandle { - &self.db_handle + pub fn db_handle(&self) -> StorageHandle { + self.db_handle.clone_for_read() } } @@ -273,4 +494,275 @@ impl Default for Workspace { fn default() -> Self { Self::new() } -} \ No newline at end of file +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normal_mutation_flow_with_guard() { + let mut workspace = Workspace::new(); + + // Normal mutation should work fine + let result = workspace.with_db_mut(|db| { + // Simple operation - create a file + let path = PathBuf::from("test.py"); + let file = db.get_or_create_file(path); + file.revision(db) // Return the revision number + }); + + // Should complete successfully - initial revision is 0 + assert_eq!(result, 0); + + // Should be able to read after mutation + let file_exists = workspace.with_db(|db| db.has_file(&PathBuf::from("test.py"))); + + assert!(file_exists); + } + + #[test] + fn test_read_access_during_no_mutation() { + let workspace = Workspace::new(); + + // Multiple concurrent reads should work + let handle1 = workspace.db_handle(); + let handle2 = workspace.db_handle(); + + // Both handles should be valid + let storage1 = handle1.into_storage(); + let storage2 = handle2.into_storage(); + + // Should be able to create databases from both + let db1 = Database::from_storage( + storage1, + workspace.file_system.clone(), + workspace.files.clone(), + ); + let db2 = Database::from_storage( + storage2, + workspace.file_system.clone(), + workspace.files.clone(), + ); + + // Both should work + assert!(!db1.has_file(&PathBuf::from("nonexistent.py"))); + assert!(!db2.has_file(&PathBuf::from("nonexistent.py"))); + } + + #[test] + #[should_panic( + expected = "Cannot access database handle for read - handle is currently taken for mutation" + )] + fn test_panic_on_read_during_mutation() { + // This test is tricky due to Rust's borrowing rules. + // Test the SafeStorageHandle directly instead of through Workspace + let mut safe_handle = SafeStorageHandle::new( + Database::new( + Arc::new(crate::fs::WorkspaceFileSystem::new( + crate::buffers::Buffers::new(), + Arc::new(crate::fs::OsFileSystem), + )), + Arc::new(DashMap::new()), + ) + .storage() + .clone() + .into_zalsa_handle(), + ); + + // Take the handle + let _handle = safe_handle.take_for_mutation(); + + // Now trying to read should panic + let _cloned_handle = safe_handle.clone_for_read(); + } + + #[test] + #[should_panic(expected = "Database handle already taken for mutation")] + fn test_panic_on_double_take() { + let mut safe_handle = SafeStorageHandle::new( + Database::new( + Arc::new(crate::fs::WorkspaceFileSystem::new( + crate::buffers::Buffers::new(), + Arc::new(crate::fs::OsFileSystem), + )), + Arc::new(DashMap::new()), + ) + .storage() + .clone() + .into_zalsa_handle(), + ); + + // First take should work + let _handle1 = safe_handle.take_for_mutation(); + + // Second take should panic + let _handle2 = safe_handle.take_for_mutation(); + } + + #[test] + #[should_panic(expected = "StorageHandle already consumed from guard")] + fn test_panic_on_double_handle_consumption() { + let mut workspace = Workspace::new(); + let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); + + // First consumption should work + let _handle1 = guard.handle(); + + // Second consumption should panic + let _handle2 = guard.handle(); + } + + #[test] + fn test_manual_restore() { + let mut workspace = Workspace::new(); + + // Take handle manually + let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); + let handle = guard.handle(); + + // Use it to create a database + let storage = handle.into_storage(); + let mut db = Database::from_storage( + storage, + workspace.file_system.clone(), + workspace.files.clone(), + ); + + // Make some changes + let path = PathBuf::from("manual_test.py"); + let _file = db.get_or_create_file(path); + + // Extract new handle and restore manually + let new_handle = db.storage().clone().into_zalsa_handle(); + guard.restore(new_handle); + + // Should be able to read now + let file_exists = workspace.with_db(|db| db.has_file(&PathBuf::from("manual_test.py"))); + + assert!(file_exists); + } + + #[test] + #[should_panic(expected = "StorageHandleGuard dropped without restoring handle")] + fn test_panic_on_guard_drop_without_restore() { + let mut workspace = Workspace::new(); + + // Create guard and consume handle but don't restore + let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); + let _handle = guard.handle(); + + // Guard drops here without restore - should panic + } + + #[test] + fn test_event_callbacks_preserved() { + // This test ensures that the new implementation preserves event callbacks + // through mutation cycles, unlike the old placeholder approach + + let mut workspace = Workspace::new(); + + // Add a file to create some state + let initial_file_count = workspace.with_db_mut(|db| { + let path = PathBuf::from("callback_test.py"); + let _file = db.get_or_create_file(path); + 1 // Return count + }); + + assert_eq!(initial_file_count, 1); + + // Perform another mutation to ensure callbacks are preserved + let final_file_count = workspace.with_db_mut(|db| { + let path = PathBuf::from("callback_test2.py"); + let _file = db.get_or_create_file(path); + + // Count files - should include both + let has_first = db.has_file(&PathBuf::from("callback_test.py")); + let has_second = db.has_file(&PathBuf::from("callback_test2.py")); + + if has_first && has_second { + 2 + } else { + 0 + } + }); + + assert_eq!(final_file_count, 2); + } + + #[test] + fn test_concurrent_read_operations() { + let workspace = Workspace::new(); + + // Multiple with_db calls should work concurrently + let result1 = workspace.with_db(|db| db.has_file(&PathBuf::from("test1.py"))); + + let result2 = workspace.with_db(|db| db.has_file(&PathBuf::from("test2.py"))); + + // Both should complete successfully + assert!(!result1); + assert!(!result2); + } + + #[test] + fn test_safe_storage_handle_state_transitions() { + let mut workspace = Workspace::new(); + + // Start in Available state - should be able to clone for read + let _handle = workspace.db_handle(); + + // Take for mutation + let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); + let handle = guard.handle(); + + // Now should be in TakenForMutation state + // Convert to storage for testing + let storage = handle.into_storage(); + let db = Database::from_storage( + storage, + workspace.file_system.clone(), + workspace.files.clone(), + ); + let new_handle = db.storage().clone().into_zalsa_handle(); + + // Restore - should return to Available state + guard.restore(new_handle); + + // Should be able to read again + let _handle = workspace.db_handle(); + } + + #[test] + #[should_panic(expected = "Cannot restore handle - it hasn't been consumed yet")] + fn test_panic_on_restore_without_consume() { + let mut workspace = Workspace::new(); + let guard = StorageHandleGuard::new(&mut workspace.db_handle); + + // Create a dummy handle for testing + let dummy_handle = Database::new( + Arc::new(crate::fs::WorkspaceFileSystem::new( + crate::buffers::Buffers::new(), + Arc::new(crate::fs::OsFileSystem), + )), + Arc::new(DashMap::new()), + ) + .storage() + .clone() + .into_zalsa_handle(); + + // Try to restore without consuming first - should panic + guard.restore(dummy_handle); + } + + #[test] + #[should_panic(expected = "StorageHandleGuard dropped without using the handle")] + fn test_panic_on_guard_drop_without_use() { + let mut workspace = Workspace::new(); + + // Create guard but don't use the handle - should panic on drop + let _guard = StorageHandleGuard::new(&mut workspace.db_handle); + + // Guard drops here without handle() being called + } +} + From 8ba72392fc2c0fd22d012a60ca3efcb55bd548db Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 16:29:21 -0500 Subject: [PATCH 40/56] move it! --- crates/djls-server/src/session.rs | 381 +++---------------------- crates/djls-workspace/src/workspace.rs | 162 ++++++++++- 2 files changed, 195 insertions(+), 348 deletions(-) diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 126bae64..125de7e2 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,86 +1,32 @@ -//! # Salsa [`StorageHandle`] Pattern for LSP +//! # LSP Session Management //! -//! This module implements a thread-safe Salsa database wrapper for use with -//! tower-lsp's async runtime. The key challenge is that tower-lsp requires -//! `Send + Sync + 'static` bounds, but Salsa's `Storage` contains thread-local -//! state and is not `Send`. -//! -//! ## The Solution: [`StorageHandle`] -//! -//! Salsa provides [`StorageHandle`] which IS `Send + Sync` because it contains -//! no thread-local state. We store the handle and create `Storage`/`Database` -//! instances on-demand. -//! -//! ## The Mutation Challenge -//! -//! When mutating Salsa inputs (e.g., updating file revisions), Salsa must -//! ensure exclusive access to prevent race conditions. It does this via -//! `cancel_others()` which: -//! -//! 1. Sets a cancellation flag (causes other threads to panic with `Cancelled`) -//! 2. Waits for all `StorageHandle` clones to drop -//! 3. Proceeds with the mutation -//! -//! If we accidentally clone the handle instead of taking ownership, step 2 -//! never completes → deadlock! -//! -//! ## The Pattern -//! -//! - **Reads**: Clone the handle freely ([`with_db`](Session::with_db)) -//! - **Mutations**: Take exclusive ownership ([`with_db_mut`](Session::with_db_mut) via [`take_db_handle_for_mutation`](Session::take_db_handle_for_mutation)) -//! -//! The explicit method names make the intent clear and prevent accidental misuse. -//! -//! [`StorageHandle`]: salsa::StorageHandle +//! This module implements the LSP session abstraction that manages project-specific +//! state and delegates workspace operations to the Workspace facade. use std::path::Path; use std::path::PathBuf; -use std::sync::Arc; -use dashmap::DashMap; use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::db::source_text; use djls_workspace::db::Database; -use djls_workspace::db::SourceFile; use djls_workspace::paths; -use djls_workspace::Buffers; -use djls_workspace::FileSystem; -use djls_workspace::OsFileSystem; +use djls_workspace::PositionEncoding; use djls_workspace::TextDocument; -use djls_workspace::WorkspaceFileSystem; -use salsa::StorageHandle; +use djls_workspace::Workspace; use tower_lsp_server::lsp_types; use url::Url; -use djls_workspace::PositionEncoding; - -/// LSP Session with thread-safe Salsa database access. -/// -/// Uses Salsa's [`StorageHandle`] pattern to maintain `Send + Sync + 'static` -/// compatibility required by tower-lsp. The handle can be safely shared -/// across threads and async boundaries. -/// -/// See [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) -/// for more information about [`StorageHandle`]. -/// -/// ## Architecture -/// -/// Two-layer system inspired by Ruff/Ty: -/// - **Layer 1**: In-memory overlays (LSP document edits) -/// - **Layer 2**: Salsa database (incremental computation cache) +/// LSP Session managing project-specific state and workspace operations. /// -/// ## Salsa Mutation Protocol +/// The Session serves as the main entry point for LSP operations, managing: +/// - Project configuration and settings +/// - Client capabilities and position encoding +/// - Workspace operations (delegated to the Workspace facade) /// -/// When mutating Salsa inputs (like changing file revisions), we must ensure -/// exclusive access to prevent race conditions. Salsa enforces this through -/// its `cancel_others()` mechanism, which waits for all [`StorageHandle`] clones -/// to drop before allowing mutations. -/// -/// We use explicit methods (`take_db_handle_for_mutation`/`restore_db_handle`) -/// to make this ownership transfer clear and prevent accidental deadlocks. -/// -/// [`StorageHandle`]: salsa::StorageHandle +/// All document lifecycle and database operations are delegated to the +/// encapsulated Workspace, which provides thread-safe Salsa database +/// management with proper mutation safety through StorageHandleGuard. pub struct Session { /// The Django project configuration project: Option, @@ -88,48 +34,18 @@ pub struct Session { /// LSP server settings settings: Settings, - /// Layer 1: Shared buffer storage for open documents - /// - /// This implements Ruff's two-layer architecture where Layer 1 contains - /// open document buffers that take precedence over disk files. The buffers - /// are shared between Session (which manages them) and - /// [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) (which reads from them). - /// - /// Key properties: - /// - Thread-safe via the Buffers abstraction - /// - Contains full [`TextDocument`](djls_workspace::TextDocument) with content, version, and metadata - /// - Never becomes Salsa inputs - only intercepted at read time - buffers: Buffers, - - /// File system abstraction with buffer interception + /// Workspace facade that encapsulates all workspace-related functionality /// - /// This [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) bridges Layer 1 (buffers) and Layer 2 (Salsa). - /// It intercepts [`FileSystem::read_to_string()`](djls_workspace::FileSystem::read_to_string()) calls to return buffer - /// content when available, falling back to disk otherwise. - file_system: Arc, - - /// Shared file tracking across all Database instances - /// - /// This is the canonical Salsa pattern from the lazy-input example. - /// The [`DashMap`] provides O(1) lookups and is shared via Arc across - /// all Database instances created from [`StorageHandle`](salsa::StorageHandle). - files: Arc>, + /// This includes document buffers, file system abstraction, and the Salsa database. + /// The workspace provides a clean interface for document lifecycle management + /// and database operations while maintaining proper isolation and thread safety. + workspace: Workspace, #[allow(dead_code)] client_capabilities: lsp_types::ClientCapabilities, /// Position encoding negotiated with client position_encoding: PositionEncoding, - - /// Layer 2: Thread-safe Salsa database handle for pure computation - /// - /// where we're using the [`StorageHandle`](salsa::StorageHandle) to create a thread-safe handle that can be - /// shared between threads. - /// - /// The database receives file content via the [`FileSystem`](djls_workspace::FileSystem) trait, which - /// is intercepted by our [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) to provide overlay content. - /// This maintains proper separation between Layer 1 and Layer 2. - db_handle: StorageHandle, } impl Session { @@ -147,16 +63,7 @@ impl Session { (None, Settings::default()) }; - let buffers = Buffers::new(); - let files = Arc::new(DashMap::new()); - let file_system = Arc::new(WorkspaceFileSystem::new( - buffers.clone(), - Arc::new(OsFileSystem), - )); - let db_handle = Database::new(file_system.clone(), files.clone()) - .storage() - .clone() - .into_zalsa_handle(); + let workspace = Workspace::new(); // Negotiate position encoding with client let position_encoding = PositionEncoding::negotiate(params); @@ -164,12 +71,9 @@ impl Session { Self { project, settings, - buffers, - file_system, - files, + workspace, client_capabilities: params.capabilities.clone(), position_encoding, - db_handle, } } /// Determines the project root path from initialization parameters. @@ -211,254 +115,62 @@ impl Session { self.position_encoding } - // TODO: Explore an abstraction around [`salsa::StorageHandle`] and the following two methods - // to make it easy in the future to avoid deadlocks. For now, this is simpler and TBH may be - // all we ever need, but still.. might be a nice CYA for future me - - /// Takes exclusive ownership of the database handle for mutation operations. - /// - /// This method extracts the [`StorageHandle`](salsa::StorageHandle) from the session, replacing it - /// with a temporary placeholder. This ensures there's exactly one handle - /// active during mutations, preventing deadlocks in Salsa's `cancel_others()`. - /// - /// ## Why Not Clone? - /// - /// Cloning would create multiple handles. When Salsa needs to mutate inputs, - /// it calls `cancel_others()` which waits for all handles to drop. With - /// multiple handles, this wait would never complete → deadlock. - /// - /// ## Panics - /// - /// This is an internal method that should only be called by - /// [`with_db_mut`](Session::with_db_mut). Multiple concurrent calls would panic when trying - /// to take an already-taken handle. - /// - /// ## Safety Note on Placeholder Handle - /// - /// This method uses `StorageHandle::new(None)` as a temporary placeholder, which - /// creates a new Salsa instance without event callbacks. While this could theoretically - /// lose state if called concurrently, the outer `Arc>>` at the - /// server level ensures this method is only called with exclusive access to the Session. - /// - /// The placeholder is immediately replaced when `restore_db_handle()` is called at the - /// end of the mutation operation, so no actual state is lost. - /// - /// A future improvement (see TODO above) would be to implement a `StorageHandleGuard` - /// abstraction that makes these state transitions more explicit and type-safe. See - /// task-152 for the planned implementation. - fn take_db_handle_for_mutation(&mut self) -> StorageHandle { - std::mem::replace(&mut self.db_handle, StorageHandle::new(None)) - } - /// Restores the database handle after a mutation operation completes. - /// - /// This should be called with the handle extracted from the database - /// after mutations are complete. It updates the session's handle to - /// reflect any changes made during the mutation. - fn restore_db_handle(&mut self, handle: StorageHandle) { - self.db_handle = handle; - } /// Execute a closure with mutable access to the database. /// - /// This method implements Salsa's required protocol for mutations: - /// 1. Takes exclusive ownership of the [`StorageHandle`](salsa::StorageHandle) - /// (no clones exist) - /// 2. Creates a temporary Database for the operation - /// 3. Executes your closure with `&mut Database` - /// 4. Extracts and restores the updated handle - /// - /// ## Example - /// - /// ```rust,ignore - /// session.with_db_mut(|db| { - /// let file = db.get_or_create_file(path); - /// file.set_revision(db).to(new_revision); // Mutation requires exclusive access - /// }); - /// ``` - /// - /// ## Why This Pattern? - /// - /// This ensures that when Salsa needs to modify inputs (via setters like - /// `set_revision`), it has exclusive access. The internal `cancel_others()` - /// call will succeed because we guarantee only one handle exists. + /// Delegates to the workspace's safe database mutation mechanism. pub fn with_db_mut(&mut self, f: F) -> R where F: FnOnce(&mut Database) -> R, { - let handle = self.take_db_handle_for_mutation(); - - let storage = handle.into_storage(); - let mut db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); - - let result = f(&mut db); - - // The database may have changed during mutations, so we need - // to extract its current handle state - let new_handle = db.storage().clone().into_zalsa_handle(); - self.restore_db_handle(new_handle); - - result + self.workspace.with_db_mut(f) } /// Execute a closure with read-only access to the database. /// - /// For read-only operations, we can safely clone the [`StorageHandle`](salsa::StorageHandle) - /// since Salsa allows multiple concurrent readers. This is more - /// efficient than taking exclusive ownership. - /// - /// ## Example - /// - /// ```rust,ignore - /// let content = session.with_db(|db| { - /// let file = db.get_file(path)?; - /// source_text(db, file).to_string() // Read-only query - /// }); - /// ``` + /// Delegates to the workspace's safe database read mechanism. pub fn with_db(&self, f: F) -> R where F: FnOnce(&Database) -> R, { - // For reads, cloning is safe and efficient - let storage = self.db_handle.clone().into_storage(); - let db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); - f(&db) + self.workspace.with_db(f) } /// Handle opening a document - sets buffer and creates file. /// - /// This method coordinates both layers: - /// - Layer 1: Stores the document content in buffers - /// - Layer 2: Creates the [`SourceFile`](djls_workspace::SourceFile) in Salsa (if path is resolvable) + /// Delegates to the workspace's document management. pub fn open_document(&mut self, url: &Url, document: TextDocument) { tracing::debug!("Opening document: {}", url); - - // Layer 1: Set buffer - self.buffers.open(url.clone(), document); - - // Layer 2: Create file and touch if it already exists - // This is crucial: if the file was already read from disk, we need to - // invalidate Salsa's cache so it re-reads through the buffer system - if let Some(path) = paths::url_to_path(url) { - self.with_db_mut(|db| { - // Check if file already exists (was previously read from disk) - let already_exists = db.has_file(&path); - let file = db.get_or_create_file(path.clone()); - - if already_exists { - // File was already read - touch to invalidate cache - db.touch_file(&path); - } else { - // New file - starts at revision 0 - tracing::debug!( - "Created new SourceFile for {}: revision {}", - path.display(), - file.revision(db) - ); - } - }); - } + self.workspace.open_document(url, document); } /// Update a document with the given changes. /// - /// This method handles both incremental updates and full document replacements, - /// coordinating both layers of the architecture: - /// - Layer 1: Updates the document content in buffers - /// - Layer 2: Bumps the file revision to trigger Salsa invalidation - /// - /// If the document is not currently open (no buffer exists), this method will - /// attempt a fallback recovery by using the first content change as a full - /// document replacement, preserving the existing language_id if possible. + /// Delegates to the workspace's document management. pub fn update_document( &mut self, url: &Url, changes: Vec, new_version: i32, ) { - // Try to apply changes to existing document - if let Some(mut document) = self.buffers.get(url) { - // Document exists - apply incremental changes - document.update(changes, new_version); - - let version = document.version(); - tracing::debug!("Updating document: {} (version {})", url, version); - - // Layer 1: Update buffer - self.buffers.update(url.clone(), document); - - // Layer 2: Touch file to trigger invalidation - if let Some(path) = paths::url_to_path(url) { - self.with_db_mut(|db| db.touch_file(&path)); - } - } else { - // Document not open - attempt fallback recovery - tracing::warn!("Document not open: {}, attempting fallback recovery", url); - - // Use first change as full content replacement - if let Some(change) = changes.into_iter().next() { - // Preserve existing language_id if document was previously opened - // This handles the case where we get changes for a document that - // somehow lost its buffer but we want to maintain its type - let language_id = self - .get_document(url) - .map_or(djls_workspace::LanguageId::Other, |doc| doc.language_id()); - - let document = - djls_workspace::TextDocument::new(change.text, new_version, language_id); - - tracing::debug!( - "Fallback: creating document {} (version {})", - url, - new_version - ); - - // Layer 1: Update buffer - self.buffers.update(url.clone(), document); - - // Layer 2: Touch file to trigger invalidation - if let Some(path) = paths::url_to_path(url) { - self.with_db_mut(|db| db.touch_file(&path)); - } - } - } + self.workspace.update_document(url, changes, new_version); } /// Handle closing a document - removes buffer and bumps revision. /// - /// This method coordinates both layers: - /// - Layer 1: Removes the buffer (falls back to disk) - /// - Layer 2: Bumps revision to trigger re-read from disk - /// - /// Returns the removed document if it existed. + /// Delegates to the workspace's document management. pub fn close_document(&mut self, url: &Url) -> Option { tracing::debug!("Closing document: {}", url); - - // Layer 1: Remove buffer (falls back to disk) - let document = self.buffers.close(url); - if let Some(ref doc) = document { - tracing::debug!( - "Removed buffer for closed document: {} (was version {})", - url, - doc.version() - ); - } - - // Layer 2: Touch file to trigger re-read from disk - if let Some(path) = paths::url_to_path(url) { - self.with_db_mut(|db| db.touch_file(&path)); - } - - document + self.workspace.close_document(url) } /// Get an open document from the buffer layer, if it exists. /// - /// This provides read-only access to Layer 1 (buffer) documents. - /// Returns None if the document is not currently open in the editor. + /// Delegates to the workspace's document management. #[must_use] pub fn get_document(&self, url: &Url) -> Option { - self.buffers.get(url) + self.workspace.get_document(url) } /// Get the current content of a file (from overlay or disk). @@ -476,42 +188,23 @@ impl Session { /// Get the current revision of a file, if it's being tracked. /// /// Returns None if the file hasn't been created yet. - pub fn file_revision(&mut self, path: &Path) -> Option { - self.with_db_mut(|db| { - db.has_file(path).then(|| { - let file = db.get_or_create_file(path.to_path_buf()); - file.revision(db) - }) - }) + pub fn file_revision(&self, path: &Path) -> Option { + self.workspace.file_revision(path) } /// Check if a file is currently being tracked in Salsa. - pub fn has_file(&mut self, path: &Path) -> bool { + pub fn has_file(&self, path: &Path) -> bool { self.with_db(|db| db.has_file(path)) } } impl Default for Session { fn default() -> Self { - let buffers = Buffers::new(); - let files = Arc::new(DashMap::new()); - let file_system = Arc::new(WorkspaceFileSystem::new( - buffers.clone(), - Arc::new(OsFileSystem), - )); - let db_handle = Database::new(file_system.clone(), files.clone()) - .storage() - .clone() - .into_zalsa_handle(); - Self { project: None, settings: Settings::default(), - db_handle, - file_system, - files, - buffers, - client_capabilities: lsp_types::ClientCapabilities::default(), + workspace: Workspace::new(), + client_capabilities: Default::default(), position_encoding: PositionEncoding::default(), } } diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index f225ead9..2a067373 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -315,15 +315,26 @@ impl Workspace { /// Open a document in the workspace. /// - /// Updates both the buffer layer and database layer. If the file exists - /// in the database, it's marked as touched to trigger invalidation. + /// Updates both the buffer layer and database layer. Creates the file in + /// the database or invalidates it if it already exists. pub fn open_document(&mut self, url: &Url, document: TextDocument) { // Layer 1: Add to buffers self.buffers.open(url.clone(), document); - // Layer 2: Update database if file exists + // Layer 2: Create file and touch if it already exists + // This matches the original Session behavior for test compatibility if let Some(path) = url_to_path(url) { - self.invalidate_file_if_exists(&path); + self.with_db_mut(|db| { + // Check if file already exists (was previously read from disk) + let already_exists = db.has_file(&path); + let _file = db.get_or_create_file(path.clone()); + + if already_exists { + // File was already read - touch to invalidate cache + db.touch_file(&path); + } + // Note: New files automatically start at revision 0, no additional action needed + }); } } @@ -499,6 +510,10 @@ impl Default for Workspace { #[cfg(test)] mod tests { use super::*; + use crate::db::source_text; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + use tempfile::tempdir; #[test] fn test_normal_mutation_flow_with_guard() { @@ -764,5 +779,144 @@ mod tests { // Guard drops here without handle() being called } + + // Error Recovery Tests + + #[test] + fn test_missing_file_returns_empty_content() { + // Tests that source_text returns "" for non-existent files + // instead of panicking or propagating errors + let mut workspace = Workspace::new(); + + // Create a file reference for non-existent path + let content = workspace.with_db_mut(|db| { + let file = db.get_or_create_file("/nonexistent/file.py".into()); + source_text(db, file).to_string() + }); + + assert_eq!(content, ""); + } + + #[test] + #[cfg(unix)] // Only on Unix systems + fn test_permission_denied_file_handling() { + // Create a file with no read permissions + let temp_dir = tempdir().unwrap(); + let file_path = temp_dir.path().join("no_read.py"); + std::fs::write(&file_path, "content").unwrap(); + + // Remove read permissions + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&file_path, + std::fs::Permissions::from_mode(0o000)).unwrap(); + + let mut workspace = Workspace::new(); + let content = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(file_path.clone()); + source_text(db, file).to_string() + }); + + // Should return empty string, not panic + assert_eq!(content, ""); + } + + #[test] + fn test_invalid_utf8_file_handling() { + // Create a file with invalid UTF-8 + let temp_dir = tempdir().unwrap(); + let file_path = temp_dir.path().join("invalid.py"); + std::fs::write(&file_path, &[0xFF, 0xFE, 0xFD]).unwrap(); + + let mut workspace = Workspace::new(); + let content = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(file_path.clone()); + source_text(db, file).to_string() + }); + + // Should handle gracefully (empty or replacement chars) + assert!(content.is_empty() || content.contains('�')); + } + + #[test] + fn test_file_deleted_after_tracking() { + let temp_dir = tempdir().unwrap(); + let file_path = temp_dir.path().join("disappearing.py"); + std::fs::write(&file_path, "original").unwrap(); + + let mut workspace = Workspace::new(); + + // First read should succeed + let content1 = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(file_path.clone()); + source_text(db, file).to_string() + }); + assert_eq!(content1, "original"); + + // Delete the file + std::fs::remove_file(&file_path).unwrap(); + + // Touch to invalidate cache + workspace.with_db_mut(|db| { + db.touch_file(&file_path); + }); + + // Second read should return empty (not panic) + let content2 = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(file_path.clone()); + source_text(db, file).to_string() + }); + assert_eq!(content2, ""); + } + + #[test] + #[cfg(unix)] + fn test_broken_symlink_handling() { + let temp_dir = tempdir().unwrap(); + let symlink_path = temp_dir.path().join("broken_link.py"); + + // Create broken symlink + std::os::unix::fs::symlink("/nonexistent/target", &symlink_path).unwrap(); + + let mut workspace = Workspace::new(); + let content = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(symlink_path.clone()); + source_text(db, file).to_string() + }); + + // Should handle gracefully + assert_eq!(content, ""); + } + + #[test] + fn test_file_modified_during_operations() { + // Tests that concurrent file modifications don't crash + let temp_dir = tempdir().unwrap(); + let file_path = temp_dir.path().join("racing.py"); + + let workspace = Arc::new(Mutex::new(Workspace::new())); + let path_clone = file_path.clone(); + let workspace_clone = workspace.clone(); + + // Writer thread + let writer = std::thread::spawn(move || { + for i in 0..10 { + std::fs::write(&path_clone, format!("version {}", i)).ok(); + std::thread::sleep(Duration::from_millis(10)); + } + }); + + // Reader thread - should never panic + for _ in 0..10 { + let content = workspace_clone.lock().unwrap().with_db_mut(|db| { + let file = db.get_or_create_file(file_path.clone()); + source_text(db, file).to_string() + }); + // Content may vary but shouldn't crash + assert!(content.is_empty() || content.starts_with("version")); + std::thread::sleep(Duration::from_millis(5)); + } + + writer.join().unwrap(); + } } From 2faa361ab4b647215accf730bc0ac16a47d33b27 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 17:09:40 -0500 Subject: [PATCH 41/56] lint and stuff --- crates/djls-server/src/completions.rs | 1 - crates/djls-server/src/session.rs | 9 +- crates/djls-server/tests/lsp_integration.rs | 4 +- crates/djls-workspace/src/buffers.rs | 5 +- crates/djls-workspace/src/db.rs | 11 +- crates/djls-workspace/src/workspace.rs | 227 +++++++++----------- 6 files changed, 115 insertions(+), 142 deletions(-) diff --git a/crates/djls-server/src/completions.rs b/crates/djls-server/src/completions.rs index 46f8d6eb..f5c6aeef 100644 --- a/crates/djls-server/src/completions.rs +++ b/crates/djls-server/src/completions.rs @@ -283,4 +283,3 @@ mod tests { assert!(completions.is_empty()); } } - diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 125de7e2..710fbd11 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -115,8 +115,6 @@ impl Session { self.position_encoding } - - /// Execute a closure with mutable access to the database. /// /// Delegates to the workspace's safe database mutation mechanism. @@ -180,7 +178,7 @@ impl Session { /// through the `FileSystem` abstraction (overlay first, then disk). pub fn file_content(&mut self, path: PathBuf) -> String { self.with_db_mut(|db| { - let file = db.get_or_create_file(path); + let file = db.get_or_create_file(&path); source_text(db, file).to_string() }) } @@ -189,7 +187,10 @@ impl Session { /// /// Returns None if the file hasn't been created yet. pub fn file_revision(&self, path: &Path) -> Option { - self.workspace.file_revision(path) + { + let this = &self.workspace; + this.with_db(|db| db.get_file(path).map(|file| file.revision(db))) + } } /// Check if a file is currently being tracked in Salsa. diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index a30ec1ec..00201262 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use std::sync::Arc; use djls_server::DjangoLanguageServer; +use djls_workspace::db::parse_template; use tempfile::TempDir; use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; @@ -263,7 +264,6 @@ async fn test_template_parsing_with_overlays() { 1, ) .await; - use djls_workspace::db::parse_template; // Parse template through the session let workspace_path = server.workspace_file(file_name); @@ -271,7 +271,7 @@ async fn test_template_parsing_with_overlays() { .server .with_session_mut(|session| { session.with_db_mut(|db| { - let file = db.get_or_create_file(workspace_path); + let file = db.get_or_create_file(&workspace_path); parse_template(db, file) }) }) diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 6f26ad16..1e3fd0fe 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -2,9 +2,10 @@ //! //! This module provides the [`Buffers`] type which represents the in-memory //! content of open files. These buffers are shared between the `Session` -//! (which manages document lifecycle) and the [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem) (which +//! (which manages document lifecycle) and the [`WorkspaceFileSystem`] (which //! reads from them). - +/// +/// [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem use std::sync::Arc; use dashmap::DashMap; diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index db335639..b3cf2ef2 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -69,6 +69,7 @@ pub struct Database { // The logs are only used for testing and demonstrating reuse: #[cfg(test)] + #[allow(dead_code)] logs: Arc>>>, } @@ -141,8 +142,8 @@ impl Database { /// /// Files are created with an initial revision of 0 and tracked in the [`Database`]'s /// `DashMap`. The `Arc` ensures cheap cloning while maintaining thread safety. - pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { - if let Some(file_ref) = self.files.get(&path) { + pub fn get_or_create_file(&mut self, path: &PathBuf) -> SourceFile { + if let Some(file_ref) = self.files.get(path) { // Copy the value (SourceFile is Copy) and drop the guard immediately let file = *file_ref; drop(file_ref); // Explicitly drop the guard to release the lock @@ -150,7 +151,7 @@ impl Database { } // File doesn't exist, so we need to create it - let kind = FileKind::from_path(&path); + let kind = FileKind::from_path(path); let file = SourceFile::new(self, kind, Arc::from(path.to_string_lossy().as_ref()), 0); self.files.insert(path.clone(), file); @@ -354,7 +355,7 @@ mod tests { let mut db = Database::new(file_system, files); // Create a SourceFile for the template - let file = db.get_or_create_file(template_path.clone()); + let file = db.get_or_create_file(&template_path); // Parse template - should get original content from disk let ast1 = parse_template(&db, file).expect("Should parse template"); @@ -410,7 +411,7 @@ mod tests { let mut db = Database::new(file_system, files); // Create a SourceFile for the template - let file = db.get_or_create_file(template_path.clone()); + let file = db.get_or_create_file(&template_path); // Parse template first time let ast1 = parse_template(&db, file).expect("Should parse"); diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index 2a067373..bcd8b11a 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -18,7 +18,7 @@ use crate::document::TextDocument; use crate::fs::{OsFileSystem, WorkspaceFileSystem}; use crate::paths::url_to_path; -/// Safe wrapper for StorageHandle that prevents misuse through type safety. +/// Safe wrapper for [`StorageHandle`](salsa::StorageHandle) that prevents misuse through type safety. /// /// This enum ensures that database handles can only be in one of two valid states, /// making invalid states unrepresentable and eliminating the need for placeholder @@ -31,14 +31,14 @@ enum SafeStorageHandle { } impl SafeStorageHandle { - /// Create a new SafeStorageHandle in the Available state + /// Create a new `SafeStorageHandle` in the `Available` state fn new(handle: StorageHandle) -> Self { Self::Available(handle) } - /// Take the handle for mutation, leaving the enum in TakenForMutation state. + /// Take the handle for mutation, leaving the enum in `TakenForMutation` state. /// - /// # Panics + /// ## Panics /// /// Panics if the handle has already been taken for mutation. fn take_for_mutation(&mut self) -> StorageHandle { @@ -52,9 +52,9 @@ impl SafeStorageHandle { } } - /// Restore the handle after mutation, returning it to Available state. + /// Restore the handle after mutation, returning it to `Available` state. /// - /// # Panics + /// ## Panics /// /// Panics if the handle is not currently taken for mutation. fn restore_from_mutation(&mut self, handle: StorageHandle) { @@ -71,7 +71,7 @@ impl SafeStorageHandle { /// Get a clone of the handle for read-only operations. /// - /// # Panics + /// ## Panics /// /// Panics if the handle is currently taken for mutation. fn clone_for_read(&self) -> StorageHandle { @@ -85,41 +85,19 @@ impl SafeStorageHandle { } } -/// State of the StorageHandleGuard during its lifetime. +/// State of the [`StorageHandleGuard`] during its lifetime. /// -/// This enum captures the exact state of the guard and the handle it manages, -/// making state transitions explicit and preventing invalid combinations. -/// -/// ## State Machine -/// -/// The valid state transitions are: -/// - `Active` → `Consumed` (via `handle()`) -/// - `Consumed` → `Restored` (via `restore()`) -/// -/// Invalid transitions will panic with specific error messages: -/// - `handle()` on `Consumed` state: "StorageHandle already consumed" -/// - `handle()` on `Restored` state: "Cannot consume handle - guard has already been restored" -/// - `restore()` on `Active` state: "Cannot restore handle - it hasn't been consumed yet" -/// - `restore()` on `Restored` state: "Handle has already been restored" -/// -/// ## Drop Behavior -/// -/// The guard will panic on drop unless it's in the `Restored` state: -/// - Drop in `Active` state: "StorageHandleGuard dropped without using the handle" -/// - Drop in `Consumed` state: "StorageHandleGuard dropped without restoring handle" -/// - Drop in `Restored` state: No panic - proper cleanup +/// See [`StorageHandleGuard`] for usage and state machine details. enum GuardState { - /// Guard holds the handle and it's ready to be consumed via `handle()` + /// Guard holds the handle, ready to be consumed Active { handle: StorageHandle }, - - /// Handle has been consumed via `handle()` method, awaiting restoration via `restore()` + /// Handle consumed, awaiting restoration Consumed, - - /// Handle has been properly restored to the SafeStorageHandle - guard is complete + /// Handle restored to [`SafeStorageHandle`] Restored, } -/// RAII guard for safe StorageHandle management during mutations. +/// RAII guard for safe [`StorageHandle`](salsa::StorageHandle) management during mutations. /// /// This guard ensures that database handles are automatically restored even if /// panics occur during mutation operations. It prevents double-takes and @@ -127,20 +105,44 @@ enum GuardState { /// /// ## State Machine /// -/// The guard follows a clear state machine: -/// - `Active` → `Consumed` (via `handle()`) -/// - `Consumed` → `Restored` (via `restore()`) -/// - `Active` → `Restored` (direct restore without consuming - future extension) +/// The guard follows these valid state transitions: +/// - `Active` → `Consumed` (via `handle()` method) +/// - `Consumed` → `Restored` (via `restore()` method) +/// +/// ## Invalid Transitions +/// +/// Invalid operations will panic with specific error messages: +/// - `handle()` on `Consumed` state: "[`StorageHandle`](salsa::StorageHandle) already consumed" +/// - `handle()` on `Restored` state: "Cannot consume handle - guard has already been restored" +/// - `restore()` on `Active` state: "Cannot restore handle - it hasn't been consumed yet" +/// - `restore()` on `Restored` state: "Handle has already been restored" +/// +/// ## Drop Behavior +/// +/// The guard will panic on drop unless it's in the `Restored` state: +/// - Drop in `Active` state: "`StorageHandleGuard` dropped without using the handle" +/// - Drop in `Consumed` state: "`StorageHandleGuard` dropped without restoring handle" +/// - Drop in `Restored` state: No panic - proper cleanup completed +/// +/// ## Usage Example +/// +/// ```rust,ignore +/// let mut guard = StorageHandleGuard::new(&mut safe_handle); +/// let handle = guard.handle(); // Active → Consumed +/// // ... perform mutations with handle ... +/// guard.restore(updated_handle); // Consumed → Restored +/// // Guard drops cleanly in Restored state +/// ``` #[must_use = "StorageHandleGuard must be used - dropping it immediately defeats the purpose"] pub struct StorageHandleGuard<'a> { - /// Reference to the workspace's SafeStorageHandle for restoration + /// Reference to the workspace's `SafeStorageHandle` for restoration safe_handle: &'a mut SafeStorageHandle, /// Current state of the guard and handle state: GuardState, } impl<'a> StorageHandleGuard<'a> { - /// Create a new guard by taking the handle from the SafeStorageHandle. + /// Create a new guard by taking the handle from the `SafeStorageHandle`. fn new(safe_handle: &'a mut SafeStorageHandle) -> Self { let handle = safe_handle.take_for_mutation(); Self { @@ -149,9 +151,9 @@ impl<'a> StorageHandleGuard<'a> { } } - /// Get the StorageHandle for mutation operations. + /// Get the [`StorageHandle`](salsa::StorageHandle) for mutation operations. /// - /// # Panics + /// ## Panics /// /// Panics if the handle has already been consumed or restored. pub fn handle(&mut self) -> StorageHandle { @@ -173,7 +175,7 @@ impl<'a> StorageHandleGuard<'a> { /// This is useful when you want to restore the handle and continue using /// the workspace in the same scope. /// - /// # Panics + /// ## Panics /// /// Panics if the handle hasn't been consumed yet, or if already restored. pub fn restore(mut self, handle: StorageHandle) { @@ -193,7 +195,7 @@ impl<'a> StorageHandleGuard<'a> { } } -impl<'a> Drop for StorageHandleGuard<'a> { +impl Drop for StorageHandleGuard<'_> { fn drop(&mut self) { // Provide specific error messages based on the exact state // Avoid double-panic during unwinding @@ -225,37 +227,20 @@ impl<'a> Drop for StorageHandleGuard<'a> { /// /// This struct provides a unified interface for managing workspace state, /// including in-memory buffers, file system abstraction, file tracking, -/// and the Salsa database handle. It follows the same initialization pattern -/// as the Session but encapsulates it in a reusable component. -/// -/// ## Components -/// -/// - **Buffers**: Thread-safe storage for open document content -/// - **WorkspaceFileSystem**: File system abstraction with buffer interception -/// - **Files**: Shared file tracking across all Database instances -/// - **Database Handle**: Thread-safe Salsa database handle for incremental computation +/// and the Salsa database handle. pub struct Workspace { - /// Layer 1: Shared buffer storage for open documents + /// Thread-safe shared buffer storage for open documents buffers: Buffers, - /// File system abstraction with buffer interception file_system: Arc, - /// Shared file tracking across all Database instances files: Arc>, - - /// Layer 2: Thread-safe Salsa database handle with safe mutation management + /// Thread-safe Salsa database handle for incremental computation with safe mutation management db_handle: SafeStorageHandle, } impl Workspace { - /// Create a new Workspace with all components initialized. - /// - /// This follows the same initialization pattern as Session: - /// 1. Creates Buffers for in-memory document storage - /// 2. Creates shared file tracking DashMap - /// 3. Creates WorkspaceFileSystem with buffer interception - /// 4. Initializes Database and extracts StorageHandle + /// Create a new [`Workspace`] with all components initialized. #[must_use] pub fn new() -> Self { let buffers = Buffers::new(); @@ -277,8 +262,6 @@ impl Workspace { } } - // Database Access Methods (AC #1) - /// Execute a read-only operation with access to the database. /// /// Creates a temporary Database instance from the handle for the closure. @@ -295,7 +278,7 @@ impl Workspace { /// Execute a mutable operation with exclusive access to the database. /// - /// Uses the StorageHandleGuard pattern to ensure the handle is safely restored + /// Uses the `StorageHandleGuard` pattern to ensure the handle is safely restored /// even if the operation panics. This eliminates the need for placeholder handles. pub fn with_db_mut(&mut self, f: F) -> R where @@ -311,8 +294,6 @@ impl Workspace { result } - // Document Lifecycle Methods (AC #2) - /// Open a document in the workspace. /// /// Updates both the buffer layer and database layer. Creates the file in @@ -322,12 +303,11 @@ impl Workspace { self.buffers.open(url.clone(), document); // Layer 2: Create file and touch if it already exists - // This matches the original Session behavior for test compatibility if let Some(path) = url_to_path(url) { self.with_db_mut(|db| { // Check if file already exists (was previously read from disk) let already_exists = db.has_file(&path); - let _file = db.get_or_create_file(path.clone()); + let _file = db.get_or_create_file(&path); if already_exists { // File was already read - touch to invalidate cache @@ -384,8 +364,6 @@ impl Workspace { document } - // File Operations (AC #3) - /// Try to read file content using read-only database access. /// /// Returns `Some(content)` if the file exists in the database, `None` otherwise. @@ -422,32 +400,29 @@ impl Workspace { path.display() ); self.with_db_mut(|db| { - let file = db.get_or_create_file(path); + let file = db.get_or_create_file(&path); source_text(db, file).to_string() }) } /// Get the revision number of a file if it exists. /// - /// Returns None if the file is not being tracked by the database. + /// Returns `None` if the file is not being tracked by the database. /// Uses read-only database access since no mutation is needed. + #[must_use] pub fn file_revision(&self, path: &Path) -> Option { self.with_db(|db| db.get_file(path).map(|file| file.revision(db))) } - // Buffer Query Method (AC #4) - /// Get a document from the buffer if it's open. /// - /// Returns a cloned TextDocument for the given URL if it exists in buffers. + /// Returns a cloned [`TextDocument`] for the given URL if it exists in buffers. #[must_use] pub fn get_document(&self, url: &Url) -> Option { self.buffers.get(url) } - // File Invalidation Helper (AC #5) - - /// Private helper: Invalidate a file if it exists in the database. + /// Invalidate a file if it exists in the database. /// /// Used by document lifecycle methods to trigger cache invalidation. fn invalidate_file_if_exists(&mut self, path: &Path) { @@ -458,8 +433,6 @@ impl Workspace { }); } - // Existing methods preserved below - /// Get a reference to the buffers. #[must_use] pub fn buffers(&self) -> &Buffers { @@ -468,7 +441,7 @@ impl Workspace { /// Get a clone of the file system. /// - /// Returns an Arc-wrapped WorkspaceFileSystem that can be shared + /// Returns an Arc-wrapped [`WorkspaceFileSystem`] that can be shared /// across threads and used for file operations. #[must_use] pub fn file_system(&self) -> Arc { @@ -477,7 +450,7 @@ impl Workspace { /// Get a clone of the files tracking map. /// - /// Returns an Arc-wrapped DashMap for O(1) file lookups that + /// Returns an Arc-wrapped [`DashMap`] for O(1) file lookups that /// can be shared across Database instances. #[must_use] pub fn files(&self) -> Arc> { @@ -492,7 +465,7 @@ impl Workspace { /// /// For mutation operations, use [`with_db_mut`](Self::with_db_mut) instead. /// - /// # Panics + /// ## Panics /// /// Panics if the handle is currently taken for mutation. #[must_use] @@ -511,6 +484,9 @@ impl Default for Workspace { mod tests { use super::*; use crate::db::source_text; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::time::Duration; use tempfile::tempdir; @@ -523,7 +499,7 @@ mod tests { let result = workspace.with_db_mut(|db| { // Simple operation - create a file let path = PathBuf::from("test.py"); - let file = db.get_or_create_file(path); + let file = db.get_or_create_file(&path); file.revision(db) // Return the revision number }); @@ -646,7 +622,7 @@ mod tests { // Make some changes let path = PathBuf::from("manual_test.py"); - let _file = db.get_or_create_file(path); + let _file = db.get_or_create_file(&path); // Extract new handle and restore manually let new_handle = db.storage().clone().into_zalsa_handle(); @@ -680,7 +656,7 @@ mod tests { // Add a file to create some state let initial_file_count = workspace.with_db_mut(|db| { let path = PathBuf::from("callback_test.py"); - let _file = db.get_or_create_file(path); + let _file = db.get_or_create_file(&path); 1 // Return count }); @@ -689,7 +665,7 @@ mod tests { // Perform another mutation to ensure callbacks are preserved let final_file_count = workspace.with_db_mut(|db| { let path = PathBuf::from("callback_test2.py"); - let _file = db.get_or_create_file(path); + let _file = db.get_or_create_file(&path); // Count files - should include both let has_first = db.has_file(&PathBuf::from("callback_test.py")); @@ -780,42 +756,38 @@ mod tests { // Guard drops here without handle() being called } - // Error Recovery Tests - #[test] fn test_missing_file_returns_empty_content() { // Tests that source_text returns "" for non-existent files // instead of panicking or propagating errors let mut workspace = Workspace::new(); - + // Create a file reference for non-existent path let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file("/nonexistent/file.py".into()); + let file = db.get_or_create_file(&PathBuf::from_str("/nonexistent/file.py").unwrap()); source_text(db, file).to_string() }); - + assert_eq!(content, ""); } #[test] - #[cfg(unix)] // Only on Unix systems + #[cfg(unix)] fn test_permission_denied_file_handling() { // Create a file with no read permissions let temp_dir = tempdir().unwrap(); let file_path = temp_dir.path().join("no_read.py"); std::fs::write(&file_path, "content").unwrap(); - + // Remove read permissions - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&file_path, - std::fs::Permissions::from_mode(0o000)).unwrap(); - + std::fs::set_permissions(&file_path, std::fs::Permissions::from_mode(0o000)).unwrap(); + let mut workspace = Workspace::new(); let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(file_path.clone()); + let file = db.get_or_create_file(&file_path); source_text(db, file).to_string() }); - + // Should return empty string, not panic assert_eq!(content, ""); } @@ -825,44 +797,44 @@ mod tests { // Create a file with invalid UTF-8 let temp_dir = tempdir().unwrap(); let file_path = temp_dir.path().join("invalid.py"); - std::fs::write(&file_path, &[0xFF, 0xFE, 0xFD]).unwrap(); - + std::fs::write(&file_path, [0xFF, 0xFE, 0xFD]).unwrap(); + let mut workspace = Workspace::new(); let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(file_path.clone()); + let file = db.get_or_create_file(&file_path); source_text(db, file).to_string() }); - + // Should handle gracefully (empty or replacement chars) assert!(content.is_empty() || content.contains('�')); } - #[test] + #[test] fn test_file_deleted_after_tracking() { let temp_dir = tempdir().unwrap(); let file_path = temp_dir.path().join("disappearing.py"); std::fs::write(&file_path, "original").unwrap(); - + let mut workspace = Workspace::new(); - + // First read should succeed let content1 = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(file_path.clone()); + let file = db.get_or_create_file(&file_path); source_text(db, file).to_string() }); assert_eq!(content1, "original"); - + // Delete the file std::fs::remove_file(&file_path).unwrap(); - + // Touch to invalidate cache workspace.with_db_mut(|db| { db.touch_file(&file_path); }); - + // Second read should return empty (not panic) let content2 = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(file_path.clone()); + let file = db.get_or_create_file(&file_path); source_text(db, file).to_string() }); assert_eq!(content2, ""); @@ -873,16 +845,16 @@ mod tests { fn test_broken_symlink_handling() { let temp_dir = tempdir().unwrap(); let symlink_path = temp_dir.path().join("broken_link.py"); - + // Create broken symlink std::os::unix::fs::symlink("/nonexistent/target", &symlink_path).unwrap(); - + let mut workspace = Workspace::new(); let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(symlink_path.clone()); + let file = db.get_or_create_file(&symlink_path); source_text(db, file).to_string() }); - + // Should handle gracefully assert_eq!(content, ""); } @@ -892,31 +864,30 @@ mod tests { // Tests that concurrent file modifications don't crash let temp_dir = tempdir().unwrap(); let file_path = temp_dir.path().join("racing.py"); - + let workspace = Arc::new(Mutex::new(Workspace::new())); let path_clone = file_path.clone(); let workspace_clone = workspace.clone(); - + // Writer thread let writer = std::thread::spawn(move || { for i in 0..10 { - std::fs::write(&path_clone, format!("version {}", i)).ok(); + std::fs::write(&path_clone, format!("version {i}")).ok(); std::thread::sleep(Duration::from_millis(10)); } }); - + // Reader thread - should never panic for _ in 0..10 { let content = workspace_clone.lock().unwrap().with_db_mut(|db| { - let file = db.get_or_create_file(file_path.clone()); + let file = db.get_or_create_file(&file_path); source_text(db, file).to_string() }); // Content may vary but shouldn't crash assert!(content.is_empty() || content.starts_with("version")); std::thread::sleep(Duration::from_millis(5)); } - + writer.join().unwrap(); } } - From 846f742c32e36c9d563c4c41b57f467440bd645d Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 17:54:21 -0500 Subject: [PATCH 42/56] clippy --- crates/djls-server/src/session.rs | 6 +++--- crates/djls-server/tests/lsp_integration.rs | 16 ++++++++-------- crates/djls-templates/src/ast.rs | 14 +++++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 710fbd11..a16d59b2 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -26,7 +26,7 @@ use url::Url; /// /// All document lifecycle and database operations are delegated to the /// encapsulated Workspace, which provides thread-safe Salsa database -/// management with proper mutation safety through StorageHandleGuard. +/// management with proper mutation safety through `StorageHandleGuard`. pub struct Session { /// The Django project configuration project: Option, @@ -186,7 +186,7 @@ impl Session { /// Get the current revision of a file, if it's being tracked. /// /// Returns None if the file hasn't been created yet. - pub fn file_revision(&self, path: &Path) -> Option { + #[must_use] pub fn file_revision(&self, path: &Path) -> Option { { let this = &self.workspace; this.with_db(|db| db.get_file(path).map(|file| file.revision(db))) @@ -194,7 +194,7 @@ impl Session { } /// Check if a file is currently being tracked in Salsa. - pub fn has_file(&self, path: &Path) -> bool { + #[must_use] pub fn has_file(&self, path: &Path) -> bool { self.with_db(|db| db.has_file(path)) } } diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index 00201262..ea10ccbe 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -392,7 +392,7 @@ async fn test_concurrent_overlay_updates() { // Open initial documents for i in 0..5 { server - .open_document(&format!("file{}.html", i), &format!("Initial {}", i), 1) + .open_document(&format!("file{i}.html"), &format!("Initial {i}"), 1) .await; } @@ -406,8 +406,8 @@ async fn test_concurrent_overlay_updates() { for version in 2..10 { server_clone .change_document( - &format!("file{}.html", i), - &format!("Updated {} v{}", i, version), + &format!("file{i}.html"), + &format!("Updated {i} v{version}"), version, ) .await; @@ -426,11 +426,11 @@ async fn test_concurrent_overlay_updates() { // Verify final state of all documents for i in 0..5 { - let content = server.get_file_content(&format!("file{}.html", i)).await; - assert_eq!(content, format!("Updated {} v9", i)); + let content = server.get_file_content(&format!("file{i}.html")).await; + assert_eq!(content, format!("Updated {i} v9")); // Each document should have had 8 changes (versions 2-9) - let revision = server.get_file_revision(&format!("file{}.html", i)).await; + let revision = server.get_file_revision(&format!("file{i}.html")).await; assert_eq!(revision, Some(8)); } } @@ -453,7 +453,7 @@ async fn test_caching_behavior() { // Parse all templates once to populate cache for i in 1..=3 { let _ = server - .get_file_content(&format!("template{}.html", i)) + .get_file_content(&format!("template{i}.html")) .await; } @@ -516,7 +516,7 @@ async fn test_revision_tracking_across_lifecycle() { // Change document multiple times for i in 2..=5 { server - .change_document(file_name, &format!("Change {}", i), i) + .change_document(file_name, &format!("Change {i}"), i) .await; assert_eq!( server.get_file_revision(file_name).await, diff --git a/crates/djls-templates/src/ast.rs b/crates/djls-templates/src/ast.rs index 1b62d4d4..ebbadf56 100644 --- a/crates/djls-templates/src/ast.rs +++ b/crates/djls-templates/src/ast.rs @@ -12,11 +12,11 @@ pub struct Ast { } impl Ast { - pub fn nodelist(&self) -> &Vec { + #[must_use] pub fn nodelist(&self) -> &Vec { &self.nodelist } - pub fn line_offsets(&self) -> &LineOffsets { + #[must_use] pub fn line_offsets(&self) -> &LineOffsets { &self.line_offsets } @@ -44,7 +44,7 @@ impl LineOffsets { self.0.push(offset); } - pub fn position_to_line_col(&self, position: usize) -> (usize, usize) { + #[must_use] pub fn position_to_line_col(&self, position: usize) -> (usize, usize) { let position = u32::try_from(position).unwrap_or_default(); let line = match self.0.binary_search(&position) { Ok(exact_line) => exact_line, // Position is at start of this line @@ -63,7 +63,7 @@ impl LineOffsets { (line + 1, col) } - pub fn line_col_to_position(&self, line: u32, col: u32) -> u32 { + #[must_use] pub fn line_col_to_position(&self, line: u32, col: u32) -> u32 { // line is 1-based, so subtract 1 to get the index self.0[(line - 1) as usize] + col } @@ -104,17 +104,17 @@ pub struct Span { } impl Span { - pub fn new(start: u32, length: u32) -> Self { + #[must_use] pub fn new(start: u32, length: u32) -> Self { Self { start, length } } #[allow(clippy::trivially_copy_pass_by_ref)] - pub fn start(&self) -> u32 { + #[must_use] pub fn start(&self) -> u32 { self.start } #[allow(clippy::trivially_copy_pass_by_ref)] - pub fn length(&self) -> u32 { + #[must_use] pub fn length(&self) -> u32 { self.length } } From 8990800f43d4acc0a0c2fe1f9ede5389ba482c37 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 18:00:19 -0500 Subject: [PATCH 43/56] clippy --- crates/djls-server/src/session.rs | 26 +++++++++++---------- crates/djls-server/tests/lsp_integration.rs | 17 +++++++++----- crates/djls-workspace/src/workspace.rs | 6 ++--- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index a16d59b2..fdfc5b04 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -176,9 +176,9 @@ impl Session { /// This is the safe way to read file content through the system. /// The file is created if it doesn't exist, and content is read /// through the `FileSystem` abstraction (overlay first, then disk). - pub fn file_content(&mut self, path: PathBuf) -> String { + pub fn file_content(&mut self, path: &PathBuf) -> String { self.with_db_mut(|db| { - let file = db.get_or_create_file(&path); + let file = db.get_or_create_file(path); source_text(db, file).to_string() }) } @@ -186,7 +186,8 @@ impl Session { /// Get the current revision of a file, if it's being tracked. /// /// Returns None if the file hasn't been created yet. - #[must_use] pub fn file_revision(&self, path: &Path) -> Option { + #[must_use] + pub fn file_revision(&self, path: &Path) -> Option { { let this = &self.workspace; this.with_db(|db| db.get_file(path).map(|file| file.revision(db))) @@ -194,7 +195,8 @@ impl Session { } /// Check if a file is currently being tracked in Salsa. - #[must_use] pub fn has_file(&self, path: &Path) -> bool { + #[must_use] + pub fn has_file(&self, path: &Path) -> bool { self.with_db(|db| db.has_file(path)) } } @@ -205,7 +207,7 @@ impl Default for Session { project: None, settings: Settings::default(), workspace: Workspace::new(), - client_capabilities: Default::default(), + client_capabilities: lsp_types::ClientCapabilities::default(), position_encoding: PositionEncoding::default(), } } @@ -231,7 +233,7 @@ mod tests { ); session.open_document(&url, document); - let content1 = session.file_content(path.clone()); + let content1 = session.file_content(&path); assert_eq!(content1, "

Original Content

"); // Update document with new content using a full replacement change @@ -243,7 +245,7 @@ mod tests { session.update_document(&url, changes, 2); // Read content again (should get new overlay content due to invalidation) - let content2 = session.file_content(path.clone()); + let content2 = session.file_content(&path); assert_eq!(content2, "

Updated Content

"); assert_ne!(content1, content2); @@ -251,7 +253,7 @@ mod tests { session.close_document(&url); // Read content again (should now read from disk, which returns empty for missing files) - let content3 = session.file_content(path.clone()); + let content3 = session.file_content(&path); assert_eq!(content3, ""); // No file on disk, returns empty } @@ -262,16 +264,16 @@ mod tests { let path1 = PathBuf::from("/test/file1.py"); let path2 = PathBuf::from("/test/file2.py"); - session.file_content(path1.clone()); - session.file_content(path2.clone()); + session.file_content(&path1); + session.file_content(&path2); // Verify files are preserved across operations assert!(session.has_file(&path1)); assert!(session.has_file(&path2)); // Files should persist even after multiple operations - let content1 = session.file_content(path1.clone()); - let content2 = session.file_content(path2.clone()); + let content1 = session.file_content(&path1); + let content2 = session.file_content(&path2); // Both should return empty (no disk content) assert_eq!(content1, ""); diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index ea10ccbe..84920d2e 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -7,6 +7,7 @@ //! The tests ensure that document changes properly invalidate cached queries //! and that overlays take precedence over disk content. +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -89,9 +90,15 @@ impl TestServer { let params = DidOpenTextDocumentParams { text_document: TextDocumentItem { uri: self.workspace_url(file_name).to_string().parse().unwrap(), - language_id: if file_name.ends_with(".html") { + language_id: if Path::new(file_name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("html")) + { "html".to_string() - } else if file_name.ends_with(".py") { + } else if Path::new(file_name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("py")) + { "python".to_string() } else { "plaintext".to_string() @@ -154,7 +161,7 @@ impl TestServer { async fn get_file_content(&self, file_name: &str) -> String { let path = self.workspace_file(file_name); self.server - .with_session_mut(|session| session.file_content(path)) + .with_session_mut(|session| session.file_content(&path)) .await } @@ -452,9 +459,7 @@ async fn test_caching_behavior() { // Parse all templates once to populate cache for i in 1..=3 { - let _ = server - .get_file_content(&format!("template{i}.html")) - .await; + let _ = server.get_file_content(&format!("template{i}.html")).await; } // Store initial revisions diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index bcd8b11a..a8735209 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -388,9 +388,9 @@ impl Workspace { /// First attempts read-only access for existing files, then escalates to write /// access only if the file needs to be created. This improves concurrency by /// avoiding unnecessary write locks. - pub fn file_content(&mut self, path: PathBuf) -> String { + pub fn file_content(&mut self, path: &PathBuf) -> String { // Try read-only access first for existing files - if let Some(content) = self.try_read_file(&path) { + if let Some(content) = self.try_read_file(path) { return content; } @@ -400,7 +400,7 @@ impl Workspace { path.display() ); self.with_db_mut(|db| { - let file = db.get_or_create_file(&path); + let file = db.get_or_create_file(path); source_text(db, file).to_string() }) } From c83332344ecf5da934ba26925739b3241632b1dc Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 18:02:14 -0500 Subject: [PATCH 44/56] clippy --- crates/djls-server/tests/lsp_integration.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index 84920d2e..e33ddc54 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -519,13 +519,13 @@ async fn test_revision_tracking_across_lifecycle() { assert_eq!(server.get_file_revision(file_name).await, Some(0)); // Change document multiple times - for i in 2..=5 { + for i in 2u64..=5 { server - .change_document(file_name, &format!("Change {i}"), i) + .change_document(file_name, &format!("Change {i}"), i.try_into().unwrap()) .await; assert_eq!( server.get_file_revision(file_name).await, - Some((i - 1) as u64), + Some(i - 1), "Revision should be {} after change {}", i - 1, i From 4fa33704e781f1b5d29290fe3d7428aa8fb2d631 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 18:02:26 -0500 Subject: [PATCH 45/56] fmt --- crates/djls-project/src/python.rs | 4 +++- crates/djls-project/src/system.rs | 4 +++- crates/djls-server/src/completions.rs | 12 ++++++++---- crates/djls-server/src/session.rs | 3 ++- crates/djls-templates/src/ast.rs | 21 ++++++++++++++------- crates/djls-workspace/src/document.rs | 6 ++++-- crates/djls-workspace/src/encoding.rs | 8 ++++++-- crates/djls-workspace/src/lib.rs | 3 --- crates/djls-workspace/src/workspace.rs | 19 +++++++++++++------ 9 files changed, 53 insertions(+), 27 deletions(-) diff --git a/crates/djls-project/src/python.rs b/crates/djls-project/src/python.rs index c3678d37..69358453 100644 --- a/crates/djls-project/src/python.rs +++ b/crates/djls-project/src/python.rs @@ -208,7 +208,9 @@ mod tests { use super::*; use crate::system::mock::MockGuard; - use crate::system::mock::{self as sys_mock}; + use crate::system::mock::{ + self as sys_mock, + }; #[test] fn test_explicit_venv_path_found() { diff --git a/crates/djls-project/src/system.rs b/crates/djls-project/src/system.rs index 88269e38..167e51c6 100644 --- a/crates/djls-project/src/system.rs +++ b/crates/djls-project/src/system.rs @@ -103,7 +103,9 @@ mod tests { use which::Error as WhichError; use super::mock::MockGuard; - use super::mock::{self as sys_mock}; + use super::mock::{ + self as sys_mock, + }; use super::*; #[test] diff --git a/crates/djls-server/src/completions.rs b/crates/djls-server/src/completions.rs index f5c6aeef..8e3180ac 100644 --- a/crates/djls-server/src/completions.rs +++ b/crates/djls-server/src/completions.rs @@ -4,10 +4,14 @@ //! and generating appropriate completion items for Django templates. use djls_project::TemplateTags; -use djls_workspace::{FileKind, PositionEncoding, TextDocument}; -use tower_lsp_server::lsp_types::{ - CompletionItem, CompletionItemKind, Documentation, InsertTextFormat, Position, -}; +use djls_workspace::FileKind; +use djls_workspace::PositionEncoding; +use djls_workspace::TextDocument; +use tower_lsp_server::lsp_types::CompletionItem; +use tower_lsp_server::lsp_types::CompletionItemKind; +use tower_lsp_server::lsp_types::Documentation; +use tower_lsp_server::lsp_types::InsertTextFormat; +use tower_lsp_server::lsp_types::Position; /// Tracks what closing characters are needed to complete a template tag. /// diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index fdfc5b04..fbd3c4b5 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -215,9 +215,10 @@ impl Default for Session { #[cfg(test)] mod tests { - use super::*; use djls_workspace::LanguageId; + use super::*; + #[test] fn test_revision_invalidation_chain() { let mut session = Session::default(); diff --git a/crates/djls-templates/src/ast.rs b/crates/djls-templates/src/ast.rs index ebbadf56..9348dec1 100644 --- a/crates/djls-templates/src/ast.rs +++ b/crates/djls-templates/src/ast.rs @@ -12,11 +12,13 @@ pub struct Ast { } impl Ast { - #[must_use] pub fn nodelist(&self) -> &Vec { + #[must_use] + pub fn nodelist(&self) -> &Vec { &self.nodelist } - #[must_use] pub fn line_offsets(&self) -> &LineOffsets { + #[must_use] + pub fn line_offsets(&self) -> &LineOffsets { &self.line_offsets } @@ -44,7 +46,8 @@ impl LineOffsets { self.0.push(offset); } - #[must_use] pub fn position_to_line_col(&self, position: usize) -> (usize, usize) { + #[must_use] + pub fn position_to_line_col(&self, position: usize) -> (usize, usize) { let position = u32::try_from(position).unwrap_or_default(); let line = match self.0.binary_search(&position) { Ok(exact_line) => exact_line, // Position is at start of this line @@ -63,7 +66,8 @@ impl LineOffsets { (line + 1, col) } - #[must_use] pub fn line_col_to_position(&self, line: u32, col: u32) -> u32 { + #[must_use] + pub fn line_col_to_position(&self, line: u32, col: u32) -> u32 { // line is 1-based, so subtract 1 to get the index self.0[(line - 1) as usize] + col } @@ -104,17 +108,20 @@ pub struct Span { } impl Span { - #[must_use] pub fn new(start: u32, length: u32) -> Self { + #[must_use] + pub fn new(start: u32, length: u32) -> Self { Self { start, length } } #[allow(clippy::trivially_copy_pass_by_ref)] - #[must_use] pub fn start(&self) -> u32 { + #[must_use] + pub fn start(&self) -> u32 { self.start } #[allow(clippy::trivially_copy_pass_by_ref)] - #[must_use] pub fn length(&self) -> u32 { + #[must_use] + pub fn length(&self) -> u32 { self.length } } diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index eb0fae76..95f7e62b 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -5,7 +5,8 @@ //! performance when handling frequent position-based operations like hover, completion, //! and diagnostics. -use tower_lsp_server::lsp_types::{Position, Range}; +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; use crate::encoding::PositionEncoding; use crate::language::LanguageId; @@ -291,9 +292,10 @@ pub enum IndexKind { #[cfg(test)] mod tests { + use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; + use super::*; use crate::language::LanguageId; - use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; #[test] fn test_incremental_update_single_change() { diff --git a/crates/djls-workspace/src/encoding.rs b/crates/djls-workspace/src/encoding.rs index 7de8f18a..f81ffeab 100644 --- a/crates/djls-workspace/src/encoding.rs +++ b/crates/djls-workspace/src/encoding.rs @@ -1,6 +1,8 @@ use std::fmt; use std::str::FromStr; -use tower_lsp_server::lsp_types::{InitializeParams, PositionEncodingKind}; + +use tower_lsp_server::lsp_types::InitializeParams; +use tower_lsp_server::lsp_types::PositionEncodingKind; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum PositionEncoding { @@ -84,8 +86,10 @@ impl TryFrom for PositionEncoding { #[cfg(test)] mod tests { + use tower_lsp_server::lsp_types::ClientCapabilities; + use tower_lsp_server::lsp_types::GeneralClientCapabilities; + use super::*; - use tower_lsp_server::lsp_types::{ClientCapabilities, GeneralClientCapabilities}; #[test] fn test_encoding_str_conversion() { diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 2791d875..7eb60b4c 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -21,7 +21,6 @@ mod language; pub mod paths; mod workspace; - use std::path::Path; pub use buffers::Buffers; @@ -34,8 +33,6 @@ pub use fs::WorkspaceFileSystem; pub use language::LanguageId; pub use workspace::Workspace; - - /// File classification for routing to analyzers. /// /// [`FileKind`] determines how a file should be processed by downstream analyzers. diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index a8735209..33dfb7ec 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -4,7 +4,8 @@ //! components including buffers, file system, file tracking, and database handle. //! This provides a clean API boundary between server and workspace layers. -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use dashmap::DashMap; @@ -13,9 +14,12 @@ use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use url::Url; use crate::buffers::Buffers; -use crate::db::{source_text, Database, SourceFile}; +use crate::db::source_text; +use crate::db::Database; +use crate::db::SourceFile; use crate::document::TextDocument; -use crate::fs::{OsFileSystem, WorkspaceFileSystem}; +use crate::fs::OsFileSystem; +use crate::fs::WorkspaceFileSystem; use crate::paths::url_to_path; /// Safe wrapper for [`StorageHandle`](salsa::StorageHandle) that prevents misuse through type safety. @@ -482,15 +486,18 @@ impl Default for Workspace { #[cfg(test)] mod tests { - use super::*; - use crate::db::source_text; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::str::FromStr; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; + use std::sync::Mutex; use std::time::Duration; + use tempfile::tempdir; + use super::*; + use crate::db::source_text; + #[test] fn test_normal_mutation_flow_with_guard() { let mut workspace = Workspace::new(); From 4858c68c3562327be9ea7d0f8ffbf3f13f5df0c1 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 21:50:29 -0500 Subject: [PATCH 46/56] trimming the fat --- crates/djls-server/src/session.rs | 125 +--- crates/djls-server/tests/lsp_integration.rs | 766 -------------------- crates/djls-workspace/src/workspace.rs | 148 ---- 3 files changed, 1 insertion(+), 1038 deletions(-) delete mode 100644 crates/djls-server/tests/lsp_integration.rs diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index fbd3c4b5..1b10e3f7 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -3,13 +3,10 @@ //! This module implements the LSP session abstraction that manages project-specific //! state and delegates workspace operations to the Workspace facade. -use std::path::Path; use std::path::PathBuf; use djls_conf::Settings; use djls_project::DjangoProject; -use djls_workspace::db::source_text; -use djls_workspace::db::Database; use djls_workspace::paths; use djls_workspace::PositionEncoding; use djls_workspace::TextDocument; @@ -76,6 +73,7 @@ impl Session { position_encoding, } } + /// Determines the project root path from initialization parameters. /// /// Tries workspace folders first (using the first one), then falls back to current directory. @@ -115,26 +113,6 @@ impl Session { self.position_encoding } - /// Execute a closure with mutable access to the database. - /// - /// Delegates to the workspace's safe database mutation mechanism. - pub fn with_db_mut(&mut self, f: F) -> R - where - F: FnOnce(&mut Database) -> R, - { - self.workspace.with_db_mut(f) - } - - /// Execute a closure with read-only access to the database. - /// - /// Delegates to the workspace's safe database read mechanism. - pub fn with_db(&self, f: F) -> R - where - F: FnOnce(&Database) -> R, - { - self.workspace.with_db(f) - } - /// Handle opening a document - sets buffer and creates file. /// /// Delegates to the workspace's document management. @@ -170,35 +148,6 @@ impl Session { pub fn get_document(&self, url: &Url) -> Option { self.workspace.get_document(url) } - - /// Get the current content of a file (from overlay or disk). - /// - /// This is the safe way to read file content through the system. - /// The file is created if it doesn't exist, and content is read - /// through the `FileSystem` abstraction (overlay first, then disk). - pub fn file_content(&mut self, path: &PathBuf) -> String { - self.with_db_mut(|db| { - let file = db.get_or_create_file(path); - source_text(db, file).to_string() - }) - } - - /// Get the current revision of a file, if it's being tracked. - /// - /// Returns None if the file hasn't been created yet. - #[must_use] - pub fn file_revision(&self, path: &Path) -> Option { - { - let this = &self.workspace; - this.with_db(|db| db.get_file(path).map(|file| file.revision(db))) - } - } - - /// Check if a file is currently being tracked in Salsa. - #[must_use] - pub fn has_file(&self, path: &Path) -> bool { - self.with_db(|db| db.has_file(path)) - } } impl Default for Session { @@ -212,75 +161,3 @@ impl Default for Session { } } } - -#[cfg(test)] -mod tests { - use djls_workspace::LanguageId; - - use super::*; - - #[test] - fn test_revision_invalidation_chain() { - let mut session = Session::default(); - - let path = PathBuf::from("/test/template.html"); - let url = Url::parse("file:///test/template.html").unwrap(); - - // Open document with initial content - let document = TextDocument::new( - "

Original Content

".to_string(), - 1, - LanguageId::Other, - ); - session.open_document(&url, document); - - let content1 = session.file_content(&path); - assert_eq!(content1, "

Original Content

"); - - // Update document with new content using a full replacement change - let changes = vec![lsp_types::TextDocumentContentChangeEvent { - range: None, - range_length: None, - text: "

Updated Content

".to_string(), - }]; - session.update_document(&url, changes, 2); - - // Read content again (should get new overlay content due to invalidation) - let content2 = session.file_content(&path); - assert_eq!(content2, "

Updated Content

"); - assert_ne!(content1, content2); - - // Close document (removes overlay, bumps revision) - session.close_document(&url); - - // Read content again (should now read from disk, which returns empty for missing files) - let content3 = session.file_content(&path); - assert_eq!(content3, ""); // No file on disk, returns empty - } - - #[test] - fn test_with_db_mut_preserves_files() { - let mut session = Session::default(); - - let path1 = PathBuf::from("/test/file1.py"); - let path2 = PathBuf::from("/test/file2.py"); - - session.file_content(&path1); - session.file_content(&path2); - - // Verify files are preserved across operations - assert!(session.has_file(&path1)); - assert!(session.has_file(&path2)); - - // Files should persist even after multiple operations - let content1 = session.file_content(&path1); - let content2 = session.file_content(&path2); - - // Both should return empty (no disk content) - assert_eq!(content1, ""); - assert_eq!(content2, ""); - - assert!(session.has_file(&path1)); - assert!(session.has_file(&path2)); - } -} diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs deleted file mode 100644 index e33ddc54..00000000 --- a/crates/djls-server/tests/lsp_integration.rs +++ /dev/null @@ -1,766 +0,0 @@ -//! Integration tests for the LSP server's overlay → revision → invalidation flow -//! -//! These tests verify the complete two-layer architecture: -//! - Layer 1: LSP overlays (in-memory document state) -//! - Layer 2: Salsa database with revision tracking -//! -//! The tests ensure that document changes properly invalidate cached queries -//! and that overlays take precedence over disk content. - -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; - -use djls_server::DjangoLanguageServer; -use djls_workspace::db::parse_template; -use tempfile::TempDir; -use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; -use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; -use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::InitializeParams; -use tower_lsp_server::lsp_types::InitializedParams; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; -use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; -use tower_lsp_server::lsp_types::TextDocumentIdentifier; -use tower_lsp_server::lsp_types::TextDocumentItem; -use tower_lsp_server::lsp_types::VersionedTextDocumentIdentifier; -use tower_lsp_server::lsp_types::WorkspaceFolder; -use tower_lsp_server::LanguageServer; -use url::Url; - -/// Test helper that manages an LSP server instance for testing -struct TestServer { - server: DjangoLanguageServer, - _temp_dir: TempDir, - workspace_root: PathBuf, -} - -impl TestServer { - /// Create a new test server with a temporary workspace - async fn new() -> Self { - // Create temporary directory for test workspace - let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let workspace_root = temp_dir.path().to_path_buf(); - - // Set up logging - let (_non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); - - // Create server (guard is moved into server, so we return it too) - let server = DjangoLanguageServer::new(guard); - - // Initialize the server - let workspace_folder = WorkspaceFolder { - uri: format!("file://{}", workspace_root.display()) - .parse() - .unwrap(), - name: "test_workspace".to_string(), - }; - - let init_params = InitializeParams { - workspace_folders: Some(vec![workspace_folder]), - ..Default::default() - }; - - server - .initialize(init_params) - .await - .expect("Failed to initialize"); - server.initialized(InitializedParams {}).await; - - Self { - server, - _temp_dir: temp_dir, - workspace_root, - } - } - - /// Helper to create a file path in the test workspace - fn workspace_file(&self, name: &str) -> PathBuf { - self.workspace_root.join(name) - } - - /// Helper to create a file URL in the test workspace - fn workspace_url(&self, name: &str) -> Url { - djls_workspace::paths::path_to_url(&self.workspace_file(name)).unwrap() - } - - /// Open a document in the LSP server - async fn open_document(&self, file_name: &str, content: &str, version: i32) { - let params = DidOpenTextDocumentParams { - text_document: TextDocumentItem { - uri: self.workspace_url(file_name).to_string().parse().unwrap(), - language_id: if Path::new(file_name) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("html")) - { - "html".to_string() - } else if Path::new(file_name) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("py")) - { - "python".to_string() - } else { - "plaintext".to_string() - }, - version, - text: content.to_string(), - }, - }; - - self.server.did_open(params).await; - } - - /// Change a document in the LSP server - async fn change_document(&self, file_name: &str, new_content: &str, version: i32) { - let params = DidChangeTextDocumentParams { - text_document: VersionedTextDocumentIdentifier { - uri: self.workspace_url(file_name).to_string().parse().unwrap(), - version, - }, - content_changes: vec![TextDocumentContentChangeEvent { - range: None, - range_length: None, - text: new_content.to_string(), - }], - }; - - self.server.did_change(params).await; - } - - /// Send incremental changes to a document - async fn change_document_incremental( - &self, - file_name: &str, - changes: Vec, - version: i32, - ) { - let params = DidChangeTextDocumentParams { - text_document: VersionedTextDocumentIdentifier { - uri: self.workspace_url(file_name).to_string().parse().unwrap(), - version, - }, - content_changes: changes, - }; - - self.server.did_change(params).await; - } - - /// Close a document in the LSP server - async fn close_document(&self, file_name: &str) { - let params = DidCloseTextDocumentParams { - text_document: TextDocumentIdentifier { - uri: self.workspace_url(file_name).to_string().parse().unwrap(), - }, - }; - - self.server.did_close(params).await; - } - - /// Get the content of a file through the session's query system - async fn get_file_content(&self, file_name: &str) -> String { - let path = self.workspace_file(file_name); - self.server - .with_session_mut(|session| session.file_content(&path)) - .await - } - - /// Write a file to disk in the test workspace - fn write_file(&self, file_name: &str, content: &str) { - let path = self.workspace_file(file_name); - std::fs::write(path, content).expect("Failed to write test file"); - } - - /// Get the revision of a file - async fn get_file_revision(&self, file_name: &str) -> Option { - let path = self.workspace_file(file_name); - self.server - .with_session_mut(|session| session.file_revision(&path)) - .await - } -} - -#[tokio::test] -async fn test_full_lsp_lifecycle() { - let server = TestServer::new().await; - let file_name = "test.html"; - - // Write initial content to disk - server.write_file(file_name, "

Disk Content

"); - - // 1. Test did_open creates overlay and file - server - .open_document(file_name, "

Overlay Content

", 1) - .await; - - // Verify overlay content is returned (not disk content) - let content = server.get_file_content(file_name).await; - assert_eq!(content, "

Overlay Content

"); - - // Verify file was created with revision 0 - let revision = server.get_file_revision(file_name).await; - assert_eq!(revision, Some(0)); - - // 2. Test did_change updates overlay and bumps revision - server - .change_document(file_name, "

Updated Content

", 2) - .await; - - // Verify content changed - let content = server.get_file_content(file_name).await; - assert_eq!(content, "

Updated Content

"); - - // Verify revision was bumped - let revision = server.get_file_revision(file_name).await; - assert_eq!(revision, Some(1)); - - // 3. Test did_close removes overlay and bumps revision - server.close_document(file_name).await; - - // Verify content now comes from disk (empty since file doesn't exist) - let content = server.get_file_content(file_name).await; - assert_eq!(content, "

Disk Content

"); - - // Verify revision was bumped again - let revision = server.get_file_revision(file_name).await; - assert_eq!(revision, Some(2)); -} - -#[tokio::test] -async fn test_overlay_precedence() { - let server = TestServer::new().await; - let file_name = "template.html"; - - // Write content to disk - server.write_file(file_name, "{% block content %}Disk{% endblock %}"); - - // Read content before overlay - should get disk content - let content = server.get_file_content(file_name).await; - assert_eq!(content, "{% block content %}Disk{% endblock %}"); - - // Open document with different content - server - .open_document(file_name, "{% block content %}Overlay{% endblock %}", 1) - .await; - - // Verify overlay content takes precedence - let content = server.get_file_content(file_name).await; - assert_eq!(content, "{% block content %}Overlay{% endblock %}"); - - // Close document - server.close_document(file_name).await; - - // Verify we're back to disk content - let content = server.get_file_content(file_name).await; - assert_eq!(content, "{% block content %}Disk{% endblock %}"); -} - -#[tokio::test] -async fn test_template_parsing_with_overlays() { - let server = TestServer::new().await; - let file_name = "template.html"; - - // Write initial template to disk - server.write_file(file_name, "{% if true %}Original{% endif %}"); - - // Open with different template content - server - .open_document( - file_name, - "{% for item in items %}{{ item }}{% endfor %}", - 1, - ) - .await; - - // Parse template through the session - let workspace_path = server.workspace_file(file_name); - let ast = server - .server - .with_session_mut(|session| { - session.with_db_mut(|db| { - let file = db.get_or_create_file(&workspace_path); - parse_template(db, file) - }) - }) - .await; - - // Verify we parsed the overlay content (for loop), not disk content (if statement) - assert!(ast.is_some()); - let ast = ast.unwrap(); - let ast_str = format!("{:?}", ast.ast); - assert!(ast_str.contains("for") || ast_str.contains("For")); - assert!(!ast_str.contains("if") && !ast_str.contains("If")); -} - -#[tokio::test] -async fn test_incremental_sync() { - let server = TestServer::new().await; - let file_name = "test.html"; - - // Open document with initial content - server.open_document(file_name, "Hello world", 1).await; - - // Apply incremental change to replace "world" with "Rust" - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 6), Position::new(0, 11))), - range_length: None, - text: "Rust".to_string(), - }]; - - server - .change_document_incremental(file_name, changes, 2) - .await; - - // Verify the incremental change was applied correctly - let content = server.get_file_content(file_name).await; - assert_eq!(content, "Hello Rust"); - - // Apply multiple incremental changes - let changes = vec![ - // Insert " programming" after "Rust" - TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 10), Position::new(0, 10))), - range_length: None, - text: " programming".to_string(), - }, - // Replace "Hello" with "Learning" - TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 0), Position::new(0, 5))), - range_length: None, - text: "Learning".to_string(), - }, - ]; - - server - .change_document_incremental(file_name, changes, 3) - .await; - - // Verify multiple changes were applied in order - let content = server.get_file_content(file_name).await; - assert_eq!(content, "Learning Rust programming"); -} - -#[tokio::test] -async fn test_incremental_sync_with_newlines() { - let server = TestServer::new().await; - let file_name = "multiline.html"; - - // Open document with multiline content - server - .open_document(file_name, "Line 1\nLine 2\nLine 3", 1) - .await; - - // Replace text spanning multiple lines - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new( - Position::new(0, 5), // After "Line " on first line - Position::new(2, 4), // Before " 3" on third line - )), - range_length: None, - text: "A\nB\nC".to_string(), - }]; - - server - .change_document_incremental(file_name, changes, 2) - .await; - - // Verify the change was applied correctly across lines - let content = server.get_file_content(file_name).await; - assert_eq!(content, "Line A\nB\nC 3"); -} - -#[tokio::test] -async fn test_multiple_documents_independent() { - let server = TestServer::new().await; - - // Open multiple documents - server.open_document("file1.html", "Content 1", 1).await; - server.open_document("file2.html", "Content 2", 1).await; - server.open_document("file3.html", "Content 3", 1).await; - - // Change one document - server.change_document("file2.html", "Updated 2", 2).await; - - // Verify only file2 was updated - assert_eq!(server.get_file_content("file1.html").await, "Content 1"); - assert_eq!(server.get_file_content("file2.html").await, "Updated 2"); - assert_eq!(server.get_file_content("file3.html").await, "Content 3"); - - // Verify revision changes - assert_eq!(server.get_file_revision("file1.html").await, Some(0)); - assert_eq!(server.get_file_revision("file2.html").await, Some(1)); - assert_eq!(server.get_file_revision("file3.html").await, Some(0)); -} - -#[tokio::test] -async fn test_concurrent_overlay_updates() { - let server = Arc::new(TestServer::new().await); - - // Open initial documents - for i in 0..5 { - server - .open_document(&format!("file{i}.html"), &format!("Initial {i}"), 1) - .await; - } - - // Spawn concurrent tasks to update different documents - let mut handles = vec![]; - - for i in 0..5 { - let server_clone = Arc::clone(&server); - let handle = tokio::spawn(async move { - // Each task updates its document multiple times - for version in 2..10 { - server_clone - .change_document( - &format!("file{i}.html"), - &format!("Updated {i} v{version}"), - version, - ) - .await; - - // Small delay to encourage interleaving - tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; - } - }); - handles.push(handle); - } - - // Wait for all tasks to complete - for handle in handles { - handle.await.expect("Task failed"); - } - - // Verify final state of all documents - for i in 0..5 { - let content = server.get_file_content(&format!("file{i}.html")).await; - assert_eq!(content, format!("Updated {i} v9")); - - // Each document should have had 8 changes (versions 2-9) - let revision = server.get_file_revision(&format!("file{i}.html")).await; - assert_eq!(revision, Some(8)); - } -} - -#[tokio::test] -async fn test_caching_behavior() { - let server = TestServer::new().await; - - // Open three template files - server - .open_document("template1.html", "{% block a %}1{% endblock %}", 1) - .await; - server - .open_document("template2.html", "{% block b %}2{% endblock %}", 1) - .await; - server - .open_document("template3.html", "{% block c %}3{% endblock %}", 1) - .await; - - // Parse all templates once to populate cache - for i in 1..=3 { - let _ = server.get_file_content(&format!("template{i}.html")).await; - } - - // Store initial revisions - let rev1_before = server.get_file_revision("template1.html").await.unwrap(); - let rev2_before = server.get_file_revision("template2.html").await.unwrap(); - let rev3_before = server.get_file_revision("template3.html").await.unwrap(); - - // Change only template2 - server - .change_document("template2.html", "{% block b %}CHANGED{% endblock %}", 2) - .await; - - // Verify only template2's revision changed - let rev1_after = server.get_file_revision("template1.html").await.unwrap(); - let rev2_after = server.get_file_revision("template2.html").await.unwrap(); - let rev3_after = server.get_file_revision("template3.html").await.unwrap(); - - assert_eq!( - rev1_before, rev1_after, - "template1 revision should not change" - ); - assert_eq!( - rev2_before + 1, - rev2_after, - "template2 revision should increment" - ); - assert_eq!( - rev3_before, rev3_after, - "template3 revision should not change" - ); - - // Verify content - assert_eq!( - server.get_file_content("template1.html").await, - "{% block a %}1{% endblock %}" - ); - assert_eq!( - server.get_file_content("template2.html").await, - "{% block b %}CHANGED{% endblock %}" - ); - assert_eq!( - server.get_file_content("template3.html").await, - "{% block c %}3{% endblock %}" - ); -} - -#[tokio::test] -async fn test_revision_tracking_across_lifecycle() { - let server = TestServer::new().await; - let file_name = "tracked.html"; - - // Create file on disk - server.write_file(file_name, "Initial"); - - // Open document - should create file with revision 0 - server.open_document(file_name, "Opened", 1).await; - assert_eq!(server.get_file_revision(file_name).await, Some(0)); - - // Change document multiple times - for i in 2u64..=5 { - server - .change_document(file_name, &format!("Change {i}"), i.try_into().unwrap()) - .await; - assert_eq!( - server.get_file_revision(file_name).await, - Some(i - 1), - "Revision should be {} after change {}", - i - 1, - i - ); - } - - // Close document - should bump revision one more time - server.close_document(file_name).await; - assert_eq!(server.get_file_revision(file_name).await, Some(5)); - - // Re-open document - file already exists, should bump revision to invalidate cache - server.open_document(file_name, "Reopened", 10).await; - assert_eq!( - server.get_file_revision(file_name).await, - Some(6), - "Revision should bump on re-open to invalidate cache" - ); - - // Change again - server.change_document(file_name, "Final", 11).await; - assert_eq!(server.get_file_revision(file_name).await, Some(7)); -} - -#[tokio::test] -async fn test_workspace_folder_priority() { - // Set up logging - let (_non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); - let server = DjangoLanguageServer::new(guard); - - // Test case 1: Workspace folders provided - should use first workspace folder - let workspace_folder1 = WorkspaceFolder { - uri: "file:///workspace/folder1".parse().unwrap(), - name: "workspace1".to_string(), - }; - let workspace_folder2 = WorkspaceFolder { - uri: "file:///workspace/folder2".parse().unwrap(), - name: "workspace2".to_string(), - }; - - let init_params = InitializeParams { - workspace_folders: Some(vec![workspace_folder1.clone(), workspace_folder2.clone()]), - ..Default::default() - }; - - server - .initialize(init_params) - .await - .expect("Failed to initialize"); - server.initialized(InitializedParams {}).await; - - // Check that the session uses the first workspace folder - let project_path = server - .with_session(|session| { - session - .project() - .map(|project| project.path().to_path_buf()) - }) - .await; - - assert_eq!(project_path, Some(PathBuf::from("/workspace/folder1"))); - - // Test case 2: No workspace folders - should fall back to current directory - let (_non_blocking2, guard2) = tracing_appender::non_blocking(std::io::sink()); - let server2 = DjangoLanguageServer::new(guard2); - - let init_params2 = InitializeParams { - workspace_folders: None, - ..Default::default() - }; - - server2 - .initialize(init_params2) - .await - .expect("Failed to initialize"); - server2.initialized(InitializedParams {}).await; - - // Check that the session falls back to current directory - let current_dir = std::env::current_dir().ok(); - let project_path2 = server2 - .with_session(|session| { - session - .project() - .map(|project| project.path().to_path_buf()) - }) - .await; - - assert_eq!(project_path2, current_dir); - - // Test case 3: Empty workspace folders array - should fall back to current directory - let (_non_blocking3, guard3) = tracing_appender::non_blocking(std::io::sink()); - let server3 = DjangoLanguageServer::new(guard3); - - let init_params3 = InitializeParams { - workspace_folders: Some(vec![]), - ..Default::default() - }; - - server3 - .initialize(init_params3) - .await - .expect("Failed to initialize"); - server3.initialized(InitializedParams {}).await; - - // Check that the session falls back to current directory - let project_path3 = server3 - .with_session(|session| { - session - .project() - .map(|project| project.path().to_path_buf()) - }) - .await; - - assert_eq!(project_path3, current_dir); -} - -#[tokio::test] -async fn test_language_id_preservation_during_fallback() { - let server = TestServer::new().await; - let file_name = "template.html"; - - // Open document with htmldjango language_id - let url = server.workspace_url(file_name); - let document = TextDocumentItem { - uri: url.to_string().parse().unwrap(), - language_id: "htmldjango".to_string(), - version: 1, - text: "{% block content %}Initial{% endblock %}".to_string(), - }; - - let params = DidOpenTextDocumentParams { - text_document: document, - }; - server.server.did_open(params).await; - - // Verify the document was opened with the correct language_id - let document = server - .server - .with_session_mut(|session| session.get_document(&url)) - .await; - match document.unwrap().language_id() { - djls_workspace::LanguageId::HtmlDjango => {} // Expected - _ => panic!("Expected HtmlDjango language_id"), - } - - // Simulate a scenario that would trigger the fallback path by sending - // a change with an invalid range that would cause apply_document_changes to fail - let params = DidChangeTextDocumentParams { - text_document: VersionedTextDocumentIdentifier { - uri: url.to_string().parse().unwrap(), - version: 2, - }, - content_changes: vec![TextDocumentContentChangeEvent { - range: Some(Range { - start: Position { - line: 100, - character: 0, - }, // Invalid position - end: Position { - line: 100, - character: 0, - }, - }), - range_length: None, - text: "Fallback content".to_string(), - }], - }; - - server.server.did_change(params).await; - - // Verify the document still has the correct language_id after fallback - let document = server - .server - .with_session_mut(|session| session.get_document(&url)) - .await; - match document.unwrap().language_id() { - djls_workspace::LanguageId::HtmlDjango => {} // Expected - _ => panic!("Expected HtmlDjango language_id after fallback"), - } - - // Also test with a Python file - let py_file_name = "views.py"; - let py_url = server.workspace_url(py_file_name); - let document = TextDocumentItem { - uri: py_url.to_string().parse().unwrap(), - language_id: "python".to_string(), - version: 1, - text: "def hello():\n return 'world'".to_string(), - }; - - let params = DidOpenTextDocumentParams { - text_document: document, - }; - server.server.did_open(params).await; - - // Verify the Python document was opened with the correct language_id - let document = server - .server - .with_session_mut(|session| session.get_document(&py_url)) - .await; - match document.unwrap().language_id() { - djls_workspace::LanguageId::Python => {} // Expected - _ => panic!("Expected Python language_id"), - } - - // Trigger fallback for Python file as well - let params = DidChangeTextDocumentParams { - text_document: VersionedTextDocumentIdentifier { - uri: py_url.to_string().parse().unwrap(), - version: 2, - }, - content_changes: vec![TextDocumentContentChangeEvent { - range: Some(Range { - start: Position { - line: 100, - character: 0, - }, // Invalid position - end: Position { - line: 100, - character: 0, - }, - }), - range_length: None, - text: "def fallback():\n pass".to_string(), - }], - }; - - server.server.did_change(params).await; - - // Verify the Python document still has the correct language_id after fallback - let document = server - .server - .with_session_mut(|session| session.get_document(&py_url)) - .await; - match document.unwrap().language_id() { - djls_workspace::LanguageId::Python => {} // Expected - _ => panic!("Expected Python language_id after fallback"), - } -} diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index 33dfb7ec..4766be27 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -14,7 +14,6 @@ use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use url::Url; use crate::buffers::Buffers; -use crate::db::source_text; use crate::db::Database; use crate::db::SourceFile; use crate::document::TextDocument; @@ -368,56 +367,6 @@ impl Workspace { document } - /// Try to read file content using read-only database access. - /// - /// Returns `Some(content)` if the file exists in the database, `None` otherwise. - /// This avoids write locks for files that are already being tracked. - fn try_read_file(&self, path: &Path) -> Option { - self.with_db(|db| { - if let Some(file) = db.get_file(path) { - tracing::debug!("Using optimized read path for {}", path.display()); - Some(source_text(db, file).to_string()) - } else { - tracing::debug!( - "File {} not in database, requiring creation", - path.display() - ); - None - } - }) - } - - /// Get file content through the database. - /// - /// First attempts read-only access for existing files, then escalates to write - /// access only if the file needs to be created. This improves concurrency by - /// avoiding unnecessary write locks. - pub fn file_content(&mut self, path: &PathBuf) -> String { - // Try read-only access first for existing files - if let Some(content) = self.try_read_file(path) { - return content; - } - - // File doesn't exist, escalate to write access to create it - tracing::debug!( - "Escalating to write access to create file {}", - path.display() - ); - self.with_db_mut(|db| { - let file = db.get_or_create_file(path); - source_text(db, file).to_string() - }) - } - - /// Get the revision number of a file if it exists. - /// - /// Returns `None` if the file is not being tracked by the database. - /// Uses read-only database access since no mutation is needed. - #[must_use] - pub fn file_revision(&self, path: &Path) -> Option { - self.with_db(|db| db.get_file(path).map(|file| file.revision(db))) - } - /// Get a document from the buffer if it's open. /// /// Returns a cloned [`TextDocument`] for the given URL if it exists in buffers. @@ -436,46 +385,6 @@ impl Workspace { } }); } - - /// Get a reference to the buffers. - #[must_use] - pub fn buffers(&self) -> &Buffers { - &self.buffers - } - - /// Get a clone of the file system. - /// - /// Returns an Arc-wrapped [`WorkspaceFileSystem`] that can be shared - /// across threads and used for file operations. - #[must_use] - pub fn file_system(&self) -> Arc { - self.file_system.clone() - } - - /// Get a clone of the files tracking map. - /// - /// Returns an Arc-wrapped [`DashMap`] for O(1) file lookups that - /// can be shared across Database instances. - #[must_use] - pub fn files(&self) -> Arc> { - self.files.clone() - } - - /// Get a cloned database handle for read operations. - /// - /// This provides access to a [`StorageHandle`](salsa::StorageHandle) for cases where - /// [`with_db`](Self::with_db) isn't sufficient. The handle is cloned to allow - /// concurrent read operations. - /// - /// For mutation operations, use [`with_db_mut`](Self::with_db_mut) instead. - /// - /// ## Panics - /// - /// Panics if the handle is currently taken for mutation. - #[must_use] - pub fn db_handle(&self) -> StorageHandle { - self.db_handle.clone_for_read() - } } impl Default for Workspace { @@ -519,35 +428,6 @@ mod tests { assert!(file_exists); } - #[test] - fn test_read_access_during_no_mutation() { - let workspace = Workspace::new(); - - // Multiple concurrent reads should work - let handle1 = workspace.db_handle(); - let handle2 = workspace.db_handle(); - - // Both handles should be valid - let storage1 = handle1.into_storage(); - let storage2 = handle2.into_storage(); - - // Should be able to create databases from both - let db1 = Database::from_storage( - storage1, - workspace.file_system.clone(), - workspace.files.clone(), - ); - let db2 = Database::from_storage( - storage2, - workspace.file_system.clone(), - workspace.files.clone(), - ); - - // Both should work - assert!(!db1.has_file(&PathBuf::from("nonexistent.py"))); - assert!(!db2.has_file(&PathBuf::from("nonexistent.py"))); - } - #[test] #[should_panic( expected = "Cannot access database handle for read - handle is currently taken for mutation" @@ -702,34 +582,6 @@ mod tests { assert!(!result2); } - #[test] - fn test_safe_storage_handle_state_transitions() { - let mut workspace = Workspace::new(); - - // Start in Available state - should be able to clone for read - let _handle = workspace.db_handle(); - - // Take for mutation - let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); - let handle = guard.handle(); - - // Now should be in TakenForMutation state - // Convert to storage for testing - let storage = handle.into_storage(); - let db = Database::from_storage( - storage, - workspace.file_system.clone(), - workspace.files.clone(), - ); - let new_handle = db.storage().clone().into_zalsa_handle(); - - // Restore - should return to Available state - guard.restore(new_handle); - - // Should be able to read again - let _handle = workspace.db_handle(); - } - #[test] #[should_panic(expected = "Cannot restore handle - it hasn't been consumed yet")] fn test_panic_on_restore_without_consume() { From 576d86c6a2f261717067b50c05b623a9da63c642 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 22:18:36 -0500 Subject: [PATCH 47/56] remove --- crates/djls-workspace/src/db.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index b3cf2ef2..2f105e95 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -202,13 +202,6 @@ impl Database { pub fn storage(&self) -> &salsa::Storage { &self.storage } - - /// Consume the database and return its storage. - /// - /// This is used when you need to take ownership of the storage. - pub fn into_storage(self) -> salsa::Storage { - self.storage - } } #[salsa::db] From 958c152bda6d602e5871279bdf934a430fcd3e83 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 23:20:42 -0500 Subject: [PATCH 48/56] cleaning --- crates/djls-server/src/completions.rs | 16 +- crates/djls-server/src/server.rs | 32 +- crates/djls-server/src/session.rs | 28 +- crates/djls-workspace/src/buffers.rs | 8 + crates/djls-workspace/src/db.rs | 13 +- crates/djls-workspace/src/lib.rs | 1 + crates/djls-workspace/src/paths.rs | 43 ++ crates/djls-workspace/src/storage.rs | 222 ++++++++++ crates/djls-workspace/src/workspace.rs | 572 +------------------------ 9 files changed, 331 insertions(+), 604 deletions(-) create mode 100644 crates/djls-workspace/src/storage.rs diff --git a/crates/djls-server/src/completions.rs b/crates/djls-server/src/completions.rs index 8e3180ac..b9054f69 100644 --- a/crates/djls-server/src/completions.rs +++ b/crates/djls-server/src/completions.rs @@ -83,7 +83,6 @@ fn get_line_info( position: Position, encoding: PositionEncoding, ) -> Option { - // Get the line content and calculate cursor position within line let content = document.content(); let lines: Vec<&str> = content.lines().collect(); @@ -94,12 +93,17 @@ fn get_line_info( let line_text = lines[line_index].to_string(); - // For UTF-16 encoding, we need to convert the character position + // Convert LSP position to character index for Vec operations. + // + // LSP default encoding is UTF-16 (emoji = 2 units), but we need + // character counts (emoji = 1 char) to index into chars[..offset]. + // + // Example: + // "h€llo" cursor after € → UTF-16: 2, chars: 2 ✓, bytes: 4 ✗ let cursor_offset_in_line = match encoding { PositionEncoding::Utf16 => { - // Convert UTF-16 position to UTF-8 character offset let utf16_pos = position.character as usize; - let mut utf8_offset = 0; + let mut char_offset = 0; // Count chars, not bytes let mut utf16_offset = 0; for ch in line_text.chars() { @@ -107,9 +111,9 @@ fn get_line_info( break; } utf16_offset += ch.len_utf16(); - utf8_offset += 1; + char_offset += 1; } - utf8_offset + char_offset } _ => position.character as usize, }; diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index 8eae018b..a13b8f63 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -8,7 +8,6 @@ use tower_lsp_server::jsonrpc::Result as LspResult; use tower_lsp_server::lsp_types; use tower_lsp_server::LanguageServer; use tracing_appender::non_blocking::WorkerGuard; -use url::Url; use crate::queue::Queue; use crate::session::Session; @@ -208,9 +207,12 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Opened document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - // Convert LSP types to our types - let url = - Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + let Some(url) = + paths::parse_lsp_uri(¶ms.text_document.uri, paths::LspContext::DidOpen) + else { + return; // Skip processing this document + }; + let language_id = djls_workspace::LanguageId::from(params.text_document.language_id.as_str()); let document = djls_workspace::TextDocument::new( @@ -228,8 +230,11 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Changed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let url = - Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + let Some(url) = + paths::parse_lsp_uri(¶ms.text_document.uri, paths::LspContext::DidChange) + else { + return; // Skip processing this change + }; session.update_document(&url, params.content_changes, params.text_document.version); }) @@ -240,8 +245,11 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Closed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let url = - Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + let Some(url) = + paths::parse_lsp_uri(¶ms.text_document.uri, paths::LspContext::DidClose) + else { + return; // Skip processing this close + }; if session.close_document(&url).is_none() { tracing::warn!("Attempted to close document without overlay: {}", url); @@ -256,8 +264,12 @@ impl LanguageServer for DjangoLanguageServer { ) -> LspResult> { let response = self .with_session_mut(|session| { - let lsp_uri = params.text_document_position.text_document.uri; - let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); + let Some(url) = paths::parse_lsp_uri( + ¶ms.text_document_position.text_document.uri, + paths::LspContext::Completion, + ) else { + return None; // Return no completions for invalid URI + }; tracing::debug!( "Completion requested for {} at {:?}", diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 1b10e3f7..d96e3e28 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -3,8 +3,6 @@ //! This module implements the LSP session abstraction that manages project-specific //! state and delegates workspace operations to the Workspace facade. -use std::path::PathBuf; - use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::paths; @@ -47,7 +45,15 @@ pub struct Session { impl Session { pub fn new(params: &lsp_types::InitializeParams) -> Self { - let project_path = Self::get_project_path(params); + let project_path = params + .workspace_folders + .as_ref() + .and_then(|folders| folders.first()) + .and_then(|folder| paths::lsp_uri_to_path(&folder.uri)) + .or_else(|| { + // Fall back to current directory + std::env::current_dir().ok() + }); let (project, settings) = if let Some(path) = &project_path { let settings = @@ -74,22 +80,6 @@ impl Session { } } - /// Determines the project root path from initialization parameters. - /// - /// Tries workspace folders first (using the first one), then falls back to current directory. - fn get_project_path(params: &lsp_types::InitializeParams) -> Option { - // Try workspace folders first - params - .workspace_folders - .as_ref() - .and_then(|folders| folders.first()) - .and_then(|folder| paths::lsp_uri_to_path(&folder.uri)) - .or_else(|| { - // Fall back to current directory - std::env::current_dir().ok() - }) - } - #[must_use] pub fn project(&self) -> Option<&DjangoProject> { self.project.as_ref() diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 1e3fd0fe..eb9c0828 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -23,6 +23,14 @@ use crate::document::TextDocument; /// The [`WorkspaceFileSystem`] holds a clone of this structure and checks /// it before falling back to disk reads. /// +/// ## Memory Management +/// +/// This structure does not implement eviction or memory limits because the +/// LSP protocol explicitly manages document lifecycle through `didOpen` and +/// `didClose` notifications. Documents are only stored while the editor has +/// them open, and are properly removed when the editor closes them. This +/// follows the battle-tested pattern used by production LSP servers like Ruff. +/// /// [`FileSystem`]: crate::fs::FileSystem /// [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem #[derive(Clone, Debug)] diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 2f105e95..e98fdfc7 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -142,12 +142,17 @@ impl Database { /// /// Files are created with an initial revision of 0 and tracked in the [`Database`]'s /// `DashMap`. The `Arc` ensures cheap cloning while maintaining thread safety. + /// + /// ## Thread Safety + /// + /// This method is inherently thread-safe despite the check-then-create pattern because + /// it requires `&mut self`, ensuring exclusive access to the Database. Only one thread + /// can call this method at a time due to Rust's ownership rules. pub fn get_or_create_file(&mut self, path: &PathBuf) -> SourceFile { if let Some(file_ref) = self.files.get(path) { - // Copy the value (SourceFile is Copy) and drop the guard immediately - let file = *file_ref; - drop(file_ref); // Explicitly drop the guard to release the lock - return file; + // Copy the value (SourceFile is Copy) + // The guard drops automatically, no need for explicit drop + return *file_ref; } // File doesn't exist, so we need to create it diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 7eb60b4c..532f88c9 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -19,6 +19,7 @@ pub mod encoding; mod fs; mod language; pub mod paths; +mod storage; mod workspace; use std::path::Path; diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 1d2b47f8..89241544 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -33,6 +33,49 @@ pub fn url_to_path(url: &Url) -> Option { Some(PathBuf::from(path.as_ref())) } +/// Context for LSP operations, used for error reporting +#[derive(Debug, Clone, Copy)] +pub enum LspContext { + /// textDocument/didOpen notification + DidOpen, + /// textDocument/didChange notification + DidChange, + /// textDocument/didClose notification + DidClose, + /// textDocument/completion request + Completion, +} + +impl std::fmt::Display for LspContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::DidOpen => write!(f, "didOpen"), + Self::DidChange => write!(f, "didChange"), + Self::DidClose => write!(f, "didClose"), + Self::Completion => write!(f, "completion"), + } + } +} + +/// Parse an LSP URI to a [`Url`], logging errors if parsing fails. +/// +/// This function is designed for use in LSP notification handlers where +/// invalid URIs should be logged but not crash the server. +pub fn parse_lsp_uri(lsp_uri: &lsp_types::Uri, context: LspContext) -> Option { + match Url::parse(lsp_uri.as_str()) { + Ok(url) => Some(url), + Err(e) => { + tracing::error!( + "Invalid URI from LSP client in {}: {} - Error: {}", + context, + lsp_uri.as_str(), + e + ); + None + } + } +} + /// Convert an LSP URI to a [`PathBuf`]. /// /// This is a convenience wrapper that parses the LSP URI string and converts it. diff --git a/crates/djls-workspace/src/storage.rs b/crates/djls-workspace/src/storage.rs new file mode 100644 index 00000000..09db4eb0 --- /dev/null +++ b/crates/djls-workspace/src/storage.rs @@ -0,0 +1,222 @@ +use salsa::StorageHandle; + +use crate::db::Database; + +/// Safe wrapper for [`StorageHandle`](salsa::StorageHandle) that prevents misuse through type safety. +/// +/// This enum ensures that database handles can only be in one of two valid states, +/// making invalid states unrepresentable and eliminating the need for placeholder +/// handles during mutations. +/// +/// ## Panic Behavior +/// +/// Methods in this type may panic when the state machine invariants are violated. +/// These panics represent **programming bugs**, not runtime errors that should be +/// handled. They indicate violations of the internal API contract, similar to how +/// `RefCell::borrow_mut()` panics on double borrows. The panics ensure that bugs +/// are caught during development rather than causing silent data corruption. +pub enum SafeStorageHandle { + /// Handle is available for use + Available(StorageHandle), + /// Handle has been taken for mutation - no handle available + TakenForMutation, +} + +impl SafeStorageHandle { + /// Create a new `SafeStorageHandle` in the `Available` state + pub fn new(handle: StorageHandle) -> Self { + Self::Available(handle) + } + + /// Take the handle for mutation, leaving the enum in `TakenForMutation` state. + /// + /// ## Panics + /// + /// Panics if the handle has already been taken for mutation. + pub fn take_for_mutation(&mut self) -> StorageHandle { + match std::mem::replace(self, Self::TakenForMutation) { + Self::Available(handle) => handle, + Self::TakenForMutation => panic!( + "Database handle already taken for mutation. This indicates a programming error - \ + ensure you're not calling multiple mutation operations concurrently or forgetting \ + to restore the handle after a previous mutation." + ), + } + } + + /// Restore the handle after mutation, returning it to `Available` state. + /// + /// ## Panics + /// + /// Panics if the handle is not currently taken for mutation. + pub fn restore_from_mutation(&mut self, handle: StorageHandle) { + match self { + Self::TakenForMutation => { + *self = Self::Available(handle); + } + Self::Available(_) => panic!( + "Cannot restore database handle - handle is not currently taken for mutation. \ + This indicates a programming error in the StorageHandleGuard implementation." + ), + } + } + + /// Get a clone of the handle for read-only operations. + /// + /// ## Panics + /// + /// Panics if the handle is currently taken for mutation. + pub fn clone_for_read(&self) -> StorageHandle { + match self { + Self::Available(handle) => handle.clone(), + Self::TakenForMutation => panic!( + "Cannot access database handle for read - handle is currently taken for mutation. \ + Wait for the current mutation operation to complete." + ), + } + } + + /// Take the handle for mutation with automatic restoration via guard. + /// This ensures the handle is always restored even if the operation panics. + pub fn take_guarded(&mut self) -> StorageHandleGuard { + StorageHandleGuard::new(self) + } +} + +/// State of the [`StorageHandleGuard`] during its lifetime. +/// +/// See [`StorageHandleGuard`] for usage and state machine details. +enum GuardState { + /// Guard holds the handle, ready to be consumed + Active { handle: StorageHandle }, + /// Handle consumed, awaiting restoration + Consumed, + /// Handle restored to [`SafeStorageHandle`] + Restored, +} + +/// RAII guard for safe [`StorageHandle`](salsa::StorageHandle) management during mutations. +/// +/// This guard ensures that database handles are automatically restored even if +/// panics occur during mutation operations. It prevents double-takes and +/// provides clear error messages for misuse. +/// +/// ## State Machine +/// +/// The guard follows these valid state transitions: +/// - `Active` → `Consumed` (via `handle()` method) +/// - `Consumed` → `Restored` (via `restore()` method) +/// +/// ## Invalid Transitions +/// +/// Invalid operations will panic with specific error messages: +/// - `handle()` on `Consumed` state: "[`StorageHandle`](salsa::StorageHandle) already consumed" +/// - `handle()` on `Restored` state: "Cannot consume handle - guard has already been restored" +/// - `restore()` on `Active` state: "Cannot restore handle - it hasn't been consumed yet" +/// - `restore()` on `Restored` state: "Handle has already been restored" +/// +/// ## Drop Behavior +/// +/// The guard will panic on drop unless it's in the `Restored` state: +/// - Drop in `Active` state: "`StorageHandleGuard` dropped without using the handle" +/// - Drop in `Consumed` state: "`StorageHandleGuard` dropped without restoring handle" +/// - Drop in `Restored` state: No panic - proper cleanup completed +/// +/// ## Usage Example +/// +/// ```rust,ignore +/// let mut guard = StorageHandleGuard::new(&mut safe_handle); +/// let handle = guard.handle(); // Active → Consumed +/// // ... perform mutations with handle ... +/// guard.restore(updated_handle); // Consumed → Restored +/// // Guard drops cleanly in Restored state +/// ``` +#[must_use = "StorageHandleGuard must be used - dropping it immediately defeats the purpose"] +pub struct StorageHandleGuard<'a> { + /// Reference to the workspace's `SafeStorageHandle` for restoration + safe_handle: &'a mut SafeStorageHandle, + /// Current state of the guard and handle + state: GuardState, +} + +impl<'a> StorageHandleGuard<'a> { + /// Create a new guard by taking the handle from the `SafeStorageHandle`. + pub fn new(safe_handle: &'a mut SafeStorageHandle) -> Self { + let handle = safe_handle.take_for_mutation(); + Self { + safe_handle, + state: GuardState::Active { handle }, + } + } + + /// Get the [`StorageHandle`](salsa::StorageHandle) for mutation operations. + /// + /// ## Panics + /// + /// Panics if the handle has already been consumed or restored. + pub fn handle(&mut self) -> StorageHandle { + match std::mem::replace(&mut self.state, GuardState::Consumed) { + GuardState::Active { handle } => handle, + GuardState::Consumed => panic!( + "StorageHandle already consumed from guard. Each guard can only provide \ + the handle once - this prevents accidental multiple uses." + ), + GuardState::Restored => panic!( + "Cannot consume handle - guard has already been restored. Once restored, \ + the guard cannot provide the handle again." + ), + } + } + + /// Restore the handle manually before the guard drops. + /// + /// This is useful when you want to restore the handle and continue using + /// the workspace in the same scope. + /// + /// ## Panics + /// + /// Panics if the handle hasn't been consumed yet, or if already restored. + pub fn restore(mut self, handle: StorageHandle) { + match self.state { + GuardState::Consumed => { + self.safe_handle.restore_from_mutation(handle); + self.state = GuardState::Restored; + } + GuardState::Active { .. } => panic!( + "Cannot restore handle - it hasn't been consumed yet. Call guard.handle() \ + first to get the handle, then restore the updated handle after mutations." + ), + GuardState::Restored => { + panic!("Handle has already been restored. Each guard can only restore once.") + } + } + } +} + +impl Drop for StorageHandleGuard<'_> { + fn drop(&mut self) { + // Provide specific error messages based on the exact state + // Avoid double-panic during unwinding + if !std::thread::panicking() { + match &self.state { + GuardState::Active { .. } => { + panic!( + "StorageHandleGuard dropped without using the handle. Either call \ + guard.handle() to consume the handle for mutations, or ensure the \ + guard is properly used in your mutation workflow." + ); + } + GuardState::Consumed => { + panic!( + "StorageHandleGuard dropped without restoring handle. You must call \ + guard.restore(updated_handle) to properly restore the database handle \ + after mutation operations complete." + ); + } + GuardState::Restored => { + // All good - proper cleanup completed + } + } + } + } +} diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index 4766be27..bfb9ac75 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -9,7 +9,6 @@ use std::path::PathBuf; use std::sync::Arc; use dashmap::DashMap; -use salsa::StorageHandle; use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use url::Url; @@ -20,211 +19,7 @@ use crate::document::TextDocument; use crate::fs::OsFileSystem; use crate::fs::WorkspaceFileSystem; use crate::paths::url_to_path; - -/// Safe wrapper for [`StorageHandle`](salsa::StorageHandle) that prevents misuse through type safety. -/// -/// This enum ensures that database handles can only be in one of two valid states, -/// making invalid states unrepresentable and eliminating the need for placeholder -/// handles during mutations. -enum SafeStorageHandle { - /// Handle is available for use - Available(StorageHandle), - /// Handle has been taken for mutation - no handle available - TakenForMutation, -} - -impl SafeStorageHandle { - /// Create a new `SafeStorageHandle` in the `Available` state - fn new(handle: StorageHandle) -> Self { - Self::Available(handle) - } - - /// Take the handle for mutation, leaving the enum in `TakenForMutation` state. - /// - /// ## Panics - /// - /// Panics if the handle has already been taken for mutation. - fn take_for_mutation(&mut self) -> StorageHandle { - match std::mem::replace(self, Self::TakenForMutation) { - Self::Available(handle) => handle, - Self::TakenForMutation => panic!( - "Database handle already taken for mutation. This indicates a programming error - \ - ensure you're not calling multiple mutation operations concurrently or forgetting \ - to restore the handle after a previous mutation." - ), - } - } - - /// Restore the handle after mutation, returning it to `Available` state. - /// - /// ## Panics - /// - /// Panics if the handle is not currently taken for mutation. - fn restore_from_mutation(&mut self, handle: StorageHandle) { - match self { - Self::TakenForMutation => { - *self = Self::Available(handle); - } - Self::Available(_) => panic!( - "Cannot restore database handle - handle is not currently taken for mutation. \ - This indicates a programming error in the StorageHandleGuard implementation." - ), - } - } - - /// Get a clone of the handle for read-only operations. - /// - /// ## Panics - /// - /// Panics if the handle is currently taken for mutation. - fn clone_for_read(&self) -> StorageHandle { - match self { - Self::Available(handle) => handle.clone(), - Self::TakenForMutation => panic!( - "Cannot access database handle for read - handle is currently taken for mutation. \ - Wait for the current mutation operation to complete." - ), - } - } -} - -/// State of the [`StorageHandleGuard`] during its lifetime. -/// -/// See [`StorageHandleGuard`] for usage and state machine details. -enum GuardState { - /// Guard holds the handle, ready to be consumed - Active { handle: StorageHandle }, - /// Handle consumed, awaiting restoration - Consumed, - /// Handle restored to [`SafeStorageHandle`] - Restored, -} - -/// RAII guard for safe [`StorageHandle`](salsa::StorageHandle) management during mutations. -/// -/// This guard ensures that database handles are automatically restored even if -/// panics occur during mutation operations. It prevents double-takes and -/// provides clear error messages for misuse. -/// -/// ## State Machine -/// -/// The guard follows these valid state transitions: -/// - `Active` → `Consumed` (via `handle()` method) -/// - `Consumed` → `Restored` (via `restore()` method) -/// -/// ## Invalid Transitions -/// -/// Invalid operations will panic with specific error messages: -/// - `handle()` on `Consumed` state: "[`StorageHandle`](salsa::StorageHandle) already consumed" -/// - `handle()` on `Restored` state: "Cannot consume handle - guard has already been restored" -/// - `restore()` on `Active` state: "Cannot restore handle - it hasn't been consumed yet" -/// - `restore()` on `Restored` state: "Handle has already been restored" -/// -/// ## Drop Behavior -/// -/// The guard will panic on drop unless it's in the `Restored` state: -/// - Drop in `Active` state: "`StorageHandleGuard` dropped without using the handle" -/// - Drop in `Consumed` state: "`StorageHandleGuard` dropped without restoring handle" -/// - Drop in `Restored` state: No panic - proper cleanup completed -/// -/// ## Usage Example -/// -/// ```rust,ignore -/// let mut guard = StorageHandleGuard::new(&mut safe_handle); -/// let handle = guard.handle(); // Active → Consumed -/// // ... perform mutations with handle ... -/// guard.restore(updated_handle); // Consumed → Restored -/// // Guard drops cleanly in Restored state -/// ``` -#[must_use = "StorageHandleGuard must be used - dropping it immediately defeats the purpose"] -pub struct StorageHandleGuard<'a> { - /// Reference to the workspace's `SafeStorageHandle` for restoration - safe_handle: &'a mut SafeStorageHandle, - /// Current state of the guard and handle - state: GuardState, -} - -impl<'a> StorageHandleGuard<'a> { - /// Create a new guard by taking the handle from the `SafeStorageHandle`. - fn new(safe_handle: &'a mut SafeStorageHandle) -> Self { - let handle = safe_handle.take_for_mutation(); - Self { - safe_handle, - state: GuardState::Active { handle }, - } - } - - /// Get the [`StorageHandle`](salsa::StorageHandle) for mutation operations. - /// - /// ## Panics - /// - /// Panics if the handle has already been consumed or restored. - pub fn handle(&mut self) -> StorageHandle { - match std::mem::replace(&mut self.state, GuardState::Consumed) { - GuardState::Active { handle } => handle, - GuardState::Consumed => panic!( - "StorageHandle already consumed from guard. Each guard can only provide \ - the handle once - this prevents accidental multiple uses." - ), - GuardState::Restored => panic!( - "Cannot consume handle - guard has already been restored. Once restored, \ - the guard cannot provide the handle again." - ), - } - } - - /// Restore the handle manually before the guard drops. - /// - /// This is useful when you want to restore the handle and continue using - /// the workspace in the same scope. - /// - /// ## Panics - /// - /// Panics if the handle hasn't been consumed yet, or if already restored. - pub fn restore(mut self, handle: StorageHandle) { - match self.state { - GuardState::Consumed => { - self.safe_handle.restore_from_mutation(handle); - self.state = GuardState::Restored; - } - GuardState::Active { .. } => panic!( - "Cannot restore handle - it hasn't been consumed yet. Call guard.handle() \ - first to get the handle, then restore the updated handle after mutations." - ), - GuardState::Restored => { - panic!("Handle has already been restored. Each guard can only restore once.") - } - } - } -} - -impl Drop for StorageHandleGuard<'_> { - fn drop(&mut self) { - // Provide specific error messages based on the exact state - // Avoid double-panic during unwinding - if !std::thread::panicking() { - match &self.state { - GuardState::Active { .. } => { - panic!( - "StorageHandleGuard dropped without using the handle. Either call \ - guard.handle() to consume the handle for mutations, or ensure the \ - guard is properly used in your mutation workflow." - ); - } - GuardState::Consumed => { - panic!( - "StorageHandleGuard dropped without restoring handle. You must call \ - guard.restore(updated_handle) to properly restore the database handle \ - after mutation operations complete." - ); - } - GuardState::Restored => { - // All good - proper cleanup completed - } - } - } - } -} +use crate::storage::SafeStorageHandle; /// Workspace facade that encapsulates all workspace components. /// @@ -287,7 +82,7 @@ impl Workspace { where F: FnOnce(&mut Database) -> R, { - let mut guard = StorageHandleGuard::new(&mut self.db_handle); + let mut guard = self.db_handle.take_guarded(); let handle = guard.handle(); let storage = handle.into_storage(); let mut db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); @@ -385,6 +180,11 @@ impl Workspace { } }); } + + #[must_use] + pub fn db_handle(&self) -> &SafeStorageHandle { + &self.db_handle + } } impl Default for Workspace { @@ -392,361 +192,3 @@ impl Default for Workspace { Self::new() } } - -#[cfg(test)] -mod tests { - #[cfg(unix)] - use std::os::unix::fs::PermissionsExt; - use std::str::FromStr; - use std::sync::Arc; - use std::sync::Mutex; - use std::time::Duration; - - use tempfile::tempdir; - - use super::*; - use crate::db::source_text; - - #[test] - fn test_normal_mutation_flow_with_guard() { - let mut workspace = Workspace::new(); - - // Normal mutation should work fine - let result = workspace.with_db_mut(|db| { - // Simple operation - create a file - let path = PathBuf::from("test.py"); - let file = db.get_or_create_file(&path); - file.revision(db) // Return the revision number - }); - - // Should complete successfully - initial revision is 0 - assert_eq!(result, 0); - - // Should be able to read after mutation - let file_exists = workspace.with_db(|db| db.has_file(&PathBuf::from("test.py"))); - - assert!(file_exists); - } - - #[test] - #[should_panic( - expected = "Cannot access database handle for read - handle is currently taken for mutation" - )] - fn test_panic_on_read_during_mutation() { - // This test is tricky due to Rust's borrowing rules. - // Test the SafeStorageHandle directly instead of through Workspace - let mut safe_handle = SafeStorageHandle::new( - Database::new( - Arc::new(crate::fs::WorkspaceFileSystem::new( - crate::buffers::Buffers::new(), - Arc::new(crate::fs::OsFileSystem), - )), - Arc::new(DashMap::new()), - ) - .storage() - .clone() - .into_zalsa_handle(), - ); - - // Take the handle - let _handle = safe_handle.take_for_mutation(); - - // Now trying to read should panic - let _cloned_handle = safe_handle.clone_for_read(); - } - - #[test] - #[should_panic(expected = "Database handle already taken for mutation")] - fn test_panic_on_double_take() { - let mut safe_handle = SafeStorageHandle::new( - Database::new( - Arc::new(crate::fs::WorkspaceFileSystem::new( - crate::buffers::Buffers::new(), - Arc::new(crate::fs::OsFileSystem), - )), - Arc::new(DashMap::new()), - ) - .storage() - .clone() - .into_zalsa_handle(), - ); - - // First take should work - let _handle1 = safe_handle.take_for_mutation(); - - // Second take should panic - let _handle2 = safe_handle.take_for_mutation(); - } - - #[test] - #[should_panic(expected = "StorageHandle already consumed from guard")] - fn test_panic_on_double_handle_consumption() { - let mut workspace = Workspace::new(); - let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); - - // First consumption should work - let _handle1 = guard.handle(); - - // Second consumption should panic - let _handle2 = guard.handle(); - } - - #[test] - fn test_manual_restore() { - let mut workspace = Workspace::new(); - - // Take handle manually - let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); - let handle = guard.handle(); - - // Use it to create a database - let storage = handle.into_storage(); - let mut db = Database::from_storage( - storage, - workspace.file_system.clone(), - workspace.files.clone(), - ); - - // Make some changes - let path = PathBuf::from("manual_test.py"); - let _file = db.get_or_create_file(&path); - - // Extract new handle and restore manually - let new_handle = db.storage().clone().into_zalsa_handle(); - guard.restore(new_handle); - - // Should be able to read now - let file_exists = workspace.with_db(|db| db.has_file(&PathBuf::from("manual_test.py"))); - - assert!(file_exists); - } - - #[test] - #[should_panic(expected = "StorageHandleGuard dropped without restoring handle")] - fn test_panic_on_guard_drop_without_restore() { - let mut workspace = Workspace::new(); - - // Create guard and consume handle but don't restore - let mut guard = StorageHandleGuard::new(&mut workspace.db_handle); - let _handle = guard.handle(); - - // Guard drops here without restore - should panic - } - - #[test] - fn test_event_callbacks_preserved() { - // This test ensures that the new implementation preserves event callbacks - // through mutation cycles, unlike the old placeholder approach - - let mut workspace = Workspace::new(); - - // Add a file to create some state - let initial_file_count = workspace.with_db_mut(|db| { - let path = PathBuf::from("callback_test.py"); - let _file = db.get_or_create_file(&path); - 1 // Return count - }); - - assert_eq!(initial_file_count, 1); - - // Perform another mutation to ensure callbacks are preserved - let final_file_count = workspace.with_db_mut(|db| { - let path = PathBuf::from("callback_test2.py"); - let _file = db.get_or_create_file(&path); - - // Count files - should include both - let has_first = db.has_file(&PathBuf::from("callback_test.py")); - let has_second = db.has_file(&PathBuf::from("callback_test2.py")); - - if has_first && has_second { - 2 - } else { - 0 - } - }); - - assert_eq!(final_file_count, 2); - } - - #[test] - fn test_concurrent_read_operations() { - let workspace = Workspace::new(); - - // Multiple with_db calls should work concurrently - let result1 = workspace.with_db(|db| db.has_file(&PathBuf::from("test1.py"))); - - let result2 = workspace.with_db(|db| db.has_file(&PathBuf::from("test2.py"))); - - // Both should complete successfully - assert!(!result1); - assert!(!result2); - } - - #[test] - #[should_panic(expected = "Cannot restore handle - it hasn't been consumed yet")] - fn test_panic_on_restore_without_consume() { - let mut workspace = Workspace::new(); - let guard = StorageHandleGuard::new(&mut workspace.db_handle); - - // Create a dummy handle for testing - let dummy_handle = Database::new( - Arc::new(crate::fs::WorkspaceFileSystem::new( - crate::buffers::Buffers::new(), - Arc::new(crate::fs::OsFileSystem), - )), - Arc::new(DashMap::new()), - ) - .storage() - .clone() - .into_zalsa_handle(); - - // Try to restore without consuming first - should panic - guard.restore(dummy_handle); - } - - #[test] - #[should_panic(expected = "StorageHandleGuard dropped without using the handle")] - fn test_panic_on_guard_drop_without_use() { - let mut workspace = Workspace::new(); - - // Create guard but don't use the handle - should panic on drop - let _guard = StorageHandleGuard::new(&mut workspace.db_handle); - - // Guard drops here without handle() being called - } - - #[test] - fn test_missing_file_returns_empty_content() { - // Tests that source_text returns "" for non-existent files - // instead of panicking or propagating errors - let mut workspace = Workspace::new(); - - // Create a file reference for non-existent path - let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(&PathBuf::from_str("/nonexistent/file.py").unwrap()); - source_text(db, file).to_string() - }); - - assert_eq!(content, ""); - } - - #[test] - #[cfg(unix)] - fn test_permission_denied_file_handling() { - // Create a file with no read permissions - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("no_read.py"); - std::fs::write(&file_path, "content").unwrap(); - - // Remove read permissions - std::fs::set_permissions(&file_path, std::fs::Permissions::from_mode(0o000)).unwrap(); - - let mut workspace = Workspace::new(); - let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(&file_path); - source_text(db, file).to_string() - }); - - // Should return empty string, not panic - assert_eq!(content, ""); - } - - #[test] - fn test_invalid_utf8_file_handling() { - // Create a file with invalid UTF-8 - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("invalid.py"); - std::fs::write(&file_path, [0xFF, 0xFE, 0xFD]).unwrap(); - - let mut workspace = Workspace::new(); - let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(&file_path); - source_text(db, file).to_string() - }); - - // Should handle gracefully (empty or replacement chars) - assert!(content.is_empty() || content.contains('�')); - } - - #[test] - fn test_file_deleted_after_tracking() { - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("disappearing.py"); - std::fs::write(&file_path, "original").unwrap(); - - let mut workspace = Workspace::new(); - - // First read should succeed - let content1 = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(&file_path); - source_text(db, file).to_string() - }); - assert_eq!(content1, "original"); - - // Delete the file - std::fs::remove_file(&file_path).unwrap(); - - // Touch to invalidate cache - workspace.with_db_mut(|db| { - db.touch_file(&file_path); - }); - - // Second read should return empty (not panic) - let content2 = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(&file_path); - source_text(db, file).to_string() - }); - assert_eq!(content2, ""); - } - - #[test] - #[cfg(unix)] - fn test_broken_symlink_handling() { - let temp_dir = tempdir().unwrap(); - let symlink_path = temp_dir.path().join("broken_link.py"); - - // Create broken symlink - std::os::unix::fs::symlink("/nonexistent/target", &symlink_path).unwrap(); - - let mut workspace = Workspace::new(); - let content = workspace.with_db_mut(|db| { - let file = db.get_or_create_file(&symlink_path); - source_text(db, file).to_string() - }); - - // Should handle gracefully - assert_eq!(content, ""); - } - - #[test] - fn test_file_modified_during_operations() { - // Tests that concurrent file modifications don't crash - let temp_dir = tempdir().unwrap(); - let file_path = temp_dir.path().join("racing.py"); - - let workspace = Arc::new(Mutex::new(Workspace::new())); - let path_clone = file_path.clone(); - let workspace_clone = workspace.clone(); - - // Writer thread - let writer = std::thread::spawn(move || { - for i in 0..10 { - std::fs::write(&path_clone, format!("version {i}")).ok(); - std::thread::sleep(Duration::from_millis(10)); - } - }); - - // Reader thread - should never panic - for _ in 0..10 { - let content = workspace_clone.lock().unwrap().with_db_mut(|db| { - let file = db.get_or_create_file(&file_path); - source_text(db, file).to_string() - }); - // Content may vary but shouldn't crash - assert!(content.is_empty() || content.starts_with("version")); - std::thread::sleep(Duration::from_millis(5)); - } - - writer.join().unwrap(); - } -} From 9a08fb6f92bcfd7106d77b0a4421e2176c853f51 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 23:35:58 -0500 Subject: [PATCH 49/56] clean it UP --- crates/djls-server/src/server.rs | 8 +- crates/djls-server/src/session.rs | 9 +- crates/djls-workspace/src/storage.rs | 120 ++++++++++++++ crates/djls-workspace/src/workspace.rs | 220 +++++++++++++++++++++++++ 4 files changed, 346 insertions(+), 11 deletions(-) diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index a13b8f63..f0ef4c1c 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -210,7 +210,7 @@ impl LanguageServer for DjangoLanguageServer { let Some(url) = paths::parse_lsp_uri(¶ms.text_document.uri, paths::LspContext::DidOpen) else { - return; // Skip processing this document + return; // Error parsing uri (unlikely), skip processing this document }; let language_id = @@ -233,7 +233,7 @@ impl LanguageServer for DjangoLanguageServer { let Some(url) = paths::parse_lsp_uri(¶ms.text_document.uri, paths::LspContext::DidChange) else { - return; // Skip processing this change + return; // Error parsing uri (unlikely), skip processing this change }; session.update_document(&url, params.content_changes, params.text_document.version); @@ -248,7 +248,7 @@ impl LanguageServer for DjangoLanguageServer { let Some(url) = paths::parse_lsp_uri(¶ms.text_document.uri, paths::LspContext::DidClose) else { - return; // Skip processing this close + return; // Error parsing uri (unlikely), skip processing this close }; if session.close_document(&url).is_none() { @@ -268,7 +268,7 @@ impl LanguageServer for DjangoLanguageServer { ¶ms.text_document_position.text_document.uri, paths::LspContext::Completion, ) else { - return None; // Return no completions for invalid URI + return None; // Error parsing uri (unlikely), return no completions }; tracing::debug!( diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index d96e3e28..b3c19b6c 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -66,17 +66,12 @@ impl Session { (None, Settings::default()) }; - let workspace = Workspace::new(); - - // Negotiate position encoding with client - let position_encoding = PositionEncoding::negotiate(params); - Self { project, settings, - workspace, + workspace: Workspace::new(), client_capabilities: params.capabilities.clone(), - position_encoding, + position_encoding: PositionEncoding::negotiate(params), } } diff --git a/crates/djls-workspace/src/storage.rs b/crates/djls-workspace/src/storage.rs index 09db4eb0..29a3880c 100644 --- a/crates/djls-workspace/src/storage.rs +++ b/crates/djls-workspace/src/storage.rs @@ -220,3 +220,123 @@ impl Drop for StorageHandleGuard<'_> { } } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use dashmap::DashMap; + + use super::*; + use crate::buffers::Buffers; + use crate::fs::{OsFileSystem, WorkspaceFileSystem}; + + fn create_test_handle() -> StorageHandle { + Database::new( + Arc::new(WorkspaceFileSystem::new( + Buffers::new(), + Arc::new(OsFileSystem), + )), + Arc::new(DashMap::new()), + ) + .storage() + .clone() + .into_zalsa_handle() + } + + #[test] + fn test_handle_lifecycle() { + // Test the happy path: take handle, use it, restore it + let mut safe_handle = SafeStorageHandle::new(create_test_handle()); + + let handle = safe_handle.take_for_mutation(); + + // Simulate using the handle to create a database + let storage = handle.into_storage(); + let db = Database::from_storage( + storage, + Arc::new(WorkspaceFileSystem::new( + Buffers::new(), + Arc::new(OsFileSystem), + )), + Arc::new(DashMap::new()), + ); + + // Get new handle after simulated mutation + let new_handle = db.storage().clone().into_zalsa_handle(); + + safe_handle.restore_from_mutation(new_handle); + + // Should be able to take it again + let _handle2 = safe_handle.take_for_mutation(); + } + + #[test] + fn test_guard_auto_restore_on_drop() { + let mut safe_handle = SafeStorageHandle::new(create_test_handle()); + + { + let mut guard = safe_handle.take_guarded(); + let handle = guard.handle(); + + // Simulate mutation + let storage = handle.into_storage(); + let db = Database::from_storage( + storage, + Arc::new(WorkspaceFileSystem::new( + Buffers::new(), + Arc::new(OsFileSystem), + )), + Arc::new(DashMap::new()), + ); + let new_handle = db.storage().clone().into_zalsa_handle(); + + guard.restore(new_handle); + } // Guard drops here in Restored state - should be clean + + // Should be able to use handle again after guard drops + let _handle = safe_handle.clone_for_read(); + } + + #[test] + #[should_panic(expected = "Database handle already taken for mutation")] + fn test_panic_on_double_mutation() { + let mut safe_handle = SafeStorageHandle::new(create_test_handle()); + + let _handle1 = safe_handle.take_for_mutation(); + // Can't take handle twice, should panic + let _handle2 = safe_handle.take_for_mutation(); + } + + #[test] + #[should_panic(expected = "Cannot access database handle for read")] + fn test_panic_on_read_during_mutation() { + let mut safe_handle = SafeStorageHandle::new(create_test_handle()); + + let _handle = safe_handle.take_for_mutation(); + // Can't read while mutating, should panic + let _read = safe_handle.clone_for_read(); + } + + #[test] + #[should_panic(expected = "Cannot restore handle - it hasn't been consumed yet")] + fn test_guard_enforces_consume_before_restore() { + let mut safe_handle = SafeStorageHandle::new(create_test_handle()); + let guard = safe_handle.take_guarded(); + + let dummy_handle = create_test_handle(); + // Try to restore without consuming, should panic + guard.restore(dummy_handle); + } + + #[test] + #[should_panic(expected = "StorageHandleGuard dropped without restoring handle")] + fn test_guard_panics_if_dropped_without_restore() { + let mut safe_handle = SafeStorageHandle::new(create_test_handle()); + + { + let mut guard = safe_handle.take_guarded(); + let _handle = guard.handle(); + } // Explicitly drop guard without restore, should panic + } +} diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index bfb9ac75..d411eb59 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -192,3 +192,223 @@ impl Default for Workspace { Self::new() } } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::Arc; + + use tempfile::tempdir; + + use super::*; + use crate::db::source_text; + use crate::LanguageId; + + #[test] + fn test_with_db_read() { + // Read-only access works + let workspace = Workspace::new(); + + let result = workspace.with_db(|db| { + // Can perform read operations + db.has_file(&PathBuf::from("test.py")) + }); + + assert!(!result); // File doesn't exist yet + } + + #[test] + fn test_with_db_mut() { + // Mutation access works + let mut workspace = Workspace::new(); + + // Create a file through mutation + workspace.with_db_mut(|db| { + let path = PathBuf::from("test.py"); + let _file = db.get_or_create_file(&path); + }); + + // Verify it exists + let exists = workspace.with_db(|db| db.has_file(&PathBuf::from("test.py"))); + assert!(exists); + } + + #[test] + fn test_concurrent_reads() { + // Multiple with_db calls can run simultaneously + let workspace = Arc::new(Workspace::new()); + + let w1 = workspace.clone(); + let w2 = workspace.clone(); + + // Spawn concurrent reads + let handle1 = + std::thread::spawn(move || w1.with_db(|db| db.has_file(&PathBuf::from("file1.py")))); + + let handle2 = + std::thread::spawn(move || w2.with_db(|db| db.has_file(&PathBuf::from("file2.py")))); + + // Both should complete without issues + assert!(!handle1.join().unwrap()); + assert!(!handle2.join().unwrap()); + } + + #[test] + fn test_sequential_mutations() { + // Multiple with_db_mut calls work in sequence + let mut workspace = Workspace::new(); + + // First mutation + workspace.with_db_mut(|db| { + let _file = db.get_or_create_file(&PathBuf::from("first.py")); + }); + + // Second mutation + workspace.with_db_mut(|db| { + let _file = db.get_or_create_file(&PathBuf::from("second.py")); + }); + + // Both files should exist + let (has_first, has_second) = workspace.with_db(|db| { + ( + db.has_file(&PathBuf::from("first.py")), + db.has_file(&PathBuf::from("second.py")), + ) + }); + + assert!(has_first); + assert!(has_second); + } + + #[test] + fn test_open_document() { + // Open doc → appears in buffers → queryable via db + let mut workspace = Workspace::new(); + let url = Url::parse("file:///test.py").unwrap(); + + // Open document + let document = TextDocument::new("print('hello')".to_string(), 1, LanguageId::Python); + workspace.open_document(&url, document); + + // Should be in buffers + assert!(workspace.buffers.get(&url).is_some()); + + // Should be queryable through database + let content = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(&PathBuf::from("/test.py")); + source_text(db, file).to_string() + }); + + assert_eq!(content, "print('hello')"); + } + + #[test] + fn test_update_document() { + // Update changes buffer content + let mut workspace = Workspace::new(); + let url = Url::parse("file:///test.py").unwrap(); + + // Open with initial content + let document = TextDocument::new("initial".to_string(), 1, LanguageId::Python); + workspace.open_document(&url, document); + + // Update content + let changes = vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: "updated".to_string(), + }]; + workspace.update_document(&url, changes, 2); + + // Verify buffer was updated + let buffer = workspace.buffers.get(&url).unwrap(); + assert_eq!(buffer.content(), "updated"); + assert_eq!(buffer.version(), 2); + } + + #[test] + fn test_close_document() { + // Close removes from buffers + let mut workspace = Workspace::new(); + let url = Url::parse("file:///test.py").unwrap(); + + // Open document + let document = TextDocument::new("content".to_string(), 1, LanguageId::Python); + workspace.open_document(&url, document.clone()); + + // Close it + let closed = workspace.close_document(&url); + assert!(closed.is_some()); + + // Should no longer be in buffers + assert!(workspace.buffers.get(&url).is_none()); + } + + #[test] + fn test_buffer_takes_precedence_over_disk() { + // Open doc content overrides file system + let temp_dir = tempdir().unwrap(); + let file_path = temp_dir.path().join("test.py"); + std::fs::write(&file_path, "disk content").unwrap(); + + let mut workspace = Workspace::new(); + let url = Url::from_file_path(&file_path).unwrap(); + + // Open document with different content than disk + let document = TextDocument::new("buffer content".to_string(), 1, LanguageId::Python); + workspace.open_document(&url, document); + + // Database should return buffer content, not disk content + let content = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(&file_path); + source_text(db, file).to_string() + }); + + assert_eq!(content, "buffer content"); + } + + #[test] + fn test_missing_file_returns_empty() { + // Non-existent files return "" not error + let mut workspace = Workspace::new(); + + let content = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(&PathBuf::from("/nonexistent.py")); + source_text(db, file).to_string() + }); + + assert_eq!(content, ""); + } + + #[test] + fn test_file_invalidation_on_touch() { + // touch_file triggers Salsa recomputation + let temp_dir = tempdir().unwrap(); + let file_path = temp_dir.path().join("test.py"); + std::fs::write(&file_path, "version 1").unwrap(); + + let mut workspace = Workspace::new(); + + // First read + let content1 = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(&file_path); + source_text(db, file).to_string() + }); + assert_eq!(content1, "version 1"); + + // Update file on disk + std::fs::write(&file_path, "version 2").unwrap(); + + // Touch to invalidate + workspace.with_db_mut(|db| { + db.touch_file(&file_path); + }); + + // Should read new content + let content2 = workspace.with_db_mut(|db| { + let file = db.get_or_create_file(&file_path); + source_text(db, file).to_string() + }); + assert_eq!(content2, "version 2"); + } +} From 9a26025bb40c255c6cf0e2c0ea8212f7ec48b1cf Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 23:36:28 -0500 Subject: [PATCH 50/56] fmt --- crates/djls-workspace/src/storage.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/djls-workspace/src/storage.rs b/crates/djls-workspace/src/storage.rs index 29a3880c..28a25fff 100644 --- a/crates/djls-workspace/src/storage.rs +++ b/crates/djls-workspace/src/storage.rs @@ -229,7 +229,8 @@ mod tests { use super::*; use crate::buffers::Buffers; - use crate::fs::{OsFileSystem, WorkspaceFileSystem}; + use crate::fs::OsFileSystem; + use crate::fs::WorkspaceFileSystem; fn create_test_handle() -> StorageHandle { Database::new( From 0780a9213acc5eee3492f245bf95169ebc76e4d6 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 23:48:07 -0500 Subject: [PATCH 51/56] use encodign --- crates/djls-server/src/session.rs | 2 +- crates/djls-workspace/src/document.rs | 25 ++++++++++++------------- crates/djls-workspace/src/workspace.rs | 6 ++++-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index b3c19b6c..dc7e60ed 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -115,7 +115,7 @@ impl Session { changes: Vec, new_version: i32, ) { - self.workspace.update_document(url, changes, new_version); + self.workspace.update_document(url, changes, new_version, self.position_encoding); } /// Handle closing a document - removes buffer and bumps revision. diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 95f7e62b..9b90f1f3 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -90,6 +90,7 @@ impl TextDocument { &mut self, changes: Vec, version: i32, + encoding: PositionEncoding, ) { // Fast path: single change without range = full document replacement if changes.len() == 1 && changes[0].range.is_none() { @@ -105,16 +106,14 @@ impl TextDocument { for change in changes { if let Some(range) = change.range { - // Convert LSP range to byte offsets - // Note: We use UTF-16 encoding by default for LSP compatibility - // This will need to use the negotiated encoding in the future + // Convert LSP range to byte offsets using the negotiated encoding let start_offset = self.line_index - .offset(range.start, &new_content, PositionEncoding::Utf16) + .offset(range.start, &new_content, encoding) as usize; let end_offset = self.line_index - .offset(range.end, &new_content, PositionEncoding::Utf16) + .offset(range.end, &new_content, encoding) as usize; // Apply the change by replacing the range @@ -308,7 +307,7 @@ mod tests { text: "Rust".to_string(), }]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "Hello Rust"); assert_eq!(doc.version(), 2); } @@ -335,7 +334,7 @@ mod tests { }, ]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "1st line\nSecond line\n3rd line"); } @@ -350,7 +349,7 @@ mod tests { text: " beautiful".to_string(), }]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "Hello beautiful world"); } @@ -365,7 +364,7 @@ mod tests { text: String::new(), }]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "Hello world"); } @@ -380,7 +379,7 @@ mod tests { text: "Completely new content".to_string(), }]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "Completely new content"); assert_eq!(doc.version(), 2); } @@ -396,7 +395,7 @@ mod tests { text: "A\nB\nC".to_string(), }]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "Line A\nB\nC 3"); } @@ -412,7 +411,7 @@ mod tests { text: "Rust".to_string(), }]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "Hello 🌍 Rust"); } @@ -427,7 +426,7 @@ mod tests { text: "\nWorld".to_string(), }]; - doc.update(changes, 2); + doc.update(changes, 2, PositionEncoding::Utf16); assert_eq!(doc.content(), "Hello\nWorld"); } diff --git a/crates/djls-workspace/src/workspace.rs b/crates/djls-workspace/src/workspace.rs index d411eb59..dfb9a2da 100644 --- a/crates/djls-workspace/src/workspace.rs +++ b/crates/djls-workspace/src/workspace.rs @@ -125,10 +125,11 @@ impl Workspace { url: &Url, changes: Vec, version: i32, + encoding: crate::encoding::PositionEncoding, ) { if let Some(mut document) = self.buffers.get(url) { // Apply incremental changes to existing document - document.update(changes, version); + document.update(changes, version, encoding); self.buffers.update(url.clone(), document); } else if let Some(first_change) = changes.into_iter().next() { // Fallback: treat first change as full replacement @@ -202,6 +203,7 @@ mod tests { use super::*; use crate::db::source_text; + use crate::encoding::PositionEncoding; use crate::LanguageId; #[test] @@ -318,7 +320,7 @@ mod tests { range_length: None, text: "updated".to_string(), }]; - workspace.update_document(&url, changes, 2); + workspace.update_document(&url, changes, 2, PositionEncoding::Utf16); // Verify buffer was updated let buffer = workspace.buffers.get(&url).unwrap(); From 27aeedeced1c9b216b998460865455357e4328ba Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 4 Sep 2025 23:52:12 -0500 Subject: [PATCH 52/56] cleaning --- crates/djls-workspace/src/document.rs | 28 ++++++++------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 9b90f1f3..5f22d332 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -101,22 +101,16 @@ impl TextDocument { } // Incremental path: apply changes to rebuild the document - // Clone current content and apply each change let mut new_content = self.content.clone(); for change in changes { if let Some(range) = change.range { // Convert LSP range to byte offsets using the negotiated encoding let start_offset = - self.line_index - .offset(range.start, &new_content, encoding) - as usize; - let end_offset = - self.line_index - .offset(range.end, &new_content, encoding) - as usize; - - // Apply the change by replacing the range + self.line_index.offset(range.start, &new_content, encoding) as usize; + let end_offset = self.line_index.offset(range.end, &new_content, encoding) as usize; + + // Apply change new_content.replace_range(start_offset..end_offset, &change.text); // Rebuild line index after each change since positions shift @@ -129,7 +123,6 @@ impl TextDocument { } } - // Store the rebuilt document self.content = new_content; self.version = version; } @@ -164,16 +157,15 @@ pub struct LineIndex { impl LineIndex { #[must_use] pub fn new(text: &str) -> Self { - let mut line_starts = vec![0]; - let mut pos_utf8 = 0; - - // Check if text is pure ASCII for optimization let kind = if text.is_ascii() { IndexKind::Ascii } else { IndexKind::Utf8 }; + let mut line_starts = vec![0]; + let mut pos_utf8 = 0; + for c in text.chars() { pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); if c == '\n' { @@ -198,12 +190,10 @@ impl LineIndex { None => return self.length, // Past end of document }; - // If position is at start of line, return line start if position.character == 0 { return line_start_utf8; } - // Find the line text let next_line_start = self .line_starts .get(position.line as usize + 1) @@ -214,16 +204,14 @@ impl LineIndex { return line_start_utf8; }; - // ASCII fast path optimization + // Fast path optimization for ASCII text, all encodings are equivalent to byte offsets if matches!(self.kind, IndexKind::Ascii) { - // For ASCII text, all encodings are equivalent to byte offsets let char_offset = position .character .min(u32::try_from(line_text.len()).unwrap_or(u32::MAX)); return line_start_utf8 + char_offset; } - // Handle different encodings for non-ASCII text match encoding { PositionEncoding::Utf8 => { // UTF-8: character positions are already byte offsets From 0ccb304a5ba7b59df4723f36712dd4829482a429 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 5 Sep 2025 00:02:13 -0500 Subject: [PATCH 53/56] clean up tests --- crates/djls-workspace/src/encoding.rs | 118 ++++++++++++++++++++------ 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/crates/djls-workspace/src/encoding.rs b/crates/djls-workspace/src/encoding.rs index f81ffeab..93cda1f1 100644 --- a/crates/djls-workspace/src/encoding.rs +++ b/crates/djls-workspace/src/encoding.rs @@ -92,39 +92,34 @@ mod tests { use super::*; #[test] - fn test_encoding_str_conversion() { - // Test FromStr trait - assert_eq!("utf-8".parse(), Ok(PositionEncoding::Utf8)); - assert_eq!("utf-16".parse(), Ok(PositionEncoding::Utf16)); - assert_eq!("utf-32".parse(), Ok(PositionEncoding::Utf32)); - assert!("invalid".parse::().is_err()); - - // Test ToString trait - assert_eq!(PositionEncoding::Utf8.to_string(), "utf-8"); - assert_eq!(PositionEncoding::Utf16.to_string(), "utf-16"); - assert_eq!(PositionEncoding::Utf32.to_string(), "utf-32"); - } - - #[test] - fn test_from_lsp_kind() { + fn test_string_parsing_and_display() { + // Valid encodings parse correctly assert_eq!( - PositionEncoding::try_from(PositionEncodingKind::new("utf-8")), + "utf-8".parse::(), Ok(PositionEncoding::Utf8) ); assert_eq!( - PositionEncoding::try_from(PositionEncodingKind::new("utf-16")), + "utf-16".parse::(), Ok(PositionEncoding::Utf16) ); assert_eq!( - PositionEncoding::try_from(PositionEncodingKind::new("utf-32")), + "utf-32".parse::(), Ok(PositionEncoding::Utf32) ); - assert!(PositionEncoding::try_from(PositionEncodingKind::new("unknown")).is_err()); + + // Invalid encoding returns error + assert!("invalid".parse::().is_err()); + assert!("UTF-8".parse::().is_err()); // case sensitive + + // Display produces correct strings + assert_eq!(PositionEncoding::Utf8.to_string(), "utf-8"); + assert_eq!(PositionEncoding::Utf16.to_string(), "utf-16"); + assert_eq!(PositionEncoding::Utf32.to_string(), "utf-32"); } #[test] - fn test_trait_conversions() { - // Test TryFrom for PositionEncoding + fn test_lsp_type_conversions() { + // TryFrom for valid encodings assert_eq!( PositionEncoding::try_from(PositionEncodingKind::new("utf-8")), Ok(PositionEncoding::Utf8) @@ -137,9 +132,11 @@ mod tests { PositionEncoding::try_from(PositionEncodingKind::new("utf-32")), Ok(PositionEncoding::Utf32) ); + + // Invalid encoding returns error assert!(PositionEncoding::try_from(PositionEncodingKind::new("unknown")).is_err()); - // Test From for PositionEncodingKind + // From produces correct LSP types assert_eq!( PositionEncodingKind::from(PositionEncoding::Utf8).as_str(), "utf-8" @@ -155,7 +152,7 @@ mod tests { } #[test] - fn test_negotiate_prefers_utf8() { + fn test_negotiate_prefers_utf8_when_all_available() { let params = InitializeParams { capabilities: ClientCapabilities { general: Some(GeneralClientCapabilities { @@ -175,7 +172,67 @@ mod tests { } #[test] - fn test_negotiate_fallback_utf16() { + fn test_negotiate_prefers_utf32_over_utf16() { + let params = InitializeParams { + capabilities: ClientCapabilities { + general: Some(GeneralClientCapabilities { + position_encodings: Some(vec![ + PositionEncodingKind::new("utf-16"), + PositionEncodingKind::new("utf-32"), + ]), + ..Default::default() + }), + ..Default::default() + }, + ..Default::default() + }; + + assert_eq!( + PositionEncoding::negotiate(¶ms), + PositionEncoding::Utf32 + ); + } + + #[test] + fn test_negotiate_accepts_utf16_when_only_option() { + let params = InitializeParams { + capabilities: ClientCapabilities { + general: Some(GeneralClientCapabilities { + position_encodings: Some(vec![PositionEncodingKind::new("utf-16")]), + ..Default::default() + }), + ..Default::default() + }, + ..Default::default() + }; + + assert_eq!( + PositionEncoding::negotiate(¶ms), + PositionEncoding::Utf16 + ); + } + + #[test] + fn test_negotiate_fallback_with_empty_encodings() { + let params = InitializeParams { + capabilities: ClientCapabilities { + general: Some(GeneralClientCapabilities { + position_encodings: Some(vec![]), + ..Default::default() + }), + ..Default::default() + }, + ..Default::default() + }; + + assert_eq!( + PositionEncoding::negotiate(¶ms), + PositionEncoding::Utf16 + ); + } + + #[test] + fn test_negotiate_fallback_with_no_capabilities() { let params = InitializeParams::default(); assert_eq!( PositionEncoding::negotiate(¶ms), @@ -184,13 +241,13 @@ mod tests { } #[test] - fn test_negotiate_prefers_utf32_over_utf16() { + fn test_negotiate_fallback_with_unknown_encodings() { let params = InitializeParams { capabilities: ClientCapabilities { general: Some(GeneralClientCapabilities { position_encodings: Some(vec![ - PositionEncodingKind::new("utf-16"), - PositionEncodingKind::new("utf-32"), + PositionEncodingKind::new("utf-7"), + PositionEncodingKind::new("ascii"), ]), ..Default::default() }), @@ -201,7 +258,12 @@ mod tests { assert_eq!( PositionEncoding::negotiate(¶ms), - PositionEncoding::Utf32 + PositionEncoding::Utf16 ); } + + #[test] + fn test_default_is_utf16() { + assert_eq!(PositionEncoding::default(), PositionEncoding::Utf16); + } } From d5c5c6efab4b241beaa6f9b2448665af0c229fd8 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 5 Sep 2025 00:11:30 -0500 Subject: [PATCH 54/56] focused tests --- crates/djls-workspace/src/fs.rs | 304 ++++++++++++++++++++------------ 1 file changed, 195 insertions(+), 109 deletions(-) diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index e424ff5f..15a519c2 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -14,16 +14,11 @@ use std::sync::Arc; use crate::buffers::Buffers; use crate::paths; -/// Trait for file system operations pub trait FileSystem: Send + Sync { - /// Read the entire contents of a file fn read_to_string(&self, path: &Path) -> io::Result; - - /// Check if a path exists fn exists(&self, path: &Path) -> bool; } -/// In-memory file system for testing #[cfg(test)] pub struct InMemoryFileSystem { files: HashMap, @@ -120,125 +115,216 @@ impl FileSystem for WorkspaceFileSystem { #[cfg(test)] mod tests { - use url::Url; - use super::*; - use crate::buffers::Buffers; - use crate::document::TextDocument; - use crate::language::LanguageId; - - #[test] - fn test_lsp_filesystem_overlay_precedence() { - let mut memory_fs = InMemoryFileSystem::new(); - memory_fs.add_file( - std::path::PathBuf::from("/test/file.py"), - "original content".to_string(), - ); - - let buffers = Buffers::new(); - let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); - - // Before adding buffer, should read from fallback - let path = std::path::Path::new("/test/file.py"); - assert_eq!(lsp_fs.read_to_string(path).unwrap(), "original content"); - - // Add buffer - this simulates having an open document with changes - let url = Url::from_file_path("/test/file.py").unwrap(); - let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); - buffers.open(url, document); - - // Now should read from buffer - assert_eq!(lsp_fs.read_to_string(path).unwrap(), "overlay content"); - } - #[test] - fn test_lsp_filesystem_fallback_when_no_overlay() { - let mut memory_fs = InMemoryFileSystem::new(); - memory_fs.add_file( - std::path::PathBuf::from("/test/file.py"), - "disk content".to_string(), - ); + mod in_memory { + use super::*; - let buffers = Buffers::new(); - let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); + #[test] + fn test_read_existing_file() { + let mut fs = InMemoryFileSystem::new(); + fs.add_file("/test.py".into(), "file content".to_string()); - // Should fall back to disk when no buffer exists - let path = std::path::Path::new("/test/file.py"); - assert_eq!(lsp_fs.read_to_string(path).unwrap(), "disk content"); - } + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "file content" + ); + } - #[test] - fn test_lsp_filesystem_other_operations_delegate() { - let mut memory_fs = InMemoryFileSystem::new(); - memory_fs.add_file( - std::path::PathBuf::from("/test/file.py"), - "content".to_string(), - ); + #[test] + fn test_read_nonexistent_file() { + let fs = InMemoryFileSystem::new(); - let buffers = Buffers::new(); - let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); + let result = fs.read_to_string(Path::new("/missing.py")); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::NotFound); + } - let path = std::path::Path::new("/test/file.py"); + #[test] + fn test_exists_returns_true_for_existing() { + let mut fs = InMemoryFileSystem::new(); + fs.add_file("/exists.py".into(), "content".to_string()); - // This should delegate to the fallback filesystem - assert!(lsp_fs.exists(path)); + assert!(fs.exists(Path::new("/exists.py"))); + } + + #[test] + fn test_exists_returns_false_for_nonexistent() { + let fs = InMemoryFileSystem::new(); + + assert!(!fs.exists(Path::new("/missing.py"))); + } } - #[test] - fn test_overlay_consistency() { - // Create an empty filesystem (no files on disk) - let memory_fs = InMemoryFileSystem::new(); - let buffers = Buffers::new(); - let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); + mod workspace { + use url::Url; - let path = std::path::Path::new("/test/overlay_only.py"); + use crate::buffers::Buffers; + use crate::document::TextDocument; + use crate::language::LanguageId; - // Before adding to overlay, file doesn't exist - assert!(!lsp_fs.exists(path)); + use super::*; - // Add file to overlay only (not on disk) - let url = Url::from_file_path("/test/overlay_only.py").unwrap(); - let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); - buffers.open(url, document); + #[test] + fn test_reads_from_buffer_when_present() { + let disk = Arc::new(InMemoryFileSystem::new()); + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers.clone(), disk); - // Now file should exist - assert!(lsp_fs.exists(path), "Overlay file should exist"); + // Add file to buffer + let url = Url::from_file_path("/test.py").unwrap(); + let doc = TextDocument::new("buffer content".to_string(), 1, LanguageId::Python); + buffers.open(url, doc); - // And we should be able to read its content - assert_eq!( - lsp_fs.read_to_string(path).unwrap(), - "overlay content", - "Should read overlay content" - ); - } + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "buffer content" + ); + } + + #[test] + fn test_reads_from_disk_when_no_buffer() { + let mut disk_fs = InMemoryFileSystem::new(); + disk_fs.add_file("/test.py".into(), "disk content".to_string()); + + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers, Arc::new(disk_fs)); + + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "disk content" + ); + } + + #[test] + fn test_buffer_overrides_disk() { + let mut disk_fs = InMemoryFileSystem::new(); + disk_fs.add_file("/test.py".into(), "disk content".to_string()); + + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(disk_fs)); + + // Add buffer with different content + let url = Url::from_file_path("/test.py").unwrap(); + let doc = TextDocument::new("buffer content".to_string(), 1, LanguageId::Python); + buffers.open(url, doc); + + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "buffer content" + ); + } + + #[test] + fn test_exists_for_buffer_only_file() { + let disk = Arc::new(InMemoryFileSystem::new()); + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers.clone(), disk); - #[test] - fn test_overlay_with_relative_path() { - // Create an empty filesystem (no files on disk) - let memory_fs = InMemoryFileSystem::new(); - let buffers = Buffers::new(); - let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); - - // Use a relative path that doesn't exist on disk - let relative_path = std::path::Path::new("nonexistent/overlay.py"); - - // Convert to absolute URL for the buffer (simulating how LSP would provide it) - let absolute_path = std::env::current_dir().unwrap().join(relative_path); - let url = Url::from_file_path(&absolute_path).unwrap(); - - // Add to overlay - let document = TextDocument::new("relative overlay".to_string(), 1, LanguageId::Python); - buffers.open(url, document); - - // The relative path should now work through the overlay - assert!( - lsp_fs.exists(relative_path), - "Relative overlay file should exist" - ); - assert_eq!( - lsp_fs.read_to_string(relative_path).unwrap(), - "relative overlay", - "Should read relative overlay content" - ); + // Add file only to buffer + let url = Url::from_file_path("/buffer_only.py").unwrap(); + let doc = TextDocument::new("content".to_string(), 1, LanguageId::Python); + buffers.open(url, doc); + + assert!(fs.exists(Path::new("/buffer_only.py"))); + } + + #[test] + fn test_exists_for_disk_only_file() { + let mut disk_fs = InMemoryFileSystem::new(); + disk_fs.add_file("/disk_only.py".into(), "content".to_string()); + + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers, Arc::new(disk_fs)); + + assert!(fs.exists(Path::new("/disk_only.py"))); + } + + #[test] + fn test_exists_for_both_buffer_and_disk() { + let mut disk_fs = InMemoryFileSystem::new(); + disk_fs.add_file("/both.py".into(), "disk".to_string()); + + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(disk_fs)); + + // Also add to buffer + let url = Url::from_file_path("/both.py").unwrap(); + let doc = TextDocument::new("buffer".to_string(), 1, LanguageId::Python); + buffers.open(url, doc); + + assert!(fs.exists(Path::new("/both.py"))); + } + + #[test] + fn test_exists_returns_false_when_nowhere() { + let disk = Arc::new(InMemoryFileSystem::new()); + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers, disk); + + assert!(!fs.exists(Path::new("/nowhere.py"))); + } + + #[test] + fn test_read_error_when_file_nowhere() { + let disk = Arc::new(InMemoryFileSystem::new()); + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers, disk); + + let result = fs.read_to_string(Path::new("/missing.py")); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::NotFound); + } + + #[test] + fn test_reflects_buffer_updates() { + let disk = Arc::new(InMemoryFileSystem::new()); + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers.clone(), disk); + + let url = Url::from_file_path("/test.py").unwrap(); + + // Initial buffer content + let doc1 = TextDocument::new("version 1".to_string(), 1, LanguageId::Python); + buffers.open(url.clone(), doc1); + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "version 1" + ); + + // Update buffer content + let doc2 = TextDocument::new("version 2".to_string(), 2, LanguageId::Python); + buffers.update(url, doc2); + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "version 2" + ); + } + + #[test] + fn test_handles_buffer_removal() { + let mut disk_fs = InMemoryFileSystem::new(); + disk_fs.add_file("/test.py".into(), "disk content".to_string()); + + let buffers = Buffers::new(); + let fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(disk_fs)); + + let url = Url::from_file_path("/test.py").unwrap(); + + // Add buffer + let doc = TextDocument::new("buffer content".to_string(), 1, LanguageId::Python); + buffers.open(url.clone(), doc); + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "buffer content" + ); + + // Remove buffer + let _ = buffers.close(&url); + assert_eq!( + fs.read_to_string(Path::new("/test.py")).unwrap(), + "disk content" + ); + } } } From b3eddc380e05f9f5b6592e62f876000ee328078f Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 5 Sep 2025 01:07:19 -0500 Subject: [PATCH 55/56] paths --- crates/djls-workspace/src/paths.rs | 189 +++++++++-------------------- 1 file changed, 54 insertions(+), 135 deletions(-) diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 89241544..06960747 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -113,174 +113,93 @@ pub fn path_to_url(path: &Path) -> Option { #[cfg(test)] mod tests { - use super::*; + use std::str::FromStr; - #[test] - fn test_url_to_path_basic() { - let url = Url::parse("file:///home/user/file.txt").unwrap(); - let path = url_to_path(&url).unwrap(); - assert_eq!(path, PathBuf::from("/home/user/file.txt")); - } + use super::*; #[test] - fn test_url_to_path_with_spaces() { - let url = Url::parse("file:///home/user/my%20file.txt").unwrap(); - let path = url_to_path(&url).unwrap(); - assert_eq!(path, PathBuf::from("/home/user/my file.txt")); + fn test_url_to_path_valid_file_url() { + let url = Url::parse("file:///home/user/test.py").unwrap(); + assert_eq!(url_to_path(&url), Some(PathBuf::from("/home/user/test.py"))); } #[test] fn test_url_to_path_non_file_scheme() { - let url = Url::parse("https://example.com/file.txt").unwrap(); - assert!(url_to_path(&url).is_none()); + let url = Url::parse("http://example.com/test.py").unwrap(); + assert_eq!(url_to_path(&url), None); } - #[cfg(windows)] #[test] - fn test_url_to_path_windows() { - let url = Url::parse("file:///C:/Users/user/file.txt").unwrap(); - let path = url_to_path(&url).unwrap(); - assert_eq!(path, PathBuf::from("C:/Users/user/file.txt")); + fn test_url_to_path_percent_encoded() { + let url = Url::parse("file:///home/user/test%20file.py").unwrap(); + assert_eq!( + url_to_path(&url), + Some(PathBuf::from("/home/user/test file.py")) + ); } #[test] - fn test_path_to_url_absolute() { - let path = if cfg!(windows) { - PathBuf::from("C:/Users/user/file.txt") - } else { - PathBuf::from("/home/user/file.txt") - }; - - let url = path_to_url(&path).unwrap(); - assert_eq!(url.scheme(), "file"); - assert!(url.path().contains("file.txt")); - } - - #[test] - fn test_round_trip() { - let original_path = if cfg!(windows) { - PathBuf::from("C:/Users/user/test file.txt") - } else { - PathBuf::from("/home/user/test file.txt") - }; - - let url = path_to_url(&original_path).unwrap(); - let converted_path = url_to_path(&url).unwrap(); - - assert_eq!(original_path, converted_path); - } - - #[test] - fn test_url_with_localhost() { - // Some systems use file://localhost/path format - let url = Url::parse("file://localhost/home/user/file.txt").unwrap(); - let path = url_to_path(&url); - - // Current implementation might not handle this correctly - // since it only checks scheme, not host - if let Some(p) = path { - assert_eq!(p, PathBuf::from("/home/user/file.txt")); - } + #[cfg(windows)] + fn test_url_to_path_windows_drive() { + let url = Url::parse("file:///C:/Users/test.py").unwrap(); + assert_eq!(url_to_path(&url), Some(PathBuf::from("C:/Users/test.py"))); } #[test] - fn test_url_with_empty_host() { - // Standard file:///path format (three slashes, empty host) - let url = Url::parse("file:///home/user/file.txt").unwrap(); - let path = url_to_path(&url).unwrap(); - assert_eq!(path, PathBuf::from("/home/user/file.txt")); + fn test_parse_lsp_uri_valid() { + let uri = lsp_types::Uri::from_str("file:///test.py").unwrap(); + let result = parse_lsp_uri(&uri, LspContext::DidOpen); + assert!(result.is_some()); + assert_eq!(result.unwrap().scheme(), "file"); } - #[cfg(windows)] + // lsp_uri_to_path tests #[test] - fn test_unc_path_to_url() { - // UNC paths like \\server\share\file.txt - let unc_path = PathBuf::from(r"\\server\share\file.txt"); - let url = path_to_url(&unc_path); - - // Check if UNC paths are handled - if let Some(u) = url { - // UNC paths should convert to file://server/share/file.txt - assert!(u.to_string().contains("server")); - assert!(u.to_string().contains("share")); - } + fn test_lsp_uri_to_path_valid_file() { + let uri = lsp_types::Uri::from_str("file:///home/user/test.py").unwrap(); + assert_eq!( + lsp_uri_to_path(&uri), + Some(PathBuf::from("/home/user/test.py")) + ); } #[test] - fn test_relative_path_with_dotdot() { - // Test relative paths with .. that might not exist - let path = PathBuf::from("../some/nonexistent/path.txt"); - let url = path_to_url(&path); - - // Should now work even for non-existent files - assert!(url.is_some(), "Should handle non-existent relative paths"); - if let Some(u) = url { - assert_eq!(u.scheme(), "file"); - assert!(u.path().ends_with("some/nonexistent/path.txt")); - } + fn test_lsp_uri_to_path_non_file() { + let uri = lsp_types::Uri::from_str("http://example.com/test.py").unwrap(); + assert_eq!(lsp_uri_to_path(&uri), None); } #[test] - fn test_non_existent_absolute_path() { - // Test that absolute paths work even if they don't exist - let path = if cfg!(windows) { - PathBuf::from("C:/NonExistent/Directory/file.txt") - } else { - PathBuf::from("/nonexistent/directory/file.txt") - }; - - let url = path_to_url(&path); - assert!(url.is_some(), "Should handle non-existent absolute paths"); - if let Some(u) = url { - assert_eq!(u.scheme(), "file"); - assert!(u.path().contains("file.txt")); - } + fn test_lsp_uri_to_path_invalid_uri() { + let uri = lsp_types::Uri::from_str("not://valid").unwrap(); + assert_eq!(lsp_uri_to_path(&uri), None); } + // path_to_url tests #[test] - fn test_non_existent_relative_path() { - // Test that relative paths work even if they don't exist - let path = PathBuf::from("nonexistent/file.txt"); - let url = path_to_url(&path); - - assert!(url.is_some(), "Should handle non-existent relative paths"); - if let Some(u) = url { - assert_eq!(u.scheme(), "file"); - assert!(u.path().ends_with("nonexistent/file.txt")); - // Should be an absolute URL - assert!(u.path().starts_with('/') || cfg!(windows)); - } + fn test_path_to_url_absolute() { + let path = Path::new("/home/user/test.py"); + let url = path_to_url(path); + assert!(url.is_some()); + assert_eq!(url.clone().unwrap().scheme(), "file"); + assert!(url.unwrap().path().contains("test.py")); } #[test] - fn test_path_with_special_chars() { - // Test paths with special characters that need encoding - let path = PathBuf::from("/home/user/file with spaces & special!.txt"); - let url = path_to_url(&path).unwrap(); - - // Should be properly percent-encoded - assert!(url.as_str().contains("%20") || url.as_str().contains("with%20spaces")); - - // Round-trip should work - let back = url_to_path(&url).unwrap(); - assert_eq!(back, path); + fn test_path_to_url_relative() { + let path = Path::new("test.py"); + let url = path_to_url(path); + assert!(url.is_some()); + assert_eq!(url.clone().unwrap().scheme(), "file"); + // Should be resolved to absolute path + assert!(url.unwrap().path().ends_with("/test.py")); } #[test] - fn test_url_with_query_or_fragment() { - // URLs with query parameters or fragments should probably be rejected - let url_with_query = Url::parse("file:///path/file.txt?query=param").unwrap(); - let url_with_fragment = Url::parse("file:///path/file.txt#section").unwrap(); - - // These should still work, extracting just the path part - let path1 = url_to_path(&url_with_query); - let path2 = url_to_path(&url_with_fragment); - - if let Some(p) = path1 { - assert_eq!(p, PathBuf::from("/path/file.txt")); - } - if let Some(p) = path2 { - assert_eq!(p, PathBuf::from("/path/file.txt")); - } + fn test_path_to_url_nonexistent_absolute() { + let path = Path::new("/definitely/does/not/exist/test.py"); + let url = path_to_url(path); + assert!(url.is_some()); + assert_eq!(url.unwrap().scheme(), "file"); } } From e42980cb97b082e4a63edf6a0f5556fd2012d927 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 5 Sep 2025 01:29:06 -0500 Subject: [PATCH 56/56] fm --- crates/djls-server/src/session.rs | 3 ++- crates/djls-workspace/src/fs.rs | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index dc7e60ed..c651bd9a 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -115,7 +115,8 @@ impl Session { changes: Vec, new_version: i32, ) { - self.workspace.update_document(url, changes, new_version, self.position_encoding); + self.workspace + .update_document(url, changes, new_version, self.position_encoding); } /// Handle closing a document - removes buffer and bumps revision. diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 15a519c2..1c4bdda0 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -159,12 +159,11 @@ mod tests { mod workspace { use url::Url; + use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; use crate::language::LanguageId; - use super::*; - #[test] fn test_reads_from_buffer_when_present() { let disk = Arc::new(InMemoryFileSystem::new());