Initial version of case-sensitive module resolver

Author: MichaReiser

crates/red_knot_python_semantic/resources/mdtest/import/case_sensitive.md

@@ -4,7 +4,7 @@
 # TODO: This test should use the real file system instead of the memory file system.
 # but we can't change the file system yet because the tests would then start failing for 
 # case-insensitive file systems.
-#system = "os"
+system = "os"
 ```
 
 Python's import system is case-sensitive even on case-insensitive file system. This means, importing

crates/red_knot_python_semantic/src/module_resolver/path.rs

@@ -63,6 +63,10 @@ impl ModulePath {
         self.relative_path.pop()
     }
 
+    pub(super) fn search_path(&self) -> &SearchPath {
+        &self.search_path
+    }
+
     #[must_use]
     pub(super) fn is_directory(&self, resolver: &ResolverContext) -> bool {
         let ModulePath {

crates/red_knot_python_semantic/src/module_resolver/resolver.rs

@@ -604,14 +604,30 @@ fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, Mod
 /// resolving modules.
 fn resolve_file_module(module: &ModulePath, resolver_state: &ResolverContext) -> Option<File> {
     // Stubs have precedence over source files
-    module
+    let file = module
         .with_pyi_extension()
         .to_file(resolver_state)
         .or_else(|| {
             module
                 .with_py_extension()
                 .and_then(|path| path.to_file(resolver_state))
-        })
+        })?;
+
+    // For system files, test if the path has the correct casing.
+    // We can skip this step for vendored files or virtual files because
+    // those file systems are case sensitive (we wouldn't get to this point).
+    if let Some(path) = file.path(resolver_state.db).as_system_path() {
+        if !resolver_state
+            .db
+            .system()
+            .path_exists_case_sensitive(path, module.search_path().as_system_path().unwrap())
+            .unwrap_or(true)
+        {
+            return None;
+        }
+    }
+
+    Some(file)
 }
 
 fn resolve_package<'a, 'db, I>(

crates/red_knot_server/src/system.rs

@@ -136,6 +136,10 @@ impl System for LSPSystem {
         self.os_system.canonicalize_path(path)
     }
 
+    fn path_exists_case_sensitive(&self, path: &SystemPath, prefix: &SystemPath) -> Result<bool> {
+        self.os_system.path_exists_case_sensitive(path, prefix)
+    }
+
     fn read_to_string(&self, path: &SystemPath) -> Result<String> {
         let document = self.system_path_to_document_ref(path)?;
 

crates/red_knot_test/src/db.rs

@@ -210,6 +210,15 @@ impl System for MdtestSystem {
         self.as_system().read_virtual_path_to_notebook(path)
     }
 
+    fn path_exists_case_sensitive(
+        &self,
+        path: &SystemPath,
+        prefix: &SystemPath,
+    ) -> ruff_db::system::Result<bool> {
+        self.as_system()
+            .path_exists_case_sensitive(&self.normalize_path(path), &self.normalize_path(prefix))
+    }
+
     fn current_directory(&self) -> &SystemPath {
         self.as_system().current_directory()
     }

crates/red_knot_wasm/src/lib.rs

@@ -260,6 +260,14 @@ impl System for WasmSystem {
         Err(ruff_notebook::NotebookError::Io(not_found()))
     }
 
+    fn path_exists_case_sensitive(
+        &self,
+        path: &SystemPath,
+        _prefix: &SystemPath,
+    ) -> ruff_db::system::Result<bool> {
+        Ok(self.path_exists(path))
+    }
+
     fn current_directory(&self) -> &SystemPath {
         self.fs.current_directory()
     }

crates/ruff_db/src/source.rs

@@ -159,7 +159,7 @@ pub enum SourceTextError {
 /// Computes the [`LineIndex`] for `file`.
 #[salsa::tracked]
 pub fn line_index(db: &dyn Db, file: File) -> LineIndex {
-    let _span = tracing::trace_span!("line_index", file = ?file).entered();
+    let _span = tracing::trace_span!("line_index", file = ?file.path(db)).entered();
 
     let source = source_text(db, file);
 

crates/ruff_db/src/system.rs

@@ -89,6 +89,8 @@ pub trait System: Debug {
         self.path_metadata(path).is_ok()
     }
 
+    fn path_exists_case_sensitive(&self, path: &SystemPath, prefix: &SystemPath) -> Result<bool>;
+
     /// Returns `true` if `path` exists and is a directory.
     fn is_directory(&self, path: &SystemPath) -> bool {
         self.path_metadata(path)

crates/ruff_db/src/system/os.rs

@@ -1,9 +1,9 @@
-use std::sync::Arc;
-use std::{any::Any, path::PathBuf};
-
 use filetime::FileTime;
-
 use ruff_notebook::{Notebook, NotebookError};
+use rustc_hash::FxHashSet;
+use std::panic::RefUnwindSafe;
+use std::sync::Arc;
+use std::{any::Any, path::PathBuf};
 
 use crate::system::{
     DirectoryEntry, FileType, GlobError, GlobErrorKind, Metadata, Result, System, SystemPath,
@@ -25,6 +25,8 @@ pub struct OsSystem {
 struct OsSystemInner {
     cwd: SystemPathBuf,
 
+    real_case_cache: RealCaseCache,
+
     /// Overrides the user's configuration directory for testing.
     /// This is an `Option<Option<..>>` to allow setting an override of `None`.
     #[cfg(feature = "testing")]
@@ -102,6 +104,27 @@ impl System for OsSystem {
         path.as_std_path().exists()
     }
 
+    fn path_exists_case_sensitive(&self, path: &SystemPath, prefix: &SystemPath) -> Result<bool> {
+        // Decide what to do if prefix isn't a prefix
+        debug_assert!(
+            path.strip_prefix(prefix).is_ok(),
+            "prefix `{prefix}` should be a prefix of `{path}`"
+        );
+
+        // Iterate over the sub-paths up to prefix and check if they match the casing as on disk.
+        for ancestor in path.ancestors() {
+            if ancestor == prefix {
+                break;
+            }
+
+            if !self.inner.real_case_cache.has_name_case(ancestor)? {
+                return Ok(false);
+            }
+        }
+
+        Ok(true)
+    }
+
     fn current_directory(&self) -> &SystemPath {
         &self.inner.cwd
     }
@@ -201,6 +224,82 @@ impl WritableSystem for OsSystem {
     }
 }
 
+#[derive(Debug, Default)]
+struct RealCaseCache {
+    by_lower_case: dashmap::DashMap<SystemPathBuf, ListedDirectory>,
+}
+
+impl RealCaseCache {
+    fn has_name_case(&self, path: &SystemPath) -> Result<bool> {
+        // TODO: Skip if the FS is known to be case-sensitive.
+        // TODO: Consider using `GetFinalPathNameByHandleW` on windows
+        // TODO: Consider using `fcntl(F_GETPATH)` on macOS (https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fcntl.2.html)
+
+        let Some(parent) = path.parent() else {
+            // TODO: Decide what to return for the root path
+            return Ok(true);
+        };
+
+        // TODO: decide what to return for the root path or files ending in `..`
+        let Some(file_name) = path.file_name() else {
+            return Ok(false);
+        };
+
+        let lower_case_path = SystemPathBuf::from(parent.as_str().to_lowercase());
+        let last_modification_time =
+            FileTime::from_last_modification_time(&parent.as_std_path().metadata()?);
+
+        let entry = self.by_lower_case.entry(lower_case_path);
+
+        if let dashmap::Entry::Occupied(entry) = &entry {
+            // Only do a cached lookup if the directory hasn't changed.
+            if entry.get().last_modification_time == last_modification_time {
+                tracing::trace!("Use cached 'real-case' entry for directory `{}`", parent);
+                return Ok(entry.get().names.contains(file_name));
+            }
+        }
+
+        tracing::trace!(
+            "Reading directory `{}` to determine the real casing of paths",
+            parent
+        );
+        let start = std::time::Instant::now();
+        let mut names = FxHashSet::default();
+
+        for entry in parent.as_std_path().read_dir()? {
+            let Ok(entry) = entry else {
+                continue;
+            };
+
+            let Ok(name) = entry.file_name().into_string() else {
+                continue;
+            };
+
+            names.insert(name);
+        }
+
+        let directory = entry.insert(ListedDirectory {
+            last_modification_time,
+            names,
+        });
+
+        tracing::trace!(
+            "Caching the real casing of `{parent}` took {:?}",
+            start.elapsed()
+        );
+
+        Ok(directory.names.contains(file_name))
+    }
+}
+
+impl RefUnwindSafe for RealCaseCache {}
+
+#[derive(Debug, Eq, PartialEq)]
+struct ListedDirectory {
+    last_modification_time: filetime::FileTime,
+    names: FxHashSet<String>,
+}
+
 #[derive(Debug)]
 struct OsDirectoryWalker;
 

crates/ruff_db/src/system/test.rs

@@ -130,6 +130,10 @@ impl System for TestSystem {
     fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
         self
     }
+
+    fn path_exists_case_sensitive(&self, path: &SystemPath, prefix: &SystemPath) -> Result<bool> {
+        self.system().path_exists_case_sensitive(path, prefix)
+    }
 }
 
 impl Default for TestSystem {
@@ -349,6 +353,12 @@ impl System for InMemorySystem {
     fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
         self
     }
+
+    #[inline]
+    fn path_exists_case_sensitive(&self, path: &SystemPath, _prefix: &SystemPath) -> Result<bool> {
+        // The memory file system is case-sensitive.
+        Ok(self.path_exists(path))
+    }
 }
 
 impl WritableSystem for InMemorySystem {