diff --git a/crates/red_knot_python_semantic/src/module_resolver/mod.rs b/crates/red_knot_python_semantic/src/module_resolver/mod.rs index 06f13271f0819..7d47249f47150 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/mod.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/mod.rs @@ -13,7 +13,6 @@ use resolver::SearchPathIterator; mod module; mod path; mod resolver; -mod state; mod typeshed; #[cfg(test)] diff --git a/crates/red_knot_python_semantic/src/module_resolver/path.rs b/crates/red_knot_python_semantic/src/module_resolver/path.rs index 546f4b857c0f5..7f80a7ad432a5 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/path.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/path.rs @@ -9,11 +9,10 @@ use ruff_db::files::{system_path_to_file, vendored_path_to_file, File, FileError use ruff_db::system::{System, SystemPath, SystemPathBuf}; use ruff_db::vendored::{VendoredPath, VendoredPathBuf}; +use super::typeshed::{typeshed_versions, TypeshedVersionsParseError, TypeshedVersionsQueryResult}; use crate::db::Db; use crate::module_name::ModuleName; - -use super::state::ResolverState; -use super::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult}; +use crate::module_resolver::resolver::ResolverContext; /// A path that points to a Python module. /// @@ -60,7 +59,7 @@ impl ModulePath { } #[must_use] - pub(crate) fn is_directory(&self, resolver: &ResolverState) -> bool { + pub(super) fn is_directory(&self, resolver: &ResolverContext) -> bool { let ModulePath { search_path, relative_path, @@ -74,7 +73,7 @@ impl ModulePath { == Err(FileError::IsADirectory) } SearchPathInner::StandardLibraryCustom(stdlib_root) => { - match query_stdlib_version(Some(stdlib_root), relative_path, resolver) { + match query_stdlib_version(relative_path, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { @@ -84,7 +83,7 @@ impl ModulePath { } } SearchPathInner::StandardLibraryVendored(stdlib_root) => { - match query_stdlib_version(None, relative_path, resolver) { + match query_stdlib_version(relative_path, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => resolver @@ -96,7 +95,7 @@ impl ModulePath { } #[must_use] - pub(crate) fn is_regular_package(&self, resolver: &ResolverState) -> bool { + pub(super) fn is_regular_package(&self, resolver: &ResolverContext) -> bool { let ModulePath { search_path, relative_path, @@ -113,7 +112,7 @@ impl ModulePath { .is_ok() } SearchPathInner::StandardLibraryCustom(search_path) => { - match query_stdlib_version(Some(search_path), relative_path, resolver) { + match query_stdlib_version(relative_path, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => system_path_to_file( @@ -124,7 +123,7 @@ impl ModulePath { } } SearchPathInner::StandardLibraryVendored(search_path) => { - match query_stdlib_version(None, relative_path, resolver) { + match query_stdlib_version(relative_path, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => resolver @@ -136,7 +135,7 @@ impl ModulePath { } #[must_use] - pub(crate) fn to_file(&self, resolver: &ResolverState) -> Option { + pub(super) fn to_file(&self, resolver: &ResolverContext) -> Option { let db = resolver.db.upcast(); let ModulePath { search_path, @@ -150,7 +149,7 @@ impl ModulePath { system_path_to_file(db, search_path.join(relative_path)).ok() } SearchPathInner::StandardLibraryCustom(stdlib_root) => { - match query_stdlib_version(Some(stdlib_root), relative_path, resolver) { + match query_stdlib_version(relative_path, resolver) { TypeshedVersionsQueryResult::DoesNotExist => None, TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { @@ -159,7 +158,7 @@ impl ModulePath { } } SearchPathInner::StandardLibraryVendored(stdlib_root) => { - match query_stdlib_version(None, relative_path, resolver) { + match query_stdlib_version(relative_path, resolver) { TypeshedVersionsQueryResult::DoesNotExist => None, TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { @@ -273,19 +272,15 @@ fn stdlib_path_to_module_name(relative_path: &Utf8Path) -> Option { #[must_use] fn query_stdlib_version( - custom_stdlib_root: Option<&SystemPath>, relative_path: &Utf8Path, - resolver: &ResolverState, + context: &ResolverContext, ) -> TypeshedVersionsQueryResult { let Some(module_name) = stdlib_path_to_module_name(relative_path) else { return TypeshedVersionsQueryResult::DoesNotExist; }; - let ResolverState { - db, - typeshed_versions, - target_version, - } = resolver; - typeshed_versions.query_module(*db, &module_name, custom_stdlib_root, *target_version) + let ResolverContext { db, target_version } = context; + + typeshed_versions(*db).query_module(&module_name, *target_version) } /// Enumeration describing the various ways in which validation of a search path might fail. @@ -293,7 +288,7 @@ fn query_stdlib_version( /// If validation fails for a search path derived from the user settings, /// a message must be displayed to the user, /// as type checking cannot be done reliably in these circumstances. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug)] pub(crate) enum SearchPathValidationError { /// The path provided by the user was not a directory NotADirectory(SystemPathBuf), @@ -304,13 +299,12 @@ pub(crate) enum SearchPathValidationError { NoStdlibSubdirectory(SystemPathBuf), /// The typeshed path provided by the user is a directory, - /// but no `stdlib/VERSIONS` file exists. - /// (This is only relevant for stdlib search paths.) - NoVersionsFile(SystemPathBuf), - - /// `stdlib/VERSIONS` is a directory. + /// but `stdlib/VERSIONS` could not be read /// (This is only relevant for stdlib search paths.) - VersionsIsADirectory(SystemPathBuf), + FailedToReadVersionsFile { + path: SystemPathBuf, + error: std::io::Error, + }, /// The path provided by the user is a directory, /// and a `stdlib/VERSIONS` file exists, but it fails to parse. @@ -325,8 +319,9 @@ impl fmt::Display for SearchPathValidationError { Self::NoStdlibSubdirectory(path) => { write!(f, "The directory at {path} has no `stdlib/` subdirectory") } - Self::NoVersionsFile(path) => write!(f, "Expected a file at {path}/stdlib/VERSIONS"), - Self::VersionsIsADirectory(path) => write!(f, "{path}/stdlib/VERSIONS is a directory."), + Self::FailedToReadVersionsFile { path, error } => { + write!(f, "Failed to read the '{path}' file: {error}") + } Self::VersionsParseError(underlying_error) => underlying_error.fmt(f), } } @@ -384,11 +379,10 @@ pub(crate) struct SearchPath(Arc); impl SearchPath { fn directory_path(system: &dyn System, root: SystemPathBuf) -> SearchPathResult { - let canonicalized = system.canonicalize_path(&root).unwrap_or(root); - if system.is_directory(&canonicalized) { - Ok(canonicalized) + if system.is_directory(&root) { + Ok(root) } else { - Err(SearchPathValidationError::NotADirectory(canonicalized)) + Err(SearchPathValidationError::NotADirectory(root)) } } @@ -407,32 +401,22 @@ impl SearchPath { } /// Create a new standard-library search path pointing to a custom directory on disk - pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: SystemPathBuf) -> SearchPathResult { + pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: &SystemPath) -> SearchPathResult { let system = db.system(); - if !system.is_directory(&typeshed) { + if !system.is_directory(typeshed) { return Err(SearchPathValidationError::NotADirectory( typeshed.to_path_buf(), )); } + let stdlib = Self::directory_path(system, typeshed.join("stdlib")).map_err(|err| match err { - SearchPathValidationError::NotADirectory(path) => { - SearchPathValidationError::NoStdlibSubdirectory(path) + SearchPathValidationError::NotADirectory(_) => { + SearchPathValidationError::NoStdlibSubdirectory(typeshed.to_path_buf()) } err => err, })?; - let typeshed_versions = - system_path_to_file(db.upcast(), stdlib.join("VERSIONS")).map_err(|err| match err { - FileError::NotFound => SearchPathValidationError::NoVersionsFile(typeshed), - FileError::IsADirectory => { - SearchPathValidationError::VersionsIsADirectory(typeshed) - } - })?; - super::typeshed::parse_typeshed_versions(db, typeshed_versions) - .as_ref() - .map_err(|validation_error| { - SearchPathValidationError::VersionsParseError(validation_error.clone()) - })?; + Ok(Self(Arc::new(SearchPathInner::StandardLibraryCustom( stdlib, )))) @@ -623,11 +607,11 @@ mod tests { use ruff_db::Db; use crate::db::tests::TestDb; - - use super::*; use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder}; use crate::python_version::PythonVersion; + use super::*; + impl ModulePath { #[must_use] fn join(&self, component: &str) -> ModulePath { @@ -661,7 +645,7 @@ mod tests { .build(); assert_eq!( - SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()) + SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) .unwrap() .to_module_path() .with_py_extension(), @@ -669,7 +653,7 @@ mod tests { ); assert_eq!( - &SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()) + &SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) .unwrap() .join("foo") .with_pyi_extension(), @@ -780,7 +764,7 @@ mod tests { let TestCase { db, stdlib, .. } = TestCaseBuilder::new() .with_custom_typeshed(MockedTypeshed::default()) .build(); - SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()) + SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) .unwrap() .to_module_path() .push("bar.py"); @@ -792,7 +776,7 @@ mod tests { let TestCase { db, stdlib, .. } = TestCaseBuilder::new() .with_custom_typeshed(MockedTypeshed::default()) .build(); - SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()) + SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()) .unwrap() .to_module_path() .push("bar.rs"); @@ -824,7 +808,7 @@ mod tests { .with_custom_typeshed(MockedTypeshed::default()) .build(); - let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()).unwrap(); + let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap(); // Must have a `.pyi` extension or no extension: let bad_absolute_path = SystemPath::new("foo/stdlib/x.py"); @@ -872,8 +856,7 @@ mod tests { .with_custom_typeshed(typeshed) .with_target_version(target_version) .build(); - let stdlib = - SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()).unwrap(); + let stdlib = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap(); (db, stdlib) } @@ -898,7 +881,7 @@ mod tests { }; let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY38); + let resolver = ResolverContext::new(&db, PythonVersion::PY38); let asyncio_regular_package = stdlib_path.join("asyncio"); assert!(asyncio_regular_package.is_directory(&resolver)); @@ -926,7 +909,7 @@ mod tests { }; let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY38); + let resolver = ResolverContext::new(&db, PythonVersion::PY38); let xml_namespace_package = stdlib_path.join("xml"); assert!(xml_namespace_package.is_directory(&resolver)); @@ -948,7 +931,7 @@ mod tests { }; let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY38); + let resolver = ResolverContext::new(&db, PythonVersion::PY38); let functools_module = stdlib_path.join("functools.pyi"); assert!(functools_module.to_file(&resolver).is_some()); @@ -964,7 +947,7 @@ mod tests { }; let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY38); + let resolver = ResolverContext::new(&db, PythonVersion::PY38); let collections_regular_package = stdlib_path.join("collections"); assert_eq!(collections_regular_package.to_file(&resolver), None); @@ -980,7 +963,7 @@ mod tests { }; let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY38); + let resolver = ResolverContext::new(&db, PythonVersion::PY38); let importlib_namespace_package = stdlib_path.join("importlib"); assert_eq!(importlib_namespace_package.to_file(&resolver), None); @@ -1001,7 +984,7 @@ mod tests { }; let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY38); + let resolver = ResolverContext::new(&db, PythonVersion::PY38); let non_existent = stdlib_path.join("doesnt_even_exist"); assert_eq!(non_existent.to_file(&resolver), None); @@ -1029,7 +1012,7 @@ mod tests { }; let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY39); + let resolver = ResolverContext::new(&db, PythonVersion::PY39); // Since we've set the target version to Py39, // `collections` should now exist as a directory, according to VERSIONS... @@ -1058,7 +1041,7 @@ mod tests { }; let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY39); + let resolver = ResolverContext::new(&db, PythonVersion::PY39); // The `importlib` directory now also exists let importlib_namespace_package = stdlib_path.join("importlib"); @@ -1082,7 +1065,7 @@ mod tests { }; let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED); - let resolver = ResolverState::new(&db, PythonVersion::PY39); + let resolver = ResolverContext::new(&db, PythonVersion::PY39); // The `xml` package no longer exists on py39: let xml_namespace_package = stdlib_path.join("xml"); diff --git a/crates/red_knot_python_semantic/src/module_resolver/resolver.rs b/crates/red_knot_python_semantic/src/module_resolver/resolver.rs index 709d74fe9be0b..c83ee773bc641 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/resolver.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/resolver.rs @@ -4,16 +4,16 @@ use std::iter::FusedIterator; use rustc_hash::{FxBuildHasher, FxHashSet}; use ruff_db::files::{File, FilePath, FileRootKind}; -use ruff_db::system::{DirectoryEntry, SystemPath, SystemPathBuf}; -use ruff_db::vendored::VendoredPath; +use ruff_db::system::{DirectoryEntry, System, SystemPath, SystemPathBuf}; +use ruff_db::vendored::{VendoredFileSystem, VendoredPath}; use crate::db::Db; use crate::module_name::ModuleName; -use crate::{Program, SearchPathSettings}; +use crate::module_resolver::typeshed::{vendored_typeshed_versions, TypeshedVersions}; +use crate::{Program, PythonVersion, SearchPathSettings}; use super::module::{Module, ModuleKind}; use super::path::{ModulePath, SearchPath, SearchPathValidationError}; -use super::state::ResolverState; /// Resolves a module name to a module. pub fn resolve_module(db: &dyn Db, module_name: ModuleName) -> Option { @@ -122,7 +122,7 @@ pub(crate) fn search_paths(db: &dyn Db) -> SearchPathIterator { Program::get(db).search_paths(db).iter(db) } -#[derive(Debug, PartialEq, Eq, Default)] +#[derive(Debug, PartialEq, Eq)] pub(crate) struct SearchPaths { /// Search paths that have been statically determined purely from reading Ruff's configuration settings. /// These shouldn't ever change unless the config settings themselves change. @@ -135,6 +135,8 @@ pub(crate) struct SearchPaths { /// in terms of module-resolution priority until we've discovered the editable installs /// for the first `site-packages` path site_packages: Vec, + + typeshed_versions: Cow<'static, TypeshedVersions>, } impl SearchPaths { @@ -148,6 +150,10 @@ impl SearchPaths { db: &dyn Db, settings: SearchPathSettings, ) -> Result { + fn canonicalize(path: SystemPathBuf, system: &dyn System) -> SystemPathBuf { + system.canonicalize_path(&path).unwrap_or(path) + } + let SearchPathSettings { extra_paths, src_root, @@ -161,45 +167,60 @@ impl SearchPaths { let mut static_paths = vec![]; for path in extra_paths { - tracing::debug!("Adding static extra search-path '{path}'"); + let path = canonicalize(path, system); + files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath); + tracing::debug!("Adding extra search-path '{path}'"); - let search_path = SearchPath::extra(system, path)?; - files.try_add_root( - db.upcast(), - search_path.as_system_path().unwrap(), - FileRootKind::LibrarySearchPath, - ); - static_paths.push(search_path); + static_paths.push(SearchPath::extra(system, path)?); } - tracing::debug!("Adding static search path '{src_root}'"); + tracing::debug!("Adding search path '{src_root}'"); static_paths.push(SearchPath::first_party(system, src_root)?); - static_paths.push(if let Some(custom_typeshed) = custom_typeshed { - tracing::debug!("Adding static custom-sdtlib search-path '{custom_typeshed}'"); + let (typeshed_versions, stdlib_path) = if let Some(custom_typeshed) = custom_typeshed { + let custom_typeshed = canonicalize(custom_typeshed, system); - let search_path = SearchPath::custom_stdlib(db, custom_typeshed)?; + tracing::debug!("Adding custom-sdtlib search-path '{custom_typeshed}'"); files.try_add_root( db.upcast(), - search_path.as_system_path().unwrap(), + &custom_typeshed, FileRootKind::LibrarySearchPath, ); - search_path + + let versions_path = custom_typeshed.join("stdlib/VERSIONS"); + + let versions_content = system.read_to_string(&versions_path).map_err(|error| { + SearchPathValidationError::FailedToReadVersionsFile { + path: versions_path, + error, + } + })?; + + let parsed: TypeshedVersions = versions_content + .parse() + .map_err(SearchPathValidationError::VersionsParseError)?; + + let search_path = SearchPath::custom_stdlib(db, &custom_typeshed)?; + + (Cow::Owned(parsed), search_path) } else { - SearchPath::vendored_stdlib() - }); + tracing::debug!("Using vendored stdlib"); + ( + Cow::Borrowed(vendored_typeshed_versions()), + SearchPath::vendored_stdlib(), + ) + }; + + static_paths.push(stdlib_path); let mut site_packages: Vec<_> = Vec::with_capacity(site_packages_paths.len()); for path in site_packages_paths { + let path = canonicalize(path, system); + tracing::debug!("Adding site-package path '{path}'"); - let search_path = SearchPath::site_packages(system, path)?; - files.try_add_root( - db.upcast(), - search_path.as_system_path().unwrap(), - FileRootKind::LibrarySearchPath, - ); - site_packages.push(search_path); + files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath); + site_packages.push(SearchPath::site_packages(system, path)?); } // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step @@ -223,16 +244,21 @@ impl SearchPaths { Ok(SearchPaths { static_paths, site_packages, + typeshed_versions, }) } - pub(crate) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> { + pub(super) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> { SearchPathIterator { db, static_paths: self.static_paths.iter(), dynamic_paths: None, } } + + pub(super) fn typeshed_versions(&self) -> &TypeshedVersions { + &self.typeshed_versions + } } /// Collect all dynamic search paths. For each `site-packages` path: @@ -250,6 +276,7 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec { let SearchPaths { static_paths, site_packages, + typeshed_versions: _, } = Program::get(db).search_paths(db); let mut dynamic_paths = Vec::new(); @@ -314,12 +341,16 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec { let installations = all_pth_files.iter().flat_map(PthFile::items); for installation in installations { + let installation = system + .canonicalize_path(&installation) + .unwrap_or(installation); + if existing_paths.insert(Cow::Owned(installation.clone())) { - match SearchPath::editable(system, installation) { + match SearchPath::editable(system, installation.clone()) { Ok(search_path) => { tracing::debug!( "Adding editable installation to module resolution path {path}", - path = search_path.as_system_path().unwrap() + path = installation ); dynamic_paths.push(search_path); } @@ -481,7 +512,7 @@ struct ModuleNameIngredient<'db> { fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, ModuleKind)> { let program = Program::get(db); let target_version = program.target_version(db); - let resolver_state = ResolverState::new(db, target_version); + let resolver_state = ResolverContext::new(db, target_version); let is_builtin_module = ruff_python_stdlib::sys::is_builtin_module(target_version.minor, name.as_str()); @@ -544,7 +575,7 @@ fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, Mod fn resolve_package<'a, 'db, I>( module_search_path: &SearchPath, components: I, - resolver_state: &ResolverState<'db>, + resolver_state: &ResolverContext<'db>, ) -> Result where I: Iterator, @@ -626,6 +657,21 @@ impl PackageKind { } } +pub(super) struct ResolverContext<'db> { + pub(crate) db: &'db dyn Db, + pub(crate) target_version: PythonVersion, +} + +impl<'db> ResolverContext<'db> { + pub(super) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self { + Self { db, target_version } + } + + pub(super) fn vendored(&self) -> &VendoredFileSystem { + self.db.vendored() + } +} + #[cfg(test)] mod tests { use ruff_db::files::{system_path_to_file, File, FilePath}; diff --git a/crates/red_knot_python_semantic/src/module_resolver/state.rs b/crates/red_knot_python_semantic/src/module_resolver/state.rs deleted file mode 100644 index cb56e5c8463fd..0000000000000 --- a/crates/red_knot_python_semantic/src/module_resolver/state.rs +++ /dev/null @@ -1,25 +0,0 @@ -use ruff_db::vendored::VendoredFileSystem; - -use super::typeshed::LazyTypeshedVersions; -use crate::db::Db; -use crate::python_version::PythonVersion; - -pub(crate) struct ResolverState<'db> { - pub(crate) db: &'db dyn Db, - pub(crate) typeshed_versions: LazyTypeshedVersions<'db>, - pub(crate) target_version: PythonVersion, -} - -impl<'db> ResolverState<'db> { - pub(crate) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self { - Self { - db, - typeshed_versions: LazyTypeshedVersions::new(), - target_version, - } - } - - pub(crate) fn vendored(&self) -> &VendoredFileSystem { - self.db.vendored() - } -} diff --git a/crates/red_knot_python_semantic/src/module_resolver/testing.rs b/crates/red_knot_python_semantic/src/module_resolver/testing.rs index 87a05001113c7..484883e6d04f4 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/testing.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/testing.rs @@ -179,6 +179,7 @@ impl TestCaseBuilder { first_party_files, site_packages_files, } = self; + TestCaseBuilder { typeshed_option: typeshed, target_version, @@ -195,6 +196,7 @@ impl TestCaseBuilder { site_packages, target_version, } = self.with_custom_typeshed(MockedTypeshed::default()).build(); + TestCase { db, src, diff --git a/crates/red_knot_python_semantic/src/module_resolver/typeshed/mod.rs b/crates/red_knot_python_semantic/src/module_resolver/typeshed/mod.rs index 97cac75fa62e0..fe6b08f5766c9 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/typeshed/mod.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/typeshed/mod.rs @@ -1,6 +1,6 @@ pub use self::vendored::vendored_typeshed_stubs; pub(super) use self::versions::{ - parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsParseError, + typeshed_versions, vendored_typeshed_versions, TypeshedVersions, TypeshedVersionsParseError, TypeshedVersionsQueryResult, }; diff --git a/crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs b/crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs index de53f0054809f..5948376bebf9e 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs @@ -1,4 +1,3 @@ -use std::cell::OnceCell; use std::collections::BTreeMap; use std::fmt; use std::num::{NonZeroU16, NonZeroUsize}; @@ -6,78 +5,12 @@ use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; use once_cell::sync::Lazy; -use ruff_db::system::SystemPath; use rustc_hash::FxHashMap; -use ruff_db::files::{system_path_to_file, File}; - use super::vendored::vendored_typeshed_stubs; use crate::db::Db; use crate::module_name::ModuleName; -use crate::python_version::PythonVersion; - -#[derive(Debug)] -pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>); - -impl<'db> LazyTypeshedVersions<'db> { - #[must_use] - pub(crate) fn new() -> Self { - Self(OnceCell::new()) - } - - /// Query whether a module exists at runtime in the stdlib on a certain Python version. - /// - /// Simply probing whether a file exists in typeshed is insufficient for this question, - /// as a module in the stdlib may have been added in Python 3.10, but the typeshed stub - /// will still be available (either in a custom typeshed dir or in our vendored copy) - /// even if the user specified Python 3.8 as the target version. - /// - /// For top-level modules and packages, the VERSIONS file can always provide an unambiguous answer - /// as to whether the module exists on the specified target version. However, VERSIONS does not - /// provide comprehensive information on all submodules, meaning that this method sometimes - /// returns [`TypeshedVersionsQueryResult::MaybeExists`]. - /// See [`TypeshedVersionsQueryResult`] for more details. - #[must_use] - pub(crate) fn query_module( - &self, - db: &'db dyn Db, - module: &ModuleName, - stdlib_root: Option<&SystemPath>, - target_version: PythonVersion, - ) -> TypeshedVersionsQueryResult { - let versions = self.0.get_or_init(|| { - let versions_path = if let Some(system_path) = stdlib_root { - system_path.join("VERSIONS") - } else { - return &VENDORED_VERSIONS; - }; - let Ok(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { - todo!( - "Still need to figure out how to handle VERSIONS files being deleted \ - from custom typeshed directories! Expected a file to exist at {versions_path}" - ) - }; - // TODO(Alex/Micha): If VERSIONS is invalid, - // this should invalidate not just the specific module resolution we're currently attempting, - // but all type inference that depends on any standard-library types. - // Unwrapping here is not correct... - parse_typeshed_versions(db, versions_file).as_ref().unwrap() - }); - versions.query_module(module, target_version) - } -} - -#[salsa::tracked(return_ref)] -pub(crate) fn parse_typeshed_versions( - db: &dyn Db, - versions_file: File, -) -> Result { - // TODO: Handle IO errors - let file_content = versions_file - .read_to_string(db.upcast()) - .unwrap_or_default(); - file_content.parse() -} +use crate::{Program, PythonVersion}; static VENDORED_VERSIONS: Lazy = Lazy::new(|| { TypeshedVersions::from_str( @@ -88,6 +21,14 @@ static VENDORED_VERSIONS: Lazy = Lazy::new(|| { .unwrap() }); +pub(crate) fn vendored_typeshed_versions() -> &'static TypeshedVersions { + &VENDORED_VERSIONS +} + +pub(crate) fn typeshed_versions(db: &dyn Db) -> &TypeshedVersions { + Program::get(db).search_paths(db).typeshed_versions() +} + #[derive(Debug, PartialEq, Eq, Clone)] pub(crate) struct TypeshedVersionsParseError { line_number: Option, @@ -164,7 +105,7 @@ impl fmt::Display for TypeshedVersionsParseErrorKind { } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub(crate) struct TypeshedVersions(FxHashMap); impl TypeshedVersions { @@ -174,7 +115,7 @@ impl TypeshedVersions { } #[must_use] - fn query_module( + pub(crate) fn query_module( &self, module: &ModuleName, target_version: PythonVersion, diff --git a/crates/red_knot_python_semantic/src/program.rs b/crates/red_knot_python_semantic/src/program.rs index 082d6b06dc774..5a43f52b0c40b 100644 --- a/crates/red_knot_python_semantic/src/program.rs +++ b/crates/red_knot_python_semantic/src/program.rs @@ -12,7 +12,6 @@ use crate::Db; pub struct Program { pub target_version: PythonVersion, - #[default] #[return_ref] pub(crate) search_paths: SearchPaths, } @@ -29,9 +28,8 @@ impl Program { let search_paths = SearchPaths::from_settings(db, search_paths) .with_context(|| "Invalid search path settings")?; - Ok(Program::builder(settings.target_version) + Ok(Program::builder(settings.target_version, search_paths) .durability(Durability::HIGH) - .search_paths(search_paths) .new(db)) }