[ty] Infer the Python version from --python=<system installation> on Unix

Author: AlexWaygood

crates/ruff_python_ast/src/python_version.rs

@@ -1,4 +1,4 @@
-use std::fmt;
+use std::{fmt, str::FromStr};
 
 /// Representation of a Python version.
 ///
@@ -123,6 +123,15 @@ impl fmt::Display for PythonVersion {
     }
 }
 
+impl FromStr for PythonVersion {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (major, minor) = s.split_once('.').ok_or(())?;
+        Self::try_from((major, minor)).map_err(|_| ())
+    }
+}
+
 #[cfg(feature = "serde")]
 mod serde {
     use super::PythonVersion;

crates/ty/docs/cli.md

@@ -82,10 +82,11 @@ pub(crate) struct CheckCommand {
     /// type definitions of first- and third-party modules that are conditional on the Python version.
     ///
     /// By default, the Python version is inferred as the lower bound of the project's
-    /// `requires-python` field from the `pyproject.toml`, if available. Otherwise, if a virtual
-    /// environment has been configured or detected and a Python version can be inferred from the
-    /// virtual environment's metadata, that version will be used. If neither of these applies, ty
-    /// will fall back to the latest stable Python version supported by ty (currently 3.13).
+    /// `requires-python` field from the `pyproject.toml`, if available. Otherwise, if a Python
+    /// installation has been configured or detected and a Python version can be inferred from the
+    /// metadata or layout of that Python installation, that version will be used. If neither of
+    /// these applies, ty will fall back to the latest stable Python version supported by ty
+    /// (currently 3.13).
     #[arg(long, value_name = "VERSION", alias = "target-version")]
     pub(crate) python_version: Option<PythonVersion>,
 

crates/ty/src/args.rs

@@ -188,6 +188,102 @@ fn config_file_annotation_showing_where_python_version_set_typing_error() -> any
     Ok(())
 }
 
+// We can't infer the Python version from the system installation on Windows,
+// because `site-packages` on Windows are located at `<sys.prefix>/Lib/site-packages`
+// rather than `<sys.prefix>/lib/pythonX.Y/site-packages`.
+#[cfg(not(windows))]
+#[test]
+fn python_version_inferred_from_system_installation() -> anyhow::Result<()> {
+    let cpython_case = CliTest::with_files([
+        ("pythons/Python3.8/bin/python", ""),
+        ("pythons/Python3.8/lib/python3.8/site-packages/foo.py", ""),
+        ("test.py", "aiter"),
+    ])?;
+
+    assert_cmd_snapshot!(cpython_case.command().arg("--python").arg("pythons/Python3.8/bin/python"), @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    error[unresolved-reference]: Name `aiter` used when not defined
+     --> test.py:1:1
+      |
+    1 | aiter
+      | ^^^^^
+      |
+    info: `aiter` was added as a builtin in Python 3.10
+    info: Python 3.8 was assumed when resolving types because of the layout of your Python installation
+    info: The primary `site-packages` directory of your installation was found at `lib/python3.8/site-packages/`
+    info: No Python version was specified on the command line or in a configuration file
+    info: rule `unresolved-reference` is enabled by default
+
+    Found 1 diagnostic
+
+    ----- stderr -----
+    WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors.
+    ");
+
+    let pypy_case = CliTest::with_files([
+        ("pythons/pypy3.8/bin/python", ""),
+        ("pythons/pypy3.8/lib/pypy3.8/site-packages/foo.py", ""),
+        ("test.py", "aiter"),
+    ])?;
+
+    assert_cmd_snapshot!(pypy_case.command().arg("--python").arg("pythons/pypy3.8/bin/python"), @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    error[unresolved-reference]: Name `aiter` used when not defined
+     --> test.py:1:1
+      |
+    1 | aiter
+      | ^^^^^
+      |
+    info: `aiter` was added as a builtin in Python 3.10
+    info: Python 3.8 was assumed when resolving types because of the layout of your Python installation
+    info: The primary `site-packages` directory of your installation was found at `lib/pypy3.8/site-packages/`
+    info: No Python version was specified on the command line or in a configuration file
+    info: rule `unresolved-reference` is enabled by default
+
+    Found 1 diagnostic
+
+    ----- stderr -----
+    WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors.
+    ");
+
+    let free_threaded_case = CliTest::with_files([
+        ("pythons/Python3.13t/bin/python", ""),
+        (
+            "pythons/Python3.13t/lib/python3.13t/site-packages/foo.py",
+            "",
+        ),
+        ("test.py", "import string.templatelib"),
+    ])?;
+
+    assert_cmd_snapshot!(free_threaded_case.command().arg("--python").arg("pythons/Python3.13t/bin/python"), @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    error[unresolved-import]: Cannot resolve imported module `string.templatelib`
+     --> test.py:1:8
+      |
+    1 | import string.templatelib
+      |        ^^^^^^^^^^^^^^^^^^
+      |
+    info: The stdlib module `string.templatelib` is only available on Python 3.14+
+    info: Python 3.13 was assumed when resolving modules because of the layout of your Python installation
+    info: The primary `site-packages` directory of your installation was found at `lib/python3.13t/site-packages/`
+    info: No Python version was specified on the command line or in a configuration file
+    info: rule `unresolved-import` is enabled by default
+
+    Found 1 diagnostic
+
+    ----- stderr -----
+    WARN ty is pre-release software and not ready for production use. Expect to encounter bugs, missing features, and fatal errors.
+    ");
+
+    Ok(())
+}
+
 #[test]
 fn pyvenv_cfg_file_annotation_showing_where_python_version_set() -> anyhow::Result<()> {
     let case = CliTest::with_files([

crates/ty/tests/cli/python_environment.rs

@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 use std::fmt;
 use std::iter::FusedIterator;
-use std::str::Split;
+use std::str::{FromStr, Split};
 
 use compact_str::format_compact;
 use rustc_hash::{FxBuildHasher, FxHashSet};
@@ -15,7 +15,9 @@ use crate::db::Db;
 use crate::module_name::ModuleName;
 use crate::module_resolver::typeshed::{TypeshedVersions, vendored_typeshed_versions};
 use crate::site_packages::{PythonEnvironment, SitePackagesPaths, SysPrefixPathOrigin};
-use crate::{Program, PythonPath, PythonVersionWithSource, SearchPathSettings};
+use crate::{
+    Program, PythonPath, PythonVersionSource, PythonVersionWithSource, SearchPathSettings,
+};
 
 use super::module::{Module, ModuleKind};
 use super::path::{ModulePath, SearchPath, SearchPathValidationError};
@@ -158,7 +160,7 @@ pub struct SearchPaths {
     /// The Python version for the search paths, if any.
     ///
     /// This is read from the `pyvenv.cfg` if present.
-    python_version: Option<PythonVersionWithSource>,
+    python_version_from_metadata: Option<PythonVersionWithSource>,
 }
 
 impl SearchPaths {
@@ -304,7 +306,7 @@ impl SearchPaths {
             static_paths,
             site_packages,
             typeshed_versions,
-            python_version,
+            python_version_from_metadata: python_version,
         })
     }
 
@@ -330,8 +332,42 @@ impl SearchPaths {
         &self.typeshed_versions
     }
 
-    pub fn python_version(&self) -> Option<&PythonVersionWithSource> {
-        self.python_version.as_ref()
+    pub fn python_version(&self) -> Option<Cow<PythonVersionWithSource>> {
+        if let Some(version) = self.python_version_from_metadata.as_ref() {
+            return Some(Cow::Borrowed(version));
+        }
+
+        if cfg!(windows) {
+            // The path to `site-packages` on Unix is
+            // `<sys.prefix>/lib/pythonX.Y/site-packages`,
+            // but on Windows it's `<sys.prefix>/Lib/site-packages`.
+            return None;
+        }
+
+        let primary_site_packages = self.site_packages.first()?.as_system_path()?;
+
+        let mut site_packages_ancestor_components = primary_site_packages
+            .components()
+            .rev()
+            .skip(1)
+            .map(|c| c.as_str());
+
+        let parent_component = site_packages_ancestor_components.next()?;
+
+        if site_packages_ancestor_components.next()? != "lib" {
+            return None;
+        }
+
+        let version = parent_component
+            .strip_prefix("python")
+            .or_else(|| parent_component.strip_prefix("pypy"))?
+            .trim_end_matches('t');
+
+        let version = PythonVersion::from_str(version).ok()?;
+        let source = PythonVersionSource::InstallationDirectoryLayout {
+            site_packages_parent_dir: Box::from(parent_component),
+        };
+        Some(Cow::Owned(PythonVersionWithSource { version, source }))
     }
 }
 
@@ -351,7 +387,7 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
         static_paths,
         site_packages,
         typeshed_versions: _,
-        python_version: _,
+        python_version_from_metadata: _,
     } = Program::get(db).search_paths(db);
 
     let mut dynamic_paths = Vec::new();

crates/ty_python_semantic/src/module_resolver/resolver.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::sync::Arc;
 
 use crate::Db;
@@ -37,8 +38,10 @@ impl Program {
         let search_paths = SearchPaths::from_settings(db, &search_paths)
             .with_context(|| "Invalid search path settings")?;
 
-        let python_version_with_source =
-            Self::resolve_python_version(python_version_with_source, search_paths.python_version());
+        let python_version_with_source = Self::resolve_python_version(
+            python_version_with_source,
+            search_paths.python_version().as_deref(),
+        );
 
         tracing::info!(
             "Python version: Python {python_version}, platform: {python_platform}",
@@ -78,8 +81,10 @@ impl Program {
 
         let search_paths = SearchPaths::from_settings(db, &search_paths)?;
 
-        let new_python_version =
-            Self::resolve_python_version(python_version_with_source, search_paths.python_version());
+        let new_python_version = Self::resolve_python_version(
+            python_version_with_source,
+            search_paths.python_version().as_deref(),
+        );
 
         if self.search_paths(db) != &search_paths {
             tracing::debug!("Updating search paths");
@@ -112,8 +117,11 @@ impl Program {
         let search_paths = SearchPaths::from_settings(db, search_path_settings)?;
 
         let current_python_version = self.python_version_with_source(db);
-        let python_version_from_environment =
-            search_paths.python_version().cloned().unwrap_or_default();
+
+        let python_version_from_environment = search_paths
+            .python_version()
+            .map(Cow::into_owned)
+            .unwrap_or_default();
 
         if current_python_version != &python_version_from_environment
             && current_python_version.source.priority()
@@ -153,6 +161,13 @@ pub enum PythonVersionSource {
     /// The virtual environment might have been configured, activated or inferred.
     PyvenvCfgFile(PythonVersionFileSource),
 
+    /// Value inferred from the layout of the Python installation.
+    ///
+    /// This only ever applies on Unix. On Unix, the `site-packages` directory
+    /// will always be at `sys.prefix/lib/pythonX.Y/site-packages`,
+    /// so we can infer the Python version from the parent directory of `site-packages`.
+    InstallationDirectoryLayout { site_packages_parent_dir: Box<str> },
+
     /// The value comes from a CLI argument, while it's left open if specified using a short argument,
     /// long argument (`--extra-paths`) or `--config key=value`.
     Cli,
@@ -169,6 +184,9 @@ impl PythonVersionSource {
             PythonVersionSource::PyvenvCfgFile(_) => PythonSourcePriority::PyvenvCfgFile,
             PythonVersionSource::ConfigFile(_) => PythonSourcePriority::ConfigFile,
             PythonVersionSource::Cli => PythonSourcePriority::Cli,
+            PythonVersionSource::InstallationDirectoryLayout { .. } => {
+                PythonSourcePriority::InstallationDirectoryLayout
+            }
         }
     }
 }
@@ -182,9 +200,10 @@ impl PythonVersionSource {
 #[cfg_attr(test, derive(strum_macros::EnumIter))]
 enum PythonSourcePriority {
     Default = 0,
-    PyvenvCfgFile = 1,
-    ConfigFile = 2,
-    Cli = 3,
+    InstallationDirectoryLayout = 1,
+    PyvenvCfgFile = 2,
+    ConfigFile = 3,
+    Cli = 4,
 }
 
 /// Information regarding the file and [`TextRange`] of the configuration
@@ -312,7 +331,9 @@ mod tests {
                         match other {
                             PythonSourcePriority::Cli => assert!(other > priority, "{other:?}"),
                             PythonSourcePriority::ConfigFile => assert_eq!(priority, other),
-                            PythonSourcePriority::PyvenvCfgFile | PythonSourcePriority::Default => {
+                            PythonSourcePriority::PyvenvCfgFile
+                            | PythonSourcePriority::Default
+                            | PythonSourcePriority::InstallationDirectoryLayout => {
                                 assert!(priority > other, "{other:?}");
                             }
                         }
@@ -327,6 +348,24 @@ mod tests {
                                 assert!(other > priority, "{other:?}");
                             }
                             PythonSourcePriority::PyvenvCfgFile => assert_eq!(priority, other),
+                            PythonSourcePriority::Default
+                            | PythonSourcePriority::InstallationDirectoryLayout => {
+                                assert!(priority > other, "{other:?}");
+                            }
+                        }
+                    }
+                }
+                PythonSourcePriority::InstallationDirectoryLayout => {
+                    for other in PythonSourcePriority::iter() {
+                        match other {
+                            PythonSourcePriority::Cli
+                            | PythonSourcePriority::ConfigFile
+                            | PythonSourcePriority::PyvenvCfgFile => {
+                                assert!(other > priority, "{other:?}");
+                            }
+                            PythonSourcePriority::InstallationDirectoryLayout => {
+                                assert_eq!(priority, other);
+                            }
                             PythonSourcePriority::Default => assert!(priority > other, "{other:?}"),
                         }
                     }

crates/ty_python_semantic/src/program.rs

@@ -61,6 +61,23 @@ pub fn add_inferred_python_version_hint_to_diagnostic(
                 or in a configuration file",
             );
         }
+        crate::PythonVersionSource::InstallationDirectoryLayout {
+            site_packages_parent_dir,
+        } => {
+            // TODO: it would also be nice to tell them how we resolved this Python installation...
+            diagnostic.info(format_args!(
+                "Python {version} was assumed when {action} \
+                because of the layout of your Python installation"
+            ));
+            diagnostic.info(format_args!(
+                "The primary `site-packages` directory of your installation was found \
+                at `lib/{site_packages_parent_dir}/site-packages/`"
+            ));
+            diagnostic.info(
+                "No Python version was specified on the command line \
+                or in a configuration file",
+            );
+        }
         crate::PythonVersionSource::Default => {
             diagnostic.info(format_args!(
                 "Python {version} was assumed when {action} \