Implement --user flag and user scheme support for uv pip

crates/uv-interpreter/python/get_interpreter_info.py

@@ -214,7 +214,7 @@ def expand_path(path: str) -> str:
         }
 
 
-def get_scheme():
+def get_scheme(user: bool = False):
     """Return the Scheme for the current interpreter.
 
     The paths returned should be absolute.
@@ -223,7 +223,7 @@ def get_scheme():
         https://github.com/pypa/pip/blob/ae5fff36b0aad6e5e0037884927eaa29163c0611/src/pip/_internal/locations/__init__.py#L230
     """
 
-    def get_sysconfig_scheme():
+    def get_sysconfig_scheme(user: bool = False):
         """Get the "scheme" corresponding to the input parameters.
 
         Uses the `sysconfig` module to get the scheme.
@@ -272,6 +272,20 @@ def _should_use_osx_framework_prefix() -> bool:
                 and is_osx_framework()
             )
 
+        def _infer_user() -> str:
+            """Try to find a user scheme for the current platform."""
+            if _PREFERRED_SCHEME_API:
+                return _PREFERRED_SCHEME_API("user")
+            if is_osx_framework() and not running_under_virtualenv():
+                suffixed = "osx_framework_user"
+            else:
+                suffixed = f"{os.name}_user"
+            if suffixed in _AVAILABLE_SCHEMES:
+                return suffixed
+            # Fall back to posix_user if user scheme is unavailable.
+            # `pip` would raise exeception when "posix_user" not in _AVAILABLE_SCHEMES
+            return "posix_user"
+
         def _infer_prefix() -> str:
             """Try to find a prefix scheme for the current platform.
 
@@ -298,11 +312,15 @@ def _infer_prefix() -> str:
             suffixed = f"{os.name}_prefix"
             if suffixed in _AVAILABLE_SCHEMES:
                 return suffixed
-            if os.name in _AVAILABLE_SCHEMES:  # On Windows, prefx is just called "nt".
+            if os.name in _AVAILABLE_SCHEMES:  # On Windows, prefix is just called "nt".
                 return os.name
             return "posix_prefix"
 
-        scheme_name = _infer_prefix()
+        if user:
+            scheme_name = _infer_user()
+        else:
+            scheme_name = _infer_prefix()
+
         paths = sysconfig.get_paths(scheme=scheme_name)
 
         # Logic here is very arbitrary, we're doing it for compatibility, don't ask.
@@ -319,7 +337,7 @@ def _infer_prefix() -> str:
             "data": paths["data"],
         }
 
-    def get_distutils_scheme():
+    def get_distutils_scheme(user: bool = False):
         """Get the "scheme" corresponding to the input parameters.
 
         Uses the deprecated `distutils` module to get the scheme.
@@ -358,6 +376,7 @@ def get_distutils_scheme():
             warnings.simplefilter("ignore")
             i = d.get_command_obj("install", create=True)
 
+        i.user = user
         i.finalize_options()
 
         scheme = {}
@@ -374,9 +393,13 @@ def get_distutils_scheme():
             scheme.update({"purelib": i.install_lib, "platlib": i.install_lib})
 
         if running_under_virtualenv():
+            if user:
+                prefix = i.install_userbase
+            else:
+                prefix = i.prefix
             # noinspection PyUnresolvedReferences
             scheme["headers"] = os.path.join(
-                i.prefix,
+                prefix,
                 "include",
                 "site",
                 f"python{get_major_minor_version()}",
@@ -400,10 +423,13 @@ def get_distutils_scheme():
     )
 
     if use_sysconfig:
-        return get_sysconfig_scheme()
+        return get_sysconfig_scheme(user)
     else:
-        return get_distutils_scheme()
+        return get_distutils_scheme(user)
+
 
+# Read the environment variable `_UV_USE_USER_SCHEME` to determine if we should use the user scheme.
+user = os.getenv("_UV_USE_USER_SCHEME", "False").upper() in {"TRUE", "1"}
 
 def get_operating_system_and_architecture():
     """Determine the Python interpreter architecture and operating system.
@@ -512,7 +538,7 @@ def get_operating_system_and_architecture():
     "base_executable": getattr(sys, "_base_executable", None),
     "sys_executable": sys.executable,
     "stdlib": sysconfig.get_path("stdlib"),
-    "scheme": get_scheme(),
+    "scheme": get_scheme(user),
     "virtualenv": get_virtualenv(),
     "platform": get_operating_system_and_architecture(),
 }

crates/uv-interpreter/src/cfg.rs

@@ -10,20 +10,23 @@ pub struct PyVenvConfiguration {
     pub(crate) virtualenv: bool,
     /// The version of the `uv` package used to create the virtual environment, if any.
     pub(crate) uv: bool,
+    /// If the virtual environment has access to the system packages, per PEP 405.
+    pub(crate) include_system_site_packages: bool,
 }
 
 impl PyVenvConfiguration {
     /// Parse a `pyvenv.cfg` file into a [`PyVenvConfiguration`].
     pub fn parse(cfg: impl AsRef<Path>) -> Result<Self, Error> {
         let mut virtualenv = false;
         let mut uv = false;
+        let mut include_system_site_packages = false;
 
         // Per https://snarky.ca/how-virtual-environments-work/, the `pyvenv.cfg` file is not a
         // valid INI file, and is instead expected to be parsed by partitioning each line on the
         // first equals sign.
         let content = fs::read_to_string(&cfg)?;
         for line in content.lines() {
-            let Some((key, _value)) = line.split_once('=') else {
+            let Some((key, value)) = line.split_once('=') else {
                 continue;
             };
             match key.trim() {
@@ -33,11 +36,18 @@ impl PyVenvConfiguration {
                 "uv" => {
                     uv = true;
                 }
+                "include-system-site-packages" => {
+                    include_system_site_packages = value.trim() == "true";
+                }
                 _ => {}
             }
         }
 
-        Ok(Self { virtualenv, uv })
+        Ok(Self {
+            virtualenv,
+            uv,
+            include_system_site_packages,
+        })
     }
 
     /// Returns true if the virtual environment was created with the `virtualenv` package.
@@ -49,6 +59,11 @@ impl PyVenvConfiguration {
     pub fn is_uv(&self) -> bool {
         self.uv
     }
+
+    /// Return true if the virtual environment has access to system site packages.
+    pub fn include_system_site_packages(&self) -> bool {
+        self.include_system_site_packages
+    }
 }
 
 #[derive(Debug, Error)]

crates/uv-interpreter/src/interpreter.rs

@@ -102,6 +102,15 @@ impl Interpreter {
         }
     }
 
+    /// Return a new [`Interpreter`] with user scheme.
+    pub fn with_user_scheme(self, cache: &Cache) -> Result<Self, Error> {
+        let info = InterpreterInfo::query_user_scheme_info(self.sys_executable(), cache)?;
+        Ok(Self {
+            scheme: info.scheme,
+            ..self
+        })
+    }
+
     /// Returns the path to the Python virtual environment.
     #[inline]
     pub fn platform(&self) -> &Platform {
@@ -523,6 +532,60 @@ impl InterpreterInfo {
 
         Ok(info)
     }
+
+    /// Return the [`InterpreterInfo`] for the given Python interpreter with user scheme.
+    pub(crate) fn query_user_scheme_info(interpreter: &Path, cache: &Cache) -> Result<Self, Error> {
+        let envs = vec![("_UV_USE_USER_SCHEME", "1")];
+        let tempdir = tempfile::tempdir_in(cache.root())?;
+        Self::setup_python_query_files(tempdir.path())?;
+
+        let output = Command::new(interpreter)
+            .arg("-m")
+            .arg("python.get_interpreter_info")
+            .envs(envs)
+            .current_dir(tempdir.path().simplified())
+            .output()
+            .map_err(|err| Error::PythonSubcommandLaunch {
+                interpreter: interpreter.to_path_buf(),
+                err,
+            })?;
+
+        // stderr isn't technically a criterion for success, but i don't know of any cases where there
+        // should be stderr output and if there is, we want to know
+        if !output.status.success() || !output.stderr.is_empty() {
+            return Err(Error::PythonSubcommandOutput {
+                message: format!(
+                    "Querying Python at `{}` failed with status {}",
+                    interpreter.display(),
+                    output.status,
+                ),
+                exit_code: output.status,
+                stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(),
+                stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
+            });
+        }
+
+        let result: InterpreterInfoResult =
+            serde_json::from_slice(&output.stdout).map_err(|err| {
+                Error::PythonSubcommandOutput {
+                    message: format!(
+                        "Querying Python at `{}` did not return the expected data: {err}",
+                        interpreter.display(),
+                    ),
+                    exit_code: output.status,
+                    stdout: String::from_utf8_lossy(&output.stdout).trim().to_string(),
+                    stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
+                }
+            })?;
+
+        match result {
+            InterpreterInfoResult::Error(err) => Err(Error::QueryScript {
+                err,
+                interpreter: interpreter.to_path_buf(),
+            }),
+            InterpreterInfoResult::Success(data) => Ok(*data),
+        }
+    }
 }
 
 #[cfg(unix)]

crates/uv-interpreter/src/python_environment.rs

@@ -1,3 +1,4 @@
+use fs_err as fs;
 use std::env;
 use std::path::{Path, PathBuf};
 
@@ -67,6 +68,45 @@ impl PythonEnvironment {
         }
     }
 
+    /// Create a [`PythonEnvironment`] with user scheme.
+    pub fn from_user_scheme(python: Option<&str>, cache: &Cache) -> Result<Self, Error> {
+        // Attempt to determine the interpreter based on the provided criteria
+        let interpreter = if let Some(requested_python) = python {
+            // If a specific Python version is requested
+            Self::from_requested_python(requested_python, cache)?.interpreter
+        } else if let Some(_venv) = detect_virtual_env()? {
+            // If a virtual environment is detected
+            Self::from_virtualenv(cache)?.interpreter
+        } else {
+            // Fallback to the default Python interpreter
+            Self::from_default_python(cache)?.interpreter
+        };
+
+        // Apply the user scheme to the determined interpreter
+        let interpreter_with_user_scheme = interpreter.with_user_scheme(cache)?;
+
+        // Ensure interpreter scheme directories exist, as per the model in pip
+        // <https://github.com/pypa/pip/blob/main/src/pip/_internal/models/scheme.py#L12>
+        let directories = vec![
+            interpreter_with_user_scheme.platlib(),
+            interpreter_with_user_scheme.purelib(),
+            interpreter_with_user_scheme.scripts(),
+            interpreter_with_user_scheme.data(),
+            interpreter_with_user_scheme.include(),
+        ];
+
+        for path in directories {
+            if !Path::new(path).exists() {
+                fs::create_dir_all(path)?;
+            }
+        }
+
+        Ok(Self {
+            root: interpreter_with_user_scheme.prefix().to_path_buf(),
+            interpreter: interpreter_with_user_scheme,
+        })
+    }
+
     /// Returns the location of the Python interpreter.
     pub fn root(&self) -> &Path {
         &self.root

crates/uv/src/commands/pip_freeze.rs

@@ -19,11 +19,14 @@ pub(crate) fn pip_freeze(
     strict: bool,
     python: Option<&str>,
     system: bool,
+    user: bool,
     cache: &Cache,
     printer: Printer,
 ) -> Result<ExitStatus> {
     // Detect the current Python interpreter.
-    let venv = if let Some(python) = python {
+    let venv = if user {
+        PythonEnvironment::from_user_scheme(python, cache)?
+    } else if let Some(python) = python {
         PythonEnvironment::from_requested_python(python, cache)?
     } else if system {
         PythonEnvironment::from_default_python(cache)?

crates/uv/src/commands/pip_install.rs

@@ -70,9 +70,10 @@ pub(crate) async fn pip_install(
     python: Option<String>,
     system: bool,
     break_system_packages: bool,
+    user: bool,
     native_tls: bool,
-    cache: Cache,
     dry_run: bool,
+    cache: Cache,
     printer: Printer,
 ) -> Result<ExitStatus> {
     let start = std::time::Instant::now();
@@ -109,7 +110,9 @@ pub(crate) async fn pip_install(
     }
 
     // Detect the current Python interpreter.
-    let venv = if let Some(python) = python.as_ref() {
+    let venv = if user {
+        PythonEnvironment::from_user_scheme(python.as_deref(), &cache)?
+    } else if let Some(python) = python.as_ref() {
         PythonEnvironment::from_requested_python(python, &cache)?
     } else if system {
         PythonEnvironment::from_default_python(&cache)?
@@ -142,6 +145,18 @@ pub(crate) async fn pip_install(
         }
     }
 
+    // Check if virtualenv has access to the system site-packages
+    // <https://github.com/pypa/pip/blob/a33caa26f2bf525eafb4ec004f9c1bd18d238a31/src/pip/_internal/commands/install.py#L686>
+    if user
+        && venv.interpreter().is_virtualenv()
+        && !venv.cfg().unwrap().include_system_site_packages()
+    {
+        return Err(anyhow::anyhow!(
+                    "Can not perform a '--user' install. User site-packages are not visible in this virtualenv {}.",
+                    venv.root().simplified_display().cyan()
+                ));
+    }
+
     let _lock = venv.lock()?;
 
     // Determine the set of installed packages.

crates/uv/src/commands/pip_list.rs

@@ -30,11 +30,14 @@ pub(crate) fn pip_list(
     format: &ListFormat,
     python: Option<&str>,
     system: bool,
+    user: bool,
     cache: &Cache,
     printer: Printer,
 ) -> Result<ExitStatus> {
     // Detect the current Python interpreter.
-    let venv = if let Some(python) = python {
+    let venv = if user {
+        PythonEnvironment::from_user_scheme(python, cache)?
+    } else if let Some(python) = python {
         PythonEnvironment::from_requested_python(python, cache)?
     } else if system {
         PythonEnvironment::from_default_python(cache)?

crates/uv/src/commands/pip_show.rs

@@ -22,6 +22,7 @@ pub(crate) fn pip_show(
     strict: bool,
     python: Option<&str>,
     system: bool,
+    user: bool,
     cache: &Cache,
     printer: Printer,
 ) -> Result<ExitStatus> {
@@ -39,7 +40,9 @@ pub(crate) fn pip_show(
     }
 
     // Detect the current Python interpreter.
-    let venv = if let Some(python) = python {
+    let venv = if user {
+        PythonEnvironment::from_user_scheme(python, cache)?
+    } else if let Some(python) = python {
         PythonEnvironment::from_requested_python(python, cache)?
     } else if system {
         PythonEnvironment::from_default_python(cache)?