Skip to content

Commit

Permalink
feat: create virtual environment (#82)
Browse files Browse the repository at this point in the history
This is the first iteration in rip to add support for #78.

Small test was added to see if the wheel install works, this will mainly
be used for #32 but we can also integrate this into the `rip` binary to
be able to add the install path there as well. @baszalmstra let me know
if you want me to add this in a separate PR once this is merged.
  • Loading branch information
tdejager authored Nov 20, 2023
1 parent 66cd42a commit e4c1a49
Show file tree
Hide file tree
Showing 8 changed files with 257 additions and 38 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/target
.idea/
*.sqlite3
**/__pycache__/**
4 changes: 2 additions & 2 deletions crates/rattler_installs_packages/src/distribution_finder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ pub fn find_distributions_in_venv(
paths: &InstallPaths,
) -> Result<Vec<Distribution>, FindDistributionError> {
// We will look for distributions in the purelib/platlib directories
let locations = [paths.mapping.get("purelib"), paths.mapping.get("platlib")]
let locations = [paths.purelib(), paths.platlib()]
.into_iter()
.filter_map(|p| Some(root.join(p?)))
.map(|p| root.join(p))
.unique()
.filter(|p| p.is_dir())
.collect_vec();
Expand Down
2 changes: 2 additions & 0 deletions crates/rattler_installs_packages/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ mod wheel;
mod sdist;
mod system_python;

mod venv;

#[cfg(feature = "resolvo")]
pub use resolve::{resolve, PinnedPackage, ResolveOptions, SDistResolution};

Expand Down
12 changes: 9 additions & 3 deletions crates/rattler_installs_packages/src/system_python.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use itertools::Itertools;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use thiserror::Error;

Expand Down Expand Up @@ -66,6 +66,7 @@ impl PythonInterpreterVersion {
// Split the version into strings separated by '.' and parse them
let parts = version_str
.split('.')
.map(str::trim)
.map(FromStr::from_str)
.collect::<Result<Vec<_>, _>>()
.map_err(|_| InvalidVersion(version_str.to_owned()))?;
Expand All @@ -88,7 +89,12 @@ impl PythonInterpreterVersion {

/// Get the python version from the system interpreter
pub fn from_system() -> Result<Self, ParsePythonInterpreterVersionError> {
let output = std::process::Command::new(system_python_executable()?)
Self::from_path(&system_python_executable()?)
}

/// Get the python version a path to the python executable
pub fn from_path(path: &Path) -> Result<Self, ParsePythonInterpreterVersionError> {
let output = std::process::Command::new(path)
.arg("--version")
.output()
.map_err(|_| FindPythonError::NotFound)?;
Expand All @@ -103,7 +109,7 @@ mod tests {

#[test]
pub fn parse_python_version() {
let version = PythonInterpreterVersion::from_python_output("Python 3.8.5").unwrap();
let version = PythonInterpreterVersion::from_python_output("Python 3.8.5\n").unwrap();
assert_eq!(version.major, 3);
assert_eq!(version.minor, 8);
assert_eq!(version.patch, 5);
Expand Down
169 changes: 169 additions & 0 deletions crates/rattler_installs_packages/src/venv.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
//! Module that helps with allowing in the creation of python virtual environments.
//! Now just use the python venv command to create the virtual environment.
//! Later on we can look into actually creating the environment by linking to the python library,
//! and creating the necessary files. See: [VEnv](https://packaging.python.org/en/latest/specifications/virtual-environments/#declaring-installation-environments-as-python-virtual-environments)
#![allow(dead_code)]
use crate::system_python::{
system_python_executable, FindPythonError, ParsePythonInterpreterVersionError,
PythonInterpreterVersion,
};
use crate::wheel::UnpackError;
use crate::{InstallPaths, UnpackWheelOptions, Wheel};
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use thiserror::Error;

/// Specifies where to find the python executable
pub enum PythonLocation {
/// Use system interpreter
System,
// Use custom interpreter
Custom(PathBuf),
}

impl PythonLocation {
/// Location of python executable
pub fn executable(&self) -> Result<PathBuf, FindPythonError> {
match self {
PythonLocation::System => system_python_executable(),
PythonLocation::Custom(path) => Ok(path.clone()),
}
}
}

#[derive(Error, Debug)]
pub enum VEnvError {
#[error(transparent)]
FindPythonError(#[from] FindPythonError),
#[error(transparent)]
ParsePythonInterpreterVersionError(#[from] ParsePythonInterpreterVersionError),
#[error("failed to run 'python -m venv': `{0}`")]
FailedToRun(String),
#[error(transparent)]
FailedToCreate(#[from] std::io::Error),
}

/// Represents a virtual environment in which wheels can be installed
pub struct VEnv {
/// Location of the virtual environment
location: PathBuf,
/// Install paths for this virtual environment
install_paths: InstallPaths,
}

impl VEnv {
fn new(location: PathBuf, install_paths: InstallPaths) -> Self {
Self {
location,
install_paths,
}
}

/// Install a wheel into this virtual environment
pub fn install_wheel(
&self,
wheel: &Wheel,
options: &UnpackWheelOptions,
) -> Result<(), UnpackError> {
wheel.unpack(&self.location, &self.install_paths, options)
}

/// Execute python script in venv
pub fn execute_script(&self, source: &Path) -> std::io::Result<Output> {
let mut cmd = Command::new(self.python_executable());
cmd.arg(source);
cmd.output()
}

/// Execute python command in venv
pub fn execute_command<S: AsRef<str>>(&self, command: S) -> std::io::Result<Output> {
let mut cmd = Command::new(self.python_executable());
cmd.arg("-c");
cmd.arg(command.as_ref());
cmd.output()
}

/// Path to python executable in venv
pub fn python_executable(&self) -> PathBuf {
let executable = if self.install_paths.is_windows() {
"python.exe"
} else {
"python"
};
self.location
.join(self.install_paths.scripts())
.join(executable)
}

/// Create a virtual environment at specified directory
/// for the platform we are running on
pub fn create(venv_dir: &Path, python: PythonLocation) -> Result<VEnv, VEnvError> {
Self::create_custom(venv_dir, python, cfg!(windows))
}

/// Create a virtual environment at specified directory
/// allows specifying if this is a windows venv
pub fn create_custom(
venv_dir: &Path,
python: PythonLocation,
windows: bool,
) -> Result<VEnv, VEnvError> {
// Find python executable
let python = python.executable()?;

// Execute command
// Don't need pip for our use-case
let output = Command::new(&python)
.arg("-m")
.arg("venv")
.arg(venv_dir)
.arg("--without-pip")
.output()?;

// Parse output
if !output.status.success() {
let stdout = String::from_utf8_lossy(&output.stderr);
return Err(VEnvError::FailedToRun(stdout.to_string()));
}

let version = PythonInterpreterVersion::from_path(&python)?;
let install_paths = InstallPaths::for_venv(version, windows);
Ok(VEnv::new(venv_dir.to_path_buf(), install_paths))
}
}

#[cfg(test)]
mod tests {
use super::VEnv;
use crate::venv::PythonLocation;
use std::path::Path;

#[test]
pub fn venv_creation() {
let venv_dir = tempfile::tempdir().unwrap();
let venv = VEnv::create(venv_dir.path(), PythonLocation::System).unwrap();
// Does python exist
assert!(venv.python_executable().is_file());

// Install wheel
let wheel = crate::wheel::Wheel::from_path(
&Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../test-data/wheels/wordle_python-2.3.32-py3-none-any.whl"),
)
.unwrap();
venv.install_wheel(&wheel, &Default::default()).unwrap();

// See if it worked
let output = venv
.execute_script(
&Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../test-data/scripts/test_wordle.py"),
)
.unwrap();

assert_eq!(
String::from_utf8(output.stdout).unwrap().trim(),
"('A d i E u ', False)"
);
}
}
100 changes: 67 additions & 33 deletions crates/rattler_installs_packages/src/wheel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use pep440_rs::Version;
use rattler_digest::Sha256;
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
collections::HashSet,
ffi::OsStr,
fs,
fs::File,
Expand Down Expand Up @@ -411,20 +411,24 @@ pub fn read_entry_to_end<R: ReadAndSeek>(
Ok(bytes)
}

/// A dictionary of installation categories to where they should be stored relative to the
/// A struct of installation categories to where they should be stored relative to the
/// installation destination.
#[derive(Debug, Clone)]
pub struct InstallPaths {
/// Mapping from category to installation path
pub mapping: HashMap<String, PathBuf>,
purelib: PathBuf,
platlib: PathBuf,
scripts: PathBuf,
data: PathBuf,
windows: bool,
}

impl InstallPaths {
/// Populates mappings of installation targets for a virtualenv layout. The mapping depends on
/// the python version and whether or not the installation targets windows. Specifucally on
/// the python version and whether or not the installation targets windows. Specifically on
/// windows some of the paths are different. :shrug:
pub fn for_venv<V: Into<PythonInterpreterVersion>>(version: V, windows: bool) -> Self {
let version = version.into();

let site_packages = if windows {
Path::new("Lib").join("site-packages")
} else {
Expand All @@ -433,29 +437,67 @@ impl InstallPaths {
version.major, version.minor
))
};
let scripts = if windows {
PathBuf::from("Scripts")
} else {
PathBuf::from("bin")
};

// Data should just be the root of the venv
let data = PathBuf::from("");

// purelib and platlib locations are not relevant when using venvs
// https://stackoverflow.com/a/27882460/3549270
Self {
mapping: HashMap::from([
(
String::from("scripts"),
if windows {
PathBuf::from("Scripts")
} else {
PathBuf::from("bin")
},
),
// purelib and platlib locations are not relevant when using venvs
// https://stackoverflow.com/a/27882460/3549270
(String::from("purelib"), site_packages.clone()),
(String::from("platlib"), site_packages),
// Move the content of the folder to the root of the venv
(String::from("data"), PathBuf::from("")),
]),
purelib: site_packages.clone(),
platlib: site_packages,
scripts,
data,
windows,
}
}

/// Determines whether this is a windows InstallPath
pub fn is_windows(&self) -> bool {
self.windows
}

/// Returns the site-packages location. This is done by searching for the purelib location.
pub fn site_packages(&self) -> Option<&PathBuf> {
self.mapping.get("purelib")
pub fn site_packages(&self) -> &Path {
&self.purelib
}

/// Reference to pure python library location.
pub fn purelib(&self) -> &Path {
&self.purelib
}

/// Reference to platform specific library location.
pub fn platlib(&self) -> &Path {
&self.platlib
}

/// Returns the binaries location.
pub fn scripts(&self) -> &Path {
&self.scripts
}

/// Returns the location of the data directory
pub fn data(&self) -> &Path {
&self.data
}

/// Matches the different categories to their install paths.
pub fn match_category<S: AsRef<str>>(&self, category: S) -> Option<&Path> {
let category = category.as_ref();
match category {
"purelib" => Some(self.purelib()),
"platlib" => Some(self.platlib()),
"scripts" => Some(self.scripts()),
"data" => Some(self.data()),
// TODO: support headers?
&_ => None,
}
}
}

Expand Down Expand Up @@ -527,13 +569,7 @@ impl Wheel {
root_is_purelib: vitals.root_is_purelib,
paths,
};
let site_packages = dest.join(
paths
.mapping
.get("purelib")
.ok_or_else(|| UnpackError::MissingInstallPath(String::from("purelib")))?
.as_path(),
);
let site_packages = dest.join(paths.site_packages());

let mut archive = self.archive.lock();

Expand Down Expand Up @@ -767,7 +803,7 @@ impl<'a> WheelPathTransformer<'a> {
(category, path)
};

match self.paths.mapping.get(category.as_ref()) {
match self.paths.match_category(category.as_ref()) {
Some(basepath) => Ok(Some((basepath.join(rest_of_path), category == "scripts"))),
None => Err(UnpackError::UnsupportedDataDirectory(category.into_owned())),
}
Expand Down Expand Up @@ -852,7 +888,6 @@ mod test {
let record_path = unpacked
.install_paths
.site_packages()
.unwrap()
.join(format!("{}/RECORD", unpacked.vitals.dist_info,));
let record_content = std::fs::read_to_string(&unpacked.tmpdir.path().join(&record_path))
.unwrap_or_else(|_| panic!("failed to read RECORD from {}", record_path.display()));
Expand All @@ -870,7 +905,6 @@ mod test {
let relative_path = unpacked
.install_paths
.site_packages()
.unwrap()
.join(format!("{}/INSTALLER", unpacked.vitals.dist_info));
let installer_content =
std::fs::read_to_string(unpacked.tmpdir.path().join(relative_path)).unwrap();
Expand Down
Loading

0 comments on commit e4c1a49

Please sign in to comment.