Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Export conda explicit specification file from project #1873

Merged
merged 8 commits into from
Sep 10, 2024
27 changes: 27 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -1178,6 +1178,33 @@ List the environments in the manifest file.
pixi project environment list
```

### `project export conda_explicit_spec`

Render a platform-specific conda [explicit specification file](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments)
for an environment. The file can be then used to create a conda environment using conda/mamba:

```shell
mamba create --name <env> --file <explicit spec file>
```

As the explicit specification file format does not support pypi-dependencies, use the `--ignore-pypi-errors` option to ignore those dependencies.

##### Arguments

1. `<OUTPUT_DIR>`: Output directory for rendered explicit environment spec files.

##### Options

- `--environment <ENVIRONMENT> (-e)`: Environment to render. Can be repeated for multiple envs. Defaults to all environments.
- `--platform <PLATFORM> (-p)`: The platform to render. Can be repeated for multiple platforms. Defaults to all platforms available for selected environments.
- `--ignore-pypi-errors`: PyPI dependencies are not supported in the conda explicit spec file. This flag allows creating the spec file even if PyPI dependencies are present.

```sh
pixi project export conda_explicit_spec output
pixi project export conda_explicit_spec -e default -e test -p linux-64 output
```


### `project platform add`

Adds a platform(s) to the manifest file and updates the lock file.
Expand Down
236 changes: 236 additions & 0 deletions src/cli/project/export/conda_explicit_spec.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};

use clap::Parser;
use miette::{Context, IntoDiagnostic};

use crate::cli::cli_config::PrefixUpdateConfig;
use crate::cli::LockFileUsageArgs;
use crate::lock_file::UpdateLockFileOptions;
use crate::Project;
use rattler_conda_types::{ExplicitEnvironmentEntry, ExplicitEnvironmentSpec, Platform};
use rattler_lock::{CondaPackage, Environment, Package};

#[derive(Debug, Parser)]
#[clap(arg_required_else_help = false)]
pub struct Args {
/// Output directory for rendered explicit environment spec files
pub output_dir: PathBuf,

/// Environment to render. Can be repeated for multiple envs. Defaults to all environments
#[arg(short, long)]
pub environment: Option<Vec<String>>,

/// The platform to render. Can be repeated for multiple platforms.
/// Defaults to all platforms available for selected environments.
#[arg(short, long)]
pub platform: Option<Vec<Platform>>,

/// PyPI dependencies are not supported in the conda explicit spec file.
/// This flag allows creating the spec file even if PyPI dependencies are present.
#[arg(long, default_value = "false")]
pub ignore_pypi_errors: bool,
baszalmstra marked this conversation as resolved.
Show resolved Hide resolved

#[clap(flatten)]
pub lock_file_usage: LockFileUsageArgs,

#[clap(flatten)]
pub prefix_update_config: PrefixUpdateConfig,
}

fn build_explicit_spec<'a>(
platform: &Platform,
conda_packages: impl IntoIterator<Item = &'a CondaPackage>,
) -> miette::Result<ExplicitEnvironmentSpec> {
let mut packages = Vec::new();

for cp in conda_packages {
let prec = cp.package_record();
let mut url = cp.url().to_owned();
let hash = prec.md5.ok_or(miette::miette!(
"Package {} does not contain an md5 hash",
prec.name.as_normalized()
))?;

url.set_fragment(Some(&format!("{:x}", hash)));

packages.push(ExplicitEnvironmentEntry {
url: url.to_owned(),
});
}

Ok(ExplicitEnvironmentSpec {
platform: Some(*platform),
packages,
})
}

fn render_explicit_spec(
target: impl AsRef<Path>,
exp_env_spec: &ExplicitEnvironmentSpec,
) -> miette::Result<()> {
if exp_env_spec.packages.is_empty() {
return Ok(());
}

let target = target.as_ref();

let mut environment = String::new();
environment.push_str("# Generated by `pixi project export`\n");
environment.push_str(exp_env_spec.to_spec_string().as_str());

fs::write(target, environment)
.into_diagnostic()
.with_context(|| format!("failed to write environment file: {}", target.display()))?;

Ok(())
}

fn render_env_platform(
output_dir: &Path,
env_name: &str,
env: &Environment,
platform: &Platform,
ignore_pypi_errors: bool,
) -> miette::Result<()> {
let packages = env.packages(*platform).ok_or(miette::miette!(
"platform '{platform}' not found for env {}",
env_name,
))?;

let mut conda_packages_from_lockfile: Vec<CondaPackage> = Vec::new();

for package in packages {
match package {
Package::Conda(p) => conda_packages_from_lockfile.push(p),
Package::Pypi(pyp) => {
if ignore_pypi_errors {
tracing::warn!(
"ignoring PyPI package {} since PyPI packages are not supported",
pyp.data().package.name
);
} else {
miette::bail!(
"PyPI packages are not supported. Specify `--ignore-pypi-errors` to ignore this error \
or `--write-pypi-requirements` to write pypi requirements to a separate requirements.txt file"
);
}
}
}
}

let ees = build_explicit_spec(platform, &conda_packages_from_lockfile)?;

tracing::info!("Creating conda explicit spec for env: {env_name} platform: {platform}");
let target = output_dir
.join(format!("{}_{}_conda_spec.txt", env_name, platform))
.into_os_string();

render_explicit_spec(target, &ees)?;

Ok(())
}

pub async fn execute(project: Project, args: Args) -> miette::Result<()> {
let lockfile = project
.update_lock_file(UpdateLockFileOptions {
lock_file_usage: args.prefix_update_config.lock_file_usage(),
no_install: args.prefix_update_config.no_install,
..UpdateLockFileOptions::default()
})
.await?
.lock_file;

let mut environments = Vec::new();
if let Some(env_names) = args.environment {
for env_name in &env_names {
environments.push((
env_name.to_string(),
lockfile
.environment(env_name)
.ok_or(miette::miette!("unknown environment {}", env_name))?,
));
}
} else {
for (env_name, env) in lockfile.environments() {
environments.push((env_name.to_string(), env));
}
};

let mut env_platform = Vec::new();

for (env_name, env) in environments {
let available_platforms: HashSet<Platform> = HashSet::from_iter(env.platforms());

if let Some(ref platforms) = args.platform {
for plat in platforms {
if available_platforms.contains(plat) {
env_platform.push((env_name.clone(), env.clone(), *plat));
} else {
tracing::warn!(
"Platform {} not available for environment {}. Skipping...",
plat,
env_name,
);
}
}
} else {
for plat in available_platforms {
env_platform.push((env_name.clone(), env.clone(), plat));
}
}
}

fs::create_dir_all(&args.output_dir).ok();

for (env_name, env, plat) in env_platform {
render_env_platform(
&args.output_dir,
&env_name,
&env,
&plat,
args.ignore_pypi_errors,
)?;
}

Ok(())
}

#[cfg(test)]
mod tests {
use std::path::Path;

use super::*;
use rattler_lock::LockFile;
use tempfile::tempdir;

#[test]
fn test_render_conda_explicit_spec() {
let path = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("src/cli/project/export/test-data/testenv/pixi.lock");
let lockfile = LockFile::from_path(&path).unwrap();

let output_dir = tempdir().unwrap();

for (env_name, env) in lockfile.environments() {
for platform in env.platforms() {
// example contains pypi dependencies so should fail if `ignore_pypi_errors` is
// false.
assert!(
render_env_platform(output_dir.path(), env_name, &env, &platform, false)
.is_err()
);
render_env_platform(output_dir.path(), env_name, &env, &platform, true).unwrap();

let file_path = output_dir
.path()
.join(format!("{}_{}_conda_spec.txt", env_name, platform));
insta::assert_snapshot!(
format!("test_render_conda_explicit_spec_{}_{}", env_name, platform),
fs::read_to_string(file_path).unwrap()
);
}
}
}
}
31 changes: 31 additions & 0 deletions src/cli/project/export/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
use std::path::PathBuf;
pub mod conda_explicit_spec;

use crate::Project;
use clap::Parser;

/// Commands to export projects to other formats
#[derive(Parser, Debug)]
pub struct Args {
/// The path to 'pixi.toml' or 'pyproject.toml'
#[clap(long, global = true)]
pub manifest_path: Option<PathBuf>,

#[clap(subcommand)]
pub command: Command,
}

#[derive(Parser, Debug)]
pub enum Command {
/// Export project environment to a conda explicit specification file
#[clap(visible_alias = "ces")]
CondaExplicitSpec(conda_explicit_spec::Args),
}

pub async fn execute(args: Args) -> miette::Result<()> {
let project = Project::load_or_else_discover(args.manifest_path.as_deref())?;
match args.command {
Command::CondaExplicitSpec(args) => conda_explicit_spec::execute(project, args).await?,
};
Ok(())
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
---
source: src/cli/project/export/conda_explicit_spec.rs
expression: "fs::read_to_string(file_path).unwrap()"
---
# Generated by `pixi project export`
# platform: linux-64
@EXPLICIT
https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d
https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py312h2ec8cdc_2.conda#b0b867af6fc74b2a0aa206da29c0f3cf
https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553
https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.8.30-hbcca054_0.conda#c27d1c142233b5bc9ca570c6e2e0c244
https://conda.anaconda.org/conda-forge/noarch/certifi-2024.8.30-pyhd8ed1ab_0.conda#12f7d00853807b0531775e9be891cb11
https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.0-py312h06ac9bb_1.conda#db9bdbaee0f524ead0471689f002781e
https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.3.2-pyhd8ed1ab_0.conda#7f4a9e3fcff3f6356ae99244a014da6a
https://conda.anaconda.org/conda-forge/noarch/h2-4.1.0-pyhd8ed1ab_0.tar.bz2#b748fbf7060927a6e82df7cb5ee8f097
https://conda.anaconda.org/conda-forge/noarch/hpack-4.0.0-pyh9f0ad1d_0.tar.bz2#914d6646c4dbb1fd3ff539830a12fd71
https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.0.1-pyhd8ed1ab_0.tar.bz2#9f765cbfab6870c8435b9eefecd7a1f4
https://conda.anaconda.org/conda-forge/noarch/idna-3.8-pyhd8ed1ab_0.conda#99e164522f6bdf23c177c8d9ae63f975
https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-hf3520f5_7.conda#b80f2f396ca2c28b8c14c437a4ed1e74
https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda#e7ba12deb7020dd080c6c70e7b6f6a3d
https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3
https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.1.0-h77fa898_1.conda#002ef4463dd1e2b44a94a4ace468f5d2
https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.1.0-h69a702a_1.conda#1efc0ad219877a73ef977af7dbb51f17
https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.1.0-h77fa898_1.conda#23c255b008c4f2ae008f81edcabaca89
https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7
https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.46.1-hadc24fc_0.conda#36f79405ab16bf271edb55b213836dac
https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.1.0-hc0a3c3a_1.conda#9dbb9699ea467983ba8a4ba89b08b066
https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-14.1.0-h4852527_1.conda#bd2598399a70bb86d8218e95548d735e
https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b
https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc
https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-h4ab18f5_1.conda#57d7dc60e9325e3de37ff8dffd18e814
https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-he02047a_1.conda#70caf8bb6cf39a0b6b7efc885f51c0fe
https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.2-hb9d3cd8_0.conda#4d638782050ab6faa27275bed57e9b4e
https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyhd8ed1ab_0.conda#844d9eb3b43095b031874477f7d70088
https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025
https://conda.anaconda.org/conda-forge/linux-64/python-3.12.5-h2ad013b_0_cpython.conda#9c56c4df45f6571b13111d8df2448692
https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.12-5_cp312.conda#0424ae29b104430108f5218a66db7260
https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4
https://conda.anaconda.org/conda-forge/noarch/requests-2.32.3-pyhd8ed1ab_0.conda#5ede4753180c7a550a443c430dc8ab52
https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda#d453b98d9c83e71da0741bb0ff4d76bc
https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h8827d51_1.conda#8bfdead4e0fff0383ae4c9c50d0531bd
https://conda.anaconda.org/conda-forge/noarch/urllib3-2.2.2-pyhd8ed1ab_1.conda#e804c43f58255e977093a2298e442bb8
https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0
https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py312hef9b889_1.conda#8b7069e9792ee4e5b4919a7a306d2e67
https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.6-ha6fb4c9_0.conda#4d056880988120e29d75bfff282e0f45
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
---
source: src/cli/project/export/conda_explicit_spec.rs
expression: "fs::read_to_string(file_path).unwrap()"
---
# Generated by `pixi project export`
# platform: osx-64
@EXPLICIT
https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda#7ed4301d437b59045be7e051a0308211
https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.8.30-h8857fd0_0.conda#b7e5424e7f06547a903d28e4651dbb21
https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.6.2-h73e2aa4_0.conda#3d1d51c8f716d97c864d12f7af329526
https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2#ccb34fb14960ad8b125962d3d79b31a9
https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.1-h4b8f8c9_0.conda#84de0078b58f899fc164303b0603ff0e
https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda#b7575b5aa92108dcc9aaab0f05f2dbce
https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-hf036a51_1.conda#e102bbf8a6ceeaf429deab8032fc8977
https://conda.anaconda.org/conda-forge/osx-64/openssl-3.3.2-hd23fc13_0.conda#2ff47134c8e292868a4609519b1ea3b6
https://conda.anaconda.org/conda-forge/osx-64/python-3.12.5-h37a9e06_0_cpython.conda#517cb4e16466f8d96ba2a72897d14c48
https://conda.anaconda.org/conda-forge/osx-64/python_abi-3.12-5_cp312.conda#c34dd4920e0addf7cfcc725809f25d8e
https://conda.anaconda.org/conda-forge/osx-64/pyyaml-6.0.2-py312hb553811_1.conda#66514594817d51c78db7109a23ad322f
https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda#f17f77f2acf4d344734bda76829ce14e
https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda#bf830ba5afc507c6232d4ef0fb1a882d
https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h8827d51_1.conda#8bfdead4e0fff0383ae4c9c50d0531bd
https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2#a72f9d4ea13d55d745ff1ed594747f10
https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h0d85af4_2.tar.bz2#d7e08fcf8259d742156188e8762b4d20
Loading
Loading