diff --git a/libmamba/include/mamba/core/activation.hpp b/libmamba/include/mamba/core/activation.hpp index fc6569b4bd..e2ad98ab20 100644 --- a/libmamba/include/mamba/core/activation.hpp +++ b/libmamba/include/mamba/core/activation.hpp @@ -258,8 +258,6 @@ namespace mamba fs::u8path hook_source_path() override; }; - std::vector get_path_dirs(const fs::u8path& prefix); - } // namespace mamba #endif diff --git a/libmamba/include/mamba/core/prefix_data.hpp b/libmamba/include/mamba/core/prefix_data.hpp index ce9919351d..4ada7c30bb 100644 --- a/libmamba/include/mamba/core/prefix_data.hpp +++ b/libmamba/include/mamba/core/prefix_data.hpp @@ -44,6 +44,8 @@ namespace mamba PrefixData(const fs::u8path& prefix_path, ChannelContext& channel_context); + void load_site_packages(); + History m_history; package_map m_package_records; fs::u8path m_prefix_path; diff --git a/libmamba/include/mamba/specs/package_info.hpp b/libmamba/include/mamba/specs/package_info.hpp index 9ee9fd783b..d1afbe4bc4 100644 --- a/libmamba/include/mamba/specs/package_info.hpp +++ b/libmamba/include/mamba/specs/package_info.hpp @@ -68,6 +68,7 @@ namespace mamba::specs PackageInfo() = default; explicit PackageInfo(std::string name); PackageInfo(std::string name, std::string version, std::string build_string, std::size_t build_number); + PackageInfo(std::string name, std::string version, std::string build_string, std::string channel); [[nodiscard]] auto json_signable() const -> nlohmann::json; [[nodiscard]] auto str() const -> std::string; diff --git a/libmamba/include/mamba/util/environment.hpp b/libmamba/include/mamba/util/environment.hpp index 483af92a79..dc54646060 100644 --- a/libmamba/include/mamba/util/environment.hpp +++ b/libmamba/include/mamba/util/environment.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "mamba/fs/filesystem.hpp" #include "mamba/util/build.hpp" @@ -93,6 +94,11 @@ namespace mamba::util */ [[nodiscard]] constexpr auto pathsep() -> char; + /** + * Return directories of the given prefix path. + */ + [[nodiscard]] auto get_path_dirs(const fs::u8path& prefix) -> std::vector; + /** * Return the full path of a program from its name. */ diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 083f42dbc0..70e921ad5e 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -10,7 +10,6 @@ #include "mamba/api/channel_loader.hpp" #include "mamba/api/configuration.hpp" #include "mamba/api/install.hpp" -#include "mamba/core/activation.hpp" #include "mamba/core/channel_context.hpp" #include "mamba/core/context.hpp" #include "mamba/core/env_lockfile.hpp" @@ -24,6 +23,7 @@ #include "mamba/download/downloader.hpp" #include "mamba/fs/filesystem.hpp" #include "mamba/solver/libsolv/solver.hpp" +#include "mamba/util/environment.hpp" #include "mamba/util/path_manip.hpp" #include "mamba/util/string.hpp" diff --git a/libmamba/src/api/list.cpp b/libmamba/src/api/list.cpp index 470555d9e5..ca5910f1d2 100644 --- a/libmamba/src/api/list.cpp +++ b/libmamba/src/api/list.cpp @@ -89,18 +89,31 @@ namespace mamba { auto channels = channel_context.make_channel(pkg_info.package_url); assert(channels.size() == 1); // A URL can only resolve to one channel - obj["base_url"] = strip_from_filename_and_platform( - channels.front().url().str(specs::CondaURL::Credentials::Remove), - pkg_info.filename, - pkg_info.platform - ); + + if (pkg_info.package_url.empty() && (pkg_info.channel == "pypi")) + { + // TODO Need to correctly set `platform`, which is empty in PyPI case + // Note that this is only a problem when using `--json` + // (otherwise, the missing info is not needed/used) + // cf. `formatted_pkgs` below + obj["base_url"] = "https://pypi.org/"; + obj["channel"] = pkg_info.channel; + } + else + { + obj["base_url"] = strip_from_filename_and_platform( + channels.front().url().str(specs::CondaURL::Credentials::Remove), + pkg_info.filename, + pkg_info.platform + ); + obj["channel"] = strip_from_filename_and_platform( + channels.front().display_name(), + pkg_info.filename, + pkg_info.platform + ); + } obj["build_number"] = pkg_info.build_number; obj["build_string"] = pkg_info.build_string; - obj["channel"] = strip_from_filename_and_platform( - channels.front().display_name(), - pkg_info.filename, - pkg_info.platform - ); obj["dist_name"] = pkg_info.str(); obj["name"] = pkg_info.name; obj["platform"] = pkg_info.platform; @@ -132,6 +145,10 @@ namespace mamba { formatted_pkgs.channel = ""; } + else if (package.second.channel == "pypi") + { + formatted_pkgs.channel = package.second.channel; + } else { auto channels = channel_context.make_channel(package.second.channel); diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index 8e3fc2fe3f..13642f08e7 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -13,7 +13,6 @@ // TODO includes to be removed after moving some functions/structs around #include "mamba/api/install.hpp" // other_pkg_mgr_spec -#include "mamba/core/activation.hpp" // get_path_dirs #include "mamba/core/context.hpp" #include "mamba/core/util.hpp" #include "mamba/fs/filesystem.hpp" @@ -33,7 +32,7 @@ namespace mamba ) { const auto get_python_path = [&] - { return util::which_in("python", get_path_dirs(target_prefix)).string(); }; + { return util::which_in("python", util::get_path_dirs(target_prefix)).string(); }; command_args cmd = { get_python_path(), "-m", "pip", "install" }; command_args cmd_extension = { "-r", spec_file, "--no-input", "--quiet" }; diff --git a/libmamba/src/core/activation.cpp b/libmamba/src/core/activation.cpp index f18485bab3..46169e7fbb 100644 --- a/libmamba/src/core/activation.cpp +++ b/libmamba/src/core/activation.cpp @@ -208,23 +208,6 @@ namespace mamba } } - std::vector get_path_dirs(const fs::u8path& prefix) - { - if (util::on_win) - { - return { prefix, - prefix / "Library" / "mingw-w64" / "bin", - prefix / "Library" / "usr" / "bin", - prefix / "Library" / "bin", - prefix / "Scripts", - prefix / "bin" }; - } - else - { - return { prefix / "bin" }; - } - } - std::vector Activator::get_PATH() { std::vector path; @@ -271,7 +254,7 @@ namespace mamba // TODO check if path_conversion does something useful here. // path_list[0:0] = list(self.path_conversion(self._get_path_dirs(prefix))) - std::vector final_path = get_path_dirs(prefix); + std::vector final_path = util::get_path_dirs(prefix); final_path.insert(final_path.end(), path_list.begin(), path_list.end()); final_path.erase(std::unique(final_path.begin(), final_path.end()), final_path.end()); std::string result = util::join(util::pathsep(), final_path).string(); @@ -285,7 +268,7 @@ namespace mamba std::vector current_path = get_PATH(); assert(!old_prefix.empty()); - std::vector old_prefix_dirs = get_path_dirs(old_prefix); + std::vector old_prefix_dirs = util::get_path_dirs(old_prefix); // remove all old paths std::vector cleaned_path; @@ -312,7 +295,7 @@ namespace mamba std::vector final_path; if (!new_prefix.empty()) { - final_path = get_path_dirs(new_prefix); + final_path = util::get_path_dirs(new_prefix); final_path.insert(final_path.end(), current_path.begin(), current_path.end()); // remove duplicates diff --git a/libmamba/src/core/prefix_data.cpp b/libmamba/src/core/prefix_data.cpp index 9c1b8c7c9b..db6647fd66 100644 --- a/libmamba/src/core/prefix_data.cpp +++ b/libmamba/src/core/prefix_data.cpp @@ -9,11 +9,14 @@ #include #include +#include + #include "mamba/core/channel_context.hpp" #include "mamba/core/output.hpp" #include "mamba/core/prefix_data.hpp" #include "mamba/core/util.hpp" #include "mamba/specs/conda_url.hpp" +#include "mamba/util/environment.hpp" #include "mamba/util/graph.hpp" #include "mamba/util/string.hpp" @@ -56,6 +59,8 @@ namespace mamba } } } + // Load packages installed with pip + load_site_packages(); } void PrefixData::add_packages(const std::vector& packages) @@ -166,10 +171,64 @@ namespace mamba auto channels = m_channel_context.make_channel(prec.channel); // If someone wrote multichannel names in repodata_record, we don't know which one is the - // correct URL. This is must never happen! + // correct URL. This must never happen! assert(channels.size() == 1); using Credentials = specs::CondaURL::Credentials; prec.channel = channels.front().platform_url(prec.platform).str(Credentials::Remove); m_package_records.insert({ prec.name, std::move(prec) }); } + + // Load python packages installed with pip in the site-packages of the prefix. + void PrefixData::load_site_packages() + { + LOG_INFO << "Loading site packages"; + + // Look for `pip` package and return if it doesn't exist + auto python_pkg_record = m_package_records.find("pip"); + if (python_pkg_record == m_package_records.end()) + { + LOG_DEBUG << "`pip` not found"; + return; + } + + // Run `pip freeze` + std::string out, err; + + const auto get_python_path = [&] + { return util::which_in("python", util::get_path_dirs(m_prefix_path)).string(); }; + + const auto args = std::array{ get_python_path(), + "-m", + "pip", + "freeze", + "--local" }; + auto [status, ec] = reproc::run( + args, + reproc::options{}, + reproc::sink::string(out), + reproc::sink::string(err) + ); + if (ec) + { + throw std::runtime_error(ec.message()); + } + + // Nothing installed with `pip` + if (out.empty()) + { + LOG_DEBUG << "Nothing installed with `pip`"; + return; + } + + auto pkgs_info_list = mamba::util::split(mamba::util::strip(out), "\n"); + for (auto& pkg_info_line : pkgs_info_list) + { + if (pkg_info_line.find("==") != std::string::npos) + { + auto pkg_info = mamba::util::split(mamba::util::strip(pkg_info_line), "=="); + auto prec = specs::PackageInfo(pkg_info[0], pkg_info[1], "pypi_0", "pypi"); + m_package_records.insert({ prec.name, std::move(prec) }); + } + } + } } // namespace mamba diff --git a/libmamba/src/specs/package_info.cpp b/libmamba/src/specs/package_info.cpp index dd4003bc33..527956bd1c 100644 --- a/libmamba/src/specs/package_info.cpp +++ b/libmamba/src/specs/package_info.cpp @@ -156,6 +156,14 @@ namespace mamba::specs { } + PackageInfo::PackageInfo(std::string n, std::string v, std::string b, std::string c) + : name(std::move(n)) + , version(std::move(v)) + , build_string(std::move(b)) + , channel(std::move(c)) + { + } + namespace { template diff --git a/libmamba/src/util/environment.cpp b/libmamba/src/util/environment.cpp index c6e2daa29b..f5046eade4 100644 --- a/libmamba/src/util/environment.cpp +++ b/libmamba/src/util/environment.cpp @@ -408,6 +408,23 @@ namespace mamba::util } } + auto get_path_dirs(const fs::u8path& prefix) -> std::vector + { + if (on_win) + { + return { prefix, + prefix / "Library" / "mingw-w64" / "bin", + prefix / "Library" / "usr" / "bin", + prefix / "Library" / "bin", + prefix / "Scripts", + prefix / "bin" }; + } + else + { + return { prefix / "bin" }; + } + } + auto which(std::string_view exe) -> fs::u8path { if (auto paths = get_env("PATH")) diff --git a/micromamba/tests/test_list.py b/micromamba/tests/test_list.py index 1865ed7dab..5a1f587755 100644 --- a/micromamba/tests/test_list.py +++ b/micromamba/tests/test_list.py @@ -40,6 +40,39 @@ def test_list_name(tmp_home, tmp_root_prefix, tmp_xtensor_env, quiet_flag): assert full_names == ["xtensor"] +env_yaml_content_to_install_numpy_with_pip = """ +channels: +- conda-forge +dependencies: +- pip +- pip: + - numpy==1.26.4 +""" + + +@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True) +def test_list_with_pip(tmp_home, tmp_root_prefix, tmp_path): + env_name = "env-list_with_pip" + tmp_root_prefix / "envs" / env_name + + env_file_yml = tmp_path / "test_env_yaml_content_to_install_numpy_with_pip.yaml" + env_file_yml.write_text(env_yaml_content_to_install_numpy_with_pip) + + helpers.create("-n", env_name, "python=3.12", "--json", no_dry_run=True) + helpers.install("-n", env_name, "-f", env_file_yml, "--json", no_dry_run=True) + + res = helpers.umamba_list("-n", env_name, "--json") + assert any( + package["name"] == "numpy" + and package["version"] == "1.26.4" + and package["base_url"] == "https://pypi.org/" + and package["build_string"] == "pypi_0" + and package["channel"] == "pypi" + and package["platform"] == "" + for package in res + ) + + @pytest.mark.parametrize("env_selector", ["name", "prefix"]) @pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True) def test_not_existing(tmp_home, tmp_root_prefix, tmp_xtensor_env, env_selector):