pip packages support with list

libmamba/include/mamba/core/activation.hpp

@@ -258,8 +258,6 @@ namespace mamba
         fs::u8path hook_source_path() override;
     };
 
-    std::vector<fs::u8path> get_path_dirs(const fs::u8path& prefix);
-
 }  // namespace mamba
 
 #endif

libmamba/include/mamba/core/prefix_data.hpp

@@ -28,9 +28,12 @@ namespace mamba
         create(const fs::u8path& prefix_path, ChannelContext& channel_context);
 
         void add_packages(const std::vector<specs::PackageInfo>& packages);
-        const package_map& records() const;
         void load_single_record(const fs::u8path& path);
 
+        const package_map& records() const;
+        const package_map& pip_records() const;
+        package_map all_pkg_mgr_records() const;
+
         History& history();
         const fs::u8path& path() const;
         std::vector<specs::PackageInfo> sorted_records() const;
@@ -44,8 +47,11 @@ namespace mamba
 
         PrefixData(const fs::u8path& prefix_path, ChannelContext& channel_context);
 
+        void load_site_packages();
+
         History m_history;
         package_map m_package_records;
+        package_map m_pip_package_records;
         fs::u8path m_prefix_path;
 
         ChannelContext& m_channel_context;

libmamba/include/mamba/specs/package_info.hpp

@@ -68,6 +68,7 @@ namespace mamba::specs
         PackageInfo() = default;
         explicit PackageInfo(std::string name);
         PackageInfo(std::string name, std::string version, std::string build_string, std::size_t build_number);
+        PackageInfo(std::string name, std::string version, std::string build_string, std::string channel);
 
         [[nodiscard]] auto json_signable() const -> nlohmann::json;
         [[nodiscard]] auto str() const -> std::string;

libmamba/include/mamba/util/environment.hpp

@@ -11,6 +11,7 @@
 #include <string>
 #include <string_view>
 #include <unordered_map>
+#include <vector>
 
 #include "mamba/fs/filesystem.hpp"
 #include "mamba/util/build.hpp"
@@ -93,6 +94,11 @@ namespace mamba::util
      */
     [[nodiscard]] constexpr auto pathsep() -> char;
 
+    /**
+     * Return directories of the given prefix path.
+     */
+    [[nodiscard]] auto get_path_dirs(const fs::u8path& prefix) -> std::vector<fs::u8path>;
+
     /**
      * Return the full path of a program from its name.
      */

libmamba/src/api/install.cpp

@@ -10,7 +10,6 @@
 #include "mamba/api/channel_loader.hpp"
 #include "mamba/api/configuration.hpp"
 #include "mamba/api/install.hpp"
-#include "mamba/core/activation.hpp"
 #include "mamba/core/channel_context.hpp"
 #include "mamba/core/context.hpp"
 #include "mamba/core/env_lockfile.hpp"

libmamba/src/api/list.cpp

@@ -67,14 +67,16 @@ namespace mamba
             {
                 regex = '^' + regex + '$';
             }
+
             std::regex spec_pat(regex);
+            auto all_records = std::move(prefix_data.all_pkg_mgr_records());
 
             if (ctx.output_params.json)
             {
                 auto jout = nlohmann::json::array();
                 std::vector<std::string> keys;
 
-                for (const auto& pkg : prefix_data.records())
+                for (const auto& pkg : all_records)
                 {
                     keys.push_back(pkg.first);
                 }
@@ -83,24 +85,33 @@ namespace mamba
                 for (const auto& key : keys)
                 {
                     auto obj = nlohmann::json();
-                    const auto& pkg_info = prefix_data.records().find(key)->second;
+                    const auto& pkg_info = all_records.find(key)->second;
 
                     if (regex.empty() || std::regex_search(pkg_info.name, spec_pat))
                     {
                         auto channels = channel_context.make_channel(pkg_info.package_url);
                         assert(channels.size() == 1);  // A URL can only resolve to one channel
-                        obj["base_url"] = strip_from_filename_and_platform(
-                            channels.front().url().str(specs::CondaURL::Credentials::Remove),
-                            pkg_info.filename,
-                            pkg_info.platform
-                        );
+
+                        if (pkg_info.package_url.empty() && (pkg_info.channel == "pypi"))
+                        {
+                            obj["base_url"] = "https://pypi.org/";
+                            obj["channel"] = pkg_info.channel;
+                        }
+                        else
+                        {
+                            obj["base_url"] = strip_from_filename_and_platform(
+                                channels.front().url().str(specs::CondaURL::Credentials::Remove),
+                                pkg_info.filename,
+                                pkg_info.platform
+                            );
+                            obj["channel"] = strip_from_filename_and_platform(
+                                channels.front().display_name(),
+                                pkg_info.filename,
+                                pkg_info.platform
+                            );
+                        }
                         obj["build_number"] = pkg_info.build_number;
                         obj["build_string"] = pkg_info.build_string;
-                        obj["channel"] = strip_from_filename_and_platform(
-                            channels.front().display_name(),
-                            pkg_info.filename,
-                            pkg_info.platform
-                        );
                         obj["dist_name"] = pkg_info.str();
                         obj["name"] = pkg_info.name;
                         obj["platform"] = pkg_info.platform;
@@ -121,7 +132,7 @@ namespace mamba
             auto requested_specs = prefix_data.history().get_requested_specs_map();
 
             // order list of packages from prefix_data by alphabetical order
-            for (const auto& package : prefix_data.records())
+            for (const auto& package : all_records)
             {
                 if (regex.empty() || std::regex_search(package.second.name, spec_pat))
                 {
@@ -132,6 +143,10 @@ namespace mamba
                     {
                         formatted_pkgs.channel = "";
                     }
+                    else if (package.second.channel == "pypi")
+                    {
+                        formatted_pkgs.channel = package.second.channel;
+                    }
                     else
                     {
                         auto channels = channel_context.make_channel(package.second.channel);

libmamba/src/api/utils.cpp

@@ -12,8 +12,7 @@
 #include <reproc/reproc.h>
 
 // TODO includes to be removed after moving some functions/structs around
-#include "mamba/api/install.hpp"      // other_pkg_mgr_spec
-#include "mamba/core/activation.hpp"  // get_path_dirs
+#include "mamba/api/install.hpp"  // other_pkg_mgr_spec
 #include "mamba/core/context.hpp"
 #include "mamba/core/util.hpp"
 #include "mamba/fs/filesystem.hpp"
@@ -33,7 +32,7 @@ namespace mamba
         )
         {
             const auto get_python_path = [&]
-            { return util::which_in("python", get_path_dirs(target_prefix)).string(); };
+            { return util::which_in("python", util::get_path_dirs(target_prefix)).string(); };
 
             command_args cmd = { get_python_path(), "-m", "pip", "install" };
             command_args cmd_extension = { "-r", spec_file, "--no-input", "--quiet" };

libmamba/src/core/activation.cpp

@@ -208,23 +208,6 @@ namespace mamba
         }
     }
 
-    std::vector<fs::u8path> get_path_dirs(const fs::u8path& prefix)
-    {
-        if (util::on_win)
-        {
-            return { prefix,
-                     prefix / "Library" / "mingw-w64" / "bin",
-                     prefix / "Library" / "usr" / "bin",
-                     prefix / "Library" / "bin",
-                     prefix / "Scripts",
-                     prefix / "bin" };
-        }
-        else
-        {
-            return { prefix / "bin" };
-        }
-    }
-
     std::vector<fs::u8path> Activator::get_PATH()
     {
         std::vector<fs::u8path> path;
@@ -271,7 +254,7 @@ namespace mamba
 
         // TODO check if path_conversion does something useful here.
         // path_list[0:0] = list(self.path_conversion(self._get_path_dirs(prefix)))
-        std::vector<fs::u8path> final_path = get_path_dirs(prefix);
+        std::vector<fs::u8path> final_path = util::get_path_dirs(prefix);
         final_path.insert(final_path.end(), path_list.begin(), path_list.end());
         final_path.erase(std::unique(final_path.begin(), final_path.end()), final_path.end());
         std::string result = util::join(util::pathsep(), final_path).string();
@@ -285,7 +268,7 @@ namespace mamba
         std::vector<fs::u8path> current_path = get_PATH();
         assert(!old_prefix.empty());
 
-        std::vector<fs::u8path> old_prefix_dirs = get_path_dirs(old_prefix);
+        std::vector<fs::u8path> old_prefix_dirs = util::get_path_dirs(old_prefix);
 
         // remove all old paths
         std::vector<fs::u8path> cleaned_path;
@@ -312,7 +295,7 @@ namespace mamba
         std::vector<fs::u8path> final_path;
         if (!new_prefix.empty())
         {
-            final_path = get_path_dirs(new_prefix);
+            final_path = util::get_path_dirs(new_prefix);
             final_path.insert(final_path.end(), current_path.begin(), current_path.end());
 
             // remove duplicates

libmamba/src/core/prefix_data.cpp

@@ -9,11 +9,14 @@
 #include <unordered_map>
 #include <utility>
 
+#include <reproc++/run.hpp>
+
 #include "mamba/core/channel_context.hpp"
 #include "mamba/core/output.hpp"
 #include "mamba/core/prefix_data.hpp"
 #include "mamba/core/util.hpp"
 #include "mamba/specs/conda_url.hpp"
+#include "mamba/util/environment.hpp"
 #include "mamba/util/graph.hpp"
 #include "mamba/util/string.hpp"
 
@@ -56,6 +59,8 @@ namespace mamba
                 }
             }
         }
+        // Load packages installed with pip
+        load_site_packages();
     }
 
     void PrefixData::add_packages(const std::vector<specs::PackageInfo>& packages)
@@ -73,6 +78,22 @@ namespace mamba
         return m_package_records;
     }
 
+    const PrefixData::package_map& PrefixData::pip_records() const
+    {
+        return m_pip_package_records;
+    }
+
+    PrefixData::package_map PrefixData::all_pkg_mgr_records() const
+    {
+        PrefixData::package_map merged_records = m_package_records;
+        // Note that if the same key (pkg name) is present in both `m_package_records` and
+        // `m_pip_package_records`, the latter is not considered
+        // (this may be modified to be completely independent in the future)
+        merged_records.insert(m_pip_package_records.begin(), m_pip_package_records.end());
+
+        return merged_records;
+    }
+
     std::vector<specs::PackageInfo> PrefixData::sorted_records() const
     {
         // TODO add_pip_as_python_dependency
@@ -166,10 +187,89 @@ namespace mamba
 
         auto channels = m_channel_context.make_channel(prec.channel);
         // If someone wrote multichannel names in repodata_record, we don't know which one is the
-        // correct URL. This is must never happen!
+        // correct URL. This must never happen!
         assert(channels.size() == 1);
         using Credentials = specs::CondaURL::Credentials;
         prec.channel = channels.front().platform_url(prec.platform).str(Credentials::Remove);
         m_package_records.insert({ prec.name, std::move(prec) });
     }
+
+    // Load python packages installed with pip in the site-packages of the prefix.
+    void PrefixData::load_site_packages()
+    {
+        LOG_INFO << "Loading site packages";
+
+        // Look for `pip` package and return if it doesn't exist
+        auto python_pkg_record = m_package_records.find("pip");
+        if (python_pkg_record == m_package_records.end())
+        {
+            LOG_DEBUG << "`pip` not found";
+            return;
+        }
+
+        // Run `pip freeze`
+        std::string out, err;
+
+        const auto get_python_path = [&]
+        { return util::which_in("python", util::get_path_dirs(m_prefix_path)).string(); };
+
+        const auto args = std::array<std::string, 5>{ get_python_path(),
+                                                      "-m",
+                                                      "pip",
+                                                      "inspect",
+                                                      "--local" };
+        auto [status, ec] = reproc::run(
+            args,
+            reproc::options{},
+            reproc::sink::string(out),
+            reproc::sink::string(err)
+        );
+        if (ec)
+        {
+            throw std::runtime_error(ec.message());
+        }
+
+        // Nothing installed with `pip`
+        if (out.empty())
+        {
+            LOG_DEBUG << "Nothing installed with `pip`";
+            return;
+        }
+
+        nlohmann::json j = nlohmann::json::parse(out);
+
+        if (j.contains("installed") && j["installed"].is_array())
+        {
+            for (const auto& package : j["installed"])
+            {
+                // Get the package metadata, if requested and installed with `pip`
+                if (package.contains("requested") && package.contains("installer")
+                    && package["requested"] == true && package["installer"] == "pip")
+                {
+                    if (package.contains("metadata"))
+                    {
+                        // NOTE As checking the presence of all used keys in the json object can be
+                        // cumbersome and might affect the code readability, the elements where the
+                        // check with `contains` is skipped are considered mandatory. If a bug is
+                        // ever to occur in the future, checking the relevant key with `contains`
+                        // should be introduced then.
+                        auto prec = specs::PackageInfo(
+                            package["metadata"]["name"],
+                            package["metadata"]["version"],
+                            "pypi_0",
+                            "pypi"
+                        );
+                        // Set platform by concatenating `sys_platform` and `platform_machine` to
+                        // have something equivalent to `conda-forge`
+                        if (j.contains("environment"))
+                        {
+                            prec.platform = j["environment"]["sys_platform"].get<std::string>() + "-"
+                                            + j["environment"]["platform_machine"].get<std::string>();
+                        }
+                        m_pip_package_records.insert({ prec.name, std::move(prec) });
+                    }
+                }
+            }
+        }
+    }
 }  // namespace mamba

libmamba/src/core/transaction.cpp

@@ -362,8 +362,6 @@ namespace mamba
     bool
     MTransaction::execute(const Context& ctx, ChannelContext& channel_context, PrefixData& prefix)
     {
-        using Solution = solver::Solution;
-
         // JSON output
         // back to the top level if any action was required
         if (!empty())

libmamba/src/specs/package_info.cpp

@@ -156,6 +156,14 @@ namespace mamba::specs
     {
     }
 
+    PackageInfo::PackageInfo(std::string n, std::string v, std::string b, std::string c)
+        : name(std::move(n))
+        , version(std::move(v))
+        , build_string(std::move(b))
+        , channel(std::move(c))
+    {
+    }
+
     namespace
     {
         template <typename T, typename U>