diff --git a/include/mamba/core/package_cache.hpp b/include/mamba/core/package_cache.hpp index 9087e8a66e..c41c93e02a 100644 --- a/include/mamba/core/package_cache.hpp +++ b/include/mamba/core/package_cache.hpp @@ -40,6 +40,7 @@ namespace mamba void set_writable(Writable writable); Writable is_writable(); fs::path get_pkgs_dir() const; + void clear_query_cache(const PackageInfo& s); bool query(const PackageInfo& s); @@ -59,11 +60,15 @@ namespace mamba MultiPackageCache(const std::vector& pkgs_dirs); PackageCacheData& first_writable(); - bool query(const PackageInfo& s); + fs::path query(const PackageInfo& s); + fs::path first_cache_path(const PackageInfo& s, bool return_empty = true); std::vector writable_caches(); + void clear_query_cache(const PackageInfo& s); + private: std::vector m_caches; + std::map m_path_cache; }; } // namespace mamba diff --git a/include/mamba/core/transaction.hpp b/include/mamba/core/transaction.hpp index 7d2bcbc96a..930d5443dc 100644 --- a/include/mamba/core/transaction.hpp +++ b/include/mamba/core/transaction.hpp @@ -100,7 +100,7 @@ namespace mamba ignore }; - MTransaction(MSolver& solver, MultiPackageCache& cache); + MTransaction(MSolver& solver, MultiPackageCache& cache, const std::string& cache_dir); ~MTransaction(); MTransaction(const MTransaction&) = delete; @@ -116,11 +116,11 @@ namespace mamba void init(); to_conda_type to_conda(); void log_json(); - bool fetch_extract_packages(const std::string& cache_dir, std::vector& repos); + bool fetch_extract_packages(std::vector& repos); bool empty(); - bool prompt(const std::string& cache_dir, std::vector& repos); + bool prompt(std::vector& repos); void print(); - bool execute(PrefixData& prefix, const fs::path& cache_dir); + bool execute(PrefixData& prefix); bool filter(Solvable* s); std::string find_python_version(); @@ -131,6 +131,7 @@ namespace mamba TransactionContext m_transaction_context; MultiPackageCache m_multi_cache; + const fs::path m_cache_path; std::vector m_to_install, m_to_remove; History::UserRequest m_history_entry; Transaction* m_transaction; diff --git a/mamba/mamba.py b/mamba/mamba.py old mode 100755 new mode 100644 index 95ab37de38..52ef7b7413 --- a/mamba/mamba.py +++ b/mamba/mamba.py @@ -247,10 +247,10 @@ def remove(args, parser): return exit_code package_cache = api.MultiPackageCache(context.pkgs_dirs) - transaction = api.Transaction(solver, package_cache) - downloaded = transaction.prompt( - PackageCacheData.first_writable().pkgs_dir, repos + transaction = api.Transaction( + solver, package_cache, PackageCacheData.first_writable().pkgs_dir ) + downloaded = transaction.prompt(repos) if not downloaded: exit(0) @@ -576,7 +576,9 @@ def install(args, parser, command="install"): return exit_code package_cache = api.MultiPackageCache(context.pkgs_dirs) - transaction = api.Transaction(solver, package_cache) + transaction = api.Transaction( + solver, package_cache, PackageCacheData.first_writable().pkgs_dir + ) mmb_specs, to_link, to_unlink = transaction.to_conda() specs_to_add = [MatchSpec(m) for m in mmb_specs[0]] @@ -584,9 +586,7 @@ def install(args, parser, command="install"): transaction.log_json() - downloaded = transaction.prompt( - PackageCacheData.first_writable().pkgs_dir, repos - ) + downloaded = transaction.prompt(repos) if not downloaded: exit(0) PackageCacheData.first_writable().reload() @@ -596,7 +596,7 @@ def install(args, parser, command="install"): if newenv and not isdir(context.target_prefix) and not context.dry_run: mkdir_p(prefix) - transaction.execute(prefix_data, PackageCacheData.first_writable().pkgs_dir) + transaction.execute(prefix_data) else: conda_transaction = to_txn( specs_to_add, diff --git a/mamba/mamba_env.py b/mamba/mamba_env.py index 826ea6e96f..8bffc9052a 100644 --- a/mamba/mamba_env.py +++ b/mamba/mamba_env.py @@ -8,6 +8,7 @@ import sys from conda.base.context import context +from conda.core.package_cache_data import PackageCacheData from conda.core.prefix_data import PrefixData from conda.core.solve import get_pinned_specs from conda.models.channel import prioritize_channels @@ -112,7 +113,9 @@ def mamba_install(prefix, specs, args, env, *_, **kwargs): exit(1) package_cache = api.MultiPackageCache(context.pkgs_dirs) - transaction = api.Transaction(solver, package_cache) + transaction = api.Transaction( + solver, package_cache, PackageCacheData.first_writable().pkgs_dir + ) if not (context.quiet or context.json): transaction.print() mmb_specs, to_link, to_unlink = transaction.to_conda() diff --git a/src/api/install.cpp b/src/api/install.cpp index d943a82376..82526632c9 100644 --- a/src/api/install.cpp +++ b/src/api/install.cpp @@ -238,7 +238,7 @@ namespace mamba } MultiPackageCache package_caches({ pkgs_dirs }); - MTransaction trans(solver, package_caches); + MTransaction trans(solver, package_caches, pkgs_dirs); if (ctx.json) { @@ -253,14 +253,14 @@ namespace mamba std::cout << std::endl; - bool yes = trans.prompt(pkgs_dirs, repo_ptrs); + bool yes = trans.prompt(repo_ptrs); if (yes) { if (create_env && !Context::instance().dry_run) { detail::create_target_directory(ctx.target_prefix); } - trans.execute(prefix_data, pkgs_dirs); + trans.execute(prefix_data); } } diff --git a/src/api/remove.cpp b/src/api/remove.cpp index dc49dd9397..2ce2a470b0 100644 --- a/src/api/remove.cpp +++ b/src/api/remove.cpp @@ -76,8 +76,9 @@ namespace mamba solver.add_jobs(specs, SOLVER_ERASE); solver.solve(); - MultiPackageCache package_caches({ ctx.root_prefix / "pkgs" }); - MTransaction trans(solver, package_caches); + const fs::path pkgs_dirs(ctx.root_prefix / "pkgs"); + MultiPackageCache package_caches({ pkgs_dirs }); + MTransaction trans(solver, package_caches, pkgs_dirs); if (ctx.json) { @@ -90,9 +91,9 @@ namespace mamba repo_ptrs.push_back(&r); } - bool yes = trans.prompt(ctx.root_prefix / "pkgs", repo_ptrs); + bool yes = trans.prompt(repo_ptrs); if (yes) - trans.execute(prefix_data, ctx.root_prefix / "pkgs"); + trans.execute(prefix_data); } } } // mamba diff --git a/src/core/package_cache.cpp b/src/core/package_cache.cpp index 92d1c680fc..7bfee31b45 100644 --- a/src/core/package_cache.cpp +++ b/src/core/package_cache.cpp @@ -57,6 +57,11 @@ namespace mamba return m_pkgs_dir; } + void PackageCacheData::clear_query_cache(const PackageInfo& s) + { + m_valid_cache.erase(s.str()); + } + PackageCacheData PackageCacheData::first_writable(const std::vector* pkgs_dirs) { const std::vector* dirs = pkgs_dirs ? pkgs_dirs : &Context::instance().pkgs_dirs; @@ -288,13 +293,44 @@ namespace mamba return res; } - bool MultiPackageCache::query(const PackageInfo& s) + fs::path MultiPackageCache::query(const PackageInfo& s) { for (auto& c : m_caches) { if (c.query(s)) - return true; + return c.get_pkgs_dir(); + } + return {}; + } + + fs::path MultiPackageCache::first_cache_path(const PackageInfo& s, bool return_empty) + { + const std::string pkg(s.str()); + const auto cache_iter(m_path_cache.find(pkg)); + if (cache_iter != m_path_cache.end()) + { + return cache_iter->second; + } + for (auto& c : m_caches) + { + const fs::path cache_path(c.get_pkgs_dir()); + if (c.query(s) && fs::exists(cache_path / strip_package_extension(s.fn))) + { + m_path_cache[pkg] = cache_path; + return cache_path; + } + } + if (return_empty) + return {}; + else + throw std::runtime_error("Cannot find cache for " + s.fn); + } + + void MultiPackageCache::clear_query_cache(const PackageInfo& s) + { + for (auto& c : m_caches) + { + c.clear_query_cache(s); } - return false; } } // namespace mamba diff --git a/src/core/package_handling.cpp b/src/core/package_handling.cpp index 452af49245..3de50ccfd7 100644 --- a/src/core/package_handling.cpp +++ b/src/core/package_handling.cpp @@ -438,7 +438,11 @@ namespace mamba for (auto& p : paths_data) { fs::path full_path = pkg_folder / p.path; - if (!fs::exists(full_path)) + // "exists" follows symlink so if the symlink doesn't link to existing target it + // will return false. There is such symlink in _openmp_mutex package. So if the file + // is a symlink we don't want to follow the symlink. The "paths_data" should include + // path of all the files and we shound't need to follow symlink. + if (!(fs::exists(full_path) || fs::is_symlink(full_path))) { if (is_warn || is_fail) { diff --git a/src/core/transaction.cpp b/src/core/transaction.cpp index edca4db0b9..f61ef6737b 100644 --- a/src/core/transaction.cpp +++ b/src/core/transaction.cpp @@ -13,6 +13,21 @@ #include "mamba/core/match_spec.hpp" #include "mamba/core/thread_utils.hpp" +namespace +{ + bool need_pkg_download(const mamba::PackageInfo& pkg_info, + mamba::MultiPackageCache& cache, + const fs::path& cache_path) + { + // See PackageDownloadExtractTarget::target() for note about when we need to download + // the package. + bool need_download(cache.first_cache_path(pkg_info).empty()); + if (need_download) + need_download = cache.query(pkg_info) != cache_path; + return need_download; + } +} // anonymouse namspace + namespace mamba { nlohmann::json solvable_to_json(Solvable* s) @@ -223,41 +238,44 @@ namespace mamba DownloadTarget* PackageDownloadExtractTarget::target(const fs::path& cache_path, MultiPackageCache& cache) { - m_cache_path = cache_path; - m_tarball_path = cache_path / m_filename; - - bool valid = cache.query(m_package_info); - - fs::path dest_dir = strip_package_extension(m_tarball_path); - bool dest_dir_exists = fs::exists(dest_dir); - - if (valid && !dest_dir_exists) - { - m_progress_proxy = Console::instance().add_progress_bar(m_name); - m_validation_result = VALIDATION_RESULT::VALID; - thread v(&PackageDownloadExtractTarget::extract_from_cache, this); - v.detach(); - return nullptr; - } - + // // tarball can be removed, it's fine if only the correct dest dir exists - if (!valid) - { - // need to download this file - LOG_INFO << "Adding " << m_name << " with " << m_url; - - m_progress_proxy = Console::instance().add_progress_bar(m_name); - m_target = std::make_unique(m_name, m_url, cache_path / m_filename); - m_target->set_finalize_callback(&PackageDownloadExtractTarget::finalize_callback, this); - m_target->set_expected_size(m_expected_size); - m_target->set_progress_bar(m_progress_proxy); - } - else + // 1. If there is extracted cache, use it, otherwise next. + // 2. If there is tarball in `cache_path` (writable), extract it, otherwise next. + // 3. Run the full download pipeline. + // + m_cache_path = cache_path; + fs::path pkg_cache_path(cache.first_cache_path(m_package_info)); + if (pkg_cache_path.empty()) { - LOG_INFO << "Using cache " << m_name; - m_finished = true; + pkg_cache_path = cache.query(m_package_info); + if (cache_path == pkg_cache_path) + { + m_tarball_path = pkg_cache_path / m_filename; + m_progress_proxy = Console::instance().add_progress_bar(m_name); + m_validation_result = VALIDATION_RESULT::VALID; + thread v(&PackageDownloadExtractTarget::extract_from_cache, this); + v.detach(); + } + else + { + m_tarball_path = cache_path / m_filename; + cache.clear_query_cache(m_package_info); + // need to download this file + LOG_INFO << "Adding " << m_name << " with " << m_url; + + m_progress_proxy = Console::instance().add_progress_bar(m_name); + m_target = std::make_unique(m_name, m_url, cache_path / m_filename); + m_target->set_finalize_callback(&PackageDownloadExtractTarget::finalize_callback, + this); + m_target->set_expected_size(m_expected_size); + m_target->set_progress_bar(m_progress_proxy); + return m_target.get(); + } } - return m_target.get(); + LOG_INFO << "Using cache " << m_name; + m_finished = true; + return nullptr; } /******************************* @@ -280,8 +298,11 @@ namespace mamba } } - MTransaction::MTransaction(MSolver& solver, MultiPackageCache& cache) + MTransaction::MTransaction(MSolver& solver, + MultiPackageCache& cache, + const std::string& cache_dir) : m_multi_cache(cache) + , m_cache_path(cache_dir) { if (!solver.is_solved()) { @@ -498,7 +519,7 @@ namespace mamba std::stack m_link_stack; }; - bool MTransaction::execute(PrefixData& prefix, const fs::path& cache_dir) + bool MTransaction::execute(PrefixData& prefix) { auto& ctx = Context::instance(); @@ -555,13 +576,17 @@ namespace mamba Console::stream() << "Changing " << PackageInfo(s).str() << " ==> " << PackageInfo(s2).str(); PackageInfo p_unlink(s); + const fs::path ul_cache_path(m_multi_cache.first_cache_path(p_unlink)); PackageInfo p_link(s2); + const fs::path l_cache_path(m_multi_cache.first_cache_path(p_link, false)); - UnlinkPackage up(p_unlink, fs::path(cache_dir), &m_transaction_context); + UnlinkPackage up(p_unlink, + ul_cache_path.empty() ? m_cache_path : ul_cache_path, + &m_transaction_context); up.execute(); rollback.record(up); - LinkPackage lp(p_link, fs::path(cache_dir), &m_transaction_context); + LinkPackage lp(p_link, l_cache_path, &m_transaction_context); lp.execute(); rollback.record(lp); @@ -573,8 +598,10 @@ namespace mamba case SOLVER_TRANSACTION_ERASE: { PackageInfo p(s); - Console::stream() << "Unlinking " << PackageInfo(s).str(); - UnlinkPackage up(p, fs::path(cache_dir), &m_transaction_context); + Console::stream() << "Unlinking " << p.str(); + const fs::path cache_path(m_multi_cache.first_cache_path(p)); + UnlinkPackage up( + p, cache_path.empty() ? m_cache_path : cache_path, &m_transaction_context); up.execute(); rollback.record(up); m_history_entry.unlink_dists.push_back(p.long_str()); @@ -584,7 +611,8 @@ namespace mamba { PackageInfo p(s); Console::stream() << "Linking " << p.str(); - LinkPackage lp(p, fs::path(cache_dir), &m_transaction_context); + const fs::path cache_path(m_multi_cache.first_cache_path(p, false)); + LinkPackage lp(p, cache_path, &m_transaction_context); lp.execute(); rollback.record(lp); m_history_entry.link_dists.push_back(p.long_str()); @@ -657,7 +685,7 @@ namespace mamba for (Solvable* s : m_to_install) { - if (this->m_multi_cache.query(s)) + if (!need_pkg_download(s, m_multi_cache, m_cache_path)) { to_link.push_back(solvable_to_json(s)); } @@ -690,10 +718,8 @@ namespace mamba add_json(to_unlink, "UNLINK"); } - bool MTransaction::fetch_extract_packages(const std::string& cache_dir, - std::vector& repos) + bool MTransaction::fetch_extract_packages(std::vector& repos) { - fs::path cache_path(cache_dir); std::vector> targets; MultiDownloadTarget multi_dl; @@ -721,7 +747,7 @@ namespace mamba } targets.emplace_back(std::make_unique(s)); - multi_dl.add(targets[targets.size() - 1]->target(cache_path, m_multi_cache)); + multi_dl.add(targets[targets.size() - 1]->target(m_cache_path, m_multi_cache)); } interruption_guard g([]() { Console::instance().init_multi_progress(); }); @@ -773,7 +799,7 @@ namespace mamba return m_to_install.size() == 0 && m_to_remove.size() == 0; } - bool MTransaction::prompt(const std::string& cache_dir, std::vector& repos) + bool MTransaction::prompt(std::vector& repos) { print(); if (Context::instance().dry_run || empty()) @@ -784,7 +810,7 @@ namespace mamba bool res = Console::prompt("Confirm changes", 'y'); if (res) { - return fetch_extract_packages(cache_dir, repos); + return fetch_extract_packages(repos); } return res; } @@ -862,21 +888,24 @@ namespace mamba { dlsize_s.s = "Ignored"; } - else if (this->m_multi_cache.query(s)) - { - dlsize_s.s = "Cached"; - dlsize_s.flag = printers::format::green; - } else { - std::stringstream s; - to_human_readable_filesize(s, dlsize); - dlsize_s.s = s.str(); - // Hacky hacky - if (static_cast(flag) - & static_cast(printers::format::green)) + if (!need_pkg_download(s, m_multi_cache, m_cache_path)) { - total_size += dlsize; + dlsize_s.s = "Cached"; + dlsize_s.flag = printers::format::green; + } + else + { + std::stringstream s; + to_human_readable_filesize(s, dlsize); + dlsize_s.s = s.str(); + // Hacky hacky + if (static_cast(flag) + & static_cast(printers::format::green)) + { + total_size += dlsize; + } } } } diff --git a/src/mamba/py_interface.cpp b/src/mamba/py_interface.cpp index 94efcee96b..159547322e 100644 --- a/src/mamba/py_interface.cpp +++ b/src/mamba/py_interface.cpp @@ -65,15 +65,15 @@ PYBIND11_MODULE(mamba_api, m) .def("clear", &MRepo::clear); py::class_(m, "Transaction") - .def(py::init()) + .def(py::init()) .def("to_conda", &MTransaction::to_conda) .def("log_json", &MTransaction::log_json) .def("print", &MTransaction::print) .def("fetch_extract_packages", &MTransaction::fetch_extract_packages) .def("prompt", &MTransaction::prompt) - .def("execute", - [](MTransaction& self, PrefixData& target_prefix, const std::string& cache_dir) - -> bool { return self.execute(target_prefix, cache_dir); }); + .def("execute", [](MTransaction& self, PrefixData& target_prefix) -> bool { + return self.execute(target_prefix); + }); py::class_(m, "Solver") .def(py::init>>())