From 2b35bcf7a489204b9c5cf82dc4eba75ccc0c96b7 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 1 Jul 2022 18:14:16 +0200 Subject: [PATCH] fix integration --- libmamba/include/mamba/core/subdirdata.hpp | 4 +- libmamba/include/mamba/core/transaction.hpp | 2 +- libmamba/include/mamba/core/url.hpp | 2 +- libmamba/src/api/channel_loader.cpp | 5 +- libmamba/src/core/subdirdata.cpp | 113 +++++++++++++------- libmamba/src/core/transaction.cpp | 33 +++--- libmamba/src/core/url.cpp | 9 +- 7 files changed, 104 insertions(+), 64 deletions(-) diff --git a/libmamba/include/mamba/core/subdirdata.hpp b/libmamba/include/mamba/core/subdirdata.hpp index 194d294814..e60002d0a5 100644 --- a/libmamba/include/mamba/core/subdirdata.hpp +++ b/libmamba/include/mamba/core/subdirdata.hpp @@ -23,7 +23,7 @@ #include "mamba/core/repo.hpp" #include "mamba/core/util.hpp" -#include "powerloader/download_target.hpp" +#include namespace decompress @@ -93,6 +93,8 @@ namespace mamba fs::path m_expired_cache_path; fs::path m_writable_pkgs_dir; + powerloader::CbReturnCode end_callback(powerloader::TransferStatus status, const std::string& msg); + int progress_callback(curl_off_t done, curl_off_t total); ProgressProxy m_progress_bar; bool m_loaded; diff --git a/libmamba/include/mamba/core/transaction.hpp b/libmamba/include/mamba/core/transaction.hpp index f37a77e66d..beed3982e7 100644 --- a/libmamba/include/mamba/core/transaction.hpp +++ b/libmamba/include/mamba/core/transaction.hpp @@ -28,7 +28,7 @@ #include "transaction_context.hpp" #include "env_lockfile.hpp" -#include "powerloader/download_target.hpp" +#include extern "C" { diff --git a/libmamba/include/mamba/core/url.hpp b/libmamba/include/mamba/core/url.hpp index 4d6cfc0563..5a07289072 100644 --- a/libmamba/include/mamba/core/url.hpp +++ b/libmamba/include/mamba/core/url.hpp @@ -7,7 +7,7 @@ #ifndef MAMBA_CORE_URL_HPP #define MAMBA_CORE_URL_HPP -#include "powerloader/url.hpp" +#include #include #include diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 7fd42bf18b..8831611f0c 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -5,7 +5,7 @@ #include "mamba/core/repo.hpp" #include "mamba/core/subdirdata.hpp" -#include "powerloader/downloader.hpp" +#include namespace mamba { @@ -48,7 +48,8 @@ namespace mamba std::vector channel_urls = ctx.channels; std::vector subdirs; - powerloader::Downloader multi_dl; + powerloader::Context plctx; + powerloader::Downloader multi_dl(plctx); std::vector> priorities; int max_prio = static_cast(channel_urls.size()); diff --git a/libmamba/src/core/subdirdata.cpp b/libmamba/src/core/subdirdata.cpp index 66275aa28f..fab1f688bb 100644 --- a/libmamba/src/core/subdirdata.cpp +++ b/libmamba/src/core/subdirdata.cpp @@ -65,6 +65,7 @@ namespace decompress namespace mamba { + using namespace std::placeholders; namespace detail { nlohmann::json read_mod_and_etag(const fs::path& file) @@ -224,7 +225,14 @@ namespace mamba { if (m_target != nullptr) { - m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this); + // m_target->cbdata = this; + m_target->end_callback = std::bind(&MSubdirData::end_callback, this, _1, _2); + // m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this); + } + if (m_progress_bar && m_target) + { + // m_target->progress_callback = this->progress_callback; + m_target->progress_callback = std::bind(&MSubdirData::progress_callback, this, _1, _2); } } @@ -251,11 +259,26 @@ namespace mamba if (m_target != nullptr) { - m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this); + // m_target->cbdata = this; + m_target->end_callback = std::bind(&MSubdirData::end_callback, this, _1, _2); + if (m_progress_bar) + { + m_target->progress_callback = std::bind(&MSubdirData::progress_callback, this, _1, _2); + } + + // m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this); } if (rhs.m_target != nullptr) { - rhs.m_target->set_finalize_callback(&MSubdirData::finalize_transfer, &rhs); + m_target->end_callback = std::bind(&MSubdirData::end_callback, &rhs, _1, _2); + + if (rhs.m_progress_bar) + { + m_target->progress_callback = std::bind(&MSubdirData::progress_callback, &rhs, _1, _2); + } + + // m_target->cbdata = &rhs; + // rhs.m_target->set_finalize_callback(&MSubdirData::finalize_transfer, &rhs); } return *this; } @@ -450,8 +473,8 @@ namespace mamba fs::path json_file, solv_file; - if (m_target->http_status == 304) - { + // if (m_target->http_status == 304) + // { // // cache still valid // LOG_INFO << "Cache is still valid"; // json_file = m_expired_cache_path / "cache" / m_json_fn; @@ -524,13 +547,13 @@ namespace mamba // r.elapsed.deactivate(); // } - m_json_cache_valid = true; - m_loaded = true; - m_temp_file.reset(nullptr); - return true; - } - else - { + // m_json_cache_valid = true; + // m_loaded = true; + // m_temp_file.reset(nullptr); + // return true; + // } + // else + // { // if (m_writable_pkgs_dir.empty()) // m_json_cache_valid = true; // m_loaded = true; @@ -539,14 +562,14 @@ namespace mamba // } // else // { - if (writable_cache_path.empty()) - { - LOG_ERROR << "Could not find any writable cache directory for repodata file"; - throw std::runtime_error("Non-writable cache error."); - } + // if (writable_cache_path.empty()) + // { + // LOG_ERROR << "Could not find any writable cache directory for repodata file"; + // throw std::runtime_error("Non-writable cache error."); + // } // } - LOG_DEBUG << "Finalized transfer of '" << m_repodata_url << "'"; + LOG_INFO << "Finalized transfer of '" << m_repodata_url << "'"; fs::path writable_cache_dir = create_cache_dir(m_writable_pkgs_dir); json_file = writable_cache_dir / m_json_fn; @@ -558,7 +581,7 @@ namespace mamba // m_mod_etag["_mod"] = m_target->mod; // m_mod_etag["_cache_control"] = m_target->cache_control; - LOG_DEBUG << "Opening '" << json_file.string() << "'"; + LOG_INFO << "Opening '" << json_file.string() << "'"; path::touch(json_file, true); std::ofstream final_file = open_ofstream(json_file); @@ -631,20 +654,42 @@ namespace mamba return result; } + int MSubdirData::progress_callback(curl_off_t done, curl_off_t total) + { + this->m_progress_bar.set_progress(done, total); + std::cout << done << " of " << total << std::endl; + return 0; + } + + powerloader::CbReturnCode MSubdirData::end_callback(powerloader::TransferStatus status, const std::string& msg) + { + if (status == powerloader::TransferStatus::kSUCCESSFUL) + { + spdlog::warn("Transfer successful... {}", msg); + std::cout << this->m_target->effective_url << std::endl; + spdlog::warn("Transfer successful... {}", this->m_target->effective_url); + this->finalize_transfer(); + } + return powerloader::CbReturnCode::kOK; + } + void MSubdirData::create_target(nlohmann::json& mod_etag) { auto& ctx = Context::instance(); m_temp_file = std::make_unique(); + std::cout << "Tempfile path: " << m_temp_file->path() << std::endl; + std::cout << "REPODATA URL " << m_repodata_url << std::endl; m_target = std::make_shared(m_repodata_url, "", m_temp_file->path()); if (!(ctx.no_progress_bars || ctx.quiet || ctx.json)) { m_progress_bar = Console::instance().add_progress_bar(m_name); // m_target->set_progress_bar(m_progress_bar); - m_target->progress_callback = [this](curl_off_t done, curl_off_t total) -> int - { - this->m_progress_bar.set_progress(done, total); - return 0; - }; + m_target->progress_callback = std::bind(&MSubdirData::progress_callback, this, _1, _2); + // m_target->progress_callback = [this](curl_off_t done, curl_off_t total) -> int + // { + // // this->m_progress_bar.set_progress(done, total); + // // std::cout << done << " of " << total << std::endl; + // }; } // if we get something _other_ than the noarch, we DO NOT throw if the file // can't be retrieved @@ -653,19 +698,15 @@ namespace mamba // m_target->set_ignore_failure(true); // } - auto x = [](powerloader::TransferStatus status, const std::string& msg, void* clientp) -> powerloader::CbReturnCode - { - auto* self = (MSubdirData*)clientp; - spdlog::warn("Status {} -- msg: {}", (int) status, msg); - if (status == powerloader::TransferStatus::kSUCCESSFUL) - { - self->finalize_transfer(); - } - return powerloader::CbReturnCode::kOK; - }; + // std::cout << "THIS: " << this << std::endl; + + // auto x = [this](powerloader::TransferStatus status, const std::string& msg) -> powerloader::CbReturnCode + // { + // this-end_callback() + // }; - m_target->endcb = x; - m_target->cbdata = this; + m_target->end_callback = std::bind(&MSubdirData::end_callback, this, _1, _2); + // m_target->cbdata = this; // m_target->set_finalize_callback(&MSubdirData::finalize_transfer, this); // m_target->set_mod_etag_headers(mod_etag); diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 98b57afcce..d4b959d1c9 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -19,8 +19,8 @@ #include "mamba/core/util_scope.hpp" -#include "powerloader/downloader.hpp" -#include "termcolor/termcolor.hpp" +#include +#include extern "C" { @@ -314,13 +314,13 @@ namespace mamba // m_download_bar.mark_as_completed(); // } - if (m_target->http_status >= 400) - { - LOG_ERROR << "Failed to download package from " << m_url << " (status " - << m_target->http_status << ")"; - m_validation_result = VALIDATION_RESULT::UNDEFINED; - return false; - } + // if (m_target->http_status >= 400) + // { + // LOG_ERROR << "Failed to download package from " << m_url << " (status " + // << m_target->http_status << ")"; + // m_validation_result = VALIDATION_RESULT::UNDEFINED; + // return false; + // } LOG_INFO << "Download finished, validating '" << m_tarball_path.string() << "'"; MainExecutor::instance().schedule(&PackageDownloadExtractTarget::validate_extract, this); @@ -404,25 +404,21 @@ namespace mamba m_tarball_path = m_cache_path / m_filename; m_target = std::make_shared(m_url, "", m_tarball_path); - // typedef CbReturnCode (*EndCb)(TransferStatus status, const std::string& msg, void* clientp); - - // m_target.endcb = &PackageDownloadExtractTarget::finalize_callback; - auto x = [](powerloader::TransferStatus status, const std::string& msg, void* clientp) -> powerloader::CbReturnCode + auto x = [this](powerloader::TransferStatus status, const std::string& msg) -> powerloader::CbReturnCode { - auto* self = (PackageDownloadExtractTarget*)clientp; spdlog::warn("Status {} -- msg: {}", (int) status, msg); if (status == powerloader::TransferStatus::kSUCCESSFUL) { - self->finalize_callback(); + this->finalize_callback(); } return powerloader::CbReturnCode::kOK; }; + m_target->expected_size = m_expected_size; m_target->checksums.push_back( powerloader::Checksum{powerloader::ChecksumType::kSHA256, m_sha256}); - m_target->endcb = x; - m_target->cbdata = this; + m_target->end_callback = x; // m_target->set_expected_size(m_expected_size); if (m_has_progress_bars) @@ -1136,7 +1132,8 @@ namespace mamba bool MTransaction::fetch_extract_packages() { std::vector> targets; - powerloader::Downloader multi_dl; + powerloader::Context plctx; + powerloader::Downloader multi_dl(plctx); auto& pbar_manager = Console::instance().init_progress_bar_manager(ProgressBarMode::aggregated); diff --git a/libmamba/src/core/url.cpp b/libmamba/src/core/url.cpp index 303a90fbad..8ddabd6314 100644 --- a/libmamba/src/core/url.cpp +++ b/libmamba/src/core/url.cpp @@ -13,11 +13,10 @@ namespace mamba { - - bool has_scheme(const std::string& url) - { - return std::regex_search(url, Context::instance().scheme_regex); - } + // bool has_scheme(const std::string& url) + // { + // return std::regex_search(url, Context::instance().scheme_regex); + // } void split_anaconda_token(const std::string& url, std::string& cleaned_url, std::string& token) {