From a511c2a132213078e75a26147be443d570e7f34f Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Thu, 25 Apr 2024 10:58:07 +0200 Subject: [PATCH] Support repodata_version:2/cep-15 with mamba parser --- libmamba/src/api/configuration.cpp | 12 ++++- libmamba/src/api/install.cpp | 6 +-- libmamba/src/solver/libsolv/helpers.cpp | 27 ++++++++-- libmamba/src/specs/package_info.cpp | 2 +- ...e-repodata-version-2-missing-base_url.json | 48 +++++++++++++++++ .../conda-forge-repodata-version-2.json | 49 +++++++++++++++++ .../src/solver/libsolv/test_database.cpp | 52 +++++++++++++++++++ 7 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 libmamba/tests/data/repodata/conda-forge-repodata-version-2-missing-base_url.json create mode 100644 libmamba/tests/data/repodata/conda-forge-repodata-version-2.json diff --git a/libmamba/src/api/configuration.cpp b/libmamba/src/api/configuration.cpp index c7a4b3f683..ff528f6932 100644 --- a/libmamba/src/api/configuration.cpp +++ b/libmamba/src/api/configuration.cpp @@ -874,6 +874,15 @@ namespace mamba } } + // cf. https://github.com/openSUSE/libsolv/issues/562 to track corresponding issue + void not_supported_option_hook(bool& value) + { + if (!value) + { + LOG_WARNING << "Parsing with libsolv does not support repodata_version 2"; + } + } + void debug_hook(bool& value) { if (value) @@ -1290,7 +1299,8 @@ namespace mamba "Default is `true`. `false` means libsolv is used.\n" ) .set_rc_configurable() - .set_env_var_names()); + .set_env_var_names() + .set_post_merge_hook(detail::not_supported_option_hook)); insert(Configurable("debug", &m_context.debug) .group("Basic") diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 9e3df895f1..b6c1bb1607 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -637,10 +637,10 @@ namespace mamba Console::instance().json_write({ { "success", true } }); // The point here is to delete the database before executing the transaction. - // The database can have high memrory impact, and so can installing pacakges as it - // requires downloading, extracgint, and launching Python interpreters for + // The database can have high memory impact, since installing packages + // requires downloading, extracting, and launching Python interpreters for // creating ``.pyc`` files. - // Ideally this whole function should be properly refactored and the transction itself + // Ideally this whole function should be properly refactored and the transaction itself // should not need the database. auto trans = [&](auto db) { diff --git a/libmamba/src/solver/libsolv/helpers.cpp b/libmamba/src/solver/libsolv/helpers.cpp index 8517f7c28c..7aa7e5f860 100644 --- a/libmamba/src/solver/libsolv/helpers.cpp +++ b/libmamba/src/solver/libsolv/helpers.cpp @@ -540,14 +540,35 @@ namespace mamba::solver::libsolv const auto lock = LockFile(filename); const auto repodata = parser.load(filename); - // An override for missing package subdir is found in at the top level + // An override for missing package subdir is found at the top level auto default_subdir = std::string(); - if (auto subdir = repodata.at_pointer("/info/subdir").get_string(); subdir.error()) + if (auto subdir = repodata.at_pointer("/info/subdir").get_string(); !subdir.error()) { default_subdir = std::string(subdir.value_unsafe()); } - const auto parsed_url = specs::CondaURL::parse(repo_url) + // Get `base_url` in case 'repodata_version': 2 + // cf. https://github.com/conda-incubator/ceps/blob/main/cep-15.md + auto base_url = repo_url; + if (auto repodata_version = repodata["repodata_version"].get_int64(); + !repodata_version.error()) + { + if (repodata_version.value_unsafe() == 2) + { + if (auto url = repodata.at_pointer("/info/base_url").get_string(); !url.error()) + { + base_url = std::string(url.value_unsafe()); + } + else + { + throw std::invalid_argument( + "'info/base_url' not found in repodata.json file while 'repodata_version': 2" + ); + } + } + } + + const auto parsed_url = specs::CondaURL::parse(base_url) .or_else([](specs::ParseError&& err) { throw std::move(err); }) .value(); diff --git a/libmamba/src/specs/package_info.cpp b/libmamba/src/specs/package_info.cpp index 309a16f7fb..28a192b4b9 100644 --- a/libmamba/src/specs/package_info.cpp +++ b/libmamba/src/specs/package_info.cpp @@ -41,7 +41,7 @@ namespace mamba::specs auto out = PackageInfo(); - // TODO decide on the bet way to group filename/channel/subdir/package_url all at once + // TODO decide on the best way to group filename/channel/subdir/package_url all at once out.package_url = util::path_or_url_to_url(spec); auto url = CondaURL(); diff --git a/libmamba/tests/data/repodata/conda-forge-repodata-version-2-missing-base_url.json b/libmamba/tests/data/repodata/conda-forge-repodata-version-2-missing-base_url.json new file mode 100644 index 0000000000..0b6270b66e --- /dev/null +++ b/libmamba/tests/data/repodata/conda-forge-repodata-version-2-missing-base_url.json @@ -0,0 +1,48 @@ +{ + "info": { + "subdir": "linux-64" + }, + "packages": { + "_libgcc_mutex-0.1-conda_forge.tar.bz2": { + "build": "conda_forge", + "build_number": 0, + "build_string": "conda_forge", + "constrains": null, + "depends": null, + "fn": "_libgcc_mutex-0.1-conda_forge.tar.bz2", + "license": "None", + "md5": "d7c89558ba9fa0495403155b64376d81", + "name": "_libgcc_mutex", + "sha256": "fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726", + "size": 2562, + "subdir": "linux-64", + "timestamp": 1578324546, + "track_features": "", + "url": "https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2", + "version": "0.1" + } + }, + "packages.conda": { + "bzip2-1.0.8-hd590300_5.conda": { + "build": "hd590300_5", + "build_number": 5, + "build_string": "hd590300_5", + "constrains": null, + "depends": [ + "libgcc-ng >=12" + ], + "fn": "bzip2-1.0.8-hd590300_5.conda", + "license": "bzip2-1.0.6", + "md5": "69b8b6202a07720f448be700e300ccf4", + "name": "bzip2", + "sha256": "242c0c324507ee172c0e0dd2045814e746bb303d1eb78870d182ceb0abc726a8", + "size": 254228, + "subdir": "linux-64", + "timestamp": 1699279927, + "track_features": "", + "url": "https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda", + "version": "1.0.8" + } + }, + "repodata_version": 2 +} diff --git a/libmamba/tests/data/repodata/conda-forge-repodata-version-2.json b/libmamba/tests/data/repodata/conda-forge-repodata-version-2.json new file mode 100644 index 0000000000..21554bd25a --- /dev/null +++ b/libmamba/tests/data/repodata/conda-forge-repodata-version-2.json @@ -0,0 +1,49 @@ +{ + "info": { + "base_url": "https://repo.anaconda.com/repo/main/linux-64/", + "subdir": "linux-64" + }, + "packages": { + "_libgcc_mutex-0.1-conda_forge.tar.bz2": { + "build": "conda_forge", + "build_number": 0, + "build_string": "conda_forge", + "constrains": null, + "depends": null, + "fn": "_libgcc_mutex-0.1-conda_forge.tar.bz2", + "license": "None", + "md5": "d7c89558ba9fa0495403155b64376d81", + "name": "_libgcc_mutex", + "sha256": "fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726", + "size": 2562, + "subdir": "linux-64", + "timestamp": 1578324546, + "track_features": "", + "url": "https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2", + "version": "0.1" + } + }, + "packages.conda": { + "bzip2-1.0.8-hd590300_5.conda": { + "build": "hd590300_5", + "build_number": 5, + "build_string": "hd590300_5", + "constrains": null, + "depends": [ + "libgcc-ng >=12" + ], + "fn": "bzip2-1.0.8-hd590300_5.conda", + "license": "bzip2-1.0.6", + "md5": "69b8b6202a07720f448be700e300ccf4", + "name": "bzip2", + "sha256": "242c0c324507ee172c0e0dd2045814e746bb303d1eb78870d182ceb0abc726a8", + "size": 254228, + "subdir": "linux-64", + "timestamp": 1699279927, + "track_features": "", + "url": "https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda", + "version": "1.0.8" + } + }, + "repodata_version": 2 +} diff --git a/libmamba/tests/src/solver/libsolv/test_database.cpp b/libmamba/tests/src/solver/libsolv/test_database.cpp index 579abc2551..d95a84b5a0 100644 --- a/libmamba/tests/src/solver/libsolv/test_database.cpp +++ b/libmamba/tests/src/solver/libsolv/test_database.cpp @@ -430,5 +430,57 @@ TEST_SUITE("solver::libsolv::database") ); } } + + SUBCASE("Add repo from repodata with repodata_version 2") + { + const auto repodata = mambatests::test_data_dir + / "repodata/conda-forge-repodata-version-2.json"; + auto repo1 = db.add_repo_from_repodata_json( + repodata, + "https://conda.anaconda.org/conda-forge/linux-64", + "conda-forge", + libsolv::PipAsPythonDependency::No, + libsolv::PackageTypes::CondaOrElseTarBz2 + ); + REQUIRE(repo1.has_value()); + CHECK_EQ(repo1->package_count(), 2); + + db.for_each_package_in_repo( + repo1.value(), + [&](const auto& p) + { + if (p.name == "_libgcc_mutex") + { + CHECK_EQ( + p.package_url, + "https://repo.anaconda.com/repo/main/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2" + ); + } + else if (p.name == "bzip2") + { + CHECK_EQ( + p.package_url, + "https://repo.anaconda.com/repo/main/linux-64/bzip2-1.0.8-hd590300_5.conda" + ); + } + } + ); + } + + SUBCASE("Add repo from repodata with repodata_version 2 with missing base_url") + { + const auto repodata = mambatests::test_data_dir + / "repodata/conda-forge-repodata-version-2-missing-base_url.json"; + CHECK_THROWS_AS( + db.add_repo_from_repodata_json( + repodata, + "https://conda.anaconda.org/conda-forge/linux-64", + "conda-forge", + libsolv::PipAsPythonDependency::No, + libsolv::PackageTypes::CondaOrElseTarBz2 + ), + std::invalid_argument + ); + } } }