From c79095119457c21ff1efabd5c50b5bbe9d3666fe Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 6 Mar 2023 00:17:05 +0100 Subject: [PATCH 01/14] initial metadata proposal --- conan/api/subapi/local.py | 2 + .../internal/cache/conan_reference_layout.py | 6 + conans/client/cmd/export.py | 1 + conans/client/cmd/uploader.py | 11 ++ conans/client/installer.py | 2 + conans/client/rest/rest_client_v2.py | 18 ++- conans/model/conan_file.py | 8 ++ conans/model/conf.py | 1 + conans/model/layout.py | 19 +++ conans/test/integration/metadata/__init__.py | 0 .../metadata/test_metadata_logs.py | 125 ++++++++++++++++++ .../test/integration/remote/rest_api_test.py | 6 +- 12 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 conans/test/integration/metadata/__init__.py create mode 100644 conans/test/integration/metadata/test_metadata_logs.py diff --git a/conan/api/subapi/local.py b/conan/api/subapi/local.py index a3925fddebf..ed45ce646a3 100644 --- a/conan/api/subapi/local.py +++ b/conan/api/subapi/local.py @@ -81,6 +81,7 @@ def source(self, path, name=None, version=None, user=None, channel=None): folder = conanfile.recipe_folder conanfile.folders.set_base_source(folder) conanfile.folders.set_base_export_sources(folder) + conanfile.folders.set_base_recipe_metadata(os.path.join(folder, "metadata")) conanfile.folders.set_base_build(None) conanfile.folders.set_base_package(None) @@ -92,6 +93,7 @@ def build(self, conanfile): """ app = ConanApp(self._conan_api.cache_folder) conanfile.folders.set_base_package(conanfile.folders.base_build) + conanfile.folders.set_base_pkg_metadata(os.path.join(conanfile.build_folder, "metadata")) run_build_method(conanfile, app.hook_manager) @staticmethod diff --git a/conan/internal/cache/conan_reference_layout.py b/conan/internal/cache/conan_reference_layout.py index 1412368b21c..a4c29ac39b4 100644 --- a/conan/internal/cache/conan_reference_layout.py +++ b/conan/internal/cache/conan_reference_layout.py @@ -50,6 +50,9 @@ def export(self): def export_sources(self): return os.path.join(self.base_folder, EXPORT_SRC_FOLDER) + def metadata(self): + return os.path.join(self.download_export(), "metadata") + def download_export(self): return os.path.join(self.base_folder, DOWNLOAD_EXPORT_FOLDER) @@ -112,6 +115,9 @@ def package(self): def download_package(self): return os.path.join(self.base_folder, DOWNLOAD_EXPORT_FOLDER) + def metadata(self): + return os.path.join(self.download_package(), "metadata") + def package_manifests(self): package_folder = self.package() readed_manifest = FileTreeManifest.load(package_folder) diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py index 35daa5cd0cf..d7f54384988 100644 --- a/conans/client/cmd/export.py +++ b/conans/client/cmd/export.py @@ -39,6 +39,7 @@ def cmd_export(app, conanfile_path, name, version, user, channel, graph_lock=Non # TODO: cache2.0 move this creation to other place mkdir(export_folder) mkdir(export_src_folder) + conanfile.folders.set_base_recipe_metadata(recipe_layout.metadata()) export_recipe(conanfile, export_folder) export_source(conanfile, export_src_folder) shutil.copy2(conanfile_path, recipe_layout.conanfile()) diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index 99468a5fd69..73f977667b0 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -153,8 +153,18 @@ def add_tgz(tgz_name, tgz_files, msg): add_tgz(EXPORT_TGZ_NAME, files, "Compressing recipe...") add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files, "Compressing recipe sources...") + self._gather_metadata(layout.metadata(), result) return result + @staticmethod + def _gather_metadata(folder, upload_files): + for root, _, files in os.walk(folder): + for f in files: + abs_path = os.path.join(root, f) + relpath = os.path.relpath(abs_path, folder) + path = os.path.join("metadata", relpath).replace("\\", "/") + upload_files[path] = abs_path + def _prepare_package(self, pref, prev_bundle): pkg_layout = self._app.cache.pkg_layout(pref) if pkg_layout.package_is_dirty(): @@ -162,6 +172,7 @@ def _prepare_package(self, pref, prev_bundle): "Remove it with 'conan remove %s -p=%s'" % (pref, pref.ref, pref.package_id)) cache_files = self._compress_package_files(pkg_layout, pref) + self._gather_metadata(pkg_layout.metadata(), cache_files) prev_bundle["files"] = cache_files def _compress_package_files(self, layout, pref): diff --git a/conans/client/installer.py b/conans/client/installer.py index fb975650a45..3a25d164d8c 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -145,6 +145,7 @@ def build_package(self, node, package_layout): conanfile.folders.set_base_package(base_package) # In local cache, generators folder always in build_folder conanfile.folders.set_base_generators(base_build) + conanfile.folders.set_base_pkg_metadata(package_layout.metadata()) if not skip_build: # In local cache, install folder always is build_folder @@ -186,6 +187,7 @@ def _install_source(self, node, remotes): conanfile.folders.set_base_source(source_folder) conanfile.folders.set_base_export_sources(source_folder) + conanfile.folders.set_base_recipe_metadata(recipe_layout.metadata()) config_source(export_source_folder, conanfile, self._hook_manager) @staticmethod diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index 50912d321a3..c32ae6b6119 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -3,7 +3,6 @@ import time from conan.api.output import ConanOutput - from conans.client.downloaders.caching_file_downloader import CachingFileDownloader from conans.client.rest.client_routes import ClientV2Router from conans.client.rest.file_uploader import FileUploader @@ -32,15 +31,20 @@ def router(self): def _get_file_list_json(self, url): data = self.get_json(url) # Discarding (.keys()) still empty metadata for files - data["files"] = list(data["files"].keys()) + # Damn windows conan-server returns paths with backslash + data["files"] = list(d.replace("\\", "/") for d in data["files"].keys()) return data def get_recipe(self, ref, dest_folder): url = self.router.recipe_snapshot(ref) data = self._get_file_list_json(url) files = data["files"] - if EXPORT_SOURCES_TGZ_NAME in files: - files.remove(EXPORT_SOURCES_TGZ_NAME) + + # Do not download by default the metadata files, except package signing + accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt"] + metadata = self._config.get("core.metadata:download", default=["sign"], check_type=list) + accepted_files.extend(f"metadata/{m}" for m in metadata) + files = [f for f in files if any(f.startswith(m) for m in accepted_files)] # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} @@ -70,6 +74,12 @@ def get_package(self, pref, dest_folder): data = self._get_file_list_json(url) files = data["files"] # If we didn't indicated reference, server got the latest, use absolute now, it's safer + # Do not download the metadata files, except package signing + accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt"] + metadata = self._config.get("core.metadata:download", default=["sign"], check_type=list) + accepted_files.extend(f"metadata/{m}" for m in metadata) + files = [f for f in files if any(f.startswith(m) for m in accepted_files)] + urls = {fn: self.router.package_file(pref, fn) for fn in files} self._download_and_save_files(urls, dest_folder, files) ret = {fn: os.path.join(dest_folder, fn) for fn in files} diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 173c406d1fa..592938a48cf 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -252,6 +252,14 @@ def build_folder(self): """ return self.folders.build_folder + @property + def recipe_metadata_folder(self): + return self.folders.recipe_metadata_folder + + @property + def pkg_metadata_folder(self): + return self.folders.pkg_metadata_folder + @property def build_path(self) -> Path: assert self.build_folder is not None, "`build_folder` is `None`" diff --git a/conans/model/conf.py b/conans/model/conf.py index cab702ddf58..06f6654b272 100644 --- a/conans/model/conf.py +++ b/conans/model/conf.py @@ -9,6 +9,7 @@ BUILT_IN_CONFS = { "core:required_conan_version": "Raise if current version does not match the defined range.", + "core.metadata:download": "List of metadata folders to download at install time", "core:non_interactive": "Disable interactive user input, raises error if input necessary", "core:default_profile": "Defines the default host profile ('default' by default)", "core:default_build_profile": "Defines the default build profile (None by default)", diff --git a/conans/model/layout.py b/conans/model/layout.py index e9149b001e3..5a6650e7e9f 100644 --- a/conans/model/layout.py +++ b/conans/model/layout.py @@ -43,6 +43,9 @@ def __init__(self): self._base_export = None self._base_export_sources = None + self._base_recipe_metadata = None + self._base_pkg_metadata = None + self.source = "" self.build = "" self.package = "" @@ -78,6 +81,8 @@ def set_base_folders(self, conanfile_folder, output_folder): self._base_build = output_folder or base_folder self._base_generators = output_folder or base_folder self._base_export_sources = output_folder or base_folder + self._base_recipe_metadata = base_folder + self._base_pkg_metadata = output_folder or base_folder @property def source_folder(self): @@ -103,6 +108,20 @@ def build_folder(self): return self._base_build return os.path.join(self._base_build, self.build) + @property + def recipe_metadata_folder(self): + return self._base_recipe_metadata + + def set_base_recipe_metadata(self, folder): + self._base_recipe_metadata = folder + + @property + def pkg_metadata_folder(self): + return self._base_pkg_metadata + + def set_base_pkg_metadata(self, folder): + self._base_pkg_metadata = folder + @property def base_build(self): return self._base_build diff --git a/conans/test/integration/metadata/__init__.py b/conans/test/integration/metadata/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conans/test/integration/metadata/test_metadata_logs.py b/conans/test/integration/metadata/test_metadata_logs.py new file mode 100644 index 00000000000..f5ed77c568e --- /dev/null +++ b/conans/test/integration/metadata/test_metadata_logs.py @@ -0,0 +1,125 @@ +import os +import textwrap + +import pytest + +from conans.model.recipe_ref import RecipeReference +from conans.test.utils.tools import TestClient +from conans.util.files import load, save + + +def test_metadata_logs(): + c = TestClient(default_server_user=True) + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save, copy + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + + def export(self): + copy(self, "*.log", src=self.recipe_folder, + dst=os.path.join(self.recipe_metadata_folder, "logs")) + + def source(self): + save(self, os.path.join(self.recipe_metadata_folder, "logs", "src.log"), "srclog!!") + + def build(self): + save(self, "mylogs.txt", "some logs!!!") + copy(self, "mylogs.txt", src=self.build_folder, + dst=os.path.join(self.pkg_metadata_folder, "logs")) + """) + c.save({"conanfile.py": conanfile, + "file.log": "log contents!"}) + c.run("create .") + # Test local cache looks good + ref = RecipeReference.loads("pkg/0.1") + ref_layout = c.get_latest_ref_layout(ref) + assert os.listdir(ref_layout.metadata()) == ["logs"] + assert os.listdir(os.path.join(ref_layout.metadata(), "logs")) == ["file.log", "src.log"] + assert load(os.path.join(ref_layout.metadata(), "logs", "file.log")) == "log contents!" + assert load(os.path.join(ref_layout.metadata(), "logs", "src.log")) == "srclog!!" + + pref = c.get_latest_package_reference(ref) + pref_layout = c.get_latest_pkg_layout(pref) + assert os.listdir(pref_layout.metadata()) == ["logs"] + assert os.listdir(os.path.join(pref_layout.metadata(), "logs")) == ["mylogs.txt"] + assert load(os.path.join(pref_layout.metadata(), "logs", "mylogs.txt")) == "some logs!!!" + + # Now upload everything + c.run("upload * -c -r=default") + assert "metadata/logs/file.log" in c.out + assert "metadata/logs/src.log" in c.out + assert "metadata/logs/mylogs.txt" in c.out + + c.run("remove * -c") + c.run("install --requires=pkg/0.1") # wont install metadata by default + assert not os.path.exists(ref_layout.metadata()) + assert not os.path.exists(pref_layout.metadata()) + + c.run("remove * -c") + save(c.cache.new_config_path, "core.metadata:download=['logs']") + c.run("install --requires=pkg/0.1") + assert load(os.path.join(ref_layout.metadata(), "logs", "file.log")) == "log contents!" + assert load(os.path.join(ref_layout.metadata(), "logs", "src.log")) == "srclog!!" + assert load(os.path.join(pref_layout.metadata(), "logs", "mylogs.txt")) == "some logs!!!" + + +def test_metadata_logs_local(): + c = TestClient() + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save, copy + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + + def layout(self): + self.folders.build = "mybuild" + self.folders.generators = "mybuild/generators" + + def export(self): + copy(self, "*.log", src=self.recipe_folder, + dst=os.path.join(self.recipe_metadata_folder, "logs")) + + def source(self): + save(self, os.path.join(self.recipe_metadata_folder, "logs", "src.log"), "srclog!!") + + def build(self): + save(self, "mylogs.txt", "some logs!!!") + copy(self, "mylogs.txt", src=self.build_folder, + dst=os.path.join(self.pkg_metadata_folder, "logs")) + """) + c.save({"conanfile.py": conanfile, + "file.log": "log contents!"}) + + c.run("source .") + assert c.load("metadata/logs/src.log") == "srclog!!" + c.run("build .") + assert c.load("mybuild/metadata/logs/mylogs.txt") == "some logs!!!" + + +@pytest.mark.skip(reason="just wip") +def test_sources_backup(): + c = TestClient(default_server_user=True) + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save, download, copy + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + + def source(self): + # Local "conan source"? use "conan metadata get pkg/0.1 srcs"? + self._conan_helpers.remote_manager.get_recipe_metadata(self.ref, "srcs") + download(self, "url") + save(self, os.path.join(self.recipe_metadata_folder, "srcs", "src.log"), "srclog!!") + """) + c.save({"conanfile.py": conanfile}) + c.run("create .") diff --git a/conans/test/integration/remote/rest_api_test.py b/conans/test/integration/remote/rest_api_test.py index 6e531d07310..e53f488da91 100644 --- a/conans/test/integration/remote/rest_api_test.py +++ b/conans/test/integration/remote/rest_api_test.py @@ -97,7 +97,7 @@ def test_get_package(self): # Get the package tmp_dir = temp_folder() self.api.get_package(pref, tmp_dir) - self.assertIn("hello.cpp", os.listdir(tmp_dir)) + self.assertIn("conanmanifest.txt", os.listdir(tmp_dir)) @pytest.mark.skipif(platform.system() != "Linux", reason="only Linux") def test_upload_huge_conan(self): @@ -198,9 +198,7 @@ def test_remove_packages(self): def _upload_package(self, package_reference, base_files=None): - files = {"conanfile.py": GenConanfile("3").with_requires("1", "12").with_exports("*"), - "hello.cpp": "hello", - "conanmanifest.txt": ""} + files = {"conanmanifest.txt": ""} if base_files: files.update(base_files) From 2661a825c2cfbf7f073afbb858c08a3cee7e9e62 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 6 Mar 2023 00:44:34 +0100 Subject: [PATCH 02/14] fix --- conans/test/integration/remote/rest_api_test.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/conans/test/integration/remote/rest_api_test.py b/conans/test/integration/remote/rest_api_test.py index e53f488da91..03ef4837b20 100644 --- a/conans/test/integration/remote/rest_api_test.py +++ b/conans/test/integration/remote/rest_api_test.py @@ -1,5 +1,4 @@ import os -import platform import unittest import pytest @@ -99,17 +98,6 @@ def test_get_package(self): self.api.get_package(pref, tmp_dir) self.assertIn("conanmanifest.txt", os.listdir(tmp_dir)) - @pytest.mark.skipif(platform.system() != "Linux", reason="only Linux") - def test_upload_huge_conan(self): - ref = RecipeReference.loads("conanhuge/1.0.0@private_user/testing#myreciperev") - files = {"file%s.cpp" % name: "File conent" for name in range(10)} - self._upload_recipe(ref, files) - - tmp = temp_folder() - files = self.api.get_recipe(ref, tmp) - self.assertIsNotNone(files) - self.assertTrue(os.path.exists(os.path.join(tmp, "file9.cpp"))) - def test_search(self): # Upload a conan1 conan_name1 = "HelloOnly/0.10@private_user/testing#myreciperev" From b1a0a723459d4f7199ab38c78f1d3a92ab11d549 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 8 Mar 2023 15:13:09 +0100 Subject: [PATCH 03/14] remove conf --- conans/model/conf.py | 1 - .../editable/test_editable_python_requires.py | 41 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 conans/test/integration/editable/test_editable_python_requires.py diff --git a/conans/model/conf.py b/conans/model/conf.py index 06f6654b272..cab702ddf58 100644 --- a/conans/model/conf.py +++ b/conans/model/conf.py @@ -9,7 +9,6 @@ BUILT_IN_CONFS = { "core:required_conan_version": "Raise if current version does not match the defined range.", - "core.metadata:download": "List of metadata folders to download at install time", "core:non_interactive": "Disable interactive user input, raises error if input necessary", "core:default_profile": "Defines the default host profile ('default' by default)", "core:default_build_profile": "Defines the default build profile (None by default)", diff --git a/conans/test/integration/editable/test_editable_python_requires.py b/conans/test/integration/editable/test_editable_python_requires.py new file mode 100644 index 00000000000..e65d69ab33f --- /dev/null +++ b/conans/test/integration/editable/test_editable_python_requires.py @@ -0,0 +1,41 @@ +import os +import re +import textwrap + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.tools import TestClient + + +def test_editable_python_requires(): + c = TestClient() + dep = textwrap.dedent(""" + from conan import ConanFile + + number = 42 + + class Dep(ConanFile): + name = "dep" + version = "1.0" + """) + pkg = textwrap.dedent(""" + from conan import ConanFile + + number = 42 + + class Dep(ConanFile): + name = "pkg" + version = "1.0" + python_requires = "dep/1.0" + + def generate(self): + self.output.info(f"NUMBER GEN: {self.python_requires['dep'].module.number}") + def build(self): + self.output.info(f"NUMBER BUILD: {self.python_requires['dep'].module.number}") + """) + + c.save({"dep/conanfile.py": dep, + "pkg/conanfile.py": pkg}) + c.run("editable add dep") + c.run("build pkg") + assert "conanfile.py (pkg/1.0): NUMBER GEN: 42" in c.out + assert "conanfile.py (pkg/1.0): NUMBER BUILD: 42" in c.out From 2a840636cc33d6f64323af84b08a8df5e419c1ef Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 8 Mar 2023 16:46:47 +0100 Subject: [PATCH 04/14] hooks tests --- .../metadata/test_metadata_logs.py | 157 ++++++++++++------ 1 file changed, 106 insertions(+), 51 deletions(-) diff --git a/conans/test/integration/metadata/test_metadata_logs.py b/conans/test/integration/metadata/test_metadata_logs.py index ee364a69f4b..391ff6e6a3e 100644 --- a/conans/test/integration/metadata/test_metadata_logs.py +++ b/conans/test/integration/metadata/test_metadata_logs.py @@ -8,8 +8,8 @@ from conans.util.files import load, save -def test_metadata_logs(): - c = TestClient(default_server_user=True) +class TestRecipeMetadataLogs: + conanfile = textwrap.dedent(""" import os from conan import ConanFile @@ -21,65 +21,120 @@ class Pkg(ConanFile): def export(self): copy(self, "*.log", src=self.recipe_folder, - dst=os.path.join(self.recipe_metadata_folder, "logs")) - - def source(self): - save(self, os.path.join(self.recipe_metadata_folder, "logs", "src.log"), "srclog!!") - - def build(self): - save(self, "mylogs.txt", "some logs!!!") - copy(self, "mylogs.txt", src=self.build_folder, - dst=os.path.join(self.pkg_metadata_folder, "logs")) - """) - c.save({"conanfile.py": conanfile, - "file.log": "log contents!"}) - c.run("create .") - # Test local cache looks good - ref = RecipeReference.loads("pkg/0.1") - ref_layout = c.get_latest_ref_layout(ref) - assert os.listdir(ref_layout.metadata()) == ["logs"] - assert os.listdir(os.path.join(ref_layout.metadata(), "logs")) == ["file.log", "src.log"] - assert load(os.path.join(ref_layout.metadata(), "logs", "file.log")) == "log contents!" - assert load(os.path.join(ref_layout.metadata(), "logs", "src.log")) == "srclog!!" - - pref = c.get_latest_package_reference(ref) - pref_layout = c.get_latest_pkg_layout(pref) - assert os.listdir(pref_layout.metadata()) == ["logs"] - assert os.listdir(os.path.join(pref_layout.metadata(), "logs")) == ["mylogs.txt"] - assert load(os.path.join(pref_layout.metadata(), "logs", "mylogs.txt")) == "some logs!!!" - - -def test_metadata_logs_local(): - c = TestClient() - conanfile = textwrap.dedent(""" - import os - from conan import ConanFile - from conan.tools.files import save, copy - - class Pkg(ConanFile): - name = "pkg" - version = "0.1" + dst=os.path.join(self.recipe_metadata_folder, "logs")) def layout(self): self.folders.build = "mybuild" self.folders.generators = "mybuild/generators" - def export(self): - copy(self, "*.log", src=self.recipe_folder, - dst=os.path.join(self.recipe_metadata_folder, "logs")) - def source(self): save(self, os.path.join(self.recipe_metadata_folder, "logs", "src.log"), "srclog!!") def build(self): save(self, "mylogs.txt", "some logs!!!") copy(self, "mylogs.txt", src=self.build_folder, - dst=os.path.join(self.pkg_metadata_folder, "logs")) + dst=os.path.join(self.pkg_metadata_folder, "logs")) """) - c.save({"conanfile.py": conanfile, - "file.log": "log contents!"}) - c.run("source .") - assert c.load("metadata/logs/src.log") == "srclog!!" - c.run("build .") - assert c.load("mybuild/metadata/logs/mylogs.txt") == "some logs!!!" + def test_metadata_logs(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": self.conanfile, + "file.log": "log contents!"}) + c.run("create .") + # Test local cache looks good + ref = RecipeReference.loads("pkg/0.1") + ref_layout = c.get_latest_ref_layout(ref) + assert os.listdir(ref_layout.metadata()) == ["logs"] + assert os.listdir(os.path.join(ref_layout.metadata(), "logs")) == ["file.log", "src.log"] + assert load(os.path.join(ref_layout.metadata(), "logs", "file.log")) == "log contents!" + assert load(os.path.join(ref_layout.metadata(), "logs", "src.log")) == "srclog!!" + + pref = c.get_latest_package_reference(ref) + pref_layout = c.get_latest_pkg_layout(pref) + assert os.listdir(pref_layout.metadata()) == ["logs"] + assert os.listdir(os.path.join(pref_layout.metadata(), "logs")) == ["mylogs.txt"] + assert load(os.path.join(pref_layout.metadata(), "logs", "mylogs.txt")) == "some logs!!!" + + def test_metadata_logs_local(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": self.conanfile, + "file.log": "log contents!"}) + c.run("source .") + assert c.load("metadata/logs/src.log") == "srclog!!" + c.run("build .") + assert c.load("mybuild/metadata/logs/mylogs.txt") == "some logs!!!" + + +class TestHooksMetadataLogs: + + @pytest.fixture() + def _client(self): + c = TestClient(default_server_user=True) + my_hook = textwrap.dedent("""\ + import os + from conan.tools.files import copy + + def post_export(conanfile): + conanfile.output.info("post_export") + copy(conanfile, "*.log", src=conanfile.recipe_folder, + dst=os.path.join(conanfile.recipe_metadata_folder, "logs")) + + def post_source(conanfile): + conanfile.output.info("post_source") + copy(conanfile, "*", src=os.path.join(conanfile.source_folder, "logs"), + dst=os.path.join(conanfile.recipe_metadata_folder, "logs")) + + def post_build(conanfile): + conanfile.output.info("post_build") + copy(conanfile, "*", src=os.path.join(conanfile.build_folder, "logs"), + dst=os.path.join(conanfile.pkg_metadata_folder, "logs")) + """) + hook_path = os.path.join(c.cache.hooks_path, "my_hook", "hook_my_hook.py") + save(hook_path, my_hook) + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save, copy + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + no_copy_source = True + + def layout(self): + self.folders.build = "mybuild" + self.folders.generators = "mybuild/generators" + + def source(self): + save(self, "logs/src.log", "srclog!!") + + def build(self): + save(self, "logs/mylogs.txt", "some logs!!!") + """) + c.save({"conanfile.py": conanfile, + "file.log": "log contents!"}) + return c + + def test_metadata_logs_hook(self, _client): + c = _client + c.run("create .") + # Test local cache looks good + ref = RecipeReference.loads("pkg/0.1") + ref_layout = c.get_latest_ref_layout(ref) + assert os.listdir(ref_layout.metadata()) == ["logs"] + assert os.listdir(os.path.join(ref_layout.metadata(), "logs")) == ["file.log", "src.log"] + assert load(os.path.join(ref_layout.metadata(), "logs", "file.log")) == "log contents!" + assert load(os.path.join(ref_layout.metadata(), "logs", "src.log")) == "srclog!!" + + pref = c.get_latest_package_reference(ref) + pref_layout = c.get_latest_pkg_layout(pref) + assert os.listdir(pref_layout.metadata()) == ["logs"] + assert os.listdir(os.path.join(pref_layout.metadata(), "logs")) == ["mylogs.txt"] + assert load(os.path.join(pref_layout.metadata(), "logs", "mylogs.txt")) == "some logs!!!" + + def test_metadata_logs_local(self, _client): + c = _client + c.run("source .") + assert c.load("metadata/logs/src.log") == "srclog!!" + c.run("build .") + assert c.load("mybuild/metadata/logs/mylogs.txt") == "some logs!!!" From 6971249baa5804a61e354502e750a5fbde45d62c Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 8 Mar 2023 17:00:53 +0100 Subject: [PATCH 05/14] removed test --- .../editable/test_editable_python_requires.py | 41 ------------------- 1 file changed, 41 deletions(-) delete mode 100644 conans/test/integration/editable/test_editable_python_requires.py diff --git a/conans/test/integration/editable/test_editable_python_requires.py b/conans/test/integration/editable/test_editable_python_requires.py deleted file mode 100644 index e65d69ab33f..00000000000 --- a/conans/test/integration/editable/test_editable_python_requires.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import re -import textwrap - -from conans.test.assets.genconanfile import GenConanfile -from conans.test.utils.tools import TestClient - - -def test_editable_python_requires(): - c = TestClient() - dep = textwrap.dedent(""" - from conan import ConanFile - - number = 42 - - class Dep(ConanFile): - name = "dep" - version = "1.0" - """) - pkg = textwrap.dedent(""" - from conan import ConanFile - - number = 42 - - class Dep(ConanFile): - name = "pkg" - version = "1.0" - python_requires = "dep/1.0" - - def generate(self): - self.output.info(f"NUMBER GEN: {self.python_requires['dep'].module.number}") - def build(self): - self.output.info(f"NUMBER BUILD: {self.python_requires['dep'].module.number}") - """) - - c.save({"dep/conanfile.py": dep, - "pkg/conanfile.py": pkg}) - c.run("editable add dep") - c.run("build pkg") - assert "conanfile.py (pkg/1.0): NUMBER GEN: 42" in c.out - assert "conanfile.py (pkg/1.0): NUMBER BUILD: 42" in c.out From 2a246d030b5599cd4837a9595a053fb98a33395d Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 9 Mar 2023 09:12:23 +0100 Subject: [PATCH 06/14] wip --- conan/cli/commands/metadata.py | 31 +++++++++++++++++++ .../metadata/test_metadata_logs.py | 20 ++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 conan/cli/commands/metadata.py diff --git a/conan/cli/commands/metadata.py b/conan/cli/commands/metadata.py new file mode 100644 index 00000000000..688634cf39b --- /dev/null +++ b/conan/cli/commands/metadata.py @@ -0,0 +1,31 @@ +import os +import shutil + +from conan.cli.command import conan_command, conan_subcommand +from conan.internal.conan_app import ConanApp +from conans.model.recipe_ref import RecipeReference + + +@conan_command(group='Misc') +def metadata(conan_api, parser, *args): + """ + Manage the recipes and packages metadata files + """ + + +@conan_subcommand() +def metadata_add(conan_api, parser, subparser, *args): + """ + Add metadata to packages in cache + """ + subparser.add_argument("ref", help="reference of recipe or package binary") + subparser.add_argument("-s", "--src", help='File or folder to add') + subparser.add_argument("-d", "--dst", help='Folder inside metadata') + args = parser.parse_args(*args) + + app = ConanApp(conan_api.cache_folder) + ref = RecipeReference.loads(args.ref) + if ref.revision is None: + ref = app.cache.get_latest_recipe_reference(ref) + folder = app.cache.ref_layout(ref).metadata() + shutil.copytree(args.src, os.path.join(folder, args.dst, args.src)) diff --git a/conans/test/integration/metadata/test_metadata_logs.py b/conans/test/integration/metadata/test_metadata_logs.py index 391ff6e6a3e..c2726abaf53 100644 --- a/conans/test/integration/metadata/test_metadata_logs.py +++ b/conans/test/integration/metadata/test_metadata_logs.py @@ -3,7 +3,10 @@ import pytest +from conan.tools.files.files import untargz from conans.model.recipe_ref import RecipeReference +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient from conans.util.files import load, save @@ -138,3 +141,20 @@ def test_metadata_logs_local(self, _client): assert c.load("metadata/logs/src.log") == "srclog!!" c.run("build .") assert c.load("mybuild/metadata/logs/mylogs.txt") == "some logs!!!" + + +class TestMetadataTestPackage: + + def test(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1"), + "test_package/conanfile.py": GenConanfile().with_test("pass")}) + c.run("create .") + c.run("metadata add pkg/0.1 --src=test_package --dst=tps") + # Test local cache looks good + ref = RecipeReference.loads("pkg/0.1") + ref_layout = c.get_latest_ref_layout(ref) + assert os.listdir(ref_layout.metadata()) == ["tps"] + tgz = os.path.join(ref_layout.metadata(), "tps", "test_package") + test_conanfile = load(os.path.join(tgz, "conanfile.py")) + assert "class HelloConan(ConanFile):" in test_conanfile From 6a8e2cfa2358caf6b807c136916230d8d501d235 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 9 Mar 2023 18:13:08 +0100 Subject: [PATCH 07/14] wip --- conan/cli/commands/metadata.py | 19 +++++++++++++++++++ .../metadata/test_metadata_logs.py | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/conan/cli/commands/metadata.py b/conan/cli/commands/metadata.py index 688634cf39b..f5eca0165ef 100644 --- a/conan/cli/commands/metadata.py +++ b/conan/cli/commands/metadata.py @@ -29,3 +29,22 @@ def metadata_add(conan_api, parser, subparser, *args): ref = app.cache.get_latest_recipe_reference(ref) folder = app.cache.ref_layout(ref).metadata() shutil.copytree(args.src, os.path.join(folder, args.dst, args.src)) + + +@conan_subcommand() +def metadata_get(conan_api, parser, subparser, *args): + """ + Add metadata to packages in cache + """ + subparser.add_argument("ref", help="reference of recipe or package binary") + subparser.add_argument("-s", "--src", help='File or folder to add') + subparser.add_argument("-d", "--dst", help='Folder inside metadata') + args = parser.parse_args(*args) + + app = ConanApp(conan_api.cache_folder) + ref = RecipeReference.loads(args.ref) + if ref.revision is None: + ref = app.cache.get_latest_recipe_reference(ref) + folder = app.cache.ref_layout(ref).metadata() + src = os.path.join(folder, args.src) + shutil.copytree(src, args.dst) diff --git a/conans/test/integration/metadata/test_metadata_logs.py b/conans/test/integration/metadata/test_metadata_logs.py index c2726abaf53..dc573eb81cb 100644 --- a/conans/test/integration/metadata/test_metadata_logs.py +++ b/conans/test/integration/metadata/test_metadata_logs.py @@ -158,3 +158,8 @@ def test(self): tgz = os.path.join(ref_layout.metadata(), "tps", "test_package") test_conanfile = load(os.path.join(tgz, "conanfile.py")) assert "class HelloConan(ConanFile):" in test_conanfile + + c.save({}, clean_first=True) + c.run("metadata get pkg/0.1 --src=tps/test_package --dst=test_package") + c.run("test test_package pkg/0.1") + print(c.out) From 5c49a7398e027b5579b36defaa449147e8d8b697 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 18 May 2023 11:11:35 +0200 Subject: [PATCH 08/14] commands wip --- conan/api/subapi/download.py | 10 ++-- conan/api/subapi/upload.py | 5 +- conan/cli/commands/download.py | 6 ++- conan/cli/commands/upload.py | 4 +- conan/internal/upload_metadata.py | 27 ++++++++--- conans/client/installer.py | 3 +- conans/client/remote_manager.py | 46 ++++++++++++++++--- conans/client/rest/rest_client.py | 8 ++-- conans/client/rest/rest_client_v2.py | 20 ++++++-- .../metadata/test_metadata_commands.py | 40 +++++++++++++--- 10 files changed, 132 insertions(+), 37 deletions(-) diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index f2353b8b7b0..b364cb2aed6 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -12,16 +12,18 @@ class DownloadAPI: def __init__(self, conan_api): self.conan_api = conan_api - def recipe(self, ref: RecipeReference, remote: Remote): + def recipe(self, ref: RecipeReference, remote: Remote, metadata=None): output = ConanOutput() app = ConanApp(self.conan_api.cache_folder) skip_download = app.cache.exists_rrev(ref) if skip_download: output.info(f"Skip recipe {ref.repr_notime()} download, already in cache") + if metadata: + app.remote_manager.get_recipe_metadata(ref, remote, metadata) return False output.info(f"Downloading recipe '{ref.repr_notime()}'") - app.remote_manager.get_recipe(ref, remote) + app.remote_manager.get_recipe(ref, remote, metadata) layout = app.cache.ref_layout(ref) conan_file_path = layout.conanfile() @@ -32,7 +34,7 @@ def recipe(self, ref: RecipeReference, remote: Remote): retrieve_exports_sources(app.remote_manager, layout, conanfile, ref, [remote]) return True - def package(self, pref: PkgReference, remote: Remote): + def package(self, pref: PkgReference, remote: Remote, metadata=None): output = ConanOutput() app = ConanApp(self.conan_api.cache_folder) if not app.cache.exists_rrev(pref.ref): @@ -42,6 +44,8 @@ def package(self, pref: PkgReference, remote: Remote): skip_download = app.cache.exists_prev(pref) if skip_download: output.info(f"Skip package {pref.repr_notime()} download, already in cache") + if metadata: + app.remote_manager.get_package_metadata(pref, remote, metadata) return False layout = app.cache.ref_layout(pref.ref) conan_file_path = layout.conanfile() diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 2364313758b..3f4fc32d8c2 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -30,7 +30,7 @@ def check_upstream(self, package_list, remote, force=False): UploadUpstreamChecker(app).check(package_list, remote, force) - def prepare(self, package_list, enabled_remotes): + def prepare(self, package_list, enabled_remotes, metadata=None): """Compress the recipes and packages and fill the upload_data objects with the complete information. It doesn't perform the upload nor checks upstream to see if the recipe is still there""" @@ -40,7 +40,8 @@ def prepare(self, package_list, enabled_remotes): signer = PkgSignaturesPlugin(app.cache) # This might add files entries to package_list with signatures signer.sign(package_list) - gather_metadata(package_list, app.cache) + # gather metadata happens after the package is created and signed + gather_metadata(package_list, app.cache, metadata) def upload(self, package_list, remote): app = ConanApp(self.conan_api.cache_folder) diff --git a/conan/cli/commands/download.py b/conan/cli/commands/download.py index 016a6e4415a..a237e26c7ef 100644 --- a/conan/cli/commands/download.py +++ b/conan/cli/commands/download.py @@ -27,6 +27,8 @@ def download(conan_api: ConanAPI, parser, *args): "(arch=x86 OR compiler=gcc)") parser.add_argument("-r", "--remote", action=OnceArgument, required=True, help='Download from this specific remote') + parser.add_argument("--metadata", action='append', + help='Upload the metadata, even if the package is not uploaded') args = parser.parse_args(*args) remote = conan_api.remotes.get(args.remote) @@ -42,9 +44,9 @@ def download(conan_api: ConanAPI, parser, *args): if parallel <= 1: for ref in refs: - conan_api.download.recipe(ref, remote) + conan_api.download.recipe(ref, remote, args.metadata) for pref in prefs: - conan_api.download.package(pref, remote) + conan_api.download.package(pref, remote, args.metadata) else: _download_parallel(parallel, conan_api, refs, prefs, remote) diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 27a04591e67..594cd07e8ff 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -33,6 +33,8 @@ def upload(conan_api: ConanAPI, parser, *args): help='Perform an integrity check, using the manifests, before upload') parser.add_argument('-c', '--confirm', default=False, action='store_true', help='Upload all matching recipes without confirmation') + parser.add_argument("--metadata", action='append', + help='Upload the metadata, even if the package is not uploaded') args = parser.parse_args(*args) @@ -54,7 +56,7 @@ def upload(conan_api: ConanAPI, parser, *args): if not args.confirm and "*" in args.reference: _ask_confirm_upload(conan_api, package_list) - conan_api.upload.prepare(package_list, enabled_remotes) + conan_api.upload.prepare(package_list, enabled_remotes, args.metadata) conan_api.upload.upload(package_list, remote) conan_api.upload.upload_backup_sources(package_list) diff --git a/conan/internal/upload_metadata.py b/conan/internal/upload_metadata.py index a1ec91ea704..72d6289b1fd 100644 --- a/conan/internal/upload_metadata.py +++ b/conan/internal/upload_metadata.py @@ -1,23 +1,36 @@ +import fnmatch import os -def _metadata_files(folder, upload_files): +def _metadata_files(folder, metadata): + result = {} for root, _, files in os.walk(folder): for f in files: abs_path = os.path.join(root, f) relpath = os.path.relpath(abs_path, folder) + if metadata: + if not any(fnmatch.fnmatch(relpath, m) for m in metadata): + continue path = os.path.join("metadata", relpath).replace("\\", "/") - upload_files[path] = abs_path + result[path] = abs_path + return result -def gather_metadata(upload_data, cache): +def gather_metadata(upload_data, cache, metadata): for rref, recipe_bundle in upload_data.refs(): - if recipe_bundle["upload"]: + if metadata or recipe_bundle["upload"]: metadata_folder = cache.ref_layout(rref).metadata() assert metadata_folder - _metadata_files(metadata_folder, recipe_bundle["files"]) + files = _metadata_files(metadata_folder, metadata) + if files: + recipe_bundle.setdefault("files", {}).update(files) + recipe_bundle["upload"] = True + for pref, pkg_bundle in upload_data.prefs(rref, recipe_bundle): - if pkg_bundle["upload"]: + if metadata or pkg_bundle["upload"]: metadata_folder = cache.pkg_layout(pref).metadata() assert metadata_folder - _metadata_files(metadata_folder, pkg_bundle["files"]) + files = _metadata_files(metadata_folder, metadata) + if files: + pkg_bundle.setdefault("files", {}).update(files) + pkg_bundle["upload"] = True diff --git a/conans/client/installer.py b/conans/client/installer.py index 56587ee9f86..56513014e15 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -285,7 +285,8 @@ def _download_pkg(self, package): node = package.nodes[0] assert node.pref.revision is not None assert node.pref.timestamp is not None - self._remote_manager.get_package(node.conanfile, node.pref, node.binary_remote) + self._remote_manager.get_package(node.conanfile, node.pref, node.binary_remote, + metadata=None) def _handle_package(self, package, install_reference, remotes, handled_count, total_count): if package.binary == BINARY_SYSTEM_TOOL: diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 534cb659164..976d8e41d6c 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -39,7 +39,7 @@ def upload_package(self, pref, files_to_upload, remote): assert pref.revision, "upload_package requires PREV" self._call_remote(remote, "upload_package", pref, files_to_upload) - def get_recipe(self, ref, remote): + def get_recipe(self, ref, remote, metadata=None): """ Read the conans from remotes Will iterate the remotes to find the conans unless remote was specified @@ -53,7 +53,9 @@ def get_recipe(self, ref, remote): download_export = layout.download_export() try: - zipped_files = self._call_remote(remote, "get_recipe", ref, download_export) + zipped_files = self._call_remote(remote, "get_recipe", ref, download_export, metadata, + only_metadata=False) + # TODO: Optimize this call, it is slow to always query all revisions remote_refs = self._call_remote(remote, "get_recipe_revisions_references", ref) ref_time = remote_refs[0].timestamp ref.timestamp = ref_time @@ -83,6 +85,20 @@ def get_recipe(self, ref, remote): # Make sure that the source dir is deleted rmdir(layout.source()) + def get_recipe_metadata(self, ref, remote, metadata): + """ + Get only the metadata for a locally existing recipe in Cache + """ + assert ref.revision, "get_recipe without revision specified" + layout = self._cache.ref_layout(ref) + download_export = layout.download_export() + try: + self._call_remote(remote, "get_recipe", ref, download_export, metadata, + only_metadata=True) + except BaseException: # So KeyboardInterrupt also cleans things + ConanOutput(scope=str(ref)).error(f"Error downloading metadata from remote '{remote.name}'") + raise + def get_recipe_sources(self, ref, layout, remote): assert ref.revision, "get_recipe_sources requires RREV" @@ -96,7 +112,7 @@ def get_recipe_sources(self, ref, layout, remote): tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] uncompress_file(tgz_file, export_sources_folder) - def get_package(self, conanfile, pref, remote): + def get_package(self, conanfile, pref, remote, metadata=None): conanfile.output.info("Retrieving package %s from remote '%s' " % (pref.package_id, remote.name)) @@ -105,15 +121,33 @@ def get_package(self, conanfile, pref, remote): pkg_layout = self._cache.get_or_create_pkg_layout(pref) pkg_layout.package_remove() # Remove first the destination folder with pkg_layout.set_dirty_context_manager(): - self._get_package(pkg_layout, pref, remote, conanfile.output) + self._get_package(pkg_layout, pref, remote, conanfile.output, metadata) + + def get_package_metadata(self, pref, remote, metadata): + output = ConanOutput(scope=str(pref)) + output.info("Retrieving package metadata %s from remote '%s' " + % (pref.package_id, remote.name)) + + assert pref.revision is not None + pkg_layout = self._cache.pkg_layout(pref) + try: + download_pkg_folder = pkg_layout.download_package() + # Download files to the pkg_tgz folder, not to the final one + self._call_remote(remote, "get_package", pref, download_pkg_folder, + metadata, only_metadata=True) + except BaseException as e: # So KeyboardInterrupt also cleans things + output.error("Exception while getting package metadata: %s" % str(pref.package_id)) + output.error("Exception: %s %s" % (type(e), str(e))) + raise - def _get_package(self, layout, pref, remote, scoped_output): + def _get_package(self, layout, pref, remote, scoped_output, metadata): try: assert pref.revision is not None download_pkg_folder = layout.download_package() # Download files to the pkg_tgz folder, not to the final one - zipped_files = self._call_remote(remote, "get_package", pref, download_pkg_folder) + zipped_files = self._call_remote(remote, "get_package", pref, download_pkg_folder, + metadata, only_metadata=False) zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)} # quick server package integrity check: for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"): diff --git a/conans/client/rest/rest_client.py b/conans/client/rest/rest_client.py index ebfe2505503..ab26bde4079 100644 --- a/conans/client/rest/rest_client.py +++ b/conans/client/rest/rest_client.py @@ -60,14 +60,14 @@ def _get_api(self): self._requester, self._config, self._verify_ssl, checksum_deploy) - def get_recipe(self, ref, dest_folder): - return self._get_api().get_recipe(ref, dest_folder) + def get_recipe(self, ref, dest_folder, metadata, only_metadata): + return self._get_api().get_recipe(ref, dest_folder, metadata, only_metadata) def get_recipe_sources(self, ref, dest_folder): return self._get_api().get_recipe_sources(ref, dest_folder) - def get_package(self, pref, dest_folder): - return self._get_api().get_package(pref, dest_folder) + def get_package(self, pref, dest_folder, metadata, only_metadata): + return self._get_api().get_package(pref, dest_folder, metadata, only_metadata) def upload_recipe(self, ref, files_to_upload): return self._get_api().upload_recipe(ref, files_to_upload) diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index 0b2ccfada3c..cca379c81be 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -1,4 +1,5 @@ import copy +import fnmatch import os from conan.api.output import ConanOutput @@ -35,12 +36,17 @@ def _get_file_list_json(self, url): data["files"] = list(d.replace("\\", "/") for d in data["files"].keys()) return data - def get_recipe(self, ref, dest_folder): + def get_recipe(self, ref, dest_folder, metadata, only_metadata): url = self.router.recipe_snapshot(ref) data = self._get_file_list_json(url) files = data["files"] accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt", "metadata/sign"] - files = [f for f in files if any(f.startswith(m) for m in accepted_files)] + if only_metadata: + accepted_files = [] + metadata = metadata or [] + metadata = [f"metadata/{m}" for m in metadata] + files = [f for f in files if any(f.startswith(m) for m in accepted_files) + or any(fnmatch.fnmatch(f, m) for m in metadata)] # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} @@ -65,13 +71,19 @@ def get_recipe_sources(self, ref, dest_folder): ret = {fn: os.path.join(dest_folder, fn) for fn in files} return ret - def get_package(self, pref, dest_folder): + def get_package(self, pref, dest_folder, metadata, only_metadata): url = self.router.package_snapshot(pref) data = self._get_file_list_json(url) files = data["files"] # Download only known files, but not metadata (except sign) accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt", "metadata/sign"] - files = [f for f in files if any(f.startswith(m) for m in accepted_files)] + if only_metadata: + accepted_files = [] + metadata = metadata or [] + metadata = [f"metadata/{m}" for m in metadata] + files = [f for f in files if any(f.startswith(m) for m in accepted_files) + or any(fnmatch.fnmatch(f, m) for m in metadata)] + # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.package_file(pref, fn) for fn in files} self._download_and_save_files(urls, dest_folder, files) diff --git a/conans/test/integration/metadata/test_metadata_commands.py b/conans/test/integration/metadata/test_metadata_commands.py index f41e986410a..a466f3b1b3e 100644 --- a/conans/test/integration/metadata/test_metadata_commands.py +++ b/conans/test/integration/metadata/test_metadata_commands.py @@ -31,16 +31,42 @@ def test_upload(self): assert "metadata/logs/mylogs.txt" in c.out assert "metadata/logs/mybuildlogs.txt" in c.out + # Add new files to the metadata + myfile = os.path.join(metadata_path, "logs", "mylogs2.txt") + save(myfile, "mylogs2!!!!") + myfile = os.path.join(pkg_metadata_path, "logs", "mybuildlogs2.txt") + save(myfile, "mybuildlogs2!!!!") + # Upload the metadata, even if the revisions exist in the server + # adding the new metadata logs files + c.run("upload * -c -r=default --metadata=*") + assert "metadata/logs/mylogs.txt" in c.out + assert "metadata/logs/mybuildlogs.txt" in c.out + assert "metadata/logs/mylogs2.txt" in c.out + assert "metadata/logs/mybuildlogs2.txt" in c.out + c.run("remove * -c") c.run("install --requires=pkg/0.1") # wont install metadata by default c.run("cache path pkg/0.1 --folder=metadata") + metadata_path = str(c.stdout).strip() + c.run(f"cache path pkg/0.1:{pid} --folder=metadata") + pkg_metadata_path = str(c.stdout).strip() assert not os.path.exists(metadata_path) assert not os.path.exists(pkg_metadata_path) - """c.run("remove * -c") - save(c.cache.new_config_path, "core.metadata:download=['logs']") - c.run("install --requires=pkg/0.1") - assert load(os.path.join(ref_layout.metadata(), "logs", "file.log")) == "log contents!" - assert load(os.path.join(ref_layout.metadata(), "logs", "src.log")) == "srclog!!" - assert load(os.path.join(pref_layout.metadata(), "logs", "mylogs.txt")) == "some logs!!!" -""" + # Forcing the download of the metadata of cache-existing things with the "download" command + c.run("download pkg/0.1 -r=default --metadata=*") + for f in "logs/mylogs.txt", "logs/mylogs2.txt": + assert os.path.isfile(os.path.join(metadata_path, f)) + for f in "logs/mybuildlogs.txt", "logs/mybuildlogs2.txt": + assert os.path.isfile(os.path.join(pkg_metadata_path, f)) + + """# Regular install can also fetch metadata + c.run("remove * -c") # If done cleanly, not incrementally + assert not os.path.exists(metadata_path) + assert not os.path.exists(pkg_metadata_path) + c.run("install --requires=pkg/0.1 --metadata=*") + for f in "logs/mylogs.txt", "logs/mylogs2.txt": + assert os.path.isfile(os.path.join(metadata_path, f)) + for f in "logs/mybuildlogs.txt", "logs/mybuildlogs2.txt": + assert os.path.isfile(os.path.join(pkg_metadata_path, f)) + """ From 8d5953342bd0d9fa7c5a06740a0ef76aeebdc1a2 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 18 May 2023 11:48:51 +0200 Subject: [PATCH 09/14] wip --- conan/api/subapi/upload.py | 5 ++--- conan/cli/commands/cache.py | 2 +- conans/client/remote_manager.py | 11 ++++++++--- .../integration/metadata/test_metadata_commands.py | 11 ----------- conans/test/integration/remote/rest_api_test.py | 6 +++--- conans/test/integration/remote/token_refresh_test.py | 6 ++++-- 6 files changed, 18 insertions(+), 23 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 3f4fc32d8c2..5b1e9a40850 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -30,18 +30,17 @@ def check_upstream(self, package_list, remote, force=False): UploadUpstreamChecker(app).check(package_list, remote, force) - def prepare(self, package_list, enabled_remotes, metadata=None): + def prepare(self, package_list, enabled_remotes, metadata): """Compress the recipes and packages and fill the upload_data objects with the complete information. It doesn't perform the upload nor checks upstream to see if the recipe is still there""" app = ConanApp(self.conan_api.cache_folder) preparator = PackagePreparator(app) preparator.prepare(package_list, enabled_remotes) + gather_metadata(package_list, app.cache, metadata) signer = PkgSignaturesPlugin(app.cache) # This might add files entries to package_list with signatures signer.sign(package_list) - # gather metadata happens after the package is created and signed - gather_metadata(package_list, app.cache, metadata) def upload(self, package_list, remote): app = ConanApp(self.conan_api.cache_folder) diff --git a/conan/cli/commands/cache.py b/conan/cli/commands/cache.py index 9308d43e2f7..582636cf797 100644 --- a/conan/cli/commands/cache.py +++ b/conan/cli/commands/cache.py @@ -68,7 +68,7 @@ def cache_clean(conan_api: ConanAPI, parser, subparser, *args): subparser.add_argument("-b", "--build", action='store_true', default=False, help="Clean build folders") subparser.add_argument("-d", "--download", action='store_true', default=False, - help="Clean download folders") + help="Clean download and metadata folders") subparser.add_argument("-t", "--temp", action='store_true', default=False, help="Clean temporary folders") subparser.add_argument('-p', '--package-query', action=OnceArgument, diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 976d8e41d6c..4367140efbe 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -90,13 +90,16 @@ def get_recipe_metadata(self, ref, remote, metadata): Get only the metadata for a locally existing recipe in Cache """ assert ref.revision, "get_recipe without revision specified" + output = ConanOutput(scope=str(ref)) + output.info("Retrieving recipe metadata from remote '%s' " % remote.name) layout = self._cache.ref_layout(ref) download_export = layout.download_export() try: self._call_remote(remote, "get_recipe", ref, download_export, metadata, only_metadata=True) except BaseException: # So KeyboardInterrupt also cleans things - ConanOutput(scope=str(ref)).error(f"Error downloading metadata from remote '{remote.name}'") + + output.error(f"Error downloading metadata from remote '{remote.name}'") raise def get_recipe_sources(self, ref, layout, remote): @@ -124,7 +127,10 @@ def get_package(self, conanfile, pref, remote, metadata=None): self._get_package(pkg_layout, pref, remote, conanfile.output, metadata) def get_package_metadata(self, pref, remote, metadata): - output = ConanOutput(scope=str(pref)) + """ + only download the metadata, not the packge itself + """ + output = ConanOutput(scope=str(pref.ref)) output.info("Retrieving package metadata %s from remote '%s' " % (pref.package_id, remote.name)) @@ -132,7 +138,6 @@ def get_package_metadata(self, pref, remote, metadata): pkg_layout = self._cache.pkg_layout(pref) try: download_pkg_folder = pkg_layout.download_package() - # Download files to the pkg_tgz folder, not to the final one self._call_remote(remote, "get_package", pref, download_pkg_folder, metadata, only_metadata=True) except BaseException as e: # So KeyboardInterrupt also cleans things diff --git a/conans/test/integration/metadata/test_metadata_commands.py b/conans/test/integration/metadata/test_metadata_commands.py index a466f3b1b3e..22b37376f9d 100644 --- a/conans/test/integration/metadata/test_metadata_commands.py +++ b/conans/test/integration/metadata/test_metadata_commands.py @@ -59,14 +59,3 @@ def test_upload(self): assert os.path.isfile(os.path.join(metadata_path, f)) for f in "logs/mybuildlogs.txt", "logs/mybuildlogs2.txt": assert os.path.isfile(os.path.join(pkg_metadata_path, f)) - - """# Regular install can also fetch metadata - c.run("remove * -c") # If done cleanly, not incrementally - assert not os.path.exists(metadata_path) - assert not os.path.exists(pkg_metadata_path) - c.run("install --requires=pkg/0.1 --metadata=*") - for f in "logs/mylogs.txt", "logs/mylogs2.txt": - assert os.path.isfile(os.path.join(metadata_path, f)) - for f in "logs/mybuildlogs.txt", "logs/mybuildlogs2.txt": - assert os.path.isfile(os.path.join(pkg_metadata_path, f)) - """ diff --git a/conans/test/integration/remote/rest_api_test.py b/conans/test/integration/remote/rest_api_test.py index 1fbb30567c6..eab9fb8b3d0 100644 --- a/conans/test/integration/remote/rest_api_test.py +++ b/conans/test/integration/remote/rest_api_test.py @@ -80,7 +80,7 @@ def test_get_conan(self): # Get the conans tmp_dir = temp_folder() - self.api.get_recipe(ref, tmp_dir) + self.api.get_recipe(ref, tmp_dir, metadata=None, only_metadata=False) self.assertIn(CONANFILE, os.listdir(tmp_dir)) self.assertIn(CONAN_MANIFEST, os.listdir(tmp_dir)) @@ -95,7 +95,7 @@ def test_get_package(self): # Get the package tmp_dir = temp_folder() - self.api.get_package(pref, tmp_dir) + self.api.get_package(pref, tmp_dir, metadata=None, only_metadata=False) # The hello.cpp file is not downloaded! self.assertNotIn("hello.cpp", os.listdir(tmp_dir)) @@ -104,7 +104,7 @@ def test_upload_huge_conan(self): self._upload_recipe(ref, {"file9.cpp": ""}) tmp = temp_folder() - files = self.api.get_recipe(ref, tmp) + files = self.api.get_recipe(ref, tmp, metadata=None, only_metadata=False) self.assertIsNotNone(files) self.assertFalse(os.path.exists(os.path.join(tmp, "file9.cpp"))) diff --git a/conans/test/integration/remote/token_refresh_test.py b/conans/test/integration/remote/token_refresh_test.py index afa231c7120..8d3fef278ad 100644 --- a/conans/test/integration/remote/token_refresh_test.py +++ b/conans/test/integration/remote/token_refresh_test.py @@ -92,7 +92,8 @@ def test_auth_with_token(self): with mock.patch("conans.client.rest.auth_manager.UserInput.request_login", return_value=("myuser", "mypassword")): - self.auth_manager.call_rest_api_method(self.remote, "get_recipe", self.ref, ".") + self.auth_manager.call_rest_api_method(self.remote, "get_recipe", self.ref, ".", + metadata=None, only_metadata=False) self.assertEqual(self.localdb.user, "myuser") self.assertEqual(self.localdb.access_token, "access_token") self.assertEqual(self.localdb.refresh_token, "refresh_token") @@ -106,7 +107,8 @@ def test_refresh_with_token(self): self.localdb.access_token = "expired" self.localdb.refresh_token = "refresh_token" - self.auth_manager.call_rest_api_method(self.remote, "get_recipe", self.ref, ".") + self.auth_manager.call_rest_api_method(self.remote, "get_recipe", self.ref, ".", + metadata=None, only_metadata=False) self.assertEqual(self.localdb.user, "myuser") self.assertEqual(self.localdb.access_token, "refreshed_access_token") self.assertEqual(self.localdb.refresh_token, "refresh_token") From 5cae1b76abd932340dcbc4d42ca24f0c667866c1 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 18 May 2023 11:55:36 +0200 Subject: [PATCH 10/14] wip --- conan/api/subapi/upload.py | 4 +++- conans/client/installer.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 5b1e9a40850..b1f5e13a142 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -33,7 +33,9 @@ def check_upstream(self, package_list, remote, force=False): def prepare(self, package_list, enabled_remotes, metadata): """Compress the recipes and packages and fill the upload_data objects with the complete information. It doesn't perform the upload nor checks upstream to see - if the recipe is still there""" + if the recipe is still there + :@param metadata: A list of patterns of metadata that should be uploaded + """ app = ConanApp(self.conan_api.cache_folder) preparator = PackagePreparator(app) preparator.prepare(package_list, enabled_remotes) diff --git a/conans/client/installer.py b/conans/client/installer.py index 56513014e15..56587ee9f86 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -285,8 +285,7 @@ def _download_pkg(self, package): node = package.nodes[0] assert node.pref.revision is not None assert node.pref.timestamp is not None - self._remote_manager.get_package(node.conanfile, node.pref, node.binary_remote, - metadata=None) + self._remote_manager.get_package(node.conanfile, node.pref, node.binary_remote) def _handle_package(self, package, install_reference, remotes, handled_count, total_count): if package.binary == BINARY_SYSTEM_TOOL: From 2030a32952c83c682a8de21bfd754551f5a3cb4d Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 18 May 2023 12:33:02 +0200 Subject: [PATCH 11/14] fix --- conan/api/subapi/upload.py | 7 +++++-- conan/internal/upload_metadata.py | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index b1f5e13a142..52ff8209a19 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -30,11 +30,14 @@ def check_upstream(self, package_list, remote, force=False): UploadUpstreamChecker(app).check(package_list, remote, force) - def prepare(self, package_list, enabled_remotes, metadata): + def prepare(self, package_list, enabled_remotes, metadata=None): """Compress the recipes and packages and fill the upload_data objects with the complete information. It doesn't perform the upload nor checks upstream to see if the recipe is still there - :@param metadata: A list of patterns of metadata that should be uploaded + :param package_list: + :param enabled_remotes: + :param metadata: A list of patterns of metadata that should be uploaded. Default None + means all metadata will be uploaded together with the pkg artifacts """ app = ConanApp(self.conan_api.cache_folder) preparator = PackagePreparator(app) diff --git a/conan/internal/upload_metadata.py b/conan/internal/upload_metadata.py index 72d6289b1fd..551ab1b109c 100644 --- a/conan/internal/upload_metadata.py +++ b/conan/internal/upload_metadata.py @@ -20,7 +20,6 @@ def gather_metadata(upload_data, cache, metadata): for rref, recipe_bundle in upload_data.refs(): if metadata or recipe_bundle["upload"]: metadata_folder = cache.ref_layout(rref).metadata() - assert metadata_folder files = _metadata_files(metadata_folder, metadata) if files: recipe_bundle.setdefault("files", {}).update(files) @@ -29,7 +28,6 @@ def gather_metadata(upload_data, cache, metadata): for pref, pkg_bundle in upload_data.prefs(rref, recipe_bundle): if metadata or pkg_bundle["upload"]: metadata_folder = cache.pkg_layout(pref).metadata() - assert metadata_folder files = _metadata_files(metadata_folder, metadata) if files: pkg_bundle.setdefault("files", {}).update(files) From 3f69d14a9011aecce51f7f093c19fc657f707dd3 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 31 May 2023 00:00:32 +0200 Subject: [PATCH 12/14] added test for test_package as metadata --- .../metadata/test_metadata_test_package.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 conans/test/integration/metadata/test_metadata_test_package.py diff --git a/conans/test/integration/metadata/test_metadata_test_package.py b/conans/test/integration/metadata/test_metadata_test_package.py new file mode 100644 index 00000000000..c118d6bfd36 --- /dev/null +++ b/conans/test/integration/metadata/test_metadata_test_package.py @@ -0,0 +1,51 @@ +import os +import shutil +import textwrap + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.tools import TestClient +from conans.util.files import save + + +class TestMetadataTestPackage: + """ It is possible to store the test_package itself in the recipe metadata and recover it + later to execute it + """ + + def test_round_trip_with_hook(self): + c = TestClient(default_server_user=True) + # TODO: Better strategy for storing clean test_package, zipping it, etc + my_hook = textwrap.dedent("""\ + import os + from conan.tools.files import copy + + def post_export(conanfile): + conanfile.output.info("Storing test_package") + folder = os.path.join(conanfile.recipe_folder, "test_package") + copy(conanfile, "*", src=folder, + dst=os.path.join(conanfile.recipe_metadata_folder, "test_package")) + """) + hook_path = os.path.join(c.cache.hooks_path, "my_hook", "hook_my_hook.py") + save(hook_path, my_hook) + c.save({"conanfile.py": GenConanfile("pkg", "0.1"), + "test_package/conanfile.py": GenConanfile().with_test("pass")}) + c.run("create .") + assert "Testing the package" in c.out + + # Now upload and remove everything + c.run("upload * -c -r=default") + assert "metadata/test_package/conanfile.py" in c.out + c.run("remove * -c") + + c.run("install --requires=pkg/0.1 -r=default") + # Recovery of the test package + # TODO: Discuss if we want better UX, in a single step or something like that + # Forcing the download of the metadata of cache-existing things with the "download" command + c.run("download pkg/0.1 -r=default --metadata=test_package*") + c.run("cache path pkg/0.1 --folder=metadata") + metadata_path = str(c.stdout).strip() + shutil.copytree(metadata_path, os.path.join(c.current_folder, "metadata")) + + # Execute the test_package + c.run("test metadata/test_package pkg/0.1") + assert "pkg/0.1 (test package): Running test()" in c.out From 7695e46b983b288b10ff6c08287d3f0358be5ef9 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 13 Jun 2023 12:10:06 +0200 Subject: [PATCH 13/14] wip --- conan/internal/upload_metadata.py | 4 ++++ .../integration/metadata/test_metadata_commands.py | 10 ++++------ .../integration/metadata/test_metadata_test_package.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/conan/internal/upload_metadata.py b/conan/internal/upload_metadata.py index 551ab1b109c..356a3e81f6b 100644 --- a/conan/internal/upload_metadata.py +++ b/conan/internal/upload_metadata.py @@ -1,6 +1,8 @@ import fnmatch import os +from conan.api.output import ConanOutput + def _metadata_files(folder, metadata): result = {} @@ -22,6 +24,7 @@ def gather_metadata(upload_data, cache, metadata): metadata_folder = cache.ref_layout(rref).metadata() files = _metadata_files(metadata_folder, metadata) if files: + ConanOutput(scope=str(rref)).info(f"Recipe metadata: {len(files)} files") recipe_bundle.setdefault("files", {}).update(files) recipe_bundle["upload"] = True @@ -30,5 +33,6 @@ def gather_metadata(upload_data, cache, metadata): metadata_folder = cache.pkg_layout(pref).metadata() files = _metadata_files(metadata_folder, metadata) if files: + ConanOutput(scope=str(pref)).info(f"Package metadata: {len(files)} files") pkg_bundle.setdefault("files", {}).update(files) pkg_bundle["upload"] = True diff --git a/conans/test/integration/metadata/test_metadata_commands.py b/conans/test/integration/metadata/test_metadata_commands.py index 22b37376f9d..c4eec4ffda1 100644 --- a/conans/test/integration/metadata/test_metadata_commands.py +++ b/conans/test/integration/metadata/test_metadata_commands.py @@ -28,8 +28,8 @@ def test_upload(self): # Now upload everything c.run("upload * -c -r=default") - assert "metadata/logs/mylogs.txt" in c.out - assert "metadata/logs/mybuildlogs.txt" in c.out + assert "pkg/0.1: Recipe metadata: 1 files" in c.out + assert "pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709: Package metadata: 1 files" in c.out # Add new files to the metadata myfile = os.path.join(metadata_path, "logs", "mylogs2.txt") @@ -39,10 +39,8 @@ def test_upload(self): # Upload the metadata, even if the revisions exist in the server # adding the new metadata logs files c.run("upload * -c -r=default --metadata=*") - assert "metadata/logs/mylogs.txt" in c.out - assert "metadata/logs/mybuildlogs.txt" in c.out - assert "metadata/logs/mylogs2.txt" in c.out - assert "metadata/logs/mybuildlogs2.txt" in c.out + assert "pkg/0.1: Recipe metadata: 2 files" in c.out + assert "pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709: Package metadata: 2 files" in c.out c.run("remove * -c") c.run("install --requires=pkg/0.1") # wont install metadata by default diff --git a/conans/test/integration/metadata/test_metadata_test_package.py b/conans/test/integration/metadata/test_metadata_test_package.py index c118d6bfd36..25f051ca489 100644 --- a/conans/test/integration/metadata/test_metadata_test_package.py +++ b/conans/test/integration/metadata/test_metadata_test_package.py @@ -34,7 +34,7 @@ def post_export(conanfile): # Now upload and remove everything c.run("upload * -c -r=default") - assert "metadata/test_package/conanfile.py" in c.out + assert "pkg/0.1: Recipe metadata: 1 files" in c.out c.run("remove * -c") c.run("install --requires=pkg/0.1 -r=default") From 59ce499eaa36fa659ca2f2b42e8489e4255d78d5 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 14 Jun 2023 10:02:22 +0200 Subject: [PATCH 14/14] review --- conan/internal/upload_metadata.py | 6 ++-- .../metadata/test_metadata_commands.py | 36 ++++++++++++++++--- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/conan/internal/upload_metadata.py b/conan/internal/upload_metadata.py index 356a3e81f6b..53ed7da03fb 100644 --- a/conan/internal/upload_metadata.py +++ b/conan/internal/upload_metadata.py @@ -18,8 +18,8 @@ def _metadata_files(folder, metadata): return result -def gather_metadata(upload_data, cache, metadata): - for rref, recipe_bundle in upload_data.refs(): +def gather_metadata(package_list, cache, metadata): + for rref, recipe_bundle in package_list.refs(): if metadata or recipe_bundle["upload"]: metadata_folder = cache.ref_layout(rref).metadata() files = _metadata_files(metadata_folder, metadata) @@ -28,7 +28,7 @@ def gather_metadata(upload_data, cache, metadata): recipe_bundle.setdefault("files", {}).update(files) recipe_bundle["upload"] = True - for pref, pkg_bundle in upload_data.prefs(rref, recipe_bundle): + for pref, pkg_bundle in package_list.prefs(rref, recipe_bundle): if metadata or pkg_bundle["upload"]: metadata_folder = cache.pkg_layout(pref).metadata() files = _metadata_files(metadata_folder, metadata) diff --git a/conans/test/integration/metadata/test_metadata_commands.py b/conans/test/integration/metadata/test_metadata_commands.py index c4eec4ffda1..4d515b72716 100644 --- a/conans/test/integration/metadata/test_metadata_commands.py +++ b/conans/test/integration/metadata/test_metadata_commands.py @@ -2,16 +2,14 @@ from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient -from conans.util.files import save +from conans.util.files import save, load class TestMetadataCommands: def test_upload(self): c = TestClient(default_server_user=True) - c.save({"conanfile.py": GenConanfile("pkg", "0.1"), - "myfile": "mycontent", - "folder/other": "other content"}) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") pid = c.created_package_id("pkg/0.1") @@ -57,3 +55,33 @@ def test_upload(self): assert os.path.isfile(os.path.join(metadata_path, f)) for f in "logs/mybuildlogs.txt", "logs/mybuildlogs2.txt": assert os.path.isfile(os.path.join(pkg_metadata_path, f)) + + def test_update_contents(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("export .") + + # Add some metadata + c.run("cache path pkg/0.1 --folder=metadata") + metadata_path = str(c.stdout).strip() + myfile = os.path.join(metadata_path, "logs", "mylogs.txt") + save(myfile, "mylogs!!!!") + + # Now upload everything + c.run("upload * -c -r=default") + assert "pkg/0.1: Recipe metadata: 1 files" in c.out + + # Update the metadata + save(myfile, "mylogs2!!!!") + # Upload the metadata, even if the revisions exist in the server + # adding the new metadata logs files + c.run("upload * -c -r=default --metadata=*") + assert "pkg/0.1: Recipe metadata: 1 files" in c.out + + c.run("remove * -c") + c.run("download pkg/0.1 -r=default --metadata=*") + c.run("cache path pkg/0.1 --folder=metadata") + metadata_path = str(c.stdout).strip() + + content = load(os.path.join(metadata_path, "logs", "mylogs.txt")) + assert "mylogs2!!!!" in content