Feature/metadata2 (#14152)

* metadata more things

* wip

* new test

* metadata improvements

conan/api/subapi/download.py

@@ -56,5 +56,5 @@ def package(self, pref: PkgReference, remote: Remote, metadata=None):
             return False
 
         output.info(f"Downloading package '{pref.repr_notime()}'")
-        app.remote_manager.get_package(pref, remote)
+        app.remote_manager.get_package(pref, remote, metadata)
         return True

conans/client/cmd/export.py

@@ -39,7 +39,9 @@ def cmd_export(app, conanfile_path, name, version, user, channel, graph_lock=Non
     # TODO: cache2.0 move this creation to other place
     mkdir(export_folder)
     mkdir(export_src_folder)
-    conanfile.folders.set_base_recipe_metadata(recipe_layout.metadata())
+    recipe_metadata = recipe_layout.metadata()
+    mkdir(recipe_metadata)
+    conanfile.folders.set_base_recipe_metadata(recipe_metadata)
     export_recipe(conanfile, export_folder)
     export_source(conanfile, export_src_folder)
     shutil.copy2(conanfile_path, recipe_layout.conanfile())

conans/client/conanfile/build.py

@@ -4,6 +4,7 @@
 
 def run_build_method(conanfile, hook_manager):
     mkdir(conanfile.build_folder)
+    mkdir(conanfile.package_metadata_folder)
     with chdir(conanfile.build_folder):
         hook_manager.execute("pre_build", conanfile=conanfile)
         if hasattr(conanfile, "build"):

conans/client/downloaders/caching_file_downloader.py

@@ -158,10 +158,10 @@ def __init__(self, requester, config, scope=None):
             raise ConanException("core.download:download_cache must be an absolute path")
         self._file_downloader = FileDownloader(requester, scope=scope)
 
-    def download(self, url, file_path, auth, verify_ssl, retry, retry_wait):
-        if not self._download_cache:
+    def download(self, url, file_path, auth, verify_ssl, retry, retry_wait, metadata=False):
+        if not self._download_cache or metadata:  # Metadata not cached and can be overwritten
             self._file_downloader.download(url, file_path, retry=retry, retry_wait=retry_wait,
-                                           verify_ssl=verify_ssl, auth=auth, overwrite=False)
+                                           verify_ssl=verify_ssl, auth=auth, overwrite=metadata)
             return
 
         download_cache = DownloadCache(self._download_cache)

conans/client/installer.py

@@ -155,7 +155,7 @@ def build_package(self, node, package_layout):
                 prev = self._package(conanfile, pref)
                 assert prev
                 node.prev = prev
-            except ConanException as exc:
+            except ConanException as exc:  # TODO: Remove this? unnecessary?
                 raise exc
 
         return node.pref
@@ -326,12 +326,14 @@ def _handle_package(self, package, install_reference, handled_count, total_count
 
         # Make sure that all nodes with same pref compute package_info()
         pkg_folder = package_layout.package()
+        pkg_metadata = package_layout.metadata()
         assert os.path.isdir(pkg_folder), "Pkg '%s' folder must exist: %s" % (str(pref), pkg_folder)
         for n in package.nodes:
             n.prev = pref.revision  # Make sure the prev is assigned
             conanfile = n.conanfile
             # Call the info method
             conanfile.folders.set_base_package(pkg_folder)
+            conanfile.folders.set_base_pkg_metadata(pkg_metadata)
             self._call_package_info(conanfile, pkg_folder, is_editable=False)
 
     def _handle_node_editable(self, install_node):

conans/client/rest/rest_client_v2.py

@@ -39,20 +39,24 @@ def _get_file_list_json(self, url):
     def get_recipe(self, ref, dest_folder, metadata, only_metadata):
         url = self.router.recipe_snapshot(ref)
         data = self._get_file_list_json(url)
-        files = data["files"]
-        accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt", "metadata/sign"]
-        if only_metadata:
-            accepted_files = []
-        metadata = metadata or []
-        metadata = [f"metadata/{m}" for m in metadata]
-        files = [f for f in files if any(f.startswith(m) for m in accepted_files)
-                 or any(fnmatch.fnmatch(f, m) for m in metadata)]
-
-        # If we didn't indicated reference, server got the latest, use absolute now, it's safer
-        urls = {fn: self.router.recipe_file(ref, fn) for fn in files}
-        self._download_and_save_files(urls, dest_folder, files, parallel=True)
-        ret = {fn: os.path.join(dest_folder, fn) for fn in files}
-        return ret
+        server_files = data["files"]
+        result = {}
+
+        if not only_metadata:
+            accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt",
+                              "metadata/sign"]
+            files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)]
+            # If we didn't indicated reference, server got the latest, use absolute now, it's safer
+            urls = {fn: self.router.recipe_file(ref, fn) for fn in files}
+            self._download_and_save_files(urls, dest_folder, files, parallel=True)
+            result.update({fn: os.path.join(dest_folder, fn) for fn in files})
+        if metadata:
+            metadata = [f"metadata/{m}" for m in metadata]
+            files = [f for f in server_files if any(fnmatch.fnmatch(f, m) for m in metadata)]
+            urls = {fn: self.router.recipe_file(ref, fn) for fn in files}
+            self._download_and_save_files(urls, dest_folder, files, parallel=True, metadata=True)
+            result.update({fn: os.path.join(dest_folder, fn) for fn in files})
+        return result
 
     def get_recipe_sources(self, ref, dest_folder):
         # If revision not specified, check latest
@@ -73,21 +77,26 @@ def get_recipe_sources(self, ref, dest_folder):
     def get_package(self, pref, dest_folder, metadata, only_metadata):
         url = self.router.package_snapshot(pref)
         data = self._get_file_list_json(url)
-        files = data["files"]
+        server_files = data["files"]
+        result = {}
         # Download only known files, but not metadata (except sign)
-        accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt", "metadata/sign"]
-        if only_metadata:
-            accepted_files = []
-        metadata = metadata or []
-        metadata = [f"metadata/{m}" for m in metadata]
-        files = [f for f in files if any(f.startswith(m) for m in accepted_files)
-                 or any(fnmatch.fnmatch(f, m) for m in metadata)]
-
-        # If we didn't indicated reference, server got the latest, use absolute now, it's safer
-        urls = {fn: self.router.package_file(pref, fn) for fn in files}
-        self._download_and_save_files(urls, dest_folder, files, scope=str(pref.ref))
-        ret = {fn: os.path.join(dest_folder, fn) for fn in files}
-        return ret
+        if not only_metadata:  # Retrieve package first, then metadata
+            accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt",
+                              "metadata/sign"]
+            files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)]
+            # If we didn't indicated reference, server got the latest, use absolute now, it's safer
+            urls = {fn: self.router.package_file(pref, fn) for fn in files}
+            self._download_and_save_files(urls, dest_folder, files, scope=str(pref.ref))
+            result.update({fn: os.path.join(dest_folder, fn) for fn in files})
+
+        if metadata:
+            metadata = [f"metadata/{m}" for m in metadata]
+            files = [f for f in server_files if any(fnmatch.fnmatch(f, m) for m in metadata)]
+            urls = {fn: self.router.package_file(pref, fn) for fn in files}
+            self._download_and_save_files(urls, dest_folder, files, scope=str(pref.ref),
+                                          metadata=True)
+            result.update({fn: os.path.join(dest_folder, fn) for fn in files})
+        return result
 
     @staticmethod
     def _is_dir(path, files):
@@ -145,7 +154,8 @@ def _upload_files(self, files, urls):
             raise ConanException("Execute upload again to retry upload the failed files: %s"
                                  % ", ".join(failed))
 
-    def _download_and_save_files(self, urls, dest_folder, files, parallel=False, scope=None):
+    def _download_and_save_files(self, urls, dest_folder, files, parallel=False, scope=None,
+                                 metadata=False):
         # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz
         # can be < conanfile, conaninfo, and sent always the last, so smaller files go first
         retry = self._config.get("core.download:retry", check_type=int, default=2)
@@ -160,13 +170,14 @@ def _download_and_save_files(self, urls, dest_folder, files, parallel=False, sco
             if parallel:
                 kwargs = {"url": resource_url, "file_path": abs_path, "retry": retry,
                           "retry_wait": retry_wait, "verify_ssl": self.verify_ssl,
-                          "auth": self.auth}
+                          "auth": self.auth, "metadata": metadata}
                 thread = ExceptionThread(target=downloader.download, kwargs=kwargs)
                 threads.append(thread)
                 thread.start()
             else:
                 downloader.download(url=resource_url, file_path=abs_path, auth=self.auth,
-                                    verify_ssl=self.verify_ssl, retry=retry, retry_wait=retry_wait)
+                                    verify_ssl=self.verify_ssl, retry=retry, retry_wait=retry_wait,
+                                    metadata=metadata)
         for t in threads:
             t.join()
         for t in threads:  # Need to join all before raising errors

conans/client/source.py

@@ -62,6 +62,7 @@ def config_source(export_source_folder, conanfile, hook_manager):
     if not os.path.exists(conanfile.folders.base_source):  # No source folder, need to get it
         with set_dirty_context_manager(conanfile.folders.base_source):
             mkdir(conanfile.source_folder)
+            mkdir(conanfile.recipe_metadata_folder)
 
             # First of all get the exported scm sources (if auto) or clone (if fixed)
             # Now move the export-sources to the right location

conans/model/conan_file.py

@@ -278,8 +278,8 @@ def recipe_metadata_folder(self):
         return self.folders.recipe_metadata_folder
 
     @property
-    def pkg_metadata_folder(self):
-        return self.folders.pkg_metadata_folder
+    def package_metadata_folder(self):
+        return self.folders.package_metadata_folder
 
     @property
     def build_path(self) -> Path:

conans/model/conanfile_interface.py

@@ -31,10 +31,18 @@ def options(self):
     def recipe_folder(self):
         return self._conanfile.recipe_folder
 
+    @property
+    def recipe_metadata_folder(self):
+        return self._conanfile.recipe_metadata_folder
+
     @property
     def package_folder(self):
         return self._conanfile.package_folder
 
+    @property
+    def package_metadata_folder(self):
+        return self._conanfile.package_metadata_folder
+
     @property
     def package_path(self) -> Path:
         assert self.package_folder is not None, "`package_folder` is `None`"

conans/model/layout.py

@@ -116,7 +116,7 @@ def set_base_recipe_metadata(self, folder):
         self._base_recipe_metadata = folder
 
     @property
-    def pkg_metadata_folder(self):
+    def package_metadata_folder(self):
         return self._base_pkg_metadata
 
     def set_base_pkg_metadata(self, folder):

conans/test/integration/metadata/test_metadata_commands.py

@@ -1,7 +1,8 @@
 import os
 
 from conans.test.assets.genconanfile import GenConanfile
-from conans.test.utils.tools import TestClient
+from conans.test.utils.test_files import temp_folder
+from conans.test.utils.tools import TestClient, NO_SETTINGS_PACKAGE_ID
 from conans.util.files import save, load
 
 
@@ -85,3 +86,115 @@ def test_update_contents(self):
 
         content = load(os.path.join(metadata_path, "logs", "mylogs.txt"))
         assert "mylogs2!!!!" in content
+
+    def test_folder_exist(self):
+        """ so we can cp -R to the metadata folder, having to create the folder in the cache
+        is weird
+        """
+        c = TestClient(default_server_user=True)
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+        c.run("create .")
+        c.run("cache path pkg/0.1 --folder=metadata")
+        metadata_path = str(c.stdout).strip()
+        assert os.path.isdir(metadata_path)
+        c.run(f"cache path pkg/0.1:{NO_SETTINGS_PACKAGE_ID} --folder=metadata")
+        pkg_metadata_path = str(c.stdout).strip()
+        assert os.path.isdir(pkg_metadata_path)
+
+    def test_direct_download_redownload(self):
+        """ When we directly download things, without "conan install" first, it is also able
+        to fetch the requested metadata
+
+        Also, re-downloading same thing shouldn't fail
+        """
+        c = TestClient(default_server_user=True)
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+        c.run("create .")
+        pid = c.created_package_id("pkg/0.1")
+
+        # Add some metadata
+        c.run("cache path pkg/0.1 --folder=metadata")
+        metadata_path = str(c.stdout).strip()
+        myfile = os.path.join(metadata_path, "logs", "mylogs.txt")
+        save(myfile, "mylogs!!!!")
+
+        c.run(f"cache path pkg/0.1:{pid} --folder=metadata")
+        pkg_metadata_path = str(c.stdout).strip()
+        myfile = os.path.join(pkg_metadata_path, "logs", "mybuildlogs.txt")
+        save(myfile, "mybuildlogs!!!!")
+
+        # Now upload everything
+        c.run("upload * -c -r=default")
+        assert "pkg/0.1: Recipe metadata: 1 files" in c.out
+        assert "pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709: Package metadata: 1 files" in c.out
+
+        c.run("remove * -c")
+
+        # Forcing the download of the metadata of cache-existing things with the "download" command
+        c.run("download pkg/0.1 -r=default --metadata=*")
+        assert os.path.isfile(os.path.join(metadata_path, "logs", "mylogs.txt"))
+        c.run(f"cache path pkg/0.1:{pid} --folder=metadata")
+        pkg_metadata_path = str(c.stdout).strip()
+        assert os.path.isfile(os.path.join(pkg_metadata_path, "logs", "mybuildlogs.txt"))
+
+        # Re-download shouldn't fail
+        c.run("download pkg/0.1 -r=default --metadata=*")
+        assert os.path.isfile(os.path.join(metadata_path, "logs", "mylogs.txt"))
+        c.run(f"cache path pkg/0.1:{pid} --folder=metadata")
+        pkg_metadata_path = str(c.stdout).strip()
+        assert os.path.isfile(os.path.join(pkg_metadata_path, "logs", "mybuildlogs.txt"))
+
+    def test_no_download_cached(self):
+        """ as the metadata can change, no checksum, no revision, cannot be cached
+        """
+        c = TestClient(default_server_user=True)
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+        c.run("create .")
+        pid = c.created_package_id("pkg/0.1")
+
+        # Add some metadata
+        c.run("cache path pkg/0.1 --folder=metadata")
+        metadata_path = str(c.stdout).strip()
+        myrecipefile = os.path.join(metadata_path, "logs", "mylogs.txt")
+        save(myrecipefile, "mylogs!!!!")
+
+        c.run(f"cache path pkg/0.1:{pid} --folder=metadata")
+        pkg_metadata_path = str(c.stdout).strip()
+        mypkgfile = os.path.join(pkg_metadata_path, "logs", "mybuildlogs.txt")
+        save(mypkgfile, "mybuildlogs!!!!")
+
+        # Now upload everything
+        c.run("upload * -c -r=default")
+        assert "pkg/0.1: Recipe metadata: 1 files" in c.out
+        assert "pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709: Package metadata: 1 files" in c.out
+
+        c2 = TestClient(servers=c.servers)
+        tmp_folder = temp_folder()
+        # MOST important part: activate cache
+        save(c2.cache.new_config_path, f"core.download:download_cache={tmp_folder}\n")
+
+        # download package and metadata
+        c2.run("download pkg/0.1 -r=default --metadata=*")
+        c2.run("cache path pkg/0.1 --folder=metadata")
+        c2_metadata_path = str(c2.stdout).strip()
+        mylogs = load(os.path.join(c2_metadata_path, "logs", "mylogs.txt"))
+        assert "mylogs!!!!" in mylogs
+        c2.run(f"cache path pkg/0.1:{pid} --folder=metadata")
+        c2_pkg_metadata_path = str(c2.stdout).strip()
+        mybuildlogs = load(os.path.join(c2_pkg_metadata_path, "logs", "mybuildlogs.txt"))
+        assert "mybuildlogs!!!!" in mybuildlogs
+
+        # Now the other client will update the metadata
+        save(myrecipefile, "mylogs2!!!!")
+        save(mypkgfile, "mybuildlogs2!!!!")
+        c.run("upload * -c -r=default --metadata=*")
+        assert "pkg/0.1: Recipe metadata: 1 files" in c.out
+        assert "pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709: Package metadata: 1 files" in c.out
+
+        # re-download of metadata in c2
+        c2.run("remove * -c")  # to make sure the download cache works
+        c2.run("download pkg/0.1 -r=default --metadata=*")
+        mylogs = load(os.path.join(c2_metadata_path, "logs", "mylogs.txt"))
+        assert "mylogs2!!!!" in mylogs
+        mybuildlogs = load(os.path.join(c2_pkg_metadata_path, "logs", "mybuildlogs.txt"))
+        assert "mybuildlogs2!!!!" in mybuildlogs

conans/test/integration/metadata/test_metadata_deploy.py

@@ -0,0 +1,52 @@
+import textwrap
+
+from conans.test.utils.tools import TestClient
+
+
+class TestMetadataDeploy:
+    """ prove we can gather metadata too with a deployer
+    """
+
+    def test_deploy(self):
+        # FIXME: It only supports package metadata deployment, because missing internal interface
+        conanfile = textwrap.dedent("""
+            import os
+            from conan import ConanFile
+            from conan.tools.files import save, copy
+
+            class Pkg(ConanFile):
+                version = "0.1"
+
+                def source(self):
+                    save(self, os.path.join(self.recipe_metadata_folder, "logs", "src.log"),
+                         f"srclog {self.name}!!")
+
+                def build(self):
+                    save(self, "mylogs.txt", f"some logs {self.name}!!!")
+                    copy(self, "mylogs.txt", src=self.build_folder,
+                         dst=os.path.join(self.package_metadata_folder, "logs"))
+            """)
+        deploy = textwrap.dedent("""
+            import os, shutil
+
+            def deploy(graph, output_folder, **kwargs):
+                conanfile = graph.root.conanfile
+                for r, d in conanfile.dependencies.items():
+                    shutil.copytree(d.package_metadata_folder, os.path.join(output_folder, "pkgs",
+                                                                            d.ref.name))
+                    # FIXME: Missing
+                    #shutil.copytree(d.recipe_metadata_folder, os.path.join(output_folder, "pkgs",
+                    #                                                         d.ref.name))
+           """)
+
+        c = TestClient()
+        c.save({"conanfile.py": conanfile,
+                "deploy.py": deploy})
+        c.run("create . --name=pkg1")
+        c.run("create . --name=pkg2")
+        c.run("install --requires=pkg1/0.1 --requires=pkg2/0.1 --deployer=deploy")
+        assert "some logs pkg1!!!" in c.load("pkgs/pkg1/logs/mylogs.txt")
+        assert "some logs pkg2!!!" in c.load("pkgs/pkg2/logs/mylogs.txt")
+        # TODO: This must pass
+        # assert "srclog pkg1!!!" in c.load("recipes/pkg1/logs/src.log")
+        # assert "srclog pkg2!!!" in c.load("recipes/pkg2/logs/src.log")