Metadata files upload/download

conan/api/subapi/cache.py

@@ -19,6 +19,12 @@ def export_path(self, ref: RecipeReference):
         ref_layout = app.cache.recipe_layout(ref)
         return ref_layout.export()
 
+    def recipe_metadata_path(self, ref: RecipeReference):
+        app = ConanApp(self.conan_api.cache_folder)
+        ref = _resolve_latest_ref(app, ref)
+        ref_layout = app.cache.ref_layout(ref)
+        return ref_layout.metadata()
+
     def export_source_path(self, ref: RecipeReference):
         app = ConanApp(self.conan_api.cache_folder)
         ref.revision = None if ref.revision == "latest" else ref.revision
@@ -37,6 +43,12 @@ def build_path(self, pref: PkgReference):
         ref_layout = app.cache.pkg_layout(pref)
         return ref_layout.build()
 
+    def package_metadata_path(self, pref: PkgReference):
+        app = ConanApp(self.conan_api.cache_folder)
+        pref = _resolve_latest_pref(app, pref)
+        ref_layout = app.cache.pkg_layout(pref)
+        return ref_layout.metadata()
+
     def package_path(self, pref: PkgReference):
         app = ConanApp(self.conan_api.cache_folder)
         pref = _resolve_latest_pref(app, pref)

conan/api/subapi/download.py

@@ -12,7 +12,7 @@ class DownloadAPI:
     def __init__(self, conan_api):
         self.conan_api = conan_api
 
-    def recipe(self, ref: RecipeReference, remote: Remote):
+    def recipe(self, ref: RecipeReference, remote: Remote, metadata=None):
         output = ConanOutput()
         app = ConanApp(self.conan_api.cache_folder)
         assert ref.revision, f"Reference '{ref}' must have revision"
@@ -22,10 +22,12 @@ def recipe(self, ref: RecipeReference, remote: Remote):
             pass
         else:
             output.info(f"Skip recipe {ref.repr_notime()} download, already in cache")
+            if metadata:
+                app.remote_manager.get_recipe_metadata(ref, remote, metadata)
             return False
 
         output.info(f"Downloading recipe '{ref.repr_notime()}'")
-        app.remote_manager.get_recipe(ref, remote)
+        app.remote_manager.get_recipe(ref, remote, metadata)
 
         layout = app.cache.ref_layout(ref)
         conan_file_path = layout.conanfile()
@@ -36,7 +38,7 @@ def recipe(self, ref: RecipeReference, remote: Remote):
         retrieve_exports_sources(app.remote_manager, layout, conanfile, ref, [remote])
         return True
 
-    def package(self, pref: PkgReference, remote: Remote):
+    def package(self, pref: PkgReference, remote: Remote, metadata=None):
         output = ConanOutput()
         app = ConanApp(self.conan_api.cache_folder)
         try:
@@ -48,6 +50,8 @@ def package(self, pref: PkgReference, remote: Remote):
         skip_download = app.cache.exists_prev(pref)
         if skip_download:
             output.info(f"Skip package {pref.repr_notime()} download, already in cache")
+            if metadata:
+                app.remote_manager.get_package_metadata(pref, remote, metadata)
             return False
         layout = app.cache.ref_layout(pref.ref)
         conan_file_path = layout.conanfile()

conan/api/subapi/upload.py

@@ -2,6 +2,7 @@
 
 from conan.api.output import ConanOutput
 from conan.internal.conan_app import ConanApp
+from conan.internal.upload_metadata import gather_metadata
 from conans.client.cmd.uploader import PackagePreparator, UploadExecutor, UploadUpstreamChecker
 from conans.client.downloaders.download_cache import DownloadCache
 from conans.client.pkg_sign import PkgSignaturesPlugin
@@ -29,13 +30,19 @@ def check_upstream(self, package_list, remote, force=False):
 
         UploadUpstreamChecker(app).check(package_list, remote, force)
 
-    def prepare(self, package_list, enabled_remotes):
+    def prepare(self, package_list, enabled_remotes, metadata=None):
         """Compress the recipes and packages and fill the upload_data objects
         with the complete information. It doesn't perform the upload nor checks upstream to see
-        if the recipe is still there"""
+        if the recipe is still there
+        :param package_list:
+        :param enabled_remotes:
+        :param metadata: A list of patterns of metadata that should be uploaded. Default None
+                         means all metadata will be uploaded together with the pkg artifacts
+        """
         app = ConanApp(self.conan_api.cache_folder)
         preparator = PackagePreparator(app)
         preparator.prepare(package_list, enabled_remotes)
+        gather_metadata(package_list, app.cache, metadata)
         signer = PkgSignaturesPlugin(app.cache)
         # This might add files entries to package_list with signatures
         signer.sign(package_list)

conan/cli/commands/cache.py

@@ -21,7 +21,7 @@ def cache_path(conan_api: ConanAPI, parser, subparser, *args):
     Show the path to the Conan cache for a given reference.
     """
     subparser.add_argument("reference", help="Recipe reference or Package reference")
-    subparser.add_argument("--folder", choices=['export_source', 'source', 'build'],
+    subparser.add_argument("--folder", choices=['export_source', 'source', 'build', 'metadata'],
                            help="Path to show. The 'build'"
                                 " requires a package reference. If not specified it shows 'exports'"
                                 " path ")
@@ -40,13 +40,17 @@ def cache_path(conan_api: ConanAPI, parser, subparser, *args):
             path = conan_api.cache.export_source_path(ref)
         elif args.folder == "source":
             path = conan_api.cache.source_path(ref)
+        elif args.folder == "metadata":
+            path = conan_api.cache.recipe_metadata_path(ref)
         else:
             raise ConanException(f"'--folder {args.folder}' requires a valid package reference")
     else:
         if args.folder is None:
             path = conan_api.cache.package_path(pref)
         elif args.folder == "build":
             path = conan_api.cache.build_path(pref)
+        elif args.folder == "metadata":
+            path = conan_api.cache.package_metadata_path(pref)
         else:
             raise ConanException(f"'--folder {args.folder}' requires a recipe reference")
     return path
@@ -64,7 +68,7 @@ def cache_clean(conan_api: ConanAPI, parser, subparser, *args):
     subparser.add_argument("-b", "--build", action='store_true', default=False,
                            help="Clean build folders")
     subparser.add_argument("-d", "--download", action='store_true', default=False,
-                           help="Clean download folders")
+                           help="Clean download and metadata folders")
     subparser.add_argument("-t", "--temp", action='store_true', default=False,
                            help="Clean temporary folders")
     subparser.add_argument('-p', '--package-query', action=OnceArgument,

conan/cli/commands/download.py

@@ -32,6 +32,9 @@ def download(conan_api: ConanAPI, parser, *args):
                              "(arch=x86 OR compiler=gcc)")
     parser.add_argument("-r", "--remote", action=OnceArgument, required=True,
                         help='Download from this specific remote')
+    parser.add_argument("-m", "--metadata", action='append',
+                        help='Download the metadata matching the pattern, even if the package is '
+                             'already in the cache and not downloaded')
     parser.add_argument("-l", "--list", help="Package list file")
 
     args = parser.parse_args(*args)
@@ -64,9 +67,9 @@ def download(conan_api: ConanAPI, parser, *args):
 
     if parallel <= 1:
         for ref in refs:
-            conan_api.download.recipe(ref, remote)
+            conan_api.download.recipe(ref, remote, args.metadata)
         for pref in prefs:
-            conan_api.download.package(pref, remote)
+            conan_api.download.package(pref, remote, args.metadata)
     else:
         _download_parallel(parallel, conan_api, refs, prefs, remote)
 

conan/cli/commands/upload.py

@@ -58,6 +58,9 @@ def upload(conan_api: ConanAPI, parser, *args):
     parser.add_argument('--dry-run', default=False, action='store_true',
                         help='Do not execute the real upload (experimental)')
     parser.add_argument("-l", "--list", help="Package list file")
+    parser.add_argument("-m", "--metadata", action='append',
+                        help='Upload the metadata, even if the package is already in the server and '
+                             'not uploaded')
 
     args = parser.parse_args(*args)
 
@@ -88,7 +91,7 @@ def upload(conan_api: ConanAPI, parser, *args):
     if not args.list and not args.confirm and "*" in args.pattern:
         _ask_confirm_upload(conan_api, package_list)
 
-    conan_api.upload.prepare(package_list, enabled_remotes)
+    conan_api.upload.prepare(package_list, enabled_remotes, args.metadata)
 
     if not args.dry_run:
         conan_api.upload.upload(package_list, remote)

conan/internal/upload_metadata.py

@@ -0,0 +1,38 @@
+import fnmatch
+import os
+
+from conan.api.output import ConanOutput
+
+
+def _metadata_files(folder, metadata):
+    result = {}
+    for root, _, files in os.walk(folder):
+        for f in files:
+            abs_path = os.path.join(root, f)
+            relpath = os.path.relpath(abs_path, folder)
+            if metadata:
+                if not any(fnmatch.fnmatch(relpath, m) for m in metadata):
+                    continue
+            path = os.path.join("metadata", relpath).replace("\\", "/")
+            result[path] = abs_path
+    return result
+
+
+def gather_metadata(package_list, cache, metadata):
+    for rref, recipe_bundle in package_list.refs():
+        if metadata or recipe_bundle["upload"]:
+            metadata_folder = cache.ref_layout(rref).metadata()
+            files = _metadata_files(metadata_folder, metadata)
+            if files:
+                ConanOutput(scope=str(rref)).info(f"Recipe metadata: {len(files)} files")
+                recipe_bundle.setdefault("files", {}).update(files)
+                recipe_bundle["upload"] = True
+
+        for pref, pkg_bundle in package_list.prefs(rref, recipe_bundle):
+            if metadata or pkg_bundle["upload"]:
+                metadata_folder = cache.pkg_layout(pref).metadata()
+                files = _metadata_files(metadata_folder, metadata)
+                if files:
+                    ConanOutput(scope=str(pref)).info(f"Package metadata: {len(files)} files")
+                    pkg_bundle.setdefault("files", {}).update(files)
+                    pkg_bundle["upload"] = True

conans/client/remote_manager.py

@@ -39,7 +39,7 @@ def upload_package(self, pref, files_to_upload, remote):
         assert pref.revision, "upload_package requires PREV"
         self._call_remote(remote, "upload_package", pref, files_to_upload)
 
-    def get_recipe(self, ref, remote):
+    def get_recipe(self, ref, remote, metadata=None):
         """
         Read the conans from remotes
         Will iterate the remotes to find the conans unless remote was specified
@@ -53,7 +53,9 @@ def get_recipe(self, ref, remote):
 
         download_export = layout.download_export()
         try:
-            zipped_files = self._call_remote(remote, "get_recipe", ref, download_export)
+            zipped_files = self._call_remote(remote, "get_recipe", ref, download_export, metadata,
+                                             only_metadata=False)
+            # TODO: Optimize this call, it is slow to always query all revisions
             remote_refs = self._call_remote(remote, "get_recipe_revisions_references", ref)
             ref_time = remote_refs[0].timestamp
             ref.timestamp = ref_time
@@ -83,6 +85,23 @@ def get_recipe(self, ref, remote):
         # Make sure that the source dir is deleted
         rmdir(layout.source())
 
+    def get_recipe_metadata(self, ref, remote, metadata):
+        """
+        Get only the metadata for a locally existing recipe in Cache
+        """
+        assert ref.revision, "get_recipe without revision specified"
+        output = ConanOutput(scope=str(ref))
+        output.info("Retrieving recipe metadata from remote '%s' " % remote.name)
+        layout = self._cache.ref_layout(ref)
+        download_export = layout.download_export()
+        try:
+            self._call_remote(remote, "get_recipe", ref, download_export, metadata,
+                              only_metadata=True)
+        except BaseException:  # So KeyboardInterrupt also cleans things
+
+            output.error(f"Error downloading metadata from remote '{remote.name}'")
+            raise
+
     def get_recipe_sources(self, ref, layout, remote):
         assert ref.revision, "get_recipe_sources requires RREV"
 
@@ -96,7 +115,7 @@ def get_recipe_sources(self, ref, layout, remote):
         tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
         uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
 
-    def get_package(self, conanfile, pref, remote):
+    def get_package(self, conanfile, pref, remote, metadata=None):
         conanfile.output.info("Retrieving package %s from remote '%s' " % (pref.package_id,
                                                                            remote.name))
 
@@ -105,15 +124,35 @@ def get_package(self, conanfile, pref, remote):
         pkg_layout = self._cache.get_or_create_pkg_layout(pref)
         pkg_layout.package_remove()  # Remove first the destination folder
         with pkg_layout.set_dirty_context_manager():
-            self._get_package(pkg_layout, pref, remote, conanfile.output)
+            self._get_package(pkg_layout, pref, remote, conanfile.output, metadata)
+
+    def get_package_metadata(self, pref, remote, metadata):
+        """
+        only download the metadata, not the packge itself
+        """
+        output = ConanOutput(scope=str(pref.ref))
+        output.info("Retrieving package metadata %s from remote '%s' "
+                    % (pref.package_id, remote.name))
+
+        assert pref.revision is not None
+        pkg_layout = self._cache.pkg_layout(pref)
+        try:
+            download_pkg_folder = pkg_layout.download_package()
+            self._call_remote(remote, "get_package", pref, download_pkg_folder,
+                              metadata, only_metadata=True)
+        except BaseException as e:  # So KeyboardInterrupt also cleans things
+            output.error("Exception while getting package metadata: %s" % str(pref.package_id))
+            output.error("Exception: %s %s" % (type(e), str(e)))
+            raise
 
-    def _get_package(self, layout, pref, remote, scoped_output):
+    def _get_package(self, layout, pref, remote, scoped_output, metadata):
         try:
             assert pref.revision is not None
 
             download_pkg_folder = layout.download_package()
             # Download files to the pkg_tgz folder, not to the final one
-            zipped_files = self._call_remote(remote, "get_package", pref, download_pkg_folder)
+            zipped_files = self._call_remote(remote, "get_package", pref, download_pkg_folder,
+                                             metadata, only_metadata=False)
             zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)}
             # quick server package integrity check:
             for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"):

conans/client/rest/rest_client.py

@@ -60,14 +60,14 @@ def _get_api(self):
                              self._requester, self._config, self._verify_ssl,
                              checksum_deploy)
 
-    def get_recipe(self, ref, dest_folder):
-        return self._get_api().get_recipe(ref, dest_folder)
+    def get_recipe(self, ref, dest_folder, metadata, only_metadata):
+        return self._get_api().get_recipe(ref, dest_folder, metadata, only_metadata)
 
     def get_recipe_sources(self, ref, dest_folder):
         return self._get_api().get_recipe_sources(ref, dest_folder)
 
-    def get_package(self, pref, dest_folder):
-        return self._get_api().get_package(pref, dest_folder)
+    def get_package(self, pref, dest_folder, metadata, only_metadata):
+        return self._get_api().get_package(pref, dest_folder, metadata, only_metadata)
 
     def upload_recipe(self, ref, files_to_upload):
         return self._get_api().upload_recipe(ref, files_to_upload)

conans/client/rest/rest_client_v2.py

@@ -1,4 +1,5 @@
 import copy
+import fnmatch
 import os
 
 from conan.api.output import ConanOutput
@@ -35,12 +36,17 @@ def _get_file_list_json(self, url):
         data["files"] = list(d.replace("\\", "/") for d in data["files"].keys())
         return data
 
-    def get_recipe(self, ref, dest_folder):
+    def get_recipe(self, ref, dest_folder, metadata, only_metadata):
         url = self.router.recipe_snapshot(ref)
         data = self._get_file_list_json(url)
         files = data["files"]
         accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt", "metadata/sign"]
-        files = [f for f in files if any(f.startswith(m) for m in accepted_files)]
+        if only_metadata:
+            accepted_files = []
+        metadata = metadata or []
+        metadata = [f"metadata/{m}" for m in metadata]
+        files = [f for f in files if any(f.startswith(m) for m in accepted_files)
+                 or any(fnmatch.fnmatch(f, m) for m in metadata)]
 
         # If we didn't indicated reference, server got the latest, use absolute now, it's safer
         urls = {fn: self.router.recipe_file(ref, fn) for fn in files}
@@ -65,13 +71,19 @@ def get_recipe_sources(self, ref, dest_folder):
         ret = {fn: os.path.join(dest_folder, fn) for fn in files}
         return ret
 
-    def get_package(self, pref, dest_folder):
+    def get_package(self, pref, dest_folder, metadata, only_metadata):
         url = self.router.package_snapshot(pref)
         data = self._get_file_list_json(url)
         files = data["files"]
         # Download only known files, but not metadata (except sign)
         accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt", "metadata/sign"]
-        files = [f for f in files if any(f.startswith(m) for m in accepted_files)]
+        if only_metadata:
+            accepted_files = []
+        metadata = metadata or []
+        metadata = [f"metadata/{m}" for m in metadata]
+        files = [f for f in files if any(f.startswith(m) for m in accepted_files)
+                 or any(fnmatch.fnmatch(f, m) for m in metadata)]
+
         # If we didn't indicated reference, server got the latest, use absolute now, it's safer
         urls = {fn: self.router.package_file(pref, fn) for fn in files}
         self._download_and_save_files(urls, dest_folder, files, scope=str(pref.ref))