Proposal for sources backup feature (#13461)

* wip

* Create DownloadCache abstracion, add upload test

* Remove unused import

* wip

* wip

* wip

* Add soures backup fetching if present

* fix test

* checking file existence prior to upload

* Split upload and download urls

* Move conf to proper place in dict

* Create policy for backup sources cache miss

* Ensure we specify a default for the conf

* Rename raise to error, raise on unknown value

* credentials

* wip

* wip

* wip

* Update conan/api/subapi/upload.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

* Update conan/api/subapi/upload.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

* Update conans/client/rest/file_uploader.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

* Update conan/api/subapi/upload.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

* Update conans/client/downloaders/caching_file_downloader.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

* Update conan/api/subapi/upload.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

* Have server folders in a dict and switch them instead of copying the tree, copytree in 3.6 is no good

* new core conf and 500-error test

* Change layout of source_credentials.json and summary.json, fix test

* Implement user-defined order, ensure a 500 in original url does not break the flow

* Remove unused conf

* Allow changed checksum workflow for backup sources

* Fix test, it expected ConanException instead of ChecksumSignatureMissmatchException

* Fix typo

* refactor and breaking test

* wip

* wip

* fixes

* wip

* Minor changes

* remove space

* fix cmake --presets message version

* improve ConanProxy performance with cache

* cmd_wrapper must get the conanfile argument

* refactor

* docstrings

* fix

---------

Co-authored-by: Rubén Rincón Blanco <rubenrb@jfrog.com>
Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

conan/api/subapi/upload.py

@@ -1,7 +1,12 @@
+import os
+
 from conan.api.output import ConanOutput
 from conan.internal.conan_app import ConanApp
 from conans.client.cmd.uploader import PackagePreparator, UploadExecutor, UploadUpstreamChecker
+from conans.client.downloaders.download_cache import DownloadCache
 from conans.client.pkg_sign import PkgSignaturesPlugin
+from conans.client.rest.file_uploader import FileUploader
+from conans.errors import ConanException, AuthenticationException, ForbiddenException
 
 
 class UploadAPI:
@@ -40,3 +45,34 @@ def upload(self, package_list, remote):
         app.remote_manager.check_credentials(remote)
         executor = UploadExecutor(app)
         executor.upload(package_list, remote)
+
+    def upload_backup_sources(self, package_list):
+        app = ConanApp(self.conan_api.cache_folder)
+        config = app.cache.new_config
+        url = config.get("core.sources:upload_url")
+        if url is None:
+            return
+        url = url if url.endswith("/") else url + "/"
+        download_cache_path = config.get("core.sources:download_cache")
+        download_cache_path = download_cache_path or app.cache.default_sources_backup_folder
+
+        files = DownloadCache(download_cache_path).get_backup_sources_files_to_upload(package_list)
+        # TODO: verify might need a config to force it to False
+        uploader = FileUploader(app.requester, verify=True, config=config)
+        # TODO: For Artifactory, we can list all files once and check from there instead
+        #  of 1 request per file, but this is more general
+        for file in files:
+            basename = os.path.basename(file)
+            full_url = url + basename
+            try:
+                # Always upload summary .json but only upload blob if it does not already exist
+                if file.endswith(".json") or not uploader.exists(full_url, auth=None):
+                    ConanOutput().info(f"Uploading file '{basename}' to backup sources server")
+                    uploader.upload(full_url, file, dedup=False, auth=None)
+                else:
+                    ConanOutput().info(f"File '{basename}' already in backup sources server, "
+                                       "skipping upload")
+            except (AuthenticationException, ForbiddenException) as e:
+                raise ConanException(f"The source backup server '{url}' needs authentication"
+                                     f"/permissions, please provide 'source_credentials.json': {e}")
+        return files

conan/cli/commands/upload.py

@@ -57,6 +57,8 @@ def upload(conan_api: ConanAPI, parser, *args):
     conan_api.upload.prepare(package_list, enabled_remotes)
     conan_api.upload.upload(package_list, remote)
 
+    conan_api.upload.upload_backup_sources(package_list)
+
 
 def _ask_confirm_upload(conan_api, upload_data):
     ui = UserInput(conan_api.config.get("core:non_interactive"))

conan/internal/conan_app.py

@@ -28,10 +28,11 @@ def wrap(self, cmd, conanfile, **kwargs):
 
 
 class ConanFileHelpers:
-    def __init__(self, requester, cmd_wrapper, global_conf):
+    def __init__(self, requester, cmd_wrapper, global_conf, cache):
         self.requester = requester
         self.cmd_wrapper = cmd_wrapper
         self.global_conf = global_conf
+        self.cache = cache
 
 
 class ConanApp(object):
@@ -56,5 +57,5 @@ def __init__(self, cache_folder):
 
         self.pyreq_loader = PyRequireLoader(self.proxy, self.range_resolver)
         cmd_wrap = CmdWrapper(self.cache)
-        conanfile_helpers = ConanFileHelpers(self.requester, cmd_wrap, global_conf)
+        conanfile_helpers = ConanFileHelpers(self.requester, cmd_wrap, global_conf, self.cache)
         self.loader = ConanFileLoader(self.pyreq_loader, conanfile_helpers)

conan/tools/files/files.py

@@ -7,11 +7,9 @@
 from contextlib import contextmanager
 from fnmatch import fnmatch
 from shutil import which
-from urllib.parse import urlparse
-from urllib.request import url2pathname
 
-from conan.api.output import ConanOutput
-from conans.client.downloaders.caching_file_downloader import CachingFileDownloader
+
+from conans.client.downloaders.caching_file_downloader import SourcesCachingDownloader
 from conans.errors import ConanException
 from conans.util.files import rmdir as _internal_rmdir
 from conans.util.sha import check_with_algorithm_sum
@@ -189,70 +187,16 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None,
     :param sha1: SHA-1 hash code to check the downloaded file
     :param sha256: SHA-256 hash code to check the downloaded file
     """
-    # TODO: Add all parameters to the new conf
-    requester = conanfile._conan_helpers.requester
-    global_conf = conanfile._conan_helpers.global_conf
     config = conanfile.conf
-    out = ConanOutput()
-    overwrite = True
 
     retry = retry if retry is not None else 2
     retry = config.get("tools.files.download:retry", check_type=int, default=retry)
     retry_wait = retry_wait if retry_wait is not None else 5
     retry_wait = config.get("tools.files.download:retry_wait", check_type=int, default=retry_wait)
 
-    download_cache = None
-    if md5 or sha1 or sha256:  # If there is no checksum, no cache is used
-        download_cache = config.get("tools.files.download:download_cache")
-        download_cache = download_cache or global_conf.get("core.download:download_cache")
-        if download_cache and not os.path.isabs(download_cache):
-            raise ConanException("core.download:download_cache must be an absolute path")
-
     filename = os.path.abspath(filename)
-
-    def _download_file(file_url):
-        # The download cache is only used if a checksum is provided, otherwise, a normal download
-        if file_url.startswith("file:"):
-            _copy_local_file_from_uri(conanfile, url=file_url, file_path=filename, md5=md5,
-                                      sha1=sha1, sha256=sha256)
-        else:
-            downloader = CachingFileDownloader(requester, download_cache=download_cache)
-            os.makedirs(os.path.dirname(filename), exist_ok=True)  # filename in subfolder must exist
-            downloader.download(url=file_url, file_path=filename, auth=auth, overwrite=overwrite,
-                                verify_ssl=verify, retry=retry, retry_wait=retry_wait,
-                                headers=headers, md5=md5, sha1=sha1, sha256=sha256,
-                                conanfile=conanfile)
-        out.writeln("")
-
-    if not isinstance(url, (list, tuple)):
-        _download_file(url)
-    else:  # We were provided several URLs to try
-        for url_it in url:
-            try:
-                _download_file(url_it)
-                break
-            except Exception as error:
-                message = "Could not download from the URL {}: {}.".format(url_it, str(error))
-                out.warning(message + " Trying another mirror.")
-        else:
-            raise ConanException("All downloads from ({}) URLs have failed.".format(len(url)))
-
-
-def _copy_local_file_from_uri(conanfile, url, file_path, md5=None, sha1=None, sha256=None):
-    file_origin = _path_from_file_uri(url)
-    shutil.copyfile(file_origin, file_path)
-
-    if md5 is not None:
-        check_md5(conanfile, file_path, md5)
-    if sha1 is not None:
-        check_sha1(conanfile, file_path, sha1)
-    if sha256 is not None:
-        check_sha256(conanfile, file_path, sha256)
-
-
-def _path_from_file_uri(uri):
-    path = urlparse(uri).path
-    return url2pathname(path)
+    downloader = SourcesCachingDownloader(conanfile)
+    downloader.download(url, filename, retry, retry_wait, verify, auth, headers, md5, sha1, sha256)
 
 
 def rename(conanfile, src, dst):

conans/client/cache/cache.py

@@ -127,6 +127,10 @@ def get_latest_package_reference(self, pref):
     def store(self):
         return self._store_folder
 
+    @property
+    def default_sources_backup_folder(self):
+        return os.path.join(self.cache_folder, "sources")
+
     @property
     def remotes_path(self):
         return os.path.join(self.cache_folder, REMOTES)