[poc][wip] Cache in local Artifactory everything that is downloaded by Conan

conans/client/conf/__init__.py

@@ -445,6 +445,14 @@ def download_cache(self):
         except ConanException:
             return None
 
+    @property
+    def sources_backup(self):
+        try:
+            sources_backup = self.get_item("storage.sources_backup")
+            return sources_backup
+        except ConanException:
+            return None
+
     @property
     def scm_to_conandata(self):
         try:

conans/client/rest/artifactory_cache.py

@@ -0,0 +1,62 @@
+from urllib.parse import urlsplit, urlunsplit
+
+from six.moves.urllib.parse import quote
+
+from conans.client.rest.file_uploader import FileUploader
+from conans.util.sha import sha256 as sha256_sum
+
+
+class ArtifactoryCacheDownloader(object):
+
+    def __init__(self, rt_base_url, file_downloader, requester, output, verify, config,
+                 user_download=False):
+        self._rt_base_url = rt_base_url  # TBD: expected full url with credentials
+        self._file_downloader = file_downloader
+        self._user_download = user_download
+        self._requester = requester
+        self._file_uploader = FileUploader(requester, output, verify, config)
+
+    def _put(self, rt_path, file_path, **props):
+        """ Put the 'local_filepath' to remote and assign given properties """
+        try:
+            matrix_params_str = ";".join(
+                ["{}={}".format(key, quote(value, safe='')) for key, value in props.items()])
+            url = self._rt_base_url + ";" + matrix_params_str + "/" + rt_path
+            self._file_uploader.upload(url, abs_path=file_path)
+        except Exception as e:
+            # TODO: Check different exceptions
+            return None
+
+    def _try_get(self, rt_path, file_path):
+        """ Try to get remote file, return None if file is not found """
+        try:
+            url = self._rt_base_url + "/" + rt_path
+            # TODO: Here we want to invoke requester, not my chained file_downloader
+            self._file_downloader.download(url=url, file_path=file_path)
+            return True
+        except Exception:
+            # TODO: Check different exceptions
+            return None
+
+    def _rt_path(self, url, checksum=None):
+        # TODO: Chain classes, use same implementation as 'file_downloader'
+        urltokens = urlsplit(url)
+        # append empty query and fragment before unsplit
+        if not self._user_download:  # removes ?signature=xxx
+            url = urlunsplit(urltokens[0:3] + ("", ""))
+        if checksum is not None:
+            url += checksum
+        h = sha256_sum(url.encode())
+        h = "{}/{}/{}".format(h[:2], h[2:4], h[4:])  # This will help Artifactory UI
+        return h
+
+    def download(self, url, file_path, md5=None, sha1=None, sha256=None, *args, **kwargs):
+        """ Intercept download call """
+        # TODO: We don't want to intercept every call, like 'conan_sources.tgz' or
+        #  'conan_package.tgz', they are already under our control. Argument from outside or
+        #  something to check names here?
+        checksum = sha256 or sha1 or md5
+        rt_path = self._rt_path(url, checksum)
+        if not self._try_get(rt_path, file_path=file_path):
+            self._file_downloader.download(url=url, file_path=file_path, *args, **kwargs)
+            self._put(rt_path, file_path=file_path, url=url)

conans/client/rest/download_cache.py

@@ -46,8 +46,10 @@ def download(self, url, file_path=None, auth=None, retry=None, retry_wait=None,
             try:
                 if not os.path.exists(cached_path):
                     try:
-                        self._file_downloader.download(url, cached_path, auth, retry, retry_wait,
-                                                       overwrite, headers)
+                        self._file_downloader.download(url, file_path=cached_path, auth=auth,
+                                                       retry=retry, retry_wait=retry_wait,
+                                                       overwrite=overwrite, headers=headers,
+                                                       md5=md5, sha1=sha1, sha256=sha256)
                         self._check_checksum(cached_path, md5, sha1, sha256)
                     except Exception:
                         if os.path.exists(cached_path):

conans/client/rest/file_downloader.py

@@ -54,7 +54,7 @@ def _download_file(self, url, auth, headers, file_path, try_resume=False):
             range_start = 0
 
         try:
-            response = self._requester.get(url, stream=True, verify=self._verify_ssl, auth=auth,
+            response = self._requester.get(url, stream=True, verify=False, auth=auth,
                                            headers=headers)
         except Exception as exc:
             raise ConanException("Error downloading file %s: '%s'" % (url, exc))

conans/client/rest/rest_client_v1.py

@@ -6,6 +6,7 @@
 from six.moves.urllib.parse import parse_qs, urljoin, urlparse, urlsplit
 
 from conans.client.remote_manager import check_compressed_files
+from conans.client.rest.artifactory_cache import ArtifactoryCacheDownloader
 from conans.client.rest.client_routes import ClientV1Router
 from conans.client.rest.download_cache import CachedFileDownloader
 from conans.client.rest.file_uploader import FileUploader
@@ -45,6 +46,10 @@ def _download_files(self, file_urls, snapshot_md5):
         Its a generator, so it yields elements for memory performance
         """
         downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config)
+        sources_backup = self._config.sources_backup
+        if sources_backup:
+            downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester,
+                                                    None, self.verify_ssl, self._config)
         download_cache = self._config.download_cache
         if download_cache:
             assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums"
@@ -189,6 +194,10 @@ def _download_files_to_folder(self, file_urls, to_folder, snapshot_md5):
         It writes downloaded files to disk (appending to file, only keeps chunks in memory)
         """
         downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config)
+        sources_backup = self._config.sources_backup
+        if sources_backup:
+            downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester,
+                                                    self._output, self.verify_ssl, self._config)
         download_cache = self._config.download_cache
         if download_cache:
             assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums"

conans/client/rest/rest_client_v2.py

@@ -4,6 +4,7 @@
 
 from conans import DEFAULT_REVISION_V1
 from conans.client.remote_manager import check_compressed_files
+from conans.client.rest.artifactory_cache import ArtifactoryCacheDownloader
 from conans.client.rest.client_routes import ClientV2Router
 from conans.client.rest.download_cache import CachedFileDownloader
 from conans.client.rest.file_uploader import FileUploader
@@ -42,6 +43,10 @@ def _get_file_list_json(self, url):
     def _get_remote_file_contents(self, url, use_cache, headers=None):
         # We don't want traces in output of these downloads, they are ugly in output
         downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config)
+        sources_backup = self._config.sources_backup
+        if sources_backup:
+            downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester,
+                                                    None, self.verify_ssl, self._config)
         if use_cache and self._config.download_cache:
             downloader = CachedFileDownloader(self._config.download_cache, downloader)
         contents = downloader.download(url, auth=self.auth, headers=headers)
@@ -217,6 +222,10 @@ def _upload_files(self, files, urls, retry, retry_wait, display_name=None):
 
     def _download_and_save_files(self, urls, dest_folder, files, use_cache):
         downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config)
+        sources_backup = self._config.sources_backup
+        if sources_backup:
+            downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester,
+                                                    self._output, self.verify_ssl, self._config)
         if use_cache and self._config.download_cache:
             downloader = CachedFileDownloader(self._config.download_cache, downloader)
         # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz

conans/client/tools/net.py

@@ -1,5 +1,6 @@
 import os
 
+from conans.client.rest.artifactory_cache import ArtifactoryCacheDownloader
 from conans.client.rest.download_cache import CachedFileDownloader
 from conans.client.rest.file_downloader import FileDownloader
 from conans.client.tools.files import check_md5, check_sha1, check_sha256, unzip
@@ -90,12 +91,17 @@ def download(url, filename, verify=True, out=None, retry=None, retry_wait=None,
     checksum = sha256 or sha1 or md5
 
     downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config)
+    sources_backup = config.sources_backup
+    if sources_backup:
+        downloader = ArtifactoryCacheDownloader(sources_backup, downloader, requester=requester,
+                                                output=out, verify=verify, config=config,
+                                                user_download=True)
     if config and config.download_cache and checksum:
         downloader = CachedFileDownloader(config.download_cache, downloader, user_download=True)
 
     def _download_file(file_url):
         # The download cache is only used if a checksum is provided, otherwise, a normal download
-        if isinstance(downloader, CachedFileDownloader):
+        if not isinstance(downloader, FileDownloader):
             downloader.download(file_url, filename, retry=retry, retry_wait=retry_wait,
                                 overwrite=overwrite, auth=auth, headers=headers, md5=md5,
                                 sha1=sha1, sha256=sha256)

conans/test/functional/cache/test_artifactory_cache.py

@@ -0,0 +1,21 @@
+import unittest
+
+from conans.test.utils.test_files import temp_folder
+from conans.test.utils.tools import TestClient
+
+
+class ArtifactoryCacheTestCase(unittest.TestCase):
+    def test_rt_cache(self):
+        client = TestClient(default_server_user=True)
+        cache_folder = temp_folder()
+        client.run('remote add conan-center https://conan.bintray.com')
+        client.run('config set storage.download_cache="%s"' % cache_folder)
+        client.run(
+            'config set storage.sources_backup="http://admin:password@0.0.0.0:8082/artifactory/conan-sources"')
+
+        client.run('install zlib/1.2.8@ -r conan-center --build=zlib')
+        print(client.out)
+        self.fail("AAA")
+
+    def test_download_cache(self):
+        pass