From a9a8fd67c8bb27c03cc13012ab74e0d9f2b5155b Mon Sep 17 00:00:00 2001 From: jgsogo Date: Wed, 25 Nov 2020 13:13:11 +0100 Subject: [PATCH 1/3] poc download rt cache --- conans/client/conf/__init__.py | 8 ++++ conans/client/rest/artifactory_cache.py | 46 +++++++++++++++++++ conans/client/rest/file_downloader.py | 2 +- conans/client/rest/rest_client_v1.py | 7 +++ conans/client/rest/rest_client_v2.py | 7 +++ conans/client/tools/net.py | 6 ++- .../cache/test_artifactory_cache.py | 21 +++++++++ 7 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 conans/client/rest/artifactory_cache.py create mode 100644 conans/test/functional/cache/test_artifactory_cache.py diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 6b7f78aee86..8c6a3a6e020 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -445,6 +445,14 @@ def download_cache(self): except ConanException: return None + @property + def artifactory_cache(self): + try: + artifactory_cache = self.get_item("storage.artifactory_cache") + return artifactory_cache + except ConanException: + return None + @property def scm_to_conandata(self): try: diff --git a/conans/client/rest/artifactory_cache.py b/conans/client/rest/artifactory_cache.py new file mode 100644 index 00000000000..1690dfb087a --- /dev/null +++ b/conans/client/rest/artifactory_cache.py @@ -0,0 +1,46 @@ +from urllib.parse import urlsplit, urlunsplit +from conans.util.sha import sha256 as sha256_sum + + +class ArtifactoryCacheDownloader(object): + + def __init__(self, rt_base_url, file_downloader, requester, user_download=False): + self._rt_base_url = rt_base_url # TBD: expected full url with credentials + self._file_downloader = file_downloader + self._user_download = user_download + self._requester = requester + + def _put(self, rt_path, file_path, **props): + """ Put the 'local_filepath' to remote and assign given properties """ + pass + + def _try_get(self, rt_path, file_path): + """ Try to get remote file, return None if file is not found """ + try: + url = self._rt_base_url + "/" + rt_path + response = self._requester.get(url, stream=True, verify=False) + + except Exception as exc: + + + def _rt_path(self, url, checksum=None): + # TODO: Chain classes, use same implementation as 'file_downloader' + urltokens = urlsplit(url) + # append empty query and fragment before unsplit + if not self._user_download: # removes ?signature=xxx + url = urlunsplit(urltokens[0:3] + ("", "")) + if checksum is not None: + url += checksum + h = sha256_sum(url.encode()) + return h + + def download(self, url, file_path, md5=None, sha1=None, sha256=None, *args, **kwargs): + """ Intercept download call """ + # TODO: We don't want to intercept every call, like 'conan_sources.tgz' or + # 'conan_package.tgz', they are already under our control. Argument from outside or + # something to check names here? + checksum = sha256 or sha1 or md5 + rt_path = self._rt_path(url, checksum) + if not self._try_get(rt_path, file_path=file_path): + self._file_downloader.download(url=url, file_path=file_path, *args, **kwargs) + self._put(rt_path, file_path=file_path, url=url) diff --git a/conans/client/rest/file_downloader.py b/conans/client/rest/file_downloader.py index 297f80f5977..84b63d91b1f 100644 --- a/conans/client/rest/file_downloader.py +++ b/conans/client/rest/file_downloader.py @@ -54,7 +54,7 @@ def _download_file(self, url, auth, headers, file_path, try_resume=False): range_start = 0 try: - response = self._requester.get(url, stream=True, verify=self._verify_ssl, auth=auth, + response = self._requester.get(url, stream=True, verify=False, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) diff --git a/conans/client/rest/rest_client_v1.py b/conans/client/rest/rest_client_v1.py index ed6e34a384f..5f99ad3e9c6 100644 --- a/conans/client/rest/rest_client_v1.py +++ b/conans/client/rest/rest_client_v1.py @@ -6,6 +6,7 @@ from six.moves.urllib.parse import parse_qs, urljoin, urlparse, urlsplit from conans.client.remote_manager import check_compressed_files +from conans.client.rest.artifactory_cache import ArtifactoryCacheDownloader from conans.client.rest.client_routes import ClientV1Router from conans.client.rest.download_cache import CachedFileDownloader from conans.client.rest.file_uploader import FileUploader @@ -45,6 +46,9 @@ def _download_files(self, file_urls, snapshot_md5): Its a generator, so it yields elements for memory performance """ downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) + artifactory_cache = self._config.artifactory_cache + if artifactory_cache: + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) download_cache = self._config.download_cache if download_cache: assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums" @@ -189,6 +193,9 @@ def _download_files_to_folder(self, file_urls, to_folder, snapshot_md5): It writes downloaded files to disk (appending to file, only keeps chunks in memory) """ downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) + artifactory_cache = self._config.artifactory_cache + if artifactory_cache: + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) download_cache = self._config.download_cache if download_cache: assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums" diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index 8f49198e2ef..838ecf9c187 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -4,6 +4,7 @@ from conans import DEFAULT_REVISION_V1 from conans.client.remote_manager import check_compressed_files +from conans.client.rest.artifactory_cache import ArtifactoryCacheDownloader from conans.client.rest.client_routes import ClientV2Router from conans.client.rest.download_cache import CachedFileDownloader from conans.client.rest.file_uploader import FileUploader @@ -42,6 +43,9 @@ def _get_file_list_json(self, url): def _get_remote_file_contents(self, url, use_cache, headers=None): # We don't want traces in output of these downloads, they are ugly in output downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) + artifactory_cache = self._config.artifactory_cache + if artifactory_cache: + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) contents = downloader.download(url, auth=self.auth, headers=headers) @@ -217,6 +221,9 @@ def _upload_files(self, files, urls, retry, retry_wait, display_name=None): def _download_and_save_files(self, urls, dest_folder, files, use_cache): downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) + artifactory_cache = self._config.artifactory_cache + if artifactory_cache: + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz diff --git a/conans/client/tools/net.py b/conans/client/tools/net.py index e5627102283..9e025ae9a64 100644 --- a/conans/client/tools/net.py +++ b/conans/client/tools/net.py @@ -1,5 +1,6 @@ import os +from conans.client.rest.artifactory_cache import ArtifactoryCacheDownloader from conans.client.rest.download_cache import CachedFileDownloader from conans.client.rest.file_downloader import FileDownloader from conans.client.tools.files import check_md5, check_sha1, check_sha256, unzip @@ -90,12 +91,15 @@ def download(url, filename, verify=True, out=None, retry=None, retry_wait=None, checksum = sha256 or sha1 or md5 downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config) + artifactory_cache = config.artifactory_cache + if artifactory_cache: + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) if config and config.download_cache and checksum: downloader = CachedFileDownloader(config.download_cache, downloader, user_download=True) def _download_file(file_url): # The download cache is only used if a checksum is provided, otherwise, a normal download - if isinstance(downloader, CachedFileDownloader): + if not isinstance(downloader, FileDownloader): downloader.download(file_url, filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) diff --git a/conans/test/functional/cache/test_artifactory_cache.py b/conans/test/functional/cache/test_artifactory_cache.py new file mode 100644 index 00000000000..f732519bb5f --- /dev/null +++ b/conans/test/functional/cache/test_artifactory_cache.py @@ -0,0 +1,21 @@ +import textwrap +import unittest + +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient + + +class ArtifactoryCacheTestCase(unittest.TestCase): + def test_rt_cache(self): + client = TestClient(default_server_user=True) + cache_folder = temp_folder() + client.run('remote add conan-center https://conan.bintray.com') + client.run('config set storage.download_cache="%s"' % cache_folder) + client.run('config set storage.artifactory_cache="http://admin:password@0.0.0.0:8082/artifactory/conan-sources"') + + client.run('install zlib/1.2.8@ -r conan-center --build=zlib') + print(client.out) + self.fail("AAA") + + def test_download_cache(self): + pass From 925d7f3b293157c0b7845a6ffa20750f2a2c4152 Mon Sep 17 00:00:00 2001 From: jgsogo Date: Wed, 25 Nov 2020 13:58:52 +0100 Subject: [PATCH 2/3] work with Artifactory, assign URL property --- conans/client/rest/artifactory_cache.py | 27 +++++++++++++++++++------ conans/client/rest/download_cache.py | 6 ++++-- conans/client/rest/rest_client_v1.py | 6 ++++-- conans/client/rest/rest_client_v2.py | 6 ++++-- conans/client/tools/net.py | 4 +++- 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/conans/client/rest/artifactory_cache.py b/conans/client/rest/artifactory_cache.py index 1690dfb087a..7e8e715e1b3 100644 --- a/conans/client/rest/artifactory_cache.py +++ b/conans/client/rest/artifactory_cache.py @@ -1,27 +1,42 @@ from urllib.parse import urlsplit, urlunsplit + +from six.moves.urllib.parse import quote + +from conans.client.rest.file_uploader import FileUploader from conans.util.sha import sha256 as sha256_sum class ArtifactoryCacheDownloader(object): - def __init__(self, rt_base_url, file_downloader, requester, user_download=False): + def __init__(self, rt_base_url, file_downloader, requester, output, verify, config, + user_download=False): self._rt_base_url = rt_base_url # TBD: expected full url with credentials self._file_downloader = file_downloader self._user_download = user_download self._requester = requester + self._file_uploader = FileUploader(requester, output, verify, config) def _put(self, rt_path, file_path, **props): """ Put the 'local_filepath' to remote and assign given properties """ - pass + try: + matrix_params_str = ";".join( + ["{}={}".format(key, quote(value, safe='')) for key, value in props.items()]) + url = self._rt_base_url + ";" + matrix_params_str + "/" + rt_path + self._file_uploader.upload(url, abs_path=file_path) + except Exception as e: + # TODO: Check different exceptions + return None def _try_get(self, rt_path, file_path): """ Try to get remote file, return None if file is not found """ try: url = self._rt_base_url + "/" + rt_path - response = self._requester.get(url, stream=True, verify=False) - - except Exception as exc: - + # TODO: Here we want to invoke requester, not my chained file_downloader + self._file_downloader.download(url=url, file_path=file_path) + return True + except Exception: + # TODO: Check different exceptions + return None def _rt_path(self, url, checksum=None): # TODO: Chain classes, use same implementation as 'file_downloader' diff --git a/conans/client/rest/download_cache.py b/conans/client/rest/download_cache.py index a55883c5ca9..54ddee54047 100644 --- a/conans/client/rest/download_cache.py +++ b/conans/client/rest/download_cache.py @@ -46,8 +46,10 @@ def download(self, url, file_path=None, auth=None, retry=None, retry_wait=None, try: if not os.path.exists(cached_path): try: - self._file_downloader.download(url, cached_path, auth, retry, retry_wait, - overwrite, headers) + self._file_downloader.download(url, file_path=cached_path, auth=auth, + retry=retry, retry_wait=retry_wait, + overwrite=overwrite, headers=headers, + md5=md5, sha1=sha1, sha256=sha256) self._check_checksum(cached_path, md5, sha1, sha256) except Exception: if os.path.exists(cached_path): diff --git a/conans/client/rest/rest_client_v1.py b/conans/client/rest/rest_client_v1.py index 5f99ad3e9c6..7e6bafeac79 100644 --- a/conans/client/rest/rest_client_v1.py +++ b/conans/client/rest/rest_client_v1.py @@ -48,7 +48,8 @@ def _download_files(self, file_urls, snapshot_md5): downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) artifactory_cache = self._config.artifactory_cache if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + None, self.verify_ssl, self._config) download_cache = self._config.download_cache if download_cache: assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums" @@ -195,7 +196,8 @@ def _download_files_to_folder(self, file_urls, to_folder, snapshot_md5): downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) artifactory_cache = self._config.artifactory_cache if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + self._output, self.verify_ssl, self._config) download_cache = self._config.download_cache if download_cache: assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums" diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index 838ecf9c187..0aaf4654da0 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -45,7 +45,8 @@ def _get_remote_file_contents(self, url, use_cache, headers=None): downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) artifactory_cache = self._config.artifactory_cache if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + None, self.verify_ssl, self._config) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) contents = downloader.download(url, auth=self.auth, headers=headers) @@ -223,7 +224,8 @@ def _download_and_save_files(self, urls, dest_folder, files, use_cache): downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) artifactory_cache = self._config.artifactory_cache if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + self._output, self.verify_ssl, self._config) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz diff --git a/conans/client/tools/net.py b/conans/client/tools/net.py index 9e025ae9a64..66a7b750369 100644 --- a/conans/client/tools/net.py +++ b/conans/client/tools/net.py @@ -93,7 +93,9 @@ def download(url, filename, verify=True, out=None, retry=None, retry_wait=None, downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config) artifactory_cache = config.artifactory_cache if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader) + downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, requester=requester, + output=out, verify=verify, config=config, + user_download=True) if config and config.download_cache and checksum: downloader = CachedFileDownloader(config.download_cache, downloader, user_download=True) From 6f0669fd3c3c86c69acb1ba00f9ddc7b2d0fa58b Mon Sep 17 00:00:00 2001 From: jgsogo Date: Wed, 25 Nov 2020 15:35:02 +0100 Subject: [PATCH 3/3] artifactory_cache -> sources_backup, split url to help RT UI --- conans/client/conf/__init__.py | 6 +++--- conans/client/rest/artifactory_cache.py | 1 + conans/client/rest/rest_client_v1.py | 12 ++++++------ conans/client/rest/rest_client_v2.py | 12 ++++++------ conans/client/tools/net.py | 6 +++--- .../test/functional/cache/test_artifactory_cache.py | 4 ++-- 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 8c6a3a6e020..bdbaf1d37f8 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -446,10 +446,10 @@ def download_cache(self): return None @property - def artifactory_cache(self): + def sources_backup(self): try: - artifactory_cache = self.get_item("storage.artifactory_cache") - return artifactory_cache + sources_backup = self.get_item("storage.sources_backup") + return sources_backup except ConanException: return None diff --git a/conans/client/rest/artifactory_cache.py b/conans/client/rest/artifactory_cache.py index 7e8e715e1b3..23565bfb155 100644 --- a/conans/client/rest/artifactory_cache.py +++ b/conans/client/rest/artifactory_cache.py @@ -47,6 +47,7 @@ def _rt_path(self, url, checksum=None): if checksum is not None: url += checksum h = sha256_sum(url.encode()) + h = "{}/{}/{}".format(h[:2], h[2:4], h[4:]) # This will help Artifactory UI return h def download(self, url, file_path, md5=None, sha1=None, sha256=None, *args, **kwargs): diff --git a/conans/client/rest/rest_client_v1.py b/conans/client/rest/rest_client_v1.py index 7e6bafeac79..59b55eec261 100644 --- a/conans/client/rest/rest_client_v1.py +++ b/conans/client/rest/rest_client_v1.py @@ -46,9 +46,9 @@ def _download_files(self, file_urls, snapshot_md5): Its a generator, so it yields elements for memory performance """ downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) - artifactory_cache = self._config.artifactory_cache - if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + sources_backup = self._config.sources_backup + if sources_backup: + downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester, None, self.verify_ssl, self._config) download_cache = self._config.download_cache if download_cache: @@ -194,9 +194,9 @@ def _download_files_to_folder(self, file_urls, to_folder, snapshot_md5): It writes downloaded files to disk (appending to file, only keeps chunks in memory) """ downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) - artifactory_cache = self._config.artifactory_cache - if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + sources_backup = self._config.sources_backup + if sources_backup: + downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester, self._output, self.verify_ssl, self._config) download_cache = self._config.download_cache if download_cache: diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index 0aaf4654da0..316b51ee8e9 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -43,9 +43,9 @@ def _get_file_list_json(self, url): def _get_remote_file_contents(self, url, use_cache, headers=None): # We don't want traces in output of these downloads, they are ugly in output downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) - artifactory_cache = self._config.artifactory_cache - if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + sources_backup = self._config.sources_backup + if sources_backup: + downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester, None, self.verify_ssl, self._config) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) @@ -222,9 +222,9 @@ def _upload_files(self, files, urls, retry, retry_wait, display_name=None): def _download_and_save_files(self, urls, dest_folder, files, use_cache): downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) - artifactory_cache = self._config.artifactory_cache - if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, self.requester, + sources_backup = self._config.sources_backup + if sources_backup: + downloader = ArtifactoryCacheDownloader(sources_backup, downloader, self.requester, self._output, self.verify_ssl, self._config) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) diff --git a/conans/client/tools/net.py b/conans/client/tools/net.py index 66a7b750369..04db4dac0fb 100644 --- a/conans/client/tools/net.py +++ b/conans/client/tools/net.py @@ -91,9 +91,9 @@ def download(url, filename, verify=True, out=None, retry=None, retry_wait=None, checksum = sha256 or sha1 or md5 downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config) - artifactory_cache = config.artifactory_cache - if artifactory_cache: - downloader = ArtifactoryCacheDownloader(artifactory_cache, downloader, requester=requester, + sources_backup = config.sources_backup + if sources_backup: + downloader = ArtifactoryCacheDownloader(sources_backup, downloader, requester=requester, output=out, verify=verify, config=config, user_download=True) if config and config.download_cache and checksum: diff --git a/conans/test/functional/cache/test_artifactory_cache.py b/conans/test/functional/cache/test_artifactory_cache.py index f732519bb5f..1d551d0a001 100644 --- a/conans/test/functional/cache/test_artifactory_cache.py +++ b/conans/test/functional/cache/test_artifactory_cache.py @@ -1,4 +1,3 @@ -import textwrap import unittest from conans.test.utils.test_files import temp_folder @@ -11,7 +10,8 @@ def test_rt_cache(self): cache_folder = temp_folder() client.run('remote add conan-center https://conan.bintray.com') client.run('config set storage.download_cache="%s"' % cache_folder) - client.run('config set storage.artifactory_cache="http://admin:password@0.0.0.0:8082/artifactory/conan-sources"') + client.run( + 'config set storage.sources_backup="http://admin:password@0.0.0.0:8082/artifactory/conan-sources"') client.run('install zlib/1.2.8@ -r conan-center --build=zlib') print(client.out)