From f98b1669fadf2cea0126cb9b749325f908355b81 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Mon, 23 Mar 2020 13:35:31 +0100 Subject: [PATCH 1/4] (#2195) Added timeout to registry download call --- core/dbt/clients/system.py | 6 +++--- core/dbt/deps/registry.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/dbt/clients/system.py b/core/dbt/clients/system.py index 713e4d4d40d..1c60e88db73 100644 --- a/core/dbt/clients/system.py +++ b/core/dbt/clients/system.py @@ -10,7 +10,7 @@ import requests import stat from typing import ( - Type, NoReturn, List, Optional, Dict, Any, Tuple, Callable + Type, NoReturn, List, Optional, Dict, Any, Tuple, Callable, Union ) import dbt.exceptions @@ -323,8 +323,8 @@ def run_cmd( return out, err -def download(url: str, path: str) -> None: - response = requests.get(url) +def download(url: str, path: str, timeout: Union[float, tuple] = None) -> None: + response = requests.get(url, timeout=timeout) with open(path, 'wb') as handle: for block in response.iter_content(1024 * 64): handle.write(block) diff --git a/core/dbt/deps/registry.py b/core/dbt/deps/registry.py index 5f6b05f6d1b..f0b75de1dd2 100644 --- a/core/dbt/deps/registry.py +++ b/core/dbt/deps/registry.py @@ -61,7 +61,7 @@ def install(self, project, renderer): system.make_directory(os.path.dirname(tar_path)) download_url = metadata.downloads.tarball - system.download(download_url, tar_path) + system.download(download_url, tar_path, timeout=10) deps_path = project.modules_path package_name = self.get_project_name(project, renderer) system.untar_package(tar_path, deps_path, package_name) From cce99b1ff271f1f41bb01f6e0bb3f254de961ba0 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Mon, 23 Mar 2020 14:14:21 +0100 Subject: [PATCH 2/4] (#2195) Added entry to CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e1d53bda28..99b2d713c15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ### Fixes - When a jinja value is undefined, give a helpful error instead of failing with cryptic "cannot pickle ParserMacroCapture" errors ([#2110](https://github.com/fishtown-analytics/dbt/issues/2110), [#2184](https://github.com/fishtown-analytics/dbt/pull/2184)) +- Added timeout to registry download call ([#2195](https://github.com/fishtown-analytics/dbt/issues/2195), [#2228](https://github.com/fishtown-analytics/dbt/pull/2228)) + +Contributors: + - [@raalsky](https://github.com/Raalsky) ([#2224](https://github.com/fishtown-analytics/dbt/pull/2224), [#2228](https://github.com/fishtown-analytics/dbt/pull/2228)) ## dbt 0.16.0 (Release date TBD) From d6cb415f76a9b350d84b3767ac5a71a35cf31501 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Mon, 23 Mar 2020 17:48:54 +0100 Subject: [PATCH 3/4] (#2195) DBT_HTTP_TIMEOUT environment variable added --- core/dbt/deps/registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/dbt/deps/registry.py b/core/dbt/deps/registry.py index f0b75de1dd2..ecbf34b9a28 100644 --- a/core/dbt/deps/registry.py +++ b/core/dbt/deps/registry.py @@ -52,6 +52,7 @@ def _fetch_metadata(self, project, renderer) -> RegistryPackageMetadata: return RegistryPackageMetadata.from_dict(dct) def install(self, project, renderer): + connection_timeout = float(os.getenv('DBT_HTTP_TIMEOUT', 10)) metadata = self.fetch_metadata(project, renderer) tar_name = '{}.{}.tar.gz'.format(self.package, self.version) @@ -61,7 +62,7 @@ def install(self, project, renderer): system.make_directory(os.path.dirname(tar_path)) download_url = metadata.downloads.tarball - system.download(download_url, tar_path, timeout=10) + system.download(download_url, tar_path, timeout=connection_timeout) deps_path = project.modules_path package_name = self.get_project_name(project, renderer) system.untar_package(tar_path, deps_path, package_name) From 1c60882525f6eb3175587046922d5ff4d55fe063 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Mon, 23 Mar 2020 17:56:37 +0100 Subject: [PATCH 4/4] (#2195) DBT_HTTP_TIMEOUT moved inside system download call --- core/dbt/clients/system.py | 3 ++- core/dbt/deps/registry.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/dbt/clients/system.py b/core/dbt/clients/system.py index 1c60e88db73..b0cfdd31ed7 100644 --- a/core/dbt/clients/system.py +++ b/core/dbt/clients/system.py @@ -324,7 +324,8 @@ def run_cmd( def download(url: str, path: str, timeout: Union[float, tuple] = None) -> None: - response = requests.get(url, timeout=timeout) + connection_timeout = timeout or float(os.getenv('DBT_HTTP_TIMEOUT', 10)) + response = requests.get(url, timeout=connection_timeout) with open(path, 'wb') as handle: for block in response.iter_content(1024 * 64): handle.write(block) diff --git a/core/dbt/deps/registry.py b/core/dbt/deps/registry.py index ecbf34b9a28..5f6b05f6d1b 100644 --- a/core/dbt/deps/registry.py +++ b/core/dbt/deps/registry.py @@ -52,7 +52,6 @@ def _fetch_metadata(self, project, renderer) -> RegistryPackageMetadata: return RegistryPackageMetadata.from_dict(dct) def install(self, project, renderer): - connection_timeout = float(os.getenv('DBT_HTTP_TIMEOUT', 10)) metadata = self.fetch_metadata(project, renderer) tar_name = '{}.{}.tar.gz'.format(self.package, self.version) @@ -62,7 +61,7 @@ def install(self, project, renderer): system.make_directory(os.path.dirname(tar_path)) download_url = metadata.downloads.tarball - system.download(download_url, tar_path, timeout=connection_timeout) + system.download(download_url, tar_path) deps_path = project.modules_path package_name = self.get_project_name(project, renderer) system.untar_package(tar_path, deps_path, package_name)