From 6c449ef064084ae19ec9161e0a3d3c342c741750 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 28 Sep 2017 14:31:03 -0400 Subject: [PATCH 01/13] semver resolution --- dbt/exceptions.py | 6 + dbt/semver.py | 281 +++++++++++++++++++++++++++++++++++++++ test/unit/test_semver.py | 98 ++++++++++++++ 3 files changed, 385 insertions(+) create mode 100644 dbt/semver.py create mode 100644 test/unit/test_semver.py diff --git a/dbt/exceptions.py b/dbt/exceptions.py index cf139f0d5e1..166c8b19df2 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -88,6 +88,12 @@ class ValidationException(RuntimeException): pass +class VersionsNotCompatibleException(ValidationException): + + def __init__(self, msg=None): + self.msg = msg + + class NotImplementedException(Exception): pass diff --git a/dbt/semver.py b/dbt/semver.py new file mode 100644 index 00000000000..0efe754a12d --- /dev/null +++ b/dbt/semver.py @@ -0,0 +1,281 @@ +import re + +from dbt.exceptions import VersionsNotCompatibleException +import dbt.utils + +_MATCHERS = "(?P\>=|\>|\<|\<=|=)?" +_NUM_NO_LEADING_ZEROS = "(0|[1-9][0-9]*)" +_ALPHA = "[0-9A-Za-z-]*" +_ALPHA_NO_LEADING_ZEROS = "(0|[1-9A-Za-z-][0-9A-Za-z-]*)" + +_BASE_VERSION_REGEX = """ +(?P{num_no_leading_zeros})\. +(?P{num_no_leading_zeros})\. +(?P{num_no_leading_zeros}) +""".format(num_no_leading_zeros=_NUM_NO_LEADING_ZEROS) + +_VERSION_EXTRA_REGEX = """ +(\- + (?P + {alpha_no_leading_zeros}(\.{alpha_no_leading_zeros})*))? +(\+ + (?P + {alpha}(\.{alpha})*))? +""".format( + alpha_no_leading_zeros=_ALPHA_NO_LEADING_ZEROS, + alpha=_ALPHA) + +_VERSION_REGEX = re.compile(""" +^ +{matchers} +{base_version_regex} +{version_extra_regex} +$ +""".format( + matchers=_MATCHERS, + base_version_regex=_BASE_VERSION_REGEX, + version_extra_regex=_VERSION_EXTRA_REGEX), + re.VERBOSE) + + +class Matchers: + GREATER_THAN = '>' + GREATER_THAN_OR_EQUAL = '>=' + LESS_THAN = '<' + LESS_THAN_OR_EQUAL = '<=' + EXACT = '=' + + +class VersionRange(dbt.utils.AttrDict): + + def _try_combine_exact(self, a, b): + if a.compare(b) == 0: + return a + else: + raise VersionsNotCompatibleException() + + def _try_combine_lower_bound_with_exact(self, lower, exact): + comparison = lower.compare(exact) + less = comparison < 0 + equal = comparison == 0 + + if(comparison < 0 or + (comparison == 0 and + lower.matcher == Matchers.GREATER_THAN_OR_EQUAL)): + return exact + + raise VersionsNotCompatibleException() + + def _try_combine_lower_bound(self, a, b): + if b.is_unbounded: + return a + elif a.is_unbounded: + return b + + if not (a.is_exact or b.is_exact): + comparison = (a.compare(b) < 0) + + if comparison: + return b + else: + return a + + elif a.is_exact: + return self._try_combine_lower_bound_with_exact(b, a) + + elif b.is_exact: + return self._try_combine_lower_bound_with_exact(a, b) + + def _try_combine_upper_bound_with_exact(self, upper, exact): + comparison = upper.compare(exact) + less = comparison < 0 + equal = comparison == 0 + + if(comparison > 0 or + (comparison == 0 and + upper.matcher == Matchers.LESS_THAN_OR_EQUAL)): + return exact + + raise VersionsNotCompatibleException() + + def _try_combine_upper_bound(self, a, b): + if b.is_unbounded: + return a + elif a.is_unbounded: + return b + + if not (a.is_exact or b.is_exact): + comparison = (a.compare(b) > 0) + + if comparison: + return b + else: + return a + + elif a.is_exact: + return self._try_combine_upper_bound_with_exact(b, a) + + elif b.is_exact: + return self._try_combine_upper_bound_with_exact(a, b) + + def reduce(self, other): + start = None + + if(self.start.is_exact and other.start.is_exact): + start = end = self._try_combine_exact(self.start, other.start) + + else: + start = self._try_combine_lower_bound(self.start, other.start) + end = self._try_combine_upper_bound(self.end, other.end) + + if start.compare(end) > 0: + raise VersionsNotCompatibleException() + + return VersionRange(start=start, end=end) + + +class VersionSpecifier(dbt.utils.AttrDict): + + def __init__(self, *args, **kwargs): + super(VersionSpecifier, self).__init__(*args, **kwargs) + + if self.matcher is None: + self.matcher = Matchers.EXACT + + @classmethod + def from_version_string(cls, version_string): + match = _VERSION_REGEX.match(version_string) + + if match is None: + # error? + return None + + return VersionSpecifier(match.groupdict()) + + def to_range(self): + range_start = UnboundedVersionSpecifier() + range_end = UnboundedVersionSpecifier() + + if self.matcher == Matchers.EXACT: + range_start = self + range_end = self + + elif self.matcher in [Matchers.GREATER_THAN, + Matchers.GREATER_THAN_OR_EQUAL]: + range_start = self + + elif self.matcher in [Matchers.LESS_THAN, + Matchers.LESS_THAN_OR_EQUAL]: + range_end = self + + return VersionRange( + start=range_start, + end=range_end) + + def compare(self, other): + if self.is_unbounded or other.is_unbounded: + return 0 + + for key in ['major', 'minor', 'patch']: + comparison = int(self[key]) - int(other[key]) + + if comparison != 0: + return comparison + + if((self.matcher == Matchers.GREATER_THAN_OR_EQUAL and + other.matcher == Matchers.LESS_THAN_OR_EQUAL) or + (self.matcher == Matchers.LESS_THAN_OR_EQUAL and + other.matcher == Matchers.GREATER_THAN_OR_EQUAL)): + return 0 + + if((self.matcher == Matchers.LESS_THAN and + other.matcher == Matchers.LESS_THAN_OR_EQUAL) or + (other.matcher == Matchers.GREATER_THAN and + self.matcher == Matchers.GREATER_THAN_OR_EQUAL) or + (self.is_upper_bound and other.is_lower_bound)): + return -1 + + if((other.matcher == Matchers.LESS_THAN and + self.matcher == Matchers.LESS_THAN_OR_EQUAL) or + (self.matcher == Matchers.GREATER_THAN and + other.matcher == Matchers.GREATER_THAN_OR_EQUAL) or + (self.is_lower_bound and other.is_upper_bound)): + return 1 + + return 0 + + @property + def is_unbounded(self): + return False + + @property + def is_lower_bound(self): + return self.matcher in [Matchers.GREATER_THAN, + Matchers.GREATER_THAN_OR_EQUAL] + + @property + def is_upper_bound(self): + return self.matcher in [Matchers.LESS_THAN, + Matchers.LESS_THAN_OR_EQUAL] + + @property + def is_exact(self): + return self.matcher == Matchers.EXACT + + +class UnboundedVersionSpecifier(VersionSpecifier): + + def __init__(self, *args, **kwargs): + super(dbt.utils.AttrDict, self).__init__(*args, **kwargs) + + @property + def is_unbounded(self): + return True + + @property + def is_lower_bound(self): + return False + + @property + def is_upper_bound(self): + return False + + @property + def is_exact(self): + return False + + +def reduce_versions(*args): + version_specifiers = [ + VersionSpecifier.from_version_string(version_string) + for version_string in args] + + for version_specifier in version_specifiers: + if not isinstance(version_specifier, VersionSpecifier): + raise Exception(version_specifier) + + try: + to_return = version_specifiers.pop().to_range() + + for version_specifier in version_specifiers: + to_return = to_return.reduce(version_specifier.to_range()) + except VersionsNotCompatibleException as e: + raise VersionsNotCompatibleException( + 'Could not find a satisfactory version from options: {}' + .format(str(args))) + + return to_return + +def versions_compatible(*args): + """ + n-to-n comparison of compatibility. Note that this can be very slow + for large numbers of arguments. + """ + if len(args) == 1: + return True + + try: + reduce_versions(*args) + return True + except VersionsNotCompatibleException as e: + return False diff --git a/test/unit/test_semver.py b/test/unit/test_semver.py new file mode 100644 index 00000000000..b7fb513692a --- /dev/null +++ b/test/unit/test_semver.py @@ -0,0 +1,98 @@ +import unittest +import itertools + +from dbt.exceptions import VersionsNotCompatibleException +from dbt.semver import VersionSpecifier, UnboundedVersionSpecifier, \ + VersionRange, reduce_versions, versions_compatible + +def create_range(start_version_string, end_version_string): + start = UnboundedVersionSpecifier() + end = UnboundedVersionSpecifier() + + if start_version_string is not None: + start = VersionSpecifier.from_version_string(start_version_string) + + if end_version_string is not None: + end = VersionSpecifier.from_version_string(end_version_string) + + return VersionRange(start=start, end=end) + +class TestSemver(unittest.TestCase): + + def assertVersionSetResult(self, inputs, output_range): + expected = create_range(*output_range) + + for permutation in itertools.permutations(inputs): + self.assertDictEqual( + reduce_versions(*permutation), + expected) + + def assertInvalidVersionSet(self, inputs): + for permutation in itertools.permutations(inputs): + with self.assertRaises(VersionsNotCompatibleException): + reduce_versions(*permutation) + + def test__versions_compatible(self): + self.assertTrue( + versions_compatible('0.0.1', '0.0.1')) + self.assertFalse( + versions_compatible('0.0.1', '0.0.2')) + self.assertTrue( + versions_compatible('>0.0.1', '0.0.2')) + + def test__reduce_versions(self): + self.assertVersionSetResult( + ['0.0.1', '0.0.1'], + ['=0.0.1', '=0.0.1']) + + self.assertVersionSetResult( + ['0.0.1'], + ['=0.0.1', '=0.0.1']) + + self.assertVersionSetResult( + ['>0.0.1'], + ['>0.0.1', None]) + + self.assertVersionSetResult( + ['<0.0.1'], + [None, '<0.0.1']) + + self.assertVersionSetResult( + ['>0.0.1', '0.0.2'], + ['=0.0.2', '=0.0.2']) + + self.assertVersionSetResult( + ['0.0.2', '>=0.0.2'], + ['=0.0.2', '=0.0.2']) + + self.assertVersionSetResult( + ['>0.0.1', '>0.0.2', '>0.0.3'], + ['>0.0.3', None]) + + self.assertVersionSetResult( + ['>0.0.1', '<0.0.3'], + ['>0.0.1', '<0.0.3']) + + self.assertVersionSetResult( + ['>0.0.1', '0.0.2', '<0.0.3'], + ['=0.0.2', '=0.0.2']) + + self.assertVersionSetResult( + ['>0.0.1', '>=0.0.1', '<0.0.3'], + ['>0.0.1', '<0.0.3']) + + self.assertVersionSetResult( + ['>0.0.1', '<0.0.3', '<=0.0.3'], + ['>0.0.1', '<0.0.3']) + + self.assertVersionSetResult( + ['<=0.0.3', '>=0.0.3'], + ['>=0.0.3', '<=0.0.3']) + + self.assertInvalidVersionSet(['>0.0.2', '0.0.1']) + self.assertInvalidVersionSet(['>0.0.2', '0.0.2']) + self.assertInvalidVersionSet(['<0.0.2', '0.0.2']) + self.assertInvalidVersionSet(['<0.0.2', '>0.0.3']) + self.assertInvalidVersionSet(['<=0.0.3', '>0.0.3']) + self.assertInvalidVersionSet(['<0.0.3', '>=0.0.3']) + self.assertInvalidVersionSet(['<0.0.3', '>0.0.3']) From 57246944c01c526e8efc353e0b1b0201d80a7b22 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 28 Sep 2017 14:32:33 -0400 Subject: [PATCH 02/13] cleanup --- dbt/semver.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt/semver.py b/dbt/semver.py index 0efe754a12d..415bfc1aa70 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -56,8 +56,6 @@ def _try_combine_exact(self, a, b): def _try_combine_lower_bound_with_exact(self, lower, exact): comparison = lower.compare(exact) - less = comparison < 0 - equal = comparison == 0 if(comparison < 0 or (comparison == 0 and @@ -88,8 +86,6 @@ def _try_combine_lower_bound(self, a, b): def _try_combine_upper_bound_with_exact(self, upper, exact): comparison = upper.compare(exact) - less = comparison < 0 - equal = comparison == 0 if(comparison > 0 or (comparison == 0 and From 9948c0f02d92a559778b9d36b3b691aaa40adedc Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 28 Sep 2017 14:33:48 -0400 Subject: [PATCH 03/13] remove unnecessary comment --- dbt/semver.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt/semver.py b/dbt/semver.py index 415bfc1aa70..b7c96a6f815 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -263,10 +263,6 @@ def reduce_versions(*args): return to_return def versions_compatible(*args): - """ - n-to-n comparison of compatibility. Note that this can be very slow - for large numbers of arguments. - """ if len(args) == 1: return True From cf65b71650e383aa27d40682e74554b377d3b1fe Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 28 Sep 2017 14:34:53 -0400 Subject: [PATCH 04/13] add test for multiples on both sides --- test/unit/test_semver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/unit/test_semver.py b/test/unit/test_semver.py index b7fb513692a..84f9f723395 100644 --- a/test/unit/test_semver.py +++ b/test/unit/test_semver.py @@ -85,6 +85,10 @@ def test__reduce_versions(self): ['>0.0.1', '<0.0.3', '<=0.0.3'], ['>0.0.1', '<0.0.3']) + self.assertVersionSetResult( + ['>0.0.1', '>0.0.2', '<0.0.3', '<0.0.4'], + ['>0.0.2', '<0.0.3']) + self.assertVersionSetResult( ['<=0.0.3', '>=0.0.3'], ['>=0.0.3', '<=0.0.3']) From 5cf1cc715249eafc190bab63b3963a22c4644c2b Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 28 Sep 2017 15:01:49 -0400 Subject: [PATCH 05/13] add resolve_to_specific_version --- dbt/semver.py | 31 ++++++++++++++++++++++++++++--- test/unit/test_semver.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/dbt/semver.py b/dbt/semver.py index b7c96a6f815..f33d231738d 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -242,9 +242,18 @@ def is_exact(self): def reduce_versions(*args): - version_specifiers = [ - VersionSpecifier.from_version_string(version_string) - for version_string in args] + version_specifiers = [] + + for version in args: + if isinstance(version, UnboundedVersionSpecifier): + continue + + elif isinstance(version, VersionSpecifier): + version_specifiers.append(version) + + else: + version_specifiers.append( + VersionSpecifier.from_version_string(version)) for version_specifier in version_specifiers: if not isinstance(version_specifier, VersionSpecifier): @@ -271,3 +280,19 @@ def versions_compatible(*args): return True except VersionsNotCompatibleException as e: return False + +def resolve_to_specific_version(requested_range, available_versions): + max_version = None + max_version_string = None + + for version_string in available_versions: + version = VersionSpecifier.from_version_string(version_string) + + if(versions_compatible(version, + requested_range.start, + requested_range.end) and + (max_version is None or max_version.compare(version) < 0)): + max_version = version + max_version_string = version_string + + return max_version_string diff --git a/test/unit/test_semver.py b/test/unit/test_semver.py index 84f9f723395..b51548ba3ef 100644 --- a/test/unit/test_semver.py +++ b/test/unit/test_semver.py @@ -3,7 +3,8 @@ from dbt.exceptions import VersionsNotCompatibleException from dbt.semver import VersionSpecifier, UnboundedVersionSpecifier, \ - VersionRange, reduce_versions, versions_compatible + VersionRange, reduce_versions, versions_compatible, \ + resolve_to_specific_version def create_range(start_version_string, end_version_string): start = UnboundedVersionSpecifier() @@ -100,3 +101,34 @@ def test__reduce_versions(self): self.assertInvalidVersionSet(['<=0.0.3', '>0.0.3']) self.assertInvalidVersionSet(['<0.0.3', '>=0.0.3']) self.assertInvalidVersionSet(['<0.0.3', '>0.0.3']) + + def test__resolve_to_specific_version(self): + self.assertEqual( + resolve_to_specific_version( + create_range('>0.0.1', None), + ['0.0.1', '0.0.2']), + '0.0.2') + + self.assertEqual( + resolve_to_specific_version( + create_range('>=0.0.2', None), + ['0.0.1', '0.0.2']), + '0.0.2') + + self.assertEqual( + resolve_to_specific_version( + create_range('>=0.0.3', None), + ['0.0.1', '0.0.2']), + None) + + self.assertEqual( + resolve_to_specific_version( + create_range('>=0.0.3', '<0.0.5'), + ['0.0.3', '0.0.4', '0.0.5']), + '0.0.4') + + self.assertEqual( + resolve_to_specific_version( + create_range(None, '<=0.0.5'), + ['0.0.3', '0.1.4', '0.0.5']), + '0.0.5') From cb95c0bd3f097170cf940a8b0fb74a5af48eec54 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Fri, 29 Sep 2017 10:01:25 -0400 Subject: [PATCH 06/13] local registry --- dbt/clients/registry.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 dbt/clients/registry.py diff --git a/dbt/clients/registry.py b/dbt/clients/registry.py new file mode 100644 index 00000000000..46b9f26677f --- /dev/null +++ b/dbt/clients/registry.py @@ -0,0 +1,23 @@ +import requests + + +def index(): + return requests.get('http://127.0.0.1:4567/api/v1/index.json').json() + + +def package(name): + return requests.get( + 'http://127.0.0.1:4567/api/v1/{}.json' + .format(name)).json() + + +def package_version(name, version): + return requests.get( + 'http://127.0.0.1:4567/api/v1/{}/{}.json' + .format(name, version)).json() + + +def get_available_versions(name): + response = package(name) + + return list(response['versions'].keys()) From 21704a913f3441b726d7aa41eaa57d1ee44d9548 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 8 Nov 2017 10:15:16 -0500 Subject: [PATCH 07/13] hacking out deps --- dbt/clients/registry.py | 29 ++++-- dbt/exceptions.py | 5 +- dbt/semver.py | 188 ++++++++++++++++++++++++++++++++++++++- dbt/task/deps.py | 135 +++++++++++++++++++++++++++- test/unit/test_semver.py | 137 +++++++++++++++++++++++++++- 5 files changed, 481 insertions(+), 13 deletions(-) diff --git a/dbt/clients/registry.py b/dbt/clients/registry.py index 46b9f26677f..8371e68b149 100644 --- a/dbt/clients/registry.py +++ b/dbt/clients/registry.py @@ -1,20 +1,33 @@ import requests -def index(): - return requests.get('http://127.0.0.1:4567/api/v1/index.json').json() +DEFAULT_REGISTRY_BASE_URL = 'http://127.0.0.1:4567/' -def package(name): +def _get_url(url, registry_base_url=None): + if registry_base_url is None: + registry_base_url = DEFAULT_REGISTRY_BASE_URL + + return '{}{}'.format(registry_base_url, url) + + +def index(registry_base_url=None): return requests.get( - 'http://127.0.0.1:4567/api/v1/{}.json' - .format(name)).json() + _get_url('api/v1/index.json', + registry_base_url)).json() -def package_version(name, version): +def package(name, registry_base_url=None): return requests.get( - 'http://127.0.0.1:4567/api/v1/{}/{}.json' - .format(name, version)).json() + _get_url('api/v1/{}.json'.format(name), + registry_base_url)).json() + + +def package_version(name, version, registry_base_url=None): + url = _get_url('api/v1/{}/{}.json'.format(name, version)) + response = requests.get(url, registry_base_url) + + return response.json() def get_available_versions(name): diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 166c8b19df2..a17a6a74ae9 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -88,8 +88,11 @@ class ValidationException(RuntimeException): pass -class VersionsNotCompatibleException(ValidationException): +class ParsingException(Exception): + pass + +class VersionsNotCompatibleException(ParsingException): def __init__(self, msg=None): self.msg = msg diff --git a/dbt/semver.py b/dbt/semver.py index f33d231738d..083bd67edaf 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -129,6 +129,31 @@ def reduce(self, other): return VersionRange(start=start, end=end) + def __str__(self): + result = [] + + if self.start.is_unbounded and self.end.is_unbounded: + return 'ANY' + + if not self.start.is_unbounded: + result.append(self.start.to_version_string()) + + if not self.end.is_unbounded: + result.append(self.end.to_version_string()) + + return ', '.join(result) + + def to_version_string_pair(self): + to_return = [] + + if not self.start.is_unbounded: + to_return.append(self.start.to_version_string()) + + if not self.end.is_unbounded: + to_return.append(self.end.to_version_string()) + + return to_return + class VersionSpecifier(dbt.utils.AttrDict): @@ -138,6 +163,27 @@ def __init__(self, *args, **kwargs): if self.matcher is None: self.matcher = Matchers.EXACT + def to_version_string(self, skip_matcher=False): + prerelease = '' + build = '' + matcher = '' + + if self.prerelease: + prerelease = '-' + self.prerelease + + if self.build: + build = '+' + self.build + + if not skip_matcher: + matcher = self.matcher + return '{}{}.{}.{}{}{}'.format( + matcher, + self.major, + self.minor, + self.patch, + prerelease, + build) + @classmethod def from_version_string(cls, version_string): match = _VERSION_REGEX.match(version_string) @@ -245,12 +291,19 @@ def reduce_versions(*args): version_specifiers = [] for version in args: - if isinstance(version, UnboundedVersionSpecifier): + if isinstance(version, UnboundedVersionSpecifier) or version is None: continue elif isinstance(version, VersionSpecifier): version_specifiers.append(version) + elif isinstance(version, VersionRange): + if not isinstance(version.start, UnboundedVersionSpecifier): + version_specifiers.append(version.start) + + if not isinstance(version.end, UnboundedVersionSpecifier): + version_specifiers.append(version.end) + else: version_specifiers.append( VersionSpecifier.from_version_string(version)) @@ -259,6 +312,10 @@ def reduce_versions(*args): if not isinstance(version_specifier, VersionSpecifier): raise Exception(version_specifier) + if not version_specifiers: + return VersionRange(start=UnboundedVersionSpecifier(), + end=UnboundedVersionSpecifier()) + try: to_return = version_specifiers.pop().to_range() @@ -271,6 +328,7 @@ def reduce_versions(*args): return to_return + def versions_compatible(*args): if len(args) == 1: return True @@ -281,6 +339,21 @@ def versions_compatible(*args): except VersionsNotCompatibleException as e: return False + +def find_possible_versions(requested_range, available_versions): + possible_versions = [] + + for version_string in available_versions: + version = VersionSpecifier.from_version_string(version_string) + + if(versions_compatible(version, + requested_range.start, + requested_range.end)): + possible_versions.append(version_string) + + return possible_versions[::-1] + + def resolve_to_specific_version(requested_range, available_versions): max_version = None max_version_string = None @@ -296,3 +369,116 @@ def resolve_to_specific_version(requested_range, available_versions): max_version_string = version_string return max_version_string + + +def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): + for name, restriction in restrictions.items(): + if not versions_compatible(*restriction): + raise VersionsNotCompatibleException('not compatible {}'.format(restriction)) + + if not unmet_dependencies: + return {}, {} + + to_return_tree = {} + to_return_install = {} + + for dependency_name, version in unmet_dependencies.items(): + print('resolving path {}'.format(dependency_name)) + dependency_restrictions = reduce_versions( + *restrictions.copy().get(dependency_name)) + + possible_matches = find_possible_versions( + dependency_restrictions, + version_index[dependency_name].keys()) + + for possible_match in possible_matches: + print('reset with {} at {}'.format(dependency_name, possible_match)) + + tree = {} + install = {} + new_restrictions = {} + new_unmet_dependencies = {} + + match_found = False + + try: + new_restrictions = restrictions.copy() + new_restrictions[dependency_name] = reduce_versions( + dependency_restrictions, + possible_match + ).to_version_string_pair() + + recursive_version_info = version_index.get(dependency_name, {}).get(possible_match) + new_unmet_dependencies = dbt.utils.deep_merge( + recursive_version_info.copy()) + + print('new unmet dependencies') + print(new_unmet_dependencies) + + new_restrictions = dbt.utils.deep_merge( + new_restrictions.copy(), + unmet_dependencies.copy(), + new_unmet_dependencies.copy()) + + new_restrictions[dependency_name] += [possible_match] + + if dependency_name in new_unmet_dependencies: + del new_unmet_dependencies[dependency_name] + + for name, restriction in new_restrictions.items(): + if not versions_compatible(*restriction): + raise VersionsNotCompatibleException('not compatible {}'.format(new_restrictions)) + + else: + match_found = True + + print('going down the stack with {}'.format(new_unmet_dependencies)) + print('and {}'.format(install)) + subtree, subinstall = resolve_dependency_tree( + version_index, + new_unmet_dependencies, + new_restrictions) + + tree.update({ + dependency_name: { + 'version': possible_match, + 'satisfies': [dependency_name], + 'dependencies': subtree + } + }) + + install = dbt.utils.deep_merge( + install, + subinstall, + {dependency_name: possible_match}) + + print('then {}'.format(install)) + + to_return_tree = dbt.utils.deep_merge( + to_return_tree, + tree) + + to_return_install = dbt.utils.deep_merge( + to_return_install, + install) + + break + + if not match_found: + raise VersionsNotCompatibleException('No match found -- exhausted this part of the ' + 'tree.') + + except VersionsNotCompatibleException as e: + print(e) + print('When attempting {} at {}'.format(dependency_name, possible_match)) + + return to_return_tree.copy(), to_return_install.copy() + + +def resolve_dependency_set(version_index, dependencies): + tree, install = resolve_dependency_tree(version_index, dependencies, dependencies) + + return { + 'install': install, + 'tree': tree, + } diff --git a/dbt/task/deps.py b/dbt/task/deps.py index 050f8874fbb..c75edd5a10f 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -4,14 +4,68 @@ import dbt.clients.git import dbt.clients.system +import dbt.clients.registry import dbt.project as project from dbt.compat import basestring from dbt.logger import GLOBAL_LOGGER as logger +from dbt.semver import VersionSpecifier, UnboundedVersionSpecifier +from dbt.utils import AttrDict from dbt.task.base_task import BaseTask +class PackageListing(AttrDict): + + @classmethod + def _convert_version_strings(cls, version_strings): + if not isinstance(version_strings, list): + version_strings = [version_strings] + + return [ + VersionSpecifier.from_version_string(version_string) + for version_string in version_strings + ] + + def incorporate(self, package, version_specifiers=None): + if version_specifiers is None: + version_specifiers = [UnboundedVersionSpecifier()] + elif not isinstance(version_specifiers, list): + # error + raise Exception('bad') + else: + for version_specifier in version_specifiers: + if not isinstance(version_specifier, VersionSpecifier): + # error + raise Exception('bad') + + if package not in self: + self[package] = version_specifiers + + else: + self[package] = self[package] + version_specifiers + + @classmethod + def create(cls, parsed_yaml): + to_return = cls({}) + + if not isinstance(parsed_yaml, list): + # error + raise Exception('bad') + + if isinstance(parsed_yaml, list): + for package in parsed_yaml: + if isinstance(package, basestring): + to_return.incorporate(package) + elif isinstance(package, dict): + (package, version_strings) = package.popitem() + to_return.incorporate( + package, + cls._convert_version_strings(version_strings)) + + return to_return + + def folder_from_git_remote(remote_spec): start = remote_spec.rfind('/') + 1 end = len(remote_spec) - (4 if remote_spec.endswith('.git') else 0) @@ -127,6 +181,83 @@ def __pull_deps_recursive(self, repos, processed_repos=None, i=0): raise e def run(self): - dbt.clients.system.make_directory(self.project['modules-path']) + listing = PackageListing.create(self.project['packages']) + visited_listing = PackageListing.create([]) + index = dbt.clients.registry.index() + + while len(listing) > 0: + (package, version_specifiers) = listing.popitem() + + if package not in index: + raise Exception('unknown package {}'.format(package)) + + version_range = dbt.semver.reduce_versions( + *version_specifiers) + + available_versions = dbt.clients.registry.get_available_versions( + package) + + # for now, pick a version and then recurse. later on, + # we'll probably want to traverse multiple options + # so we can match packages. not going to make a difference + # right now. + target_version = dbt.semver.resolve_to_specific_version( + version_range, + available_versions) + + if target_version is None: + logger.error( + 'Could not find a matching version for package {}!' + .format(package)) + logger.error( + ' Requested range: {}'.format(version_range)) + logger.error( + ' Available versions: {}'.format( + ', '.join(available_versions))) + raise Exception('bad') + + visited_listing.incorporate( + package, + [VersionSpecifier.from_version_string(target_version)]) + + target_version_metadata = dbt.clients.registry.package_version( + package, target_version) + + dependencies = target_version_metadata.get('dependencies', {}) + + for package, versions in dependencies.items(): + listing.incorporate( + package, + [VersionSpecifier.from_version_string(version) + for version in versions]) + + for package, version_specifiers in visited_listing.items(): + version_string = version_specifiers[0].to_version_string(True) + version_info = dbt.clients.registry.package_version( + package, version_string) + + import requests + + tar_path = os.path.realpath('{}/downloads/{}.{}.tar.gz'.format( + self.project['modules-path'], + package, + version_string)) + + logger.info("Pulling {}@{} from hub.getdbt.com...".format( + package, version_string)) + + dbt.clients.system.make_directory( + os.path.dirname(tar_path)) + + response = requests.get(version_info.get('downloads').get('tarball')) + + with open(tar_path, 'wb') as handle: + for block in response.iter_content(1024*64): + handle.write(block) + + import tarfile + + with tarfile.open(tar_path, 'r') as tarball: + tarball.extractall(self.project['modules-path']) - self.__pull_deps_recursive(self.project['repositories']) + logger.info(" -> Success.") diff --git a/test/unit/test_semver.py b/test/unit/test_semver.py index b51548ba3ef..72bb7294c42 100644 --- a/test/unit/test_semver.py +++ b/test/unit/test_semver.py @@ -4,7 +4,8 @@ from dbt.exceptions import VersionsNotCompatibleException from dbt.semver import VersionSpecifier, UnboundedVersionSpecifier, \ VersionRange, reduce_versions, versions_compatible, \ - resolve_to_specific_version + resolve_to_specific_version, resolve_dependency_set + def create_range(start_version_string, end_version_string): start = UnboundedVersionSpecifier() @@ -18,6 +19,7 @@ def create_range(start_version_string, end_version_string): return VersionRange(start=start, end=end) + class TestSemver(unittest.TestCase): def assertVersionSetResult(self, inputs, output_range): @@ -132,3 +134,136 @@ def test__resolve_to_specific_version(self): create_range(None, '<=0.0.5'), ['0.0.3', '0.1.4', '0.0.5']), '0.0.5') + + def test__resolve_dependency_set__multiple_deps(self): + self.maxDiff = None + self.assertDictEqual( + resolve_dependency_set( + version_index={ + 'a': { + '0.0.1': { + 'b': ['=0.0.1'], + }, + '0.0.2': { + 'b': ['=0.0.1'], + }, + '0.0.3': { + 'b': ['>=0.0.1'], + 'c': ['=0.0.2'], + } + }, + 'b': { + '0.0.1': { + 'c': ['=0.0.2'], + }, + '0.0.2': { + 'c': ['=0.0.1'], + } + }, + 'c': { + '0.0.1': {}, + '0.0.2': {} + } + }, + dependencies={ + 'a': [] + } + ), + { + 'install': { + 'a': '0.0.3', + 'b': '0.0.1', + 'c': '0.0.2', + }, + 'tree': { + 'a': { + 'version': '0.0.3', + 'satisfies': ['a'], + 'dependencies': { + 'b': { + 'version': '0.0.1', + 'satisfies': ['b'], + 'dependencies': { + 'c': { + 'version': '0.0.2', + 'satisfies': ['c'], + 'dependencies': {} + } + } + }, + 'c': { + 'version': '0.0.2', + 'satisfies': ['c'], + 'dependencies': {}, + } + } + } + } + }) + + def test__resolve_dependency_set(self): + self.maxDiff = None + self.assertDictEqual( + resolve_dependency_set( + version_index={ + 'a': { + '0.0.1': { + 'b': ['=0.0.1'], + }, + '0.0.2': { + 'b': ['=0.0.1'], + }, + '0.0.3': { + 'b': ['>=0.0.1'], + 'c': ['=0.0.2'], + } + }, + 'b': { + '0.0.1': { + 'c': ['=0.0.1'], + }, + '0.0.2': { + 'c': ['=0.0.2'], + } + }, + 'c': { + '0.0.1': {}, + '0.0.2': {} + } + }, + dependencies={ + 'a': [] + } + ), + { + 'install': { + 'a': '0.0.3', + 'b': '0.0.2', + 'c': '0.0.2', + }, + 'tree': { + 'a': { + 'version': '0.0.3', + 'satisfies': ['a'], + 'dependencies': { + 'b': { + 'version': '0.0.2', + 'satisfies': ['b'], + 'dependencies': { + 'c': { + 'version': '0.0.2', + 'satisfies': ['c'], + 'dependencies': {} + } + } + }, + 'c': { + 'version': '0.0.2', + 'satisfies': ['c'], + 'dependencies': {} + } + + } + } + } + }) From 4e2cb401d6f62a9b58dfa60d2607d82108a03079 Mon Sep 17 00:00:00 2001 From: Buck Ryan Date: Mon, 26 Feb 2018 19:14:45 -0500 Subject: [PATCH 08/13] Buck pkg mgmt (#645) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * only load hooks and archives once (#540) * sets schema for node before parsing raw sql (#541) * Fix/env vars (#543) * fix for bad env_var exception * overwrite target with compiled values * fixes env vars, adds test. Auto-compile profile/target args * improvements for code that runs in hooks (#544) * improvements for code that runs in hooks * fix error message note * typo * Update CHANGELOG.md * bump version (#546) * add scope to service account json creds initializer (#547) * bump 0.9.0a3 --> 0.9.0a4 (#548) * Fix README links (#554) * Update README.md * handle empty profiles.yml file (#555) * return empty string (instead of None) to avoid polluting rendered sql (#566) * tojson was added in jinja 2.9 (#563) * tojson was added in jinja 2.9 * requirements * fix package-defined schema test macros (#562) * fix package-defined schema test macros * create a dummy Relation in parsing * fix for bq quoting (#565) * bump snowflake, remove pyasn1 (#570) * bump snowflake, remove pyasn1 * change requirements.txt * allow macros to return non-text values (#571) * revert jinja version, implement tojson hack (#572) * bump to 090a5 * update changelog * bump (#574) * 090 docs (#575) * 090 docs * Update CHANGELOG.md * Update CHANGELOG.md * Raise CompilationException on duplicate model (#568) * Raise CompilationException on duplicate model Extend tests * Ignore disabled models in parse_sql_nodes Extend tests for duplicate model * Fix preexisting models * Use double quotes consistently Rename model-1 to model-disabled * Fix unit tests * Raise exception on duplicate model across packages Extend tests * Make run_started_at timezone aware (#553) (#556) * Make run_started_at timezone aware Set run_started_at timezone to UTC Enable timezone change in models Extend requirements Extend tests * Address comments from code review Create modules namespace to context Move pytz to modules Add new dependencies to setup.py * Add warning for missing constraints. Fixes #592 (#600) * Add warning for missing constraints. Fixes #592 * fix unit tests * fix schema tests used in, or defined in packages (#599) * fix schema tests used in, or defined in packages * don't hardcode dbt test namespace * fix/actually run tests * rm junk * run hooks in correct order, fixes #590 (#601) * run hooks in correct order, fixes #590 * add tests * fix tests * pep8 * change req for snowflake to fix crypto install issue (#612) From cffi callback : Traceback (most recent call last): File "c:\projects\dbt\.tox\pywin\lib\site-packages\OpenSSL\SSL.py", line 313, in wrapper _lib.X509_up_ref(x509) AttributeError: module 'lib' has no attribute 'X509_up_ref' From cffi callback : * Update python version in Makefile from 3.5 to 3.6 (#613) * Fix/snowflake custom schema (#626) * Fixes already opened transaction issue For https://github.com/fishtown-analytics/dbt/issues/602 * Fixes https://github.com/fishtown-analytics/dbt/issues/621 * Create schema in archival flow (#625) * Fix for pre-hooks outside of transactions (#623) * Fix for pre-hooks outside of transactions https://github.com/fishtown-analytics/dbt/issues/576 * improve tests * Fixes already opened transaction issue (#622) For https://github.com/fishtown-analytics/dbt/issues/602 * Accept string for postgres port number (#583) (#624) * Accept string for postgres port number (#583) * s/str/basestring/g * print correct run time (include hooks) (#607) * add support for late binding views (Redshift) (#614) * add support for late binding views (Redshift) * fix bind logic * wip for get_columns_in_table * fix get_columns_in_table * fix for default value in bind config * pep8 * skip tests that depend on nonexistent or disabled models (#617) * skip tests that depend on nonexistent or disabled models * pep8, Fixes https://github.com/fishtown-analytics/dbt/issues/616 * refactor * fix for adapter macro called within packages (#630) * fix for adapter macro called within packages * better error message * Update CHANGELOG.md (#632) * Update CHANGELOG.md * Update CHANGELOG.md * Bump version: 0.9.0 → 0.9.1 * more helpful exception for registry funcs * Rework deps to support local & git * pylint and cleanup * make modules directory first * Refactor registry client for cleanliness and better error handling * init converter script * create modules directory only if non-existent * Only check the hub registry for registry packages * Incorporate changes from Drew's branch Diff of original changes: https://github.com/fishtown-analytics/dbt/pull/591/files * lint * include a portion of the actual name in destination directory * Install dependencies using actual name; better exceptions * Error if two dependencies have same name * Process dependencies one level at a time Included in this change is a refactor of the deps run function for clarity. Also I changed the resolve_version function to update the object in place. I prefer the immutability of this function as it was, but the rest of the code doesn't really operate that way. And I ran into some bugs due to this discrepancy. * update var name * Provide support for repositories in project yml * Download files in a temp directory The downloads directory causes problems with the run command because this directory is not a dbt project. Need to download it elsewhere. * pin some versions * pep8-ify * some PR feedback changes around logging * PR feedback round 2 * Fix for redshift varchar bug (#647) * Fix for redshift varchar bug * pep8 on a sql string, smh * Set global variable overrides on the command line with --vars (#640) * Set global variable overrides on the command line with --vars * pep8 * integration tests for cli vars * Seed rewrite (#618) * loader for seed data files * Functioning rework of seed task * Make CompilerRunner fns private and impl. SeedRunner.compile Trying to distinguish between the public/private interface for this class. And the SeedRunner doesn't need the functionality in the compile function, it just needs a compile function to exist for use in the compilation process. * Test changes and fixes * make the DB setup script usable locally * convert simple copy test to use seeed * Fixes to get Snowflake working * New seed flag and make it non-destructive by default * Convert update SQL script to another seed * cleanup * implement bigquery csv load * context handling of StringIO * Better typing * strip seeder and csvkit dependency * update bigquery to use new data typing and to fix unicode issue * update seed test * fix abstract functions in base adapter * support time type * try pinning crypto, pyopenssl versions * remove unnecessary version pins * insert all at once, rather than one query per row * do not quote field names on creation * bad * quiet down parsedatetime logger * pep8 * UI updates + node conformity for seed nodes * add seed to list of resource types, cleanup * show option for CSVs * typo * pep8 * move agate import to avoid strange warnings * deprecation warning for --drop-existing * quote column names in seed files * revert quoting change (breaks Snowflake). Hush warnings * use hub url * Show installed version, silence semver regex warnings * sort versions to make tests deterministic. Prefer higher versions * pep8, fix comparison functions for py3 * make compare function return value in {-1, 0, 1} * fix for deleting git dirs on windows? * use system client rmdir instead of shutil directly * debug logging to identify appveyor issue * less restrictive error retry * rm debug logging --- .bumpversion.cfg | 2 +- CHANGELOG.md | 117 +++- Makefile | 4 +- README.md | 14 +- converter.py | 73 +++ dbt/adapters/bigquery.py | 64 +- dbt/adapters/default.py | 95 ++- dbt/adapters/postgres.py | 71 +++ dbt/adapters/redshift.py | 62 ++ dbt/adapters/snowflake.py | 16 +- dbt/clients/git.py | 32 +- dbt/clients/jinja.py | 2 + dbt/clients/registry.py | 53 +- dbt/clients/system.py | 49 +- dbt/compilation.py | 11 +- dbt/config.py | 5 +- dbt/context/common.py | 35 +- dbt/context/parser.py | 5 + dbt/context/runtime.py | 2 +- dbt/contracts/connection.py | 2 +- dbt/contracts/graph/parsed.py | 5 + dbt/contracts/graph/unparsed.py | 6 +- dbt/deprecations.py | 23 +- dbt/exceptions.py | 63 +- dbt/hooks.py | 5 +- .../global_project/macros/adapters/common.sql | 29 +- .../macros/adapters/redshift.sql | 10 + .../macros/materializations/archive.sql | 4 + .../macros/materializations/helpers.sql | 7 +- dbt/loader.py | 55 +- dbt/logger.py | 1 + dbt/main.py | 43 +- dbt/model.py | 3 +- dbt/node_runners.py | 92 ++- dbt/node_types.py | 4 +- dbt/parser.py | 134 +++- dbt/project.py | 21 +- dbt/runner.py | 8 +- dbt/seeder.py | 139 ----- dbt/semver.py | 132 ++-- dbt/task/deps.py | 554 ++++++++++------- dbt/task/init.py | 2 +- dbt/task/seed.py | 51 +- dbt/tracking.py | 3 +- dbt/ui/printer.py | 16 + dbt/utils.py | 129 +++- dbt/version.py | 6 +- dev_requirements.txt | 2 + docs/about/contributing.md | 1 - docs/about/license.md | 1 - docs/about/overview.md | 1 - docs/about/release-notes.md | 0 docs/about/viewpoint.md | 1 - docs/guide/archival.md | 1 - docs/guide/best-practices.md | 1 - docs/guide/building-models.md | 1 - docs/guide/configuring-models.md | 1 - docs/guide/context-variables.md | 1 - docs/guide/database-optimizations.md | 1 - docs/guide/macros.md | 1 - docs/guide/package-management.md | 1 - docs/guide/setup.md | 1 - docs/guide/testing.md | 1 - docs/guide/upgrading.md | 1 - docs/guide/usage.md | 1 - docs/guide/using-hooks.md | 1 - docs/index.md | 1 - requirements.txt | 7 +- sample.dbt_project.yml | 4 +- setup.py | 9 +- .../001_simple_copy_test/models/schema.yml | 8 + .../seed-initial/seed.csv | 101 +++ .../001_simple_copy_test/seed-update/seed.csv | 201 ++++++ .../integration/001_simple_copy_test/seed.sql | 111 ---- .../001_simple_copy_test/test_simple_copy.py | 42 +- .../001_simple_copy_test/update.sql | 101 --- .../integration/005_simple_seed_test/seed.sql | 6 +- .../008_schema_tests_test/macros/tests.sql | 2 +- .../models-custom/schema.yml | 6 +- .../008_schema_tests_test/models/disabled.sql | 6 - .../008_schema_tests_test/models/schema.yml | 7 - .../test_schema_tests.py | 11 +- .../test_invalid_models.py | 2 +- .../test_context_vars.py | 14 +- .../014_hook_tests/test_model_hooks.py | 17 +- .../014_hook_tests/test_run_hooks.py | 19 +- .../023_exit_codes_test/data-bad/data.csv | 4 +- .../023_exit_codes_test/test_exit_codes.py | 7 +- .../models-1/model-enabled-1/model.sql | 8 + .../models-1/model-enabled-2/model.sql | 8 + .../models-2/model-disabled/model.sql | 8 + .../models-2/model-enabled/model.sql | 8 + .../models-3/table.sql | 7 + .../models-4/table.sql | 8 + .../025_duplicate_model_test/seed.sql | 587 ++++++++++++++++++ .../test_duplicate_model.py | 155 +++++ .../025_timezones_test/models/timezones.sql | 10 + .../025_timezones_test/test_timezones.py | 55 ++ .../models_complex/complex_model.sql | 6 + .../028_cli_vars/models_complex/schema.yml | 7 + .../028_cli_vars/models_simple/schema.yml | 5 + .../models_simple/simple_model.sql | 4 + .../integration/028_cli_vars/test_cli_vars.py | 54 ++ test/integration/base.py | 9 +- test/setup_db.sh | 19 +- test/unit/test_graph.py | 29 - test/unit/test_parser.py | 42 -- 107 files changed, 3020 insertions(+), 973 deletions(-) create mode 100755 converter.py delete mode 100644 dbt/seeder.py delete mode 100644 docs/about/contributing.md delete mode 100644 docs/about/license.md delete mode 100644 docs/about/overview.md delete mode 100644 docs/about/release-notes.md delete mode 100644 docs/about/viewpoint.md delete mode 100644 docs/guide/archival.md delete mode 100644 docs/guide/best-practices.md delete mode 100644 docs/guide/building-models.md delete mode 100644 docs/guide/configuring-models.md delete mode 100644 docs/guide/context-variables.md delete mode 100644 docs/guide/database-optimizations.md delete mode 100644 docs/guide/macros.md delete mode 100644 docs/guide/package-management.md delete mode 100644 docs/guide/setup.md delete mode 100644 docs/guide/testing.md delete mode 100644 docs/guide/upgrading.md delete mode 100644 docs/guide/usage.md delete mode 100644 docs/guide/using-hooks.md delete mode 100644 docs/index.md create mode 100644 test/integration/001_simple_copy_test/models/schema.yml create mode 100644 test/integration/001_simple_copy_test/seed-initial/seed.csv create mode 100644 test/integration/001_simple_copy_test/seed-update/seed.csv delete mode 100644 test/integration/001_simple_copy_test/seed.sql delete mode 100644 test/integration/001_simple_copy_test/update.sql delete mode 100644 test/integration/008_schema_tests_test/models/disabled.sql create mode 100644 test/integration/025_duplicate_model_test/models-1/model-enabled-1/model.sql create mode 100644 test/integration/025_duplicate_model_test/models-1/model-enabled-2/model.sql create mode 100644 test/integration/025_duplicate_model_test/models-2/model-disabled/model.sql create mode 100644 test/integration/025_duplicate_model_test/models-2/model-enabled/model.sql create mode 100644 test/integration/025_duplicate_model_test/models-3/table.sql create mode 100644 test/integration/025_duplicate_model_test/models-4/table.sql create mode 100644 test/integration/025_duplicate_model_test/seed.sql create mode 100644 test/integration/025_duplicate_model_test/test_duplicate_model.py create mode 100644 test/integration/025_timezones_test/models/timezones.sql create mode 100644 test/integration/025_timezones_test/test_timezones.py create mode 100644 test/integration/028_cli_vars/models_complex/complex_model.sql create mode 100644 test/integration/028_cli_vars/models_complex/schema.yml create mode 100644 test/integration/028_cli_vars/models_simple/schema.yml create mode 100644 test/integration/028_cli_vars/models_simple/simple_model.sql create mode 100644 test/integration/028_cli_vars/test_cli_vars.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ca582ec0883..cd6c9e3ab87 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.9.0a2 +current_version = 0.9.1 commit = True tag = True diff --git a/CHANGELOG.md b/CHANGELOG.md index 27f65f556d2..0713dfa19f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,94 @@ -## dbt 0.9.0 Alpha 3 (September 26, 2017) +## dbt 0.9.1 (January 2, 2018) + +This release fixes bugs and adds supports for late binding views on Redshift. + +### Changes +- Support late binding views on Redshift ([#614](https://github.com/fishtown-analytics/dbt/pull/614)) ([docs](https://docs.getdbt.com/docs/warehouse-specific-configurations#section-late-binding-views)) +- Make `run_started_at` timezone-aware ([#553](https://github.com/fishtown-analytics/dbt/pull/553)) (Contributed by [@mturzanska](https://github.com/mturzanska)) ([docs](https://docs.getdbt.com/v0.9/reference#run_started_at)) + +### Bugfixes + +- Include hook run time in reported model run time ([#607](https://github.com/fishtown-analytics/dbt/pull/607)) +- Add warning for missing test constraints ([#600](https://github.com/fishtown-analytics/dbt/pull/600)) +- Fix for schema tests used or defined in packages ([#599](https://github.com/fishtown-analytics/dbt/pull/599)) +- Run hooks in defined order ([#601](https://github.com/fishtown-analytics/dbt/pull/601)) +- Skip tests that depend on nonexistent models ([#617](https://github.com/fishtown-analytics/dbt/pull/617)) +- Fix for `adapter_macro` called within a package ([#630](https://github.com/fishtown-analytics/dbt/pull/630)) + + +## dbt 0.9.0 (October 25, 2017) + +This release focuses on improvements to macros, materializations, and package management. Check out [the blog post](https://blog.fishtownanalytics.com/whats-new-in-dbt-0-9-0-dd36f3572ac6) to learn more about what's possible in this new version of dbt. + +### Installation + +Full installation instructions for macOS, Windows, and Linux can be found [here](https://docs.getdbt.com/v0.9/docs/installation). If you use Windows or Linux, installation works the same as with previous versions of dbt. If you use macOS and Homebrew to install dbt, note that installation instructions have changed: + +#### macOS Installation Instructions +```bash +brew update +brew tap fishtown-analytics/dbt +brew install dbt +``` + +### Overview + +- More powerful macros and materializations +- Custom model schemas +- BigQuery improvements +- Bugfixes +- Documentation (0.9.0 docs can be found [here](https://docs.getdbt.com/v0.9/)) + + +### Breaking Changes +- `adapter` functions must be namespaced to the `adapter` context variable. To fix this error, use `adapter.already_exists` instead of just `already_exists`, or similar for other [adapter functions](https://docs.getdbt.com/reference#adapter). + + +### Bugfixes +- Handle lingering `__dbt_tmp` relations ([#511](https://github.com/fishtown-analytics/dbt/pull/511)) +- Run tests defined in an ephemeral directory ([#509](https://github.com/fishtown-analytics/dbt/pull/509)) + + +### Changes +- use `adapter`, `ref`, and `var` inside of macros ([#466](https://github.com/fishtown-analytics/dbt/pull/466/files)) +- Build custom tests and materializations in dbt packages ([#466](https://github.com/fishtown-analytics/dbt/pull/466/files)) +- Support pre- and post- hooks that run outside of a transaction ([#510](https://github.com/fishtown-analytics/dbt/pull/510)) +- Support table materializations for BigQuery ([#507](https://github.com/fishtown-analytics/dbt/pull/507)) +- Support querying external data sources in BigQuery ([#507](https://github.com/fishtown-analytics/dbt/pull/507)) +- Override which schema models are materialized in ([#522](https://github.com/fishtown-analytics/dbt/pull/522)) ([docs](https://docs.getdbt.com/v0.9/docs/using-custom-schemas)) +- Make `{{ ref(...) }}` return the same type of object as `{{ this }} `([#530](https://github.com/fishtown-analytics/dbt/pull/530)) +- Replace schema test CTEs with subqueries to speed them up for Postgres ([#536](https://github.com/fishtown-analytics/dbt/pull/536)) ([@ronnyli](https://github.com/ronnyli)) + - Bump Snowflake dependency, remove pyasn1 ([#570](https://github.com/fishtown-analytics/dbt/pull/570)) + + +### Documentation +- Document how to [create a package](https://docs.getdbt.com/v0.9/docs/building-packages) +- Document how to [make a materialization](https://docs.getdbt.com/v0.9/docs/creating-new-materializations) +- Document how to [make custom schema tests](https://docs.getdbt.com/v0.9/docs/custom-schema-tests) +- Document how to [use hooks to vacuum](https://docs.getdbt.com/v0.9/docs/using-hooks#section-using-hooks-to-vacuum) +- Document [all context variables](https://docs.getdbt.com/v0.9/reference) + + +### New Contributors +- [@ronnyli](https://github.com/ronnyli) ([#536](https://github.com/fishtown-analytics/dbt/pull/536)) + + +## dbt 0.9.0 Alpha 5 (October 24, 2017) + +### Overview + - Bump Snowflake dependency, remove pyasn1 ([#570](https://github.com/fishtown-analytics/dbt/pull/570)) + +## dbt 0.9.0 Alpha 4 (October 3, 2017) + +### Bugfixes + - Fix for federated queries on BigQuery with Service Account json credentials ([#547](https://github.com/fishtown-analytics/dbt/pull/547)) + +## dbt 0.9.0 Alpha 3 (October 3, 2017) ### Overview - Bugfixes - Faster schema tests on Postgres + - Fix for broken environment variables ### Improvements @@ -12,6 +98,7 @@ - Fix broken integration tests ([#539](https://github.com/fishtown-analytics/dbt/pull/539)) - Fix for `--non-destructive` on views ([#539](https://github.com/fishtown-analytics/dbt/pull/539)) - Fix for package models materialized in the wrong schema ([#538](https://github.com/fishtown-analytics/dbt/pull/538)) +- Fix for broken environment variables ([#543](https://github.com/fishtown-analytics/dbt/pull/543)) ### New Contributors @@ -30,7 +117,7 @@ - Parity for `statement` interface on BigQuery ([#526](https://github.com/fishtown-analytics/dbt/pull/526)) ### Changes -- Override which schema models are materialized in ([#522](https://github.com/fishtown-analytics/dbt/pull/522)) ([docs](https://dbt.readme.io/v0.9/docs/using-custom-schemas)) +- Override which schema models are materialized in ([#522](https://github.com/fishtown-analytics/dbt/pull/522)) ([docs](https://docs.getdbt.com/v0.9/docs/using-custom-schemas)) - Make `{{ ref(...) }}` return the same type of object as `{{ this }} `([#530](https://github.com/fishtown-analytics/dbt/pull/530)) @@ -41,10 +128,10 @@ - More powerful macros - BigQuery improvements - Bugfixes -- Documentation (0.9.0 docs can be found [here](https://dbt.readme.io/v0.9/)) +- Documentation (0.9.0 docs can be found [here](https://docs.getdbt.com/v0.9/)) ### Breaking Changes -dbt 0.9.0 Alpha 1 introduces a number of new features intended to help dbt-ers write flexible, reusable code. The majority of these changes involve the `macro` and `materialization` Jinja blocks. As this is an alpha release, there may exist bugs or incompatibilites, particularly surrounding these two blocks. A list of known breaking changes is provided below. If you find new bugs, or have questions about dbt 0.9.0, please don't hesitate to reach out in [slack](http://ac-slackin.herokuapp.com/) or [open a new issue](https://github.com/fishtown-analytics/dbt/issues/new?milestone=0.9.0+alpha-1). +dbt 0.9.0 Alpha 1 introduces a number of new features intended to help dbt-ers write flexible, reusable code. The majority of these changes involve the `macro` and `materialization` Jinja blocks. As this is an alpha release, there may exist bugs or incompatibilites, particularly surrounding these two blocks. A list of known breaking changes is provided below. If you find new bugs, or have questions about dbt 0.9.0, please don't hesitate to reach out in [slack](http://slack.getdbt.com/) or [open a new issue](https://github.com/fishtown-analytics/dbt/issues/new?milestone=0.9.0+alpha-1). ##### 1. Adapter functions must be namespaced to the `adapter` context variable This will manifest as a compilation error that looks like: @@ -53,7 +140,7 @@ Compilation Error in model {your_model} (models/path/to/your_model.sql) 'already_exists' is undefined ``` -To fix this error, use `adapter.already_exists` instead of just `already_exists`, or similar for other [adapter functions](https://dbt.readme.io/reference#adapter). +To fix this error, use `adapter.already_exists` instead of just `already_exists`, or similar for other [adapter functions](https://docs.getdbt.com/reference#adapter). ### Bugfixes - Handle lingering `__dbt_tmp` relations ([#511](https://github.com/fishtown-analytics/dbt/pull/511)) @@ -67,9 +154,9 @@ To fix this error, use `adapter.already_exists` instead of just `already_exists` - Support querying external data sources in BigQuery ([#507](https://github.com/fishtown-analytics/dbt/pull/507)) ### Documentation -- Document how to [create a package](https://dbt.readme.io/v0.8/docs/building-packages) -- Document how to [make a materialization](https://dbt.readme.io/v0.8/docs/creating-new-materializations) -- Document how to [make custom schema tests](https://dbt.readme.io/v0.8/docs/custom-schema-tests) +- Document how to [create a package](https://docs.getdbt.com/v0.8/docs/building-packages) +- Document how to [make a materialization](https://docs.getdbt.com/v0.8/docs/creating-new-materializations) +- Document how to [make custom schema tests](https://docs.getdbt.com/v0.8/docs/custom-schema-tests) ## dbt 0.8.3 (July 14, 2017) @@ -93,9 +180,9 @@ To fix this error, use `adapter.already_exists` instead of just `already_exists` - Add context function to pull in environment variables ([#450](https://github.com/fishtown-analytics/dbt/issues/450)) ### Documentation -- Document target configuration for BigQuery [here](https://dbt.readme.io/v0.8/docs/supported-databases#section-bigquery) -- Document dbt exit codes [here](https://dbt.readme.io/v0.8/reference#exit-codes) -- Document environment variable usage [here](https://dbt.readme.io/v0.8/reference#env_var) +- Document target configuration for BigQuery [here](https://docs.getdbt.com/v0.8/docs/supported-databases#section-bigquery) +- Document dbt exit codes [here](https://docs.getdbt.com/v0.8/reference#exit-codes) +- Document environment variable usage [here](https://docs.getdbt.com/v0.8/reference#env_var) ## dbt 0.8.2 (May 31, 2017) @@ -131,7 +218,7 @@ To fix this error, use `adapter.already_exists` instead of just `already_exists` ### Overview - Bugfixes - Reintroduce `compile` command -- Moved docs to [readme.io](https://dbt.readme.io/) +- Moved docs to [readme.io](https://docs.getdbt.com/) ### Bugfixes @@ -434,7 +521,7 @@ With the dbt 0.5.4 release, dbt now features a robust integration test suite. Th You can check out the DBT roadmap [here](https://github.com/fishtown-analytics/dbt/milestones). In the next few weeks, we'll be working on [bugfixes](https://github.com/fishtown-analytics/dbt/milestone/11), [minor features](https://github.com/fishtown-analytics/dbt/milestone/15), [improved macro support](https://github.com/fishtown-analytics/dbt/milestone/14), and [expanded control over runtime materialization configs](https://github.com/fishtown-analytics/dbt/milestone/9). -As always, feel free to reach out to us on [Slack](http://ac-slackin.herokuapp.com/) with any questions or comments! +As always, feel free to reach out to us on [Slack](http://slack.getdbt.com/) with any questions or comments! --- @@ -486,7 +573,7 @@ As `dbt` has grown, we found this implementation to be a little unwieldy and har The additions of automated testing and a more comprehensive manual testing process will go a long way to ensuring the future stability of dbt. We're going to get started on these tasks soon, and you can follow our progress here: https://github.com/fishtown-analytics/dbt/milestone/16 . -As always, feel free to [reach out to us on Slack](http://ac-slackin.herokuapp.com/) with any questions or concerns: +As always, feel free to [reach out to us on Slack](http://slack.getdbt.com/) with any questions or concerns: @@ -623,7 +710,7 @@ pip install --upgrade dbt ### And another thing -- Join us on [slack](http://ac-slackin.herokuapp.com/) with questions or comments +- Join us on [slack](http://slack.getdbt.com/) with questions or comments Made with ♥️ by 🐟🏙 📈 diff --git a/Makefile b/Makefile index 72b2836347c..c6d937a9ba9 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,11 @@ test: test-unit: @echo "Unit test run starting..." - @time docker-compose run test tox -e unit-py27,unit-py35,pep8 + @time docker-compose run test tox -e unit-py27,unit-py36,pep8 test-integration: @echo "Integration test run starting..." - @time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py35,integration-snowflake-py27,integration-snowflake-py35,integration-bigquery-py27,integration-bigquery-py35 + @time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py36,integration-snowflake-py27,integration-snowflake-py36,integration-bigquery-py27,integration-bigquery-py36 test-new: @echo "Test run starting..." diff --git a/README.md b/README.md index a79ca14832f..ba680c9e9d3 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ dbt (data build tool) helps analysts write reliable, modular code using a workfl A dbt project primarily consists of "models". These models are SQL `select` statements that filter, aggregate, and otherwise transform data to facilitate analytics. Analysts use dbt to [aggregate pageviews into sessions](https://github.com/fishtown-analytics/snowplow), calculate [ad spend ROI](https://github.com/fishtown-analytics/facebook-ads), or report on [email campaign performance](https://github.com/fishtown-analytics/mailchimp). -These models frequently build on top of one another. Fortunately, dbt makes it easy to [manage relationships](https://dbt.readme.io/reference#ref) between models, [test](https://dbt.readme.io/docs/testing) your assumptions, and [visualize](https://graph.sinterdata.com/) your projects. +These models frequently build on top of one another. Fortunately, dbt makes it easy to [manage relationships](https://docs.getdbt.com/reference#ref) between models, [test](https://docs.getdbt.com/docs/testing) your assumptions, and [visualize](https://graph.sinterdata.com/) your projects. -Still reading? Check out the [docs](https://dbt.readme.io/docs/overview) for more information. +Still reading? Check out the [docs](https://docs.getdbt.com/docs/overview) for more information. ![dbt dag](/etc/dag.png?raw=true) @@ -25,7 +25,7 @@ Still reading? Check out the [docs](https://dbt.readme.io/docs/overview) for mor --- -[![Code Climate](https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg)](https://codeclimate.com/github/fishtown-analytics/dbt) [![Slack](https://ac-slackin.herokuapp.com/badge.svg)](https://ac-slackin.herokuapp.com) +[![Code Climate](https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg)](https://codeclimate.com/github/fishtown-analytics/dbt) [![Slack](https://slack.getdbt.com/badge.svg)](https://slack.getdbt.com) ### Testing @@ -43,7 +43,7 @@ Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, [PyPA Code of Conduct]: https://www.pypa.io/en/latest/code-of-conduct/ -[slack-url]: http://ac-slackin.herokuapp.com/ -[Installation]: https://dbt.readme.io/docs/installation -[What is dbt]: https://dbt.readme.io/docs/overview -[dbt viewpoint]: https://dbt.readme.io/docs/viewpoint +[slack-url]: https://slack.getdbt.com/ +[Installation]: https://docs.getdbt.com/docs/installation +[What is dbt]: https://docs.getdbt.com/docs/overview +[dbt viewpoint]: https://docs.getdbt.com/docs/viewpoint diff --git a/converter.py b/converter.py new file mode 100755 index 00000000000..e9a6cb22141 --- /dev/null +++ b/converter.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +import json +import yaml +import sys +import argparse +from datetime import datetime, timezone +import dbt.clients.registry as registry + + +def yaml_type(fname): + with open(fname) as f: + return yaml.load(f) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--project", type=yaml_type, default="dbt_project.yml") + parser.add_argument("--namespace", required=True) + return parser.parse_args() + + +def get_full_name(args): + return "{}/{}".format(args.namespace, args.project["name"]) + + +def init_project_in_packages(args, packages): + full_name = get_full_name(args) + if full_name not in packages: + packages[full_name] = { + "name": args.project["name"], + "namespace": args.namespace, + "latest": args.project["version"], + "assets": {}, + "versions": {}, + } + return packages[full_name] + + +def add_version_to_package(args, project_json): + project_json["versions"][args.project["version"]] = { + "id": "{}/{}".format(get_full_name(args), args.project["version"]), + "name": args.project["name"], + "version": args.project["version"], + "description": "", + "published_at": datetime.now(timezone.utc).astimezone().isoformat(), + "packages": args.project.get("packages") or [], + "works_with": [], + "_source": { + "type": "github", + "url": "", + "readme": "", + }, + "downloads": { + "tarball": "", + "format": "tgz", + "sha1": "", + }, + } + + +def main(): + args = parse_args() + packages = registry.packages() + project_json = init_project_in_packages(args, packages) + if args.project["version"] in project_json["versions"]: + raise Exception("Version {} already in packages JSON" + .format(args.project["version"]), + file=sys.stderr) + add_version_to_package(args, project_json) + print(json.dumps(packages, indent=2)) + +if __name__ == "__main__": + main() diff --git a/dbt/adapters/bigquery.py b/dbt/adapters/bigquery.py index e6f45402873..4e3664986ea 100644 --- a/dbt/adapters/bigquery.py +++ b/dbt/adapters/bigquery.py @@ -40,7 +40,9 @@ class BigQueryAdapter(PostgresAdapter): def handle_error(cls, error, message, sql): logger.debug(message.format(sql=sql)) logger.debug(error) - error_msg = "\n".join([error['message'] for error in error.errors]) + error_msg = "\n".join( + [item['message'] for item in error.errors]) + raise dbt.exceptions.DatabaseException(error_msg) @classmethod @@ -99,7 +101,7 @@ def get_bigquery_credentials(cls, config): elif method == 'service-account-json': details = config.get('keyfile_json') - return creds.from_service_account_info(details) + return creds.from_service_account_info(details, scopes=cls.SCOPE) error = ('Invalid `method` in profile: "{}"'.format(method)) raise dbt.exceptions.FailedToConnectException(error) @@ -341,6 +343,12 @@ def get_existing_schemas(cls, profile, model_name=None): all_datasets = client.list_datasets() return [ds.name for ds in all_datasets] + @classmethod + def get_columns_in_table(cls, profile, schema_name, table_name, + model_name=None): + raise dbt.exceptions.NotImplementedException( + '`get_columns_in_table` is not implemented for this adapter!') + @classmethod def check_schema_exists(cls, profile, schema, model_name=None): conn = cls.get_connection(profile, model_name) @@ -366,7 +374,8 @@ def warning_on_hooks(cls, hook_type): dbt.ui.printer.COLOR_FG_YELLOW) @classmethod - def add_query(cls, profile, sql, model_name=None, auto_begin=True): + def add_query(cls, profile, sql, model_name=None, auto_begin=True, + bindings=None): if model_name in ['on-run-start', 'on-run-end']: cls.warning_on_hooks(model_name) else: @@ -389,3 +398,52 @@ def quote_schema_and_table(cls, profile, schema, table, model_name=None): return '{}.{}.{}'.format(cls.quote(project), cls.quote(schema), cls.quote(table)) + + @classmethod + def convert_text_type(cls, agate_table, col_idx): + return "string" + + @classmethod + def convert_number_type(cls, agate_table, col_idx): + import agate + decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) + return "float64" if decimals else "int64" + + @classmethod + def convert_boolean_type(cls, agate_table, col_idx): + return "bool" + + @classmethod + def convert_datetime_type(cls, agate_table, col_idx): + return "datetime" + + @classmethod + def create_csv_table(cls, profile, schema, table_name, agate_table): + pass + + @classmethod + def reset_csv_table(cls, profile, schema, table_name, agate_table, + full_refresh=False): + cls.drop(profile, schema, table_name, "table") + + @classmethod + def _agate_to_schema(cls, agate_table): + bq_schema = [] + for idx, col_name in enumerate(agate_table.column_names): + type_ = cls.convert_agate_type(agate_table, idx) + bq_schema.append( + google.cloud.bigquery.SchemaField(col_name, type_)) + return bq_schema + + @classmethod + def load_csv_rows(cls, profile, schema, table_name, agate_table): + bq_schema = cls._agate_to_schema(agate_table) + dataset = cls.get_dataset(profile, schema, None) + table = dataset.table(table_name, schema=bq_schema) + conn = cls.get_connection(profile, None) + client = conn.get('handle') + with open(agate_table.original_abspath, "rb") as f: + job = table.upload_from_file(f, "CSV", rewind=True, + client=client, skip_leading_rows=1) + with cls.exception_handler(profile, "LOAD TABLE"): + cls.poll_until_job_completes(job, cls.get_timeout(conn)) diff --git a/dbt/adapters/default.py b/dbt/adapters/default.py index a31c73a44fb..a2046e64c17 100644 --- a/dbt/adapters/default.py +++ b/dbt/adapters/default.py @@ -94,6 +94,22 @@ def cancel_connection(cls, project, connection): raise dbt.exceptions.NotImplementedException( '`cancel_connection` is not implemented for this adapter!') + @classmethod + def create_csv_table(cls, profile, schema, table_name, agate_table): + raise dbt.exceptions.NotImplementedException( + '`create_csv_table` is not implemented for this adapter!') + + @classmethod + def reset_csv_table(cls, profile, schema, table_name, agate_table, + full_refresh=False): + raise dbt.exceptions.NotImplementedException( + '`reset_csv_table` is not implemented for this adapter!') + + @classmethod + def load_csv_rows(cls, profile, schema, table_name, agate_table): + raise dbt.exceptions.NotImplementedException( + '`load_csv_rows` is not implemented for this adapter!') + ### # FUNCTIONS THAT SHOULD BE ABSTRACT ### @@ -174,8 +190,7 @@ def get_missing_columns(cls, profile, if col_name in missing_columns] @classmethod - def get_columns_in_table(cls, profile, schema_name, table_name, - model_name=None): + def _get_columns_in_table_sql(cls, schema_name, table_name): sql = """ select column_name, data_type, character_maximum_length from information_schema.columns @@ -186,6 +201,13 @@ def get_columns_in_table(cls, profile, schema_name, table_name, sql += (" AND table_schema = '{schema_name}'" .format(schema_name=schema_name)) + return sql + + @classmethod + def get_columns_in_table(cls, profile, schema_name, table_name, + model_name=None): + + sql = cls._get_columns_in_table_sql(schema_name, table_name) connection, cursor = cls.add_query( profile, sql, model_name) @@ -501,7 +523,8 @@ def close(cls, connection): return connection @classmethod - def add_query(cls, profile, sql, model_name=None, auto_begin=True): + def add_query(cls, profile, sql, model_name=None, auto_begin=True, + bindings=None): connection = cls.get_connection(profile, model_name) connection_name = connection.get('name') @@ -516,7 +539,7 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True): pre = time.time() cursor = connection.get('handle').cursor() - cursor.execute(sql) + cursor.execute(sql, bindings) logger.debug("SQL status: %s in %0.2f seconds", cls.get_status(cursor), (time.time() - pre)) @@ -597,9 +620,71 @@ def already_exists(cls, profile, schema, table, model_name=None): @classmethod def quote(cls, identifier): - return '"{}"'.format(identifier) + return '"{}"'.format(identifier.replace('"', '""')) @classmethod def quote_schema_and_table(cls, profile, schema, table, model_name=None): return '{}.{}'.format(cls.quote(schema), cls.quote(table)) + + @classmethod + def handle_csv_table(cls, profile, schema, table_name, agate_table, + full_refresh=False): + existing = cls.query_for_existing(profile, schema) + existing_type = existing.get(table_name) + if existing_type and existing_type != "table": + raise dbt.exceptions.RuntimeException( + "Cannot seed to '{}', it is a view".format(table_name)) + if existing_type: + cls.reset_csv_table(profile, schema, table_name, agate_table, + full_refresh=full_refresh) + else: + cls.create_csv_table(profile, schema, table_name, agate_table) + cls.load_csv_rows(profile, schema, table_name, agate_table) + cls.commit_if_has_connection(profile, None) + + @classmethod + def convert_text_type(cls, agate_table, col_idx): + raise dbt.exceptions.NotImplementedException( + '`convert_text_type` is not implemented for this adapter!') + + @classmethod + def convert_number_type(cls, agate_table, col_idx): + raise dbt.exceptions.NotImplementedException( + '`convert_number_type` is not implemented for this adapter!') + + @classmethod + def convert_boolean_type(cls, agate_table, col_idx): + raise dbt.exceptions.NotImplementedException( + '`convert_boolean_type` is not implemented for this adapter!') + + @classmethod + def convert_datetime_type(cls, agate_table, col_idx): + raise dbt.exceptions.NotImplementedException( + '`convert_datetime_type` is not implemented for this adapter!') + + @classmethod + def convert_date_type(cls, agate_table, col_idx): + raise dbt.exceptions.NotImplementedException( + '`convert_date_type` is not implemented for this adapter!') + + @classmethod + def convert_time_type(cls, agate_table, col_idx): + raise dbt.exceptions.NotImplementedException( + '`convert_time_type` is not implemented for this adapter!') + + @classmethod + def convert_agate_type(cls, agate_table, col_idx): + import agate + agate_type = agate_table.column_types[col_idx] + conversions = [ + (agate.Text, cls.convert_text_type), + (agate.Number, cls.convert_number_type), + (agate.Boolean, cls.convert_boolean_type), + (agate.DateTime, cls.convert_datetime_type), + (agate.Date, cls.convert_date_type), + (agate.TimeDelta, cls.convert_time_type), + ] + for agate_cls, func in conversions: + if isinstance(agate_type, agate_cls): + return func(agate_table, col_idx) diff --git a/dbt/adapters/postgres.py b/dbt/adapters/postgres.py index 8836f0bca92..5a0b489af6b 100644 --- a/dbt/adapters/postgres.py +++ b/dbt/adapters/postgres.py @@ -5,6 +5,7 @@ import dbt.adapters.default import dbt.compat import dbt.exceptions +from dbt.utils import max_digits from dbt.logger import GLOBAL_LOGGER as logger @@ -165,3 +166,73 @@ def cancel_connection(cls, profile, connection): res = cursor.fetchone() logger.debug("Cancel query '{}': {}".format(connection_name, res)) + + @classmethod + def convert_text_type(cls, agate_table, col_idx): + return "text" + + @classmethod + def convert_number_type(cls, agate_table, col_idx): + import agate + column = agate_table.columns[col_idx] + precision = max_digits(column.values_without_nulls()) + # agate uses the term Precision but in this context, it is really the + # scale - ie. the number of decimal places + scale = agate_table.aggregate(agate.MaxPrecision(col_idx)) + if not scale: + return "integer" + return "numeric({}, {})".format(precision, scale) + + @classmethod + def convert_boolean_type(cls, agate_table, col_idx): + return "boolean" + + @classmethod + def convert_datetime_type(cls, agate_table, col_idx): + return "timestamp without time zone" + + @classmethod + def convert_date_type(cls, agate_table, col_idx): + return "date" + + @classmethod + def convert_time_type(cls, agate_table, col_idx): + return "time" + + @classmethod + def create_csv_table(cls, profile, schema, table_name, agate_table): + col_sqls = [] + for idx, col_name in enumerate(agate_table.column_names): + type_ = cls.convert_agate_type(agate_table, idx) + col_sqls.append('{} {}'.format(col_name, type_)) + sql = 'create table "{}"."{}" ({})'.format(schema, table_name, + ", ".join(col_sqls)) + return cls.add_query(profile, sql) + + @classmethod + def reset_csv_table(cls, profile, schema, table_name, agate_table, + full_refresh=False): + if full_refresh: + cls.drop_table(profile, schema, table_name, None) + cls.create_csv_table(profile, schema, table_name, agate_table) + else: + cls.truncate(profile, schema, table_name) + + @classmethod + def load_csv_rows(cls, profile, schema, table_name, agate_table): + bindings = [] + placeholders = [] + cols_sql = ", ".join(c for c in agate_table.column_names) + + for row in agate_table.rows: + bindings += row + placeholders.append("({})".format( + ", ".join("%s" for _ in agate_table.column_names))) + + sql = ('insert into {}.{} ({}) values {}' + .format(cls.quote(schema), + cls.quote(table_name), + cols_sql, + ",\n".join(placeholders))) + + cls.add_query(profile, sql, bindings=bindings) diff --git a/dbt/adapters/redshift.py b/dbt/adapters/redshift.py index e4e0db9d543..d3009e3e5ce 100644 --- a/dbt/adapters/redshift.py +++ b/dbt/adapters/redshift.py @@ -17,6 +17,57 @@ def type(cls): def date_function(cls): return 'getdate()' + @classmethod + def _get_columns_in_table_sql(cls, schema_name, table_name): + # TODO : how do we make this a macro? + if schema_name is None: + table_schema_filter = '1=1' + else: + table_schema_filter = "table_schema = '{schema_name}'".format( + schema_name=schema_name) + + sql = """ + with bound_views as ( + select + table_schema, + column_name, + data_type, + character_maximum_length + + from information_schema.columns + where table_name = '{table_name}' + ), + + unbound_views as ( + select + view_schema, + col_name, + col_type, + case + when col_type like 'character%' + then nullif(REGEXP_SUBSTR(col_type, '[0-9]+'), '')::int + else null + end as character_maximum_length + + from pg_get_late_binding_view_cols() + cols(view_schema name, view_name name, col_name name, + col_type varchar, col_num int) + where view_name = '{table_name}' + ), + + unioned as ( + select * from bound_views + union all + select * from unbound_views + ) + + select column_name, data_type, character_maximum_length + from unioned + where {table_schema_filter} + """.format(table_name=table_name, + table_schema_filter=table_schema_filter).strip() + return sql + @classmethod def drop(cls, profile, schema, relation, relation_type, model_name=None): global drop_lock @@ -43,3 +94,14 @@ def drop(cls, profile, schema, relation, relation_type, model_name=None): finally: drop_lock.release() + + @classmethod + def convert_text_type(cls, agate_table, col_idx): + column = agate_table.columns[col_idx] + lens = (len(d.encode("utf-8")) for d in column.values_without_nulls()) + max_len = max(lens) if lens else 64 + return "varchar({})".format(max_len) + + @classmethod + def convert_time_type(cls, agate_table, col_idx): + return "varchar(24)" diff --git a/dbt/adapters/snowflake.py b/dbt/adapters/snowflake.py index 801628e2dbc..08385a19c91 100644 --- a/dbt/adapters/snowflake.py +++ b/dbt/adapters/snowflake.py @@ -147,7 +147,11 @@ def add_begin_query(cls, profile, name): def create_schema(cls, profile, schema, model_name=None): logger.debug('Creating schema "%s".', schema) sql = cls.get_create_schema_sql(schema) - return cls.add_query(profile, sql, model_name, select_schema=False) + res = cls.add_query(profile, sql, model_name, select_schema=False) + + cls.commit_if_has_connection(profile, model_name) + + return res @classmethod def get_existing_schemas(cls, profile, model_name=None): @@ -177,7 +181,7 @@ def check_schema_exists(cls, profile, schema, model_name=None): @classmethod def add_query(cls, profile, sql, model_name=None, auto_begin=True, - select_schema=True): + select_schema=True, bindings=None): # snowflake only allows one query per api call. queries = sql.strip().split(";") cursor = None @@ -189,6 +193,11 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True, model_name, auto_begin) + if bindings: + # The snowflake connector is more strict than, eg., psycopg2 - + # which allows any iterable thing to be passed as a binding. + bindings = tuple(bindings) + for individual_query in queries: # hack -- after the last ';', remove comments and don't run # empty queries. this avoids using exceptions as flow control, @@ -201,7 +210,8 @@ def add_query(cls, profile, sql, model_name=None, auto_begin=True, continue connection, cursor = super(PostgresAdapter, cls).add_query( - profile, individual_query, model_name, auto_begin) + profile, individual_query, model_name, auto_begin, + bindings=bindings) return connection, cursor diff --git a/dbt/clients/git.py b/dbt/clients/git.py index 9183acef4f3..ab94603d03f 100644 --- a/dbt/clients/git.py +++ b/dbt/clients/git.py @@ -1,3 +1,4 @@ +import re import os.path from dbt.clients.system import run_cmd, rmdir @@ -29,7 +30,7 @@ def checkout(cwd, repo, branch=None): if branch is None: branch = 'master' - logger.info(' Checking out branch {}.'.format(branch)) + logger.debug(' Checking out branch {}.'.format(branch)) run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', branch]) run_cmd(cwd, ['git', 'fetch', '--tags', '--depth', '1', 'origin', branch]) @@ -59,3 +60,32 @@ def get_current_sha(cwd): def remove_remote(cwd): return run_cmd(cwd, ['git', 'remote', 'rm', 'origin']) + + +def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False, + branch=None): + _, err = clone(repo, cwd, dirname=dirname, remove_git_dir=remove_git_dir) + exists = re.match("fatal: destination path '(.+)' already exists", + err.decode('utf-8')) + directory = None + start_sha = None + if exists: + directory = exists.group(1) + logger.debug('Updating existing dependency %s.', directory) + else: + matches = re.match("Cloning into '(.+)'", err.decode('utf-8')) + directory = matches.group(1) + logger.debug('Pulling new dependency %s.', directory) + full_path = os.path.join(cwd, directory) + start_sha = get_current_sha(full_path) + checkout(full_path, repo, branch) + end_sha = get_current_sha(full_path) + if exists: + if start_sha == end_sha: + logger.debug(' Already at %s, nothing to do.', start_sha[:7]) + else: + logger.debug(' Updated checkout from %s to %s.', + start_sha[:7], end_sha[:7]) + else: + logger.debug(' Checked out at %s.', end_sha[:7]) + return directory diff --git a/dbt/clients/jinja.py b/dbt/clients/jinja.py index cfdb50da56d..463d4d4df98 100644 --- a/dbt/clients/jinja.py +++ b/dbt/clients/jinja.py @@ -49,6 +49,8 @@ def call(*args, **kwargs): try: return macro(*args, **kwargs) + except dbt.exceptions.MacroReturn as e: + return e.value except (TypeError, jinja2.exceptions.TemplateRuntimeError) as e: dbt.exceptions.raise_compiler_error( diff --git a/dbt/clients/registry.py b/dbt/clients/registry.py index 8371e68b149..0873cc509ad 100644 --- a/dbt/clients/registry.py +++ b/dbt/clients/registry.py @@ -1,7 +1,14 @@ +from functools import wraps +import six import requests +from dbt.exceptions import RegistryException +from dbt.utils import memoized +import os - -DEFAULT_REGISTRY_BASE_URL = 'http://127.0.0.1:4567/' +if os.getenv('DBT_PACKAGE_HUB_URL'): + DEFAULT_REGISTRY_BASE_URL = os.getenv('DBT_PACKAGE_HUB_URL') +else: + DEFAULT_REGISTRY_BASE_URL = 'https://hub.getdbt.com/' def _get_url(url, registry_base_url=None): @@ -11,26 +18,44 @@ def _get_url(url, registry_base_url=None): return '{}{}'.format(registry_base_url, url) +def _wrap_exceptions(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except requests.exceptions.ConnectionError as e: + six.raise_from( + RegistryException('Unable to connect to registry hub'), e) + return wrapper + + +@_wrap_exceptions +def _get(path, registry_base_url=None): + url = _get_url(path, registry_base_url) + resp = requests.get(url) + resp.raise_for_status() + return resp.json() + + def index(registry_base_url=None): - return requests.get( - _get_url('api/v1/index.json', - registry_base_url)).json() + return _get('api/v1/index.json', registry_base_url) + + +index_cached = memoized(index) + + +def packages(registry_base_url=None): + return _get('api/v1/packages.json', registry_base_url) def package(name, registry_base_url=None): - return requests.get( - _get_url('api/v1/{}.json'.format(name), - registry_base_url)).json() + return _get('api/v1/{}.json'.format(name), registry_base_url) def package_version(name, version, registry_base_url=None): - url = _get_url('api/v1/{}/{}.json'.format(name, version)) - response = requests.get(url, registry_base_url) - - return response.json() + return _get('api/v1/{}/{}.json'.format(name, version), registry_base_url) def get_available_versions(name): response = package(name) - - return list(response['versions'].keys()) + return list(response['versions']) diff --git a/dbt/clients/system.py b/dbt/clients/system.py index f94108f7e66..114939df5d1 100644 --- a/dbt/clients/system.py +++ b/dbt/clients/system.py @@ -5,6 +5,9 @@ import shutil import subprocess import sys +import tarfile +import requests +import stat import dbt.compat @@ -99,12 +102,28 @@ def write_file(path, contents=''): return True +def _windows_rmdir_readonly(func, path, exc): + exception_val = exc[1] + if exception_val.errno == errno.EACCES: + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise + + def rmdir(path): """ - Make a file at `path` assuming that the directory it resides in already - exists. The file is saved with contents `contents` + Recursively deletes a directory. Includes an error handler to retry with + different permissions on Windows. Otherwise, removing directories (eg. + cloned via git) can cause rmtree to throw a PermissionError exception """ - return shutil.rmtree(path) + logger.debug("DEBUG** Window rmdir sys.platform: {}".format(sys.platform)) + if sys.platform == 'win32': + onerror = _windows_rmdir_readonly + else: + onerror = None + + return shutil.rmtree(path, onerror=onerror) def open_dir_cmd(): @@ -133,3 +152,27 @@ def run_cmd(cwd, cmd): logger.debug('STDERR: "{}"'.format(err)) return out, err + + +def download(url, path): + response = requests.get(url) + with open(path, 'wb') as handle: + for block in response.iter_content(1024*64): + handle.write(block) + + +def rename(from_path, to_path, force=False): + if os.path.exists(to_path) and force: + rmdir(to_path) + os.rename(from_path, to_path) + + +def untar_package(tar_path, dest_dir, rename_to=None): + tar_dir_name = None + with tarfile.open(tar_path, 'r') as tarball: + tarball.extractall(dest_dir) + tar_dir_name = os.path.commonprefix(tarball.getnames()) + if rename_to: + downloaded_path = os.path.join(dest_dir, tar_dir_name) + desired_path = os.path.join(dest_dir, rename_to) + dbt.clients.system.rename(downloaded_path, desired_path, force=True) diff --git a/dbt/compilation.py b/dbt/compilation.py index 706009a77ca..39bc4af302c 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -34,17 +34,10 @@ def print_compile_stats(stats): NodeType.Analysis: 'analyses', NodeType.Macro: 'macros', NodeType.Operation: 'operations', + NodeType.Seed: 'seed files', } - results = { - NodeType.Model: 0, - NodeType.Test: 0, - NodeType.Archive: 0, - NodeType.Analysis: 0, - NodeType.Macro: 0, - NodeType.Operation: 0, - } - + results = {k: 0 for k in names.keys()} results.update(stats) stat_line = ", ".join( diff --git a/dbt/config.py b/dbt/config.py index 639cb161caf..51681da6ddc 100644 --- a/dbt/config.py +++ b/dbt/config.py @@ -31,7 +31,10 @@ def read_profile(profiles_dir): def read_config(profiles_dir): profile = read_profile(profiles_dir) - return profile.get('config', {}) + if profile is None: + return {} + else: + return profile.get('config', {}) def send_anonymous_usage_stats(config): diff --git a/dbt/context/common.py b/dbt/context/common.py index 9524eb4ed5e..072cb62a7e4 100644 --- a/dbt/context/common.py +++ b/dbt/context/common.py @@ -1,5 +1,6 @@ import json import os +import pytz import voluptuous from dbt.adapters.factory import get_adapter @@ -128,6 +129,7 @@ def call(name, status, data=[]): 'status': status, 'data': data }) + return '' return call @@ -220,6 +222,7 @@ def write(node, target_path, subdirectory): def fn(payload): node['build_path'] = dbt.writer.write_node( node, target_path, subdirectory, payload) + return '' return fn @@ -231,13 +234,22 @@ def fn(string): return fn -def fromjson(node): - def fn(string, default=None): - try: - return json.loads(string) - except ValueError as e: - return default - return fn +def fromjson(string, default=None): + try: + return json.loads(string) + except ValueError as e: + return default + + +def tojson(value, default=None): + try: + return json.dumps(value) + except ValueError as e: + return default + + +def _return(value): + raise dbt.exceptions.MacroReturn(value) def generate(model, project, flat_graph, provider=None): @@ -277,15 +289,20 @@ def generate(model, project, flat_graph, provider=None): "graph": flat_graph, "log": log, "model": model, + "modules": { + "pytz": pytz, + }, "post_hooks": post_hooks, "pre_hooks": pre_hooks, "ref": provider.ref(model, project, profile, flat_graph), + "return": _return, "schema": model.get('schema', schema), "sql": model.get('injected_sql'), "sql_now": adapter.date_function(), - "fromjson": fromjson(model), + "fromjson": fromjson, + "tojson": tojson, "target": target, - "this": dbt.utils.Relation(adapter, model, use_temp=True) + "this": dbt.utils.Relation(profile, adapter, model, use_temp=True) }) context = _add_tracking(context) diff --git a/dbt/context/parser.py b/dbt/context/parser.py index 1720c01a46f..bf254bb3636 100644 --- a/dbt/context/parser.py +++ b/dbt/context/parser.py @@ -3,6 +3,8 @@ import dbt.context.common +from dbt.adapters.factory import get_adapter + execute = False @@ -16,6 +18,9 @@ def ref(*args): else: dbt.exceptions.ref_invalid_args(model, args) + adapter = get_adapter(profile) + return dbt.utils.Relation(profile, adapter, model) + return ref diff --git a/dbt/context/runtime.py b/dbt/context/runtime.py index bac4f5c40b3..1a67ab1f5b1 100644 --- a/dbt/context/runtime.py +++ b/dbt/context/runtime.py @@ -52,7 +52,7 @@ def do_ref(*args): model['extra_ctes'][target_model_id] = None adapter = get_adapter(profile) - return dbt.utils.Relation(adapter, target_model) + return dbt.utils.Relation(profile, adapter, target_model) return do_ref diff --git a/dbt/contracts/connection.py b/dbt/contracts/connection.py index 27a5317c511..2524b8e371e 100644 --- a/dbt/contracts/connection.py +++ b/dbt/contracts/connection.py @@ -20,7 +20,7 @@ Required('host'): basestring, Required('user'): basestring, Required('pass'): basestring, - Required('port'): All(int, Range(min=0, max=65535)), + Required('port'): Any(All(int, Range(min=0, max=65535)), basestring), Required('schema'): basestring, }) diff --git a/dbt/contracts/graph/parsed.py b/dbt/contracts/graph/parsed.py index d25d8d1b976..e5dd0cba9ab 100644 --- a/dbt/contracts/graph/parsed.py +++ b/dbt/contracts/graph/parsed.py @@ -1,4 +1,5 @@ from voluptuous import Schema, Required, All, Any, Length, ALLOW_EXTRA +from voluptuous import Optional import dbt.exceptions @@ -15,6 +16,7 @@ hook_contract = Schema({ Required('sql'): basestring, Required('transaction'): bool, + Required('index'): int, }) config_contract = Schema({ @@ -42,6 +44,9 @@ Required('empty'): bool, Required('config'): config_contract, Required('tags'): All(set), + + # For csv files + Optional('agate_table'): object, }) parsed_nodes_contract = Schema({ diff --git a/dbt/contracts/graph/unparsed.py b/dbt/contracts/graph/unparsed.py index 1a7e75e50d9..fc4daae7103 100644 --- a/dbt/contracts/graph/unparsed.py +++ b/dbt/contracts/graph/unparsed.py @@ -1,4 +1,4 @@ -from voluptuous import Schema, Required, All, Any, Length +from voluptuous import Schema, Required, All, Any, Length, Optional from dbt.compat import basestring from dbt.contracts.common import validate_with @@ -15,13 +15,15 @@ Required('path'): basestring, Required('original_file_path'): basestring, Required('raw_sql'): basestring, + Optional('index'): int, }) unparsed_node_contract = unparsed_base_contract.extend({ Required('resource_type'): Any(NodeType.Model, NodeType.Test, NodeType.Analysis, - NodeType.Operation) + NodeType.Operation, + NodeType.Seed) }) unparsed_nodes_contract = Schema([unparsed_node_contract]) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index a4e1a222934..2b11b5c3a69 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -11,14 +11,19 @@ def show(self, *args, **kwargs): logger.info("* Deprecation Warning: {}\n".format(desc)) active_deprecations.add(self.name) -# Leaving this as an example. Make sure to add new ones to deprecations_list -# - Connor -# -# class DBTRunTargetDeprecation(DBTDeprecation): -# name = 'run-target' -# description = """profiles.yml configuration option 'run-target' is -# deprecated. Please use 'target' instead. The 'run-target' option will be -# removed (in favor of 'target') in DBT version 0.7.0""" + +class DBTRepositoriesDeprecation(DBTDeprecation): + name = "repositories" + description = """dbt_project.yml configuration option 'repositories' is + deprecated. Please use 'packages' instead. The 'repositories' option will + be removed in a later version of DBT.""" + + +class SeedDropExistingDeprecation(DBTDeprecation): + name = 'drop-existing' + description = """The --drop-existing argument has been deprecated. Please + use --full-refresh instead. The --drop-existing option will be removed in a + future version of dbt.""" def warn(name, *args, **kwargs): @@ -37,6 +42,8 @@ def warn(name, *args, **kwargs): active_deprecations = set() deprecations_list = [ + DBTRepositoriesDeprecation(), + SeedDropExistingDeprecation() ] deprecations = {d.name: d for d in deprecations_list} diff --git a/dbt/exceptions.py b/dbt/exceptions.py index a17a6a74ae9..4feeb3c34d5 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -1,10 +1,20 @@ from dbt.compat import basestring +from dbt.logger import GLOBAL_LOGGER as logger class Exception(BaseException): pass +class MacroReturn(BaseException): + """ + Hack of all hacks + """ + + def __init__(self, value): + self.value = value + + class InternalException(Exception): pass @@ -20,6 +30,9 @@ def type(self): return 'Runtime' def node_to_string(self, node): + if node is None: + return "" + return "{} {} ({})".format( node.get('resource_type'), node.get('name', 'unknown'), @@ -92,11 +105,19 @@ class ParsingException(Exception): pass -class VersionsNotCompatibleException(ParsingException): +class DependencyException(Exception): + pass + + +class SemverException(Exception): def __init__(self, msg=None): self.msg = msg +class VersionsNotCompatibleException(SemverException): + pass + + class NotImplementedException(Exception): pass @@ -116,6 +137,10 @@ def raise_database_error(msg, node=None): raise DatabaseException(msg, node) +def raise_dependency_error(msg): + raise DependencyException(msg) + + def ref_invalid_args(model, args): raise_compiler_error( "ref() takes at most two arguments ({} given)".format(len(args)), @@ -143,18 +168,22 @@ def ref_bad_context(model, target_model_name, target_model_package): raise_compiler_error(error_msg, model) -def ref_target_not_found(model, target_model_name, target_model_package): +def get_target_not_found_msg(model, target_model_name, target_model_package): target_package_string = '' if target_model_package is not None: target_package_string = "in package '{}' ".format(target_model_package) - raise_compiler_error( - "Model '{}' depends on model '{}' {}which was not found." - .format(model.get('unique_id'), - target_model_name, - target_package_string), - model) + return ("Model '{}' depends on model '{}' {}which was not found or is" + " disabled".format(model.get('unique_id'), + target_model_name, + target_package_string)) + + +def ref_target_not_found(model, target_model_name, target_model_package): + msg = get_target_not_found_msg(model, target_model_name, + target_model_package) + raise_compiler_error(msg, model) def ref_disabled_dependency(model, target_model): @@ -221,7 +250,25 @@ def missing_relation(relation_name, model=None): model) +def package_not_found(package_name): + raise_dependency_error( + "Package {} was not found in the package index".format(package_name)) + + +def package_version_not_found(package_name, version_range, available_versions): + base_msg = ('Could not find a matching version for package {}\n' + ' Requested range: {}\n' + ' Available versions: {}') + raise_dependency_error(base_msg.format(package_name, + version_range, + available_versions)) + + def invalid_materialization_argument(name, argument): raise_compiler_error( "materialization '{}' received unknown argument '{}'." .format(name, argument)) + + +class RegistryException(Exception): + pass diff --git a/dbt/hooks.py b/dbt/hooks.py index 7df63354869..dc87e19b9d2 100644 --- a/dbt/hooks.py +++ b/dbt/hooks.py @@ -21,12 +21,13 @@ def _parse_hook_to_dict(hook_string): return hook_dict -def get_hook_dict(hook): +def get_hook_dict(hook, index): if isinstance(hook, dict): hook_dict = hook else: hook_dict = _parse_hook_to_dict(to_string(hook)) + hook_dict['index'] = index return hook_dict @@ -36,5 +37,5 @@ def get_hooks(model, hook_key): if not isinstance(hooks, (list, tuple)): hooks = [hooks] - wrapped = [get_hook_dict(hook) for hook in hooks] + wrapped = [get_hook_dict(hook, i) for i, hook in enumerate(hooks)] return wrapped diff --git a/dbt/include/global_project/macros/adapters/common.sql b/dbt/include/global_project/macros/adapters/common.sql index 2863a578b9a..806ddf66e95 100644 --- a/dbt/include/global_project/macros/adapters/common.sql +++ b/dbt/include/global_project/macros/adapters/common.sql @@ -1,14 +1,37 @@ {% macro adapter_macro(name) -%} + {% set original_name = name %} + {% if '.' in name %} + {% set package_name, name = name.split(".", 1) %} + {% else %} + {% set package_name = none %} + {% endif %} + + {% if package_name is none %} + {% set package_context = context %} + {% elif package_name in context %} + {% set package_context = context[package_name] %} + {% else %} + {% set error_msg %} + In adapter_macro: could not find package '{{package_name}}', called with '{{original_name}}' + {% endset %} + {{ exceptions.raise_compiler_error(error_msg | trim) }} + {% endif %} + {%- set separator = '__' -%} {%- set search_name = adapter.type() + separator + name -%} {%- set default_name = 'default' + separator + name -%} - {%- if context.get(search_name) is not none -%} - {{ context[search_name](*varargs, **kwargs) }} + + {%- if package_context.get(search_name) is not none -%} + {{ package_context[search_name](*varargs, **kwargs) }} {%- else -%} - {{ context[default_name](*varargs, **kwargs) }} + {{ package_context[default_name](*varargs, **kwargs) }} {%- endif -%} {%- endmacro %} +{% macro create_schema(schema_name) %} + create schema if not exists "{{ schema_name }}"; +{% endmacro %} + {% macro create_table_as(temporary, identifier, sql) -%} {{ adapter_macro('create_table_as', temporary, identifier, sql) }} diff --git a/dbt/include/global_project/macros/adapters/redshift.sql b/dbt/include/global_project/macros/adapters/redshift.sql index 972bf594d5a..6fc15111455 100644 --- a/dbt/include/global_project/macros/adapters/redshift.sql +++ b/dbt/include/global_project/macros/adapters/redshift.sql @@ -52,6 +52,16 @@ {%- endmacro %} +{% macro redshift__create_view_as(identifier, sql) -%} + + {% set bind_qualifier = '' if config.get('bind', default=True) else 'with no schema binding' %} + + create view "{{ schema }}"."{{ identifier }}" as ( + {{ sql }} + ) {{ bind_qualifier }}; +{% endmacro %} + + {% macro redshift__create_archive_table(schema, identifier, columns) -%} create table if not exists "{{ schema }}"."{{ identifier }}" ( {{ column_list_for_create_table(columns) }} diff --git a/dbt/include/global_project/macros/materializations/archive.sql b/dbt/include/global_project/macros/materializations/archive.sql index ae3eed76458..e91daf7d0e8 100644 --- a/dbt/include/global_project/macros/materializations/archive.sql +++ b/dbt/include/global_project/macros/materializations/archive.sql @@ -91,6 +91,10 @@ column('dbt_updated_at', 'timestamp', None) ] -%} + {% call statement() %} + {{ create_schema(target_schema) }} + {% endcall %} + {% call statement() %} {{ create_archive_table(target_schema, target_table, dest_columns) }} {% endcall %} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/dbt/include/global_project/macros/materializations/helpers.sql index 1171be00f69..1a20316c2f3 100644 --- a/dbt/include/global_project/macros/materializations/helpers.sql +++ b/dbt/include/global_project/macros/materializations/helpers.sql @@ -1,5 +1,10 @@ {% macro run_hooks(hooks, inside_transaction=True) %} {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% if not inside_transaction and loop.first %} + {% call statement(auto_begin=inside_transaction) %} + commit; + {% endcall %} + {% endif %} {% call statement(auto_begin=inside_transaction) %} {{ hook.get('sql') }} {% endcall %} @@ -22,7 +27,7 @@ {% macro make_hook_config(sql, inside_transaction) %} - {{ {"sql": sql, "transaction": inside_transaction} | tojson }} + {{ tojson({"sql": sql, "transaction": inside_transaction}) }} {% endmacro %} diff --git a/dbt/loader.py b/dbt/loader.py index ae1ad706b36..1d067e04f7e 100644 --- a/dbt/loader.py +++ b/dbt/loader.py @@ -72,6 +72,30 @@ def load_project(cls, root_project, all_projects, project, project_name, class ModelLoader(ResourceLoader): + @classmethod + def load_all(cls, root_project, all_projects, macros=None): + to_return = {} + + for project_name, project in all_projects.items(): + project_loaded = cls.load_project(root_project, + all_projects, + project, project_name, + macros) + + to_return.update(project_loaded) + + # Check for duplicate model names + names_models = {} + for model, attribs in to_return.items(): + name = attribs['name'] + existing_name = names_models.get(name) + if existing_name is not None: + raise dbt.exceptions.CompilationException( + 'Found models with the same name: \n- %s\n- %s' % ( + model, existing_name)) + names_models[name] = model + return to_return + @classmethod def load_project(cls, root_project, all_projects, project, project_name, macros): @@ -130,11 +154,16 @@ def load_project(cls, root_project, all_projects, project, project_name, macros=macros) +# ArchiveLoader and RunHookLoader operate on configs, so we just need to run +# them both once, not for each project class ArchiveLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name, - macros): + def load_all(cls, root_project, all_projects, macros=None): + return cls.load_project(root_project, all_projects, macros) + + @classmethod + def load_project(cls, root_project, all_projects, macros): return dbt.parser.parse_archives_from_projects(root_project, all_projects, macros) @@ -143,12 +172,29 @@ def load_project(cls, root_project, all_projects, project, project_name, class RunHookLoader(ResourceLoader): @classmethod - def load_project(cls, root_project, all_projects, project, project_name, - macros): + def load_all(cls, root_project, all_projects, macros=None): + return cls.load_project(root_project, all_projects, macros) + + @classmethod + def load_project(cls, root_project, all_projects, macros): return dbt.parser.load_and_parse_run_hooks(root_project, all_projects, macros) +class SeedLoader(ResourceLoader): + + @classmethod + def load_project(cls, root_project, all_projects, project, project_name, + macros): + return dbt.parser.load_and_parse_seeds( + package_name=project_name, + root_project=root_project, + all_projects=all_projects, + root_dir=project.get('project-root'), + relative_dirs=project.get('data-paths', []), + resource_type=NodeType.Seed) + + # node loaders GraphLoader.register(ModelLoader, 'nodes') GraphLoader.register(AnalysisLoader, 'nodes') @@ -156,3 +202,4 @@ def load_project(cls, root_project, all_projects, project, project_name, GraphLoader.register(DataTestLoader, 'nodes') GraphLoader.register(RunHookLoader, 'nodes') GraphLoader.register(ArchiveLoader, 'nodes') +GraphLoader.register(SeedLoader, 'nodes') diff --git a/dbt/logger.py b/dbt/logger.py index 2daa036e6c1..7ce14b7ca7e 100644 --- a/dbt/logger.py +++ b/dbt/logger.py @@ -12,6 +12,7 @@ logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('google').setLevel(logging.CRITICAL) logging.getLogger('snowflake.connector').setLevel(logging.CRITICAL) +logging.getLogger('parsedatetime').setLevel(logging.CRITICAL) # Colorama needs some help on windows because we're using logger.info # intead of print(). If the Windows env doesn't have a TERM var set, diff --git a/dbt/main.py b/dbt/main.py index a5a18fa471f..3282090156f 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -22,13 +22,14 @@ import dbt.config as config import dbt.ui.printer import dbt.compat +import dbt.deprecations from dbt.utils import ExitCodes PROFILES_HELP_MESSAGE = """ For more information on configuring profiles, please consult the dbt docs: -https://dbt.readme.io/docs/configure-your-profile +https://docs.getdbt.com/docs/configure-your-profile """ @@ -214,6 +215,8 @@ def invoke_dbt(parsed): targets = proj.cfg.get('outputs', {}).keys() if parsed.target in targets: proj.cfg['target'] = parsed.target + # make sure we update the target if this is overriden on the cli + proj.compile_and_update_target() else: logger.info("Encountered an error while reading the project:") logger.info(" ERROR Specified target {} is not a valid option " @@ -230,7 +233,15 @@ def invoke_dbt(parsed): return None flags.NON_DESTRUCTIVE = getattr(proj.args, 'non_destructive', False) - flags.FULL_REFRESH = getattr(proj.args, 'full_refresh', False) + + arg_drop_existing = getattr(proj.args, 'drop_existing', False) + arg_full_refresh = getattr(proj.args, 'full_refresh', False) + + if arg_drop_existing: + dbt.deprecations.warn('drop-existing') + flags.FULL_REFRESH = True + elif arg_full_refresh: + flags.FULL_REFRESH = True logger.debug("running dbt with arguments %s", parsed) @@ -293,6 +304,16 @@ def parse_args(args): help='Which target to load for the given profile' ) + base_subparser.add_argument( + '--vars', + type=str, + default='{}', + help=""" + Supply variables to the project. This argument overrides + variables defined in your dbt_project.yml file. This argument + should be a YAML string, eg. '{my_variable: my_value}'""" + ) + sub = subs.add_parser('init', parents=[base_subparser]) sub.add_argument('project_name', type=str, help='Name of the new project') sub.set_defaults(cls=init_task.InitTask, which='init') @@ -373,13 +394,23 @@ def parse_args(args): fully-recalculate the incremental table from the model definition. """) - sub = subs.add_parser('seed', parents=[base_subparser]) - sub.add_argument( + seed_sub = subs.add_parser('seed', parents=[base_subparser]) + seed_sub.add_argument( '--drop-existing', action='store_true', - help="Drop existing seed tables and recreate them" + help='(DEPRECATED) Use --full-refresh instead.' + ) + seed_sub.add_argument( + '--full-refresh', + action='store_true', + help='Drop existing seed tables and recreate them' + ) + seed_sub.add_argument( + '--show', + action='store_true', + help='Show a sample of the loaded data in the terminal' ) - sub.set_defaults(cls=seed_task.SeedTask, which='seed') + seed_sub.set_defaults(cls=seed_task.SeedTask, which='seed') sub = subs.add_parser('test', parents=[base_subparser]) sub.add_argument( diff --git a/dbt/model.py b/dbt/model.py index 5d5010f3154..298070f1b2e 100644 --- a/dbt/model.py +++ b/dbt/model.py @@ -20,7 +20,8 @@ class SourceConfig(object): 'sort', 'sql_where', 'unique_key', - 'sort_type' + 'sort_type', + 'bind' ] def __init__(self, active_project, own_project, fqn): diff --git a/dbt/node_runners.py b/dbt/node_runners.py index e69d40b4705..d877c887c17 100644 --- a/dbt/node_runners.py +++ b/dbt/node_runners.py @@ -180,12 +180,20 @@ def get_model_schemas(cls, flat_graph): return schemas + @classmethod + def before_hooks(self, project, adapter, flat_graph): + pass + @classmethod def before_run(self, project, adapter, flat_graph): pass @classmethod - def after_run(self, project, adapter, results, flat_graph, elapsed): + def after_run(self, project, adapter, results, flat_graph): + pass + + @classmethod + def after_hooks(self, project, adapter, results, flat_graph, elapsed): pass @@ -205,14 +213,14 @@ def execute(self, compiled_node, existing, flat_graph): return RunModelResult(compiled_node) def compile(self, flat_graph): - return self.compile_node(self.adapter, self.project, self.node, - flat_graph) + return self._compile_node(self.adapter, self.project, self.node, + flat_graph) @classmethod - def compile_node(cls, adapter, project, node, flat_graph): + def _compile_node(cls, adapter, project, node, flat_graph): compiler = dbt.compilation.Compiler(project) node = compiler.compile_node(node, flat_graph) - node = cls.inject_runtime_config(adapter, project, node) + node = cls._inject_runtime_config(adapter, project, node) if(node['injected_sql'] is not None and not (dbt.utils.is_type(node, NodeType.Archive))): @@ -230,15 +238,15 @@ def compile_node(cls, adapter, project, node, flat_graph): return node @classmethod - def inject_runtime_config(cls, adapter, project, node): + def _inject_runtime_config(cls, adapter, project, node): wrapped_sql = node.get('wrapped_sql') - context = cls.node_context(adapter, project, node) + context = cls._node_context(adapter, project, node) sql = dbt.clients.jinja.get_rendered(wrapped_sql, context) node['wrapped_sql'] = sql return node @classmethod - def node_context(cls, adapter, project, node): + def _node_context(cls, adapter, project, node): profile = project.run_environment() def call_get_columns_in_table(schema_name, table_name): @@ -263,6 +271,14 @@ def call_table_exists(schema, table): "already_exists": call_table_exists, } + @classmethod + def create_schemas(cls, project, adapter, flat_graph): + profile = project.run_environment() + required_schemas = cls.get_model_schemas(flat_graph) + existing_schemas = set(adapter.get_existing_schemas(profile)) + for schema in (required_schemas - existing_schemas): + adapter.create_schema(profile, schema) + class ModelRunner(CompileRunner): @@ -287,15 +303,17 @@ def run_hooks(cls, project, adapter, flat_graph, hook_type): compiled_hooks = [] for hook in hooks: - compiled = cls.compile_node(adapter, project, hook, flat_graph) + compiled = cls._compile_node(adapter, project, hook, flat_graph) model_name = compiled.get('name') statement = compiled['wrapped_sql'] - hook_dict = dbt.hooks.get_hook_dict(statement) + hook_index = hook.get('index', len(hooks)) + hook_dict = dbt.hooks.get_hook_dict(statement, index=hook_index) compiled_hooks.append(hook_dict) - for hook in compiled_hooks: + ordered_hooks = sorted(compiled_hooks, key=lambda h: h.get('index', 0)) + for hook in ordered_hooks: if dbt.flags.STRICT_MODE: dbt.contracts.graph.parsed.validate_hook(hook) @@ -316,6 +334,13 @@ def safe_run_hooks(cls, project, adapter, flat_graph, hook_type): def create_schemas(cls, project, adapter, flat_graph): profile = project.run_environment() required_schemas = cls.get_model_schemas(flat_graph) + + # Snowflake needs to issue a "use {schema}" query, where schema + # is the one defined in the profile. Create this schema if it + # does not exist, otherwise subsequent queries will fail. Generally, + # dbt expects that this schema will exist anyway. + required_schemas.add(adapter.get_default_schema(profile)) + existing_schemas = set(adapter.get_existing_schemas(profile)) for schema in (required_schemas - existing_schemas): @@ -343,8 +368,11 @@ def print_results_line(cls, results, execution_time): .format(stat_line=stat_line, execution=execution)) @classmethod - def after_run(cls, project, adapter, results, flat_graph, elapsed): + def after_run(cls, project, adapter, results, flat_graph): cls.safe_run_hooks(project, adapter, flat_graph, RunHookType.End) + + @classmethod + def after_hooks(cls, project, adapter, results, flat_graph, elapsed): cls.print_results_line(results, elapsed) def describe_node(self): @@ -454,3 +482,43 @@ def describe_node(self): def print_result_line(self, result): dbt.ui.printer.print_archive_result_line(result, self.node_index, self.num_nodes) + + +class SeedRunner(ModelRunner): + + def describe_node(self): + schema_name = self.node.get('schema') + return "seed file {}.{}".format(schema_name, self.node["name"]) + + @classmethod + def before_run(cls, project, adapter, flat_graph): + cls.create_schemas(project, adapter, flat_graph) + + def before_execute(self): + description = self.describe_node() + dbt.ui.printer.print_start_line(description, self.node_index, + self.num_nodes) + + def execute(self, compiled_node, existing_, flat_graph): + schema = compiled_node["schema"] + table_name = compiled_node["name"] + table = compiled_node["agate_table"] + self.adapter.handle_csv_table(self.profile, schema, table_name, table, + full_refresh=dbt.flags.FULL_REFRESH) + + if dbt.flags.FULL_REFRESH: + status = 'CREATE {}'.format(len(table.rows)) + else: + status = 'INSERT {}'.format(len(table.rows)) + + return RunModelResult(compiled_node, status=status) + + def compile(self, flat_graph): + return self.node + + def print_result_line(self, result): + schema_name = self.node.get('schema') + dbt.ui.printer.print_seed_result_line(result, + schema_name, + self.node_index, + self.num_nodes) diff --git a/dbt/node_types.py b/dbt/node_types.py index ae498d3ea47..5d3ef275c83 100644 --- a/dbt/node_types.py +++ b/dbt/node_types.py @@ -7,6 +7,7 @@ class NodeType(object): Archive = 'archive' Macro = 'macro' Operation = 'operation' + Seed = 'seed' @classmethod def executable(cls): @@ -15,7 +16,8 @@ def executable(cls): cls.Test, cls.Archive, cls.Analysis, - cls.Operation + cls.Operation, + cls.Seed, ] diff --git a/dbt/parser.py b/dbt/parser.py index 810685f3e44..9eb1fb694aa 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -3,7 +3,9 @@ import re import hashlib import collections +import agate +import dbt.exceptions import dbt.flags import dbt.model import dbt.utils @@ -91,17 +93,13 @@ def process_refs(flat_graph, current_project): node.get('package_name')) if target_model is None: - dbt.exceptions.ref_target_not_found( - node, - target_model_name, - target_model_package) - - if (dbt.utils.is_enabled(node) and not - dbt.utils.is_enabled(target_model)): - if dbt.utils.is_type(node, NodeType.Model): - dbt.exceptions.ref_disabled_dependency(node, target_model) - else: - node.get('config', {})['enabled'] = False + # This may raise. Even if it doesn't, we don't want to add + # this node to the graph b/c there is no destination node + node.get('config', {})['enabled'] = False + dbt.utils.invalid_ref_fail_unless_test(node, + target_model_name, + target_model_package) + continue target_model_id = target_model.get('unique_id') @@ -206,7 +204,7 @@ def parse_node(node, node_path, root_project_config, package_project_config, root_project_config, package_project_config, fqn) node['unique_id'] = node_path - node['empty'] = (len(node.get('raw_sql').strip()) == 0) + node['empty'] = ('raw_sql' in node and len(node['raw_sql'].strip()) == 0) node['fqn'] = fqn node['tags'] = tags node['config_reference'] = config @@ -218,6 +216,11 @@ def parse_node(node, node_path, root_project_config, package_project_config, config_dict.update(config.config) node['config'] = config_dict + # Set this temporarily so get_rendered() below has access to a schema + profile = dbt.utils.get_profile_from_project(root_project_config) + default_schema = profile.get('schema', 'public') + node['schema'] = default_schema + context = dbt.context.parser.generate(node, root_project_config, {"macros": macros}) @@ -232,7 +235,6 @@ def parse_node(node, node_path, root_project_config, package_project_config, adapter.release_connection(profile, node.get('name')) # Special macro defined in the global project - default_schema = context.get('schema') schema_override = config.config.get('schema') get_schema = context.get('generate_schema_name', lambda x: default_schema) node['schema'] = get_schema(schema_override) @@ -268,14 +270,27 @@ def parse_sql_nodes(nodes, root_project, projects, tags=None, macros=None): package_name, node.get('name')) - # TODO if this is set, raise a compiler error - to_return[node_path] = parse_node(node, - node_path, - root_project, - projects.get(package_name), - projects, - tags=tags, - macros=macros) + node_parsed = parse_node(node, + node_path, + root_project, + projects.get(package_name), + projects, + tags=tags, + macros=macros) + + # Ignore disabled nodes + if not node_parsed['config']['enabled']: + continue + + # Check for duplicate model names + existing_node = to_return.get(node_path) + if existing_node is not None: + raise dbt.exceptions.CompilationException( + 'Found models with the same name:\n- %s\n- %s' % ( + existing_node.get('original_file_path'), + node.get('original_file_path'))) + + to_return[node_path] = node_parsed dbt.contracts.graph.parsed.validate_nodes(to_return) @@ -376,7 +391,8 @@ def load_and_parse_run_hook_type(root_project, all_projects, hook_type, 'path': hook_path, 'original_file_path': hook_path, 'package_name': project_name, - 'raw_sql': hook + 'raw_sql': hook, + 'index': i }) tags = {hook_type} @@ -448,8 +464,12 @@ def parse_schema_tests(tests, root_project, projects, macros=None): if test_yml is None: continue + no_tests_warning = ("* WARNING: No constraints found for model" + " '{}' in file {}\n") for model_name, test_spec in test_yml.items(): if test_spec is None or test_spec.get('constraints') is None: + test_path = test.get('original_file_path', '') + logger.warning(no_tests_warning.format(model_name, test_path)) continue for test_type, configs in test_spec.get('constraints', {}).items(): @@ -467,14 +487,20 @@ def parse_schema_tests(tests, root_project, projects, macros=None): for config in configs: package_name = test.get('package_name') + test_namespace = None split = test_type.split('.') if len(split) > 1: test_type = split[1] package_name = split[0] + test_namespace = package_name to_add = parse_schema_test( - test, model_name, config, test_type, + test, + model_name, + config, + test_namespace, + test_type, root_project, projects.get(package_name), all_projects=projects, @@ -529,8 +555,8 @@ def as_kwarg(key, value): return "{key}={value}".format(key=key, value=formatted_value) -def parse_schema_test(test_base, model_name, test_config, test_type, - root_project_config, package_project_config, +def parse_schema_test(test_base, model_name, test_config, test_namespace, + test_type, root_project_config, package_project_config, all_projects, macros=None): if isinstance(test_config, (basestring, int, float, bool)): @@ -541,9 +567,14 @@ def parse_schema_test(test_base, model_name, test_config, test_type, # sort the dict so the keys are rendered deterministically (for tests) kwargs = [as_kwarg(key, test_args[key]) for key in sorted(test_args)] + if test_namespace is None: + macro_name = "test_{}".format(test_type) + else: + macro_name = "{}.test_{}".format(test_namespace, test_type) + raw_sql = "{{{{ {macro}(model=ref('{model}'), {kwargs}) }}}}".format(**{ 'model': model_name, - 'macro': "test_{}".format(test_type), + 'macro': macro_name, 'kwargs': ", ".join(kwargs) }) @@ -667,3 +698,54 @@ def parse_archives_from_project(project): }) return archives + + +def parse_seed_file(file_match, root_dir, package_name): + abspath = file_match['absolute_path'] + logger.debug("Parsing {}".format(abspath)) + to_return = {} + table_name = os.path.basename(abspath)[:-4] + node = { + 'unique_id': get_path(NodeType.Seed, package_name, table_name), + 'path': file_match['relative_path'], + 'name': table_name, + 'root_path': root_dir, + 'resource_type': NodeType.Seed, + # Give this raw_sql so it conforms to the node spec, + # use dummy text so it doesn't look like an empty node + 'raw_sql': '-- csv --', + 'package_name': package_name, + 'depends_on': {'nodes': []}, + 'original_file_path': os.path.join(file_match.get('searched_path'), + file_match.get('relative_path')), + } + try: + table = agate.Table.from_csv(abspath) + except ValueError as e: + dbt.exceptions.raise_compiler_error(str(e), node) + table.original_abspath = abspath + node['agate_table'] = table + return node + + +def load_and_parse_seeds(package_name, root_project, all_projects, root_dir, + relative_dirs, resource_type, tags=None, macros=None): + extension = "[!.#~]*.csv" + if dbt.flags.STRICT_MODE: + dbt.contracts.project.validate_list(all_projects) + file_matches = dbt.clients.system.find_matching( + root_dir, + relative_dirs, + extension) + result = {} + for file_match in file_matches: + node = parse_seed_file(file_match, root_dir, package_name) + node_path = node['unique_id'] + parsed = parse_node(node, node_path, root_project, + all_projects.get(package_name), + all_projects, tags=tags, macros=macros) + # parsed['empty'] = False + result[node_path] = parsed + + dbt.contracts.graph.parsed.validate_nodes(result) + return result diff --git a/dbt/project.py b/dbt/project.py index 8e3ec0f72bf..bdb1f811076 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -80,11 +80,19 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None, if self.profile_to_load in self.profiles: self.cfg.update(self.profiles[self.profile_to_load]) + self.compile_and_update_target() + else: raise DbtProjectError( "Could not find profile named '{}'" .format(self.profile_to_load), self) + global_vars = dbt.utils.parse_cli_vars(getattr(args, 'vars', '{}')) + if 'vars' not in self.cfg['models']: + self.cfg['models']['vars'] = {} + + self.cfg['models']['vars'].update(global_vars) + def __str__(self): return pprint.pformat({'project': self.cfg, 'profiles': self.profiles}) @@ -120,8 +128,7 @@ def compile_target(self, target_cfg): is_str = isinstance(value, dbt.compat.basestring) if is_str: - node = "config key: '{}'".format(key) - compiled_val = dbt.clients.jinja.get_rendered(value, ctx, node) + compiled_val = dbt.clients.jinja.get_rendered(value, ctx) else: compiled_val = value @@ -129,6 +136,10 @@ def compile_target(self, target_cfg): return compiled + def compile_and_update_target(self): + target = self.cfg['target'] + self.cfg['outputs'][target].update(self.run_environment()) + def run_environment(self): target_name = self.cfg['target'] if target_name in self.cfg['outputs']: @@ -219,7 +230,11 @@ def read_profiles(profiles_dir=None): raw_profiles = dbt.config.read_profile(profiles_dir) - profiles = {k: v for (k, v) in raw_profiles.items() if k != 'config'} + if raw_profiles is None: + profiles = {} + else: + profiles = {k: v for (k, v) in raw_profiles.items() if k != 'config'} + return profiles diff --git a/dbt/runner.py b/dbt/runner.py index e18ea22a08a..efb9ff7427e 100644 --- a/dbt/runner.py +++ b/dbt/runner.py @@ -26,7 +26,7 @@ def __init__(self, project, target_path, args): profile = self.project.run_environment() # TODO validate the number of threads - if self.args.threads is None: + if not getattr(self.args, "threads", None): self.threads = profile.get('threads', 1) else: self.threads = self.args.threads @@ -203,11 +203,13 @@ def run_from_graph(self, Selector, Runner, query): logger.info("") try: - Runner.before_run(self.project, adapter, flat_graph) + Runner.before_hooks(self.project, adapter, flat_graph) started = time.time() + Runner.before_run(self.project, adapter, flat_graph) res = self.execute_nodes(linker, Runner, flat_graph, dep_list) + Runner.after_run(self.project, adapter, res, flat_graph) elapsed = time.time() - started - Runner.after_run(self.project, adapter, res, flat_graph, elapsed) + Runner.after_hooks(self.project, adapter, res, flat_graph, elapsed) finally: adapter.cleanup_connections() diff --git a/dbt/seeder.py b/dbt/seeder.py deleted file mode 100644 index d30e5f131cc..00000000000 --- a/dbt/seeder.py +++ /dev/null @@ -1,139 +0,0 @@ -import os -import fnmatch -from csvkit import table as csv_table, sql as csv_sql -from sqlalchemy.dialects import postgresql as postgresql_dialect -import psycopg2 - -from dbt.source import Source -from dbt.logger import GLOBAL_LOGGER as logger -from dbt.adapters.factory import get_adapter -import dbt.exceptions - - -class Seeder: - def __init__(self, project): - self.project = project - run_environment = self.project.run_environment() - - def find_csvs(self): - return Source(self.project).get_csvs(self.project['data-paths']) - - def drop_table(self, cursor, schema, table): - sql = 'drop table if exists "{schema}"."{table}" cascade'.format( - schema=schema, table=table - ) - logger.info("Dropping table {}.{}".format(schema, table)) - cursor.execute(sql) - - def truncate_table(self, cursor, schema, table): - sql = 'truncate table "{schema}"."{table}"'.format( - schema=schema, table=table - ) - logger.info("Truncating table {}.{}".format(schema, table)) - cursor.execute(sql) - - def create_table(self, cursor, schema, table, virtual_table): - sql_table = csv_sql.make_table(virtual_table, db_schema=schema) - create_table_sql = csv_sql.make_create_table_statement( - sql_table, dialect='postgresql' - ) - logger.info("Creating table {}.{}".format(schema, table)) - cursor.execute(create_table_sql) - - def insert_into_table(self, cursor, schema, table, virtual_table): - headers = virtual_table.headers() - - header_csv = ", ".join(['"{}"'.format(h) for h in headers]) - base_insert = ('INSERT INTO "{schema}"."{table}" ({header_csv}) ' - 'VALUES '.format( - schema=schema, - table=table, - header_csv=header_csv - )) - records = [] - - def quote_or_null(s): - if s is None: - return 'null' - else: - return "'{}'".format(s) - - for row in virtual_table.to_rows(): - record_csv = ', '.join([quote_or_null(val) for val in row]) - record_csv_wrapped = "({})".format(record_csv) - records.append(record_csv_wrapped) - insert_sql = "{} {}".format(base_insert, ",\n".join(records)) - logger.info("Inserting {} records into table {}.{}" - .format(len(virtual_table.to_rows()), schema, table)) - cursor.execute(insert_sql) - - def existing_tables(self, cursor, schema): - sql = ("select tablename as name from pg_tables where " - "schemaname = '{schema}'".format(schema=schema)) - - cursor.execute(sql) - existing = set([row[0] for row in cursor.fetchall()]) - return existing - - def do_seed(self, schema, cursor, drop_existing): - existing_tables = self.existing_tables(cursor, schema) - - csvs = self.find_csvs() - statuses = [] - for csv in csvs: - - table_name = csv.name - fh = open(csv.filepath) - virtual_table = csv_table.Table.from_csv(fh, table_name) - - if table_name in existing_tables: - if drop_existing: - self.drop_table(cursor, schema, table_name) - self.create_table( - cursor, - schema, - table_name, - virtual_table - ) - else: - self.truncate_table(cursor, schema, table_name) - else: - self.create_table(cursor, schema, table_name, virtual_table) - - try: - self.insert_into_table( - cursor, schema, table_name, virtual_table - ) - statuses.append(True) - - except psycopg2.ProgrammingError as e: - statuses.append(False) - logger.info( - 'Encountered an error while inserting into table "{}"."{}"' - .format(schema, table_name) - ) - logger.info( - 'Check for formatting errors in {}'.format(csv.filepath) - ) - logger.info( - 'Try --drop-existing to delete and recreate the table ' - 'instead' - ) - logger.info(str(e)) - return all(statuses) - - def seed(self, drop_existing=False): - profile = self.project.run_environment() - - if profile.get('type') == 'snowflake': - raise dbt.exceptions.NotImplementedException( - "`seed` operation is not supported for snowflake.") - - adapter = get_adapter(profile) - connection = adapter.get_connection(profile) - - schema = connection.get('credentials', {}).get('schema') - - with connection.get('handle') as handle: - with handle.cursor() as cursor: - return self.do_seed(schema, cursor, drop_existing) diff --git a/dbt/semver.py b/dbt/semver.py index 083bd67edaf..77d71b5d3f4 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -1,20 +1,23 @@ import re +import logging from dbt.exceptions import VersionsNotCompatibleException import dbt.utils -_MATCHERS = "(?P\>=|\>|\<|\<=|=)?" -_NUM_NO_LEADING_ZEROS = "(0|[1-9][0-9]*)" -_ALPHA = "[0-9A-Za-z-]*" -_ALPHA_NO_LEADING_ZEROS = "(0|[1-9A-Za-z-][0-9A-Za-z-]*)" +logger = logging.getLogger(__name__) -_BASE_VERSION_REGEX = """ +_MATCHERS = r"(?P\>=|\>|\<|\<=|=)?" +_NUM_NO_LEADING_ZEROS = r"(0|[1-9][0-9]*)" +_ALPHA = r"[0-9A-Za-z-]*" +_ALPHA_NO_LEADING_ZEROS = r"(0|[1-9A-Za-z-][0-9A-Za-z-]*)" + +_BASE_VERSION_REGEX = r""" (?P{num_no_leading_zeros})\. (?P{num_no_leading_zeros})\. (?P{num_no_leading_zeros}) """.format(num_no_leading_zeros=_NUM_NO_LEADING_ZEROS) -_VERSION_EXTRA_REGEX = """ +_VERSION_EXTRA_REGEX = r""" (\- (?P {alpha_no_leading_zeros}(\.{alpha_no_leading_zeros})*))? @@ -25,7 +28,7 @@ alpha_no_leading_zeros=_ALPHA_NO_LEADING_ZEROS, alpha=_ALPHA) -_VERSION_REGEX = re.compile(""" +_VERSION_REGEX = re.compile(r""" ^ {matchers} {base_version_regex} @@ -57,9 +60,9 @@ def _try_combine_exact(self, a, b): def _try_combine_lower_bound_with_exact(self, lower, exact): comparison = lower.compare(exact) - if(comparison < 0 or - (comparison == 0 and - lower.matcher == Matchers.GREATER_THAN_OR_EQUAL)): + if (comparison < 0 or + (comparison == 0 and + lower.matcher == Matchers.GREATER_THAN_OR_EQUAL)): return exact raise VersionsNotCompatibleException() @@ -87,9 +90,9 @@ def _try_combine_lower_bound(self, a, b): def _try_combine_upper_bound_with_exact(self, upper, exact): comparison = upper.compare(exact) - if(comparison > 0 or - (comparison == 0 and - upper.matcher == Matchers.LESS_THAN_OR_EQUAL)): + if (comparison > 0 or + (comparison == 0 and + upper.matcher == Matchers.LESS_THAN_OR_EQUAL)): return exact raise VersionsNotCompatibleException() @@ -188,12 +191,15 @@ def to_version_string(self, skip_matcher=False): def from_version_string(cls, version_string): match = _VERSION_REGEX.match(version_string) - if match is None: - # error? - return None + if not match: + raise dbt.exceptions.SemverException( + 'Could not parse version "{}"'.format(version_string)) return VersionSpecifier(match.groupdict()) + def __str__(self): + return self.to_version_string() + def to_range(self): range_start = UnboundedVersionSpecifier() range_end = UnboundedVersionSpecifier() @@ -221,31 +227,48 @@ def compare(self, other): for key in ['major', 'minor', 'patch']: comparison = int(self[key]) - int(other[key]) - if comparison != 0: - return comparison + if comparison > 0: + return 1 + elif comparison < 0: + return -1 - if((self.matcher == Matchers.GREATER_THAN_OR_EQUAL and - other.matcher == Matchers.LESS_THAN_OR_EQUAL) or - (self.matcher == Matchers.LESS_THAN_OR_EQUAL and - other.matcher == Matchers.GREATER_THAN_OR_EQUAL)): + equal = ((self.matcher == Matchers.GREATER_THAN_OR_EQUAL and + other.matcher == Matchers.LESS_THAN_OR_EQUAL) or + (self.matcher == Matchers.LESS_THAN_OR_EQUAL and + other.matcher == Matchers.GREATER_THAN_OR_EQUAL)) + if equal: return 0 - if((self.matcher == Matchers.LESS_THAN and - other.matcher == Matchers.LESS_THAN_OR_EQUAL) or - (other.matcher == Matchers.GREATER_THAN and - self.matcher == Matchers.GREATER_THAN_OR_EQUAL) or - (self.is_upper_bound and other.is_lower_bound)): + lt = ((self.matcher == Matchers.LESS_THAN and + other.matcher == Matchers.LESS_THAN_OR_EQUAL) or + (other.matcher == Matchers.GREATER_THAN and + self.matcher == Matchers.GREATER_THAN_OR_EQUAL) or + (self.is_upper_bound and other.is_lower_bound)) + if lt: return -1 - if((other.matcher == Matchers.LESS_THAN and - self.matcher == Matchers.LESS_THAN_OR_EQUAL) or - (self.matcher == Matchers.GREATER_THAN and - other.matcher == Matchers.GREATER_THAN_OR_EQUAL) or - (self.is_lower_bound and other.is_upper_bound)): + gt = ((other.matcher == Matchers.LESS_THAN and + self.matcher == Matchers.LESS_THAN_OR_EQUAL) or + (self.matcher == Matchers.GREATER_THAN and + other.matcher == Matchers.GREATER_THAN_OR_EQUAL) or + (self.is_lower_bound and other.is_upper_bound)) + if gt: return 1 return 0 + def __lt__(self, other): + return self.compare(other) == -1 + + def __gt__(self, other): + return self.compare(other) == 1 + + def __eq___(self, other): + return self.compare(other) == 0 + + def __cmp___(self, other): + return self.compare(other) + @property def is_unbounded(self): return False @@ -270,6 +293,9 @@ class UnboundedVersionSpecifier(VersionSpecifier): def __init__(self, *args, **kwargs): super(dbt.utils.AttrDict, self).__init__(*args, **kwargs) + def __str__(self): + return "*" + @property def is_unbounded(self): return True @@ -324,7 +350,7 @@ def reduce_versions(*args): except VersionsNotCompatibleException as e: raise VersionsNotCompatibleException( 'Could not find a satisfactory version from options: {}' - .format(str(args))) + .format([str(a) for a in args])) return to_return @@ -349,9 +375,10 @@ def find_possible_versions(requested_range, available_versions): if(versions_compatible(version, requested_range.start, requested_range.end)): - possible_versions.append(version_string) + possible_versions.append(version) - return possible_versions[::-1] + sorted_versions = sorted(possible_versions, reverse=True) + return [v.to_version_string(skip_matcher=True) for v in sorted_versions] def resolve_to_specific_version(requested_range, available_versions): @@ -374,7 +401,8 @@ def resolve_to_specific_version(requested_range, available_versions): def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): for name, restriction in restrictions.items(): if not versions_compatible(*restriction): - raise VersionsNotCompatibleException('not compatible {}'.format(restriction)) + raise VersionsNotCompatibleException( + 'not compatible {}'.format(restriction)) if not unmet_dependencies: return {}, {} @@ -383,7 +411,7 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): to_return_install = {} for dependency_name, version in unmet_dependencies.items(): - print('resolving path {}'.format(dependency_name)) + logger.debug('resolving path %s', dependency_name) dependency_restrictions = reduce_versions( *restrictions.copy().get(dependency_name)) @@ -392,7 +420,8 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): version_index[dependency_name].keys()) for possible_match in possible_matches: - print('reset with {} at {}'.format(dependency_name, possible_match)) + logger.debug('reset with %s at %s', + dependency_name, possible_match) tree = {} install = {} @@ -408,12 +437,13 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): possible_match ).to_version_string_pair() - recursive_version_info = version_index.get(dependency_name, {}).get(possible_match) + ver = version_index.get(dependency_name, {}) + recursive_version_info = ver.get(possible_match) new_unmet_dependencies = dbt.utils.deep_merge( recursive_version_info.copy()) - print('new unmet dependencies') - print(new_unmet_dependencies) + logger.debug('new unmet dependencies %s', + new_unmet_dependencies) new_restrictions = dbt.utils.deep_merge( new_restrictions.copy(), @@ -427,13 +457,14 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): for name, restriction in new_restrictions.items(): if not versions_compatible(*restriction): - raise VersionsNotCompatibleException('not compatible {}'.format(new_restrictions)) + raise VersionsNotCompatibleException( + 'not compatible {}'.format(new_restrictions)) else: match_found = True - print('going down the stack with {}'.format(new_unmet_dependencies)) - print('and {}'.format(install)) + logger.debug('going down the stack with %s and %s', + new_unmet_dependencies, install) subtree, subinstall = resolve_dependency_tree( version_index, new_unmet_dependencies, @@ -452,7 +483,7 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): subinstall, {dependency_name: possible_match}) - print('then {}'.format(install)) + logger.debug('then %s', install) to_return_tree = dbt.utils.deep_merge( to_return_tree, @@ -465,18 +496,19 @@ def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): break if not match_found: - raise VersionsNotCompatibleException('No match found -- exhausted this part of the ' - 'tree.') + raise VersionsNotCompatibleException( + 'No match found -- exhausted this part of the tree.') except VersionsNotCompatibleException as e: - print(e) - print('When attempting {} at {}'.format(dependency_name, possible_match)) + logger.debug('%s -- When attempting %s at %s', + e, dependency_name, possible_match) return to_return_tree.copy(), to_return_install.copy() def resolve_dependency_set(version_index, dependencies): - tree, install = resolve_dependency_tree(version_index, dependencies, dependencies) + tree, install = resolve_dependency_tree( + version_index, dependencies, dependencies) return { 'install': install, diff --git a/dbt/task/deps.py b/dbt/task/deps.py index c75edd5a10f..5d80f881fd9 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -1,11 +1,14 @@ import os -import errno -import re +import shutil +import hashlib +import tempfile +import six +import dbt.deprecations import dbt.clients.git import dbt.clients.system -import dbt.clients.registry -import dbt.project as project +import dbt.clients.registry as registry +from dbt.clients.yaml_helper import load_yaml_text from dbt.compat import basestring from dbt.logger import GLOBAL_LOGGER as logger @@ -14,250 +17,339 @@ from dbt.task.base_task import BaseTask +DOWNLOADS_PATH = os.path.join(tempfile.gettempdir(), "dbt-downloads") -class PackageListing(AttrDict): + +class Package(object): + def __init__(self, name): + self.name = name + self._cached_metadata = None + + def __str__(self): + version = getattr(self, 'version', None) + if not version: + return self.name + version_str = version[0] \ + if len(version) == 1 else '' + return '{}@{}'.format(self.name, version_str) @classmethod - def _convert_version_strings(cls, version_strings): - if not isinstance(version_strings, list): - version_strings = [version_strings] - - return [ - VersionSpecifier.from_version_string(version_string) - for version_string in version_strings - ] - - def incorporate(self, package, version_specifiers=None): - if version_specifiers is None: - version_specifiers = [UnboundedVersionSpecifier()] - elif not isinstance(version_specifiers, list): - # error - raise Exception('bad') - else: - for version_specifier in version_specifiers: - if not isinstance(version_specifier, VersionSpecifier): - # error - raise Exception('bad') + def version_to_list(cls, version): + if version is None: + return [] + if not isinstance(version, (list, basestring)): + dbt.exceptions.raise_dependency_error( + 'version must be list or string, got {}' + .format(type(version))) + if not isinstance(version, list): + version = [version] + return version + + def _resolve_version(self): + pass + + def resolve_version(self): + try: + self._resolve_version() + except dbt.exceptions.VersionsNotCompatibleException as e: + new_msg = ('Version error for package {}: {}' + .format(self.name, e)) + six.raise_from(dbt.exceptions.DependencyException(new_msg), e) + + def version_name(self): + raise NotImplementedError() + + def _fetch_metadata(self, project): + raise NotImplementedError() + + def fetch_metadata(self, project): + if not self._cached_metadata: + self._cached_metadata = self._fetch_metadata(project) + return self._cached_metadata + + def get_project_name(self, project): + metadata = self.fetch_metadata(project) + return metadata["name"] + + def get_installation_path(self, project): + dest_dirname = self.get_project_name(project) + return os.path.join(project['modules-path'], dest_dirname) + + +class RegistryPackage(Package): + def __init__(self, package, version): + super(RegistryPackage, self).__init__(package) + self.package = package + self._version = self._sanitize_version(version) + + @classmethod + def _sanitize_version(cls, version): + version = [v if isinstance(v, VersionSpecifier) + else VersionSpecifier.from_version_string(v) + for v in cls.version_to_list(version)] + return version or [UnboundedVersionSpecifier()] + + @property + def version(self): + return self._version + + @version.setter + def version(self, version): + self._version = self._sanitize_version(version) + + def version_name(self): + self._check_version_pinned() + version_string = self.version[0].to_version_string(skip_matcher=True) + return version_string + + def incorporate(self, other): + return RegistryPackage(self.package, self.version + other.version) + + def _check_in_index(self): + index = registry.index_cached() + if self.package not in index: + dbt.exceptions.package_not_found(self.package) + + def _resolve_version(self): + self._check_in_index() + range_ = dbt.semver.reduce_versions(*self.version) + available = registry.get_available_versions(self.package) + # for now, pick a version and then recurse. later on, + # we'll probably want to traverse multiple options + # so we can match packages. not going to make a difference + # right now. + target = dbt.semver.resolve_to_specific_version(range_, available) + if not target: + dbt.exceptions.package_version_not_found( + self.package, range_, available) + self.version = target + + def _check_version_pinned(self): + if len(self.version) != 1: + dbt.exceptions.raise_dependency_error( + 'Cannot fetch metadata until the version is pinned.') + + def _fetch_metadata(self, project): + version_string = self.version_name() + return registry.package_version(self.package, version_string) + + def install(self, project): + version_string = self.version_name() + metadata = self.fetch_metadata(project) + + tar_name = '{}.{}.tar.gz'.format(self.package, version_string) + tar_path = os.path.realpath(os.path.join(DOWNLOADS_PATH, tar_name)) + dbt.clients.system.make_directory(os.path.dirname(tar_path)) + + download_url = metadata.get('downloads').get('tarball') + dbt.clients.system.download(download_url, tar_path) + deps_path = project['modules-path'] + package_name = self.get_project_name(project) + dbt.clients.system.untar_package(tar_path, deps_path, package_name) + + +class GitPackage(Package): + def __init__(self, git, version): + super(GitPackage, self).__init__(git) + self.git = git + self._checkout_name = hashlib.md5(six.b(git)).hexdigest() + self._version = self._sanitize_version(version) - if package not in self: - self[package] = version_specifiers + @classmethod + def _sanitize_version(cls, version): + return cls.version_to_list(version) or ['master'] + + @property + def version(self): + return self._version + + @version.setter + def version(self, version): + self._version = self._sanitize_version(version) + + def version_name(self): + return self._version[0] + + def incorporate(self, other): + return GitPackage(self.git, self.version + other.version) + + def _resolve_version(self): + requested = set(self.version) + if len(requested) != 1: + dbt.exceptions.raise_dependency_error( + 'git dependencies should contain exactly one version. ' + '{} contains: {}'.format(self.git, requested)) + self.version = requested.pop() + + def _checkout(self, project): + """Performs a shallow clone of the repository into the downloads + directory. This function can be called repeatedly. If the project has + already been checked out at this version, it will be a no-op. Returns + the path to the checked out directory.""" + if len(self.version) != 1: + dbt.exceptions.raise_dependency_error( + 'Cannot checkout repository until the version is pinned.') + dir_ = dbt.clients.git.clone_and_checkout( + self.git, DOWNLOADS_PATH, branch=self.version[0], + dirname=self._checkout_name) + return os.path.join(DOWNLOADS_PATH, dir_) + + def _fetch_metadata(self, project): + path = self._checkout(project) + with open(os.path.join(path, 'dbt_project.yml')) as f: + return load_yaml_text(f.read()) + + def install(self, project): + dest_path = self.get_installation_path(project) + if os.path.exists(dest_path): + dbt.clients.system.rmdir(dest_path) + shutil.move(self._checkout(project), dest_path) + + +class LocalPackage(Package): + def __init__(self, local): + super(LocalPackage, self).__init__(local) + self.local = local + + def incorporate(self, _): + return LocalPackage(self.local) + + def version_name(self): + return ''.format(self.local) + + def _fetch_metadata(self, project): + with open(os.path.join(self.local, 'dbt_project.yml')) as f: + return load_yaml_text(f.read()) + + def install(self, project): + dest_path = self.get_installation_path(project) + if os.path.exists(dest_path): + dbt.clients.system.rmdir(dest_path) + shutil.copytree(self.local, dest_path) + + +def _parse_package(dict_): + only_1_keys = ['package', 'git', 'local'] + specified = [k for k in only_1_keys if dict_.get(k)] + if len(specified) > 1: + dbt.exceptions.raise_dependency_error( + 'Packages should not contain more than one of {}; ' + 'yours has {} of them - {}' + .format(only_1_keys, len(specified), specified)) + if dict_.get('package'): + return RegistryPackage(dict_['package'], dict_.get('version')) + if dict_.get('git'): + return GitPackage(dict_['git'], dict_.get('version')) + if dict_.get('local'): + return LocalPackage(dict_['local']) + dbt.exceptions.raise_dependency_error( + 'Malformed package definition. Must contain package, git, or local.') + + +class PackageListing(AttrDict): + def incorporate(self, package): + if not isinstance(package, Package): + package = _parse_package(package) + if package.name not in self: + self[package.name] = package else: - self[package] = self[package] + version_specifiers + self[package.name] = self[package.name].incorporate(package) @classmethod def create(cls, parsed_yaml): to_return = cls({}) - if not isinstance(parsed_yaml, list): - # error - raise Exception('bad') - - if isinstance(parsed_yaml, list): - for package in parsed_yaml: - if isinstance(package, basestring): - to_return.incorporate(package) - elif isinstance(package, dict): - (package, version_strings) = package.popitem() - to_return.incorporate( - package, - cls._convert_version_strings(version_strings)) - + dbt.exceptions.raise_dependency_error( + 'Package definitions must be a list, got: {}' + .format(type(parsed_yaml))) + for package in parsed_yaml: + to_return.incorporate(package) return to_return + def incorporate_from_yaml(self, parsed_yaml): + listing = self.create(parsed_yaml) + for _, package in listing.items(): + self.incorporate(package) -def folder_from_git_remote(remote_spec): - start = remote_spec.rfind('/') + 1 - end = len(remote_spec) - (4 if remote_spec.endswith('.git') else 0) - return remote_spec[start:end] - -class DepsTask(BaseTask): - def __pull_repo(self, repo, branch=None): - modules_path = self.project['modules-path'] - - out, err = dbt.clients.git.clone(repo, modules_path) - - exists = re.match("fatal: destination path '(.+)' already exists", - err.decode('utf-8')) - - folder = None - start_sha = None - - if exists: - folder = exists.group(1) - logger.info('Updating existing dependency {}.'.format(folder)) - else: - matches = re.match("Cloning into '(.+)'", err.decode('utf-8')) - folder = matches.group(1) - logger.info('Pulling new dependency {}.'.format(folder)) - - dependency_path = os.path.join(modules_path, folder) - start_sha = dbt.clients.git.get_current_sha(dependency_path) - dbt.clients.git.checkout(dependency_path, repo, branch) - end_sha = dbt.clients.git.get_current_sha(dependency_path) - - if exists: - if start_sha == end_sha: - logger.info(' Already at {}, nothing to do.'.format( - start_sha[:7])) - else: - logger.info(' Updated checkout from {} to {}.'.format( - start_sha[:7], end_sha[:7])) - else: - logger.info(' Checked out at {}.'.format(end_sha[:7])) - - return folder - - def __split_at_branch(self, repo_spec): - parts = repo_spec.split("@") - error = RuntimeError( - "Invalid dep specified: '{}' -- not a repo we can clone".format( - repo_spec - ) +def _split_at_branch(repo_spec): + parts = repo_spec.split('@') + error = RuntimeError( + "Invalid dep specified: '{}' -- not a repo we can clone".format( + repo_spec ) - - repo = None - if repo_spec.startswith("git@"): - if len(parts) == 1: - raise error - if len(parts) == 2: - repo, branch = repo_spec, None - elif len(parts) == 3: - repo, branch = "@".join(parts[:2]), parts[2] - else: - if len(parts) == 1: - repo, branch = parts[0], None - elif len(parts) == 2: - repo, branch = parts - - if repo is None: + ) + repo = None + if repo_spec.startswith('git@'): + if len(parts) == 1: raise error + if len(parts) == 2: + repo, branch = repo_spec, None + elif len(parts) == 3: + repo, branch = '@'.join(parts[:2]), parts[2] + else: + if len(parts) == 1: + repo, branch = parts[0], None + elif len(parts) == 2: + repo, branch = parts + if repo is None: + raise error + return repo, branch + + +def _convert_repo(repo_spec): + repo, branch = _split_at_branch(repo_spec) + return { + 'git': repo, + 'version': branch, + } + + +def _read_packages(project_yaml): + packages = project_yaml.get('packages', []) + repos = project_yaml.get('repositories', []) + if repos: + dbt.deprecations.warn('repositories') + packages += [_convert_repo(r) for r in repos] + return packages - return repo, branch - - def __pull_deps_recursive(self, repos, processed_repos=None, i=0): - if processed_repos is None: - processed_repos = set() - for repo_string in repos: - repo, branch = self.__split_at_branch(repo_string) - repo_folder = folder_from_git_remote(repo) - - try: - if repo_folder in processed_repos: - logger.info( - "skipping already processed dependency {}" - .format(repo_folder) - ) - else: - dep_folder = self.__pull_repo(repo, branch) - dep_project = project.read_project( - os.path.join(self.project['modules-path'], - dep_folder, - 'dbt_project.yml'), - self.project.profiles_dir, - profile_to_load=self.project.profile_to_load - ) - processed_repos.add(dep_folder) - self.__pull_deps_recursive( - dep_project['repositories'], processed_repos, i+1 - ) - except IOError as e: - if e.errno == errno.ENOENT: - error_string = basestring(e) - - if 'dbt_project.yml' in error_string: - error_string = ("'{}' is not a valid dbt project - " - "dbt_project.yml not found" - .format(repo)) - - elif 'git' in error_string: - error_string = ("Git CLI is a dependency of dbt, but " - "it is not installed!") - - raise dbt.exceptions.RuntimeException(error_string) - - else: - raise e - - def run(self): - listing = PackageListing.create(self.project['packages']) - visited_listing = PackageListing.create([]) - index = dbt.clients.registry.index() - - while len(listing) > 0: - (package, version_specifiers) = listing.popitem() - - if package not in index: - raise Exception('unknown package {}'.format(package)) - - version_range = dbt.semver.reduce_versions( - *version_specifiers) - - available_versions = dbt.clients.registry.get_available_versions( - package) - - # for now, pick a version and then recurse. later on, - # we'll probably want to traverse multiple options - # so we can match packages. not going to make a difference - # right now. - target_version = dbt.semver.resolve_to_specific_version( - version_range, - available_versions) - - if target_version is None: - logger.error( - 'Could not find a matching version for package {}!' - .format(package)) - logger.error( - ' Requested range: {}'.format(version_range)) - logger.error( - ' Available versions: {}'.format( - ', '.join(available_versions))) - raise Exception('bad') - - visited_listing.incorporate( - package, - [VersionSpecifier.from_version_string(target_version)]) - - target_version_metadata = dbt.clients.registry.package_version( - package, target_version) - - dependencies = target_version_metadata.get('dependencies', {}) - for package, versions in dependencies.items(): - listing.incorporate( - package, - [VersionSpecifier.from_version_string(version) - for version in versions]) - - for package, version_specifiers in visited_listing.items(): - version_string = version_specifiers[0].to_version_string(True) - version_info = dbt.clients.registry.package_version( - package, version_string) - - import requests - - tar_path = os.path.realpath('{}/downloads/{}.{}.tar.gz'.format( - self.project['modules-path'], - package, - version_string)) - - logger.info("Pulling {}@{} from hub.getdbt.com...".format( - package, version_string)) - - dbt.clients.system.make_directory( - os.path.dirname(tar_path)) - - response = requests.get(version_info.get('downloads').get('tarball')) - - with open(tar_path, 'wb') as handle: - for block in response.iter_content(1024*64): - handle.write(block) - - import tarfile - - with tarfile.open(tar_path, 'r') as tarball: - tarball.extractall(self.project['modules-path']) +class DepsTask(BaseTask): + def _check_for_duplicate_project_names(self, final_deps): + seen = set() + for _, package in final_deps.items(): + project_name = package.get_project_name(self.project) + if project_name in seen: + dbt.exceptions.raise_dependency_error( + 'Found duplicate project {}. This occurs when a dependency' + ' has the same project name as some other dependency.' + .format(project_name)) + seen.add(project_name) - logger.info(" -> Success.") + def run(self): + dbt.clients.system.make_directory(self.project['modules-path']) + dbt.clients.system.make_directory(DOWNLOADS_PATH) + + packages = _read_packages(self.project) + if not packages: + logger.info('Warning: No packages found in dbt_project.yml') + return + + pending_deps = PackageListing.create(packages) + final_deps = PackageListing.create([]) + while pending_deps: + sub_deps = PackageListing.create([]) + for name, package in pending_deps.items(): + final_deps.incorporate(package) + final_deps[name].resolve_version() + target_metadata = final_deps[name].fetch_metadata(self.project) + sub_deps.incorporate_from_yaml(_read_packages(target_metadata)) + pending_deps = sub_deps + + self._check_for_duplicate_project_names(final_deps) + + for _, package in final_deps.items(): + logger.info('Installing %s', package) + package.install(self.project) + logger.info(' Installed at version %s\n', package.version_name()) diff --git a/dbt/task/init.py b/dbt/task/init.py index 26b90b6ff92..cace35a0294 100644 --- a/dbt/task/init.py +++ b/dbt/task/init.py @@ -9,7 +9,7 @@ from dbt.task.base_task import BaseTask STARTER_REPO = 'https://github.com/fishtown-analytics/dbt-starter-project.git' -DOCS_URL = 'https://dbt.readme.io/docs/configure-your-profile' +DOCS_URL = 'https://docs.getdbt.com/docs/configure-your-profile' SAMPLE_PROFILES_YML_FILE = 'https://github.com/fishtown-analytics/dbt/blob/master/sample.profiles.yml' # noqa ON_COMPLETE_MESSAGE = """ diff --git a/dbt/task/seed.py b/dbt/task/seed.py index fabc14eb412..1f3ac5f7e42 100644 --- a/dbt/task/seed.py +++ b/dbt/task/seed.py @@ -1,12 +1,47 @@ -import os -from dbt.seeder import Seeder -from dbt.task.base_task import BaseTask +import random +from dbt.logger import GLOBAL_LOGGER as logger +from dbt.node_runners import SeedRunner +from dbt.node_types import NodeType +from dbt.runner import RunManager +from dbt.task.base_task import RunnableTask +import dbt.ui.printer -class SeedTask(BaseTask): +class SeedTask(RunnableTask): def run(self): - seeder = Seeder(self.project) - self.success = seeder.seed(self.args.drop_existing) + runner = RunManager( + self.project, + self.project["target-path"], + self.args, + ) + query = { + "include": ["*"], + "exclude": [], + "resource_types": [NodeType.Seed], + } + results = runner.run_flat(query, SeedRunner) - def interpret_results(self, results): - return self.success + if self.args.show: + self.show_tables(results) + + dbt.ui.printer.print_run_end_messages(results) + return results + + def show_table(self, result): + table = result.node['agate_table'] + rand_table = table.order_by(lambda x: random.random()) + + schema = result.node['schema'] + name = result.node['name'] + + header = "Random sample of table: {}.{}".format(schema, name) + logger.info("") + logger.info(header) + logger.info("-" * len(header)) + rand_table.print_table(max_rows=10, max_columns=None) + logger.info("") + + def show_tables(self, results): + for result in results: + if not result.errored: + self.show_table(result) diff --git a/dbt/tracking.py b/dbt/tracking.py index 39e5b7b0436..0a724e878a9 100644 --- a/dbt/tracking.py +++ b/dbt/tracking.py @@ -4,6 +4,7 @@ from snowplow_tracker import SelfDescribingJson, disable_contracts from datetime import datetime +import pytz import platform import uuid import yaml @@ -44,7 +45,7 @@ def __init__(self): self.id = None self.invocation_id = str(uuid.uuid4()) - self.run_started_at = datetime.now() + self.run_started_at = datetime.now(tz=pytz.utc) def state(self): return "do not track" if self.do_not_track else "tracking" diff --git a/dbt/ui/printer.py b/dbt/ui/printer.py index 33904c1c2b4..871bedd2c63 100644 --- a/dbt/ui/printer.py +++ b/dbt/ui/printer.py @@ -178,6 +178,22 @@ def print_archive_result_line(result, index, total): result.execution_time) +def print_seed_result_line(result, schema_name, index, total): + model = result.node + + info, status = get_printable_result(result, 'loaded', 'loading') + + print_fancy_output_line( + "{info} seed file {schema}.{relation}".format( + info=info, + schema=schema_name, + relation=model.get('name')), + status, + index, + total, + result.execution_time) + + def interpret_run_result(result): if result.errored or result.failed: return 'error' diff --git a/dbt/utils.py b/dbt/utils.py index cd2b32e73c0..d0a591ec4b4 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -1,6 +1,8 @@ import os import hashlib import itertools +import collections +import functools import dbt.exceptions import dbt.flags @@ -9,6 +11,7 @@ from dbt.compat import basestring from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_types import NodeType +from dbt.clients import yaml_helper DBTConfigKeys = [ @@ -22,7 +25,8 @@ 'sort_type', 'pre-hook', 'post-hook', - 'vars' + 'vars', + 'bind', ] @@ -33,9 +37,7 @@ class ExitCodes(object): class Relation(object): - def __init__(self, adapter, node, use_temp=False): - self._adapter = adapter - + def __init__(self, profile, adapter, node, use_temp=False): self.node = node self.schema = node.get('schema') self.name = node.get('name') @@ -48,16 +50,36 @@ def __init__(self, adapter, node, use_temp=False): self.materialized = get_materialization(node) self.sql = node.get('injected_sql') + self.do_quote = self._get_quote_function(profile, adapter) + + def _get_quote_function(self, profile, adapter): + + # make a closure so we don't need to store the profile + # on the `Relation` object. That shouldn't be accessible in user-land + def quote(schema, table): + return adapter.quote_schema_and_table( + profile=profile, + schema=schema, + table=table + ) + + return quote + def _get_table_name(self, node): return model_immediate_name(node, dbt.flags.NON_DESTRUCTIVE) + def final_name(self): + if self.materialized == 'ephemeral': + msg = "final_name() was called on an ephemeral model" + dbt.exceptions.raise_compiler_error(msg, self.node) + else: + return self.do_quote(self.schema, self.name) + def __repr__(self): if self.materialized == 'ephemeral': return '__dbt__CTE__{}'.format(self.name) else: - return self._adapter.quote_schema_and_table(profile=None, - schema=self.schema, - table=self.table) + return self.do_quote(self.schema, self.table) def coalesce(*args): @@ -67,6 +89,12 @@ def coalesce(*args): return None +def get_profile_from_project(project): + target_name = project.get('target', {}) + profile = project.get('outputs', {}).get(target_name, {}) + return profile + + def get_model_name_or_none(model): if model is None: name = '' @@ -214,10 +242,10 @@ def merge(*args): if len(args) == 1: return args[0] - l = list(args) - last = l.pop(len(l)-1) + lst = list(args) + last = lst.pop(len(lst)-1) - return _merge(merge(*l), last) + return _merge(merge(*lst), last) def _merge(a, b): @@ -238,10 +266,10 @@ def deep_merge(*args): if len(args) == 1: return args[0] - l = list(args) - last = l.pop(len(l)-1) + lst = list(args) + last = lst.pop(len(lst)-1) - return _deep_merge(deep_merge(*l), last) + return _deep_merge(deep_merge(*lst), last) def _deep_merge(destination, source): @@ -335,3 +363,78 @@ def get_hashed_contents(model): def flatten_nodes(dep_list): return list(itertools.chain.from_iterable(dep_list)) + + +class memoized(object): + '''Decorator. Caches a function's return value each time it is called. If + called later with the same arguments, the cached value is returned (not + reevaluated). + + Taken from https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize''' + def __init__(self, func): + self.func = func + self.cache = {} + + def __call__(self, *args): + if not isinstance(args, collections.Hashable): + # uncacheable. a list, for instance. + # better to not cache than blow up. + return self.func(*args) + if args in self.cache: + return self.cache[args] + value = self.func(*args) + self.cache[args] = value + return value + + def __repr__(self): + '''Return the function's docstring.''' + return self.func.__doc__ + + def __get__(self, obj, objtype): + '''Support instance methods.''' + return functools.partial(self.__call__, obj) + + +def max_digits(values): + """Given a series of decimal.Decimal values, find the maximum + number of digits (on both sides of the decimal point) used by the + values.""" + max_ = 0 + for value in values: + if value is None: + continue + sign, digits, exponent = value.normalize().as_tuple() + max_ = max(len(digits), max_) + return max_ + + +def invalid_ref_fail_unless_test(node, target_model_name, + target_model_package): + if node.get('resource_type') == NodeType.Test: + warning = dbt.exceptions.get_target_not_found_msg( + node, + target_model_name, + target_model_package) + logger.debug("WARNING: {}".format(warning)) + else: + dbt.exceptions.ref_target_not_found( + node, + target_model_name, + target_model_package) + + +def parse_cli_vars(var_string): + try: + cli_vars = yaml_helper.load_yaml_text(var_string) + var_type = type(cli_vars) + if var_type == dict: + return cli_vars + else: + type_name = var_type.__name__ + dbt.exceptions.raise_compiler_error( + "The --vars argument must be a YAML dictionary, but was " + "of type '{}'".format(type_name)) + except dbt.exceptions.ValidationException as e: + logger.error( + "The YAML provided in the --vars argument is not valid.\n") + raise diff --git a/dbt/version.py b/dbt/version.py index 05d3e5e6898..b4719560fde 100644 --- a/dbt/version.py +++ b/dbt/version.py @@ -31,7 +31,7 @@ def get_latest_version(): try: f = urlopen(REMOTE_VERSION_FILE) contents = f.read() - except: + except Exception: contents = '' if hasattr(contents, 'decode'): contents = contents.decode('utf-8') @@ -42,7 +42,7 @@ def not_latest(): return """Your version of dbt is out of date! You can find instructions for upgrading here: - https://dbt.readme.io/docs/installation + https://docs.getdbt.com/docs/installation """ @@ -67,6 +67,6 @@ def is_latest(): return installed == latest -__version__ = '0.9.0a2' +__version__ = '0.9.1' installed = get_version() latest = get_latest_version() diff --git a/dev_requirements.txt b/dev_requirements.txt index bd576fdfb5c..d205dbd8f9d 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,8 @@ +freezegun==0.3.9 nose>=1.3.7 mock>=1.3.0 pep8>=1.6.2 +pytz==2017.2 bumpversion==0.5.3 coverage==4.2 tox==2.5.0 diff --git a/docs/about/contributing.md b/docs/about/contributing.md deleted file mode 100644 index 8afeb267268..00000000000 --- a/docs/about/contributing.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/contributing](https://dbt.readme.io/docs/contributing) diff --git a/docs/about/license.md b/docs/about/license.md deleted file mode 100644 index 8f34cd21140..00000000000 --- a/docs/about/license.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/license](https://dbt.readme.io/docs/license) diff --git a/docs/about/overview.md b/docs/about/overview.md deleted file mode 100644 index 4c7903710bf..00000000000 --- a/docs/about/overview.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/overview](https://dbt.readme.io/docs/overview) diff --git a/docs/about/release-notes.md b/docs/about/release-notes.md deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/about/viewpoint.md b/docs/about/viewpoint.md deleted file mode 100644 index 44ca92142a7..00000000000 --- a/docs/about/viewpoint.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/viewpoint](https://dbt.readme.io/docs/viewpoint) diff --git a/docs/guide/archival.md b/docs/guide/archival.md deleted file mode 100644 index fc05c855c34..00000000000 --- a/docs/guide/archival.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/reference#archive](https://dbt.readme.io/reference#archive) diff --git a/docs/guide/best-practices.md b/docs/guide/best-practices.md deleted file mode 100644 index 16353aad49f..00000000000 --- a/docs/guide/best-practices.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/best-practices](https://dbt.readme.io/docs/best-practices) diff --git a/docs/guide/building-models.md b/docs/guide/building-models.md deleted file mode 100644 index 3fe103dde3f..00000000000 --- a/docs/guide/building-models.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/building-models](https://dbt.readme.io/docs/building-models) diff --git a/docs/guide/configuring-models.md b/docs/guide/configuring-models.md deleted file mode 100644 index 658e1f53f91..00000000000 --- a/docs/guide/configuring-models.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/configuring-models](https://dbt.readme.io/docs/configuring-models) diff --git a/docs/guide/context-variables.md b/docs/guide/context-variables.md deleted file mode 100644 index fdf55f1838f..00000000000 --- a/docs/guide/context-variables.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/reference#adapter](https://dbt.readme.io/reference#adapter) diff --git a/docs/guide/database-optimizations.md b/docs/guide/database-optimizations.md deleted file mode 100644 index 65e0cf7d63f..00000000000 --- a/docs/guide/database-optimizations.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/performance-optimization](https://dbt.readme.io/docs/performance-optimization) diff --git a/docs/guide/macros.md b/docs/guide/macros.md deleted file mode 100644 index 75f9c06212f..00000000000 --- a/docs/guide/macros.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/macros](https://dbt.readme.io/docs/macros) diff --git a/docs/guide/package-management.md b/docs/guide/package-management.md deleted file mode 100644 index 3904bf0fa65..00000000000 --- a/docs/guide/package-management.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/package-management](https://dbt.readme.io/docs/package-management) diff --git a/docs/guide/setup.md b/docs/guide/setup.md deleted file mode 100644 index 59149af82e7..00000000000 --- a/docs/guide/setup.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/installation](https://dbt.readme.io/docs/installation) diff --git a/docs/guide/testing.md b/docs/guide/testing.md deleted file mode 100644 index 48ff985e0e8..00000000000 --- a/docs/guide/testing.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/testing](https://dbt.readme.io/docs/testing) diff --git a/docs/guide/upgrading.md b/docs/guide/upgrading.md deleted file mode 100644 index 59149af82e7..00000000000 --- a/docs/guide/upgrading.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/installation](https://dbt.readme.io/docs/installation) diff --git a/docs/guide/usage.md b/docs/guide/usage.md deleted file mode 100644 index 00fd1be86c7..00000000000 --- a/docs/guide/usage.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/reference#run](https://dbt.readme.io/reference#run) diff --git a/docs/guide/using-hooks.md b/docs/guide/using-hooks.md deleted file mode 100644 index aac3c183dd0..00000000000 --- a/docs/guide/using-hooks.md +++ /dev/null @@ -1 +0,0 @@ -Moved to [https://dbt.readme.io/docs/using-hooks](https://dbt.readme.io/docs/using-hooks) diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 5d455cd9145..00000000000 --- a/docs/index.md +++ /dev/null @@ -1 +0,0 @@ -The dbt docs have moved to [https://dbt.readme.io/](https://dbt.readme.io/) diff --git a/requirements.txt b/requirements.txt index 906e96c7b17..1985b3e5a5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,15 @@ argparse>=1.2.1 +freezegun==0.3.9 Jinja2>=2.8 +pytz==2017.2 PyYAML>=3.11 psycopg2==2.7.1 sqlparse==0.2.3 networkx==1.11 -csvkit==0.9.1 snowplow-tracker==0.7.2 celery==3.1.23 voluptuous==0.10.5 -snowflake-connector-python==1.4.0 colorama==0.3.9 google-cloud-bigquery==0.26.0 -pyasn1==0.2.3 +snowflake-connector-python>=1.4.9 +agate>=1.6,<2 diff --git a/sample.dbt_project.yml b/sample.dbt_project.yml index 726e3db22b8..158a64b411d 100644 --- a/sample.dbt_project.yml +++ b/sample.dbt_project.yml @@ -3,7 +3,7 @@ # that dbt needs in order to build your models. This file is _required_ # # For more information, consult: -# https://dbt.readme.io/reference#configuration +# https://docs.getdbt.com/reference#configuration # @@ -197,7 +197,7 @@ repositories: # value changes for a given user record (identified by the `id` field), dbt # will record a new record in `users_archived` table which reflects the # changed state of that row. For more information on this command, consult -# the dbt documentation: https://dbt.readme.io/reference#archive +# the dbt documentation: https://docs.getdbt.com/reference#archive archive: - source_schema: public target_schema: public diff --git a/setup.py b/setup.py index 7b639a97963..23ecc1bc669 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ package_name = "dbt" -package_version = "0.9.0a2" +package_version = "0.9.1" setup( name=package_name, @@ -31,18 +31,19 @@ 'scripts/dbt', ], install_requires=[ + 'freezegun==0.3.9', 'Jinja2>=2.8', + 'pytz==2017.2', 'PyYAML>=3.11', 'psycopg2==2.7.1', 'sqlparse==0.2.3', 'networkx==1.11', - 'csvkit==0.9.1', 'snowplow-tracker==0.7.2', 'celery==3.1.23', 'voluptuous==0.10.5', - 'snowflake-connector-python>=1.3.16', + 'snowflake-connector-python>=1.4.9', 'colorama==0.3.9', 'google-cloud-bigquery==0.26.0', - 'pyasn1==0.2.3', + 'agate>=1.6,<2', ] ) diff --git a/test/integration/001_simple_copy_test/models/schema.yml b/test/integration/001_simple_copy_test/models/schema.yml new file mode 100644 index 00000000000..f8eb708f45f --- /dev/null +++ b/test/integration/001_simple_copy_test/models/schema.yml @@ -0,0 +1,8 @@ + + +# Confirm that this does not throw an exception for +# a missing ref to the disabled model +disabled: + constraints: + unique: + - id diff --git a/test/integration/001_simple_copy_test/seed-initial/seed.csv b/test/integration/001_simple_copy_test/seed-initial/seed.csv new file mode 100644 index 00000000000..640af6c4ee6 --- /dev/null +++ b/test/integration/001_simple_copy_test/seed-initial/seed.csv @@ -0,0 +1,101 @@ +id,first_name,last_name,email,gender,ip_address +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 +21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 +93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 diff --git a/test/integration/001_simple_copy_test/seed-update/seed.csv b/test/integration/001_simple_copy_test/seed-update/seed.csv new file mode 100644 index 00000000000..5b93306a280 --- /dev/null +++ b/test/integration/001_simple_copy_test/seed-update/seed.csv @@ -0,0 +1,201 @@ +id,first_name,last_name,email,gender,ip_address +1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 +2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 +3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 +4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 +5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 +6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 +7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 +8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 +9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 +10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 +11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 +12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 +13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 +14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 +15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 +16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 +17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 +18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 +19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 +20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 +21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 +22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 +23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 +24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 +25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 +26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 +27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 +28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 +29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 +30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 +31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 +32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 +33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 +34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 +35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 +36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 +37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 +38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 +39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 +40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 +41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 +42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 +43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 +44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 +45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 +46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 +47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 +48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 +49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 +50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 +51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 +52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 +53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 +54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 +55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 +56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 +57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 +58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 +59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 +60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 +61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 +62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 +63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 +64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 +65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 +66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 +67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 +68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 +69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 +70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 +71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 +72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 +73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 +74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 +75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 +76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 +77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 +78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 +79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 +80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 +81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 +82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 +83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 +84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 +85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 +86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 +87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 +88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 +89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 +90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 +91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 +92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 +93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 +94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 +95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 +96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 +97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 +98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 +99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 +100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 +101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175 +102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182 +103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106 +104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114 +105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238 +106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40 +107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115 +108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194 +109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131 +110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168 +111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54 +112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195 +113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214 +114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251 +115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235 +116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176 +117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12 +118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123 +119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63 +120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148 +121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249 +122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249 +123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46 +124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202 +125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139 +126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78 +127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107 +128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217 +129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213 +130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60 +131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87 +132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153 +133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24 +134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58 +135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184 +136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137 +137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104 +138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101 +139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18 +140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249 +141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243 +142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244 +143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119 +144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81 +145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192 +146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197 +147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144 +148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163 +149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194 +150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160 +151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20 +152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110 +153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245 +154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4 +155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252 +156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209 +157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87 +158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86 +159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202 +160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128 +161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26 +162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149 +163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255 +164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58 +165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179 +166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93 +167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22 +168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52 +169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86 +170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168 +171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218 +172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69 +173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211 +174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12 +175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28 +176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1 +177,Anne,Wells,awells24@apache.org,Female,180.168.81.153 +178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21 +179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178 +180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30 +181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194 +182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86 +183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184 +184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52 +185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210 +186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129 +187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251 +188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135 +189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149 +190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220 +191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214 +192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197 +193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136 +194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49 +195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186 +196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187 +197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46 +198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13 +199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253 +200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117 diff --git a/test/integration/001_simple_copy_test/seed.sql b/test/integration/001_simple_copy_test/seed.sql deleted file mode 100644 index 28309b400f8..00000000000 --- a/test/integration/001_simple_copy_test/seed.sql +++ /dev/null @@ -1,111 +0,0 @@ -create table "{schema}"."seed" ( - id BIGSERIAL PRIMARY KEY, - first_name VARCHAR(50), - last_name VARCHAR(50), - email VARCHAR(50), - gender VARCHAR(50), - ip_address VARCHAR(20) -); - - -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values -('Jack', 'Hunter', 'jhunter0@pbs.org', 'Male', '59.80.20.168'), -('Kathryn', 'Walker', 'kwalker1@ezinearticles.com', 'Female', '194.121.179.35'), -('Gerald', 'Ryan', 'gryan2@com.com', 'Male', '11.3.212.243'), -('Bonnie', 'Spencer', 'bspencer3@ameblo.jp', 'Female', '216.32.196.175'), -('Harold', 'Taylor', 'htaylor4@people.com.cn', 'Male', '253.10.246.136'), -('Jacqueline', 'Griffin', 'jgriffin5@t.co', 'Female', '16.13.192.220'), -('Wanda', 'Arnold', 'warnold6@google.nl', 'Female', '232.116.150.64'), -('Craig', 'Ortiz', 'cortiz7@sciencedaily.com', 'Male', '199.126.106.13'), -('Gary', 'Day', 'gday8@nih.gov', 'Male', '35.81.68.186'), -('Rose', 'Wright', 'rwright9@yahoo.co.jp', 'Female', '236.82.178.100'), -('Raymond', 'Kelley', 'rkelleya@fc2.com', 'Male', '213.65.166.67'), -('Gerald', 'Robinson', 'grobinsonb@disqus.com', 'Male', '72.232.194.193'), -('Mildred', 'Martinez', 'mmartinezc@samsung.com', 'Female', '198.29.112.5'), -('Dennis', 'Arnold', 'darnoldd@google.com', 'Male', '86.96.3.250'), -('Judy', 'Gray', 'jgraye@opensource.org', 'Female', '79.218.162.245'), -('Theresa', 'Garza', 'tgarzaf@epa.gov', 'Female', '21.59.100.54'), -('Gerald', 'Robertson', 'grobertsong@csmonitor.com', 'Male', '131.134.82.96'), -('Philip', 'Hernandez', 'phernandezh@adobe.com', 'Male', '254.196.137.72'), -('Julia', 'Gonzalez', 'jgonzalezi@cam.ac.uk', 'Female', '84.240.227.174'), -('Andrew', 'Davis', 'adavisj@patch.com', 'Male', '9.255.67.25'), -('Kimberly', 'Harper', 'kharperk@foxnews.com', 'Female', '198.208.120.253'), -('Mark', 'Martin', 'mmartinl@marketwatch.com', 'Male', '233.138.182.153'), -('Cynthia', 'Ruiz', 'cruizm@google.fr', 'Female', '18.178.187.201'), -('Samuel', 'Carroll', 'scarrolln@youtu.be', 'Male', '128.113.96.122'), -('Jennifer', 'Larson', 'jlarsono@vinaora.com', 'Female', '98.234.85.95'), -('Ashley', 'Perry', 'aperryp@rakuten.co.jp', 'Female', '247.173.114.52'), -('Howard', 'Rodriguez', 'hrodriguezq@shutterfly.com', 'Male', '231.188.95.26'), -('Amy', 'Brooks', 'abrooksr@theatlantic.com', 'Female', '141.199.174.118'), -('Louise', 'Warren', 'lwarrens@adobe.com', 'Female', '96.105.158.28'), -('Tina', 'Watson', 'twatsont@myspace.com', 'Female', '251.142.118.177'), -('Janice', 'Kelley', 'jkelleyu@creativecommons.org', 'Female', '239.167.34.233'), -('Terry', 'Mccoy', 'tmccoyv@bravesites.com', 'Male', '117.201.183.203'), -('Jeffrey', 'Morgan', 'jmorganw@surveymonkey.com', 'Male', '78.101.78.149'), -('Louis', 'Harvey', 'lharveyx@sina.com.cn', 'Male', '51.50.0.167'), -('Philip', 'Miller', 'pmillery@samsung.com', 'Male', '103.255.222.110'), -('Willie', 'Marshall', 'wmarshallz@ow.ly', 'Male', '149.219.91.68'), -('Patrick', 'Lopez', 'plopez10@redcross.org', 'Male', '250.136.229.89'), -('Adam', 'Jenkins', 'ajenkins11@harvard.edu', 'Male', '7.36.112.81'), -('Benjamin', 'Cruz', 'bcruz12@linkedin.com', 'Male', '32.38.98.15'), -('Ruby', 'Hawkins', 'rhawkins13@gmpg.org', 'Female', '135.171.129.255'), -('Carlos', 'Barnes', 'cbarnes14@a8.net', 'Male', '240.197.85.140'), -('Ruby', 'Griffin', 'rgriffin15@bravesites.com', 'Female', '19.29.135.24'), -('Sean', 'Mason', 'smason16@icq.com', 'Male', '159.219.155.249'), -('Anthony', 'Payne', 'apayne17@utexas.edu', 'Male', '235.168.199.218'), -('Steve', 'Cruz', 'scruz18@pcworld.com', 'Male', '238.201.81.198'), -('Anthony', 'Garcia', 'agarcia19@flavors.me', 'Male', '25.85.10.18'), -('Doris', 'Lopez', 'dlopez1a@sphinn.com', 'Female', '245.218.51.238'), -('Susan', 'Nichols', 'snichols1b@freewebs.com', 'Female', '199.99.9.61'), -('Wanda', 'Ferguson', 'wferguson1c@yahoo.co.jp', 'Female', '236.241.135.21'), -('Andrea', 'Pierce', 'apierce1d@google.co.uk', 'Female', '132.40.10.209'), -('Lawrence', 'Phillips', 'lphillips1e@jugem.jp', 'Male', '72.226.82.87'), -('Judy', 'Gilbert', 'jgilbert1f@multiply.com', 'Female', '196.250.15.142'), -('Eric', 'Williams', 'ewilliams1g@joomla.org', 'Male', '222.202.73.126'), -('Ralph', 'Romero', 'rromero1h@sogou.com', 'Male', '123.184.125.212'), -('Jean', 'Wilson', 'jwilson1i@ocn.ne.jp', 'Female', '176.106.32.194'), -('Lori', 'Reynolds', 'lreynolds1j@illinois.edu', 'Female', '114.181.203.22'), -('Donald', 'Moreno', 'dmoreno1k@bbc.co.uk', 'Male', '233.249.97.60'), -('Steven', 'Berry', 'sberry1l@eepurl.com', 'Male', '186.193.50.50'), -('Theresa', 'Shaw', 'tshaw1m@people.com.cn', 'Female', '120.37.71.222'), -('John', 'Stephens', 'jstephens1n@nationalgeographic.com', 'Male', '191.87.127.115'), -('Richard', 'Jacobs', 'rjacobs1o@state.tx.us', 'Male', '66.210.83.155'), -('Andrew', 'Lawson', 'alawson1p@over-blog.com', 'Male', '54.98.36.94'), -('Peter', 'Morgan', 'pmorgan1q@rambler.ru', 'Male', '14.77.29.106'), -('Nicole', 'Garrett', 'ngarrett1r@zimbio.com', 'Female', '21.127.74.68'), -('Joshua', 'Kim', 'jkim1s@edublogs.org', 'Male', '57.255.207.41'), -('Ralph', 'Roberts', 'rroberts1t@people.com.cn', 'Male', '222.143.131.109'), -('George', 'Montgomery', 'gmontgomery1u@smugmug.com', 'Male', '76.75.111.77'), -('Gerald', 'Alvarez', 'galvarez1v@flavors.me', 'Male', '58.157.186.194'), -('Donald', 'Olson', 'dolson1w@whitehouse.gov', 'Male', '69.65.74.135'), -('Carlos', 'Morgan', 'cmorgan1x@pbs.org', 'Male', '96.20.140.87'), -('Aaron', 'Stanley', 'astanley1y@webnode.com', 'Male', '163.119.217.44'), -('Virginia', 'Long', 'vlong1z@spiegel.de', 'Female', '204.150.194.182'), -('Robert', 'Berry', 'rberry20@tripadvisor.com', 'Male', '104.19.48.241'), -('Antonio', 'Brooks', 'abrooks21@unesco.org', 'Male', '210.31.7.24'), -('Ruby', 'Garcia', 'rgarcia22@ovh.net', 'Female', '233.218.162.214'), -('Jack', 'Hanson', 'jhanson23@blogtalkradio.com', 'Male', '31.55.46.199'), -('Kathryn', 'Nelson', 'knelson24@walmart.com', 'Female', '14.189.146.41'), -('Jason', 'Reed', 'jreed25@printfriendly.com', 'Male', '141.189.89.255'), -('George', 'Coleman', 'gcoleman26@people.com.cn', 'Male', '81.189.221.144'), -('Rose', 'King', 'rking27@ucoz.com', 'Female', '212.123.168.231'), -('Johnny', 'Holmes', 'jholmes28@boston.com', 'Male', '177.3.93.188'), -('Katherine', 'Gilbert', 'kgilbert29@altervista.org', 'Female', '199.215.169.61'), -('Joshua', 'Thomas', 'jthomas2a@ustream.tv', 'Male', '0.8.205.30'), -('Julie', 'Perry', 'jperry2b@opensource.org', 'Female', '60.116.114.192'), -('Richard', 'Perry', 'rperry2c@oracle.com', 'Male', '181.125.70.232'), -('Kenneth', 'Ruiz', 'kruiz2d@wikimedia.org', 'Male', '189.105.137.109'), -('Jose', 'Morgan', 'jmorgan2e@webnode.com', 'Male', '101.134.215.156'), -('Donald', 'Campbell', 'dcampbell2f@goo.ne.jp', 'Male', '102.120.215.84'), -('Debra', 'Collins', 'dcollins2g@uol.com.br', 'Female', '90.13.153.235'), -('Jesse', 'Johnson', 'jjohnson2h@stumbleupon.com', 'Male', '225.178.125.53'), -('Elizabeth', 'Stone', 'estone2i@histats.com', 'Female', '123.184.126.221'), -('Angela', 'Rogers', 'arogers2j@goodreads.com', 'Female', '98.104.132.187'), -('Emily', 'Dixon', 'edixon2k@mlb.com', 'Female', '39.190.75.57'), -('Albert', 'Scott', 'ascott2l@tinypic.com', 'Male', '40.209.13.189'), -('Barbara', 'Peterson', 'bpeterson2m@ow.ly', 'Female', '75.249.136.180'), -('Adam', 'Greene', 'agreene2n@fastcompany.com', 'Male', '184.173.109.144'), -('Earl', 'Sanders', 'esanders2o@hc360.com', 'Male', '247.34.90.117'), -('Angela', 'Brooks', 'abrooks2p@mtv.com', 'Female', '10.63.249.126'), -('Harold', 'Foster', 'hfoster2q@privacy.gov.au', 'Male', '139.214.40.244'), -('Carl', 'Meyer', 'cmeyer2r@disqus.com', 'Male', '204.117.7.88'); diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index 5f67d72eb9d..fcf720f7aac 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -11,57 +11,61 @@ def setUp(self): def schema(self): return "simple_copy_001" + @staticmethod + def dir(path): + return "test/integration/001_simple_copy_test/" + path.lstrip("/") + @property def models(self): - return "test/integration/001_simple_copy_test/models" + return self.dir("models") - @attr(type='postgres') + @attr(type="postgres") def test__postgres__simple_copy(self): - self.use_default_project() - self.use_profile('postgres') - self.run_sql_file("test/integration/001_simple_copy_test/seed.sql") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) + self.use_profile("postgres") + self.run_dbt(["seed"]) self.run_dbt() self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - self.run_sql_file("test/integration/001_simple_copy_test/update.sql") - + self.use_default_project({"data-paths": [self.dir("seed-update")]}) + self.run_dbt(["seed"]) self.run_dbt() self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - @attr(type='postgres') + @attr(type="postgres") def test__postgres__dbt_doesnt_run_empty_models(self): - self.use_default_project() - self.use_profile('postgres') - self.run_sql_file("test/integration/001_simple_copy_test/seed.sql") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) + self.use_profile("postgres") + self.run_dbt(["seed"]) self.run_dbt() models = self.get_models_in_schema() - self.assertFalse('empty' in models.keys()) - self.assertFalse('disabled' in models.keys()) + self.assertFalse("empty" in models.keys()) + self.assertFalse("disabled" in models.keys()) - @attr(type='snowflake') + @attr(type="snowflake") def test__snowflake__simple_copy(self): - self.use_default_project() - self.use_profile('snowflake') - self.run_sql_file("test/integration/001_simple_copy_test/seed.sql") + self.use_default_project({"data-paths": [self.dir("seed-initial")]}) + self.use_profile("snowflake") + self.run_dbt(["seed"]) self.run_dbt() self.assertTablesEqual("seed","view") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") - self.run_sql_file("test/integration/001_simple_copy_test/update.sql") - + self.use_default_project({"data-paths": [self.dir("seed-update")]}) + self.run_dbt(["seed"]) self.run_dbt() self.assertTablesEqual("seed","view") diff --git a/test/integration/001_simple_copy_test/update.sql b/test/integration/001_simple_copy_test/update.sql deleted file mode 100644 index e78d5d7d6df..00000000000 --- a/test/integration/001_simple_copy_test/update.sql +++ /dev/null @@ -1,101 +0,0 @@ -insert into "{schema}"."seed" (first_name, last_name, email, gender, ip_address) values -('Michael', 'Perez', 'mperez0@chronoengine.com', 'Male', '106.239.70.175'), -('Shawn', 'Mccoy', 'smccoy1@reddit.com', 'Male', '24.165.76.182'), -('Kathleen', 'Payne', 'kpayne2@cargocollective.com', 'Female', '113.207.168.106'), -('Jimmy', 'Cooper', 'jcooper3@cargocollective.com', 'Male', '198.24.63.114'), -('Katherine', 'Rice', 'krice4@typepad.com', 'Female', '36.97.186.238'), -('Sarah', 'Ryan', 'sryan5@gnu.org', 'Female', '119.117.152.40'), -('Martin', 'Mcdonald', 'mmcdonald6@opera.com', 'Male', '8.76.38.115'), -('Frank', 'Robinson', 'frobinson7@wunderground.com', 'Male', '186.14.64.194'), -('Jennifer', 'Franklin', 'jfranklin8@mail.ru', 'Female', '91.216.3.131'), -('Henry', 'Welch', 'hwelch9@list-manage.com', 'Male', '176.35.182.168'), -('Fred', 'Snyder', 'fsnydera@reddit.com', 'Male', '217.106.196.54'), -('Amy', 'Dunn', 'adunnb@nba.com', 'Female', '95.39.163.195'), -('Kathleen', 'Meyer', 'kmeyerc@cdc.gov', 'Female', '164.142.188.214'), -('Steve', 'Ferguson', 'sfergusond@reverbnation.com', 'Male', '138.22.204.251'), -('Teresa', 'Hill', 'thille@dion.ne.jp', 'Female', '82.84.228.235'), -('Amanda', 'Harper', 'aharperf@mail.ru', 'Female', '16.123.56.176'), -('Kimberly', 'Ray', 'krayg@xing.com', 'Female', '48.66.48.12'), -('Johnny', 'Knight', 'jknighth@jalbum.net', 'Male', '99.30.138.123'), -('Virginia', 'Freeman', 'vfreemani@tiny.cc', 'Female', '225.172.182.63'), -('Anna', 'Austin', 'aaustinj@diigo.com', 'Female', '62.111.227.148'), -('Willie', 'Hill', 'whillk@mail.ru', 'Male', '0.86.232.249'), -('Sean', 'Harris', 'sharrisl@zdnet.com', 'Male', '117.165.133.249'), -('Mildred', 'Adams', 'madamsm@usatoday.com', 'Female', '163.44.97.46'), -('David', 'Graham', 'dgrahamn@zimbio.com', 'Male', '78.13.246.202'), -('Victor', 'Hunter', 'vhuntero@ehow.com', 'Male', '64.156.179.139'), -('Aaron', 'Ruiz', 'aruizp@weebly.com', 'Male', '34.194.68.78'), -('Benjamin', 'Brooks', 'bbrooksq@jalbum.net', 'Male', '20.192.189.107'), -('Lisa', 'Wilson', 'lwilsonr@japanpost.jp', 'Female', '199.152.130.217'), -('Benjamin', 'King', 'bkings@comsenz.com', 'Male', '29.189.189.213'), -('Christina', 'Williamson', 'cwilliamsont@boston.com', 'Female', '194.101.52.60'), -('Jane', 'Gonzalez', 'jgonzalezu@networksolutions.com', 'Female', '109.119.12.87'), -('Thomas', 'Owens', 'towensv@psu.edu', 'Male', '84.168.213.153'), -('Katherine', 'Moore', 'kmoorew@naver.com', 'Female', '183.150.65.24'), -('Jennifer', 'Stewart', 'jstewartx@yahoo.com', 'Female', '38.41.244.58'), -('Sara', 'Tucker', 'stuckery@topsy.com', 'Female', '181.130.59.184'), -('Harold', 'Ortiz', 'hortizz@vkontakte.ru', 'Male', '198.231.63.137'), -('Shirley', 'James', 'sjames10@yelp.com', 'Female', '83.27.160.104'), -('Dennis', 'Johnson', 'djohnson11@slate.com', 'Male', '183.178.246.101'), -('Louise', 'Weaver', 'lweaver12@china.com.cn', 'Female', '1.14.110.18'), -('Maria', 'Armstrong', 'marmstrong13@prweb.com', 'Female', '181.142.1.249'), -('Gloria', 'Cruz', 'gcruz14@odnoklassniki.ru', 'Female', '178.232.140.243'), -('Diana', 'Spencer', 'dspencer15@ifeng.com', 'Female', '125.153.138.244'), -('Kelly', 'Nguyen', 'knguyen16@altervista.org', 'Female', '170.13.201.119'), -('Jane', 'Rodriguez', 'jrodriguez17@biblegateway.com', 'Female', '12.102.249.81'), -('Scott', 'Brown', 'sbrown18@geocities.jp', 'Male', '108.174.99.192'), -('Norma', 'Cruz', 'ncruz19@si.edu', 'Female', '201.112.156.197'), -('Marie', 'Peters', 'mpeters1a@mlb.com', 'Female', '231.121.197.144'), -('Lillian', 'Carr', 'lcarr1b@typepad.com', 'Female', '206.179.164.163'), -('Judy', 'Nichols', 'jnichols1c@t-online.de', 'Female', '158.190.209.194'), -('Billy', 'Long', 'blong1d@yahoo.com', 'Male', '175.20.23.160'), -('Howard', 'Reid', 'hreid1e@exblog.jp', 'Male', '118.99.196.20'), -('Laura', 'Ferguson', 'lferguson1f@tuttocitta.it', 'Female', '22.77.87.110'), -('Anne', 'Bailey', 'abailey1g@geocities.com', 'Female', '58.144.159.245'), -('Rose', 'Morgan', 'rmorgan1h@ehow.com', 'Female', '118.127.97.4'), -('Nicholas', 'Reyes', 'nreyes1i@google.ru', 'Male', '50.135.10.252'), -('Joshua', 'Kennedy', 'jkennedy1j@house.gov', 'Male', '154.6.163.209'), -('Paul', 'Watkins', 'pwatkins1k@upenn.edu', 'Male', '177.236.120.87'), -('Kathryn', 'Kelly', 'kkelly1l@businessweek.com', 'Female', '70.28.61.86'), -('Adam', 'Armstrong', 'aarmstrong1m@techcrunch.com', 'Male', '133.235.24.202'), -('Norma', 'Wallace', 'nwallace1n@phoca.cz', 'Female', '241.119.227.128'), -('Timothy', 'Reyes', 'treyes1o@google.cn', 'Male', '86.28.23.26'), -('Elizabeth', 'Patterson', 'epatterson1p@sun.com', 'Female', '139.97.159.149'), -('Edward', 'Gomez', 'egomez1q@google.fr', 'Male', '158.103.108.255'), -('David', 'Cox', 'dcox1r@friendfeed.com', 'Male', '206.80.80.58'), -('Brenda', 'Wood', 'bwood1s@over-blog.com', 'Female', '217.207.44.179'), -('Adam', 'Walker', 'awalker1t@blogs.com', 'Male', '253.211.54.93'), -('Michael', 'Hart', 'mhart1u@wix.com', 'Male', '230.206.200.22'), -('Jesse', 'Ellis', 'jellis1v@google.co.uk', 'Male', '213.254.162.52'), -('Janet', 'Powell', 'jpowell1w@un.org', 'Female', '27.192.194.86'), -('Helen', 'Ford', 'hford1x@creativecommons.org', 'Female', '52.160.102.168'), -('Gerald', 'Carpenter', 'gcarpenter1y@about.me', 'Male', '36.30.194.218'), -('Kathryn', 'Oliver', 'koliver1z@army.mil', 'Female', '202.63.103.69'), -('Alan', 'Berry', 'aberry20@gov.uk', 'Male', '246.157.112.211'), -('Harry', 'Andrews', 'handrews21@ameblo.jp', 'Male', '195.108.0.12'), -('Andrea', 'Hall', 'ahall22@hp.com', 'Female', '149.162.163.28'), -('Barbara', 'Wells', 'bwells23@behance.net', 'Female', '224.70.72.1'), -('Anne', 'Wells', 'awells24@apache.org', 'Female', '180.168.81.153'), -('Harry', 'Harper', 'hharper25@rediff.com', 'Male', '151.87.130.21'), -('Jack', 'Ray', 'jray26@wufoo.com', 'Male', '220.109.38.178'), -('Phillip', 'Hamilton', 'phamilton27@joomla.org', 'Male', '166.40.47.30'), -('Shirley', 'Hunter', 'shunter28@newsvine.com', 'Female', '97.209.140.194'), -('Arthur', 'Daniels', 'adaniels29@reuters.com', 'Male', '5.40.240.86'), -('Virginia', 'Rodriguez', 'vrodriguez2a@walmart.com', 'Female', '96.80.164.184'), -('Christina', 'Ryan', 'cryan2b@hibu.com', 'Female', '56.35.5.52'), -('Theresa', 'Mendoza', 'tmendoza2c@vinaora.com', 'Female', '243.42.0.210'), -('Jason', 'Cole', 'jcole2d@ycombinator.com', 'Male', '198.248.39.129'), -('Phillip', 'Bryant', 'pbryant2e@rediff.com', 'Male', '140.39.116.251'), -('Adam', 'Torres', 'atorres2f@sun.com', 'Male', '101.75.187.135'), -('Margaret', 'Johnston', 'mjohnston2g@ucsd.edu', 'Female', '159.30.69.149'), -('Paul', 'Payne', 'ppayne2h@hhs.gov', 'Male', '199.234.140.220'), -('Todd', 'Willis', 'twillis2i@businessweek.com', 'Male', '191.59.136.214'), -('Willie', 'Oliver', 'woliver2j@noaa.gov', 'Male', '44.212.35.197'), -('Frances', 'Robertson', 'frobertson2k@go.com', 'Female', '31.117.65.136'), -('Gregory', 'Hawkins', 'ghawkins2l@joomla.org', 'Male', '91.3.22.49'), -('Lisa', 'Perkins', 'lperkins2m@si.edu', 'Female', '145.95.31.186'), -('Jacqueline', 'Anderson', 'janderson2n@cargocollective.com', 'Female', '14.176.0.187'), -('Shirley', 'Diaz', 'sdiaz2o@ucla.edu', 'Female', '207.12.95.46'), -('Nicole', 'Meyer', 'nmeyer2p@flickr.com', 'Female', '231.79.115.13'), -('Mary', 'Gray', 'mgray2q@constantcontact.com', 'Female', '210.116.64.253'), -('Jean', 'Mcdonald', 'jmcdonald2r@baidu.com', 'Female', '122.239.235.117'); diff --git a/test/integration/005_simple_seed_test/seed.sql b/test/integration/005_simple_seed_test/seed.sql index 5ad307e67ff..d4eea3ae495 100644 --- a/test/integration/005_simple_seed_test/seed.sql +++ b/test/integration/005_simple_seed_test/seed.sql @@ -1,8 +1,8 @@ create table {schema}.seed_expected ( id INTEGER, - first_name VARCHAR(11), - email VARCHAR(31), - ip_address VARCHAR(15), + first_name TEXT, + email TEXT, + ip_address TEXT, birthday TIMESTAMP WITHOUT TIME ZONE ); diff --git a/test/integration/008_schema_tests_test/macros/tests.sql b/test/integration/008_schema_tests_test/macros/tests.sql index 3e196fee9eb..e962f7896a8 100644 --- a/test/integration/008_schema_tests_test/macros/tests.sql +++ b/test/integration/008_schema_tests_test/macros/tests.sql @@ -19,7 +19,7 @@ from {{ model }} where {{ field }} in ( {% for value in values %} - {{ value }} {% if not loop.last %} , {% endif %} + '{{ value }}' {% if not loop.last %} , {% endif %} {% endfor %} ) diff --git a/test/integration/008_schema_tests_test/models-custom/schema.yml b/test/integration/008_schema_tests_test/models-custom/schema.yml index dab76f25462..15c9b95552a 100644 --- a/test/integration/008_schema_tests_test/models-custom/schema.yml +++ b/test/integration/008_schema_tests_test/models-custom/schema.yml @@ -11,12 +11,12 @@ table_copy: # fails every_value_is_blue: - - color + - favorite_color # passes rejected_values: - - { field: 'color', values: ['orange', 'purple'] } + - { field: 'favorite_color', values: ['orange', 'purple'] } # passes dbt_utils.equality: - - "{{ ref('seed') }}" + - ref('table_copy') diff --git a/test/integration/008_schema_tests_test/models/disabled.sql b/test/integration/008_schema_tests_test/models/disabled.sql deleted file mode 100644 index cc592715f43..00000000000 --- a/test/integration/008_schema_tests_test/models/disabled.sql +++ /dev/null @@ -1,6 +0,0 @@ - -{{ config(enabled=False) }} - -select 1 as not_unique -union all -select 1 diff --git a/test/integration/008_schema_tests_test/models/schema.yml b/test/integration/008_schema_tests_test/models/schema.yml index 28bd8b5ffed..2cf90e28248 100644 --- a/test/integration/008_schema_tests_test/models/schema.yml +++ b/test/integration/008_schema_tests_test/models/schema.yml @@ -53,10 +53,3 @@ table_failure_summary: relationships: - { from: favorite_color, to: ref('table_copy'), field: favorite_color } - - -# this will fail if run, but we've disabled this model -disabled: - constraints: - unique: - - not_unique diff --git a/test/integration/008_schema_tests_test/test_schema_tests.py b/test/integration/008_schema_tests_test/test_schema_tests.py index 8e986030228..8ec45103cd7 100644 --- a/test/integration/008_schema_tests_test/test_schema_tests.py +++ b/test/integration/008_schema_tests_test/test_schema_tests.py @@ -89,10 +89,14 @@ def schema(self): @property def project_config(self): + # dbt-utils containts a schema test (equality) + # dbt-integration-project contains a schema.yml file + # both should work! return { "macro-paths": ["test/integration/008_schema_tests_test/macros"], "repositories": [ - 'https://github.com/fishtown-analytics/dbt-utils' + 'https://github.com/fishtown-analytics/dbt-utils', + 'https://github.com/fishtown-analytics/dbt-integration-project' ] } @@ -113,5 +117,8 @@ def test_schema_tests(self): self.run_dbt() test_results = self.run_schema_validations() + expected_failures = ['unique', 'every_value_is_blue'] + for result in test_results: - print(result.node, result.errored, result.skipped, result.status) + if result.errored: + self.assertTrue(result.node['name'] in expected_failures) diff --git a/test/integration/011_invalid_model_tests/test_invalid_models.py b/test/integration/011_invalid_model_tests/test_invalid_models.py index f0f21b77e22..ae2ee46c0eb 100644 --- a/test/integration/011_invalid_model_tests/test_invalid_models.py +++ b/test/integration/011_invalid_model_tests/test_invalid_models.py @@ -53,4 +53,4 @@ def test_view_with_incremental_attributes(self): # should throw self.assertTrue(False) except RuntimeError as e: - self.assertTrue("which is disabled" in str(e)) + self.assertTrue("which was not found" in str(e)) diff --git a/test/integration/013_context_var_tests/test_context_vars.py b/test/integration/013_context_var_tests/test_context_vars.py index b2e8fe3df79..dc19d3583c8 100644 --- a/test/integration/013_context_var_tests/test_context_vars.py +++ b/test/integration/013_context_var_tests/test_context_vars.py @@ -10,6 +10,9 @@ def setUp(self): DBTIntegrationTest.setUp(self) os.environ["DBT_TEST_013_ENV_VAR"] = "1" + os.environ["DBT_TEST_013_USER"] = "root" + os.environ["DBT_TEST_013_PASS"] = "password" + self.fields = [ 'this', 'this.name', @@ -42,13 +45,15 @@ def profile_config(self): return { 'test': { 'outputs': { + # don't use env_var's here so the integration tests can run + # seed sql statements and the like. default target is used 'dev': { 'type': 'postgres', 'threads': 1, 'host': 'database', 'port': 5432, - 'user': 'root', - 'pass': 'password', + 'user': "root", + 'pass': "password", 'dbname': 'dbt', 'schema': self.unique_schema() }, @@ -57,8 +62,9 @@ def profile_config(self): 'threads': 1, 'host': 'database', 'port': 5432, - 'user': 'root', - 'pass': 'password', + # root/password + 'user': "{{ env_var('DBT_TEST_013_USER') }}", + 'pass': "{{ env_var('DBT_TEST_013_PASS') }}", 'dbname': 'dbt', 'schema': self.unique_schema() } diff --git a/test/integration/014_hook_tests/test_model_hooks.py b/test/integration/014_hook_tests/test_model_hooks.py index 27e3174ba54..fa5ed397a16 100644 --- a/test/integration/014_hook_tests/test_model_hooks.py +++ b/test/integration/014_hook_tests/test_model_hooks.py @@ -95,8 +95,21 @@ def project_config(self): 'macro-paths': ['test/integration/014_hook_tests/macros'], 'models': { 'test': { - 'pre-hook': MODEL_PRE_HOOK, - 'post-hook': MODEL_POST_HOOK, + 'pre-hook': [ + # inside transaction (runs second) + MODEL_PRE_HOOK, + + # outside transaction (runs first) + {"sql": "vacuum {{ this.schema }}.on_model_hook", "transaction": False}, + ], + + 'post-hook':[ + # outside transaction (runs second) + {"sql": "vacuum {{ this.schema }}.on_model_hook", "transaction": False}, + + # inside transaction (runs first) + MODEL_POST_HOOK, + ] } } } diff --git a/test/integration/014_hook_tests/test_run_hooks.py b/test/integration/014_hook_tests/test_run_hooks.py index 3bb077dc7b4..b249eb3d709 100644 --- a/test/integration/014_hook_tests/test_run_hooks.py +++ b/test/integration/014_hook_tests/test_run_hooks.py @@ -34,8 +34,19 @@ def project_config(self): return { 'macro-paths': ['test/integration/014_hook_tests/macros'], - "on-run-start": "{{ custom_run_hook('start', target, run_started_at, invocation_id) }}", - "on-run-end": "{{ custom_run_hook('end', target, run_started_at, invocation_id) }}", + # The create and drop table statements here validate that these hooks run + # in the same order that they are defined. Drop before create is an error. + # Also check that the table does not exist below. + "on-run-start": [ + "{{ custom_run_hook('start', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.start_hook_order_test ( id int )", + "drop table {{ target.schema }}.start_hook_order_test", + ], + "on-run-end": [ + "{{ custom_run_hook('end', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.end_hook_order_test ( id int )", + "drop table {{ target.schema }}.end_hook_order_test", + ] } @property @@ -76,3 +87,7 @@ def test_pre_and_post_run_hooks(self): self.check_hooks('start') self.check_hooks('end') + + + self.assertTableDoesNotExist("start_hook_order_test") + self.assertTableDoesNotExist("end_hook_order_test") diff --git a/test/integration/023_exit_codes_test/data-bad/data.csv b/test/integration/023_exit_codes_test/data-bad/data.csv index 37ff5b7356a..fcc8e001bbd 100644 --- a/test/integration/023_exit_codes_test/data-bad/data.csv +++ b/test/integration/023_exit_codes_test/data-bad/data.csv @@ -1,2 +1,2 @@ -"a,b,c -"1,\2,3,a,a,a +a,b,c +1,\2,3,a,a,a diff --git a/test/integration/023_exit_codes_test/test_exit_codes.py b/test/integration/023_exit_codes_test/test_exit_codes.py index e3640b7e80c..dfc4a9a2931 100644 --- a/test/integration/023_exit_codes_test/test_exit_codes.py +++ b/test/integration/023_exit_codes_test/test_exit_codes.py @@ -214,5 +214,8 @@ def project_config(self): @attr(type='postgres') def test_seed(self): - _, success = self.run_dbt_and_check(['seed']) - self.assertFalse(success) + try: + _, success = self.run_dbt_and_check(['seed']) + self.assertTrue(False) + except dbt.exceptions.CompilationException as e: + pass diff --git a/test/integration/025_duplicate_model_test/models-1/model-enabled-1/model.sql b/test/integration/025_duplicate_model_test/models-1/model-enabled-1/model.sql new file mode 100644 index 00000000000..1ee2efab480 --- /dev/null +++ b/test/integration/025_duplicate_model_test/models-1/model-enabled-1/model.sql @@ -0,0 +1,8 @@ +{{ + config( + enabled=True, + materialized="table", + ) +}} + +select 1 diff --git a/test/integration/025_duplicate_model_test/models-1/model-enabled-2/model.sql b/test/integration/025_duplicate_model_test/models-1/model-enabled-2/model.sql new file mode 100644 index 00000000000..1ee2efab480 --- /dev/null +++ b/test/integration/025_duplicate_model_test/models-1/model-enabled-2/model.sql @@ -0,0 +1,8 @@ +{{ + config( + enabled=True, + materialized="table", + ) +}} + +select 1 diff --git a/test/integration/025_duplicate_model_test/models-2/model-disabled/model.sql b/test/integration/025_duplicate_model_test/models-2/model-disabled/model.sql new file mode 100644 index 00000000000..410a2920cd9 --- /dev/null +++ b/test/integration/025_duplicate_model_test/models-2/model-disabled/model.sql @@ -0,0 +1,8 @@ +{{ + config( + enabled=False, + materialized="table", + ) +}} + +select 1 diff --git a/test/integration/025_duplicate_model_test/models-2/model-enabled/model.sql b/test/integration/025_duplicate_model_test/models-2/model-enabled/model.sql new file mode 100644 index 00000000000..40cda769245 --- /dev/null +++ b/test/integration/025_duplicate_model_test/models-2/model-enabled/model.sql @@ -0,0 +1,8 @@ +{{ + config( + enabled=True, + materialized="table", + ) +}} + +select 1 as value diff --git a/test/integration/025_duplicate_model_test/models-3/table.sql b/test/integration/025_duplicate_model_test/models-3/table.sql new file mode 100644 index 00000000000..aa015eca4b8 --- /dev/null +++ b/test/integration/025_duplicate_model_test/models-3/table.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized = 'table', + ) +}} + +select 1 as item diff --git a/test/integration/025_duplicate_model_test/models-4/table.sql b/test/integration/025_duplicate_model_test/models-4/table.sql new file mode 100644 index 00000000000..706fcd15716 --- /dev/null +++ b/test/integration/025_duplicate_model_test/models-4/table.sql @@ -0,0 +1,8 @@ +{{ + config( + materialized = 'table', + enabled = False, + ) +}} + +select 1 as item diff --git a/test/integration/025_duplicate_model_test/seed.sql b/test/integration/025_duplicate_model_test/seed.sql new file mode 100644 index 00000000000..5bb6f00beaa --- /dev/null +++ b/test/integration/025_duplicate_model_test/seed.sql @@ -0,0 +1,587 @@ +create table {schema}.seed ( + id INTEGER, + first_name VARCHAR(11), + email VARCHAR(31), + ip_address VARCHAR(15), + updated_at TIMESTAMP WITHOUT TIME ZONE +); + + +INSERT INTO {schema}.seed + ("id","first_name","email","ip_address","updated_at") +VALUES + (1,'Larry','lking0@miitbeian.gov.cn','69.135.206.194','2008-09-12 19:08:31'), + (2,'Larry','lperkins1@toplist.cz','64.210.133.162','1978-05-09 04:15:14'), + (3,'Anna','amontgomery2@miitbeian.gov.cn','168.104.64.114','2011-10-16 04:07:57'), + (4,'Sandra','sgeorge3@livejournal.com','229.235.252.98','1973-07-19 10:52:43'), + (5,'Fred','fwoods4@google.cn','78.229.170.124','2012-09-30 16:38:29'), + (6,'Stephen','shanson5@livejournal.com','182.227.157.105','1995-11-07 21:40:50'), + (7,'William','wmartinez6@upenn.edu','135.139.249.50','1982-09-05 03:11:59'), + (8,'Jessica','jlong7@hao123.com','203.62.178.210','1991-10-16 11:03:15'), + (9,'Douglas','dwhite8@tamu.edu','178.187.247.1','1979-10-01 09:49:48'), + (10,'Lisa','lcoleman9@nydailynews.com','168.234.128.249','2011-05-26 07:45:49'), + (11,'Ralph','rfieldsa@home.pl','55.152.163.149','1972-11-18 19:06:11'), + (12,'Louise','lnicholsb@samsung.com','141.116.153.154','2014-11-25 20:56:14'), + (13,'Clarence','cduncanc@sfgate.com','81.171.31.133','2011-11-17 07:02:36'), + (14,'Daniel','dfranklind@omniture.com','8.204.211.37','1980-09-13 00:09:04'), + (15,'Katherine','klanee@auda.org.au','176.96.134.59','1997-08-22 19:36:56'), + (16,'Billy','bwardf@wikia.com','214.108.78.85','2003-10-19 02:14:47'), + (17,'Annie','agarzag@ocn.ne.jp','190.108.42.70','1988-10-28 15:12:35'), + (18,'Shirley','scolemanh@fastcompany.com','109.251.164.84','1988-08-24 10:50:57'), + (19,'Roger','rfrazieri@scribd.com','38.145.218.108','1985-12-31 15:17:15'), + (20,'Lillian','lstanleyj@goodreads.com','47.57.236.17','1970-06-08 02:09:05'), + (21,'Aaron','arodriguezk@nps.gov','205.245.118.221','1985-10-11 23:07:49'), + (22,'Patrick','pparkerl@techcrunch.com','19.8.100.182','2006-03-29 12:53:56'), + (23,'Phillip','pmorenom@intel.com','41.38.254.103','2011-11-07 15:35:43'), + (24,'Henry','hgarcian@newsvine.com','1.191.216.252','2008-08-28 08:30:44'), + (25,'Irene','iturnero@opera.com','50.17.60.190','1994-04-01 07:15:02'), + (26,'Andrew','adunnp@pen.io','123.52.253.176','2000-11-01 06:03:25'), + (27,'David','dgutierrezq@wp.com','238.23.203.42','1988-01-25 07:29:18'), + (28,'Henry','hsanchezr@cyberchimps.com','248.102.2.185','1983-01-01 13:36:37'), + (29,'Evelyn','epetersons@gizmodo.com','32.80.46.119','1979-07-16 17:24:12'), + (30,'Tammy','tmitchellt@purevolume.com','249.246.167.88','2001-04-03 10:00:23'), + (31,'Jacqueline','jlittleu@domainmarket.com','127.181.97.47','1986-02-11 21:35:50'), + (32,'Earl','eortizv@opera.com','166.47.248.240','1996-07-06 08:16:27'), + (33,'Juan','jgordonw@sciencedirect.com','71.77.2.200','1987-01-31 03:46:44'), + (34,'Diane','dhowellx@nyu.edu','140.94.133.12','1994-06-11 02:30:05'), + (35,'Randy','rkennedyy@microsoft.com','73.255.34.196','2005-05-26 20:28:39'), + (36,'Janice','jriveraz@time.com','22.214.227.32','1990-02-09 04:16:52'), + (37,'Laura','lperry10@diigo.com','159.148.145.73','2015-03-17 05:59:25'), + (38,'Gary','gray11@statcounter.com','40.193.124.56','1970-01-27 10:04:51'), + (39,'Jesse','jmcdonald12@typepad.com','31.7.86.103','2009-03-14 08:14:29'), + (40,'Sandra','sgonzalez13@goodreads.com','223.80.168.239','1993-05-21 14:08:54'), + (41,'Scott','smoore14@archive.org','38.238.46.83','1980-08-30 11:16:56'), + (42,'Phillip','pevans15@cisco.com','158.234.59.34','2011-12-15 23:26:31'), + (43,'Steven','sriley16@google.ca','90.247.57.68','2011-10-29 19:03:28'), + (44,'Deborah','dbrown17@hexun.com','179.125.143.240','1995-04-10 14:36:07'), + (45,'Lori','lross18@ow.ly','64.80.162.180','1980-12-27 16:49:15'), + (46,'Sean','sjackson19@tumblr.com','240.116.183.69','1988-06-12 21:24:45'), + (47,'Terry','tbarnes1a@163.com','118.38.213.137','1997-09-22 16:43:19'), + (48,'Dorothy','dross1b@ebay.com','116.81.76.49','2005-02-28 13:33:24'), + (49,'Samuel','swashington1c@house.gov','38.191.253.40','1989-01-19 21:15:48'), + (50,'Ralph','rcarter1d@tinyurl.com','104.84.60.174','2007-08-11 10:21:49'), + (51,'Wayne','whudson1e@princeton.edu','90.61.24.102','1983-07-03 16:58:12'), + (52,'Rose','rjames1f@plala.or.jp','240.83.81.10','1995-06-08 11:46:23'), + (53,'Louise','lcox1g@theglobeandmail.com','105.11.82.145','2016-09-19 14:45:51'), + (54,'Kenneth','kjohnson1h@independent.co.uk','139.5.45.94','1976-08-17 11:26:19'), + (55,'Donna','dbrown1i@amazon.co.uk','19.45.169.45','2006-05-27 16:51:40'), + (56,'Johnny','jvasquez1j@trellian.com','118.202.238.23','1975-11-17 08:42:32'), + (57,'Patrick','pramirez1k@tamu.edu','231.25.153.198','1997-08-06 11:51:09'), + (58,'Helen','hlarson1l@prweb.com','8.40.21.39','1993-08-04 19:53:40'), + (59,'Patricia','pspencer1m@gmpg.org','212.198.40.15','1977-08-03 16:37:27'), + (60,'Joseph','jspencer1n@marriott.com','13.15.63.238','2005-07-23 20:22:06'), + (61,'Phillip','pschmidt1o@blogtalkradio.com','177.98.201.190','1976-05-19 21:47:44'), + (62,'Joan','jwebb1p@google.ru','105.229.170.71','1972-09-07 17:53:47'), + (63,'Phyllis','pkennedy1q@imgur.com','35.145.8.244','2000-01-01 22:33:37'), + (64,'Katherine','khunter1r@smh.com.au','248.168.205.32','1991-01-09 06:40:24'), + (65,'Laura','lvasquez1s@wiley.com','128.129.115.152','1997-10-23 12:04:56'), + (66,'Juan','jdunn1t@state.gov','44.228.124.51','2004-11-10 05:07:35'), + (67,'Judith','jholmes1u@wiley.com','40.227.179.115','1977-08-02 17:01:45'), + (68,'Beverly','bbaker1v@wufoo.com','208.34.84.59','2016-03-06 20:07:23'), + (69,'Lawrence','lcarr1w@flickr.com','59.158.212.223','1988-09-13 06:07:21'), + (70,'Gloria','gwilliams1x@mtv.com','245.231.88.33','1995-03-18 22:32:46'), + (71,'Steven','ssims1y@cbslocal.com','104.50.58.255','2001-08-05 21:26:20'), + (72,'Betty','bmills1z@arstechnica.com','103.177.214.220','1981-12-14 21:26:54'), + (73,'Mildred','mfuller20@prnewswire.com','151.158.8.130','2000-04-19 10:13:55'), + (74,'Donald','dday21@icq.com','9.178.102.255','1972-12-03 00:58:24'), + (75,'Eric','ethomas22@addtoany.com','85.2.241.227','1992-11-01 05:59:30'), + (76,'Joyce','jarmstrong23@sitemeter.com','169.224.20.36','1985-10-24 06:50:01'), + (77,'Maria','mmartinez24@amazonaws.com','143.189.167.135','2005-10-05 05:17:42'), + (78,'Harry','hburton25@youtube.com','156.47.176.237','1978-03-26 05:53:33'), + (79,'Kevin','klawrence26@hao123.com','79.136.183.83','1994-10-12 04:38:52'), + (80,'David','dhall27@prweb.com','133.149.172.153','1976-12-15 16:24:24'), + (81,'Kathy','kperry28@twitter.com','229.242.72.228','1979-03-04 02:58:56'), + (82,'Adam','aprice29@elegantthemes.com','13.145.21.10','1982-11-07 11:46:59'), + (83,'Brandon','bgriffin2a@va.gov','73.249.128.212','2013-10-30 05:30:36'), + (84,'Henry','hnguyen2b@discovery.com','211.36.214.242','1985-01-09 06:37:27'), + (85,'Eric','esanchez2c@edublogs.org','191.166.188.251','2004-05-01 23:21:42'), + (86,'Jason','jlee2d@jimdo.com','193.92.16.182','1973-01-08 09:05:39'), + (87,'Diana','drichards2e@istockphoto.com','19.130.175.245','1994-10-05 22:50:49'), + (88,'Andrea','awelch2f@abc.net.au','94.155.233.96','2002-04-26 08:41:44'), + (89,'Louis','lwagner2g@miitbeian.gov.cn','26.217.34.111','2003-08-25 07:56:39'), + (90,'Jane','jsims2h@seesaa.net','43.4.220.135','1987-03-20 20:39:04'), + (91,'Larry','lgrant2i@si.edu','97.126.79.34','2000-09-07 20:26:19'), + (92,'Louis','ldean2j@prnewswire.com','37.148.40.127','2011-09-16 20:12:14'), + (93,'Jennifer','jcampbell2k@xing.com','38.106.254.142','1988-07-15 05:06:49'), + (94,'Wayne','wcunningham2l@google.com.hk','223.28.26.187','2009-12-15 06:16:54'), + (95,'Lori','lstevens2m@icq.com','181.250.181.58','1984-10-28 03:29:19'), + (96,'Judy','jsimpson2n@marriott.com','180.121.239.219','1986-02-07 15:18:10'), + (97,'Phillip','phoward2o@usa.gov','255.247.0.175','2002-12-26 08:44:45'), + (98,'Gloria','gwalker2p@usa.gov','156.140.7.128','1997-10-04 07:58:58'), + (99,'Paul','pjohnson2q@umn.edu','183.59.198.197','1991-11-14 12:33:55'), + (100,'Frank','fgreene2r@blogspot.com','150.143.68.121','2010-06-12 23:55:39'), + (101,'Deborah','dknight2s@reverbnation.com','222.131.211.191','1970-07-08 08:54:23'), + (102,'Sandra','sblack2t@tripadvisor.com','254.183.128.254','2000-04-12 02:39:36'), + (103,'Edward','eburns2u@dailymotion.com','253.89.118.18','1993-10-10 10:54:01'), + (104,'Anthony','ayoung2v@ustream.tv','118.4.193.176','1978-08-26 17:07:29'), + (105,'Donald','dlawrence2w@wp.com','139.200.159.227','2007-07-21 20:56:20'), + (106,'Matthew','mfreeman2x@google.fr','205.26.239.92','2014-12-05 17:05:39'), + (107,'Sean','ssanders2y@trellian.com','143.89.82.108','1993-07-14 21:45:02'), + (108,'Sharon','srobinson2z@soundcloud.com','66.234.247.54','1977-04-06 19:07:03'), + (109,'Jennifer','jwatson30@t-online.de','196.102.127.7','1998-03-07 05:12:23'), + (110,'Clarence','cbrooks31@si.edu','218.93.234.73','2002-11-06 17:22:25'), + (111,'Jose','jflores32@goo.gl','185.105.244.231','1995-01-05 06:32:21'), + (112,'George','glee33@adobe.com','173.82.249.196','2015-01-04 02:47:46'), + (113,'Larry','lhill34@linkedin.com','66.5.206.195','2010-11-02 10:21:17'), + (114,'Marie','mmeyer35@mysql.com','151.152.88.107','1990-05-22 20:52:51'), + (115,'Clarence','cwebb36@skype.com','130.198.55.217','1972-10-27 07:38:54'), + (116,'Sarah','scarter37@answers.com','80.89.18.153','1971-08-24 19:29:30'), + (117,'Henry','hhughes38@webeden.co.uk','152.60.114.174','1973-01-27 09:00:42'), + (118,'Teresa','thenry39@hao123.com','32.187.239.106','2015-11-06 01:48:44'), + (119,'Billy','bgutierrez3a@sun.com','52.37.70.134','2002-03-19 03:20:19'), + (120,'Anthony','agibson3b@github.io','154.251.232.213','1991-04-19 01:08:15'), + (121,'Sandra','sromero3c@wikia.com','44.124.171.2','1998-09-06 20:30:34'), + (122,'Paula','pandrews3d@blogs.com','153.142.118.226','2003-06-24 16:31:24'), + (123,'Terry','tbaker3e@csmonitor.com','99.120.45.219','1970-12-09 23:57:21'), + (124,'Lois','lwilson3f@reuters.com','147.44.171.83','1971-01-09 22:28:51'), + (125,'Sara','smorgan3g@nature.com','197.67.192.230','1992-01-28 20:33:24'), + (126,'Charles','ctorres3h@china.com.cn','156.115.216.2','1993-10-02 19:36:34'), + (127,'Richard','ralexander3i@marriott.com','248.235.180.59','1999-02-03 18:40:55'), + (128,'Christina','charper3j@cocolog-nifty.com','152.114.116.129','1978-09-13 00:37:32'), + (129,'Steve','sadams3k@economist.com','112.248.91.98','2004-03-21 09:07:43'), + (130,'Katherine','krobertson3l@ow.ly','37.220.107.28','1977-03-18 19:28:50'), + (131,'Donna','dgibson3m@state.gov','222.218.76.221','1999-02-01 06:46:16'), + (132,'Christina','cwest3n@mlb.com','152.114.6.160','1979-12-24 15:30:35'), + (133,'Sandra','swillis3o@meetup.com','180.71.49.34','1984-09-27 08:05:54'), + (134,'Clarence','cedwards3p@smugmug.com','10.64.180.186','1979-04-16 16:52:10'), + (135,'Ruby','rjames3q@wp.com','98.61.54.20','2007-01-13 14:25:52'), + (136,'Sarah','smontgomery3r@tripod.com','91.45.164.172','2009-07-25 04:34:30'), + (137,'Sarah','soliver3s@eventbrite.com','30.106.39.146','2012-05-09 22:12:33'), + (138,'Deborah','dwheeler3t@biblegateway.com','59.105.213.173','1999-11-09 08:08:44'), + (139,'Deborah','dray3u@i2i.jp','11.108.186.217','2014-02-04 03:15:19'), + (140,'Paul','parmstrong3v@alexa.com','6.250.59.43','2009-12-21 10:08:53'), + (141,'Aaron','abishop3w@opera.com','207.145.249.62','1996-04-25 23:20:23'), + (142,'Henry','hsanders3x@google.ru','140.215.203.171','2012-01-29 11:52:32'), + (143,'Anne','aanderson3y@1688.com','74.150.102.118','1982-04-03 13:46:17'), + (144,'Victor','vmurphy3z@hugedomains.com','222.155.99.152','1987-11-03 19:58:41'), + (145,'Evelyn','ereid40@pbs.org','249.122.33.117','1977-12-14 17:09:57'), + (146,'Brian','bgonzalez41@wikia.com','246.254.235.141','1991-02-24 00:45:58'), + (147,'Sandra','sgray42@squarespace.com','150.73.28.159','1972-07-28 17:26:32'), + (148,'Alice','ajones43@a8.net','78.253.12.177','2002-12-05 16:57:46'), + (149,'Jessica','jhanson44@mapquest.com','87.229.30.160','1994-01-30 11:40:04'), + (150,'Louise','lbailey45@reuters.com','191.219.31.101','2011-09-07 21:11:45'), + (151,'Christopher','cgonzalez46@printfriendly.com','83.137.213.239','1984-10-24 14:58:04'), + (152,'Gregory','gcollins47@yandex.ru','28.176.10.115','1998-07-25 17:17:10'), + (153,'Jane','jperkins48@usnews.com','46.53.164.159','1979-08-19 15:25:00'), + (154,'Phyllis','plong49@yahoo.co.jp','208.140.88.2','1985-07-06 02:16:36'), + (155,'Adam','acarter4a@scribd.com','78.48.148.204','2005-07-20 03:31:09'), + (156,'Frank','fweaver4b@angelfire.com','199.180.255.224','2011-03-04 23:07:54'), + (157,'Ronald','rmurphy4c@cloudflare.com','73.42.97.231','1991-01-11 10:39:41'), + (158,'Richard','rmorris4d@e-recht24.de','91.9.97.223','2009-01-17 21:05:15'), + (159,'Rose','rfoster4e@woothemes.com','203.169.53.16','1991-04-21 02:09:38'), + (160,'George','ggarrett4f@uiuc.edu','186.61.5.167','1989-11-11 11:29:42'), + (161,'Victor','vhamilton4g@biblegateway.com','121.229.138.38','2012-06-22 18:01:23'), + (162,'Mark','mbennett4h@businessinsider.com','209.184.29.203','1980-04-16 15:26:34'), + (163,'Martin','mwells4i@ifeng.com','97.223.55.105','2010-05-26 14:08:18'), + (164,'Diana','dstone4j@google.ru','90.155.52.47','2013-02-11 00:14:54'), + (165,'Walter','wferguson4k@blogger.com','30.63.212.44','1986-02-20 17:46:46'), + (166,'Denise','dcoleman4l@vistaprint.com','10.209.153.77','1992-05-13 20:14:14'), + (167,'Philip','pknight4m@xing.com','15.28.135.167','2000-09-11 18:41:13'), + (168,'Russell','rcarr4n@youtube.com','113.55.165.50','2008-07-10 17:49:27'), + (169,'Donna','dburke4o@dion.ne.jp','70.0.105.111','1992-02-10 17:24:58'), + (170,'Anne','along4p@squidoo.com','36.154.58.107','2012-08-19 23:35:31'), + (171,'Clarence','cbanks4q@webeden.co.uk','94.57.53.114','1972-03-11 21:46:44'), + (172,'Betty','bbowman4r@cyberchimps.com','178.115.209.69','2013-01-13 21:34:51'), + (173,'Andrew','ahudson4s@nytimes.com','84.32.252.144','1998-09-15 14:20:04'), + (174,'Keith','kgordon4t@cam.ac.uk','189.237.211.102','2009-01-22 05:34:38'), + (175,'Patrick','pwheeler4u@mysql.com','47.22.117.226','1984-09-05 22:33:15'), + (176,'Jesse','jfoster4v@mapquest.com','229.95.131.46','1990-01-20 12:19:15'), + (177,'Arthur','afisher4w@jugem.jp','107.255.244.98','1983-10-13 11:08:46'), + (178,'Nicole','nryan4x@wsj.com','243.211.33.221','1974-05-30 23:19:14'), + (179,'Bruce','bjohnson4y@sfgate.com','17.41.200.101','1992-09-23 02:02:19'), + (180,'Terry','tcox4z@reference.com','20.189.120.106','1982-02-13 12:43:14'), + (181,'Ashley','astanley50@kickstarter.com','86.3.56.98','1976-05-09 01:27:16'), + (182,'Michael','mrivera51@about.me','72.118.249.0','1971-11-11 17:28:37'), + (183,'Steven','sgonzalez52@mozilla.org','169.112.247.47','2002-08-24 14:59:25'), + (184,'Kathleen','kfuller53@bloglovin.com','80.93.59.30','2002-03-11 13:41:29'), + (185,'Nicole','nhenderson54@usda.gov','39.253.60.30','1995-04-24 05:55:07'), + (186,'Ralph','rharper55@purevolume.com','167.147.142.189','1980-02-10 18:35:45'), + (187,'Heather','hcunningham56@photobucket.com','96.222.196.229','2007-06-15 05:37:50'), + (188,'Nancy','nlittle57@cbc.ca','241.53.255.175','2007-07-12 23:42:48'), + (189,'Juan','jramirez58@pinterest.com','190.128.84.27','1978-11-07 23:37:37'), + (190,'Beverly','bfowler59@chronoengine.com','54.144.230.49','1979-03-31 23:27:28'), + (191,'Shirley','sstevens5a@prlog.org','200.97.231.248','2011-12-06 07:08:50'), + (192,'Annie','areyes5b@squidoo.com','223.32.182.101','2011-05-28 02:42:09'), + (193,'Jack','jkelley5c@tiny.cc','47.34.118.150','1981-12-05 17:31:40'), + (194,'Keith','krobinson5d@1und1.de','170.210.209.31','1999-03-09 11:05:43'), + (195,'Joseph','jmiller5e@google.com.au','136.74.212.139','1984-10-08 13:18:20'), + (196,'Annie','aday5f@blogspot.com','71.99.186.69','1986-02-18 12:27:34'), + (197,'Nancy','nperez5g@liveinternet.ru','28.160.6.107','1983-10-20 17:51:20'), + (198,'Tammy','tward5h@ucoz.ru','141.43.164.70','1980-03-31 04:45:29'), + (199,'Doris','dryan5i@ted.com','239.117.202.188','1985-07-03 03:17:53'), + (200,'Rose','rmendoza5j@photobucket.com','150.200.206.79','1973-04-21 21:36:40'), + (201,'Cynthia','cbutler5k@hubpages.com','80.153.174.161','2001-01-20 01:42:26'), + (202,'Samuel','soliver5l@people.com.cn','86.127.246.140','1970-09-02 02:19:00'), + (203,'Carl','csanchez5m@mysql.com','50.149.237.107','1993-12-01 07:02:09'), + (204,'Kathryn','kowens5n@geocities.jp','145.166.205.201','2004-07-06 18:39:33'), + (205,'Nicholas','nnichols5o@parallels.com','190.240.66.170','2014-11-11 18:52:19'), + (206,'Keith','kwillis5p@youtube.com','181.43.206.100','1998-06-13 06:30:51'), + (207,'Justin','jwebb5q@intel.com','211.54.245.74','2000-11-04 16:58:26'), + (208,'Gary','ghicks5r@wikipedia.org','196.154.213.104','1992-12-01 19:48:28'), + (209,'Martin','mpowell5s@flickr.com','153.67.12.241','1983-06-30 06:24:32'), + (210,'Brenda','bkelley5t@xinhuanet.com','113.100.5.172','2005-01-08 20:50:22'), + (211,'Edward','eray5u@a8.net','205.187.246.65','2011-09-26 08:04:44'), + (212,'Steven','slawson5v@senate.gov','238.150.250.36','1978-11-22 02:48:09'), + (213,'Robert','rthompson5w@furl.net','70.7.89.236','2001-09-12 08:52:07'), + (214,'Jack','jporter5x@diigo.com','220.172.29.99','1976-07-26 14:29:21'), + (215,'Lisa','ljenkins5y@oakley.com','150.151.170.180','2010-03-20 19:21:16'), + (216,'Theresa','tbell5z@mayoclinic.com','247.25.53.173','2001-03-11 05:36:40'), + (217,'Jimmy','jstephens60@weather.com','145.101.93.235','1983-04-12 09:35:30'), + (218,'Louis','lhunt61@amazon.co.jp','78.137.6.253','1997-08-29 19:34:34'), + (219,'Lawrence','lgilbert62@ted.com','243.132.8.78','2015-04-08 22:06:56'), + (220,'David','dgardner63@4shared.com','204.40.46.136','1971-07-09 03:29:11'), + (221,'Charles','ckennedy64@gmpg.org','211.83.233.2','2011-02-26 11:55:04'), + (222,'Lillian','lbanks65@msu.edu','124.233.12.80','2010-05-16 20:29:02'), + (223,'Ernest','enguyen66@baidu.com','82.45.128.148','1996-07-04 10:07:04'), + (224,'Ryan','rrussell67@cloudflare.com','202.53.240.223','1983-08-05 12:36:29'), + (225,'Donald','ddavis68@ustream.tv','47.39.218.137','1989-05-27 02:30:56'), + (226,'Joe','jscott69@blogspot.com','140.23.131.75','1973-03-16 12:21:31'), + (227,'Anne','amarshall6a@google.ca','113.162.200.197','1988-12-09 03:38:29'), + (228,'Willie','wturner6b@constantcontact.com','85.83.182.249','1991-10-06 01:51:10'), + (229,'Nicole','nwilson6c@sogou.com','30.223.51.135','1977-05-29 19:54:56'), + (230,'Janet','jwheeler6d@stumbleupon.com','153.194.27.144','2011-03-13 12:48:47'), + (231,'Lois','lcarr6e@statcounter.com','0.41.36.53','1993-02-06 04:52:01'), + (232,'Shirley','scruz6f@tmall.com','37.156.39.223','2007-02-18 17:47:01'), + (233,'Patrick','pford6g@reverbnation.com','36.198.200.89','1977-03-06 15:47:24'), + (234,'Lisa','lhudson6h@usatoday.com','134.213.58.137','2014-10-28 01:56:56'), + (235,'Pamela','pmartinez6i@opensource.org','5.151.127.202','1987-11-30 16:44:47'), + (236,'Larry','lperez6j@infoseek.co.jp','235.122.96.148','1979-01-18 06:33:45'), + (237,'Pamela','pramirez6k@census.gov','138.233.34.163','2012-01-29 10:35:20'), + (238,'Daniel','dcarr6l@php.net','146.21.152.242','1984-11-17 08:22:59'), + (239,'Patrick','psmith6m@indiegogo.com','136.222.199.36','2001-05-30 22:16:44'), + (240,'Raymond','rhenderson6n@hc360.com','116.31.112.38','2000-01-05 20:35:41'), + (241,'Teresa','treynolds6o@miitbeian.gov.cn','198.126.205.220','1996-11-08 01:27:31'), + (242,'Johnny','jmason6p@flickr.com','192.8.232.114','2013-05-14 05:35:50'), + (243,'Angela','akelly6q@guardian.co.uk','234.116.60.197','1977-08-20 02:05:17'), + (244,'Douglas','dcole6r@cmu.edu','128.135.212.69','2016-10-26 17:40:36'), + (245,'Frances','fcampbell6s@twitpic.com','94.22.243.235','1987-04-26 07:07:13'), + (246,'Donna','dgreen6t@chron.com','227.116.46.107','2011-07-25 12:59:54'), + (247,'Benjamin','bfranklin6u@redcross.org','89.141.142.89','1974-05-03 20:28:18'), + (248,'Randy','rpalmer6v@rambler.ru','70.173.63.178','2011-12-20 17:40:18'), + (249,'Melissa','mmurray6w@bbb.org','114.234.118.137','1991-02-26 12:45:44'), + (250,'Jean','jlittle6x@epa.gov','141.21.163.254','1991-08-16 04:57:09'), + (251,'Daniel','dolson6y@nature.com','125.75.104.97','2010-04-23 06:25:54'), + (252,'Kathryn','kwells6z@eventbrite.com','225.104.28.249','2015-01-31 02:21:50'), + (253,'Theresa','tgonzalez70@ox.ac.uk','91.93.156.26','1971-12-11 10:31:31'), + (254,'Beverly','broberts71@bluehost.com','244.40.158.89','2013-09-21 13:02:31'), + (255,'Pamela','pmurray72@netscape.com','218.54.95.216','1985-04-16 00:34:00'), + (256,'Timothy','trichardson73@amazonaws.com','235.49.24.229','2000-11-11 09:48:28'), + (257,'Mildred','mpalmer74@is.gd','234.125.95.132','1992-05-25 02:25:02'), + (258,'Jessica','jcampbell75@google.it','55.98.30.140','2014-08-26 00:26:34'), + (259,'Beverly','bthomas76@cpanel.net','48.78.228.176','1970-08-18 10:40:05'), + (260,'Eugene','eward77@cargocollective.com','139.226.204.2','1996-12-04 23:17:00'), + (261,'Andrea','aallen78@webnode.com','160.31.214.38','2009-07-06 07:22:37'), + (262,'Justin','jruiz79@merriam-webster.com','150.149.246.122','2005-06-06 11:44:19'), + (263,'Kenneth','kedwards7a@networksolutions.com','98.82.193.128','2001-07-03 02:00:10'), + (264,'Rachel','rday7b@miibeian.gov.cn','114.15.247.221','1994-08-18 19:45:40'), + (265,'Russell','rmiller7c@instagram.com','184.130.152.253','1977-11-06 01:58:12'), + (266,'Bonnie','bhudson7d@cornell.edu','235.180.186.206','1990-12-03 22:45:24'), + (267,'Raymond','rknight7e@yandex.ru','161.2.44.252','1995-08-25 04:31:19'), + (268,'Bonnie','brussell7f@elpais.com','199.237.57.207','1991-03-29 08:32:06'), + (269,'Marie','mhenderson7g@elpais.com','52.203.131.144','2004-06-04 21:50:28'), + (270,'Alan','acarr7h@trellian.com','147.51.205.72','2005-03-03 10:51:31'), + (271,'Barbara','bturner7i@hugedomains.com','103.160.110.226','2004-08-04 13:42:40'), + (272,'Christina','cdaniels7j@census.gov','0.238.61.251','1972-10-18 12:47:33'), + (273,'Jeremy','jgomez7k@reuters.com','111.26.65.56','2013-01-13 10:41:35'), + (274,'Laura','lwood7l@icio.us','149.153.38.205','2011-06-25 09:33:59'), + (275,'Matthew','mbowman7m@auda.org.au','182.138.206.172','1999-03-05 03:25:36'), + (276,'Denise','dparker7n@icq.com','0.213.88.138','2011-11-04 09:43:06'), + (277,'Phillip','pparker7o@discuz.net','219.242.165.240','1973-10-19 04:22:29'), + (278,'Joan','jpierce7p@salon.com','63.31.213.202','1989-04-09 22:06:24'), + (279,'Irene','ibaker7q@cbc.ca','102.33.235.114','1992-09-04 13:00:57'), + (280,'Betty','bbowman7r@ted.com','170.91.249.242','2015-09-28 08:14:22'), + (281,'Teresa','truiz7s@boston.com','82.108.158.207','1999-07-18 05:17:09'), + (282,'Helen','hbrooks7t@slideshare.net','102.87.162.187','2003-01-06 15:45:29'), + (283,'Karen','kgriffin7u@wunderground.com','43.82.44.184','2010-05-28 01:56:37'), + (284,'Lisa','lfernandez7v@mtv.com','200.238.218.220','1993-04-03 20:33:51'), + (285,'Jesse','jlawrence7w@timesonline.co.uk','95.122.105.78','1990-01-05 17:28:43'), + (286,'Terry','tross7x@macromedia.com','29.112.114.133','2009-08-29 21:32:17'), + (287,'Angela','abradley7y@icq.com','177.44.27.72','1989-10-04 21:46:06'), + (288,'Maria','mhart7z@dailymotion.com','55.27.55.202','1975-01-21 01:22:57'), + (289,'Raymond','randrews80@pinterest.com','88.90.78.67','1992-03-16 21:37:40'), + (290,'Kathy','krice81@bluehost.com','212.63.196.102','2000-12-14 03:06:44'), + (291,'Cynthia','cramos82@nymag.com','107.89.190.6','2005-06-28 02:02:33'), + (292,'Kimberly','kjones83@mysql.com','86.169.101.101','2007-06-13 22:56:49'), + (293,'Timothy','thansen84@microsoft.com','108.100.254.90','2003-04-04 10:31:57'), + (294,'Carol','cspencer85@berkeley.edu','75.118.144.187','1999-03-30 14:53:21'), + (295,'Louis','lmedina86@latimes.com','141.147.163.24','1991-04-11 17:53:13'), + (296,'Margaret','mcole87@google.fr','53.184.26.83','1991-12-19 01:54:10'), + (297,'Mary','mgomez88@yellowpages.com','208.56.57.99','1976-05-21 18:05:08'), + (298,'Amanda','aanderson89@geocities.com','147.73.15.252','1987-08-22 15:05:28'), + (299,'Kathryn','kgarrett8a@nature.com','27.29.177.220','1976-07-15 04:25:04'), + (300,'Dorothy','dmason8b@shareasale.com','106.210.99.193','1990-09-03 21:39:31'), + (301,'Lois','lkennedy8c@amazon.de','194.169.29.187','2007-07-29 14:09:31'), + (302,'Irene','iburton8d@washingtonpost.com','196.143.110.249','2013-09-05 11:32:46'), + (303,'Betty','belliott8e@wired.com','183.105.222.199','1979-09-19 19:29:13'), + (304,'Bobby','bmeyer8f@census.gov','36.13.161.145','2014-05-24 14:34:39'), + (305,'Ann','amorrison8g@sfgate.com','72.154.54.137','1978-10-05 14:22:34'), + (306,'Daniel','djackson8h@wunderground.com','144.95.32.34','1990-07-27 13:23:05'), + (307,'Joe','jboyd8i@alibaba.com','187.105.86.178','2011-09-28 16:46:32'), + (308,'Ralph','rdunn8j@fc2.com','3.19.87.255','1984-10-18 08:00:40'), + (309,'Craig','ccarter8k@gizmodo.com','235.152.76.215','1998-07-04 12:15:21'), + (310,'Paula','pdean8l@hhs.gov','161.100.173.197','1973-02-13 09:38:55'), + (311,'Andrew','agarrett8m@behance.net','199.253.123.218','1991-02-14 13:36:32'), + (312,'Janet','jhowell8n@alexa.com','39.189.139.79','2012-11-24 20:17:33'), + (313,'Keith','khansen8o@godaddy.com','116.186.223.196','1987-08-23 21:22:05'), + (314,'Nicholas','nedwards8p@state.gov','142.175.142.11','1977-03-28 18:27:27'), + (315,'Jacqueline','jallen8q@oaic.gov.au','189.66.135.192','1994-10-26 11:44:26'), + (316,'Frank','fgardner8r@mapy.cz','154.77.119.169','1983-01-29 19:19:51'), + (317,'Eric','eharrison8s@google.cn','245.139.65.123','1984-02-04 09:54:36'), + (318,'Gregory','gcooper8t@go.com','171.147.0.221','2004-06-14 05:22:08'), + (319,'Jean','jfreeman8u@rakuten.co.jp','67.243.121.5','1977-01-07 18:23:43'), + (320,'Juan','jlewis8v@shinystat.com','216.181.171.189','2001-08-23 17:32:43'), + (321,'Randy','rwilliams8w@shinystat.com','105.152.146.28','1983-02-17 00:05:50'), + (322,'Stephen','shart8x@sciencedirect.com','196.131.205.148','2004-02-15 10:12:03'), + (323,'Annie','ahunter8y@example.com','63.36.34.103','2003-07-23 21:15:25'), + (324,'Melissa','mflores8z@cbc.ca','151.230.217.90','1983-11-02 14:53:56'), + (325,'Jane','jweaver90@about.me','0.167.235.217','1987-07-29 00:13:44'), + (326,'Anthony','asmith91@oracle.com','97.87.48.41','2001-05-31 18:44:11'), + (327,'Terry','tdavis92@buzzfeed.com','46.20.12.51','2015-09-12 23:13:55'), + (328,'Brandon','bmontgomery93@gravatar.com','252.101.48.186','2010-10-28 08:26:27'), + (329,'Chris','cmurray94@bluehost.com','25.158.167.97','2004-05-05 16:10:31'), + (330,'Denise','dfuller95@hugedomains.com','216.210.149.28','1979-04-20 08:57:24'), + (331,'Arthur','amcdonald96@sakura.ne.jp','206.42.36.213','2009-08-15 03:26:16'), + (332,'Jesse','jhoward97@google.cn','46.181.118.30','1974-04-18 14:08:41'), + (333,'Frank','fsimpson98@domainmarket.com','163.220.211.87','2006-06-30 14:46:52'), + (334,'Janice','jwoods99@pen.io','229.245.237.182','1988-04-06 11:52:58'), + (335,'Rebecca','rroberts9a@huffingtonpost.com','148.96.15.80','1976-10-05 08:44:16'), + (336,'Joshua','jray9b@opensource.org','192.253.12.198','1971-12-25 22:27:07'), + (337,'Joyce','jcarpenter9c@statcounter.com','125.171.46.215','2001-12-31 22:08:13'), + (338,'Andrea','awest9d@privacy.gov.au','79.101.180.201','1983-02-18 20:07:47'), + (339,'Christine','chudson9e@yelp.com','64.198.43.56','1997-09-08 08:03:43'), + (340,'Joe','jparker9f@earthlink.net','251.215.148.153','1973-11-04 05:08:18'), + (341,'Thomas','tkim9g@answers.com','49.187.34.47','1991-08-07 21:13:48'), + (342,'Janice','jdean9h@scientificamerican.com','4.197.117.16','2009-12-08 02:35:49'), + (343,'James','jmitchell9i@umich.edu','43.121.18.147','2011-04-28 17:04:09'), + (344,'Charles','cgardner9j@purevolume.com','197.78.240.240','1998-02-11 06:47:07'), + (345,'Robert','rhenderson9k@friendfeed.com','215.84.180.88','2002-05-10 15:33:14'), + (346,'Chris','cgray9l@4shared.com','249.70.192.240','1998-10-03 16:43:42'), + (347,'Gloria','ghayes9m@hibu.com','81.103.138.26','1999-12-26 11:23:13'), + (348,'Edward','eramirez9n@shareasale.com','38.136.90.136','2010-08-19 08:01:06'), + (349,'Cheryl','cbutler9o@google.ca','172.180.78.172','1995-05-27 20:03:52'), + (350,'Margaret','mwatkins9p@sfgate.com','3.20.198.6','2014-10-21 01:42:58'), + (351,'Rebecca','rwelch9q@examiner.com','45.81.42.208','2001-02-08 12:19:06'), + (352,'Joe','jpalmer9r@phpbb.com','163.202.92.190','1970-01-05 11:29:12'), + (353,'Sandra','slewis9s@dyndns.org','77.215.201.236','1974-01-05 07:04:04'), + (354,'Todd','tfranklin9t@g.co','167.125.181.82','2009-09-28 10:13:58'), + (355,'Joseph','jlewis9u@webmd.com','244.204.6.11','1990-10-21 15:49:57'), + (356,'Alan','aknight9v@nydailynews.com','152.197.95.83','1996-03-08 08:43:17'), + (357,'Sharon','sdean9w@123-reg.co.uk','237.46.40.26','1985-11-30 12:09:24'), + (358,'Annie','awright9x@cafepress.com','190.45.231.111','2000-08-24 11:56:06'), + (359,'Diane','dhamilton9y@youtube.com','85.146.171.196','2015-02-24 02:03:57'), + (360,'Antonio','alane9z@auda.org.au','61.63.146.203','2001-05-13 03:43:34'), + (361,'Matthew','mallena0@hhs.gov','29.97.32.19','1973-02-19 23:43:32'), + (362,'Bonnie','bfowlera1@soup.io','251.216.99.53','2013-08-01 15:35:41'), + (363,'Margaret','mgraya2@examiner.com','69.255.151.79','1998-01-23 22:24:59'), + (364,'Joan','jwagnera3@printfriendly.com','192.166.120.61','1973-07-13 00:30:22'), + (365,'Catherine','cperkinsa4@nytimes.com','58.21.24.214','2006-11-19 11:52:26'), + (366,'Mark','mcartera5@cpanel.net','220.33.102.142','2007-09-09 09:43:27'), + (367,'Paula','ppricea6@msn.com','36.182.238.124','2009-11-11 09:13:05'), + (368,'Catherine','cgreena7@army.mil','228.203.58.19','2005-08-09 16:52:15'), + (369,'Helen','hhamiltona8@symantec.com','155.56.194.99','2005-02-01 05:40:36'), + (370,'Jane','jmeyera9@ezinearticles.com','133.244.113.213','2013-11-06 22:10:23'), + (371,'Wanda','wevansaa@bloglovin.com','233.125.192.48','1994-12-26 23:43:42'), + (372,'Mark','mmarshallab@tumblr.com','114.74.60.47','2016-09-29 18:03:01'), + (373,'Andrew','amartinezac@google.cn','182.54.37.130','1976-06-06 17:04:17'), + (374,'Helen','hmoralesad@e-recht24.de','42.45.4.123','1977-03-28 19:06:59'), + (375,'Bonnie','bstoneae@php.net','196.149.79.137','1970-02-05 17:05:58'), + (376,'Douglas','dfreemanaf@nasa.gov','215.65.124.218','2008-11-20 21:51:55'), + (377,'Willie','wwestag@army.mil','35.189.92.118','1992-07-24 05:08:08'), + (378,'Cheryl','cwagnerah@upenn.edu','228.239.222.141','2010-01-25 06:29:01'), + (379,'Sandra','swardai@baidu.com','63.11.113.240','1985-05-23 08:07:37'), + (380,'Julie','jrobinsonaj@jugem.jp','110.58.202.50','2015-03-05 09:42:07'), + (381,'Larry','lwagnerak@shop-pro.jp','98.234.25.24','1975-07-22 22:22:02'), + (382,'Juan','jcastilloal@yelp.com','24.174.74.202','2007-01-17 09:32:43'), + (383,'Donna','dfrazieram@artisteer.com','205.26.147.45','1990-02-11 20:55:46'), + (384,'Rachel','rfloresan@w3.org','109.60.216.162','1983-05-22 22:42:18'), + (385,'Robert','rreynoldsao@theguardian.com','122.65.209.130','2009-05-01 18:02:51'), + (386,'Donald','dbradleyap@etsy.com','42.54.35.126','1997-01-16 16:31:52'), + (387,'Rachel','rfisheraq@nih.gov','160.243.250.45','2006-02-17 22:05:49'), + (388,'Nicholas','nhamiltonar@princeton.edu','156.211.37.111','1976-06-21 03:36:29'), + (389,'Timothy','twhiteas@ca.gov','36.128.23.70','1975-09-24 03:51:18'), + (390,'Diana','dbradleyat@odnoklassniki.ru','44.102.120.184','1983-04-27 09:02:50'), + (391,'Billy','bfowlerau@jimdo.com','91.200.68.196','1995-01-29 06:57:35'), + (392,'Bruce','bandrewsav@ucoz.com','48.12.101.125','1992-10-27 04:31:39'), + (393,'Linda','lromeroaw@usa.gov','100.71.233.19','1992-06-08 15:13:18'), + (394,'Debra','dwatkinsax@ucoz.ru','52.160.233.193','2001-11-11 06:51:01'), + (395,'Katherine','kburkeay@wix.com','151.156.242.141','2010-06-14 19:54:28'), + (396,'Martha','mharrisonaz@youku.com','21.222.10.199','1989-10-16 14:17:55'), + (397,'Dennis','dwellsb0@youtu.be','103.16.29.3','1985-12-21 06:05:51'), + (398,'Gloria','grichardsb1@bloglines.com','90.147.120.234','1982-08-27 01:04:43'), + (399,'Brenda','bfullerb2@t.co','33.253.63.90','2011-04-20 05:00:35'), + (400,'Larry','lhendersonb3@disqus.com','88.95.132.128','1982-08-31 02:15:12'), + (401,'Richard','rlarsonb4@wisc.edu','13.48.231.150','1979-04-15 14:08:09'), + (402,'Terry','thuntb5@usa.gov','65.91.103.240','1998-05-15 11:50:49'), + (403,'Harry','hburnsb6@nasa.gov','33.38.21.244','1981-04-12 14:02:20'), + (404,'Diana','dellisb7@mlb.com','218.229.81.135','1997-01-29 00:17:25'), + (405,'Jack','jburkeb8@tripadvisor.com','210.227.182.216','1984-03-09 17:24:03'), + (406,'Julia','jlongb9@fotki.com','10.210.12.104','2005-10-26 03:54:13'), + (407,'Lois','lscottba@msu.edu','188.79.136.138','1973-02-02 18:40:39'), + (408,'Sandra','shendersonbb@shareasale.com','114.171.220.108','2012-06-09 18:22:26'), + (409,'Irene','isanchezbc@cdbaby.com','109.255.50.119','1983-09-28 21:11:27'), + (410,'Emily','ebrooksbd@bandcamp.com','227.81.93.79','1970-08-31 21:08:01'), + (411,'Michelle','mdiazbe@businessweek.com','236.249.6.226','1993-05-22 08:07:07'), + (412,'Tammy','tbennettbf@wisc.edu','145.253.239.152','1978-12-31 20:24:51'), + (413,'Christine','cgreenebg@flickr.com','97.25.140.118','1978-07-17 12:55:30'), + (414,'Patricia','pgarzabh@tuttocitta.it','139.246.192.211','1984-02-27 13:40:08'), + (415,'Kimberly','kromerobi@aol.com','73.56.88.247','1976-09-16 14:22:04'), + (416,'George','gjohnstonbj@fda.gov','240.36.245.185','1979-07-24 14:36:02'), + (417,'Eugene','efullerbk@sciencedaily.com','42.38.105.140','2012-09-12 01:56:41'), + (418,'Andrea','astevensbl@goo.gl','31.152.207.204','1979-05-24 11:06:21'), + (419,'Shirley','sreidbm@scientificamerican.com','103.60.31.241','1984-02-23 04:07:41'), + (420,'Terry','tmorenobn@blinklist.com','92.161.34.42','1994-06-25 14:01:35'), + (421,'Christopher','cmorenobo@go.com','158.86.176.82','1973-09-05 09:18:47'), + (422,'Dennis','dhansonbp@ning.com','40.160.81.75','1982-01-20 10:19:41'), + (423,'Beverly','brussellbq@de.vu','138.32.56.204','1997-11-06 07:20:19'), + (424,'Howard','hparkerbr@163.com','103.171.134.171','2015-06-24 15:37:10'), + (425,'Helen','hmccoybs@fema.gov','61.200.4.71','1995-06-20 08:59:10'), + (426,'Ann','ahudsonbt@cafepress.com','239.187.71.125','1977-04-11 07:59:28'), + (427,'Tina','twestbu@nhs.uk','80.213.117.74','1992-08-19 05:54:44'), + (428,'Terry','tnguyenbv@noaa.gov','21.93.118.95','1991-09-19 23:22:55'), + (429,'Ashley','aburtonbw@wix.com','233.176.205.109','2009-11-10 05:01:20'), + (430,'Eric','emyersbx@1und1.de','168.91.212.67','1987-08-10 07:16:20'), + (431,'Barbara','blittleby@lycos.com','242.14.189.239','2008-08-02 12:13:04'), + (432,'Sean','sevansbz@instagram.com','14.39.177.13','2007-04-16 17:28:49'), + (433,'Shirley','sburtonc0@newsvine.com','34.107.138.76','1980-12-10 02:19:29'), + (434,'Patricia','pfreemanc1@so-net.ne.jp','219.213.142.117','1987-03-01 02:25:45'), + (435,'Paula','pfosterc2@vkontakte.ru','227.14.138.141','1972-09-22 12:59:34'), + (436,'Nicole','nstewartc3@1688.com','8.164.23.115','1998-10-27 00:10:17'), + (437,'Earl','ekimc4@ovh.net','100.26.244.177','2013-01-22 10:05:46'), + (438,'Beverly','breedc5@reuters.com','174.12.226.27','1974-09-22 07:29:36'), + (439,'Lawrence','lbutlerc6@a8.net','105.164.42.164','1992-06-05 00:43:40'), + (440,'Charles','cmoorec7@ucoz.com','252.197.131.69','1990-04-09 02:34:05'), + (441,'Alice','alawsonc8@live.com','183.73.220.232','1989-02-28 09:11:04'), + (442,'Dorothy','dcarpenterc9@arstechnica.com','241.47.200.14','2005-05-02 19:57:21'), + (443,'Carolyn','cfowlerca@go.com','213.109.55.202','1978-09-10 20:18:20'), + (444,'Anthony','alongcb@free.fr','169.221.158.204','1984-09-13 01:59:23'), + (445,'Annie','amoorecc@e-recht24.de','50.34.148.61','2009-03-26 03:41:07'), + (446,'Carlos','candrewscd@ihg.com','236.69.59.212','1972-03-29 22:42:48'), + (447,'Beverly','bramosce@google.ca','164.250.184.49','1982-11-10 04:34:01'), + (448,'Teresa','tlongcf@umich.edu','174.88.53.223','1987-05-17 12:48:00'), + (449,'Roy','rboydcg@uol.com.br','91.58.243.215','1974-06-16 17:59:54'), + (450,'Ashley','afieldsch@tamu.edu','130.138.11.126','1983-09-15 05:52:36'), + (451,'Judith','jhawkinsci@cmu.edu','200.187.103.245','2003-10-22 12:24:03'), + (452,'Rebecca','rwestcj@ocn.ne.jp','72.85.3.103','1980-11-13 11:01:26'), + (453,'Raymond','rporterck@infoseek.co.jp','146.33.216.151','1982-05-17 23:58:03'), + (454,'Janet','jmarshallcl@odnoklassniki.ru','52.46.193.166','1998-10-04 00:02:21'), + (455,'Shirley','speterscm@salon.com','248.126.31.15','1987-01-30 06:04:59'), + (456,'Annie','abowmancn@economist.com','222.213.248.59','2006-03-14 23:52:59'), + (457,'Jean','jlarsonco@blogspot.com','71.41.25.195','2007-09-08 23:49:45'), + (458,'Phillip','pmoralescp@stanford.edu','74.119.87.28','2011-03-14 20:25:40'), + (459,'Norma','nrobinsoncq@economist.com','28.225.21.54','1989-10-21 01:22:43'), + (460,'Kimberly','kclarkcr@dion.ne.jp','149.171.132.153','2008-06-27 02:27:30'), + (461,'Ruby','rmorriscs@ucla.edu','177.85.163.249','2016-01-28 16:43:44'), + (462,'Jonathan','jcastilloct@tripod.com','78.4.28.77','2000-05-24 17:33:06'), + (463,'Edward','ebryantcu@jigsy.com','140.31.98.193','1992-12-17 08:32:47'), + (464,'Chris','chamiltoncv@eepurl.com','195.171.234.206','1970-12-05 03:42:19'), + (465,'Michael','mweavercw@reference.com','7.233.133.213','1987-03-29 02:30:54'), + (466,'Howard','hlawrencecx@businessweek.com','113.225.124.224','1990-07-30 07:20:57'), + (467,'Philip','phowardcy@comsenz.com','159.170.247.249','2010-10-15 10:18:37'), + (468,'Mary','mmarshallcz@xing.com','125.132.189.70','2007-07-19 13:48:47'), + (469,'Scott','salvarezd0@theguardian.com','78.49.103.230','1987-10-31 06:10:44'), + (470,'Wayne','wcarrolld1@blog.com','238.1.120.204','1980-11-19 03:26:10'), + (471,'Jennifer','jwoodsd2@multiply.com','92.20.224.49','2010-05-06 22:17:04'), + (472,'Raymond','rwelchd3@toplist.cz','176.158.35.240','2007-12-12 19:02:51'), + (473,'Steven','sdixond4@wisc.edu','167.55.237.52','1984-05-05 11:44:37'), + (474,'Ralph','rjamesd5@ameblo.jp','241.190.50.133','2000-07-06 08:44:37'), + (475,'Jason','jrobinsond6@hexun.com','138.119.139.56','2006-02-03 05:27:45'), + (476,'Doris','dwoodd7@fema.gov','180.220.156.190','1978-05-11 20:14:20'), + (477,'Elizabeth','eberryd8@youtu.be','74.188.53.229','2006-11-18 08:29:06'), + (478,'Irene','igilbertd9@privacy.gov.au','194.152.218.1','1985-09-17 02:46:52'), + (479,'Jessica','jdeanda@ameblo.jp','178.103.93.118','1974-06-07 19:04:05'), + (480,'Rachel','ralvarezdb@phoca.cz','17.22.223.174','1999-03-08 02:43:25'), + (481,'Kenneth','kthompsondc@shinystat.com','229.119.91.234','2007-05-15 13:17:32'), + (482,'Harold','hmurraydd@parallels.com','133.26.188.80','1993-11-15 03:42:07'), + (483,'Paula','phowellde@samsung.com','34.215.28.216','1993-11-29 15:55:00'), + (484,'Ruth','rpiercedf@tripadvisor.com','111.30.130.123','1986-08-17 10:19:38'), + (485,'Phyllis','paustindg@vk.com','50.84.34.178','1994-04-13 03:05:24'), + (486,'Laura','lfosterdh@usnews.com','37.8.101.33','2001-06-30 08:58:59'), + (487,'Eric','etaylordi@com.com','103.183.253.45','2006-09-15 20:18:46'), + (488,'Doris','driveradj@prweb.com','247.16.2.199','1989-05-08 09:27:09'), + (489,'Ryan','rhughesdk@elegantthemes.com','103.234.153.232','1989-08-01 18:36:06'), + (490,'Steve','smoralesdl@jigsy.com','3.76.84.207','2011-03-13 17:01:05'), + (491,'Louis','lsullivandm@who.int','78.135.44.208','1975-11-26 16:01:23'), + (492,'Catherine','ctuckerdn@seattletimes.com','93.137.106.21','1990-03-13 16:14:56'), + (493,'Ann','adixondo@gmpg.org','191.136.222.111','2002-06-05 14:22:18'), + (494,'Johnny','jhartdp@amazon.com','103.252.198.39','1988-07-30 23:54:49'), + (495,'Susan','srichardsdq@skype.com','126.247.192.11','2005-01-09 12:08:14'), + (496,'Brenda','bparkerdr@skype.com','63.232.216.86','1974-05-18 05:58:29'), + (497,'Tammy','tmurphyds@constantcontact.com','56.56.37.112','2014-08-05 18:22:25'), + (498,'Larry','lhayesdt@wordpress.com','162.146.13.46','1997-02-26 14:01:53'), + (499,'Evelyn','ethomasdu@hhs.gov','6.241.88.250','2007-09-14 13:03:34'), + (500,'Paula','pshawdv@networksolutions.com','123.27.47.249','2003-10-30 21:19:20'); + +create table {schema}.seed_config_expected_1 as ( + + select *, 'default'::text as c1, 'default'::text as c2, 'was true'::text as some_bool from {schema}.seed + +); + +create table {schema}.seed_config_expected_2 as ( + + select *, 'abc'::text as c1, 'def'::text as c2, 'was true'::text as some_bool from {schema}.seed + +); + +create table {schema}.seed_config_expected_3 as ( + + select *, 'ghi'::text as c1, 'jkl'::text as c2, 'was true'::text as some_bool from {schema}.seed + +); + +create table {schema}.seed_summary ( + year timestamp without time zone, + count bigint +); + +INSERT INTO {schema}.seed_summary + ("year","count") +VALUES + ('1970-01-01 00:00:00',10), + ('1971-01-01 00:00:00',6), + ('1972-01-01 00:00:00',9), + ('1973-01-01 00:00:00',12), + ('1974-01-01 00:00:00',8), + ('1975-01-01 00:00:00',5), + ('1976-01-01 00:00:00',11), + ('1977-01-01 00:00:00',13), + ('1978-01-01 00:00:00',11), + ('1979-01-01 00:00:00',13), + ('1980-01-01 00:00:00',9), + ('1981-01-01 00:00:00',3), + ('1982-01-01 00:00:00',9), + ('1983-01-01 00:00:00',15), + ('1984-01-01 00:00:00',13), + ('1985-01-01 00:00:00',11), + ('1986-01-01 00:00:00',5), + ('1987-01-01 00:00:00',14), + ('1988-01-01 00:00:00',9), + ('1989-01-01 00:00:00',10), + ('1990-01-01 00:00:00',12), + ('1991-01-01 00:00:00',16), + ('1992-01-01 00:00:00',15), + ('1993-01-01 00:00:00',11), + ('1994-01-01 00:00:00',10), + ('1995-01-01 00:00:00',10), + ('1996-01-01 00:00:00',6), + ('1997-01-01 00:00:00',11), + ('1998-01-01 00:00:00',12), + ('1999-01-01 00:00:00',9), + ('2000-01-01 00:00:00',13), + ('2001-01-01 00:00:00',14), + ('2002-01-01 00:00:00',9), + ('2003-01-01 00:00:00',8), + ('2004-01-01 00:00:00',9), + ('2005-01-01 00:00:00',14), + ('2006-01-01 00:00:00',9), + ('2007-01-01 00:00:00',16), + ('2008-01-01 00:00:00',6), + ('2009-01-01 00:00:00',15), + ('2010-01-01 00:00:00',13), + ('2011-01-01 00:00:00',23), + ('2012-01-01 00:00:00',9), + ('2013-01-01 00:00:00',10), + ('2014-01-01 00:00:00',9), + ('2015-01-01 00:00:00',10), + ('2016-01-01 00:00:00',5); + diff --git a/test/integration/025_duplicate_model_test/test_duplicate_model.py b/test/integration/025_duplicate_model_test/test_duplicate_model.py new file mode 100644 index 00000000000..e9250c5db39 --- /dev/null +++ b/test/integration/025_duplicate_model_test/test_duplicate_model.py @@ -0,0 +1,155 @@ +from nose.plugins.attrib import attr + +from dbt.exceptions import CompilationException +from test.integration.base import DBTIntegrationTest + + +class TestDuplicateModelEnabled(DBTIntegrationTest): + + def setUp(self): + DBTIntegrationTest.setUp(self) + + @property + def schema(self): + return "duplicate_model_025" + + @property + def models(self): + return "test/integration/025_duplicate_model_test/models-1" + + @property + def profile_config(self): + return { + "test": { + "outputs": { + "dev": { + "type": "postgres", + "threads": 1, + "host": "database", + "port": 5432, + "user": "root", + "pass": "password", + "dbname": "dbt", + "schema": self.unique_schema() + }, + }, + "target": "dev" + } + } + + @attr(type="postgres") + def test_duplicate_model_enabled(self): + message = "Found models with the same name:.*" + with self.assertRaisesRegexp(CompilationException, message): + self.run_dbt(["run"]) + + +class TestDuplicateModelDisabled(DBTIntegrationTest): + + def setUp(self): + DBTIntegrationTest.setUp(self) + + @property + def schema(self): + return "duplicate_model_025" + + @property + def models(self): + return "test/integration/025_duplicate_model_test/models-2" + + @property + def profile_config(self): + return { + "test": { + "outputs": { + "dev": { + "type": "postgres", + "threads": 1, + "host": "database", + "port": 5432, + "user": "root", + "pass": "password", + "dbname": "dbt", + "schema": self.unique_schema() + }, + }, + "target": "dev" + } + } + + @attr(type="postgres") + def test_duplicate_model_disabled(self): + try: + self.run_dbt(["run"]) + except CompilationException: + self.fail( + "Compilation Exception raised on disabled model") + query = "select value from {schema}.model" \ + .format(schema=self.unique_schema()) + result = self.run_sql(query, fetch="one")[0] + assert result == 1 + + +class TestDuplicateModelEnabledAcrossPackages(DBTIntegrationTest): + + def setUp(self): + DBTIntegrationTest.setUp(self) + + @property + def schema(self): + return "duplicate_model_025" + + @property + def models(self): + return "test/integration/025_duplicate_model_test/models-3" + + @property + def project_config(self): + return { + "repositories": [ + 'https://github.com/fishtown-analytics/dbt-integration-project@master' + ] + } + + @attr(type="postgres") + def test_duplicate_model_enabled_across_packages(self): + self.run_dbt(["deps"]) + message = "Found models with the same name:.*" + with self.assertRaisesRegexp(CompilationException, message): + self.run_dbt(["run"]) + + +class TestDuplicateModelDisabledAcrossPackages(DBTIntegrationTest): + + def setUp(self): + DBTIntegrationTest.setUp(self) + self.run_sql_file("test/integration/025_duplicate_model_test/seed.sql") + + @property + def schema(self): + return "duplicate_model_025" + + @property + def models(self): + return "test/integration/025_duplicate_model_test/models-4" + + @property + def project_config(self): + return { + "repositories": [ + 'https://github.com/fishtown-analytics/dbt-integration-project@master' + ] + } + + @attr(type="postgres") + def test_duplicate_model_disabled_across_packages(self): + self.run_dbt(["deps"]) + try: + self.run_dbt(["run"]) + except CompilationException: + self.fail( + "Compilation Exception raised on disabled model") + query = "select 1 from {schema}.table" \ + .format(schema=self.unique_schema()) + result = self.run_sql(query, fetch="one")[0] + assert result == 1 diff --git a/test/integration/025_timezones_test/models/timezones.sql b/test/integration/025_timezones_test/models/timezones.sql new file mode 100644 index 00000000000..87d565487e1 --- /dev/null +++ b/test/integration/025_timezones_test/models/timezones.sql @@ -0,0 +1,10 @@ + +{{ + config( + materialized='table' + ) +}} + +select + '{{ run_started_at.astimezone(modules.pytz.timezone("America/New_York")) }}' as run_started_at_est, + '{{ run_started_at }}' as run_started_at_utc diff --git a/test/integration/025_timezones_test/test_timezones.py b/test/integration/025_timezones_test/test_timezones.py new file mode 100644 index 00000000000..af995eaef31 --- /dev/null +++ b/test/integration/025_timezones_test/test_timezones.py @@ -0,0 +1,55 @@ +from freezegun import freeze_time +from nose.plugins.attrib import attr +from test.integration.base import DBTIntegrationTest + + +class TestTimezones(DBTIntegrationTest): + + def setUp(self): + DBTIntegrationTest.setUp(self) + + @property + def schema(self): + return "timezones_025" + + @property + def models(self): + return "test/integration/025_timezones_test/models" + + @property + def profile_config(self): + return { + 'test': { + 'outputs': { + 'dev': { + 'type': 'postgres', + 'threads': 1, + 'host': 'database', + 'port': 5432, + 'user': "root", + 'pass': "password", + 'dbname': 'dbt', + 'schema': self.unique_schema() + }, + }, + 'target': 'dev' + } + } + + @property + def query(self): + return """ + select + run_started_at_est, + run_started_at_utc + from {schema}.timezones + """.format(schema=self.unique_schema()) + + @freeze_time("2017-01-01 03:00:00", tz_offset=0) + @attr(type='postgres') + def test_run_started_at(self): + self.run_dbt(['run']) + result = self.run_sql(self.query, fetch='all')[0] + est, utc = result + self.assertEqual(utc, '2017-01-01 03:00:00+00:00') + self.assertEqual(est, '2016-12-31 22:00:00-05:00') diff --git a/test/integration/028_cli_vars/models_complex/complex_model.sql b/test/integration/028_cli_vars/models_complex/complex_model.sql new file mode 100644 index 00000000000..1022c648100 --- /dev/null +++ b/test/integration/028_cli_vars/models_complex/complex_model.sql @@ -0,0 +1,6 @@ + +select + '{{ var("variable_1") }}'::varchar as var_1, + '{{ var("variable_2")[0] }}'::varchar as var_2, + '{{ var("variable_3")["value"] }}'::varchar as var_3 + diff --git a/test/integration/028_cli_vars/models_complex/schema.yml b/test/integration/028_cli_vars/models_complex/schema.yml new file mode 100644 index 00000000000..1b9f3156167 --- /dev/null +++ b/test/integration/028_cli_vars/models_complex/schema.yml @@ -0,0 +1,7 @@ + +complex_model: + constraints: + accepted_values: + - {field: var_1, values: ["abc"]} + - {field: var_2, values: ["def"]} + - {field: var_3, values: ["jkl"]} diff --git a/test/integration/028_cli_vars/models_simple/schema.yml b/test/integration/028_cli_vars/models_simple/schema.yml new file mode 100644 index 00000000000..98e80979a55 --- /dev/null +++ b/test/integration/028_cli_vars/models_simple/schema.yml @@ -0,0 +1,5 @@ + +simple_model: + constraints: + accepted_values: + - {field: simple, values: ["abc"]} diff --git a/test/integration/028_cli_vars/models_simple/simple_model.sql b/test/integration/028_cli_vars/models_simple/simple_model.sql new file mode 100644 index 00000000000..084bd57012e --- /dev/null +++ b/test/integration/028_cli_vars/models_simple/simple_model.sql @@ -0,0 +1,4 @@ + +select + '{{ var("simple") }}'::varchar as simple + diff --git a/test/integration/028_cli_vars/test_cli_vars.py b/test/integration/028_cli_vars/test_cli_vars.py new file mode 100644 index 00000000000..347a20e4dac --- /dev/null +++ b/test/integration/028_cli_vars/test_cli_vars.py @@ -0,0 +1,54 @@ +from nose.plugins.attrib import attr +from test.integration.base import DBTIntegrationTest +import yaml + + +class TestCLIVars(DBTIntegrationTest): + @property + def schema(self): + return "cli_vars_028" + + @property + def models(self): + return "test/integration/028_cli_vars/models_complex" + + @attr(type='postgres') + def test__cli_vars_longform(self): + self.use_default_project() + self.use_profile('postgres') + + cli_vars = { + "variable_1": "abc", + "variable_2": ["def", "ghi"], + "variable_3": { + "value": "jkl" + } + } + self.run_dbt(["run", "--vars", yaml.dump(cli_vars)]) + self.run_dbt(["test"]) + + +class TestCLIVarsSimple(DBTIntegrationTest): + @property + def schema(self): + return "cli_vars_028" + + @property + def models(self): + return "test/integration/028_cli_vars/models_simple" + + @attr(type='postgres') + def test__cli_vars_shorthand(self): + self.use_default_project() + self.use_profile('postgres') + + self.run_dbt(["run", "--vars", "simple: abc"]) + self.run_dbt(["test"]) + + @attr(type='postgres') + def test__cli_vars_longer(self): + self.use_default_project() + self.use_profile('postgres') + + self.run_dbt(["run", "--vars", "{simple: abc, unused: def}"]) + self.run_dbt(["test"]) diff --git a/test/integration/base.py b/test/integration/base.py index 78aa2729d06..05d48487809 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -186,19 +186,20 @@ def setUp(self): self.run_sql('DROP SCHEMA IF EXISTS "{}" CASCADE'.format(self.unique_schema())) self.run_sql('CREATE SCHEMA "{}"'.format(self.unique_schema())) - def use_default_project(self): + def use_default_project(self, overrides=None): # create a dbt_project.yml base_project_config = { 'name': 'test', 'version': '1.0', 'test-paths': [], 'source-paths': [self.models], - 'profile': 'test' + 'profile': 'test', } project_config = {} project_config.update(base_project_config) project_config.update(self.project_config) + project_config.update(overrides or {}) with open("dbt_project.yml", 'w') as f: yaml.safe_dump(project_config, f, default_flow_style=True) @@ -431,3 +432,7 @@ def assertTableColumnsEqual(self, table_a, table_b, table_a_schema=None, table_b table_a_result, table_b_result ) + + def assertEquals(self, *args, **kwargs): + # assertEquals is deprecated. This makes the warnings less chatty + self.assertEqual(*args, **kwargs) diff --git a/test/setup_db.sh b/test/setup_db.sh index e315d76f02f..d373b7195a0 100644 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -1,12 +1,19 @@ +#!/bin/bash set -x +# If you want to run this script for your own postgresql (run with +# docker-compose) it will look like this: +# PGHOST=127.0.0.1 PGUSER=root PGPASSWORD=password PGDATABASE=postgres \ +# bash test/setup.sh +PGUSER="${PGUSER:-postgres}" + createdb dbt -psql -c "CREATE ROLE root WITH UNENCRYPTED PASSWORD 'password';" -U postgres -psql -c "ALTER ROLE root WITH LOGIN;" -U postgres -psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root;" -U postgres +psql -c "CREATE ROLE root WITH PASSWORD 'password';" +psql -c "ALTER ROLE root WITH LOGIN;" +psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root;" -psql -c "CREATE ROLE noaccess WITH UNENCRYPTED PASSWORD 'password' NOSUPERUSER;" -U postgres; -psql -c "ALTER ROLE noaccess WITH LOGIN;" -U postgres -psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" -U postgres; +psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" +psql -c "ALTER ROLE noaccess WITH LOGIN;" +psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" set +x diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index 3ae465cafb9..25ca2a26de2 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -190,35 +190,6 @@ def test__model_materializations(self): .get('materialized') self.assertEquals(actual, expected) - def test__model_enabled(self): - self.use_models({ - 'model_one': 'select * from events', - 'model_two': "select * from {{ref('model_one')}}", - }) - - cfg = { - "models": { - "materialized": "table", - "test_models_compile": { - "model_one": {"enabled": True}, - "model_two": {"enabled": False}, - } - } - } - - compiler = self.get_compiler(self.get_project(cfg)) - graph, linker = compiler.compile() - - six.assertCountEqual( - self, linker.nodes(), - ['model.test_models_compile.model_one', - 'model.test_models_compile.model_two']) - - six.assertCountEqual( - self, linker.edges(), - [('model.test_models_compile.model_one', - 'model.test_models_compile.model_two',)]) - def test__model_incremental(self): self.use_models({ 'model_one': 'select * from events' diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index d8606c2a6a8..d20c4b3ae27 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -1101,48 +1101,6 @@ def test__other_project_config(self): 'raw_sql': self.find_input_by_name( models, 'view').get('raw_sql') }, - 'model.snowplow.disabled': { - 'name': 'disabled', - 'schema': 'analytics', - 'resource_type': 'model', - 'unique_id': 'model.snowplow.disabled', - 'fqn': ['snowplow', 'disabled'], - 'empty': False, - 'package_name': 'snowplow', - 'refs': [], - 'depends_on': { - 'nodes': [], - 'macros': [] - }, - 'path': 'disabled.sql', - 'original_file_path': 'disabled.sql', - 'root_path': get_os_path('/usr/src/app'), - 'config': disabled_config, - 'tags': set(), - 'raw_sql': self.find_input_by_name( - models, 'disabled').get('raw_sql') - }, - 'model.snowplow.package': { - 'name': 'package', - 'schema': 'analytics', - 'resource_type': 'model', - 'unique_id': 'model.snowplow.package', - 'fqn': ['snowplow', 'views', 'package'], - 'empty': False, - 'package_name': 'snowplow', - 'refs': [], - 'depends_on': { - 'nodes': [], - 'macros': [] - }, - 'path': get_os_path('views/package.sql'), - 'original_file_path': get_os_path('views/package.sql'), - 'root_path': get_os_path('/usr/src/app'), - 'config': sort_config, - 'tags': set(), - 'raw_sql': self.find_input_by_name( - models, 'package').get('raw_sql') - }, 'model.snowplow.multi_sort': { 'name': 'multi_sort', 'schema': 'analytics', From f1129cd930616fdfdd34b7135295db01970a19d9 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 26 Feb 2018 19:36:58 -0500 Subject: [PATCH 09/13] s/version/revision for git packages --- dbt/task/deps.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/dbt/task/deps.py b/dbt/task/deps.py index 5d80f881fd9..a01991c67e5 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -59,6 +59,9 @@ def resolve_version(self): def version_name(self): raise NotImplementedError() + def nice_version_name(self): + raise NotImplementedError() + def _fetch_metadata(self, project): raise NotImplementedError() @@ -102,6 +105,9 @@ def version_name(self): version_string = self.version[0].to_version_string(skip_matcher=True) return version_string + def nice_version_name(self): + return "version {}".format(self.version_name()) + def incorporate(self, other): return RegistryPackage(self.package, self.version + other.version) @@ -170,6 +176,9 @@ def version(self, version): def version_name(self): return self._version[0] + def nice_version_name(self): + return "revision {}".format(self.version_name()) + def incorporate(self, other): return GitPackage(self.git, self.version + other.version) @@ -217,6 +226,9 @@ def incorporate(self, _): def version_name(self): return ''.format(self.local) + def nice_version_name(self): + return self.version_name() + def _fetch_metadata(self, project): with open(os.path.join(self.local, 'dbt_project.yml')) as f: return load_yaml_text(f.read()) @@ -239,7 +251,11 @@ def _parse_package(dict_): if dict_.get('package'): return RegistryPackage(dict_['package'], dict_.get('version')) if dict_.get('git'): - return GitPackage(dict_['git'], dict_.get('version')) + if dict_.get('version'): + msg = ("Keyword 'version' specified for git package {}.\nDid " + "you mean 'revision'?".format(dict_.get('git'))) + dbt.exceptions.raise_dependency_error(msg) + return GitPackage(dict_['git'], dict_.get('revision')) if dict_.get('local'): return LocalPackage(dict_['local']) dbt.exceptions.raise_dependency_error( @@ -352,4 +368,4 @@ def run(self): for _, package in final_deps.items(): logger.info('Installing %s', package) package.install(self.project) - logger.info(' Installed at version %s\n', package.version_name()) + logger.info(' Installed from %s\n', package.nice_version_name()) From 5a072233e19a2de79741fc78fe07bb663fe05644 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Mon, 26 Feb 2018 19:43:01 -0500 Subject: [PATCH 10/13] more s/version/revision, deprecation cleanup --- dbt/deprecations.py | 13 +++---------- dbt/task/deps.py | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index 9bf5c48b05f..e7c43fb6fe0 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -21,16 +21,9 @@ class DBTRepositoriesDeprecation(DBTDeprecation): class SeedDropExistingDeprecation(DBTDeprecation): name = 'drop-existing' - description = """The --drop-existing argument has been deprecated. Please - use --full-refresh instead. The --drop-existing option will be removed in a - future version of dbt.""" - - -class SeedDropExistingDeprecation(DBTDeprecation): - name = 'drop-existing' - description = """The --drop-existing argument has been deprecated. Please - use --full-refresh instead. The --drop-existing option will be removed in a - future version of dbt.""" + description = """The --drop-existing argument to `dbt seed` has been + deprecated. Please use --full-refresh instead. The --drop-existing option + will be removed in a future version of dbt.""" def warn(name, *args, **kwargs): diff --git a/dbt/task/deps.py b/dbt/task/deps.py index a01991c67e5..33e9d1e8f1a 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -318,7 +318,7 @@ def _convert_repo(repo_spec): repo, branch = _split_at_branch(repo_spec) return { 'git': repo, - 'version': branch, + 'revision': branch, } From fac6e0a1dcfd82802e362dd79b689861a19aa481 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 27 Feb 2018 15:55:25 -0500 Subject: [PATCH 11/13] remove unused semver codepath --- dbt/semver.py | 118 ---------------------------------- test/unit/test_semver.py | 133 --------------------------------------- 2 files changed, 251 deletions(-) diff --git a/dbt/semver.py b/dbt/semver.py index 77d71b5d3f4..006513fd315 100644 --- a/dbt/semver.py +++ b/dbt/semver.py @@ -396,121 +396,3 @@ def resolve_to_specific_version(requested_range, available_versions): max_version_string = version_string return max_version_string - - -def resolve_dependency_tree(version_index, unmet_dependencies, restrictions): - for name, restriction in restrictions.items(): - if not versions_compatible(*restriction): - raise VersionsNotCompatibleException( - 'not compatible {}'.format(restriction)) - - if not unmet_dependencies: - return {}, {} - - to_return_tree = {} - to_return_install = {} - - for dependency_name, version in unmet_dependencies.items(): - logger.debug('resolving path %s', dependency_name) - dependency_restrictions = reduce_versions( - *restrictions.copy().get(dependency_name)) - - possible_matches = find_possible_versions( - dependency_restrictions, - version_index[dependency_name].keys()) - - for possible_match in possible_matches: - logger.debug('reset with %s at %s', - dependency_name, possible_match) - - tree = {} - install = {} - new_restrictions = {} - new_unmet_dependencies = {} - - match_found = False - - try: - new_restrictions = restrictions.copy() - new_restrictions[dependency_name] = reduce_versions( - dependency_restrictions, - possible_match - ).to_version_string_pair() - - ver = version_index.get(dependency_name, {}) - recursive_version_info = ver.get(possible_match) - new_unmet_dependencies = dbt.utils.deep_merge( - recursive_version_info.copy()) - - logger.debug('new unmet dependencies %s', - new_unmet_dependencies) - - new_restrictions = dbt.utils.deep_merge( - new_restrictions.copy(), - unmet_dependencies.copy(), - new_unmet_dependencies.copy()) - - new_restrictions[dependency_name] += [possible_match] - - if dependency_name in new_unmet_dependencies: - del new_unmet_dependencies[dependency_name] - - for name, restriction in new_restrictions.items(): - if not versions_compatible(*restriction): - raise VersionsNotCompatibleException( - 'not compatible {}'.format(new_restrictions)) - - else: - match_found = True - - logger.debug('going down the stack with %s and %s', - new_unmet_dependencies, install) - subtree, subinstall = resolve_dependency_tree( - version_index, - new_unmet_dependencies, - new_restrictions) - - tree.update({ - dependency_name: { - 'version': possible_match, - 'satisfies': [dependency_name], - 'dependencies': subtree - } - }) - - install = dbt.utils.deep_merge( - install, - subinstall, - {dependency_name: possible_match}) - - logger.debug('then %s', install) - - to_return_tree = dbt.utils.deep_merge( - to_return_tree, - tree) - - to_return_install = dbt.utils.deep_merge( - to_return_install, - install) - - break - - if not match_found: - raise VersionsNotCompatibleException( - 'No match found -- exhausted this part of the tree.') - - except VersionsNotCompatibleException as e: - logger.debug('%s -- When attempting %s at %s', - e, dependency_name, possible_match) - - return to_return_tree.copy(), to_return_install.copy() - - -def resolve_dependency_set(version_index, dependencies): - tree, install = resolve_dependency_tree( - version_index, dependencies, dependencies) - - return { - 'install': install, - 'tree': tree, - } diff --git a/test/unit/test_semver.py b/test/unit/test_semver.py index 72bb7294c42..7f76de158f8 100644 --- a/test/unit/test_semver.py +++ b/test/unit/test_semver.py @@ -134,136 +134,3 @@ def test__resolve_to_specific_version(self): create_range(None, '<=0.0.5'), ['0.0.3', '0.1.4', '0.0.5']), '0.0.5') - - def test__resolve_dependency_set__multiple_deps(self): - self.maxDiff = None - self.assertDictEqual( - resolve_dependency_set( - version_index={ - 'a': { - '0.0.1': { - 'b': ['=0.0.1'], - }, - '0.0.2': { - 'b': ['=0.0.1'], - }, - '0.0.3': { - 'b': ['>=0.0.1'], - 'c': ['=0.0.2'], - } - }, - 'b': { - '0.0.1': { - 'c': ['=0.0.2'], - }, - '0.0.2': { - 'c': ['=0.0.1'], - } - }, - 'c': { - '0.0.1': {}, - '0.0.2': {} - } - }, - dependencies={ - 'a': [] - } - ), - { - 'install': { - 'a': '0.0.3', - 'b': '0.0.1', - 'c': '0.0.2', - }, - 'tree': { - 'a': { - 'version': '0.0.3', - 'satisfies': ['a'], - 'dependencies': { - 'b': { - 'version': '0.0.1', - 'satisfies': ['b'], - 'dependencies': { - 'c': { - 'version': '0.0.2', - 'satisfies': ['c'], - 'dependencies': {} - } - } - }, - 'c': { - 'version': '0.0.2', - 'satisfies': ['c'], - 'dependencies': {}, - } - } - } - } - }) - - def test__resolve_dependency_set(self): - self.maxDiff = None - self.assertDictEqual( - resolve_dependency_set( - version_index={ - 'a': { - '0.0.1': { - 'b': ['=0.0.1'], - }, - '0.0.2': { - 'b': ['=0.0.1'], - }, - '0.0.3': { - 'b': ['>=0.0.1'], - 'c': ['=0.0.2'], - } - }, - 'b': { - '0.0.1': { - 'c': ['=0.0.1'], - }, - '0.0.2': { - 'c': ['=0.0.2'], - } - }, - 'c': { - '0.0.1': {}, - '0.0.2': {} - } - }, - dependencies={ - 'a': [] - } - ), - { - 'install': { - 'a': '0.0.3', - 'b': '0.0.2', - 'c': '0.0.2', - }, - 'tree': { - 'a': { - 'version': '0.0.3', - 'satisfies': ['a'], - 'dependencies': { - 'b': { - 'version': '0.0.2', - 'satisfies': ['b'], - 'dependencies': { - 'c': { - 'version': '0.0.2', - 'satisfies': ['c'], - 'dependencies': {} - } - } - }, - 'c': { - 'version': '0.0.2', - 'satisfies': ['c'], - 'dependencies': {} - } - - } - } - } - }) From 7afe427861bf45b8fdf96b23860795775f57dae6 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 27 Feb 2018 16:37:40 -0500 Subject: [PATCH 12/13] plus symlinks!!! --- dbt/clients/system.py | 27 +++++++++++++++++++++++++++ dbt/exceptions.py | 8 ++++++++ dbt/task/deps.py | 21 ++++++++++++++++++--- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/dbt/clients/system.py b/dbt/clients/system.py index 114939df5d1..de4f4c7b335 100644 --- a/dbt/clients/system.py +++ b/dbt/clients/system.py @@ -10,6 +10,7 @@ import stat import dbt.compat +import dbt.exceptions from dbt.logger import GLOBAL_LOGGER as logger @@ -95,6 +96,20 @@ def make_file(path, contents='', overwrite=False): return False +def make_symlink(source, link_path): + """ + Create a symlink at `link_path` referring to `source`. + """ + if not supports_symlinks(): + dbt.exceptions.system_error('create a symbolic link') + + return os.symlink(source, link_path) + + +def supports_symlinks(): + return getattr(os, "symlink", None) is not None + + def write_file(path, contents=''): make_directory(os.path.dirname(path)) dbt.compat.write_file(path, contents) @@ -126,6 +141,18 @@ def rmdir(path): return shutil.rmtree(path, onerror=onerror) +def remove_file(path): + return os.remove(path) + + +def path_exists(path): + return os.path.lexists(path) + + +def path_is_symlink(path): + return os.path.islink(path) + + def open_dir_cmd(): # https://docs.python.org/2/library/sys.html#sys.platform if sys.platform == 'win32': diff --git a/dbt/exceptions.py b/dbt/exceptions.py index 798c5874897..c5076f5132e 100644 --- a/dbt/exceptions.py +++ b/dbt/exceptions.py @@ -274,5 +274,13 @@ def invalid_materialization_argument(name, argument): .format(name, argument)) +def system_error(operation_name): + raise_compiler_error( + "dbt encountered an error when attempting to {}. " + "If this error persists, please create an issue at: \n\n" + "https://github.com/fishtown-analytics/dbt" + .format(operation_name)) + + class RegistryException(Exception): pass diff --git a/dbt/task/deps.py b/dbt/task/deps.py index 33e9d1e8f1a..a0c4f2ca27d 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -235,9 +235,23 @@ def _fetch_metadata(self, project): def install(self, project): dest_path = self.get_installation_path(project) - if os.path.exists(dest_path): - dbt.clients.system.rmdir(dest_path) - shutil.copytree(self.local, dest_path) + + can_create_symlink = dbt.clients.system.supports_symlinks() + + if dbt.clients.system.path_exists(dest_path): + if not dbt.clients.system.path_is_symlink(dest_path): + dbt.clients.system.rmdir(dest_path) + else: + dbt.clients.system.remove_file(dest_path) + + if can_create_symlink: + logger.debug(' Creating symlink to local dependency.') + dbt.clients.system.make_symlink(self.local, dest_path) + + else: + logger.debug(' Symlinks are not available on this ' + 'OS, copying dependency.') + shutil.copytree(self.local, dest_path) def _parse_package(dict_): @@ -354,6 +368,7 @@ def run(self): pending_deps = PackageListing.create(packages) final_deps = PackageListing.create([]) + while pending_deps: sub_deps = PackageListing.create([]) for name, package in pending_deps.items(): From 707161a95c88c27a6cd1219f0b06371b9dfde9fe Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Tue, 27 Feb 2018 16:38:39 -0500 Subject: [PATCH 13/13] get rid of reference to removed function --- test/unit/test_semver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_semver.py b/test/unit/test_semver.py index 7f76de158f8..05cde5b450f 100644 --- a/test/unit/test_semver.py +++ b/test/unit/test_semver.py @@ -4,7 +4,7 @@ from dbt.exceptions import VersionsNotCompatibleException from dbt.semver import VersionSpecifier, UnboundedVersionSpecifier, \ VersionRange, reduce_versions, versions_compatible, \ - resolve_to_specific_version, resolve_dependency_set + resolve_to_specific_version def create_range(start_version_string, end_version_string):