diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f04d1e92cfa..00c2666d1f2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.12.2rc1 +current_version = 0.13.0a1 parse = (?P\d+) \.(?P\d+) \.(?P\d+) @@ -20,7 +20,15 @@ values = [bumpversion:part:num] first_value = 1 -[bumpversion:file:setup.py] +[bumpversion:file:core/setup.py] -[bumpversion:file:dbt/version.py] +[bumpversion:file:core/dbt/version.py] + +[bumpversion:file:plugins/postgres/setup.py] + +[bumpversion:file:plugins/redshift/setup.py] + +[bumpversion:file:plugins/snowflake/setup.py] + +[bumpversion:file:plugins/bigquery/setup.py] diff --git a/.coveragerc b/.coveragerc index 23cf94ed431..18244411816 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,4 @@ [report] include = - dbt/* + core/dbt/* + plugins/adapters/dbt/* diff --git a/CHANGELOG.md b/core/CHANGELOG.md similarity index 100% rename from CHANGELOG.md rename to core/CHANGELOG.md diff --git a/CONTRIBUTING.md b/core/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to core/CONTRIBUTING.md diff --git a/License.md b/core/License.md similarity index 100% rename from License.md rename to core/License.md diff --git a/MANIFEST.in b/core/MANIFEST.in similarity index 100% rename from MANIFEST.in rename to core/MANIFEST.in diff --git a/README.md b/core/README.md similarity index 100% rename from README.md rename to core/README.md diff --git a/core/dbt/__init__.py b/core/dbt/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/core/dbt/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/core/dbt/adapters/__init__.py b/core/dbt/adapters/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/core/dbt/adapters/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/dbt/adapters/base/__init__.py b/core/dbt/adapters/base/__init__.py similarity index 81% rename from dbt/adapters/base/__init__.py rename to core/dbt/adapters/base/__init__.py index a509f6b0ed9..98f96abe1b4 100644 --- a/dbt/adapters/base/__init__.py +++ b/core/dbt/adapters/base/__init__.py @@ -2,3 +2,4 @@ from dbt.adapters.base.relation import BaseRelation from dbt.adapters.base.connections import BaseConnectionManager, Credentials from dbt.adapters.base.impl import BaseAdapter +from dbt.adapters.base.plugin import AdapterPlugin diff --git a/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py similarity index 100% rename from dbt/adapters/base/connections.py rename to core/dbt/adapters/base/connections.py diff --git a/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py similarity index 100% rename from dbt/adapters/base/impl.py rename to core/dbt/adapters/base/impl.py diff --git a/dbt/adapters/base/meta.py b/core/dbt/adapters/base/meta.py similarity index 100% rename from dbt/adapters/base/meta.py rename to core/dbt/adapters/base/meta.py diff --git a/core/dbt/adapters/base/plugin.py b/core/dbt/adapters/base/plugin.py new file mode 100644 index 00000000000..242bc681b8a --- /dev/null +++ b/core/dbt/adapters/base/plugin.py @@ -0,0 +1,26 @@ +import os + +from dbt.config.project import Project + + +class AdapterPlugin(object): + """Defines the basic requirements for a dbt adapter plugin. + + :param type adapter: An adapter class, derived from BaseAdapter + :param type credentials: A credentials object, derived from Credentials + :param str project_name: The name of this adapter plugin's associated dbt + project. + :param str include_path: The path to this adapter plugin's root + :param Optional[List[str]] dependencies: A list of adapter names that this\ + adapter depends upon. + """ + def __init__(self, adapter, credentials, include_path, dependencies=None): + self.adapter = adapter + self.credentials = credentials + self.include_path = include_path + project_path = os.path.join(self.include_path, adapter.type()) + project = Project.from_project_root(project_path, {}) + self.project_name = project.project_name + if dependencies is None: + dependencies = [] + self.dependencies = dependencies diff --git a/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py similarity index 100% rename from dbt/adapters/base/relation.py rename to core/dbt/adapters/base/relation.py diff --git a/dbt/adapters/cache.py b/core/dbt/adapters/cache.py similarity index 100% rename from dbt/adapters/cache.py rename to core/dbt/adapters/cache.py diff --git a/dbt/adapters/factory.py b/core/dbt/adapters/factory.py similarity index 81% rename from dbt/adapters/factory.py rename to core/dbt/adapters/factory.py index bef5e66dfa3..f3145c690f1 100644 --- a/dbt/adapters/factory.py +++ b/core/dbt/adapters/factory.py @@ -2,10 +2,10 @@ import dbt.exceptions from importlib import import_module +from dbt.include.global_project import PACKAGES import threading - ADAPTER_TYPES = {} _ADAPTERS = {} @@ -28,25 +28,31 @@ def get_relation_class_by_name(adapter_name): return adapter.Relation -def load_adapter(adapter_name): - """Load an adapter package with the class of adapter_name, put it in the - ADAPTER_TYPES dict, and return its associated Credentials - """ +def load_plugin(adapter_name): try: mod = import_module('.'+adapter_name, 'dbt.adapters') except ImportError: raise dbt.exceptions.RuntimeException( "Could not find adapter type {}!".format(adapter_name) ) - if mod.Adapter.type() != adapter_name: + plugin = mod.Plugin + + if plugin.adapter.type() != adapter_name: raise dbt.exceptions.RuntimeException( 'Expected to find adapter with type named {}, got adapter with ' 'type {}' - .format(adapter_name, mod.Adapter.type()) + .format(adapter_name, plugin.adapter.type()) ) - ADAPTER_TYPES[adapter_name] = mod.Adapter - return mod.Credentials + with _ADAPTER_LOCK: + ADAPTER_TYPES[adapter_name] = plugin.adapter + + PACKAGES[plugin.project_name] = plugin.include_path + + for dep in plugin.dependencies: + load_plugin(dep) + + return plugin.credentials def get_adapter(config): diff --git a/dbt/adapters/sql/__init__.py b/core/dbt/adapters/sql/__init__.py similarity index 100% rename from dbt/adapters/sql/__init__.py rename to core/dbt/adapters/sql/__init__.py diff --git a/dbt/adapters/sql/connections.py b/core/dbt/adapters/sql/connections.py similarity index 100% rename from dbt/adapters/sql/connections.py rename to core/dbt/adapters/sql/connections.py diff --git a/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py similarity index 100% rename from dbt/adapters/sql/impl.py rename to core/dbt/adapters/sql/impl.py diff --git a/dbt/api/__init__.py b/core/dbt/api/__init__.py similarity index 100% rename from dbt/api/__init__.py rename to core/dbt/api/__init__.py diff --git a/dbt/api/object.py b/core/dbt/api/object.py similarity index 100% rename from dbt/api/object.py rename to core/dbt/api/object.py diff --git a/dbt/__init__.py b/core/dbt/clients/__init__.py similarity index 100% rename from dbt/__init__.py rename to core/dbt/clients/__init__.py diff --git a/dbt/clients/agate_helper.py b/core/dbt/clients/agate_helper.py similarity index 100% rename from dbt/clients/agate_helper.py rename to core/dbt/clients/agate_helper.py diff --git a/dbt/clients/gcloud.py b/core/dbt/clients/gcloud.py similarity index 100% rename from dbt/clients/gcloud.py rename to core/dbt/clients/gcloud.py diff --git a/dbt/clients/git.py b/core/dbt/clients/git.py similarity index 100% rename from dbt/clients/git.py rename to core/dbt/clients/git.py diff --git a/dbt/clients/jinja.py b/core/dbt/clients/jinja.py similarity index 100% rename from dbt/clients/jinja.py rename to core/dbt/clients/jinja.py diff --git a/dbt/clients/registry.py b/core/dbt/clients/registry.py similarity index 100% rename from dbt/clients/registry.py rename to core/dbt/clients/registry.py diff --git a/dbt/clients/system.py b/core/dbt/clients/system.py similarity index 100% rename from dbt/clients/system.py rename to core/dbt/clients/system.py diff --git a/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py similarity index 100% rename from dbt/clients/yaml_helper.py rename to core/dbt/clients/yaml_helper.py diff --git a/dbt/compat.py b/core/dbt/compat.py similarity index 100% rename from dbt/compat.py rename to core/dbt/compat.py diff --git a/dbt/compilation.py b/core/dbt/compilation.py similarity index 100% rename from dbt/compilation.py rename to core/dbt/compilation.py diff --git a/core/dbt/config/__init__.py b/core/dbt/config/__init__.py new file mode 100644 index 00000000000..b5280511ef7 --- /dev/null +++ b/core/dbt/config/__init__.py @@ -0,0 +1,22 @@ + +from .renderer import ConfigRenderer +from .profile import Profile, UserConfig +from .project import Project +from .profile import read_profile +from .profile import PROFILES_DIR +from .runtime import RuntimeConfig + + +def read_profiles(profiles_dir=None): + """This is only used in main, for some error handling""" + if profiles_dir is None: + profiles_dir = PROFILES_DIR + + raw_profiles = read_profile(profiles_dir) + + if raw_profiles is None: + profiles = {} + else: + profiles = {k: v for (k, v) in raw_profiles.items() if k != 'config'} + + return profiles diff --git a/core/dbt/config/profile.py b/core/dbt/config/profile.py new file mode 100644 index 00000000000..bb5c91cc246 --- /dev/null +++ b/core/dbt/config/profile.py @@ -0,0 +1,370 @@ +import os +import pprint + +from dbt.adapters.factory import load_plugin +from dbt.clients.system import load_file_contents +from dbt.clients.yaml_helper import load_yaml_text +from dbt.contracts.project import ProfileConfig +from dbt.exceptions import DbtProfileError +from dbt.exceptions import DbtProjectError +from dbt.exceptions import ValidationException +from dbt.exceptions import RuntimeException +from dbt.logger import GLOBAL_LOGGER as logger +from dbt.utils import parse_cli_vars + +from .renderer import ConfigRenderer + +DEFAULT_THREADS = 1 +DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True +DEFAULT_USE_COLORS = True +DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser('~'), '.dbt') +PROFILES_DIR = os.path.expanduser( + os.environ.get('DBT_PROFILES_DIR', DEFAULT_PROFILES_DIR) +) + +INVALID_PROFILE_MESSAGE = """ +dbt encountered an error while trying to read your profiles.yml file. + +{error_string} +""" + + +NO_SUPPLIED_PROFILE_ERROR = """\ +dbt cannot run because no profile was specified for this dbt project. +To specify a profile for this project, add a line like the this to +your dbt_project.yml file: + +profile: [profile name] + +Here, [profile name] should be replaced with a profile name +defined in your profiles.yml file. You can find profiles.yml here: + +{profiles_file}/profiles.yml +""".format(profiles_file=PROFILES_DIR) + + +def read_profile(profiles_dir): + path = os.path.join(profiles_dir, 'profiles.yml') + + contents = None + if os.path.isfile(path): + try: + contents = load_file_contents(path, strip=False) + return load_yaml_text(contents) + except ValidationException as e: + msg = INVALID_PROFILE_MESSAGE.format(error_string=e) + raise ValidationException(msg) + + return {} + + +class UserConfig(object): + def __init__(self, send_anonymous_usage_stats, use_colors): + self.send_anonymous_usage_stats = send_anonymous_usage_stats + self.use_colors = use_colors + + @classmethod + def from_dict(cls, cfg=None): + if cfg is None: + cfg = {} + send_anonymous_usage_stats = cfg.get( + 'send_anonymous_usage_stats', + DEFAULT_SEND_ANONYMOUS_USAGE_STATS + ) + use_colors = cfg.get( + 'use_colors', + DEFAULT_USE_COLORS + ) + return cls(send_anonymous_usage_stats, use_colors) + + def to_dict(self): + return { + 'send_anonymous_usage_stats': self.send_anonymous_usage_stats, + 'use_colors': self.use_colors, + } + + @classmethod + def from_directory(cls, directory): + user_cfg = None + profile = read_profile(directory) + if profile: + user_cfg = profile.get('config', {}) + return cls.from_dict(user_cfg) + + +class Profile(object): + def __init__(self, profile_name, target_name, config, threads, + credentials): + self.profile_name = profile_name + self.target_name = target_name + if isinstance(config, dict): + config = UserConfig.from_dict(config) + self.config = config + self.threads = threads + self.credentials = credentials + + def to_profile_info(self, serialize_credentials=False): + """Unlike to_project_config, this dict is not a mirror of any existing + on-disk data structure. It's used when creating a new profile from an + existing one. + + :param serialize_credentials bool: If True, serialize the credentials. + Otherwise, the Credentials object will be copied. + :returns dict: The serialized profile. + """ + result = { + 'profile_name': self.profile_name, + 'target_name': self.target_name, + 'config': self.config.to_dict(), + 'threads': self.threads, + 'credentials': self.credentials.incorporate(), + } + if serialize_credentials: + result['credentials'] = result['credentials'].serialize() + return result + + def __str__(self): + return pprint.pformat(self.to_profile_info()) + + def __eq__(self, other): + if not (isinstance(other, self.__class__) and + isinstance(self, other.__class__)): + return False + return False + return self.to_profile_info() == other.to_profile_info() + + def validate(self): + if self.credentials: + self.credentials.validate() + try: + ProfileConfig(**self.to_profile_info(serialize_credentials=True)) + except ValidationException as exc: + raise DbtProfileError(str(exc)) + + @staticmethod + def _credentials_from_profile(profile, profile_name, target_name): + # credentials carry their 'type' in their actual type, not their + # attributes. We do want this in order to pick our Credentials class. + if 'type' not in profile: + raise DbtProfileError( + 'required field "type" not found in profile {} and target {}' + .format(profile_name, target_name)) + + typename = profile.pop('type') + + try: + cls = load_plugin(typename) + credentials = cls(**profile) + except RuntimeException as e: + raise DbtProfileError( + 'Credentials in profile "{}", target "{}" invalid: {}' + .format(profile_name, target_name, str(e)) + ) + return credentials + + @staticmethod + def pick_profile_name(args_profile_name, project_profile_name=None): + profile_name = project_profile_name + if args_profile_name is not None: + profile_name = args_profile_name + if profile_name is None: + raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR) + return profile_name + + @staticmethod + def _get_profile_data(profile, profile_name, target_name): + if 'outputs' not in profile: + raise DbtProfileError( + "outputs not specified in profile '{}'".format(profile_name) + ) + outputs = profile['outputs'] + + if target_name not in outputs: + outputs = '\n'.join(' - {}'.format(output) + for output in outputs) + msg = ("The profile '{}' does not have a target named '{}'. The " + "valid target names for this profile are:\n{}" + .format(profile_name, target_name, outputs)) + raise DbtProfileError(msg, result_type='invalid_target') + profile_data = outputs[target_name] + return profile_data + + @classmethod + def from_credentials(cls, credentials, threads, profile_name, target_name, + user_cfg=None): + """Create a profile from an existing set of Credentials and the + remaining information. + + :param credentials dict: The credentials dict for this profile. + :param threads int: The number of threads to use for connections. + :param profile_name str: The profile name used for this profile. + :param target_name str: The target name used for this profile. + :param user_cfg Optional[dict]: The user-level config block from the + raw profiles, if specified. + :raises DbtProfileError: If the profile is invalid. + :returns Profile: The new Profile object. + """ + config = UserConfig.from_dict(user_cfg) + profile = cls( + profile_name=profile_name, + target_name=target_name, + config=config, + threads=threads, + credentials=credentials + ) + profile.validate() + return profile + + @classmethod + def render_profile(cls, raw_profile, profile_name, target_override, + cli_vars): + """This is a containment zone for the hateful way we're rendering + profiles. + """ + renderer = ConfigRenderer(cli_vars=cli_vars) + + # rendering profiles is a bit complex. Two constraints cause trouble: + # 1) users should be able to use environment/cli variables to specify + # the target in their profile. + # 2) Missing environment/cli variables in profiles/targets that don't + # end up getting selected should not cause errors. + # so first we'll just render the target name, then we use that rendered + # name to extract a profile that we can render. + if target_override is not None: + target_name = target_override + elif 'target' in raw_profile: + # render the target if it was parsed from yaml + target_name = renderer.render_value(raw_profile['target']) + else: + target_name = 'default' + logger.debug( + "target not specified in profile '{}', using '{}'" + .format(profile_name, target_name) + ) + + raw_profile_data = cls._get_profile_data( + raw_profile, profile_name, target_name + ) + + profile_data = renderer.render_profile_data(raw_profile_data) + return target_name, profile_data + + @classmethod + def from_raw_profile_info(cls, raw_profile, profile_name, cli_vars, + user_cfg=None, target_override=None, + threads_override=None): + """Create a profile from its raw profile information. + + (this is an intermediate step, mostly useful for unit testing) + + :param raw_profile dict: The profile data for a single profile, from + disk as yaml and its values rendered with jinja. + :param profile_name str: The profile name used. + :param cli_vars dict: The command-line variables passed as arguments, + as a dict. + :param user_cfg Optional[dict]: The global config for the user, if it + was present. + :param target_override Optional[str]: The target to use, if provided on + the command line. + :param threads_override Optional[str]: The thread count to use, if + provided on the command line. + :raises DbtProfileError: If the profile is invalid or missing, or the + target could not be found + :returns Profile: The new Profile object. + """ + # user_cfg is not rendered since it only contains booleans. + # TODO: should it be, and the values coerced to bool? + target_name, profile_data = cls.render_profile( + raw_profile, profile_name, target_override, cli_vars + ) + + # valid connections never include the number of threads, but it's + # stored on a per-connection level in the raw configs + threads = profile_data.pop('threads', DEFAULT_THREADS) + if threads_override is not None: + threads = threads_override + + credentials = cls._credentials_from_profile( + profile_data, profile_name, target_name + ) + + return cls.from_credentials( + credentials=credentials, + profile_name=profile_name, + target_name=target_name, + threads=threads, + user_cfg=user_cfg + ) + + @classmethod + def from_raw_profiles(cls, raw_profiles, profile_name, cli_vars, + target_override=None, threads_override=None): + """ + :param raw_profiles dict: The profile data, from disk as yaml. + :param profile_name str: The profile name to use. + :param cli_vars dict: The command-line variables passed as arguments, + as a dict. + :param target_override Optional[str]: The target to use, if provided on + the command line. + :param threads_override Optional[str]: The thread count to use, if + provided on the command line. + :raises DbtProjectError: If there is no profile name specified in the + project or the command line arguments + :raises DbtProfileError: If the profile is invalid or missing, or the + target could not be found + :returns Profile: The new Profile object. + """ + if profile_name not in raw_profiles: + raise DbtProjectError( + "Could not find profile named '{}'".format(profile_name) + ) + + # First, we've already got our final decision on profile name, and we + # don't render keys, so we can pluck that out + raw_profile = raw_profiles[profile_name] + + user_cfg = raw_profiles.get('config') + + return cls.from_raw_profile_info( + raw_profile=raw_profile, + profile_name=profile_name, + cli_vars=cli_vars, + user_cfg=user_cfg, + target_override=target_override, + threads_override=threads_override, + ) + + @classmethod + def from_args(cls, args, project_profile_name=None, cli_vars=None): + """Given the raw profiles as read from disk and the name of the desired + profile if specified, return the profile component of the runtime + config. + + :param args argparse.Namespace: The arguments as parsed from the cli. + :param cli_vars dict: The command-line variables passed as arguments, + as a dict. + :param project_profile_name Optional[str]: The profile name, if + specified in a project. + :raises DbtProjectError: If there is no profile name specified in the + project or the command line arguments, or if the specified profile + is not found + :raises DbtProfileError: If the profile is invalid or missing, or the + target could not be found. + :returns Profile: The new Profile object. + """ + if cli_vars is None: + cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) + + threads_override = getattr(args, 'threads', None) + target_override = getattr(args, 'target', None) + raw_profiles = read_profile(args.profiles_dir) + profile_name = cls.pick_profile_name(args.profile, + project_profile_name) + + return cls.from_raw_profiles( + raw_profiles=raw_profiles, + profile_name=profile_name, + cli_vars=cli_vars, + target_override=target_override, + threads_override=threads_override + ) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py new file mode 100644 index 00000000000..8366755ef89 --- /dev/null +++ b/core/dbt/config/project.py @@ -0,0 +1,442 @@ + +from copy import deepcopy +import hashlib +import os +import pprint + +from dbt import compat +from dbt.clients.system import resolve_path_from_base +from dbt.clients.system import path_exists +from dbt.clients.system import load_file_contents +from dbt.clients.yaml_helper import load_yaml_text +from dbt.exceptions import DbtProjectError +from dbt.exceptions import RecursionException +from dbt.exceptions import SemverException +from dbt.exceptions import ValidationException +from dbt.logger import GLOBAL_LOGGER as logger +from dbt.semver import VersionSpecifier +from dbt.semver import versions_compatible +from dbt.version import get_installed_version +from dbt.ui import printer +from dbt.utils import deep_map +from dbt.utils import parse_cli_vars +from dbt.utils import DBTConfigKeys + +from dbt.contracts.project import Project as ProjectContract +from dbt.contracts.project import PackageConfig + +from .renderer import ConfigRenderer + + +UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE = """\ +WARNING: Configuration paths exist in your dbt_project.yml file which do not \ +apply to any resources. +There are {} unused configuration paths:\n{} +""" + + +INVALID_VERSION_ERROR = """\ +This version of dbt is not supported with the '{package}' package. + Installed version of dbt: {installed} + Required version of dbt for '{package}': {version_spec} +Check the requirements for the '{package}' package, or run dbt again with \ +--no-version-check +""" + + +IMPOSSIBLE_VERSION_ERROR = """\ +The package version requirement can never be satisfied for the '{package} +package. + Required versions of dbt for '{package}': {version_spec} +Check the requirements for the '{package}' package, or run dbt again with \ +--no-version-check +""" + + +def _list_if_none(value): + if value is None: + value = [] + return value + + +def _dict_if_none(value): + if value is None: + value = {} + return value + + +def _list_if_none_or_string(value): + value = _list_if_none(value) + if isinstance(value, compat.basestring): + return [value] + return value + + +def _load_yaml(path): + contents = load_file_contents(path) + return load_yaml_text(contents) + + +def _get_config_paths(config, path=(), paths=None): + if paths is None: + paths = set() + + for key, value in config.items(): + if isinstance(value, dict): + if key in DBTConfigKeys: + if path not in paths: + paths.add(path) + else: + _get_config_paths(value, path + (key,), paths) + else: + if path not in paths: + paths.add(path) + + return frozenset(paths) + + +def _is_config_used(path, fqns): + if fqns: + for fqn in fqns: + if len(path) <= len(fqn) and fqn[:len(path)] == path: + return True + return False + + +def package_data_from_root(project_root): + package_filepath = resolve_path_from_base( + 'packages.yml', project_root + ) + + if path_exists(package_filepath): + packages_dict = _load_yaml(package_filepath) + else: + packages_dict = None + return packages_dict + + +def package_config_from_data(packages_data): + if packages_data is None: + packages_data = {'packages': []} + + try: + packages = PackageConfig(**packages_data) + except ValidationException as e: + raise DbtProjectError('Invalid package config: {}'.format(str(e))) + return packages + + +def _parse_versions(versions): + """Parse multiple versions as read from disk. The versions value may be any + one of: + - a single version string ('>0.12.1') + - a single string specifying multiple comma-separated versions + ('>0.11.1,<=0.12.2') + - an array of single-version strings (['>0.11.1', '<=0.12.2']) + + Regardless, this will return a list of VersionSpecifiers + """ + if isinstance(versions, compat.basestring): + versions = versions.split(',') + return [VersionSpecifier.from_version_string(v) for v in versions] + + +class Project(object): + def __init__(self, project_name, version, project_root, profile_name, + source_paths, macro_paths, data_paths, test_paths, + analysis_paths, docs_paths, target_path, clean_targets, + log_path, modules_path, quoting, models, on_run_start, + on_run_end, archive, seeds, dbt_version, packages): + self.project_name = project_name + self.version = version + self.project_root = project_root + self.profile_name = profile_name + self.source_paths = source_paths + self.macro_paths = macro_paths + self.data_paths = data_paths + self.test_paths = test_paths + self.analysis_paths = analysis_paths + self.docs_paths = docs_paths + self.target_path = target_path + self.clean_targets = clean_targets + self.log_path = log_path + self.modules_path = modules_path + self.quoting = quoting + self.models = models + self.on_run_start = on_run_start + self.on_run_end = on_run_end + self.archive = archive + self.seeds = seeds + self.dbt_version = dbt_version + self.packages = packages + + @staticmethod + def _preprocess(project_dict): + """Pre-process certain special keys to convert them from None values + into empty containers, and to turn strings into arrays of strings. + """ + handlers = { + ('archive',): _list_if_none, + ('on-run-start',): _list_if_none_or_string, + ('on-run-end',): _list_if_none_or_string, + } + + for k in ('models', 'seeds'): + handlers[(k,)] = _dict_if_none + handlers[(k, 'vars')] = _dict_if_none + handlers[(k, 'pre-hook')] = _list_if_none_or_string + handlers[(k, 'post-hook')] = _list_if_none_or_string + handlers[('seeds', 'column_types')] = _dict_if_none + + def converter(value, keypath): + if keypath in handlers: + handler = handlers[keypath] + return handler(value) + else: + return value + + return deep_map(converter, project_dict) + + @classmethod + def from_project_config(cls, project_dict, packages_dict=None): + """Create a project from its project and package configuration, as read + by yaml.safe_load(). + + :param project_dict dict: The dictionary as read from disk + :param packages_dict Optional[dict]: If it exists, the packages file as + read from disk. + :raises DbtProjectError: If the project is missing or invalid, or if + the packages file exists and is invalid. + :returns Project: The project, with defaults populated. + """ + try: + project_dict = cls._preprocess(project_dict) + except RecursionException: + raise DbtProjectError( + 'Cycle detected: Project input has a reference to itself', + project=project_dict + ) + # just for validation. + try: + ProjectContract(**project_dict) + except ValidationException as e: + raise DbtProjectError(str(e)) + + # name/version are required in the Project definition, so we can assume + # they are present + name = project_dict['name'] + version = project_dict['version'] + # this is added at project_dict parse time and should always be here + # once we see it. + project_root = project_dict['project-root'] + # this is only optional in the sense that if it's not present, it needs + # to have been a cli argument. + profile_name = project_dict.get('profile') + # these are optional + source_paths = project_dict.get('source-paths', ['models']) + macro_paths = project_dict.get('macro-paths', ['macros']) + data_paths = project_dict.get('data-paths', ['data']) + test_paths = project_dict.get('test-paths', ['test']) + analysis_paths = project_dict.get('analysis-paths', []) + docs_paths = project_dict.get('docs-paths', source_paths[:]) + target_path = project_dict.get('target-path', 'target') + # should this also include the modules path by default? + clean_targets = project_dict.get('clean-targets', [target_path]) + log_path = project_dict.get('log-path', 'logs') + modules_path = project_dict.get('modules-path', 'dbt_modules') + # in the default case we'll populate this once we know the adapter type + quoting = project_dict.get('quoting', {}) + + models = project_dict.get('models', {}) + on_run_start = project_dict.get('on-run-start', []) + on_run_end = project_dict.get('on-run-end', []) + archive = project_dict.get('archive', []) + seeds = project_dict.get('seeds', {}) + dbt_raw_version = project_dict.get('require-dbt-version', '>=0.0.0') + + try: + dbt_version = _parse_versions(dbt_raw_version) + except SemverException as e: + raise DbtProjectError(str(e)) + + packages = package_config_from_data(packages_dict) + + project = cls( + project_name=name, + version=version, + project_root=project_root, + profile_name=profile_name, + source_paths=source_paths, + macro_paths=macro_paths, + data_paths=data_paths, + test_paths=test_paths, + analysis_paths=analysis_paths, + docs_paths=docs_paths, + target_path=target_path, + clean_targets=clean_targets, + log_path=log_path, + modules_path=modules_path, + quoting=quoting, + models=models, + on_run_start=on_run_start, + on_run_end=on_run_end, + archive=archive, + seeds=seeds, + dbt_version=dbt_version, + packages=packages + ) + # sanity check - this means an internal issue + project.validate() + return project + + def __str__(self): + cfg = self.to_project_config(with_packages=True) + return pprint.pformat(cfg) + + def __eq__(self, other): + if not (isinstance(other, self.__class__) and + isinstance(self, other.__class__)): + return False + return self.to_project_config(with_packages=True) == \ + other.to_project_config(with_packages=True) + + def to_project_config(self, with_packages=False): + """Return a dict representation of the config that could be written to + disk with `yaml.safe_dump` to get this configuration. + + :param with_packages bool: If True, include the serialized packages + file in the root. + :returns dict: The serialized profile. + """ + result = deepcopy({ + 'name': self.project_name, + 'version': self.version, + 'project-root': self.project_root, + 'profile': self.profile_name, + 'source-paths': self.source_paths, + 'macro-paths': self.macro_paths, + 'data-paths': self.data_paths, + 'test-paths': self.test_paths, + 'analysis-paths': self.analysis_paths, + 'docs-paths': self.docs_paths, + 'target-path': self.target_path, + 'clean-targets': self.clean_targets, + 'log-path': self.log_path, + 'quoting': self.quoting, + 'models': self.models, + 'on-run-start': self.on_run_start, + 'on-run-end': self.on_run_end, + 'archive': self.archive, + 'seeds': self.seeds, + 'require-dbt-version': [ + v.to_version_string() for v in self.dbt_version + ], + }) + if with_packages: + result.update(self.packages.serialize()) + return result + + def validate(self): + try: + ProjectContract(**self.to_project_config()) + except ValidationException as exc: + raise DbtProjectError(str(exc)) + + @classmethod + def from_project_root(cls, project_root, cli_vars): + """Create a project from a root directory. Reads in dbt_project.yml and + packages.yml, if it exists. + + :param project_root str: The path to the project root to load. + :raises DbtProjectError: If the project is missing or invalid, or if + the packages file exists and is invalid. + :returns Project: The project, with defaults populated. + """ + project_root = os.path.normpath(project_root) + project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml') + + # get the project.yml contents + if not path_exists(project_yaml_filepath): + raise DbtProjectError( + 'no dbt_project.yml found at expected path {}' + .format(project_yaml_filepath) + ) + + if isinstance(cli_vars, compat.basestring): + cli_vars = parse_cli_vars(cli_vars) + renderer = ConfigRenderer(cli_vars) + + project_dict = _load_yaml(project_yaml_filepath) + rendered_project = renderer.render_project(project_dict) + rendered_project['project-root'] = project_root + packages_dict = package_data_from_root(project_root) + return cls.from_project_config(rendered_project, packages_dict) + + @classmethod + def from_current_directory(cls, cli_vars): + return cls.from_project_root(os.getcwd(), cli_vars) + + def hashed_name(self): + return hashlib.md5(self.project_name.encode('utf-8')).hexdigest() + + def get_resource_config_paths(self): + """Return a dictionary with 'seeds' and 'models' keys whose values are + lists of lists of strings, where each inner list of strings represents + a configured path in the resource. + """ + return { + 'models': _get_config_paths(self.models), + 'seeds': _get_config_paths(self.seeds), + } + + def get_unused_resource_config_paths(self, resource_fqns, disabled): + """Return a list of lists of strings, where each inner list of strings + represents a type + FQN path of a resource configuration that is not + used. + """ + disabled_fqns = frozenset(tuple(fqn) for fqn in disabled) + resource_config_paths = self.get_resource_config_paths() + unused_resource_config_paths = [] + for resource_type, config_paths in resource_config_paths.items(): + used_fqns = resource_fqns.get(resource_type, frozenset()) + fqns = used_fqns | disabled_fqns + + for config_path in config_paths: + if not _is_config_used(config_path, fqns): + unused_resource_config_paths.append( + (resource_type,) + config_path + ) + return unused_resource_config_paths + + def warn_for_unused_resource_config_paths(self, resource_fqns, disabled): + unused = self.get_unused_resource_config_paths(resource_fqns, disabled) + if len(unused) == 0: + return + + msg = UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE.format( + len(unused), + '\n'.join('- {}'.format('.'.join(u)) for u in unused) + ) + logger.info(printer.yellow(msg)) + + def validate_version(self): + """Ensure this package works with the installed version of dbt.""" + installed = get_installed_version() + if not versions_compatible(*self.dbt_version): + msg = IMPOSSIBLE_VERSION_ERROR.format( + package=self.project_name, + version_spec=[ + x.to_version_string() for x in self.dbt_version + ] + ) + raise DbtProjectError(msg) + + if not versions_compatible(installed, *self.dbt_version): + msg = INVALID_VERSION_ERROR.format( + package=self.project_name, + installed=installed.to_version_string(), + version_spec=[ + x.to_version_string() for x in self.dbt_version + ] + ) + raise DbtProjectError(msg) diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py new file mode 100644 index 00000000000..35c66ae880a --- /dev/null +++ b/core/dbt/config/renderer.py @@ -0,0 +1,95 @@ +from dbt import compat +from dbt.clients.jinja import get_rendered +from dbt.context.common import env_var +from dbt.context.common import Var +from dbt.exceptions import DbtProfileError +from dbt.exceptions import DbtProjectError +from dbt.exceptions import RecursionException +from dbt.utils import deep_map + + +class ConfigRenderer(object): + """A renderer provides configuration rendering for a given set of cli + variables and a render type. + """ + def __init__(self, cli_vars): + self.context = {'env_var': env_var} + self.context['var'] = Var(None, self.context, cli_vars) + + @staticmethod + def _is_hook_or_model_vars_path(keypath): + if not keypath: + return False + + first = keypath[0] + # run hooks + if first in {'on-run-start', 'on-run-end'}: + return True + # models have two things to avoid + if first in {'seeds', 'models'}: + # model-level hooks + if 'pre-hook' in keypath or 'post-hook' in keypath: + return True + # model-level 'vars' declarations + if 'vars' in keypath: + return True + + return False + + def _render_project_entry(self, value, keypath): + """Render an entry, in case it's jinja. This is meant to be passed to + deep_map. + + If the parsed entry is a string and has the name 'port', this will + attempt to cast it to an int, and on failure will return the parsed + string. + + :param value Any: The value to potentially render + :param key str: The key to convert on. + :return Any: The rendered entry. + """ + # hooks should be treated as raw sql, they'll get rendered later. + # Same goes for 'vars' declarations inside 'models'/'seeds'. + if self._is_hook_or_model_vars_path(keypath): + return value + + return self.render_value(value) + + def render_value(self, value, keypath=None): + # keypath is ignored. + # if it wasn't read as a string, ignore it + if not isinstance(value, compat.basestring): + return value + + return get_rendered(value, self.context) + + def _render_profile_data(self, value, keypath): + result = self.render_value(value) + if len(keypath) == 1 and keypath[-1] == 'port': + try: + result = int(result) + except ValueError: + # let the validator or connection handle this + pass + return result + + def render_project(self, as_parsed): + """Render the parsed data, returning a new dict (or whatever was read). + """ + try: + return deep_map(self._render_project_entry, as_parsed) + except RecursionException: + raise DbtProjectError( + 'Cycle detected: Project input has a reference to itself', + project=as_parsed + ) + + def render_profile_data(self, as_parsed): + """Render the chosen profile entry, as it was parsed.""" + try: + return deep_map(self._render_profile_data, as_parsed) + except RecursionException: + raise DbtProfileError( + 'Cycle detected: Profile input has a reference to itself', + project=as_parsed + ) diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py new file mode 100644 index 00000000000..ee654474a5b --- /dev/null +++ b/core/dbt/config/runtime.py @@ -0,0 +1,190 @@ + +from copy import deepcopy +import pprint + +from dbt.utils import parse_cli_vars +from dbt.contracts.project import Configuration +from dbt.exceptions import DbtProjectError +from dbt.exceptions import ValidationException +from dbt.adapters.factory import get_relation_class_by_name + +from .profile import Profile +from .project import Project + + +class RuntimeConfig(Project, Profile): + """The runtime configuration, as constructed from its components. There's a + lot because there is a lot of stuff! + """ + def __init__(self, project_name, version, project_root, source_paths, + macro_paths, data_paths, test_paths, analysis_paths, + docs_paths, target_path, clean_targets, log_path, + modules_path, quoting, models, on_run_start, on_run_end, + archive, seeds, dbt_version, profile_name, target_name, + config, threads, credentials, packages, args): + # 'vars' + self.args = args + self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) + # 'project' + Project.__init__( + self, + project_name=project_name, + version=version, + project_root=project_root, + profile_name=profile_name, + source_paths=source_paths, + macro_paths=macro_paths, + data_paths=data_paths, + test_paths=test_paths, + analysis_paths=analysis_paths, + docs_paths=docs_paths, + target_path=target_path, + clean_targets=clean_targets, + log_path=log_path, + modules_path=modules_path, + quoting=quoting, + models=models, + on_run_start=on_run_start, + on_run_end=on_run_end, + archive=archive, + seeds=seeds, + dbt_version=dbt_version, + packages=packages + ) + # 'profile' + Profile.__init__( + self, + profile_name=profile_name, + target_name=target_name, + config=config, + threads=threads, + credentials=credentials + ) + self.validate() + + @classmethod + def from_parts(cls, project, profile, args): + """Instantiate a RuntimeConfig from its components. + + :param profile Profile: A parsed dbt Profile. + :param project Project: A parsed dbt Project. + :param args argparse.Namespace: The parsed command-line arguments. + :returns RuntimeConfig: The new configuration. + """ + quoting = deepcopy( + get_relation_class_by_name(profile.credentials.type) + .DEFAULTS['quote_policy'] + ) + quoting.update(project.quoting) + return cls( + project_name=project.project_name, + version=project.version, + project_root=project.project_root, + source_paths=project.source_paths, + macro_paths=project.macro_paths, + data_paths=project.data_paths, + test_paths=project.test_paths, + analysis_paths=project.analysis_paths, + docs_paths=project.docs_paths, + target_path=project.target_path, + clean_targets=project.clean_targets, + log_path=project.log_path, + modules_path=project.modules_path, + quoting=quoting, + models=project.models, + on_run_start=project.on_run_start, + on_run_end=project.on_run_end, + archive=project.archive, + seeds=project.seeds, + dbt_version=project.dbt_version, + packages=project.packages, + profile_name=profile.profile_name, + target_name=profile.target_name, + config=profile.config, + threads=profile.threads, + credentials=profile.credentials, + args=args + ) + + def new_project(self, project_root): + """Given a new project root, read in its project dictionary, supply the + existing project's profile info, and create a new project file. + + :param project_root str: A filepath to a dbt project. + :raises DbtProfileError: If the profile is invalid. + :raises DbtProjectError: If project is missing or invalid. + :returns RuntimeConfig: The new configuration. + """ + # copy profile + profile = Profile(**self.to_profile_info()) + profile.validate() + # load the new project and its packages. Don't pass cli variables. + project = Project.from_project_root(project_root, {}) + + cfg = self.from_parts( + project=project, + profile=profile, + args=deepcopy(self.args), + ) + # force our quoting back onto the new project. + cfg.quoting = deepcopy(self.quoting) + return cfg + + def serialize(self): + """Serialize the full configuration to a single dictionary. For any + instance that has passed validate() (which happens in __init__), it + matches the Configuration contract. + + Note that args are not serialized. + + :returns dict: The serialized configuration. + """ + result = self.to_project_config(with_packages=True) + result.update(self.to_profile_info(serialize_credentials=True)) + result['cli_vars'] = deepcopy(self.cli_vars) + return result + + def __str__(self): + return pprint.pformat(self.serialize()) + + def validate(self): + """Validate the configuration against its contract. + + :raises DbtProjectError: If the configuration fails validation. + """ + try: + Configuration(**self.serialize()) + except ValidationException as e: + raise DbtProjectError(str(e)) + + if getattr(self.args, 'version_check', False): + self.validate_version() + + @classmethod + def from_args(cls, args): + """Given arguments, read in dbt_project.yml from the current directory, + read in packages.yml if it exists, and use them to find the profile to + load. + + :param args argparse.Namespace: The arguments as parsed from the cli. + :raises DbtProjectError: If the project is invalid or missing. + :raises DbtProfileError: If the profile is invalid or missing. + :raises ValidationException: If the cli variables are invalid. + """ + cli_vars = parse_cli_vars(getattr(args, 'vars', '{}')) + + # build the project and read in packages.yml + project = Project.from_current_directory(cli_vars) + + # build the profile + profile = Profile.from_args( + args=args, + project_profile_name=project.profile_name, + cli_vars=cli_vars + ) + + return cls.from_parts( + project=project, + profile=profile, + args=args + ) diff --git a/dbt/adapters/__init__.py b/core/dbt/context/__init__.py similarity index 100% rename from dbt/adapters/__init__.py rename to core/dbt/context/__init__.py diff --git a/dbt/context/common.py b/core/dbt/context/common.py similarity index 95% rename from dbt/context/common.py rename to core/dbt/context/common.py index d55e43d049d..61b7aeab66d 100644 --- a/dbt/context/common.py +++ b/core/dbt/context/common.py @@ -7,6 +7,8 @@ from dbt.compat import basestring from dbt.node_types import NodeType from dbt.contracts.graph.parsed import ParsedMacro, ParsedNode +from dbt.include.global_project import PACKAGES +from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME import dbt.clients.jinja import dbt.clients.agate_helper @@ -84,6 +86,20 @@ def commit(self): return self.adapter.commit_if_has_connection(self.model.get('name')) +def _add_macro_map(context, package_name, macro_map): + """Update an existing context in-place, adding the given macro map to the + appropriate package namespace. Adapter packages get inserted into the + global namespace. + """ + key = package_name + if package_name in PACKAGES: + key = GLOBAL_PROJECT_NAME + if key not in context: + context[key] = {} + + context[key].update(macro_map) + + def _add_macros(context, model, manifest): macros_to_add = {'global': [], 'local': []} @@ -96,15 +112,12 @@ def _add_macros(context, model, manifest): macro.name: macro.generator(context) } - if context.get(package_name) is None: - context[package_name] = {} - - context.get(package_name, {}) \ - .update(macro_map) + # adapter packages are part of the global project space + _add_macro_map(context, package_name, macro_map) if package_name == model.package_name: macros_to_add['local'].append(macro_map) - elif package_name == dbt.include.GLOBAL_PROJECT_NAME: + elif package_name in PACKAGES: macros_to_add['global'].append(macro_map) # Load global macros before local macros -- local takes precedence diff --git a/dbt/context/parser.py b/core/dbt/context/parser.py similarity index 100% rename from dbt/context/parser.py rename to core/dbt/context/parser.py diff --git a/dbt/context/runtime.py b/core/dbt/context/runtime.py similarity index 100% rename from dbt/context/runtime.py rename to core/dbt/context/runtime.py diff --git a/dbt/clients/__init__.py b/core/dbt/contracts/__init__.py similarity index 100% rename from dbt/clients/__init__.py rename to core/dbt/contracts/__init__.py diff --git a/dbt/contracts/common.py b/core/dbt/contracts/common.py similarity index 100% rename from dbt/contracts/common.py rename to core/dbt/contracts/common.py diff --git a/dbt/contracts/connection.py b/core/dbt/contracts/connection.py similarity index 100% rename from dbt/contracts/connection.py rename to core/dbt/contracts/connection.py diff --git a/dbt/context/__init__.py b/core/dbt/contracts/graph/__init__.py similarity index 100% rename from dbt/context/__init__.py rename to core/dbt/contracts/graph/__init__.py diff --git a/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py similarity index 100% rename from dbt/contracts/graph/compiled.py rename to core/dbt/contracts/graph/compiled.py diff --git a/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py similarity index 100% rename from dbt/contracts/graph/manifest.py rename to core/dbt/contracts/graph/manifest.py diff --git a/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py similarity index 100% rename from dbt/contracts/graph/parsed.py rename to core/dbt/contracts/graph/parsed.py diff --git a/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py similarity index 100% rename from dbt/contracts/graph/unparsed.py rename to core/dbt/contracts/graph/unparsed.py diff --git a/dbt/contracts/project.py b/core/dbt/contracts/project.py similarity index 100% rename from dbt/contracts/project.py rename to core/dbt/contracts/project.py diff --git a/dbt/contracts/results.py b/core/dbt/contracts/results.py similarity index 100% rename from dbt/contracts/results.py rename to core/dbt/contracts/results.py diff --git a/dbt/deprecations.py b/core/dbt/deprecations.py similarity index 100% rename from dbt/deprecations.py rename to core/dbt/deprecations.py diff --git a/dbt/exceptions.py b/core/dbt/exceptions.py similarity index 100% rename from dbt/exceptions.py rename to core/dbt/exceptions.py diff --git a/dbt/flags.py b/core/dbt/flags.py similarity index 100% rename from dbt/flags.py rename to core/dbt/flags.py diff --git a/dbt/contracts/__init__.py b/core/dbt/graph/__init__.py similarity index 100% rename from dbt/contracts/__init__.py rename to core/dbt/graph/__init__.py diff --git a/dbt/graph/selector.py b/core/dbt/graph/selector.py similarity index 100% rename from dbt/graph/selector.py rename to core/dbt/graph/selector.py diff --git a/dbt/hooks.py b/core/dbt/hooks.py similarity index 100% rename from dbt/hooks.py rename to core/dbt/hooks.py diff --git a/core/dbt/include/__init__.py b/core/dbt/include/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/core/dbt/include/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/core/dbt/include/global_project/__init__.py b/core/dbt/include/global_project/__init__.py new file mode 100644 index 00000000000..d0415108083 --- /dev/null +++ b/core/dbt/include/global_project/__init__.py @@ -0,0 +1,11 @@ +import os + +PACKAGE_PATH = os.path.dirname(os.path.dirname(__file__)) +PROJECT_NAME = 'dbt' + +DOCS_INDEX_FILE_PATH = os.path.normpath( + os.path.join(PACKAGE_PATH, "index.html")) + + +# Adapter registration will add to this +PACKAGES = {PROJECT_NAME: PACKAGE_PATH} diff --git a/dbt/include/global_project/dbt_project.yml b/core/dbt/include/global_project/dbt_project.yml similarity index 100% rename from dbt/include/global_project/dbt_project.yml rename to core/dbt/include/global_project/dbt_project.yml diff --git a/dbt/include/global_project/docs/overview.md b/core/dbt/include/global_project/docs/overview.md similarity index 100% rename from dbt/include/global_project/docs/overview.md rename to core/dbt/include/global_project/docs/overview.md diff --git a/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql similarity index 100% rename from dbt/include/global_project/macros/adapters/common.sql rename to core/dbt/include/global_project/macros/adapters/common.sql diff --git a/dbt/include/global_project/macros/core.sql b/core/dbt/include/global_project/macros/core.sql similarity index 100% rename from dbt/include/global_project/macros/core.sql rename to core/dbt/include/global_project/macros/core.sql diff --git a/dbt/include/global_project/macros/etc/datetime.sql b/core/dbt/include/global_project/macros/etc/datetime.sql similarity index 100% rename from dbt/include/global_project/macros/etc/datetime.sql rename to core/dbt/include/global_project/macros/etc/datetime.sql diff --git a/dbt/include/global_project/macros/etc/get_custom_schema.sql b/core/dbt/include/global_project/macros/etc/get_custom_schema.sql similarity index 100% rename from dbt/include/global_project/macros/etc/get_custom_schema.sql rename to core/dbt/include/global_project/macros/etc/get_custom_schema.sql diff --git a/dbt/include/global_project/macros/etc/is_incremental.sql b/core/dbt/include/global_project/macros/etc/is_incremental.sql similarity index 100% rename from dbt/include/global_project/macros/etc/is_incremental.sql rename to core/dbt/include/global_project/macros/etc/is_incremental.sql diff --git a/dbt/include/global_project/macros/materializations/archive/archive.sql b/core/dbt/include/global_project/macros/materializations/archive/archive.sql similarity index 89% rename from dbt/include/global_project/macros/materializations/archive/archive.sql rename to core/dbt/include/global_project/macros/materializations/archive/archive.sql index 06eabcade2e..1fb266535de 100644 --- a/dbt/include/global_project/macros/materializations/archive/archive.sql +++ b/core/dbt/include/global_project/macros/materializations/archive/archive.sql @@ -10,10 +10,6 @@ md5("dbt_pk" || '|' || "dbt_updated_at") {% endmacro %} -{% macro bigquery__archive_scd_hash() %} - to_hex(md5(concat(cast(`dbt_pk` as string), '|', cast(`dbt_updated_at` as string)))) -{% endmacro %} - {% macro create_temporary_table(sql, relation) %} {{ return(adapter_macro('create_temporary_table', sql, relation)) }} {% endmacro %} @@ -25,12 +21,6 @@ {{ return(relation) }} {% endmacro %} -{% macro bigquery__create_temporary_table(sql, relation) %} - {% set tmp_relation = adapter.create_temporary_table(sql) %} - {{ return(tmp_relation) }} -{% endmacro %} - - {# Add new columns to the table if applicable #} @@ -46,10 +36,6 @@ {% endfor %} {% endmacro %} -{% macro bigquery__create_columns(relation, columns) %} - {{ adapter.alter_table_add_columns(relation, columns) }} -{% endmacro %} - {# Run the update part of an archive query. Different databases have tricky differences in their `update` semantics. Table projection is @@ -68,14 +54,6 @@ and {{ adapter.quote('change_type') }} = 'update'; {% endmacro %} -{% macro bigquery__archive_update(target_relation, tmp_relation) %} - update {{ target_relation }} as dest - set dest.{{ adapter.quote('valid_to') }} = tmp.{{ adapter.quote('valid_to') }} - from {{ tmp_relation }} as tmp - where tmp.{{ adapter.quote('scd_id') }} = dest.{{ adapter.quote('scd_id') }} - and {{ adapter.quote('change_type') }} = 'update'; -{% endmacro %} - {# Cross-db compatible archival implementation diff --git a/dbt/include/global_project/macros/materializations/common/merge.sql b/core/dbt/include/global_project/macros/materializations/common/merge.sql similarity index 73% rename from dbt/include/global_project/macros/materializations/common/merge.sql rename to core/dbt/include/global_project/macros/materializations/common/merge.sql index bc537925abd..9903f7afd4e 100644 --- a/dbt/include/global_project/macros/materializations/common/merge.sql +++ b/core/dbt/include/global_project/macros/materializations/common/merge.sql @@ -4,7 +4,8 @@ {{ adapter_macro('get_merge_sql', target, source, unique_key, dest_columns) }} {%- endmacro %} -{% macro default__get_merge_sql(target, source, unique_key, dest_columns) -%} + +{% macro common_get_merge_sql(target, source, unique_key, dest_columns) -%} {%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%} merge into {{ target }} as DBT_INTERNAL_DEST @@ -31,10 +32,12 @@ {% endmacro %} -{% macro redshift__get_merge_sql(target, source, unique_key, dest_columns) -%} - {{ exceptions.raise_compiler_error('get_merge_sql is not implemented for Redshift') }} -{% endmacro %} +{% macro default__get_merge_sql(target, source, unique_key, dest_columns) -%} + {% set typename = adapter.type() %} + + {{ exceptions.raise_compiler_error( + 'get_merge_sql is not implemented for {}'.format(typename) + ) + }} -{% macro postgres__get_merge_sql(target, source, unique_key, dest_columns) -%} - {{ exceptions.raise_compiler_error('get_merge_sql is not implemented for Postgres') }} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/helpers.sql b/core/dbt/include/global_project/macros/materializations/helpers.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/helpers.sql rename to core/dbt/include/global_project/macros/materializations/helpers.sql diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/incremental/incremental.sql rename to core/dbt/include/global_project/macros/materializations/incremental/incremental.sql diff --git a/dbt/include/global_project/macros/materializations/seed/seed.sql b/core/dbt/include/global_project/macros/materializations/seed/seed.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/seed/seed.sql rename to core/dbt/include/global_project/macros/materializations/seed/seed.sql diff --git a/dbt/include/global_project/macros/materializations/table/table.sql b/core/dbt/include/global_project/macros/materializations/table/table.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/table/table.sql rename to core/dbt/include/global_project/macros/materializations/table/table.sql diff --git a/dbt/include/global_project/macros/materializations/view/bq_snowflake_view.sql b/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql similarity index 82% rename from dbt/include/global_project/macros/materializations/view/bq_snowflake_view.sql rename to core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql index d5ceafec383..20b2d5df9db 100644 --- a/dbt/include/global_project/macros/materializations/view/bq_snowflake_view.sql +++ b/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql @@ -9,15 +9,6 @@ {%- endif -%} {% endmacro %} -{% macro bigquery__handle_existing_table(full_refresh, non_destructive_mode, old_relation) %} - {%- if full_refresh and not non_destructive_mode -%} - {{ adapter.drop_relation(old_relation) }} - {%- else -%} - {{ exceptions.relation_wrong_type(old_relation, 'view') }} - {%- endif -%} -{% endmacro %} - - {# /* Core materialization implementation. BigQuery and Snowflake are similar because both can use `create or replace view` where the resulting view schema @@ -30,7 +21,7 @@ */ #} -{% macro impl_view_materialization(run_outside_transaction_hooks=True) %} +{% macro create_or_replace_view(run_outside_transaction_hooks=True) %} {%- set identifier = model['alias'] -%} {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} @@ -88,11 +79,3 @@ {{ run_hooks(post_hooks, inside_transaction=False) }} {% endif %} {% endmacro %} - -{% materialization view, adapter='bigquery' -%} - {{ impl_view_materialization(run_outside_transaction_hooks=False) }} -{%- endmaterialization %} - -{% materialization view, adapter='snowflake' -%} - {{ impl_view_materialization() }} -{%- endmaterialization %} diff --git a/dbt/include/global_project/macros/materializations/view/view.sql b/core/dbt/include/global_project/macros/materializations/view/view.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/view/view.sql rename to core/dbt/include/global_project/macros/materializations/view/view.sql diff --git a/dbt/include/global_project/macros/operations/catalog/get_catalog.sql b/core/dbt/include/global_project/macros/operations/catalog/get_catalog.sql similarity index 100% rename from dbt/include/global_project/macros/operations/catalog/get_catalog.sql rename to core/dbt/include/global_project/macros/operations/catalog/get_catalog.sql diff --git a/dbt/include/global_project/macros/operations/relations/get_relations.sql b/core/dbt/include/global_project/macros/operations/relations/get_relations.sql similarity index 100% rename from dbt/include/global_project/macros/operations/relations/get_relations.sql rename to core/dbt/include/global_project/macros/operations/relations/get_relations.sql diff --git a/dbt/include/global_project/macros/schema_tests/accepted_values.sql b/core/dbt/include/global_project/macros/schema_tests/accepted_values.sql similarity index 100% rename from dbt/include/global_project/macros/schema_tests/accepted_values.sql rename to core/dbt/include/global_project/macros/schema_tests/accepted_values.sql diff --git a/dbt/include/global_project/macros/schema_tests/not_null.sql b/core/dbt/include/global_project/macros/schema_tests/not_null.sql similarity index 100% rename from dbt/include/global_project/macros/schema_tests/not_null.sql rename to core/dbt/include/global_project/macros/schema_tests/not_null.sql diff --git a/dbt/include/global_project/macros/schema_tests/relationships.sql b/core/dbt/include/global_project/macros/schema_tests/relationships.sql similarity index 100% rename from dbt/include/global_project/macros/schema_tests/relationships.sql rename to core/dbt/include/global_project/macros/schema_tests/relationships.sql diff --git a/dbt/include/global_project/macros/schema_tests/unique.sql b/core/dbt/include/global_project/macros/schema_tests/unique.sql similarity index 100% rename from dbt/include/global_project/macros/schema_tests/unique.sql rename to core/dbt/include/global_project/macros/schema_tests/unique.sql diff --git a/dbt/include/index.html b/core/dbt/include/index.html similarity index 100% rename from dbt/include/index.html rename to core/dbt/include/index.html diff --git a/dbt/linker.py b/core/dbt/linker.py similarity index 100% rename from dbt/linker.py rename to core/dbt/linker.py diff --git a/dbt/links.py b/core/dbt/links.py similarity index 100% rename from dbt/links.py rename to core/dbt/links.py diff --git a/dbt/loader.py b/core/dbt/loader.py similarity index 100% rename from dbt/loader.py rename to core/dbt/loader.py diff --git a/dbt/logger.py b/core/dbt/logger.py similarity index 100% rename from dbt/logger.py rename to core/dbt/logger.py diff --git a/dbt/main.py b/core/dbt/main.py similarity index 100% rename from dbt/main.py rename to core/dbt/main.py diff --git a/dbt/node_runners.py b/core/dbt/node_runners.py similarity index 99% rename from dbt/node_runners.py rename to core/dbt/node_runners.py index 27f23c6347e..93224161809 100644 --- a/dbt/node_runners.py +++ b/core/dbt/node_runners.py @@ -13,7 +13,6 @@ import dbt.ui.printer import dbt.flags import dbt.schema -import dbt.templates import dbt.writer import six diff --git a/dbt/node_types.py b/core/dbt/node_types.py similarity index 100% rename from dbt/node_types.py rename to core/dbt/node_types.py diff --git a/dbt/parser/__init__.py b/core/dbt/parser/__init__.py similarity index 100% rename from dbt/parser/__init__.py rename to core/dbt/parser/__init__.py diff --git a/dbt/parser/analysis.py b/core/dbt/parser/analysis.py similarity index 100% rename from dbt/parser/analysis.py rename to core/dbt/parser/analysis.py diff --git a/dbt/parser/archives.py b/core/dbt/parser/archives.py similarity index 100% rename from dbt/parser/archives.py rename to core/dbt/parser/archives.py diff --git a/dbt/parser/base.py b/core/dbt/parser/base.py similarity index 98% rename from dbt/parser/base.py rename to core/dbt/parser/base.py index fbcd3050c98..0bf18a7df5d 100644 --- a/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -8,6 +8,7 @@ import dbt.clients.jinja import dbt.context.parser +from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME from dbt.utils import coalesce from dbt.logger import GLOBAL_LOGGER as logger from dbt.contracts.graph.parsed import ParsedNode @@ -76,7 +77,7 @@ def get_schema_func(self): if get_schema_macro is None: get_schema_macro = self.macro_manifest.find_macro_by_name( 'generate_schema_name', - dbt.include.GLOBAL_PROJECT_NAME + GLOBAL_PROJECT_NAME ) if get_schema_macro is None: def get_schema(_): diff --git a/dbt/parser/base_sql.py b/core/dbt/parser/base_sql.py similarity index 100% rename from dbt/parser/base_sql.py rename to core/dbt/parser/base_sql.py diff --git a/dbt/parser/data_test.py b/core/dbt/parser/data_test.py similarity index 100% rename from dbt/parser/data_test.py rename to core/dbt/parser/data_test.py diff --git a/dbt/parser/docs.py b/core/dbt/parser/docs.py similarity index 100% rename from dbt/parser/docs.py rename to core/dbt/parser/docs.py diff --git a/dbt/parser/hooks.py b/core/dbt/parser/hooks.py similarity index 100% rename from dbt/parser/hooks.py rename to core/dbt/parser/hooks.py diff --git a/dbt/parser/macros.py b/core/dbt/parser/macros.py similarity index 100% rename from dbt/parser/macros.py rename to core/dbt/parser/macros.py diff --git a/dbt/parser/models.py b/core/dbt/parser/models.py similarity index 100% rename from dbt/parser/models.py rename to core/dbt/parser/models.py diff --git a/dbt/parser/schemas.py b/core/dbt/parser/schemas.py similarity index 100% rename from dbt/parser/schemas.py rename to core/dbt/parser/schemas.py diff --git a/dbt/parser/seeds.py b/core/dbt/parser/seeds.py similarity index 100% rename from dbt/parser/seeds.py rename to core/dbt/parser/seeds.py diff --git a/dbt/parser/source_config.py b/core/dbt/parser/source_config.py similarity index 100% rename from dbt/parser/source_config.py rename to core/dbt/parser/source_config.py diff --git a/dbt/parser/util.py b/core/dbt/parser/util.py similarity index 100% rename from dbt/parser/util.py rename to core/dbt/parser/util.py diff --git a/dbt/profiler.py b/core/dbt/profiler.py similarity index 100% rename from dbt/profiler.py rename to core/dbt/profiler.py diff --git a/dbt/runner.py b/core/dbt/runner.py similarity index 100% rename from dbt/runner.py rename to core/dbt/runner.py diff --git a/dbt/schema.py b/core/dbt/schema.py similarity index 100% rename from dbt/schema.py rename to core/dbt/schema.py diff --git a/dbt/semver.py b/core/dbt/semver.py similarity index 100% rename from dbt/semver.py rename to core/dbt/semver.py diff --git a/dbt/ssh_forward.py b/core/dbt/ssh_forward.py similarity index 100% rename from dbt/ssh_forward.py rename to core/dbt/ssh_forward.py diff --git a/dbt/contracts/graph/__init__.py b/core/dbt/task/__init__.py similarity index 100% rename from dbt/contracts/graph/__init__.py rename to core/dbt/task/__init__.py diff --git a/dbt/task/archive.py b/core/dbt/task/archive.py similarity index 100% rename from dbt/task/archive.py rename to core/dbt/task/archive.py diff --git a/dbt/task/base_task.py b/core/dbt/task/base_task.py similarity index 100% rename from dbt/task/base_task.py rename to core/dbt/task/base_task.py diff --git a/dbt/task/clean.py b/core/dbt/task/clean.py similarity index 100% rename from dbt/task/clean.py rename to core/dbt/task/clean.py diff --git a/dbt/task/compile.py b/core/dbt/task/compile.py similarity index 100% rename from dbt/task/compile.py rename to core/dbt/task/compile.py diff --git a/dbt/task/debug.py b/core/dbt/task/debug.py similarity index 100% rename from dbt/task/debug.py rename to core/dbt/task/debug.py diff --git a/dbt/task/deps.py b/core/dbt/task/deps.py similarity index 100% rename from dbt/task/deps.py rename to core/dbt/task/deps.py diff --git a/dbt/task/generate.py b/core/dbt/task/generate.py similarity index 99% rename from dbt/task/generate.py rename to core/dbt/task/generate.py index 6cc1df5e02c..96d5789eb10 100644 --- a/dbt/task/generate.py +++ b/core/dbt/task/generate.py @@ -5,7 +5,7 @@ from dbt.adapters.factory import get_adapter from dbt.clients.system import write_json from dbt.compat import bigint -from dbt.include import DOCS_INDEX_FILE_PATH +from dbt.include.global_project import DOCS_INDEX_FILE_PATH import dbt.ui.printer import dbt.utils import dbt.compilation diff --git a/dbt/task/init.py b/core/dbt/task/init.py similarity index 100% rename from dbt/task/init.py rename to core/dbt/task/init.py diff --git a/dbt/task/run.py b/core/dbt/task/run.py similarity index 100% rename from dbt/task/run.py rename to core/dbt/task/run.py diff --git a/dbt/task/seed.py b/core/dbt/task/seed.py similarity index 100% rename from dbt/task/seed.py rename to core/dbt/task/seed.py diff --git a/dbt/task/serve.py b/core/dbt/task/serve.py similarity index 94% rename from dbt/task/serve.py rename to core/dbt/task/serve.py index 3a94ffbafed..d5e58762466 100644 --- a/dbt/task/serve.py +++ b/core/dbt/task/serve.py @@ -2,7 +2,7 @@ import os import webbrowser -from dbt.include import DOCS_INDEX_FILE_PATH +from dbt.include.global_project import DOCS_INDEX_FILE_PATH from dbt.compat import SimpleHTTPRequestHandler, TCPServer from dbt.logger import GLOBAL_LOGGER as logger diff --git a/dbt/task/test.py b/core/dbt/task/test.py similarity index 100% rename from dbt/task/test.py rename to core/dbt/task/test.py diff --git a/dbt/tracking.py b/core/dbt/tracking.py similarity index 100% rename from dbt/tracking.py rename to core/dbt/tracking.py diff --git a/dbt/graph/__init__.py b/core/dbt/ui/__init__.py similarity index 100% rename from dbt/graph/__init__.py rename to core/dbt/ui/__init__.py diff --git a/dbt/ui/colors.py b/core/dbt/ui/colors.py similarity index 100% rename from dbt/ui/colors.py rename to core/dbt/ui/colors.py diff --git a/dbt/ui/printer.py b/core/dbt/ui/printer.py similarity index 100% rename from dbt/ui/printer.py rename to core/dbt/ui/printer.py diff --git a/dbt/utils.py b/core/dbt/utils.py similarity index 98% rename from dbt/utils.py rename to core/dbt/utils.py index 788c73f760f..b5c2001edaf 100644 --- a/dbt/utils.py +++ b/core/dbt/utils.py @@ -13,7 +13,7 @@ import dbt.exceptions import dbt.flags -from dbt.include import GLOBAL_DBT_MODULES_PATH +from dbt.include.global_project import PACKAGES from dbt.compat import basestring, DECIMALS from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_types import NodeType @@ -216,10 +216,8 @@ def dependencies_for_path(config, module_path): def dependency_projects(config): - module_paths = [ - GLOBAL_DBT_MODULES_PATH, - os.path.join(config.project_root, config.modules_path) - ] + module_paths = list(PACKAGES.values()) + module_paths.append(os.path.join(config.project_root, config.modules_path)) for module_path in module_paths: for entry in dependencies_for_path(config, module_path): diff --git a/dbt/version.py b/core/dbt/version.py similarity index 98% rename from dbt/version.py rename to core/dbt/version.py index a990b6614f3..752926a2f59 100644 --- a/dbt/version.py +++ b/core/dbt/version.py @@ -57,5 +57,5 @@ def get_version_information(): .format(version_msg)) -__version__ = '0.12.2rc1' +__version__ = '0.13.0a1' installed = get_installed_version() diff --git a/dbt/writer.py b/core/dbt/writer.py similarity index 100% rename from dbt/writer.py rename to core/dbt/writer.py diff --git a/core/etc/dag.png b/core/etc/dag.png new file mode 100644 index 00000000000..ad76924a43f Binary files /dev/null and b/core/etc/dag.png differ diff --git a/events/schemas/com.fishtownanalytics/invocation_env_context.json b/core/events/schemas/com.fishtownanalytics/invocation_env_context.json similarity index 100% rename from events/schemas/com.fishtownanalytics/invocation_env_context.json rename to core/events/schemas/com.fishtownanalytics/invocation_env_context.json diff --git a/events/schemas/com.fishtownanalytics/invocation_event.json b/core/events/schemas/com.fishtownanalytics/invocation_event.json similarity index 100% rename from events/schemas/com.fishtownanalytics/invocation_event.json rename to core/events/schemas/com.fishtownanalytics/invocation_event.json diff --git a/events/schemas/com.fishtownanalytics/platform_context.json b/core/events/schemas/com.fishtownanalytics/platform_context.json similarity index 100% rename from events/schemas/com.fishtownanalytics/platform_context.json rename to core/events/schemas/com.fishtownanalytics/platform_context.json diff --git a/events/schemas/com.fishtownanalytics/run_model_context.json b/core/events/schemas/com.fishtownanalytics/run_model_context.json similarity index 100% rename from events/schemas/com.fishtownanalytics/run_model_context.json rename to core/events/schemas/com.fishtownanalytics/run_model_context.json diff --git a/core/scripts/create_adapter_plugins.py b/core/scripts/create_adapter_plugins.py new file mode 100644 index 00000000000..9b0349f1876 --- /dev/null +++ b/core/scripts/create_adapter_plugins.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +import argparse +import os +import sys + +pj = os.path.join + +PROJECT_TEMPLATE = ''' +name: dbt_{adapter} +version: {version} + +macro-paths: ["macros"] +''' + +NAMESPACE_INIT_TEMPLATE = ''' +__path__ = __import__('pkgutil').extend_path(__path__, __name__) +'''.lstrip() + + +# TODO: make this not default to fishtown for everything! +SETUP_PY_TEMPLATE = ''' +#!/usr/bin/env python +from setuptools import find_packages +from distutils.core import setup + +package_name = "dbt-{adapter}" +package_version = "{version}" +description = """The {adapter} adpter plugin for dbt (data build tool)""" + +setup( + name=package_name, + version=package_version, + description=description, + long_description_content_type=description, + author={author_name}, + author_email={author_email}, + url={url}, + packages=find_packages(), + install_requires=[ + 'dbt-core=={dbt_core_version}', + {dependencies} + ] +) +'''.lstrip() + + + +ADAPTER_INIT_TEMPLATE = ''' +from dbt.adapters.{adapter}.connections import {title_adapter}ConnectionManager +from dbt.adapters.{adapter}.connections import {title_adapter}Credentials +from dbt.adapters.{adapter}.impl import {title_adapter}Adapter + +from dbt.adapters.base import AdapterPlugin +from dbt.include import {adapter} + +Plugin = AdapterPlugin( + adapter={title_adapter}Adapter, + credentials={title_adapter}Credentials, + include_path={adapter}.PACKAGE_PATH) +'''.lstrip() + + +INCLUDE_INIT_TEMPLATE = ''' +import os +PACKAGE_PATH = os.path.dirname(os.path.dirname(__file__)) +'''.lstrip() + + +def parse_args(argv=None): + if argv is None: + argv = sys.argv[1:] + parser = argparse.ArgumentParser() + parser.add_argument('root') + parser.add_argument('adapter') + parser.add_argument('--title-case', '-t', default=None) + parser.add_argument('--dependency', action='append') + parser.add_argument('--dbt-core-version', default='0.13.0') + parser.add_argument('--email') + parser.add_argument('--author') + parser.add_argument('--url') + parser.add_argument('--package-version', default='0.0.1') + parser.add_argument('--project-version', default='1.0') + parsed = parser.parse_args() + + if parsed.title_case is None: + parsed.title_case = parsed.adapter.title() + + if parsed.dependency: + #['a', 'b'] => "'a',\n 'b'"; ['a'] -> "'a'," + parsed.dependency = '\n '.join( + "'{}',".format(d) for d in parsed.dependency + ) + else: + parsed.dependency = '' + + if parsed.email is not None: + parsed.email = "'{}'".format(parsed.email) + else: + parsed.email = '' + if parsed.author is not None: + parsed.author = "'{}'".format(parsed.author) + else: + parsed.author = '' + if parsed.url is not None: + parsed.url = "'{}'".format(parsed.url) + else: + parsed.url = '' + return parsed + + + +def main(): + parsed = parse_args() + dest = pj(parsed.root, parsed.adapter) + if os.path.exists(dest): + raise Exception('path exists') + + adapters_path = pj(dest, 'dbt', 'adapters', parsed.adapter) + include_path = pj(dest, 'dbt', 'include', parsed.adapter) + os.makedirs(adapters_path) + os.makedirs(pj(include_path, 'macros')) + + # namespaces! + with open(pj(dest, 'dbt', '__init__.py'), 'w') as fp: + fp.write(NAMESPACE_INIT_TEMPLATE) + with open(pj(dest, 'dbt', 'adapters', '__init__.py'), 'w') as fp: + fp.write(NAMESPACE_INIT_TEMPLATE) + with open(pj(dest, 'dbt', 'include', '__init__.py'), 'w') as fp: + fp.write(NAMESPACE_INIT_TEMPLATE) + + # setup script! + with open(pj(dest, 'setup.py'), 'w') as fp: + fp.write(SETUP_PY_TEMPLATE.format(adapter=parsed.adapter, + version=parsed.package_version, + author_name=parsed.author, + author_email=parsed.email, + url=parsed.url, + dbt_core_version=parsed.dbt_core_version, + dependencies=parsed.dependency)) + + + # adapter stuff! + with open(pj(adapters_path, '__init__.py'), 'w') as fp: + fp.write(ADAPTER_INIT_TEMPLATE.format(adapter=parsed.adapter, + title_adapter=parsed.title_case)) + + # macro/project stuff! + with open(pj(include_path, '__init__.py'), 'w') as fp: + fp.write(INCLUDE_INIT_TEMPLATE) + + with open(pj(include_path, 'dbt_project.yml'), 'w') as fp: + fp.write(PROJECT_TEMPLATE.format(adapter=parsed.adapter, + version=parsed.project_version)) + + # TODO: + # - bare class impls for mandatory subclasses + # (ConnectionManager, Credentials, Adapter) + # - impls of mandatory abstract methods w/explicit NotImplementedErrors + + +if __name__ == '__main__': + main() diff --git a/scripts/dbt b/core/scripts/dbt similarity index 100% rename from scripts/dbt rename to core/scripts/dbt diff --git a/scripts/upgrade_dbt_schema_tests_v1_to_v2.py b/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py similarity index 100% rename from scripts/upgrade_dbt_schema_tests_v1_to_v2.py rename to core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py diff --git a/setup.py b/core/setup.py similarity index 69% rename from setup.py rename to core/setup.py index 95e3a7d1593..68c5495e8fd 100644 --- a/setup.py +++ b/core/setup.py @@ -6,22 +6,12 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -package_name = "dbt" -package_version = "0.12.2rc1" +package_name = "dbt-core" +package_version = "0.13.0a1" description = """dbt (data build tool) is a command line tool that helps \ analysts and engineers transform data in their warehouse more effectively""" -boto_requirements = [ - 'boto3>=1.6.23,<1.8.0', - 'botocore>=1.9.23,<1.11.0', -] - -postgres_requirements = [ - 'psycopg2>=2.7.5,<2.8', -] - - setup( name=package_name, version=package_version, @@ -50,15 +40,6 @@ def read(fname): scripts=[ 'scripts/dbt', ], - extras_require={ - # when we split out adapters into their own python packages, these - # extras requirements lists will just be the appropriate adapter - # packages - 'snowflake': ['snowflake-connector-python>=1.4.9'] + boto_requirements, - 'bigquery': ['google-cloud-bigquery>=1.0.0,<2'], - 'redshift': boto_requirements[:] + postgres_requirements[:], - 'postgres': postgres_requirements[:], - }, install_requires=[ 'Jinja2>=2.10', 'PyYAML>=3.11', diff --git a/dbt/adapters/postgres/__init__.py b/dbt/adapters/postgres/__init__.py deleted file mode 100644 index d63b2d32b58..00000000000 --- a/dbt/adapters/postgres/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from dbt.adapters.postgres.connections import PostgresConnectionManager -from dbt.adapters.postgres.connections import PostgresCredentials -from dbt.adapters.postgres.impl import PostgresAdapter - -Adapter = PostgresAdapter -Credentials = PostgresCredentials diff --git a/dbt/adapters/redshift/__init__.py b/dbt/adapters/redshift/__init__.py deleted file mode 100644 index aa47613a429..00000000000 --- a/dbt/adapters/redshift/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from dbt.adapters.redshift.connections import RedshiftConnectionManager -from dbt.adapters.redshift.connections import RedshiftCredentials -from dbt.adapters.redshift.impl import RedshiftAdapter - -Adapter = RedshiftAdapter -Credentials = RedshiftCredentials diff --git a/dbt/config.py b/dbt/config.py deleted file mode 100644 index 65365aeec61..00000000000 --- a/dbt/config.py +++ /dev/null @@ -1,1077 +0,0 @@ -import os.path -import os -from copy import deepcopy -import hashlib -import pprint - -import dbt.exceptions -import dbt.clients.yaml_helper -import dbt.clients.system -import dbt.utils -from dbt.compat import basestring -from dbt.contracts.project import Project as ProjectContract, Configuration, \ - PackageConfig, ProfileConfig -from dbt.exceptions import DbtProjectError, DbtProfileError, RecursionException -from dbt.context.common import env_var, Var -from dbt import compat -from dbt.adapters.factory import load_adapter, get_relation_class_by_name - -from dbt.logger import GLOBAL_LOGGER as logger -from dbt.utils import DBTConfigKeys -from dbt.version import get_installed_version -from dbt.semver import VersionSpecifier, versions_compatible -import dbt.ui.printer - -DEFAULT_THREADS = 1 -DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True -DEFAULT_USE_COLORS = True -DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser('~'), '.dbt') -PROFILES_DIR = os.path.expanduser( - os.environ.get('DBT_PROFILES_DIR', DEFAULT_PROFILES_DIR) - ) - -INVALID_PROFILE_MESSAGE = """ -dbt encountered an error while trying to read your profiles.yml file. - -{error_string} -""" - - -NO_SUPPLIED_PROFILE_ERROR = """\ -dbt cannot run because no profile was specified for this dbt project. -To specify a profile for this project, add a line like the this to -your dbt_project.yml file: - -profile: [profile name] - -Here, [profile name] should be replaced with a profile name -defined in your profiles.yml file. You can find profiles.yml here: - -{profiles_file}/profiles.yml -""".format(profiles_file=PROFILES_DIR) - - -UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE = """\ -WARNING: Configuration paths exist in your dbt_project.yml file which do not \ -apply to any resources. -There are {} unused configuration paths:\n{} -""" - -INVALID_VERSION_ERROR = """\ -This version of dbt is not supported with the '{package}' package. - Installed version of dbt: {installed} - Required version of dbt for '{package}': {version_spec} - -Check the requirements for the '{package}' package, or run dbt again with \ ---no-version-check -""" - -IMPOSSIBLE_VERSION_ERROR = """\ -The package version requirement can never be satisfied for the '{package} -package. - Required versions of dbt for '{package}': {version_spec} - -Check the requirements for the '{package}' package, or run dbt again with \ ---no-version-check -""" - - -def read_profile(profiles_dir): - path = os.path.join(profiles_dir, 'profiles.yml') - - contents = None - if os.path.isfile(path): - try: - contents = dbt.clients.system.load_file_contents(path, strip=False) - return dbt.clients.yaml_helper.load_yaml_text(contents) - except dbt.exceptions.ValidationException as e: - msg = INVALID_PROFILE_MESSAGE.format(error_string=e) - raise dbt.exceptions.ValidationException(msg) - - return {} - - -def read_profiles(profiles_dir=None): - """This is only used in main, for some error handling""" - if profiles_dir is None: - profiles_dir = PROFILES_DIR - - raw_profiles = read_profile(profiles_dir) - - if raw_profiles is None: - profiles = {} - else: - profiles = {k: v for (k, v) in raw_profiles.items() if k != 'config'} - - return profiles - - -class ConfigRenderer(object): - """A renderer provides configuration rendering for a given set of cli - variables and a render type. - """ - def __init__(self, cli_vars): - self.context = {'env_var': env_var} - self.context['var'] = Var(None, self.context, cli_vars) - - @staticmethod - def _is_hook_or_model_vars_path(keypath): - if not keypath: - return False - - first = keypath[0] - # run hooks - if first in {'on-run-start', 'on-run-end'}: - return True - # models have two things to avoid - if first in {'seeds', 'models'}: - # model-level hooks - if 'pre-hook' in keypath or 'post-hook' in keypath: - return True - # model-level 'vars' declarations - if 'vars' in keypath: - return True - - return False - - def _render_project_entry(self, value, keypath): - """Render an entry, in case it's jinja. This is meant to be passed to - dbt.utils.deep_map. - - If the parsed entry is a string and has the name 'port', this will - attempt to cast it to an int, and on failure will return the parsed - string. - - :param value Any: The value to potentially render - :param key str: The key to convert on. - :return Any: The rendered entry. - """ - # hooks should be treated as raw sql, they'll get rendered later. - # Same goes for 'vars' declarations inside 'models'/'seeds'. - if self._is_hook_or_model_vars_path(keypath): - return value - - return self.render_value(value) - - def render_value(self, value, keypath=None): - # keypath is ignored. - # if it wasn't read as a string, ignore it - if not isinstance(value, compat.basestring): - return value - - return dbt.clients.jinja.get_rendered(value, self.context) - - def _render_profile_data(self, value, keypath): - result = self.render_value(value) - if len(keypath) == 1 and keypath[-1] == 'port': - try: - result = int(result) - except ValueError: - # let the validator or connection handle this - pass - return result - - def render_project(self, as_parsed): - """Render the parsed data, returning a new dict (or whatever was read). - """ - try: - return dbt.utils.deep_map(self._render_project_entry, as_parsed) - except RecursionException: - raise DbtProjectError( - 'Cycle detected: Project input has a reference to itself', - project=project_dict - ) - - def render_profile_data(self, as_parsed): - """Render the chosen profile entry, as it was parsed.""" - try: - return dbt.utils.deep_map(self._render_profile_data, as_parsed) - except RecursionException: - raise DbtProfileError( - 'Cycle detected: Profile input has a reference to itself', - project=as_parsed - ) - - -def _list_if_none(value): - if value is None: - value = [] - return value - - -def _dict_if_none(value): - if value is None: - value = {} - return value - - -def _list_if_none_or_string(value): - value = _list_if_none(value) - if isinstance(value, compat.basestring): - return [value] - return value - - -def _parse_versions(versions): - """Parse multiple versions as read from disk. The versions value may be any - one of: - - a single version string ('>0.12.1') - - a single string specifying multiple comma-separated versions - ('>0.11.1,<=0.12.2') - - an array of single-version strings (['>0.11.1', '<=0.12.2']) - - Regardless, this will return a list of VersionSpecifiers - """ - if isinstance(versions, basestring): - versions = versions.split(',') - return [VersionSpecifier.from_version_string(v) for v in versions] - - -class Project(object): - def __init__(self, project_name, version, project_root, profile_name, - source_paths, macro_paths, data_paths, test_paths, - analysis_paths, docs_paths, target_path, clean_targets, - log_path, modules_path, quoting, models, on_run_start, - on_run_end, archive, seeds, dbt_version, packages): - self.project_name = project_name - self.version = version - self.project_root = project_root - self.profile_name = profile_name - self.source_paths = source_paths - self.macro_paths = macro_paths - self.data_paths = data_paths - self.test_paths = test_paths - self.analysis_paths = analysis_paths - self.docs_paths = docs_paths - self.target_path = target_path - self.clean_targets = clean_targets - self.log_path = log_path - self.modules_path = modules_path - self.quoting = quoting - self.models = models - self.on_run_start = on_run_start - self.on_run_end = on_run_end - self.archive = archive - self.seeds = seeds - self.dbt_version = dbt_version - self.packages = packages - - @staticmethod - def _preprocess(project_dict): - """Pre-process certain special keys to convert them from None values - into empty containers, and to turn strings into arrays of strings. - """ - handlers = { - ('archive',): _list_if_none, - ('on-run-start',): _list_if_none_or_string, - ('on-run-end',): _list_if_none_or_string, - } - - for k in ('models', 'seeds'): - handlers[(k,)] = _dict_if_none - handlers[(k, 'vars')] = _dict_if_none - handlers[(k, 'pre-hook')] = _list_if_none_or_string - handlers[(k, 'post-hook')] = _list_if_none_or_string - handlers[('seeds', 'column_types')] = _dict_if_none - - def converter(value, keypath): - if keypath in handlers: - handler = handlers[keypath] - return handler(value) - else: - return value - - return dbt.utils.deep_map(converter, project_dict) - - @classmethod - def from_project_config(cls, project_dict, packages_dict=None): - """Create a project from its project and package configuration, as read - by yaml.safe_load(). - - :param project_dict dict: The dictionary as read from disk - :param packages_dict Optional[dict]: If it exists, the packages file as - read from disk. - :raises DbtProjectError: If the project is missing or invalid, or if - the packages file exists and is invalid. - :returns Project: The project, with defaults populated. - """ - try: - project_dict = cls._preprocess(project_dict) - except RecursionException: - raise DbtProjectError( - 'Cycle detected: Project input has a reference to itself', - project=project_dict - ) - # just for validation. - try: - ProjectContract(**project_dict) - except dbt.exceptions.ValidationException as e: - raise DbtProjectError(str(e)) - - # name/version are required in the Project definition, so we can assume - # they are present - name = project_dict['name'] - version = project_dict['version'] - # this is added at project_dict parse time and should always be here - # once we see it. - project_root = project_dict['project-root'] - # this is only optional in the sense that if it's not present, it needs - # to have been a cli argument. - profile_name = project_dict.get('profile') - # these are optional - source_paths = project_dict.get('source-paths', ['models']) - macro_paths = project_dict.get('macro-paths', ['macros']) - data_paths = project_dict.get('data-paths', ['data']) - test_paths = project_dict.get('test-paths', ['test']) - analysis_paths = project_dict.get('analysis-paths', []) - docs_paths = project_dict.get('docs-paths', source_paths[:]) - target_path = project_dict.get('target-path', 'target') - # should this also include the modules path by default? - clean_targets = project_dict.get('clean-targets', [target_path]) - log_path = project_dict.get('log-path', 'logs') - modules_path = project_dict.get('modules-path', 'dbt_modules') - # in the default case we'll populate this once we know the adapter type - quoting = project_dict.get('quoting', {}) - - models = project_dict.get('models', {}) - on_run_start = project_dict.get('on-run-start', []) - on_run_end = project_dict.get('on-run-end', []) - archive = project_dict.get('archive', []) - seeds = project_dict.get('seeds', {}) - dbt_raw_version = project_dict.get('require-dbt-version', '>=0.0.0') - - try: - dbt_version = _parse_versions(dbt_raw_version) - except dbt.exceptions.SemverException as e: - raise DbtProjectError(str(e)) - - packages = package_config_from_data(packages_dict) - - project = cls( - project_name=name, - version=version, - project_root=project_root, - profile_name=profile_name, - source_paths=source_paths, - macro_paths=macro_paths, - data_paths=data_paths, - test_paths=test_paths, - analysis_paths=analysis_paths, - docs_paths=docs_paths, - target_path=target_path, - clean_targets=clean_targets, - log_path=log_path, - modules_path=modules_path, - quoting=quoting, - models=models, - on_run_start=on_run_start, - on_run_end=on_run_end, - archive=archive, - seeds=seeds, - dbt_version=dbt_version, - packages=packages - ) - # sanity check - this means an internal issue - project.validate() - return project - - def __str__(self): - cfg = self.to_project_config(with_packages=True) - return pprint.pformat(cfg) - - def __eq__(self, other): - if not (isinstance(other, self.__class__) and - isinstance(self, other.__class__)): - return False - return self.to_project_config(with_packages=True) == \ - other.to_project_config(with_packages=True) - - def to_project_config(self, with_packages=False): - """Return a dict representation of the config that could be written to - disk with `yaml.safe_dump` to get this configuration. - - :param with_packages bool: If True, include the serialized packages - file in the root. - :returns dict: The serialized profile. - """ - result = deepcopy({ - 'name': self.project_name, - 'version': self.version, - 'project-root': self.project_root, - 'profile': self.profile_name, - 'source-paths': self.source_paths, - 'macro-paths': self.macro_paths, - 'data-paths': self.data_paths, - 'test-paths': self.test_paths, - 'analysis-paths': self.analysis_paths, - 'docs-paths': self.docs_paths, - 'target-path': self.target_path, - 'clean-targets': self.clean_targets, - 'log-path': self.log_path, - 'quoting': self.quoting, - 'models': self.models, - 'on-run-start': self.on_run_start, - 'on-run-end': self.on_run_end, - 'archive': self.archive, - 'seeds': self.seeds, - 'require-dbt-version': [ - v.to_version_string() for v in self.dbt_version - ], - }) - if with_packages: - result.update(self.packages.serialize()) - return result - - def validate(self): - try: - ProjectContract(**self.to_project_config()) - except dbt.exceptions.ValidationException as exc: - raise DbtProjectError(str(exc)) - - @classmethod - def from_project_root(cls, project_root, cli_vars): - """Create a project from a root directory. Reads in dbt_project.yml and - packages.yml, if it exists. - - :param project_root str: The path to the project root to load. - :raises DbtProjectError: If the project is missing or invalid, or if - the packages file exists and is invalid. - :returns Project: The project, with defaults populated. - """ - project_root = os.path.normpath(project_root) - project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml') - - # get the project.yml contents - if not dbt.clients.system.path_exists(project_yaml_filepath): - raise DbtProjectError( - 'no dbt_project.yml found at expected path {}' - .format(project_yaml_filepath) - ) - - if isinstance(cli_vars, compat.basestring): - cli_vars = dbt.utils.parse_cli_vars(cli_vars) - renderer = ConfigRenderer(cli_vars) - - project_dict = _load_yaml(project_yaml_filepath) - rendered_project = renderer.render_project(project_dict) - rendered_project['project-root'] = project_root - packages_dict = package_data_from_root(project_root) - return cls.from_project_config(rendered_project, packages_dict) - - @classmethod - def from_current_directory(cls, cli_vars): - return cls.from_project_root(os.getcwd(), cli_vars) - - def hashed_name(self): - return hashlib.md5(self.project_name.encode('utf-8')).hexdigest() - - def get_resource_config_paths(self): - """Return a dictionary with 'seeds' and 'models' keys whose values are - lists of lists of strings, where each inner list of strings represents - a configured path in the resource. - """ - return { - 'models': _get_config_paths(self.models), - 'seeds': _get_config_paths(self.seeds), - } - - def get_unused_resource_config_paths(self, resource_fqns, disabled): - """Return a list of lists of strings, where each inner list of strings - represents a type + FQN path of a resource configuration that is not - used. - """ - disabled_fqns = frozenset(tuple(fqn) for fqn in disabled) - resource_config_paths = self.get_resource_config_paths() - unused_resource_config_paths = [] - for resource_type, config_paths in resource_config_paths.items(): - used_fqns = resource_fqns.get(resource_type, frozenset()) - fqns = used_fqns | disabled_fqns - - for config_path in config_paths: - if not _is_config_used(config_path, fqns): - unused_resource_config_paths.append( - (resource_type,) + config_path - ) - return unused_resource_config_paths - - def warn_for_unused_resource_config_paths(self, resource_fqns, disabled): - unused = self.get_unused_resource_config_paths(resource_fqns, disabled) - if len(unused) == 0: - return - - msg = UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE.format( - len(unused), - '\n'.join('- {}'.format('.'.join(u)) for u in unused) - ) - logger.info(dbt.ui.printer.yellow(msg)) - - def validate_version(self): - """Ensure this package works with the installed version of dbt.""" - installed = get_installed_version() - if not versions_compatible(*self.dbt_version): - msg = IMPOSSIBLE_VERSION_ERROR.format( - package=self.project_name, - version_spec=[ - x.to_version_string() for x in self.dbt_version - ] - ) - raise DbtProjectError(msg) - - if not versions_compatible(installed, *self.dbt_version): - msg = INVALID_VERSION_ERROR.format( - package=self.project_name, - installed=installed.to_version_string(), - version_spec=[ - x.to_version_string() for x in self.dbt_version - ] - ) - raise DbtProjectError(msg) - - -class UserConfig(object): - def __init__(self, send_anonymous_usage_stats, use_colors): - self.send_anonymous_usage_stats = send_anonymous_usage_stats - self.use_colors = use_colors - - @classmethod - def from_dict(cls, cfg=None): - if cfg is None: - cfg = {} - send_anonymous_usage_stats = cfg.get( - 'send_anonymous_usage_stats', - DEFAULT_SEND_ANONYMOUS_USAGE_STATS - ) - use_colors = cfg.get( - 'use_colors', - DEFAULT_USE_COLORS - ) - return cls(send_anonymous_usage_stats, use_colors) - - def to_dict(self): - return { - 'send_anonymous_usage_stats': self.send_anonymous_usage_stats, - 'use_colors': self.use_colors, - } - - @classmethod - def from_directory(cls, directory): - user_cfg = None - profile = read_profile(directory) - if profile: - user_cfg = profile.get('config', {}) - return cls.from_dict(user_cfg) - - -class Profile(object): - def __init__(self, profile_name, target_name, config, threads, - credentials): - self.profile_name = profile_name - self.target_name = target_name - if isinstance(config, dict): - config = UserConfig.from_dict(config) - self.config = config - self.threads = threads - self.credentials = credentials - - def to_profile_info(self, serialize_credentials=False): - """Unlike to_project_config, this dict is not a mirror of any existing - on-disk data structure. It's used when creating a new profile from an - existing one. - - :param serialize_credentials bool: If True, serialize the credentials. - Otherwise, the Credentials object will be copied. - :returns dict: The serialized profile. - """ - result = { - 'profile_name': self.profile_name, - 'target_name': self.target_name, - 'config': self.config.to_dict(), - 'threads': self.threads, - 'credentials': self.credentials.incorporate(), - } - if serialize_credentials: - result['credentials'] = result['credentials'].serialize() - return result - - def __str__(self): - return pprint.pformat(self.to_profile_info()) - - def __eq__(self, other): - if not (isinstance(other, self.__class__) and - isinstance(self, other.__class__)): - return False - return False - return self.to_profile_info() == other.to_profile_info() - - def validate(self): - if self.credentials: - self.credentials.validate() - try: - ProfileConfig(**self.to_profile_info(serialize_credentials=True)) - except dbt.exceptions.ValidationException as exc: - raise DbtProfileError(str(exc)) - - @staticmethod - def _credentials_from_profile(profile, profile_name, target_name): - # credentials carry their 'type' in their actual type, not their - # attributes. We do want this in order to pick our Credentials class. - if 'type' not in profile: - raise DbtProfileError( - 'required field "type" not found in profile {} and target {}' - .format(profile_name, target_name)) - - typename = profile.pop('type') - - try: - cls = load_adapter(typename) - credentials = cls(**profile) - except dbt.exceptions.RuntimeException as e: - raise DbtProfileError( - 'Credentials in profile "{}", target "{}" invalid: {}' - .format(profile_name, target_name, str(e)) - ) - return credentials - - @staticmethod - def pick_profile_name(args_profile_name, project_profile_name=None): - profile_name = project_profile_name - if args_profile_name is not None: - profile_name = args_profile_name - if profile_name is None: - raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR) - return profile_name - - @staticmethod - def _get_profile_data(profile, profile_name, target_name): - if 'outputs' not in profile: - raise DbtProfileError( - "outputs not specified in profile '{}'".format(profile_name) - ) - outputs = profile['outputs'] - - if target_name not in outputs: - outputs = '\n'.join(' - {}'.format(output) - for output in outputs) - msg = ("The profile '{}' does not have a target named '{}'. The " - "valid target names for this profile are:\n{}" - .format(profile_name, target_name, outputs)) - raise DbtProfileError(msg, result_type='invalid_target') - profile_data = outputs[target_name] - return profile_data - - @classmethod - def from_credentials(cls, credentials, threads, profile_name, target_name, - user_cfg=None): - """Create a profile from an existing set of Credentials and the - remaining information. - - :param credentials dict: The credentials dict for this profile. - :param threads int: The number of threads to use for connections. - :param profile_name str: The profile name used for this profile. - :param target_name str: The target name used for this profile. - :param user_cfg Optional[dict]: The user-level config block from the - raw profiles, if specified. - :raises DbtProfileError: If the profile is invalid. - :returns Profile: The new Profile object. - """ - config = UserConfig.from_dict(user_cfg) - profile = cls( - profile_name=profile_name, - target_name=target_name, - config=config, - threads=threads, - credentials=credentials - ) - profile.validate() - return profile - - @classmethod - def render_profile(cls, raw_profile, profile_name, target_override, - cli_vars): - """This is a containment zone for the hateful way we're rendering - profiles. - """ - renderer = ConfigRenderer(cli_vars=cli_vars) - - # rendering profiles is a bit complex. Two constraints cause trouble: - # 1) users should be able to use environment/cli variables to specify - # the target in their profile. - # 2) Missing environment/cli variables in profiles/targets that don't - # end up getting selected should not cause errors. - # so first we'll just render the target name, then we use that rendered - # name to extract a profile that we can render. - if target_override is not None: - target_name = target_override - elif 'target' in raw_profile: - # render the target if it was parsed from yaml - target_name = renderer.render_value(raw_profile['target']) - else: - target_name = 'default' - logger.debug( - "target not specified in profile '{}', using '{}'" - .format(profile_name, target_name) - ) - - raw_profile_data = cls._get_profile_data( - raw_profile, profile_name, target_name - ) - - profile_data = renderer.render_profile_data(raw_profile_data) - return target_name, profile_data - - @classmethod - def from_raw_profile_info(cls, raw_profile, profile_name, cli_vars, - user_cfg=None, target_override=None, - threads_override=None): - """Create a profile from its raw profile information. - - (this is an intermediate step, mostly useful for unit testing) - - :param raw_profile dict: The profile data for a single profile, from - disk as yaml and its values rendered with jinja. - :param profile_name str: The profile name used. - :param cli_vars dict: The command-line variables passed as arguments, - as a dict. - :param user_cfg Optional[dict]: The global config for the user, if it - was present. - :param target_override Optional[str]: The target to use, if provided on - the command line. - :param threads_override Optional[str]: The thread count to use, if - provided on the command line. - :raises DbtProfileError: If the profile is invalid or missing, or the - target could not be found - :returns Profile: The new Profile object. - """ - # user_cfg is not rendered since it only contains booleans. - # TODO: should it be, and the values coerced to bool? - target_name, profile_data = cls.render_profile( - raw_profile, profile_name, target_override, cli_vars - ) - - # valid connections never include the number of threads, but it's - # stored on a per-connection level in the raw configs - threads = profile_data.pop('threads', DEFAULT_THREADS) - if threads_override is not None: - threads = threads_override - - credentials = cls._credentials_from_profile( - profile_data, profile_name, target_name - ) - - return cls.from_credentials( - credentials=credentials, - profile_name=profile_name, - target_name=target_name, - threads=threads, - user_cfg=user_cfg - ) - - @classmethod - def from_raw_profiles(cls, raw_profiles, profile_name, cli_vars, - target_override=None, threads_override=None): - """ - :param raw_profiles dict: The profile data, from disk as yaml. - :param profile_name str: The profile name to use. - :param cli_vars dict: The command-line variables passed as arguments, - as a dict. - :param target_override Optional[str]: The target to use, if provided on - the command line. - :param threads_override Optional[str]: The thread count to use, if - provided on the command line. - :raises DbtProjectError: If there is no profile name specified in the - project or the command line arguments - :raises DbtProfileError: If the profile is invalid or missing, or the - target could not be found - :returns Profile: The new Profile object. - """ - if profile_name not in raw_profiles: - raise DbtProjectError( - "Could not find profile named '{}'".format(profile_name) - ) - - # First, we've already got our final decision on profile name, and we - # don't render keys, so we can pluck that out - raw_profile = raw_profiles[profile_name] - - user_cfg = raw_profiles.get('config') - - return cls.from_raw_profile_info( - raw_profile=raw_profile, - profile_name=profile_name, - cli_vars=cli_vars, - user_cfg=user_cfg, - target_override=target_override, - threads_override=threads_override, - ) - - @classmethod - def from_args(cls, args, project_profile_name=None, cli_vars=None): - """Given the raw profiles as read from disk and the name of the desired - profile if specified, return the profile component of the runtime - config. - - :param args argparse.Namespace: The arguments as parsed from the cli. - :param cli_vars dict: The command-line variables passed as arguments, - as a dict. - :param project_profile_name Optional[str]: The profile name, if - specified in a project. - :raises DbtProjectError: If there is no profile name specified in the - project or the command line arguments, or if the specified profile - is not found - :raises DbtProfileError: If the profile is invalid or missing, or the - target could not be found. - :returns Profile: The new Profile object. - """ - if cli_vars is None: - cli_vars = dbt.utils.parse_cli_vars(getattr(args, 'vars', '{}')) - - threads_override = getattr(args, 'threads', None) - target_override = getattr(args, 'target', None) - raw_profiles = read_profile(args.profiles_dir) - profile_name = cls.pick_profile_name(args.profile, - project_profile_name) - - return cls.from_raw_profiles( - raw_profiles=raw_profiles, - profile_name=profile_name, - cli_vars=cli_vars, - target_override=target_override, - threads_override=threads_override - ) - - -def package_config_from_data(packages_data): - if packages_data is None: - packages_data = {'packages': []} - - try: - packages = PackageConfig(**packages_data) - except dbt.exceptions.ValidationException as e: - raise DbtProjectError('Invalid package config: {}'.format(str(e))) - return packages - - -def package_data_from_root(project_root): - package_filepath = dbt.clients.system.resolve_path_from_base( - 'packages.yml', project_root - ) - - if dbt.clients.system.path_exists(package_filepath): - packages_dict = _load_yaml(package_filepath) - else: - packages_dict = None - return packages_dict - - -def package_config_from_root(project_root): - packages_dict = package_data_from_root(project_root) - return package_config_from_data(packages_dict) - - -class RuntimeConfig(Project, Profile): - """The runtime configuration, as constructed from its components. There's a - lot because there is a lot of stuff! - """ - def __init__(self, project_name, version, project_root, source_paths, - macro_paths, data_paths, test_paths, analysis_paths, - docs_paths, target_path, clean_targets, log_path, - modules_path, quoting, models, on_run_start, on_run_end, - archive, seeds, dbt_version, profile_name, target_name, - config, threads, credentials, packages, args): - # 'vars' - self.args = args - self.cli_vars = dbt.utils.parse_cli_vars(getattr(args, 'vars', '{}')) - # 'project' - Project.__init__( - self, - project_name=project_name, - version=version, - project_root=project_root, - profile_name=profile_name, - source_paths=source_paths, - macro_paths=macro_paths, - data_paths=data_paths, - test_paths=test_paths, - analysis_paths=analysis_paths, - docs_paths=docs_paths, - target_path=target_path, - clean_targets=clean_targets, - log_path=log_path, - modules_path=modules_path, - quoting=quoting, - models=models, - on_run_start=on_run_start, - on_run_end=on_run_end, - archive=archive, - seeds=seeds, - dbt_version=dbt_version, - packages=packages - ) - # 'profile' - Profile.__init__( - self, - profile_name=profile_name, - target_name=target_name, - config=config, - threads=threads, - credentials=credentials - ) - self.validate() - - @classmethod - def from_parts(cls, project, profile, args): - """Instantiate a RuntimeConfig from its components. - - :param profile Profile: A parsed dbt Profile. - :param project Project: A parsed dbt Project. - :param args argparse.Namespace: The parsed command-line arguments. - :returns RuntimeConfig: The new configuration. - """ - quoting = deepcopy( - get_relation_class_by_name(profile.credentials.type) - .DEFAULTS['quote_policy'] - ) - quoting.update(project.quoting) - return cls( - project_name=project.project_name, - version=project.version, - project_root=project.project_root, - source_paths=project.source_paths, - macro_paths=project.macro_paths, - data_paths=project.data_paths, - test_paths=project.test_paths, - analysis_paths=project.analysis_paths, - docs_paths=project.docs_paths, - target_path=project.target_path, - clean_targets=project.clean_targets, - log_path=project.log_path, - modules_path=project.modules_path, - quoting=quoting, - models=project.models, - on_run_start=project.on_run_start, - on_run_end=project.on_run_end, - archive=project.archive, - seeds=project.seeds, - dbt_version=project.dbt_version, - packages=project.packages, - profile_name=profile.profile_name, - target_name=profile.target_name, - config=profile.config, - threads=profile.threads, - credentials=profile.credentials, - args=args - ) - - def new_project(self, project_root): - """Given a new project root, read in its project dictionary, supply the - existing project's profile info, and create a new project file. - - :param project_root str: A filepath to a dbt project. - :raises DbtProfileError: If the profile is invalid. - :raises DbtProjectError: If project is missing or invalid. - :returns RuntimeConfig: The new configuration. - """ - # copy profile - profile = Profile(**self.to_profile_info()) - profile.validate() - # load the new project and its packages. Don't pass cli variables. - project = Project.from_project_root(project_root, {}) - - cfg = self.from_parts( - project=project, - profile=profile, - args=deepcopy(self.args), - ) - # force our quoting back onto the new project. - cfg.quoting = deepcopy(self.quoting) - return cfg - - def serialize(self): - """Serialize the full configuration to a single dictionary. For any - instance that has passed validate() (which happens in __init__), it - matches the Configuration contract. - - Note that args are not serialized. - - :returns dict: The serialized configuration. - """ - result = self.to_project_config(with_packages=True) - result.update(self.to_profile_info(serialize_credentials=True)) - result['cli_vars'] = deepcopy(self.cli_vars) - return result - - def __str__(self): - return pprint.pformat(self.serialize()) - - def validate(self): - """Validate the configuration against its contract. - - :raises DbtProjectError: If the configuration fails validation. - """ - try: - Configuration(**self.serialize()) - except dbt.exceptions.ValidationException as e: - raise DbtProjectError(str(e)) - - if getattr(self.args, 'version_check', False): - self.validate_version() - - @classmethod - def from_args(cls, args): - """Given arguments, read in dbt_project.yml from the current directory, - read in packages.yml if it exists, and use them to find the profile to - load. - - :param args argparse.Namespace: The arguments as parsed from the cli. - :raises DbtProjectError: If the project is invalid or missing. - :raises DbtProfileError: If the profile is invalid or missing. - :raises ValidationException: If the cli variables are invalid. - """ - cli_vars = dbt.utils.parse_cli_vars(getattr(args, 'vars', '{}')) - - # build the project and read in packages.yml - project = Project.from_current_directory(cli_vars) - - # build the profile - profile = Profile.from_args( - args=args, - project_profile_name=project.profile_name, - cli_vars=cli_vars - ) - - return cls.from_parts( - project=project, - profile=profile, - args=args - ) - - -def _load_yaml(path): - contents = dbt.clients.system.load_file_contents(path) - return dbt.clients.yaml_helper.load_yaml_text(contents) - - -def _get_config_paths(config, path=(), paths=None): - if paths is None: - paths = set() - - for key, value in config.items(): - if isinstance(value, dict): - if key in DBTConfigKeys: - if path not in paths: - paths.add(path) - else: - _get_config_paths(value, path + (key,), paths) - else: - if path not in paths: - paths.add(path) - - return frozenset(paths) - - -def _is_config_used(path, fqns): - if fqns: - for fqn in fqns: - if len(path) <= len(fqn) and fqn[:len(path)] == path: - return True - return False diff --git a/dbt/include/__init__.py b/dbt/include/__init__.py deleted file mode 100644 index 4a940f8a518..00000000000 --- a/dbt/include/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ - -import os - -GLOBAL_DBT_MODULES_PATH = os.path.dirname(__file__) -GLOBAL_PROJECT_NAME = 'dbt' - -DOCS_INDEX_FILE_PATH = os.path.normpath( - os.path.join(GLOBAL_DBT_MODULES_PATH, "index.html")) diff --git a/dbt/task/__init__.py b/dbt/task/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbt/templates.py b/dbt/templates.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbt/ui/__init__.py b/dbt/ui/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/plugins/bigquery/dbt/__init__.py b/plugins/bigquery/dbt/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/bigquery/dbt/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/bigquery/dbt/adapters/__init__.py b/plugins/bigquery/dbt/adapters/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/bigquery/dbt/adapters/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/dbt/adapters/bigquery/__init__.py b/plugins/bigquery/dbt/adapters/bigquery/__init__.py similarity index 54% rename from dbt/adapters/bigquery/__init__.py rename to plugins/bigquery/dbt/adapters/bigquery/__init__.py index c9822036cf6..5707c1690e9 100644 --- a/dbt/adapters/bigquery/__init__.py +++ b/plugins/bigquery/dbt/adapters/bigquery/__init__.py @@ -3,5 +3,10 @@ from dbt.adapters.bigquery.relation import BigQueryRelation from dbt.adapters.bigquery.impl import BigQueryAdapter -Adapter = BigQueryAdapter -Credentials = BigQueryCredentials +from dbt.adapters.base import AdapterPlugin +from dbt.include import bigquery + +Plugin = AdapterPlugin( + adapter=BigQueryAdapter, + credentials=BigQueryCredentials, + include_path=bigquery.PACKAGE_PATH) diff --git a/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py similarity index 100% rename from dbt/adapters/bigquery/connections.py rename to plugins/bigquery/dbt/adapters/bigquery/connections.py diff --git a/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py similarity index 100% rename from dbt/adapters/bigquery/impl.py rename to plugins/bigquery/dbt/adapters/bigquery/impl.py diff --git a/dbt/adapters/bigquery/relation.py b/plugins/bigquery/dbt/adapters/bigquery/relation.py similarity index 100% rename from dbt/adapters/bigquery/relation.py rename to plugins/bigquery/dbt/adapters/bigquery/relation.py diff --git a/plugins/bigquery/dbt/include/__init__.py b/plugins/bigquery/dbt/include/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/bigquery/dbt/include/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/bigquery/dbt/include/bigquery/__init__.py b/plugins/bigquery/dbt/include/bigquery/__init__.py new file mode 100644 index 00000000000..87098354afd --- /dev/null +++ b/plugins/bigquery/dbt/include/bigquery/__init__.py @@ -0,0 +1,2 @@ +import os +PACKAGE_PATH = os.path.dirname(os.path.dirname(__file__)) diff --git a/plugins/bigquery/dbt/include/bigquery/dbt_project.yml b/plugins/bigquery/dbt/include/bigquery/dbt_project.yml new file mode 100644 index 00000000000..edae5386994 --- /dev/null +++ b/plugins/bigquery/dbt/include/bigquery/dbt_project.yml @@ -0,0 +1,5 @@ + +name: dbt_bigquery +version: 1.0 + +macro-paths: ["macros"] diff --git a/dbt/include/global_project/macros/adapters/bigquery.sql b/plugins/bigquery/dbt/include/bigquery/macros/adapters.sql similarity index 100% rename from dbt/include/global_project/macros/adapters/bigquery.sql rename to plugins/bigquery/dbt/include/bigquery/macros/adapters.sql diff --git a/dbt/include/global_project/macros/etc/bigquery.sql b/plugins/bigquery/dbt/include/bigquery/macros/etc.sql similarity index 100% rename from dbt/include/global_project/macros/etc/bigquery.sql rename to plugins/bigquery/dbt/include/bigquery/macros/etc.sql diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/archive.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/archive.sql new file mode 100644 index 00000000000..5548b71a7e8 --- /dev/null +++ b/plugins/bigquery/dbt/include/bigquery/macros/materializations/archive.sql @@ -0,0 +1,23 @@ +{% macro bigquery__create_temporary_table(sql, relation) %} + {% set tmp_relation = adapter.create_temporary_table(sql) %} + {{ return(tmp_relation) }} +{% endmacro %} + + +{% macro bigquery__archive_scd_hash() %} + to_hex(md5(concat(cast(`dbt_pk` as string), '|', cast(`dbt_updated_at` as string)))) +{% endmacro %} + + +{% macro bigquery__create_columns(relation, columns) %} + {{ adapter.alter_table_add_columns(relation, columns) }} +{% endmacro %} + + +{% macro bigquery__archive_update(target_relation, tmp_relation) %} + update {{ target_relation }} as dest + set dest.{{ adapter.quote('valid_to') }} = tmp.{{ adapter.quote('valid_to') }} + from {{ tmp_relation }} as tmp + where tmp.{{ adapter.quote('scd_id') }} = dest.{{ adapter.quote('scd_id') }} + and {{ adapter.quote('change_type') }} = 'update'; +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental/bigquery_incremental.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/incremental/bigquery_incremental.sql rename to plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/merge.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/merge.sql new file mode 100644 index 00000000000..8e8f42a3563 --- /dev/null +++ b/plugins/bigquery/dbt/include/bigquery/macros/materializations/merge.sql @@ -0,0 +1,3 @@ +{% macro bigquery__get_merge_sql(target, source, unique_key, dest_columns) %} + {{ common_get_merge_sql(target, source, unique_key, dest_columns) }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/seed/bigquery.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/seed/bigquery.sql rename to plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql diff --git a/dbt/include/global_project/macros/materializations/table/bigquery_table.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/table/bigquery_table.sql rename to plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql diff --git a/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql b/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql new file mode 100644 index 00000000000..561c38bb5c3 --- /dev/null +++ b/plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql @@ -0,0 +1,13 @@ + +{% macro bigquery__handle_existing_table(full_refresh, non_destructive_mode, old_relation) %} + {%- if full_refresh and not non_destructive_mode -%} + {{ adapter.drop_relation(old_relation) }} + {%- else -%} + {{ exceptions.relation_wrong_type(old_relation, 'view') }} + {%- endif -%} +{% endmacro %} + + +{% materialization view, adapter='bigquery' -%} + {{ create_or_replace_view(run_outside_transaction_hooks=False) }} +{%- endmaterialization %} diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py new file mode 100644 index 00000000000..5c743575d46 --- /dev/null +++ b/plugins/bigquery/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +from setuptools import find_packages +from distutils.core import setup + +package_name = "dbt-bigquery" +package_version = "0.13.0a1" +description = """The bigquery adapter plugin for dbt (data build tool)""" + + +setup( + name=package_name, + version=package_version, + description=description, + long_description_content_type=description, + author="Fishtown Analytics", + author_email="info@fishtownanalytics.com", + url="https://github.com/fishtown-analytics/dbt", + packages=find_packages(), + package_data={ + 'dbt': [ + 'include/bigquery/macros/*.sql', + 'include/bigquery/macros/**/*.sql', + ] + }, + install_requires=[ + 'dbt-core=={}'.format(package_version), + 'google-cloud-bigquery>=1.0.0,<2', + ] +) diff --git a/plugins/postgres/dbt/__init__.py b/plugins/postgres/dbt/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/postgres/dbt/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/postgres/dbt/adapters/__init__.py b/plugins/postgres/dbt/adapters/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/postgres/dbt/adapters/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/postgres/dbt/adapters/postgres/__init__.py b/plugins/postgres/dbt/adapters/postgres/__init__.py new file mode 100644 index 00000000000..f139484e807 --- /dev/null +++ b/plugins/postgres/dbt/adapters/postgres/__init__.py @@ -0,0 +1,11 @@ +from dbt.adapters.postgres.connections import PostgresConnectionManager +from dbt.adapters.postgres.connections import PostgresCredentials +from dbt.adapters.postgres.impl import PostgresAdapter + +from dbt.adapters.base import AdapterPlugin +from dbt.include import postgres + +Plugin = AdapterPlugin( + adapter=PostgresAdapter, + credentials=PostgresCredentials, + include_path=postgres.PACKAGE_PATH) diff --git a/dbt/adapters/postgres/connections.py b/plugins/postgres/dbt/adapters/postgres/connections.py similarity index 100% rename from dbt/adapters/postgres/connections.py rename to plugins/postgres/dbt/adapters/postgres/connections.py diff --git a/dbt/adapters/postgres/impl.py b/plugins/postgres/dbt/adapters/postgres/impl.py similarity index 100% rename from dbt/adapters/postgres/impl.py rename to plugins/postgres/dbt/adapters/postgres/impl.py diff --git a/plugins/postgres/dbt/include/__init__.py b/plugins/postgres/dbt/include/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/postgres/dbt/include/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/postgres/dbt/include/postgres/__init__.py b/plugins/postgres/dbt/include/postgres/__init__.py new file mode 100644 index 00000000000..87098354afd --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/__init__.py @@ -0,0 +1,2 @@ +import os +PACKAGE_PATH = os.path.dirname(os.path.dirname(__file__)) diff --git a/plugins/postgres/dbt/include/postgres/dbt_project.yml b/plugins/postgres/dbt/include/postgres/dbt_project.yml new file mode 100644 index 00000000000..266eba33db9 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/dbt_project.yml @@ -0,0 +1,5 @@ + +name: dbt_postgres +version: 1.0 + +macro-paths: ["macros"] diff --git a/dbt/include/global_project/macros/catalog/postgres_catalog.sql b/plugins/postgres/dbt/include/postgres/macros/catalog.sql similarity index 100% rename from dbt/include/global_project/macros/catalog/postgres_catalog.sql rename to plugins/postgres/dbt/include/postgres/macros/catalog.sql diff --git a/dbt/include/global_project/macros/relations/postgres_relations.sql b/plugins/postgres/dbt/include/postgres/macros/relations.sql similarity index 100% rename from dbt/include/global_project/macros/relations/postgres_relations.sql rename to plugins/postgres/dbt/include/postgres/macros/relations.sql diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py new file mode 100644 index 00000000000..c91de0f680a --- /dev/null +++ b/plugins/postgres/setup.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +from setuptools import find_packages +from distutils.core import setup + +package_name = "dbt-postgres" +package_version = "0.13.0a1" +description = """The postgres adpter plugin for dbt (data build tool)""" + +setup( + name=package_name, + version=package_version, + description=description, + long_description_content_type=description, + author="Fishtown Analytics", + author_email="info@fishtownanalytics.com", + url="https://github.com/fishtown-analytics/dbt", + packages=find_packages(), + package_data={ + 'dbt': [ + 'include/postgres/macros/*.sql', + ] + }, + install_requires=[ + 'dbt-core=={}'.format(package_version), + 'psycopg2>=2.7.5,<2.8', + ] +) diff --git a/plugins/redshift/dbt/__init__.py b/plugins/redshift/dbt/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/redshift/dbt/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/redshift/dbt/adapters/__init__.py b/plugins/redshift/dbt/adapters/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/redshift/dbt/adapters/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/redshift/dbt/adapters/redshift/__init__.py b/plugins/redshift/dbt/adapters/redshift/__init__.py new file mode 100644 index 00000000000..336eb55d17d --- /dev/null +++ b/plugins/redshift/dbt/adapters/redshift/__init__.py @@ -0,0 +1,13 @@ +from dbt.adapters.redshift.connections import RedshiftConnectionManager +from dbt.adapters.redshift.connections import RedshiftCredentials +from dbt.adapters.redshift.impl import RedshiftAdapter + + +from dbt.adapters.base import AdapterPlugin +from dbt.include import redshift + +Plugin = AdapterPlugin( + adapter=RedshiftAdapter, + credentials=RedshiftCredentials, + include_path=redshift.PACKAGE_PATH, + dependencies=['postgres']) diff --git a/dbt/adapters/redshift/connections.py b/plugins/redshift/dbt/adapters/redshift/connections.py similarity index 100% rename from dbt/adapters/redshift/connections.py rename to plugins/redshift/dbt/adapters/redshift/connections.py diff --git a/dbt/adapters/redshift/impl.py b/plugins/redshift/dbt/adapters/redshift/impl.py similarity index 100% rename from dbt/adapters/redshift/impl.py rename to plugins/redshift/dbt/adapters/redshift/impl.py diff --git a/plugins/redshift/dbt/include/__init__.py b/plugins/redshift/dbt/include/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/redshift/dbt/include/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/redshift/dbt/include/redshift/__init__.py b/plugins/redshift/dbt/include/redshift/__init__.py new file mode 100644 index 00000000000..ef10d7896ca --- /dev/null +++ b/plugins/redshift/dbt/include/redshift/__init__.py @@ -0,0 +1,3 @@ +import os +from dbt.include.postgres import PACKAGE_PATH as POSTGRES_PACKAGE_PATH +PACKAGE_PATH = os.path.dirname(os.path.dirname(__file__)) diff --git a/plugins/redshift/dbt/include/redshift/dbt_project.yml b/plugins/redshift/dbt/include/redshift/dbt_project.yml new file mode 100644 index 00000000000..edcd805ab7a --- /dev/null +++ b/plugins/redshift/dbt/include/redshift/dbt_project.yml @@ -0,0 +1,5 @@ + +name: dbt_redshift +version: 1.0 + +macro-paths: ["macros"] diff --git a/dbt/include/global_project/macros/adapters/redshift.sql b/plugins/redshift/dbt/include/redshift/macros/adapters.sql similarity index 100% rename from dbt/include/global_project/macros/adapters/redshift.sql rename to plugins/redshift/dbt/include/redshift/macros/adapters.sql diff --git a/dbt/include/global_project/macros/catalog/redshift_catalog.sql b/plugins/redshift/dbt/include/redshift/macros/catalog.sql similarity index 100% rename from dbt/include/global_project/macros/catalog/redshift_catalog.sql rename to plugins/redshift/dbt/include/redshift/macros/catalog.sql diff --git a/dbt/include/global_project/macros/relations/redshift_relations.sql b/plugins/redshift/dbt/include/redshift/macros/relations.sql similarity index 100% rename from dbt/include/global_project/macros/relations/redshift_relations.sql rename to plugins/redshift/dbt/include/redshift/macros/relations.sql diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py new file mode 100644 index 00000000000..a62328edf0d --- /dev/null +++ b/plugins/redshift/setup.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +from setuptools import find_packages +from distutils.core import setup + +package_name = "dbt-redshift" +package_version = "0.13.0a1" +description = """The redshift adapter plugin for dbt (data build tool)""" + + +setup( + name=package_name, + version=package_version, + description=description, + long_description_content_type=description, + author="Fishtown Analytics", + author_email="info@fishtownanalytics.com", + url="https://github.com/fishtown-analytics/dbt", + packages=find_packages(), + package_data={ + 'dbt': [ + 'include/redshift/macros/*.sql', + ] + }, + install_requires=[ + 'dbt-core=={}'.format(package_version), + 'dbt-postgres=={}'.format(package_version), + 'boto3>=1.6.23,<1.8.0', + 'botocore>=1.9.23,<1.11.0', + 'psycopg2>=2.7.5,<2.8', + ] +) diff --git a/plugins/snowflake/dbt/__init__.py b/plugins/snowflake/dbt/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/snowflake/dbt/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/snowflake/dbt/adapters/__init__.py b/plugins/snowflake/dbt/adapters/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/snowflake/dbt/adapters/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/dbt/adapters/snowflake/__init__.py b/plugins/snowflake/dbt/adapters/snowflake/__init__.py similarity index 55% rename from dbt/adapters/snowflake/__init__.py rename to plugins/snowflake/dbt/adapters/snowflake/__init__.py index c2b982df79a..1ac7dcbdf2f 100644 --- a/dbt/adapters/snowflake/__init__.py +++ b/plugins/snowflake/dbt/adapters/snowflake/__init__.py @@ -3,6 +3,10 @@ from dbt.adapters.snowflake.relation import SnowflakeRelation from dbt.adapters.snowflake.impl import SnowflakeAdapter +from dbt.adapters.base import AdapterPlugin +from dbt.include import snowflake -Adapter = SnowflakeAdapter -Credentials = SnowflakeCredentials +Plugin = AdapterPlugin( + adapter=SnowflakeAdapter, + credentials=SnowflakeCredentials, + include_path=snowflake.PACKAGE_PATH) diff --git a/dbt/adapters/snowflake/connections.py b/plugins/snowflake/dbt/adapters/snowflake/connections.py similarity index 100% rename from dbt/adapters/snowflake/connections.py rename to plugins/snowflake/dbt/adapters/snowflake/connections.py diff --git a/dbt/adapters/snowflake/impl.py b/plugins/snowflake/dbt/adapters/snowflake/impl.py similarity index 100% rename from dbt/adapters/snowflake/impl.py rename to plugins/snowflake/dbt/adapters/snowflake/impl.py diff --git a/dbt/adapters/snowflake/relation.py b/plugins/snowflake/dbt/adapters/snowflake/relation.py similarity index 100% rename from dbt/adapters/snowflake/relation.py rename to plugins/snowflake/dbt/adapters/snowflake/relation.py diff --git a/plugins/snowflake/dbt/include/__init__.py b/plugins/snowflake/dbt/include/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/plugins/snowflake/dbt/include/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/plugins/snowflake/dbt/include/snowflake/__init__.py b/plugins/snowflake/dbt/include/snowflake/__init__.py new file mode 100644 index 00000000000..87098354afd --- /dev/null +++ b/plugins/snowflake/dbt/include/snowflake/__init__.py @@ -0,0 +1,2 @@ +import os +PACKAGE_PATH = os.path.dirname(os.path.dirname(__file__)) diff --git a/plugins/snowflake/dbt/include/snowflake/dbt_project.yml b/plugins/snowflake/dbt/include/snowflake/dbt_project.yml new file mode 100644 index 00000000000..587a22b5232 --- /dev/null +++ b/plugins/snowflake/dbt/include/snowflake/dbt_project.yml @@ -0,0 +1,5 @@ + +name: dbt_snowflake +version: 1.0 + +macro-paths: ["macros"] diff --git a/dbt/include/global_project/macros/adapters/snowflake.sql b/plugins/snowflake/dbt/include/snowflake/macros/adapters.sql similarity index 100% rename from dbt/include/global_project/macros/adapters/snowflake.sql rename to plugins/snowflake/dbt/include/snowflake/macros/adapters.sql diff --git a/dbt/include/global_project/macros/catalog/snowflake_catalog.sql b/plugins/snowflake/dbt/include/snowflake/macros/catalog.sql similarity index 100% rename from dbt/include/global_project/macros/catalog/snowflake_catalog.sql rename to plugins/snowflake/dbt/include/snowflake/macros/catalog.sql diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql new file mode 100644 index 00000000000..ac92f2ef26e --- /dev/null +++ b/plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql @@ -0,0 +1,3 @@ +{% macro snowflake__get_merge_sql(target, source, unique_key, dest_columns) %} + {{ common_get_merge_sql(target, source, unique_key, dest_columns) }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/table/snowflake_table.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql similarity index 100% rename from dbt/include/global_project/macros/materializations/table/snowflake_table.sql rename to plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql diff --git a/plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql b/plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql new file mode 100644 index 00000000000..d1dde6e052a --- /dev/null +++ b/plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql @@ -0,0 +1,3 @@ +{% materialization view, adapter='snowflake' -%} + {{ create_or_replace_view() }} +{%- endmaterialization %} diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py new file mode 100644 index 00000000000..24be4e60ee1 --- /dev/null +++ b/plugins/snowflake/setup.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +from setuptools import find_packages +from distutils.core import setup + +package_name = "dbt-snowflake" +package_version = "0.13.0a1" +description = """The snowflake adapter plugin for dbt (data build tool)""" + + +setup( + name=package_name, + version=package_version, + description=description, + long_description_content_type=description, + author="Fishtown Analytics", + author_email="info@fishtownanalytics.com", + url="https://github.com/fishtown-analytics/dbt", + packages=find_packages(), + package_data={ + 'dbt': [ + 'include/snowflake/macros/*.sql', + 'include/snowflake/macros/**/*.sql', + ] + }, + install_requires=[ + 'dbt-core=={}'.format(package_version), + 'snowflake-connector-python>=1.4.9', + 'boto3>=1.6.23,<1.8.0', + 'botocore>=1.9.23,<1.11.0', + ] +) diff --git a/requirements.txt b/requirements.txt index c84b33a5a62..1c185b37d46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,5 @@ --e .[postgres,snowflake,redshift,bigquery] +-e ./core +-e ./plugins/postgres +-e ./plugins/redshift +-e ./plugins/snowflake +-e ./plugins/bigquery diff --git a/test/integration/006_simple_dependency_test/test_local_dependency.py b/test/integration/006_simple_dependency_test/test_local_dependency.py index d09aa9de848..17c4c50e16f 100644 --- a/test/integration/006_simple_dependency_test/test_local_dependency.py +++ b/test/integration/006_simple_dependency_test/test_local_dependency.py @@ -63,7 +63,7 @@ def configured_schema(self): return 'configured_{}_macro'.format(self.unique_schema()) @attr(type='postgres') - @mock.patch('dbt.config.get_installed_version') + @mock.patch('dbt.config.project.get_installed_version') def test_postgres_local_dependency_out_of_date(self, mock_get): mock_get.return_value = dbt.semver.VersionSpecifier.from_version_string('0.0.1') self.run_dbt(['deps']) @@ -72,7 +72,7 @@ def test_postgres_local_dependency_out_of_date(self, mock_get): self.assertIn('--no-version-check', str(e.exception)) @attr(type='postgres') - @mock.patch('dbt.config.get_installed_version') + @mock.patch('dbt.config.project.get_installed_version') def test_postgres_local_dependency_out_of_date_no_check(self, mock_get): mock_get.return_value = dbt.semver.VersionSpecifier.from_version_string('0.0.1') self.run_dbt(['deps']) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 3e85e3122ce..979cf12f7b3 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -30,6 +30,19 @@ def _read_file(path): return fp.read() +def _normalize(path): + """On windows, neither is enough on its own: + + >>> normcase('C:\\documents/ALL CAPS/subdir\\..') + 'c:\\documents\\all caps\\subdir\\..' + >>> normpath('C:\\documents/ALL CAPS/subdir\\..') + 'C:\\documents\\ALL CAPS' + >>> normpath(normcase('C:\\documents/ALL CAPS/subdir\\..')) + 'c:\\documents\\all caps' + """ + return os.path.normcase(os.path.normpath(path)) + + class TestDocsGenerate(DBTIntegrationTest): def setUp(self): super(TestDocsGenerate,self).setUp() @@ -41,8 +54,9 @@ def schema(self): @staticmethod def dir(path): - return os.path.normpath( - os.path.join('test/integration/029_docs_generate_tests', path)) + return _normalize( + os.path.join('test/integration/029_docs_generate_tests', path) + ) @property def models(self): @@ -77,8 +91,8 @@ def run_and_generate(self, extra=None, seed_count=1, model_count=1): self.assertEqual(len(self.run_dbt(["seed"])), seed_count) self.assertEqual(len(self.run_dbt()), model_count) - os.remove(os.path.normpath('target/manifest.json')) - os.remove(os.path.normpath('target/run_results.json')) + os.remove(_normalize('target/manifest.json')) + os.remove(_normalize('target/run_results.json')) self.generate_start_time = datetime.utcnow() self.run_dbt(['docs', 'generate']) @@ -717,15 +731,16 @@ def verify_manifest_macros(self, manifest): self.assertTrue(len(macro['raw_sql']) > 10) without_sql = {k: v for k, v in macro.items() if k != 'raw_sql'} # Windows means we can't hard-code this. - helpers_path = os.path.normpath('macros/materializations/helpers.sql') + helpers_path = _normalize('macros/materializations/helpers.sql') self.assertEqual( without_sql, { 'path': helpers_path, 'original_file_path': helpers_path, 'package_name': 'dbt', - 'root_path': os.path.join(os.getcwd(), 'dbt','include', - 'global_project'), + 'root_path': _normalize(os.path.join( + os.getcwd(), 'core', 'dbt','include', 'global_project' + )), 'name': 'column_list', 'unique_id': 'macro.dbt.column_list', 'tags': [], @@ -843,7 +858,7 @@ def expected_seeded_manifest(self): 'name': 'not_null_model_id', 'original_file_path': self.dir('models/schema.yml'), 'package_name': 'test', - 'path': os.path.normpath('schema_test/not_null_model_id.sql'), + 'path': _normalize('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ test_not_null(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', @@ -872,7 +887,7 @@ def expected_seeded_manifest(self): 'name': 'nothing_model_', 'original_file_path': self.dir('models/schema.yml'), 'package_name': 'test', - 'path': os.path.normpath('schema_test/nothing_model_.sql'), + 'path': _normalize('schema_test/nothing_model_.sql'), 'raw_sql': "{{ test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', @@ -902,7 +917,7 @@ def expected_seeded_manifest(self): 'name': 'unique_model_id', 'original_file_path': self.dir('models/schema.yml'), 'package_name': 'test', - 'path': os.path.normpath('schema_test/unique_model_id.sql'), + 'path': _normalize('schema_test/unique_model_id.sql'), 'raw_sql': "{{ test_unique(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', @@ -1633,7 +1648,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'fail': None, 'node': { 'alias': 'model', - 'build_path': os.path.normpath( + 'build_path': _normalize( 'target/compiled/test/model.sql' ), 'columns': { @@ -1689,7 +1704,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'fail': None, 'node': { 'alias': 'seed', - 'build_path': os.path.normpath( + 'build_path': _normalize( 'target/compiled/test/seed.csv' ), 'columns': {}, @@ -1734,7 +1749,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'fail': None, 'node': { 'alias': 'not_null_model_id', - 'build_path': os.path.normpath('target/compiled/test/schema_test/not_null_model_id.sql'), + 'build_path': _normalize('target/compiled/test/schema_test/not_null_model_id.sql'), 'column_name': 'id', 'columns': {}, 'compiled': True, @@ -1759,7 +1774,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'name': 'not_null_model_id', 'original_file_path': self.dir('models/schema.yml'), 'package_name': 'test', - 'path': os.path.normpath('schema_test/not_null_model_id.sql'), + 'path': _normalize('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ test_not_null(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', @@ -1778,7 +1793,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'fail': None, 'node': { 'alias': 'nothing_model_', - 'build_path': os.path.normpath('target/compiled/test/schema_test/nothing_model_.sql'), + 'build_path': _normalize('target/compiled/test/schema_test/nothing_model_.sql'), 'columns': {}, 'compiled': True, 'compiled_sql': AnyStringWith('select 0'), @@ -1802,7 +1817,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'name': 'nothing_model_', 'original_file_path': self.dir('models/schema.yml'), 'package_name': 'test', - 'path': os.path.normpath('schema_test/nothing_model_.sql'), + 'path': _normalize('schema_test/nothing_model_.sql'), 'raw_sql': "{{ test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', @@ -1821,7 +1836,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'fail': None, 'node': { 'alias': 'unique_model_id', - 'build_path': os.path.normpath('target/compiled/test/schema_test/unique_model_id.sql'), + 'build_path': _normalize('target/compiled/test/schema_test/unique_model_id.sql'), 'column_name': 'id', 'columns': {}, 'compiled': True, @@ -1846,7 +1861,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False): 'name': 'unique_model_id', 'original_file_path': self.dir('models/schema.yml'), 'package_name': 'test', - 'path': os.path.normpath('schema_test/unique_model_id.sql'), + 'path': _normalize('schema_test/unique_model_id.sql'), 'raw_sql': "{{ test_unique(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', @@ -1890,7 +1905,7 @@ def expected_postgres_references_run_results(self): 'fail': None, 'node': { 'alias': 'ephemeral_summary', - 'build_path': os.path.normpath( + 'build_path': _normalize( 'target/compiled/test/ephemeral_summary.sql' ), 'columns': { @@ -1973,7 +1988,7 @@ def expected_postgres_references_run_results(self): 'fail': None, 'node': { 'alias': 'view_summary', - 'build_path': os.path.normpath( + 'build_path': _normalize( 'target/compiled/test/view_summary.sql' ), 'alias': 'view_summary', @@ -2055,7 +2070,7 @@ def expected_postgres_references_run_results(self): 'fail': None, 'node': { 'alias': 'seed', - 'build_path': os.path.normpath( + 'build_path': _normalize( 'target/compiled/test/seed.csv' ), 'columns': {}, diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 7f924e02adf..92f268c33fa 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -475,7 +475,7 @@ def test_invalid_env_vars(self): self.env_override['env_value_port'] = 'hello' self.args.target = 'with-vars' with mock.patch.dict(os.environ, self.env_override): - with self.assertRaises(dbt.config.DbtProfileError) as exc: + with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: self.from_args() self.assertIn("not of type 'integer'", str(exc.exception)) @@ -800,7 +800,7 @@ def test__get_unused_resource_config_paths_empty(self): ))}, []) self.assertEqual(len(unused), 0) - @mock.patch.object(dbt.config, 'logger') + @mock.patch.object(dbt.config.project, 'logger') def test__warn_for_unused_resource_config_paths_empty(self, mock_logger): project = dbt.config.Project.from_project_config( self.default_project_data @@ -884,7 +884,7 @@ def test__get_unused_resource_config_paths(self): self.assertEqual(len(unused), 1) self.assertEqual(unused[0], ('models', 'my_test_project', 'baz')) - @mock.patch.object(dbt.config, 'logger') + @mock.patch.object(dbt.config.project, 'logger') def test__warn_for_unused_resource_config_paths(self, mock_logger): project = dbt.config.Project.from_project_config( self.default_project_data @@ -892,7 +892,7 @@ def test__warn_for_unused_resource_config_paths(self, mock_logger): unused = project.warn_for_unused_resource_config_paths(self.used, []) mock_logger.info.assert_called_once() - @mock.patch.object(dbt.config, 'logger') + @mock.patch.object(dbt.config.project, 'logger') def test__warn_for_unused_resource_config_paths_disabled(self, mock_logger): project = dbt.config.Project.from_project_config( self.default_project_data diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index dc63b0e3b83..f989d2a167e 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -9,7 +9,6 @@ import dbt.flags import dbt.linker import dbt.config -import dbt.templates import dbt.utils try: diff --git a/tox.ini b/tox.ini index 366bf7a3e4a..4feae5dc355 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = unit-py27, unit-py36, integration-postgres-py27, integration-postgres- [testenv:pep8] basepython = python3.6 -commands = /bin/bash -c '$(which pep8) dbt/ --exclude dbt/templates.py' +commands = /bin/bash -c '$(which pep8) core/dbt plugins/*/dbt' deps = -r{toxinidir}/dev_requirements.txt @@ -12,14 +12,22 @@ deps = basepython = python2.7 commands = /bin/bash -c '$(which nosetests) -v {posargs} test/unit' deps = - -e .[postgres,snowflake,redshift,bigquery] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres + -e {toxinidir}/plugins/snowflake + -e {toxinidir}/plugins/bigquery + -e {toxinidir}/plugins/redshift -r{toxinidir}/dev_requirements.txt [testenv:unit-py36] basepython = python3.6 commands = /bin/bash -c '{envpython} $(which nosetests) -v {posargs} test/unit' deps = - -e .[postgres,snowflake,redshift,bigquery] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres + -e {toxinidir}/plugins/snowflake + -e {toxinidir}/plugins/bigquery + -e {toxinidir}/plugins/redshift -r{toxinidir}/dev_requirements.txt [testenv:integration-postgres-py27] @@ -29,7 +37,8 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=postgres {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -e .[postgres] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres -r{toxinidir}/dev_requirements.txt [testenv:integration-snowflake-py27] @@ -39,7 +48,8 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=snowflake {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -e .[snowflake] + -e {toxinidir}/core + -e {toxinidir}/plugins/snowflake -r{toxinidir}/dev_requirements.txt [testenv:integration-bigquery-py27] @@ -49,7 +59,8 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=bigquery {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -e .[bigquery] + -e {toxinidir}/core + -e {toxinidir}/plugins/bigquery -r{toxinidir}/dev_requirements.txt [testenv:integration-redshift-py27] @@ -59,7 +70,9 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=redshift {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -e .[redshift] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres + -e {toxinidir}/plugins/redshift -r{toxinidir}/dev_requirements.txt [testenv:integration-postgres-py36] @@ -69,7 +82,8 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=postgres --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov {posargs} test/integration/*' deps = - -e .[postgres] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres -r{toxinidir}/dev_requirements.txt [testenv:integration-snowflake-py36] @@ -79,7 +93,8 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=snowflake {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -e .[snowflake] + -e {toxinidir}/core + -e {toxinidir}/plugins/snowflake -r{toxinidir}/dev_requirements.txt [testenv:integration-bigquery-py36] @@ -89,7 +104,8 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=bigquery {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -e .[bigquery] + -e {toxinidir}/core + -e {toxinidir}/plugins/bigquery -r{toxinidir}/dev_requirements.txt [testenv:integration-redshift-py36] @@ -99,7 +115,9 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v -a type=redshift {posargs} --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration/*' deps = - -e .[redshift] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres + -e {toxinidir}/plugins/redshift -r{toxinidir}/dev_requirements.txt [testenv:explicit-py27] @@ -109,7 +127,11 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v {posargs}' deps = - -e .[postgres,snowflake,redshift,bigquery] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres + -e {toxinidir}/plugins/snowflake + -e {toxinidir}/plugins/bigquery + -e {toxinidir}/plugins/redshift -r{toxinidir}/dev_requirements.txt [testenv:explicit-py36] @@ -119,7 +141,11 @@ setenv = HOME=/home/dbt_test_user commands = /bin/bash -c '{envpython} $(which nosetests) -v {posargs}' deps = - -e .[postgres,snowflake,redshift,bigquery] + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres + -e {toxinidir}/plugins/snowflake + -e {toxinidir}/plugins/bigquery + -e {toxinidir}/plugins/redshift -r{toxinidir}/dev_requirements.txt [testenv:pywin] @@ -130,5 +156,9 @@ setenv = DBT_INVOCATION_ENV = ci-appveyor commands = nosetests -v -a type=postgres -a type=snowflake -a type=bigquery --with-coverage --cover-branches --cover-html --cover-html-dir=htmlcov test/integration test/unit deps = - -e .[postgres,snowflake,redshift,bigquery] - -rdev_requirements.txt + -e {toxinidir}/core + -e {toxinidir}/plugins/postgres + -e {toxinidir}/plugins/snowflake + -e {toxinidir}/plugins/bigquery + -e {toxinidir}/plugins/redshift + -r{toxinidir}/dev_requirements.txt