Merge pull request #2640 from fishtown-analytics/feature/yaml-selections

Feature/yaml selections

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Add more helpful error message for misconfiguration in profiles.yml ([#2569](https://github.com/fishtown-analytics/dbt/issues/2569), [#2627](https://github.com/fishtown-analytics/dbt/pull/2627))
 ### Fixes
 - Adapter plugins can once again override plugins defined in core ([#2548](https://github.com/fishtown-analytics/dbt/issues/2548), [#2590](https://github.com/fishtown-analytics/dbt/pull/2590))
+- Added `--selector` argument and support for `selectors.yml` file to define selection mechanisms. ([#2172](https://github.com/fishtown-analytics/dbt/issues/2172), [#2640](https://github.com/fishtown-analytics/dbt/pull/2640))
 
 Contributors:
 - [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581))

core/dbt/config/project.py

@@ -19,6 +19,8 @@
 from dbt.exceptions import RecursionException
 from dbt.exceptions import SemverException
 from dbt.exceptions import validator_error_message
+from dbt.exceptions import RuntimeException
+from dbt.graph import SelectionSpec
 from dbt.helper_types import NoValue
 from dbt.semver import VersionSpecifier
 from dbt.semver import versions_compatible
@@ -37,6 +39,11 @@
 from hologram import ValidationError
 
 from .renderer import DbtProjectYamlRenderer
+from .selectors import (
+    selector_config_from_data,
+    selector_data_from_root,
+    SelectorConfig,
+)
 
 
 INVALID_VERSION_ERROR = """\
@@ -211,10 +218,12 @@ class PartialProject:
 
     def render(self, renderer):
         packages_dict = package_data_from_root(self.project_root)
+        selectors_dict = selector_data_from_root(self.project_root)
         return Project.render_from_dict(
             self.project_root,
             self.project_dict,
             packages_dict,
+            selectors_dict,
             renderer,
         )
 
@@ -310,6 +319,7 @@ class Project:
     vars: VarProvider
     dbt_version: List[VersionSpecifier]
     packages: Dict[str, Any]
+    selectors: SelectorConfig
     query_comment: QueryComment
     config_version: int
 
@@ -351,6 +361,7 @@ def from_project_config(
         cls,
         project_dict: Dict[str, Any],
         packages_dict: Optional[Dict[str, Any]] = None,
+        selectors_dict: Optional[Dict[str, Any]] = None,
     ) -> 'Project':
         """Create a project from its project and package configuration, as read
         by yaml.safe_load().
@@ -464,6 +475,11 @@ def from_project_config(
         except ValidationError as e:
             raise DbtProjectError(validator_error_message(e)) from e
 
+        try:
+            selectors = selector_config_from_data(selectors_dict)
+        except ValidationError as e:
+            raise DbtProjectError(validator_error_message(e)) from e
+
         project = cls(
             project_name=name,
             version=version,
@@ -488,6 +504,7 @@ def from_project_config(
             snapshots=snapshots,
             dbt_version=dbt_version,
             packages=packages,
+            selectors=selectors,
             query_comment=query_comment,
             sources=sources,
             vars=vars_value,
@@ -568,14 +585,21 @@ def render_from_dict(
         project_root: str,
         project_dict: Dict[str, Any],
         packages_dict: Dict[str, Any],
+        selectors_dict: Dict[str, Any],
         renderer: DbtProjectYamlRenderer,
     ) -> 'Project':
         rendered_project = renderer.render_data(project_dict)
         rendered_project['project-root'] = project_root
         package_renderer = renderer.get_package_renderer()
         rendered_packages = package_renderer.render_data(packages_dict)
+        selectors_renderer = renderer.get_selector_renderer()
+        rendered_selectors = selectors_renderer.render_data(selectors_dict)
         try:
-            return cls.from_project_config(rendered_project, rendered_packages)
+            return cls.from_project_config(
+                rendered_project,
+                rendered_packages,
+                rendered_selectors,
+            )
         except DbtProjectError as exc:
             if exc.path is None:
                 exc.path = os.path.join(project_root, 'dbt_project.yml')
@@ -659,6 +683,14 @@ def as_v1(self, all_projects: Iterable[str]):
         project.packages = self.packages
         return project
 
+    def get_selector(self, name: str) -> SelectionSpec:
+        if name not in self.selectors:
+            raise RuntimeException(
+                f'Could not find selector named {name}, expected one of '
+                f'{list(self.selectors)}'
+            )
+        return self.selectors[name]
+
 
 def v2_vars_to_v1(
     dst: Dict[str, Any], src_vars: Dict[str, Any], project_names: Set[str]

core/dbt/config/renderer.py

@@ -69,6 +69,9 @@ def name(self):
     def get_package_renderer(self) -> BaseRenderer:
         return PackageRenderer(self.context)
 
+    def get_selector_renderer(self) -> BaseRenderer:
+        return SelectorRenderer(self.context)
+
     def should_render_keypath_v1(self, keypath: Keypath) -> bool:
         if not keypath:
             return True
@@ -206,3 +209,9 @@ class PackageRenderer(BaseRenderer):
     @property
     def name(self):
         return 'Packages config'
+
+
+class SelectorRenderer(BaseRenderer):
+    @property
+    def name(self):
+        return 'Selector config'

core/dbt/config/runtime.py

@@ -106,6 +106,7 @@ def from_parts(
             snapshots=project.snapshots,
             dbt_version=project.dbt_version,
             packages=project.packages,
+            selectors=project.selectors,
             query_comment=project.query_comment,
             sources=project.sources,
             vars=project.vars,
@@ -361,8 +362,9 @@ def load_projects(
                 project = self.new_project(str(path))
             except DbtProjectError as e:
                 raise DbtProjectError(
-                    'Failed to read package at {}: {}'
-                    .format(path, e)
+                    f'Failed to read package: {e}',
+                    result_type='invalid_project',
+                    path=path,
                 ) from e
             else:
                 yield project.project_name, project
@@ -501,6 +503,7 @@ def from_parts(
             snapshots=project.snapshots,
             dbt_version=project.dbt_version,
             packages=project.packages,
+            selectors=project.selectors,
             query_comment=project.query_comment,
             sources=project.sources,
             vars=project.vars,

core/dbt/config/selectors.py

@@ -0,0 +1,104 @@
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+from hologram import ValidationError
+
+from .renderer import SelectorRenderer
+
+from dbt.clients.system import (
+    load_file_contents,
+    path_exists,
+    resolve_path_from_base,
+)
+from dbt.clients.yaml_helper import load_yaml_text
+from dbt.contracts.selection import SelectorFile
+from dbt.exceptions import DbtSelectorsError, RuntimeException
+from dbt.graph import parse_from_selectors_definition, SelectionSpec
+
+MALFORMED_SELECTOR_ERROR = """\
+The selectors.yml file in this project is malformed. Please double check
+the contents of this file and fix any errors before retrying.
+
+You can find more information on the syntax for this file here:
+https://docs.getdbt.com/docs/package-management
+
+Validator Error:
+{error}
+"""
+
+
+class SelectorConfig(Dict[str, SelectionSpec]):
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'SelectorConfig':
+        try:
+            selector_file = SelectorFile.from_dict(data)
+            selectors = parse_from_selectors_definition(selector_file)
+        except (ValidationError, RuntimeException) as exc:
+            raise DbtSelectorsError(
+                f'Could not read selector file data: {exc}',
+                result_type='invalid_selector',
+            ) from exc
+
+        return cls(selectors)
+
+    @classmethod
+    def render_from_dict(
+        cls,
+        data: Dict[str, Any],
+        renderer: SelectorRenderer,
+    ) -> 'SelectorConfig':
+        try:
+            rendered = renderer.render_data(data)
+        except (ValidationError, RuntimeException) as exc:
+            raise DbtSelectorsError(
+                f'Could not render selector data: {exc}',
+                result_type='invalid_selector',
+            ) from exc
+        return cls.from_dict(rendered)
+
+    @classmethod
+    def from_path(
+        cls, path: Path, renderer: SelectorRenderer,
+    ) -> 'SelectorConfig':
+        try:
+            data = load_yaml_text(load_file_contents(str(path)))
+        except (ValidationError, RuntimeException) as exc:
+            raise DbtSelectorsError(
+                f'Could not read selector file: {exc}',
+                result_type='invalid_selector',
+                path=path,
+            ) from exc
+
+        try:
+            return cls.render_from_dict(data, renderer)
+        except DbtSelectorsError as exc:
+            exc.path = path
+            raise
+
+
+def selector_data_from_root(project_root: str) -> Dict[str, Any]:
+    selector_filepath = resolve_path_from_base(
+        'selectors.yml', project_root
+    )
+
+    if path_exists(selector_filepath):
+        selectors_dict = load_yaml_text(load_file_contents(selector_filepath))
+    else:
+        selectors_dict = None
+    return selectors_dict
+
+
+def selector_config_from_data(
+    selectors_data: Optional[Dict[str, Any]]
+) -> SelectorConfig:
+    if selectors_data is None:
+        selectors_data = {'selectors': []}
+
+    try:
+        selectors = SelectorConfig.from_dict(selectors_data)
+    except ValidationError as e:
+        raise DbtSelectorsError(
+            MALFORMED_SELECTOR_ERROR.format(error=str(e.message)),
+            result_type='invalid_selector',
+        ) from e
+    return selectors

core/dbt/contracts/graph/unparsed.py

@@ -1,5 +1,9 @@
 from dbt.node_types import NodeType
-from dbt.contracts.util import Replaceable, Mergeable
+from dbt.contracts.util import (
+    AdditionalPropertiesMixin,
+    Mergeable,
+    Replaceable,
+)
 # trigger the PathEncoder
 import dbt.helper_types  # noqa:F401
 from dbt.exceptions import CompilationException
@@ -177,32 +181,12 @@ def __bool__(self):
 
 
 @dataclass
-class AdditionalPropertiesAllowed(ExtensibleJsonSchemaMixin):
+class AdditionalPropertiesAllowed(
+    AdditionalPropertiesMixin,
+    ExtensibleJsonSchemaMixin
+):
     _extra: Dict[str, Any] = field(default_factory=dict)
 
-    @property
-    def extra(self):
-        return self._extra
-
-    @classmethod
-    def from_dict(cls, data, validate=True):
-        self = super().from_dict(data=data, validate=validate)
-        keys = self.to_dict(validate=False, omit_none=False)
-        for key, value in data.items():
-            if key not in keys:
-                self._extra[key] = value
-        return self
-
-    def to_dict(self, omit_none=True, validate=False):
-        data = super().to_dict(omit_none=omit_none, validate=validate)
-        data.update(self._extra)
-        return data
-
-    def replace(self, **kwargs):
-        dct = self.to_dict(omit_none=False, validate=False)
-        dct.update(kwargs)
-        return self.from_dict(dct)
-
 
 @dataclass
 class ExternalPartition(AdditionalPropertiesAllowed, Replaceable):

core/dbt/contracts/rpc.py

@@ -44,13 +44,15 @@ class RPCCompileParameters(RPCParameters):
     threads: Optional[int] = None
     models: Union[None, str, List[str]] = None
     exclude: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
 
 
 @dataclass
 class RPCSnapshotParameters(RPCParameters):
     threads: Optional[int] = None
     select: Union[None, str, List[str]] = None
     exclude: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
 
 
 @dataclass
@@ -64,6 +66,7 @@ class RPCSeedParameters(RPCParameters):
     threads: Optional[int] = None
     select: Union[None, str, List[str]] = None
     exclude: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
     show: bool = False
 
 

core/dbt/contracts/selection.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from hologram import JsonSchemaMixin
+
+from typing import List, Dict, Any, Union
+
+
+@dataclass
+class SelectorDefinition(JsonSchemaMixin):
+    name: str
+    definition: Union[str, Dict[str, Any]]
+
+
+@dataclass
+class SelectorFile(JsonSchemaMixin):
+    selectors: List[SelectorDefinition]
+    version: int = 2
+
+
+# @dataclass
+# class SelectorCollection:
+#     packages: Dict[str, List[SelectorFile]] = field(default_factory=dict)