Merge pull request dbt-labs#2640 from fishtown-analytics/feature/yaml…

…-selections

Feature/yaml selections

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Add more helpful error message for misconfiguration in profiles.yml ([#2569](https://github.com/fishtown-analytics/dbt/issues/2569), [#2627](https://github.com/fishtown-analytics/dbt/pull/2627))
 ### Fixes
 - Adapter plugins can once again override plugins defined in core ([#2548](https://github.com/fishtown-analytics/dbt/issues/2548), [#2590](https://github.com/fishtown-analytics/dbt/pull/2590))
+- Added `--selector` argument and support for `selectors.yml` file to define selection mechanisms. ([#2172](https://github.com/fishtown-analytics/dbt/issues/2172), [#2640](https://github.com/fishtown-analytics/dbt/pull/2640))
 
 Contributors:
 - [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581))

core/dbt/config/project.py

@@ -19,6 +19,8 @@
 from dbt.exceptions import RecursionException
 from dbt.exceptions import SemverException
 from dbt.exceptions import validator_error_message
+from dbt.exceptions import RuntimeException
+from dbt.graph import SelectionSpec
 from dbt.helper_types import NoValue
 from dbt.semver import VersionSpecifier
 from dbt.semver import versions_compatible
@@ -37,6 +39,11 @@
 from hologram import ValidationError
 
 from .renderer import DbtProjectYamlRenderer
+from .selectors import (
+ selector_config_from_data,
+ selector_data_from_root,
+ SelectorConfig,
+)
 
 
 INVALID_VERSION_ERROR = """\
@@ -211,10 +218,12 @@ class PartialProject:
 
  def render(self, renderer):
  packages_dict = package_data_from_root(self.project_root)
+ selectors_dict = selector_data_from_root(self.project_root)
  return Project.render_from_dict(
  self.project_root,
  self.project_dict,
  packages_dict,
+ selectors_dict,
  renderer,
  )
 
@@ -310,6 +319,7 @@ class Project:
  vars: VarProvider
  dbt_version: List[VersionSpecifier]
  packages: Dict[str, Any]
+ selectors: SelectorConfig
  query_comment: QueryComment
  config_version: int
 
@@ -351,6 +361,7 @@ def from_project_config(
  cls,
  project_dict: Dict[str, Any],
  packages_dict: Optional[Dict[str, Any]] = None,
+ selectors_dict: Optional[Dict[str, Any]] = None,
  ) -> 'Project':
  """Create a project from its project and package configuration, as read
  by yaml.safe_load().
@@ -464,6 +475,11 @@ def from_project_config(
  except ValidationError as e:
  raise DbtProjectError(validator_error_message(e)) from e
 
+ try:
+ selectors = selector_config_from_data(selectors_dict)
+ except ValidationError as e:
+ raise DbtProjectError(validator_error_message(e)) from e
+
  project = cls(
  project_name=name,
  version=version,
@@ -488,6 +504,7 @@ def from_project_config(
  snapshots=snapshots,
  dbt_version=dbt_version,
  packages=packages,
+ selectors=selectors,
  query_comment=query_comment,
  sources=sources,
  vars=vars_value,
@@ -568,14 +585,21 @@ def render_from_dict(
  project_root: str,
  project_dict: Dict[str, Any],
  packages_dict: Dict[str, Any],
+ selectors_dict: Dict[str, Any],
  renderer: DbtProjectYamlRenderer,
  ) -> 'Project':
  rendered_project = renderer.render_data(project_dict)
  rendered_project['project-root'] = project_root
  package_renderer = renderer.get_package_renderer()
  rendered_packages = package_renderer.render_data(packages_dict)
+ selectors_renderer = renderer.get_selector_renderer()
+ rendered_selectors = selectors_renderer.render_data(selectors_dict)
  try:
- return cls.from_project_config(rendered_project, rendered_packages)
+ return cls.from_project_config(
+ rendered_project,
+ rendered_packages,
+ rendered_selectors,
+ )
  except DbtProjectError as exc:
  if exc.path is None:
  exc.path = os.path.join(project_root, 'dbt_project.yml')
@@ -659,6 +683,14 @@ def as_v1(self, all_projects: Iterable[str]):
  project.packages = self.packages
  return project
 
+ def get_selector(self, name: str) -> SelectionSpec:
+ if name not in self.selectors:
+ raise RuntimeException(
+ f'Could not find selector named {name}, expected one of '
+ f'{list(self.selectors)}'
+ )
+ return self.selectors[name]
+
 
 def v2_vars_to_v1(
  dst: Dict[str, Any], src_vars: Dict[str, Any], project_names: Set[str]

core/dbt/config/renderer.py

@@ -69,6 +69,9 @@ def name(self):
  def get_package_renderer(self) -> BaseRenderer:
  return PackageRenderer(self.context)
 
+ def get_selector_renderer(self) -> BaseRenderer:
+ return SelectorRenderer(self.context)
+
  def should_render_keypath_v1(self, keypath: Keypath) -> bool:
  if not keypath:
  return True
@@ -206,3 +209,9 @@ class PackageRenderer(BaseRenderer):
  @property
  def name(self):
  return 'Packages config'
+
+
+class SelectorRenderer(BaseRenderer):
+ @property
+ def name(self):
+ return 'Selector config'

core/dbt/config/runtime.py

@@ -106,6 +106,7 @@ def from_parts(
  snapshots=project.snapshots,
  dbt_version=project.dbt_version,
  packages=project.packages,
+ selectors=project.selectors,
  query_comment=project.query_comment,
  sources=project.sources,
  vars=project.vars,
@@ -361,8 +362,9 @@ def load_projects(
  project = self.new_project(str(path))
  except DbtProjectError as e:
  raise DbtProjectError(
- 'Failed to read package at {}: {}'
- .format(path, e)
+ f'Failed to read package: {e}',
+ result_type='invalid_project',
+ path=path,
  ) from e
  else:
  yield project.project_name, project
@@ -501,6 +503,7 @@ def from_parts(
  snapshots=project.snapshots,
  dbt_version=project.dbt_version,
  packages=project.packages,
+ selectors=project.selectors,
  query_comment=project.query_comment,
  sources=project.sources,
  vars=project.vars,

core/dbt/config/selectors.py

@@ -0,0 +1,104 @@
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+from hologram import ValidationError
+
+from .renderer import SelectorRenderer
+
+from dbt.clients.system import (
+ load_file_contents,
+ path_exists,
+ resolve_path_from_base,
+)
+from dbt.clients.yaml_helper import load_yaml_text
+from dbt.contracts.selection import SelectorFile
+from dbt.exceptions import DbtSelectorsError, RuntimeException
+from dbt.graph import parse_from_selectors_definition, SelectionSpec
+
+MALFORMED_SELECTOR_ERROR = """\
+The selectors.yml file in this project is malformed. Please double check
+the contents of this file and fix any errors before retrying.
+
+You can find more information on the syntax for this file here:
+https://docs.getdbt.com/docs/package-management
+
+Validator Error:
+{error}
+"""
+
+
+class SelectorConfig(Dict[str, SelectionSpec]):
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> 'SelectorConfig':
+ try:
+ selector_file = SelectorFile.from_dict(data)
+ selectors = parse_from_selectors_definition(selector_file)
+ except (ValidationError, RuntimeException) as exc:
+ raise DbtSelectorsError(
+ f'Could not read selector file data: {exc}',
+ result_type='invalid_selector',
+ ) from exc
+
+ return cls(selectors)
+
+ @classmethod
+ def render_from_dict(
+ cls,
+ data: Dict[str, Any],
+ renderer: SelectorRenderer,
+ ) -> 'SelectorConfig':
+ try:
+ rendered = renderer.render_data(data)
+ except (ValidationError, RuntimeException) as exc:
+ raise DbtSelectorsError(
+ f'Could not render selector data: {exc}',
+ result_type='invalid_selector',
+ ) from exc
+ return cls.from_dict(rendered)
+
+ @classmethod
+ def from_path(
+ cls, path: Path, renderer: SelectorRenderer,
+ ) -> 'SelectorConfig':
+ try:
+ data = load_yaml_text(load_file_contents(str(path)))
+ except (ValidationError, RuntimeException) as exc:
+ raise DbtSelectorsError(
+ f'Could not read selector file: {exc}',
+ result_type='invalid_selector',
+ path=path,
+ ) from exc
+
+ try:
+ return cls.render_from_dict(data, renderer)
+ except DbtSelectorsError as exc:
+ exc.path = path
+ raise
+
+
+def selector_data_from_root(project_root: str) -> Dict[str, Any]:
+ selector_filepath = resolve_path_from_base(
+ 'selectors.yml', project_root
+ )
+
+ if path_exists(selector_filepath):
+ selectors_dict = load_yaml_text(load_file_contents(selector_filepath))
+ else:
+ selectors_dict = None
+ return selectors_dict
+
+
+def selector_config_from_data(
+ selectors_data: Optional[Dict[str, Any]]
+) -> SelectorConfig:
+ if selectors_data is None:
+ selectors_data = {'selectors': []}
+
+ try:
+ selectors = SelectorConfig.from_dict(selectors_data)
+ except ValidationError as e:
+ raise DbtSelectorsError(
+ MALFORMED_SELECTOR_ERROR.format(error=str(e.message)),
+ result_type='invalid_selector',
+ ) from e
+ return selectors

core/dbt/contracts/graph/unparsed.py

@@ -1,5 +1,9 @@
 from dbt.node_types import NodeType
-from dbt.contracts.util import Replaceable, Mergeable
+from dbt.contracts.util import (
+ AdditionalPropertiesMixin,
+ Mergeable,
+ Replaceable,
+)
 # trigger the PathEncoder
 import dbt.helper_types # noqa:F401
 from dbt.exceptions import CompilationException
@@ -177,32 +181,12 @@ def __bool__(self):
 
 
 @dataclass
-class AdditionalPropertiesAllowed(ExtensibleJsonSchemaMixin):
+class AdditionalPropertiesAllowed(
+ AdditionalPropertiesMixin,
+ ExtensibleJsonSchemaMixin
+):
  _extra: Dict[str, Any] = field(default_factory=dict)
 
- @property
- def extra(self):
- return self._extra
-
- @classmethod
- def from_dict(cls, data, validate=True):
- self = super().from_dict(data=data, validate=validate)
- keys = self.to_dict(validate=False, omit_none=False)
- for key, value in data.items():
- if key not in keys:
- self._extra[key] = value
- return self
-
- def to_dict(self, omit_none=True, validate=False):
- data = super().to_dict(omit_none=omit_none, validate=validate)
- data.update(self._extra)
- return data
-
- def replace(self, **kwargs):
- dct = self.to_dict(omit_none=False, validate=False)
- dct.update(kwargs)
- return self.from_dict(dct)
-
 
 @dataclass
 class ExternalPartition(AdditionalPropertiesAllowed, Replaceable):

core/dbt/contracts/rpc.py

@@ -44,13 +44,15 @@ class RPCCompileParameters(RPCParameters):
  threads: Optional[int] = None
  models: Union[None, str, List[str]] = None
  exclude: Union[None, str, List[str]] = None
+ selector: Optional[str] = None
 
 
 @dataclass
 class RPCSnapshotParameters(RPCParameters):
  threads: Optional[int] = None
  select: Union[None, str, List[str]] = None
  exclude: Union[None, str, List[str]] = None
+ selector: Optional[str] = None
 
 
 @dataclass
@@ -64,6 +66,7 @@ class RPCSeedParameters(RPCParameters):
  threads: Optional[int] = None
  select: Union[None, str, List[str]] = None
  exclude: Union[None, str, List[str]] = None
+ selector: Optional[str] = None
  show: bool = False
 
 

core/dbt/contracts/selection.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from hologram import JsonSchemaMixin
+
+from typing import List, Dict, Any, Union
+
+
+@dataclass
+class SelectorDefinition(JsonSchemaMixin):
+ name: str
+ definition: Union[str, Dict[str, Any]]
+
+
+@dataclass
+class SelectorFile(JsonSchemaMixin):
+ selectors: List[SelectorDefinition]
+ version: int = 2
+
+
+# @dataclass
+# class SelectorCollection:
+# packages: Dict[str, List[SelectorFile]] = field(default_factory=dict)