dbt Contexts

Rewrite/reorganize contexts
 - now they are objects that have a to_dict()
 - special decorators on properties/methods to indicate membership
 - reorganize contexts/ to remove many many import cycles

Context behavior changes:
 - 'dbt_version' is alwways available
 - 'target' is available as long as the profile has been parsed
 - 'project_name' is available in: query headers, hooks, models, and macro execution

Profiles/projects now render at load time
 - reading a profile or project file requires a ConfigRenderer
 - projects get an extra-fiddly special load for use during initial parsing
    - it returns the profile name and a function that, given a renderer, can return the rest of the Project
    - idea is: use the profile name to load the Profile, use that to build a ConfigRenderer that has a TargetContext, render the project with that
 - profiles.yml is rendered with the 'base' context
 - dbt_project.yml/schema.yml/packages.yml are rendered with the 'target' context: 'base' context + 'target'
 - docs are rendered with the docs context: 'target' context + the 'doc' function
 - query headers are rendered with the query header context: 'target' context + macros + 'project_name'
 - executed macros/models should have the same context as previously (query headers + adapter/model/etc related functions)

Moved actual ref/source searching into the manifest
Moved the rest of the parse utils into parser/manifest.py
Made the ref/source resolvers defined in the provider context a bit more sane/well-defined
Picked consistent-but-not-great names for all the various context generation functions
Moved write_node into ParsedNode

core/dbt/adapters/base/connections.py

@@ -14,8 +14,9 @@
 from dbt.contracts.connection import (
  Connection, Identifier, ConnectionState, AdapterRequiredConfig, LazyHandle
 )
+from dbt.contracts.graph.manifest import Manifest
 from dbt.adapters.base.query_headers import (
- QueryStringSetter, MacroQueryStringSetter,
+ MacroQueryStringSetter,
 )
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -39,13 +40,10 @@ def __init__(self, profile: AdapterRequiredConfig):
  self.profile = profile
  self.thread_connections: Dict[Hashable, Connection] = {}
  self.lock: RLock = dbt.flags.MP_CONTEXT.RLock()
- self.query_header = QueryStringSetter(self.profile)
+ self.query_header: Optional[MacroQueryStringSetter] = None
 
- def set_query_header(self, manifest=None) -> None:
- if manifest is not None:
- self.query_header = MacroQueryStringSetter(self.profile, manifest)
- else:
- self.query_header = QueryStringSetter(self.profile)
+ def set_query_header(self, manifest: Manifest) -> None:
+ self.query_header = MacroQueryStringSetter(self.profile, manifest)
 
  @staticmethod
  def get_thread_identifier() -> Hashable:
@@ -285,6 +283,8 @@ def commit_if_has_connection(self) -> None:
  self.commit()
 
  def _add_query_comment(self, sql: str) -> str:
+ if self.query_header is None:
+ return sql
  return self.query_header.add(sql)
 
  @abc.abstractmethod

core/dbt/adapters/base/impl.py

@@ -245,12 +245,14 @@ def connection_named(
  self, name: str, node: Optional[CompileResultNode] = None
  ) -> Iterator[None]:
  try:
- self.connections.query_header.set(name, node)
+ if self.connections.query_header is not None:
+ self.connections.query_header.set(name, node)
  self.acquire_connection(name)
  yield
  finally:
  self.release_connection()
- self.connections.query_header.reset()
+ if self.connections.query_header is not None:
+ self.connections.query_header.reset()
 
  @contextmanager
  def connection_for(
@@ -980,12 +982,12 @@ def execute_macro(
  'dbt could not find a macro with the name "{}" in {}'
  .format(macro_name, package_name)
  )
- # This causes a reference cycle, as dbt.context.runtime.generate()
+ # This causes a reference cycle, as generate_runtime_macro()
  # ends up calling get_adapter, so the import has to be here.
- import dbt.context.operation
- macro_context = dbt.context.operation.generate(
- model=macro,
- runtime_config=self.config,
+ from dbt.context.providers import generate_runtime_macro
+ macro_context = generate_runtime_macro(
+ macro=macro,
+ config=self.config,
  manifest=manifest,
  package_name=project
  )

core/dbt/adapters/base/plugin.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Type
 
 from dbt.adapters.base import BaseAdapter, Credentials
+from dbt.exceptions import CompilationException
 
 
 class AdapterPlugin:
@@ -23,8 +24,12 @@ def __init__(
  self.adapter: Type[BaseAdapter] = adapter
  self.credentials: Type[Credentials] = credentials
  self.include_path: str = include_path
- project = Project.from_project_root(include_path, {})
- self.project_name: str = project.project_name
+ partial = Project.partial_load(include_path)
+ if partial.project_name is None:
+ raise CompilationException(
+ f'Invalid project at {include_path}: name not set!'
+ )
+ self.project_name: str = partial.project_name
  self.dependencies: List[str]
  if dependencies is None:
  self.dependencies = []

core/dbt/adapters/base/query_headers.py

@@ -3,8 +3,7 @@
 
 from dbt.clients.jinja import QueryStringGenerator
 
-# this generates an import cycle, as usual
-from dbt.context.base import QueryHeaderContext
+from dbt.context.configured import generate_query_header_context
 from dbt.contracts.connection import AdapterRequiredConfig
 from dbt.contracts.graph.compiled import CompileResultNode
 from dbt.contracts.graph.manifest import Manifest
@@ -68,9 +67,9 @@ def set(self, comment: Optional[str]):
 QueryStringFunc = Callable[[str, Optional[NodeWrapper]], str]
 
 
-class QueryStringSetter:
- """The base query string setter. This is only used once."""
- def __init__(self, config: AdapterRequiredConfig):
+class MacroQueryStringSetter:
+ def __init__(self, config: AdapterRequiredConfig, manifest: Manifest):
+  self.manifest = manifest
  self.config = config
 
  comment_macro = self._get_comment_macro()
@@ -88,17 +87,22 @@ def __init__(self, config: AdapterRequiredConfig):
  self.comment = _QueryComment(None)
  self.reset()
 
- def _get_context(self):
- return QueryHeaderContext(self.config).to_dict()
-
- def _get_comment_macro(self) -> Optional[str]:
+ def _get_comment_macro(self):
+ if (
+ self.config.query_comment != NoValue() and
+ self.config.query_comment
+ ):
+ return self.config.query_comment
  # if the query comment is null/empty string, there is no comment at all
- if not self.config.query_comment:
+ elif not self.config.query_comment:
  return None
  else:
  # else, the default
  return DEFAULT_QUERY_COMMENT
 
+ def _get_context(self) -> Dict[str, Any]:
+ return generate_query_header_context(self.config, self.manifest)
+
  def add(self, sql: str) -> str:
  return self.comment.add(sql)
 
@@ -111,21 +115,3 @@ def set(self, name: str, node: Optional[CompileResultNode]):
  wrapped = NodeWrapper(node)
  comment_str = self.generator(name, wrapped)
  self.comment.set(comment_str)
-
-
-class MacroQueryStringSetter(QueryStringSetter):
- def __init__(self, config: AdapterRequiredConfig, manifest: Manifest):
- self.manifest = manifest
- super().__init__(config)
-
- def _get_comment_macro(self):
- if (
- self.config.query_comment != NoValue() and
- self.config.query_comment
- ):
- return self.config.query_comment
- else:
- return super()._get_comment_macro()
-
- def _get_context(self) -> Dict[str, Any]:
- return QueryHeaderContext(self.config).to_dict(self.manifest.macros)

core/dbt/clients/jinja.py

@@ -4,7 +4,7 @@
 import tempfile
 from contextlib import contextmanager
 from typing import (
- List, Union, Set, Optional, Dict, Any, Callable, Iterator, Type
+ List, Union, Set, Optional, Dict, Any, Callable, Iterator, Type, NoReturn
 )
 
 import jinja2
@@ -374,7 +374,7 @@ def get_rendered(string, ctx, node=None,
  return render_template(template, ctx, node)
 
 
-def undefined_error(msg):
+def undefined_error(msg) -> NoReturn:
  raise jinja2.exceptions.UndefinedError(msg)
 
 

core/dbt/compilation.py

@@ -10,7 +10,7 @@
 from dbt.node_types import NodeType
 from dbt.linker import Linker
 
-import dbt.context.runtime
+from dbt.context.providers import generate_runtime_model
 import dbt.contracts.project
 import dbt.exceptions
 import dbt.flags
@@ -146,8 +146,9 @@ def compile_node(self, node, manifest, extra_context=None):
  })
  compiled_node = _compiled_type_for(node).from_dict(data)
 
- context = dbt.context.runtime.generate(
- compiled_node, self.config, manifest)
+ context = generate_runtime_model(
+ compiled_node, self.config, manifest
+ )
  context.update(extra_context)
 
  compiled_node.compiled_sql = dbt.clients.jinja.get_rendered(
@@ -253,13 +254,11 @@ def compile_node(adapter, config, node, manifest, extra_context, write=True):
  logger.debug('Writing injected SQL for node "{}"'.format(
  node.unique_id))
 
- written_path = dbt.writer.write_node(
- node,
+ node.build_path = node.write_node(
  config.target_path,
  'compiled',
- node.injected_sql)
-
- node.build_path = written_path
+ node.injected_sql
+ )
 
  return node
 

core/dbt/config/profile.py

@@ -6,15 +6,15 @@
 
 from dbt.clients.system import load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
-from dbt.contracts.connection import Credentials
+from dbt.contracts.connection import Credentials, HasCredentials
 from dbt.contracts.project import ProfileConfig, UserConfig
 from dbt.exceptions import DbtProfileError
 from dbt.exceptions import DbtProjectError
 from dbt.exceptions import ValidationException
 from dbt.exceptions import RuntimeException
 from dbt.exceptions import validator_error_message
 from dbt.logger import GLOBAL_LOGGER as logger
-from dbt.utils import parse_cli_vars, coerce_dict_str
+from dbt.utils import coerce_dict_str
 
 from .renderer import ConfigRenderer
 
@@ -73,7 +73,7 @@ def read_user_config(directory: str) -> UserConfig:
 
 
 @dataclass
-class Profile:
+class Profile(HasCredentials):
  profile_name: str
  target_name: str
  config: UserConfig
@@ -217,13 +217,11 @@ def render_profile(
  raw_profile: Dict[str, Any],
  profile_name: str,
  target_override: Optional[str],
- cli_vars: Dict[str, Any],
+ renderer: ConfigRenderer,
  ) -> Tuple[str, Dict[str, Any]]:
  """This is a containment zone for the hateful way we're rendering
  profiles.
  """
- renderer = ConfigRenderer(cli_vars=cli_vars)
-
  # rendering profiles is a bit complex. Two constraints cause trouble:
  # 1) users should be able to use environment/cli variables to specify
  # the target in their profile.
@@ -255,7 +253,7 @@ def from_raw_profile_info(
  cls,
  raw_profile: Dict[str, Any],
  profile_name: str,
- cli_vars: Dict[str, Any],
+ renderer: ConfigRenderer,
  user_cfg: Optional[Dict[str, Any]] = None,
  target_override: Optional[str] = None,
  threads_override: Optional[int] = None,
@@ -267,8 +265,7 @@ def from_raw_profile_info(
  :param raw_profile: The profile data for a single profile, from
  disk as yaml and its values rendered with jinja.
  :param profile_name: The profile name used.
- :param cli_vars: The command-line variables passed as arguments,
- as a dict.
+ :param renderer: The config renderer.
  :param user_cfg: The global config for the user, if it
  was present.
  :param target_override: The target to use, if provided on
@@ -285,7 +282,7 @@ def from_raw_profile_info(
 
  # TODO: should it be, and the values coerced to bool?
  target_name, profile_data = cls.render_profile(
- raw_profile, profile_name, target_override, cli_vars
+ raw_profile, profile_name, target_override, renderer
  )
 
  # valid connections never include the number of threads, but it's
@@ -311,15 +308,14 @@ def from_raw_profiles(
  cls,
  raw_profiles: Dict[str, Any],
  profile_name: str,
- cli_vars: Dict[str, Any],
+ renderer: ConfigRenderer,
  target_override: Optional[str] = None,
  threads_override: Optional[int] = None,
  ) -> 'Profile':
  """
  :param raw_profiles: The profile data, from disk as yaml.
  :param profile_name: The profile name to use.
- :param cli_vars: The command-line variables passed as arguments, as a
- dict.
+ :param renderer: The config renderer.
  :param target_override: The target to use, if provided on the command
  line.
  :param threads_override: The thread count to use, if provided on the
@@ -344,17 +340,18 @@ def from_raw_profiles(
  return cls.from_raw_profile_info(
  raw_profile=raw_profile,
  profile_name=profile_name,
- cli_vars=cli_vars,
+ renderer=renderer,
  user_cfg=user_cfg,
  target_override=target_override,
  threads_override=threads_override,
  )
 
  @classmethod
- def from_args(
+ def render_from_args(
  cls,
  args: Any,
- project_profile_name: Optional[str] = None,
+ renderer: ConfigRenderer,
+ project_profile_name: Optional[str],
  ) -> 'Profile':
  """Given the raw profiles as read from disk and the name of the desired
  profile if specified, return the profile component of the runtime
@@ -370,7 +367,6 @@ def from_args(
  target could not be found.
  :returns Profile: The new Profile object.
  """
- cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
  threads_override = getattr(args, 'threads', None)
  target_override = getattr(args, 'target', None)
  raw_profiles = read_profile(args.profiles_dir)
@@ -380,7 +376,7 @@ def from_args(
  return cls.from_raw_profiles(
  raw_profiles=raw_profiles,
  profile_name=profile_name,
- cli_vars=cli_vars,
+ renderer=renderer,
  target_override=target_override,
  threads_override=threads_override
  )