(#4884) Add support for ratio metrics

core/dbt/compilation.py

@@ -397,6 +397,8 @@ def link_node(self, linker: Linker, node: GraphMemberNode, manifest: Manifest):
  linker.dependency(node.unique_id, (manifest.nodes[dependency].unique_id))
  elif dependency in manifest.sources:
  linker.dependency(node.unique_id, (manifest.sources[dependency].unique_id))
+ elif dependency in manifest.metrics:
+ linker.dependency(node.unique_id, (manifest.metrics[dependency].unique_id))
  else:
  dependency_not_found(node, dependency)
 

core/dbt/context/providers.py

@@ -40,6 +40,9 @@
  ParsedSeedNode,
  ParsedSourceDefinition,
 )
+from dbt.contracts.graph.metrics import (
+ MetricReference, ResolvedMetricReference
+)
 from dbt.exceptions import (
  CompilationException,
  ParsingException,
@@ -197,7 +200,7 @@ def Relation(self):
  return self.db_wrapper.Relation
 
  @abc.abstractmethod
- def __call__(self, *args: str) -> Union[str, RelationProxy]:
+ def __call__(self, *args: str) -> Union[str, RelationProxy, MetricReference]:
  pass
 
 
@@ -263,6 +266,42 @@ def __call__(self, *args: str) -> RelationProxy:
  return self.resolve(args[0], args[1])
 
 
+class BaseMetricResolver(BaseResolver):
+ def resolve(self, name: str, package: Optional[str] = None) -> MetricReference:
+ ...
+
+ def _repack_args(self, name: str, package: Optional[str]) -> List[str]:
+ if package is None:
+ return [name]
+ else:
+ return [package, name]
+
+ def validate_args(self, name: str, package: Optional[str]):
+ if not isinstance(name, str):
+ raise CompilationException(
+ f"The name argument to metric() must be a string, got " f"{type(name)}"
+ )
+
+ if package is not None and not isinstance(package, str):
+ raise CompilationException(
+ f"The package argument to metric() must be a string or None, got " f"{type(package)}"
+ )
+
+ def __call__(self, *args: str) -> MetricReference:
+ name: str
+ package: Optional[str] = None
+
+ if len(args) == 1:
+ name = args[0]
+ elif len(args) == 2:
+ package, name = args
+ else:
+ # TODO: Use a different error!
+ ref_invalid_args(self.model, args)
+ self.validate_args(name, package)
+ return self.resolve(name, package)
+
+
 class Config(Protocol):
  def __init__(self, model, context_config: Optional[ContextConfig]):
  ...
@@ -509,6 +548,35 @@ def resolve(self, source_name: str, table_name: str):
  return self.Relation.create_from_source(target_source)
 
 
+# metric` implementations
+class ParseMetricResolver(BaseMetricResolver):
+ def resolve(self, name: str, package: Optional[str] = None) -> MetricReference:
+ self.model.metrics.append(self._repack_args(name, package))
+
+ return MetricReference(name, package)
+
+
+class RuntimeMetricResolver(BaseMetricResolver):
+ def resolve(self, target_name: str, target_package: Optional[str] = None) -> MetricReference:
+ target_metric = self.manifest.resolve_metric(
+ target_name,
+ target_package,
+ self.current_project,
+ self.model.package_name,
+ )
+
+ if target_metric is None or isinstance(target_metric, Disabled):
+ # TODO : Use a different exception!!
+ ref_target_not_found(
+ self.model,
+ target_name,
+ target_package,
+ disabled=isinstance(target_metric, Disabled),
+ )
+
+ return ResolvedMetricReference(target_metric)
+
+
 # `var` implementations.
 class ModelConfiguredVar(Var):
  def __init__(
@@ -566,6 +634,7 @@ class Provider(Protocol):
  Var: Type[ModelConfiguredVar]
  ref: Type[BaseRefResolver]
  source: Type[BaseSourceResolver]
+ metric: Type[BaseMetricResolver]
 
 
 class ParseProvider(Provider):
@@ -575,6 +644,7 @@ class ParseProvider(Provider):
  Var = ParseVar
  ref = ParseRefResolver
  source = ParseSourceResolver
+ metric = ParseMetricResolver
 
 
 class GenerateNameProvider(Provider):
@@ -584,6 +654,7 @@ class GenerateNameProvider(Provider):
  Var = RuntimeVar
  ref = ParseRefResolver
  source = ParseSourceResolver
+ metric = ParseMetricResolver
 
 
 class RuntimeProvider(Provider):
@@ -593,6 +664,7 @@ class RuntimeProvider(Provider):
  Var = RuntimeVar
  ref = RuntimeRefResolver
  source = RuntimeSourceResolver
+ metric = RuntimeMetricResolver
 
 
 class OperationProvider(RuntimeProvider):
@@ -776,6 +848,10 @@ def ref(self) -> Callable:
  def source(self) -> Callable:
  return self.provider.source(self.db_wrapper, self.model, self.config, self.manifest)
 
+ @contextproperty
+ def metric(self) -> Callable:
+ return self.provider.metric(self.db_wrapper, self.model, self.config, self.manifest)
+
  @contextproperty("config")
  def ctx_config(self) -> Config:
  """The `config` variable exists to handle end-user configuration for
@@ -1286,6 +1362,15 @@ def generate_runtime_macro_context(
  return ctx.to_dict()
 
 
+def generate_runtime_metric_context(
+ metric: ParsedMetric,
+ config: RuntimeConfig,
+ manifest: Manifest,
+) -> Dict[str, Any]:
+ ctx = ProviderContext(metric, config, manifest, RuntimeProvider(), None)
+ return ctx.to_dict()
+
+
 class ExposureRefResolver(BaseResolver):
  def __call__(self, *args) -> str:
  if len(args) not in (1, 2):
@@ -1362,6 +1447,12 @@ def generate_parse_metrics(
  project,
  manifest,
  ),
+ "metric": ParseMetricResolver(
+ None,
+ metric,
+ project,
+ manifest,
+ ),
  }
 
 

core/dbt/contracts/graph/manifest.py

@@ -183,6 +183,39 @@ def perform_lookup(self, unique_id: UniqueID, manifest) -> ManifestNode:
  return manifest.nodes[unique_id]
 
 
+class MetricLookup(dbtClassMixin):
+ def __init__(self, manifest: "Manifest"):
+ self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+ self.populate(manifest)
+
+ def get_unique_id(self, search_name, package: Optional[PackageName]):
+ return find_unique_id_for_package(self.storage, search_name, package)
+
+ def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"):
+ unique_id = self.get_unique_id(search_name, package)
+ if unique_id is not None:
+ return self.perform_lookup(unique_id, manifest)
+ return None
+
+ def add_metric(self, metric: ParsedMetric):
+ if metric.search_name not in self.storage:
+ self.storage[metric.search_name] = {}
+
+ self.storage[metric.search_name][metric.package_name] = metric.unique_id
+
+ def populate(self, manifest):
+ for metric in manifest.metrics.values():
+ if hasattr(metric, "name"):
+ self.add_metric(metric)
+
+ def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> ParsedMetric:
+ if unique_id not in manifest.metrics:
+ raise dbt.exceptions.InternalException(
+ f"Metric {unique_id} found in cache but not found in manifest"
+ )
+ return manifest.metrics[unique_id]
+
+
 # This handles both models/seeds/snapshots and sources
 class DisabledLookup(dbtClassMixin):
  def __init__(self, manifest: "Manifest"):
@@ -434,6 +467,9 @@ class Disabled(Generic[D]):
  target: D
 
 
+MaybeMetricNode = Optional[ParsedMetric]
+
+
 MaybeDocumentation = Optional[ParsedDocumentation]
 
 
@@ -595,6 +631,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
  _ref_lookup: Optional[RefableLookup] = field(
  default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
  )
+ _metric_lookup: Optional[MetricLookup] = field(
+ default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+ )
  _disabled_lookup: Optional[DisabledLookup] = field(
  default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
  )
@@ -833,6 +872,12 @@ def ref_lookup(self) -> RefableLookup:
  self._ref_lookup = RefableLookup(self)
  return self._ref_lookup
 
+ @property
+ def metric_lookup(self) -> MetricLookup:
+ if self._metric_lookup is None:
+ self._metric_lookup = MetricLookup(self)
+ return self._metric_lookup
+
  def rebuild_ref_lookup(self):
  self._ref_lookup = RefableLookup(self)
 
@@ -908,6 +953,31 @@ def resolve_source(
  return Disabled(disabled[0])
  return None
 
+ def resolve_metric(
+ self,
+ target_metric_name: str,
+ target_metric_package: Optional[str],
+ current_project: str,
+ node_package: str,
+ ) -> MaybeMetricNode:
+ metric: Optional[ParsedMetric] = None
+ disabled: Optional[List[ParsedMetric]] = None
+
+ candidates = _search_packages(current_project, node_package, target_metric_package)
+ for pkg in candidates:
+ metric = self.metric_lookup.find(target_metric_name, pkg, self)
+ if metric is not None:
+ # Skip if the metric is disabled!
+ return metric
+
+ # TODO: Figure this part out...
+ # if disabled is None:
+ # disabled = self.disabled_lookup.find(search_name, pkg)
+
+ if disabled:
+ return Disabled(disabled[0])
+ return None
+
  # Called by DocsRuntimeContext.doc
  def resolve_doc(
  self,
@@ -1072,6 +1142,7 @@ def __reduce_ex__(self, protocol):
  self._doc_lookup,
  self._source_lookup,
  self._ref_lookup,
+ self._metric_lookup,
  self._disabled_lookup,
  self._analysis_lookup,
  )

core/dbt/contracts/graph/metrics.py

@@ -0,0 +1,30 @@
+
+class MetricReference(object):
+ def __init__(self, metric_name, package_name=None):
+ self.metric_name = metric_name
+ self.package_name = package_name
+
+
+class ResolvedMetricReference(MetricReference):
+ """
+ Simple proxy over a ParsedMetric which delegates property
+ lookups to the underlying node. Also adds helper functions
+ for working with metrics (ie. __str__ and templating functions)
+ """
+ def __init__(self, node):
+ super().__init__(node.name, node.package_name)
+ self.node = node
+
+ def __getattr__(self, key):
+ return getattr(self.node, key)
+
+ @property
+ def is_derived(self):
+ return self.node.is_derived
+
+ def qualify(self, namespace):
+ return f'{namespace}.{self.metric_name}'
+
+ def __str__(self):
+ return self.node.name
+