Dc/8546 semantic models in graph selection (#8589)

.changes/unreleased/Fixes-20230925-233306.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: semantic models in graph selection
+time: 2023-09-25T23:33:06.754344+01:00
+custom:
+ Author: dave-connors-3 michelleark
+ Issue: "8589"

core/dbt/cli/params.py

@@ -368,6 +368,7 @@
  type=ChoiceTuple(
  [
  "metric",
+ "semantic_model",
  "source",
  "analysis",
  "model",

core/dbt/compilation.py

@@ -183,10 +183,10 @@ def link_node(self, node: GraphMemberNode, manifest: Manifest):
  def link_graph(self, manifest: Manifest):
  for source in manifest.sources.values():
  self.add_node(source.unique_id)
- for semantic_model in manifest.semantic_models.values():
- self.add_node(semantic_model.unique_id)
  for node in manifest.nodes.values():
  self.link_node(node, manifest)
+ for semantic_model in manifest.semantic_models.values():
+ self.link_node(semantic_model, manifest)
  for exposure in manifest.exposures.values():
  self.link_node(exposure, manifest)
  for metric in manifest.metrics.values():

core/dbt/contracts/graph/nodes.py

@@ -1676,6 +1676,56 @@ def primary_entity_reference(self) -> Optional[EntityReference]:
  else None
  )
 
+ def same_model(self, old: "SemanticModel") -> bool:
+ return self.model == old.same_model
+
+ def same_node_relation(self, old: "SemanticModel") -> bool:
+ return self.node_relation == old.node_relation
+
+ def same_description(self, old: "SemanticModel") -> bool:
+ return self.description == old.description
+
+ def same_defaults(self, old: "SemanticModel") -> bool:
+ return self.defaults == old.defaults
+
+ def same_entities(self, old: "SemanticModel") -> bool:
+ return self.entities == old.entities
+
+ def same_dimensions(self, old: "SemanticModel") -> bool:
+ return self.dimensions == old.dimensions
+
+ def same_measures(self, old: "SemanticModel") -> bool:
+ return self.measures == old.measures
+
+ def same_config(self, old: "SemanticModel") -> bool:
+ return self.config == old.config
+
+ def same_primary_entity(self, old: "SemanticModel") -> bool:
+ return self.primary_entity == old.primary_entity
+
+ def same_group(self, old: "SemanticModel") -> bool:
+ return self.group == old.group
+
+ def same_contents(self, old: Optional["SemanticModel"]) -> bool:
+ # existing when it didn't before is a change!
+ # metadata/tags changes are not "changes"
+ if old is None:
+ return True
+
+ return (
+ self.same_model(old)
+ and self.same_node_relation(old)
+ and self.same_description(old)
+ and self.same_defaults(old)
+ and self.same_entities(old)
+ and self.same_dimensions(old)
+ and self.same_measures(old)
+ and self.same_config(old)
+ and self.same_primary_entity(old)
+ and self.same_group(old)
+ and True
+ )
+
 
 # ====================================
 # Patches

core/dbt/graph/cli.py

@@ -21,7 +21,7 @@
 
 INTERSECTION_DELIMITER = ","
 
-DEFAULT_INCLUDES: List[str] = ["fqn:*", "source:*", "exposure:*", "metric:*"]
+DEFAULT_INCLUDES: List[str] = ["fqn:*", "source:*", "exposure:*", "metric:*", "semantic_model:*"]
 DEFAULT_EXCLUDES: List[str] = []
 
 

core/dbt/graph/selector_methods.py

@@ -18,6 +18,7 @@
  ResultNode,
  ManifestNode,
  ModelNode,
+ SemanticModel,
 )
 from dbt.contracts.graph.unparsed import UnparsedVersion
 from dbt.contracts.state import PreviousState
@@ -53,6 +54,7 @@ class MethodName(StrEnum):
  SourceStatus = "source_status"
  Wildcard = "wildcard"
  Version = "version"
+ SemanticModel = "semantic_model"
 
 
 def is_selected_node(fqn: List[str], node_selector: str, is_versioned: bool) -> bool:
@@ -144,6 +146,16 @@ def metric_nodes(self, included_nodes: Set[UniqueId]) -> Iterator[Tuple[UniqueId
  continue
  yield unique_id, metric
 
+ def semantic_model_nodes(
+ self, included_nodes: Set[UniqueId]
+ ) -> Iterator[Tuple[UniqueId, SemanticModel]]:
+
+ for key, semantic_model in self.manifest.semantic_models.items():
+ unique_id = UniqueId(key)
+ if unique_id not in included_nodes:
+ continue
+ yield unique_id, semantic_model
+
  def all_nodes(
  self, included_nodes: Set[UniqueId]
  ) -> Iterator[Tuple[UniqueId, SelectorTarget]]:
@@ -152,6 +164,7 @@ def all_nodes(
  self.source_nodes(included_nodes),
  self.exposure_nodes(included_nodes),
  self.metric_nodes(included_nodes),
+ self.semantic_model_nodes(included_nodes),
  )
 
  def configurable_nodes(
@@ -167,6 +180,7 @@ def non_source_nodes(
  self.parsed_nodes(included_nodes),
  self.exposure_nodes(included_nodes),
  self.metric_nodes(included_nodes),
+ self.semantic_model_nodes(included_nodes),
  )
 
  def groupable_nodes(
@@ -210,8 +224,8 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
 
  :param str selector: The selector or node name
  """
- parsed_nodes = list(self.parsed_nodes(included_nodes))
- for node, real_node in parsed_nodes:
+ non_source_nodes = list(self.non_source_nodes(included_nodes))
+ for node, real_node in non_source_nodes:
  if self.node_is_match(selector, real_node.fqn, real_node.is_versioned):
  yield node
 
@@ -322,6 +336,31 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
  yield node
 
 
+class SemanticModelSelectorMethod(SelectorMethod):
+ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
+ parts = selector.split(".")
+ target_package = SELECTOR_GLOB
+ if len(parts) == 1:
+ target_name = parts[0]
+ elif len(parts) == 2:
+ target_package, target_name = parts
+ else:
+ msg = (
+ 'Invalid semantic model selector value "{}". Semantic models must be of '
+ "the form ${{semantic_model_name}} or "
+ "${{semantic_model_package.semantic_model_name}}"
+ ).format(selector)
+ raise DbtRuntimeError(msg)
+
+ for node, real_node in self.semantic_model_nodes(included_nodes):
+ if not fnmatch(real_node.package_name, target_package):
+ continue
+ if not fnmatch(real_node.name, target_name):
+ continue
+
+ yield node
+
+
 class PathSelectorMethod(SelectorMethod):
  def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
  """Yields nodes from included that match the given path."""
@@ -431,7 +470,7 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu
  resource_type = NodeType(selector)
  except ValueError as exc:
  raise DbtRuntimeError(f'Invalid resource_type selector "{selector}"') from exc
- for node, real_node in self.parsed_nodes(included_nodes):
+ for node, real_node in self.all_nodes(included_nodes):
  if real_node.resource_type == resource_type:
  yield node
 
@@ -539,7 +578,7 @@ def check_macros_modified(self, node):
  def check_modified_content(
  self, old: Optional[SelectorTarget], new: SelectorTarget, adapter_type: str
  ) -> bool:
- if isinstance(new, (SourceDefinition, Exposure, Metric)):
+ if isinstance(new, (SourceDefinition, Exposure, Metric, SemanticModel)):
  # these all overwrite `same_contents`
  different_contents = not new.same_contents(old) # type: ignore
  else:
@@ -761,6 +800,7 @@ class MethodManager:
  MethodName.Result: ResultSelectorMethod,
  MethodName.SourceStatus: SourceStatusSelectorMethod,
  MethodName.Version: VersionSelectorMethod,
+ MethodName.SemanticModel: SemanticModelSelectorMethod,
  }
 
  def __init__(

core/dbt/task/list.py

@@ -1,6 +1,6 @@
 import json
 
-from dbt.contracts.graph.nodes import Exposure, SourceDefinition, Metric
+from dbt.contracts.graph.nodes import Exposure, SourceDefinition, Metric, SemanticModel
 from dbt.flags import get_flags
 from dbt.graph import ResourceTypeSelector
 from dbt.task.runnable import GraphRunnableTask
@@ -28,6 +28,7 @@ class ListTask(GraphRunnableTask):
  NodeType.Source,
  NodeType.Exposure,
  NodeType.Metric,
+ NodeType.SemanticModel,
  )
  )
  ALL_RESOURCE_VALUES = DEFAULT_RESOURCE_VALUES | frozenset((NodeType.Analysis,))
@@ -74,6 +75,8 @@ def _iterate_selected_nodes(self):
  yield self.manifest.exposures[node]
  elif node in self.manifest.metrics:
  yield self.manifest.metrics[node]
+ elif node in self.manifest.semantic_models:
+ yield self.manifest.semantic_models[node]
  else:
  raise DbtRuntimeError(
  f'Got an unexpected result from node selection: "{node}"'
@@ -97,6 +100,10 @@ def generate_selectors(self):
  # metrics are searched for by pkg.metric_name
  metric_selector = ".".join([node.package_name, node.name])
  yield f"metric:{metric_selector}"
+ elif node.resource_type == NodeType.SemanticModel:
+ assert isinstance(node, SemanticModel)
+ semantic_model_selector = ".".join([node.package_name, node.name])
+ yield f"semantic_model:{semantic_model_selector}"
  else:
  # everything else is from `fqn`
  yield ".".join(node.fqn)

tests/functional/list/fixtures.py

@@ -46,7 +46,16 @@
 
 {{ config(materialized='ephemeral') }}
 
-select 1 as id
+select
+ 1 as id,
+ {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as created_at
+
+"""
+
+models__metric_flow = """
+
+select
+ {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date_day
 
 """
 
@@ -103,6 +112,38 @@
 
 """
 
+semantic_models__sm_yml = """
+semantic_models:
+ - name: my_sm
+ model: ref('outer')
+ defaults:
+ agg_time_dimension: created_at
+ entities:
+ - name: my_entity
+ type: primary
+ expr: id
+ dimensions:
+ - name: created_at
+ type: time
+ type_params:
+ time_granularity: day
+ measures:
+ - name: total_outer_count
+ agg: count
+ expr: 1
+
+"""
+
+metrics__m_yml = """
+metrics:
+ - name: total_outer
+ type: simple
+ description: The total count of outer
+ label: Total Outer
+ type_params:
+ measure: total_outer_count
+"""
+
 
 @pytest.fixture(scope="class")
 def snapshots():
@@ -122,6 +163,9 @@ def models():
  "incremental.sql": models__incremental_sql,
  "docs.md": models__docs_md,
  "outer.sql": models__outer_sql,
+ "metricflow_time_spine.sql": models__metric_flow,
+ "sm.yml": semantic_models__sm_yml,
+ "m.yml": metrics__m_yml,
  "sub": {"inner.sql": models__sub__inner_sql},
  }
 
@@ -141,6 +185,16 @@ def analyses():
  return {"a.sql": analyses__a_sql}
 
 
+@pytest.fixture(scope="class")
+def semantic_models():
+ return {"sm.yml": semantic_models__sm_yml}
+
+
+@pytest.fixture(scope="class")
+def metrics():
+ return {"m.yml": metrics__m_yml}
+
+
 @pytest.fixture(scope="class")
 def project_files(
  project_root,