diff --git a/.changes/unreleased/Features-20230712-123724.yaml b/.changes/unreleased/Features-20230712-123724.yaml new file mode 100644 index 00000000000..635b56715ca --- /dev/null +++ b/.changes/unreleased/Features-20230712-123724.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Begin populating `depends_on` of metric nodes +time: 2023-07-12T12:37:24.01449-07:00 +custom: + Author: QMalcolm gshank + Issue: "7854" diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 125fe8abf8a..d9a52865460 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -173,6 +173,8 @@ def link_node(self, node: GraphMemberNode, manifest: Manifest): self.dependency(node.unique_id, (manifest.sources[dependency].unique_id)) elif dependency in manifest.metrics: self.dependency(node.unique_id, (manifest.metrics[dependency].unique_id)) + elif dependency in manifest.semantic_models: + self.dependency(node.unique_id, (manifest.semantic_models[dependency].unique_id)) else: raise GraphDependencyNotFoundError(node, dependency) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index b278970c154..20d2dc5f394 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -1,9 +1,11 @@ import enum +from collections import defaultdict from dataclasses import dataclass, field from itertools import chain, islice from mashumaro.mixins.msgpack import DataClassMessagePackMixin from multiprocessing.synchronize import Lock from typing import ( + DefaultDict, Dict, List, Optional, @@ -297,6 +299,49 @@ def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> Metric: return manifest.metrics[unique_id] +class SemanticModelByMeasureLookup(dbtClassMixin): + """Lookup utility for finding SemanticModel by measure + + This is possible because measure names are supposed to be unique across + the semantic models in a manifest. + """ + + def __init__(self, manifest: "Manifest"): + self.storage: DefaultDict[str, Dict[PackageName, UniqueID]] = defaultdict(dict) + self.populate(manifest) + + def get_unique_id(self, search_name: str, package: Optional[PackageName]): + return find_unique_id_for_package(self.storage, search_name, package) + + def find( + self, search_name: str, package: Optional[PackageName], manifest: "Manifest" + ) -> Optional[SemanticModel]: + """Tries to find a SemanticModel based on a measure name""" + unique_id = self.get_unique_id(search_name, package) + if unique_id is not None: + return self.perform_lookup(unique_id, manifest) + return None + + def add(self, semantic_model: SemanticModel): + """Sets all measures for a SemanticModel as paths to the SemanticModel's `unique_id`""" + for measure in semantic_model.measures: + self.storage[measure.name][semantic_model.package_name] = semantic_model.unique_id + + def populate(self, manifest: "Manifest"): + """Populate storage with all the measure + package paths to the Manifest's SemanticModels""" + for semantic_model in manifest.semantic_models.values(): + self.add(semantic_model=semantic_model) + + def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SemanticModel: + """Tries to get a SemanticModel from the Manifest""" + semantic_model = manifest.semantic_models.get(unique_id) + if semantic_model is None: + raise dbt.exceptions.DbtInternalError( + f"Semantic model `{unique_id}` found in cache but not found in manifest" + ) + return semantic_model + + # This handles both models/seeds/snapshots and sources/metrics/exposures class DisabledLookup(dbtClassMixin): def __init__(self, manifest: "Manifest"): @@ -710,6 +755,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): _metric_lookup: Optional[MetricLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) + _semantic_model_by_measure_lookup: Optional[SemanticModelByMeasureLookup] = field( + default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} + ) _disabled_lookup: Optional[DisabledLookup] = field( default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} ) @@ -960,6 +1008,13 @@ def metric_lookup(self) -> MetricLookup: self._metric_lookup = MetricLookup(self) return self._metric_lookup + @property + def semantic_model_by_measure_lookup(self) -> SemanticModelByMeasureLookup: + """Gets (and creates if necessary) the lookup utility for getting SemanticModels by measures""" + if self._semantic_model_by_measure_lookup is None: + self._semantic_model_by_measure_lookup = SemanticModelByMeasureLookup(self) + return self._semantic_model_by_measure_lookup + def rebuild_ref_lookup(self): self._ref_lookup = RefableLookup(self) @@ -1087,6 +1142,25 @@ def resolve_metric( return Disabled(disabled[0]) return None + def resolve_semantic_model_for_measure( + self, + target_measure_name: str, + current_project: str, + node_package: str, + target_package: Optional[str] = None, + ) -> Optional[SemanticModel]: + """Tries to find the SemanticModel that a measure belongs to""" + candidates = _packages_to_search(current_project, node_package, target_package) + + for pkg in candidates: + semantic_model = self.semantic_model_by_measure_lookup.find( + target_measure_name, pkg, self + ) + if semantic_model is not None: + return semantic_model + + return None + # Called by DocsRuntimeContext.doc def resolve_doc( self, @@ -1328,6 +1402,7 @@ def __reduce_ex__(self, protocol): self._source_lookup, self._ref_lookup, self._metric_lookup, + self._semantic_model_by_measure_lookup, self._disabled_lookup, self._analysis_lookup, ) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index fbcfb6d3c56..62664664ae9 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -386,6 +386,11 @@ def replace(self, **kwargs): return self.from_dict(dct) +@dataclass +class SemanticModelConfig(BaseConfig): + enabled: bool = True + + @dataclass class MetricConfig(BaseConfig): enabled: bool = True diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 3fa2e098f45..94710a4cfb7 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -52,6 +52,7 @@ MeasureReference, LinkableElementReference, SemanticModelReference, + TimeDimensionReference, ) from dbt_semantic_interfaces.references import MetricReference as DSIMetricReference from dbt_semantic_interfaces.type_enums import MetricType, TimeGranularity @@ -65,6 +66,7 @@ ExposureConfig, EmptySnapshotConfig, SnapshotConfig, + SemanticModelConfig, ) @@ -1482,6 +1484,7 @@ class SemanticModel(GraphNode): depends_on: DependsOn = field(default_factory=DependsOn) refs: List[RefArgs] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) + config: SemanticModelConfig = field(default_factory=SemanticModelConfig) @property def entity_references(self) -> List[LinkableElementReference]: @@ -1540,6 +1543,29 @@ def depends_on_nodes(self): def depends_on_macros(self): return self.depends_on.macros + def checked_agg_time_dimension_for_measure( + self, measure_reference: MeasureReference + ) -> TimeDimensionReference: + measure: Optional[Measure] = None + for measure in self.measures: + if measure.reference == measure_reference: + measure = measure + + assert ( + measure is not None + ), f"No measure with name ({measure_reference.element_name}) in semantic_model with name ({self.name})" + + if self.defaults is not None: + default_agg_time_dimesion = self.defaults.agg_time_dimension + + agg_time_dimension_name = measure.agg_time_dimension or default_agg_time_dimesion + assert agg_time_dimension_name is not None, ( + f"Aggregation time dimension for measure {measure.name} is not set! This should either be set directly on " + f"the measure specification in the model, or else defaulted to the primary time dimension in the data " + f"source containing the measure." + ) + return TimeDimensionReference(element_name=agg_time_dimension_name) + # ==================================== # Patches diff --git a/core/dbt/contracts/graph/semantic_models.py b/core/dbt/contracts/graph/semantic_models.py index 492c3188808..3ccf705ed85 100644 --- a/core/dbt/contracts/graph/semantic_models.py +++ b/core/dbt/contracts/graph/semantic_models.py @@ -142,13 +142,6 @@ class Measure(dbtClassMixin): non_additive_dimension: Optional[NonAdditiveDimension] = None agg_time_dimension: Optional[str] = None - @property - def checked_agg_time_dimension(self) -> TimeDimensionReference: - if self.agg_time_dimension is not None: - return TimeDimensionReference(element_name=self.agg_time_dimension) - else: - raise Exception("Measure is missing agg_time_dimension!") - @property def reference(self) -> MeasureReference: return MeasureReference(element_name=self.name) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 3d9c932a1bf..bb47cb51867 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1454,7 +1454,7 @@ def _process_metric_node( current_project: str, metric: Metric, ) -> None: - """Sets a metric's input_measures""" + """Sets a metric's `input_measures` and `depends_on` properties""" # This ensures that if this metrics input_measures have already been set # we skip the work. This could happen either due to recursion or if multiple @@ -1468,6 +1468,18 @@ def _process_metric_node( metric.type_params.measure is not None ), f"{metric} should have a measure defined, but it does not." metric.type_params.input_measures.append(metric.type_params.measure) + target_semantic_model = manifest.resolve_semantic_model_for_measure( + target_measure_name=metric.type_params.measure.name, + current_project=current_project, + node_package=metric.package_name, + ) + if target_semantic_model is None: + raise dbt.exceptions.ParsingError( + f"A semantic model having a measure `{metric.type_params.measure.name}` does not exist but was referenced.", + node=metric, + ) + + metric.depends_on.add_node(target_semantic_model.unique_id) elif metric.type is MetricType.DERIVED or metric.type is MetricType.RATIO: input_metrics = metric.input_metrics @@ -1502,6 +1514,7 @@ def _process_metric_node( manifest=manifest, current_project=current_project, metric=target_metric ) metric.type_params.input_measures.extend(target_metric.type_params.input_measures) + metric.depends_on.add_node(target_metric.unique_id) else: assert_values_exhausted(metric.type) diff --git a/core/setup.py b/core/setup.py index c56315997be..ea8332a6102 100644 --- a/core/setup.py +++ b/core/setup.py @@ -78,7 +78,7 @@ "minimal-snowplow-tracker~=0.0.2", # DSI is under active development, so we're pinning to specific dev versions for now. # TODO: Before RC/final release, update to use ~= pinning. - "dbt-semantic-interfaces==0.1.0.dev8", + "dbt-semantic-interfaces~=0.1.0.dev10", # ---- # Expect compatibility with all new versions of these packages, so lower bounds only. "packaging>20.9", diff --git a/tests/functional/access/test_access.py b/tests/functional/access/test_access.py index 4e9551d08a3..424616970f7 100644 --- a/tests/functional/access/test_access.py +++ b/tests/functional/access/test_access.py @@ -112,7 +112,7 @@ group: analytics - name: people_model description: "some people" - access: private + access: public group: analytics """ @@ -124,6 +124,31 @@ select 1 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at """ +people_semantic_model_yml = """ +semantic_models: + - name: semantic_people + model: ref('people_model') + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at +""" + people_metric_yml = """ metrics: @@ -203,6 +228,10 @@ group: package """ +metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day +""" + class TestAccess: @pytest.fixture(scope="class") @@ -278,10 +307,12 @@ def test_access_attribute(self, project): write_file(v5_schema_yml, project.project_root, "models", "schema.yml") rm_file(project.project_root, "models", "simple_exposure.yml") write_file(people_model_sql, "models", "people_model.sql") + write_file(people_semantic_model_yml, "models", "people_semantic_model.yml") write_file(people_metric_yml, "models", "people_metric.yml") + write_file(metricflow_time_spine_sql, "models", "metricflow_time_spine.sql") # Should succeed manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 4 + assert len(manifest.nodes) == 5 manifest = get_manifest(project.project_root) metric_id = "metric.test.number_of_people" assert manifest.metrics[metric_id].group == "analytics" diff --git a/tests/functional/artifacts/test_previous_version_state.py b/tests/functional/artifacts/test_previous_version_state.py index 401ff2b4dc1..7ed5fb61310 100644 --- a/tests/functional/artifacts/test_previous_version_state.py +++ b/tests/functional/artifacts/test_previous_version_state.py @@ -113,6 +113,10 @@ select 9 as id """ +metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day +""" + # Use old attribute names (v1.0-1.2) to test forward/backward compatibility with the rename in v1.3 models__schema_yml = """ version: 2 @@ -127,6 +131,32 @@ tests: - not_null +semantic_models: + - name: semantic_people + model: ref('my_model') + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + - name: customers + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at + metrics: - name: my_metric label: Count records @@ -208,6 +238,7 @@ def models(self): "schema.yml": models__schema_yml, "somedoc.md": docs__somedoc_md, "disabled_model.sql": models__disabled_model_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, } @pytest.fixture(scope="class") @@ -250,10 +281,10 @@ def test_project(self, project): # This is mainly used to test changes to the test project in isolation from # the other noise. results = run_dbt(["run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) # model, snapshot, seed, singular test, generic test, analysis - assert len(manifest.nodes) == 7 + assert len(manifest.nodes) == 8 assert len(manifest.sources) == 1 assert len(manifest.exposures) == 1 assert len(manifest.metrics) == 1 @@ -297,7 +328,7 @@ def compare_previous_state( ] if expect_pass: results = run_dbt(cli_args, expect_pass=expect_pass) - assert len(results) == 0 + assert len(results) == 1 else: with pytest.raises(IncompatibleSchemaError): run_dbt(cli_args, expect_pass=expect_pass) diff --git a/tests/functional/exposures/fixtures.py b/tests/functional/exposures/fixtures.py index 8b97c657aff..809df9e901c 100644 --- a/tests/functional/exposures/fixtures.py +++ b/tests/functional/exposures/fixtures.py @@ -7,6 +7,11 @@ """ +metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day +""" + + source_schema_yml = """version: 2 sources: @@ -15,6 +20,28 @@ - name: test_table """ + +semantic_models_schema_yml = """version: 2 + +semantic_models: + - name: semantic_model + model: ref('model') + dimensions: + - name: created_at + type: time + measures: + - name: distinct_metrics + agg: count_distinct + expr: id + entities: + - name: model + type: primary + expr: id + defaults: + agg_time_dimension: created_at +""" + + metrics_schema_yml = """version: 2 metrics: diff --git a/tests/functional/exposures/test_exposure_configs.py b/tests/functional/exposures/test_exposure_configs.py index 199a6368a4a..34c5570a84e 100644 --- a/tests/functional/exposures/test_exposure_configs.py +++ b/tests/functional/exposures/test_exposure_configs.py @@ -12,6 +12,8 @@ enabled_yaml_level_exposure_yml, invalid_config_exposure_yml, source_schema_yml, + metricflow_time_spine_sql, + semantic_models_schema_yml, metrics_schema_yml, ) @@ -30,9 +32,11 @@ class TestExposureEnabledConfigProjectLevel(ExposureConfigTests): def models(self): return { "model.sql": models_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, "second_model.sql": second_model_sql, "exposure.yml": simple_exposure_yml, "schema.yml": source_schema_yml, + "semantic_models.yml": semantic_models_schema_yml, "metrics.yml": metrics_schema_yml, } diff --git a/tests/functional/exposures/test_exposures.py b/tests/functional/exposures/test_exposures.py index 97849fa0835..1988dd976b3 100644 --- a/tests/functional/exposures/test_exposures.py +++ b/tests/functional/exposures/test_exposures.py @@ -7,6 +7,8 @@ simple_exposure_yml, source_schema_yml, metrics_schema_yml, + semantic_models_schema_yml, + metricflow_time_spine_sql, ) @@ -16,8 +18,10 @@ def models(self): return { "exposure.yml": simple_exposure_yml, "model.sql": models_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, "second_model.sql": second_model_sql, "schema.yml": source_schema_yml, + "semantic_models.yml": semantic_models_schema_yml, "metrics.yml": metrics_schema_yml, } diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index 9f5b4a37ae5..87fc67c10c6 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -17,9 +17,36 @@ models_people_sql = """ select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at union all -select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at +select 2 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at union all -select 1 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at +select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at +""" + +semantic_model_people_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at """ basic_metrics_yml = """ @@ -63,6 +90,10 @@ expr: "average_tenure + 1" """ +metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023, 'mm/dd/yyyy') as date_day +""" + models_people_metrics_yml = """ version: 2 @@ -296,8 +327,7 @@ label: {{ m.label }} type: {{ m.type }} type_params: {{ m.type_params }} - filters {{ m.filter }} - window: {{ m.window }} + filter: {{ m.filter }} {% endfor %} {% endif %} @@ -345,6 +375,35 @@ - payment_type """ +purchasing_model_sql = """ +select purchased_at, payment_type, payment_total from {{ ref('mock_purchase_data') }} +""" + +semantic_model_purchasing_yml = """ +version: 2 + +semantic_models: + - name: semantic_purchasing + model: ref('purchasing') + measures: + - name: num_orders + agg: COUNT + expr: purchased_at + - name: order_revenue + agg: SUM + expr: payment_total + dimensions: + - name: purchased_at + type: TIME + entities: + - name: purchase + type: primary + expr: '1' + defaults: + agg_time_dimension: purchased_at + +""" + derived_metric_yml = """ version: 2 metrics: diff --git a/tests/functional/metrics/test_metric_configs.py b/tests/functional/metrics/test_metric_configs.py index efc3f37b26a..03b8fe2275c 100644 --- a/tests/functional/metrics/test_metric_configs.py +++ b/tests/functional/metrics/test_metric_configs.py @@ -8,10 +8,12 @@ from tests.functional.metrics.fixtures import ( models_people_sql, models_people_metrics_yml, + metricflow_time_spine_sql, disabled_metric_level_schema_yml, enabled_metric_level_schema_yml, models_people_metrics_sql, invalid_config_metric_yml, + semantic_model_people_yml, ) @@ -29,6 +31,8 @@ class TestMetricEnabledConfigProjectLevel(MetricConfigTests): def models(self): return { "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": models_people_metrics_yml, } @@ -69,6 +73,8 @@ class TestConfigYamlMetricLevel(MetricConfigTests): def models(self): return { "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": disabled_metric_level_schema_yml, } @@ -85,6 +91,8 @@ class TestMetricConfigsInheritence(MetricConfigTests): def models(self): return { "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": enabled_metric_level_schema_yml, } @@ -112,6 +120,8 @@ class TestDisabledMetricRef(MetricConfigTests): def models(self): return { "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, "people_metrics.sql": models_people_metrics_sql, "schema.yml": models_people_metrics_yml, } @@ -152,6 +162,8 @@ class TestInvalidMetric(MetricConfigTests): def models(self): return { "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": invalid_config_metric_yml, } @@ -167,6 +179,8 @@ class TestDisabledMetric(MetricConfigTests): def models(self): return { "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, "schema.yml": models_people_metrics_yml, } diff --git a/tests/functional/metrics/test_metrics.py b/tests/functional/metrics/test_metrics.py index 0f1ebe67fdf..3cc0ea412b7 100644 --- a/tests/functional/metrics/test_metrics.py +++ b/tests/functional/metrics/test_metrics.py @@ -22,6 +22,10 @@ derived_metric_yml, invalid_metric_without_timestamp_with_time_grains_yml, invalid_metric_without_timestamp_with_window_yml, + metricflow_time_spine_sql, + semantic_model_people_yml, + semantic_model_purchasing_yml, + purchasing_model_sql, ) @@ -30,6 +34,8 @@ class TestSimpleMetrics: def models(self): return { "people_metrics.yml": models_people_metrics_yml, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, "people.sql": models_people_sql, } @@ -197,12 +203,42 @@ def test_invalid_derived_metrics(self, project): run_dbt(["run"]) +class TestMetricDependsOn: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + } + + def test_metric_depends_on(self, project): + manifest = run_dbt(["parse"]) + assert isinstance(manifest, Manifest) + + expected_depends_on_for_number_of_people = ["semantic_model.test.semantic_people"] + expected_depends_on_for_average_tenure = [ + "metric.test.collective_tenure", + "metric.test.number_of_people", + ] + + number_of_people_metric = manifest.metrics["metric.test.number_of_people"] + assert number_of_people_metric.depends_on.nodes == expected_depends_on_for_number_of_people + + average_tenure_metric = manifest.metrics["metric.test.average_tenure"] + assert average_tenure_metric.depends_on.nodes == expected_depends_on_for_average_tenure + + class TestDerivedMetric: @pytest.fixture(scope="class") def models(self): return { - "derived_metric.yml": derived_metric_yml, "downstream_model.sql": downstream_model_sql, + "purchasing.sql": purchasing_model_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_purchasing_yml, + "derived_metric.yml": derived_metric_yml, } # not strictly necessary to use "real" mock data for this test @@ -214,7 +250,6 @@ def seeds(self): "mock_purchase_data.csv": mock_purchase_data_csv, } - @pytest.mark.skip("TODO bring back once we start populating metric `depends_on`") def test_derived_metric( self, project, @@ -245,7 +280,6 @@ def test_derived_metric( # make sure the 'expression' metric depends on the two upstream metrics derived_metric = manifest.metrics["metric.test.average_order_value"] - assert sorted(derived_metric.metrics) == [["count_orders"], ["sum_order_revenue"]] assert sorted(derived_metric.depends_on.nodes) == [ "metric.test.count_orders", "metric.test.sum_order_revenue", @@ -264,7 +298,6 @@ def test_derived_metric( "type", "type_params", "filter", - "window", ]: expected_value = getattr(parsed_metric_node, property) assert f"{property}: {expected_value}" in compiled_code diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py index 12ec77b178e..9da5a6f5025 100644 --- a/tests/functional/partial_parsing/fixtures.py +++ b/tests/functional/partial_parsing/fixtures.py @@ -397,6 +397,10 @@ """ +metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day +""" + env_var_schema3_yml = """ models: @@ -421,6 +425,33 @@ """ +people_semantic_models_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at +""" + env_var_metrics_yml = """ metrics: diff --git a/tests/functional/partial_parsing/test_pp_disabled_config.py b/tests/functional/partial_parsing/test_pp_disabled_config.py index 01df216d758..03d2e8a728b 100644 --- a/tests/functional/partial_parsing/test_pp_disabled_config.py +++ b/tests/functional/partial_parsing/test_pp_disabled_config.py @@ -5,12 +5,34 @@ select 1 as fun """ +metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day +""" + schema1_yml = """ version: 2 models: - name: model_one +semantic_models: + - name: semantic_people + model: ref('model_one') + dimensions: + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: people + agg: count + expr: fun + entities: + - name: fun + type: primary + defaults: + agg_time_dimension: created_at + metrics: - name: number_of_people @@ -39,6 +61,24 @@ models: - name: model_one +semantic_models: + - name: semantic_people + model: ref('model_one') + dimensions: + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: people + agg: count + expr: fun + entities: + - name: fun + type: primary + defaults: + agg_time_dimension: created_at + metrics: - name: number_of_people @@ -71,6 +111,24 @@ models: - name: model_one +semantic_models: + - name: semantic_people + model: ref('model_one') + dimensions: + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: people + agg: count + expr: fun + entities: + - name: fun + type: primary + defaults: + agg_time_dimension: created_at + metrics: - name: number_of_people @@ -108,6 +166,7 @@ class TestDisabled: def models(self): return { "model_one.sql": model_one_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, "schema.yml": schema1_yml, } @@ -116,10 +175,8 @@ def test_pp_disabled(self, project): expected_metric = "metric.test.number_of_people" run_dbt(["seed"]) - results = run_dbt(["run"]) - assert len(results) == 1 + manifest = run_dbt(["parse"]) - manifest = get_manifest(project.project_root) assert expected_exposure in manifest.exposures assert expected_metric in manifest.metrics assert expected_exposure not in manifest.disabled @@ -128,7 +185,7 @@ def test_pp_disabled(self, project): # Update schema file with disabled metric and exposure write_file(schema2_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure not in manifest.exposures assert expected_metric not in manifest.metrics @@ -138,7 +195,7 @@ def test_pp_disabled(self, project): # Update schema file with enabled metric and exposure write_file(schema1_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure in manifest.exposures assert expected_metric in manifest.metrics @@ -148,7 +205,7 @@ def test_pp_disabled(self, project): # Update schema file - remove exposure, enable metric write_file(schema3_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure not in manifest.exposures assert expected_metric in manifest.metrics @@ -158,7 +215,7 @@ def test_pp_disabled(self, project): # Update schema file - add back exposure, remove metric write_file(schema4_yml, project.project_root, "models", "schema.yml") results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) assert expected_exposure not in manifest.exposures assert expected_metric not in manifest.metrics diff --git a/tests/functional/partial_parsing/test_pp_metrics.py b/tests/functional/partial_parsing/test_pp_metrics.py index d6e837f7b55..da994e09808 100644 --- a/tests/functional/partial_parsing/test_pp_metrics.py +++ b/tests/functional/partial_parsing/test_pp_metrics.py @@ -3,6 +3,8 @@ from dbt.tests.util import run_dbt, write_file, get_manifest from tests.functional.partial_parsing.fixtures import ( people_sql, + metricflow_time_spine_sql, + people_semantic_models_yml, people_metrics_yml, people_metrics2_yml, metric_model_a_sql, @@ -17,19 +19,26 @@ class TestMetrics: def models(self): return { "people.sql": people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, } def test_metrics(self, project): # initial run results = run_dbt(["run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 1 + assert len(manifest.nodes) == 2 - # Add metrics yaml file + # Add metrics yaml file (and necessary semantic models yaml) + write_file( + people_semantic_models_yml, + project.project_root, + "models", + "people_semantic_models.yml", + ) write_file(people_metrics_yml, project.project_root, "models", "people_metrics.yml") results = run_dbt(["run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) assert len(manifest.metrics) == 2 metric_people_id = "metric.test.number_of_people" @@ -48,7 +57,7 @@ def test_metrics(self, project): # Change metrics yaml files write_file(people_metrics2_yml, project.project_root, "models", "people_metrics.yml") results = run_dbt(["run"]) - assert len(results) == 1 + assert len(results) == 2 manifest = get_manifest(project.project_root) metric_people = manifest.metrics[metric_people_id] expected_meta = {"my_meta": "replaced"} diff --git a/tests/functional/partial_parsing/test_pp_vars.py b/tests/functional/partial_parsing/test_pp_vars.py index a73d250eb30..f57fca06b1e 100644 --- a/tests/functional/partial_parsing/test_pp_vars.py +++ b/tests/functional/partial_parsing/test_pp_vars.py @@ -17,8 +17,10 @@ env_var_schema3_yml, env_var_schema_yml, env_var_sources_yml, + metricflow_time_spine_sql, model_color_sql, model_one_sql, + people_semantic_models_yml, people_sql, raw_customers_csv, test_color_sql, @@ -228,6 +230,12 @@ def test_env_vars_models(self, project): # Add a metrics file with env_vars os.environ["ENV_VAR_METRICS"] = "TeStInG" write_file(people_sql, project.project_root, "models", "people.sql") + write_file( + metricflow_time_spine_sql, project.project_root, "models", "metricflow_time_spine.sql" + ) + write_file( + people_semantic_models_yml, project.project_root, "models", "semantic_models.yml" + ) write_file(env_var_metrics_yml, project.project_root, "models", "metrics.yml") results = run_dbt(["run"]) manifest = get_manifest(project.project_root)