From 8b2948de024514f63325a6263ef28498d4128733 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Mon, 13 Nov 2023 15:45:14 +0000 Subject: [PATCH 01/15] Fix #8652: Use seed value if rows not specified --- .../unreleased/Fixes-20231113-154535.yaml | 6 ++ core/dbt/parser/unit_tests.py | 27 +++++++- .../unit_testing/test_unit_testing.py | 61 +++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20231113-154535.yaml diff --git a/.changes/unreleased/Fixes-20231113-154535.yaml b/.changes/unreleased/Fixes-20231113-154535.yaml new file mode 100644 index 00000000000..13b900ec2dd --- /dev/null +++ b/.changes/unreleased/Fixes-20231113-154535.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use seed value if rows not specified +time: 2023-11-13T15:45:35.008565Z +custom: + Author: aranke + Issue: "8652" diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index c93f70b2997..91a977400b8 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -1,5 +1,9 @@ +from csv import DictReader +from pathlib import Path from typing import List, Set, Dict, Any +from dbt_extractor import py_extract_from_source, ExtractionError # type: ignore + from dbt.config import RuntimeConfig from dbt.context.context_config import ContextConfig from dbt.context.providers import generate_parse_exposure, get_rendered @@ -14,7 +18,7 @@ UnitTestConfig, ) from dbt.contracts.graph.unparsed import UnparsedUnitTest -from dbt.exceptions import ParsingError, InvalidUnitTestGivenInput +from dbt.exceptions import ParsingError, InvalidUnitTestGivenInput, DbtInternalError from dbt.graph import UniqueId from dbt.node_types import NodeType from dbt.parser.schemas import ( @@ -27,7 +31,6 @@ ParseResult, ) from dbt.utils import get_pseudo_test_path -from dbt_extractor import py_extract_from_source, ExtractionError # type: ignore class UnitTestManifestLoader: @@ -203,6 +206,22 @@ def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: self.schema_parser = schema_parser self.yaml = yaml + def load_rows_from_seed(self, seed_name): + rows = [] + + try: + seed_node = self.manifest.ref_lookup.perform_lookup( + f"seed.{self.project.project_name}.{seed_name}", self.manifest + ) + seed_path = Path(seed_node.root_path) / seed_node.original_file_path + with open(seed_path, "r") as f: + for row in DictReader(f): + rows.append(row) + except DbtInternalError: + pass + finally: + return rows + def parse(self) -> ParseResult: for data in self.get_key_dicts(): unit_test = self._get_unit_test(data) @@ -214,8 +233,12 @@ def parse(self) -> ParseResult: unit_test_fqn = [self.project.project_name] + model_name_split + [unit_test.name] unit_test_config = self._build_unit_test_config(unit_test_fqn, unit_test.config) + # self.manifest.ref_lookup.perform_lookup('seed.test.my_favorite_source', self.manifest) + # Check that format and type of rows matches for each given input for input in unit_test.given: + if input.rows is None: + input.rows = self.load_rows_from_seed(input.input.split("'")[1]) input.validate_fixture("input", unit_test.name) unit_test.expect.validate_fixture("expected", unit_test.name) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 2a631c23efe..e475f162e96 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -100,3 +100,64 @@ def test_basic(self, project): # Select by model name results = run_dbt(["unit-test", "--select", "my_incremental_model"], expect_pass=True) assert len(results) == 2 + + +my_new_model = """ +select +my_favorite_seed.id, +a + b as c +from {{ ref('my_favorite_seed') }} as my_favorite_seed +inner join {{ ref('my_favorite_model') }} as my_favorite_model +on my_favorite_seed.id = my_second_favorite_model.id +""" + +my_second_favorite_model = """ +select +2 as id, +3 as b +""" + +seed_my_favorite_seed = """id,a +1,5 +2,4 +3,3 +4,2 +5,1 +""" + +test_my_model_implicit_seed = """ +unit_tests: + - name: t + model: my_new_model + given: + - input: ref('my_favorite_seed') + - input: ref('my_second_favorite_model') + rows: + - {id: 1, b: 2} + expect: + rows: + - {id: 1, c: 7} +""" + + +class TestUnitTestImplicitSeed: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_favorite_seed.csv": seed_my_favorite_seed} + + @pytest.fixture(scope="class") + def models(self): + return { + "my_new_model.sql": my_new_model, + "my_second_favorite_model.sql": my_second_favorite_model, + "schema.yml": test_my_model_implicit_seed, + } + + def test_basic(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + # assert len(results) == 1 + + # Select by model name + results = run_dbt(["unit-test", "--select", "my_new_model"], expect_pass=True) + assert len(results) == 1 From d1c8f01d9582ffc2caad720ab24b8f0806100169 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 14:48:18 +0000 Subject: [PATCH 02/15] More testing --- .../unit_testing/test_unit_testing.py | 70 +++++++++++++++++-- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index e475f162e96..e504e1c93da 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -108,10 +108,10 @@ def test_basic(self, project): a + b as c from {{ ref('my_favorite_seed') }} as my_favorite_seed inner join {{ ref('my_favorite_model') }} as my_favorite_model -on my_favorite_seed.id = my_second_favorite_model.id +on my_favorite_seed.id = my_favorite_model.id """ -my_second_favorite_model = """ +my_favorite_model = """ select 2 as id, 3 as b @@ -125,13 +125,43 @@ def test_basic(self, project): 5,1 """ -test_my_model_implicit_seed = """ +schema_yml_explicit_seed = """ unit_tests: - name: t model: my_new_model given: - input: ref('my_favorite_seed') - - input: ref('my_second_favorite_model') + rows: + - {id: 1, a: 10} + - input: ref('my_favorite_model') + rows: + - {id: 1, b: 2} + expect: + rows: + - {id: 1, c: 12} +""" + +schema_yml_implicit_seed = """ +unit_tests: + - name: t + model: my_new_model + given: + - input: ref('my_favorite_seed') + - input: ref('my_favorite_model') + rows: + - {id: 1, b: 2} + expect: + rows: + - {id: 1, c: 7} +""" + +schema_yml_nonexistent_seed = """ +unit_tests: + - name: t + model: my_new_model + given: + - input: ref('my_second_favorite_seed') + - input: ref('my_favorite_model') rows: - {id: 1, b: 2} expect: @@ -140,7 +170,7 @@ def test_basic(self, project): """ -class TestUnitTestImplicitSeed: +class TestUnitTestBaseSeed: @pytest.fixture(scope="class") def seeds(self): return {"my_favorite_seed.csv": seed_my_favorite_seed} @@ -149,8 +179,8 @@ def seeds(self): def models(self): return { "my_new_model.sql": my_new_model, - "my_second_favorite_model.sql": my_second_favorite_model, - "schema.yml": test_my_model_implicit_seed, + "my_favorite_model.sql": my_favorite_model, + "schema.yml": self.schema_yml, } def test_basic(self, project): @@ -161,3 +191,29 @@ def test_basic(self, project): # Select by model name results = run_dbt(["unit-test", "--select", "my_new_model"], expect_pass=True) assert len(results) == 1 + + +class TestUnitTestExplicitSeed(TestUnitTestBaseSeed): + schema_yml = schema_yml_explicit_seed + + +class TestUnitTestImplicitSeed(TestUnitTestBaseSeed): + schema_yml = schema_yml_implicit_seed + + +class TestUnitTestNonexistentSeed(TestUnitTestBaseSeed): + @pytest.fixture(scope="class") + def models(self): + return { + "my_new_model.sql": my_new_model, + "my_favorite_model.sql": my_favorite_model, + "schema.yml": schema_yml_nonexistent_seed, + } + + def test_basic(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + # Select by model name + with pytest.raises(AttributeError): + run_dbt(["unit-test", "--select", "my_new_model"], expect_pass=False) From d6961cac89ed99c7dc510bc1a9a5f61f1157074b Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 15:03:13 +0000 Subject: [PATCH 03/15] Stop being clever --- .../unit_testing/test_unit_testing.py | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index e504e1c93da..14784ce1c41 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -170,7 +170,7 @@ def test_basic(self, project): """ -class TestUnitTestBaseSeed: +class TestUnitTestExplicitSeed: @pytest.fixture(scope="class") def seeds(self): return {"my_favorite_seed.csv": seed_my_favorite_seed} @@ -180,7 +180,7 @@ def models(self): return { "my_new_model.sql": my_new_model, "my_favorite_model.sql": my_favorite_model, - "schema.yml": self.schema_yml, + "schema.yml": schema_yml_explicit_seed, } def test_basic(self, project): @@ -193,15 +193,30 @@ def test_basic(self, project): assert len(results) == 1 -class TestUnitTestExplicitSeed(TestUnitTestBaseSeed): - schema_yml = schema_yml_explicit_seed +class TestUnitTestImplicitSeed: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_favorite_seed.csv": seed_my_favorite_seed} + @pytest.fixture(scope="class") + def models(self): + return { + "my_new_model.sql": my_new_model, + "my_favorite_model.sql": my_favorite_model, + "schema.yml": schema_yml_implicit_seed, + } -class TestUnitTestImplicitSeed(TestUnitTestBaseSeed): - schema_yml = schema_yml_implicit_seed + def test_basic(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + # assert len(results) == 1 + + # Select by model name + results = run_dbt(["unit-test", "--select", "my_new_model"], expect_pass=True) + assert len(results) == 1 -class TestUnitTestNonexistentSeed(TestUnitTestBaseSeed): +class TestUnitTestNonexistentSeed: @pytest.fixture(scope="class") def models(self): return { From 46f2b527c1977d2ad62dfd5a94fea454ea2db728 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 15:04:57 +0000 Subject: [PATCH 04/15] Update Fixes-20231113-154535.yaml --- .changes/unreleased/Fixes-20231113-154535.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Fixes-20231113-154535.yaml b/.changes/unreleased/Fixes-20231113-154535.yaml index 13b900ec2dd..f352830921f 100644 --- a/.changes/unreleased/Fixes-20231113-154535.yaml +++ b/.changes/unreleased/Fixes-20231113-154535.yaml @@ -1,5 +1,5 @@ kind: Fixes -body: Use seed value if rows not specified +body: Use seed file from disk for unit testing if rows not specified in YAML config time: 2023-11-13T15:45:35.008565Z custom: Author: aranke From 60fd28e8cdaa52144d2f370f44b53c379093e70c Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 15:21:57 +0000 Subject: [PATCH 05/15] add back seed --- tests/functional/unit_testing/test_unit_testing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 14784ce1c41..bbd0cc943f0 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -217,6 +217,10 @@ def test_basic(self, project): class TestUnitTestNonexistentSeed: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_favorite_seed.csv": seed_my_favorite_seed} + @pytest.fixture(scope="class") def models(self): return { From 3abb089b345e756fa542a355c5de410b9cefabf4 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 17:50:17 +0000 Subject: [PATCH 06/15] remove comments, add type annotations and docstring --- core/dbt/parser/unit_tests.py | 9 ++++----- tests/functional/unit_testing/test_unit_testing.py | 8 +++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 91a977400b8..ec23c9cc603 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -206,8 +206,9 @@ def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: self.schema_parser = schema_parser self.yaml = yaml - def load_rows_from_seed(self, seed_name): - rows = [] + def _load_rows_from_seed(self, seed_name: str) -> List[Dict[str, Any]]: + """Read rows from seed file on disk if not specified in YAML config. If seed file doesn't exist, return empty list.""" + rows: List[Dict[str, Any]] = [] try: seed_node = self.manifest.ref_lookup.perform_lookup( @@ -233,12 +234,10 @@ def parse(self) -> ParseResult: unit_test_fqn = [self.project.project_name] + model_name_split + [unit_test.name] unit_test_config = self._build_unit_test_config(unit_test_fqn, unit_test.config) - # self.manifest.ref_lookup.perform_lookup('seed.test.my_favorite_source', self.manifest) - # Check that format and type of rows matches for each given input for input in unit_test.given: if input.rows is None: - input.rows = self.load_rows_from_seed(input.input.split("'")[1]) + input.rows = self._load_rows_from_seed(input.input.split("'")[1]) input.validate_fixture("input", unit_test.name) unit_test.expect.validate_fixture("expected", unit_test.name) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index bbd0cc943f0..dc7affc119c 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -183,10 +183,9 @@ def models(self): "schema.yml": schema_yml_explicit_seed, } - def test_basic(self, project): + def test_explicit_seed(self, project): run_dbt(["seed"]) run_dbt(["run"]) - # assert len(results) == 1 # Select by model name results = run_dbt(["unit-test", "--select", "my_new_model"], expect_pass=True) @@ -206,10 +205,9 @@ def models(self): "schema.yml": schema_yml_implicit_seed, } - def test_basic(self, project): + def test_implicit_seed(self, project): run_dbt(["seed"]) run_dbt(["run"]) - # assert len(results) == 1 # Select by model name results = run_dbt(["unit-test", "--select", "my_new_model"], expect_pass=True) @@ -229,7 +227,7 @@ def models(self): "schema.yml": schema_yml_nonexistent_seed, } - def test_basic(self, project): + def test_nonexistent_seed(self, project): run_dbt(["seed"]) run_dbt(["run"]) From bf95d3fff98704844e5f06316a7960db91e31d8f Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 18:05:40 +0000 Subject: [PATCH 07/15] use py_extract_from_source --- core/dbt/parser/unit_tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index ec23c9cc603..0dab9ee716a 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -206,6 +206,10 @@ def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: self.schema_parser = schema_parser self.yaml = yaml + def _get_seed_name_from_ref(self, ref: str) -> str: + """Extract seed name from ref string.""" + return py_extract_from_source("{{ " + ref + " }}")["refs"][0]["name"] + def _load_rows_from_seed(self, seed_name: str) -> List[Dict[str, Any]]: """Read rows from seed file on disk if not specified in YAML config. If seed file doesn't exist, return empty list.""" rows: List[Dict[str, Any]] = [] @@ -237,7 +241,8 @@ def parse(self) -> ParseResult: # Check that format and type of rows matches for each given input for input in unit_test.given: if input.rows is None: - input.rows = self._load_rows_from_seed(input.input.split("'")[1]) + seed_name = self._get_seed_name_from_ref(input.input) + input.rows = self._load_rows_from_seed(seed_name) input.validate_fixture("input", unit_test.name) unit_test.expect.validate_fixture("expected", unit_test.name) From cae056f0009ef070d973ba1d06fb11ea15f7a1d8 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 18:18:00 +0000 Subject: [PATCH 08/15] Localize ParsingError --- core/dbt/parser/unit_tests.py | 21 ++++++++++++------- .../unit_testing/test_unit_testing.py | 10 ++++----- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 0dab9ee716a..40d3cafd0c3 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -214,18 +214,23 @@ def _load_rows_from_seed(self, seed_name: str) -> List[Dict[str, Any]]: """Read rows from seed file on disk if not specified in YAML config. If seed file doesn't exist, return empty list.""" rows: List[Dict[str, Any]] = [] + package_name = self.project.project_name + try: seed_node = self.manifest.ref_lookup.perform_lookup( - f"seed.{self.project.project_name}.{seed_name}", self.manifest + f"seed.{package_name}.{seed_name}", self.manifest ) - seed_path = Path(seed_node.root_path) / seed_node.original_file_path - with open(seed_path, "r") as f: - for row in DictReader(f): - rows.append(row) except DbtInternalError: - pass - finally: - return rows + raise ParsingError( + f"Unable to find seed '{package_name}.{seed_name}' for unit tests in directories: {self.project.seed_paths}" + ) + + seed_path = Path(seed_node.root_path) / seed_node.original_file_path + with open(seed_path, "r") as f: + for row in DictReader(f): + rows.append(row) + + return rows def parse(self) -> ParseResult: for data in self.get_key_dicts(): diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index dc7affc119c..9043c947fa2 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -1,6 +1,6 @@ import pytest from dbt.tests.util import run_dbt, write_file, get_manifest, get_artifact -from dbt.exceptions import DuplicateResourceNameError +from dbt.exceptions import DuplicateResourceNameError, ParsingError from fixtures import ( my_model_vars_sql, my_model_a_sql, @@ -228,9 +228,7 @@ def models(self): } def test_nonexistent_seed(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - - # Select by model name - with pytest.raises(AttributeError): + with pytest.raises( + ParsingError, match="Unable to find seed 'test.my_second_favorite_seed' for unit tests" + ): run_dbt(["unit-test", "--select", "my_new_model"], expect_pass=False) From a80a4e56a0f823c38d0c02a92270cd77b4e7b39b Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 18:23:35 +0000 Subject: [PATCH 09/15] Use ref_lookup.find instead --- core/dbt/parser/unit_tests.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 40d3cafd0c3..345d2b59662 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -18,7 +18,7 @@ UnitTestConfig, ) from dbt.contracts.graph.unparsed import UnparsedUnitTest -from dbt.exceptions import ParsingError, InvalidUnitTestGivenInput, DbtInternalError +from dbt.exceptions import ParsingError, InvalidUnitTestGivenInput from dbt.graph import UniqueId from dbt.node_types import NodeType from dbt.parser.schemas import ( @@ -216,11 +216,9 @@ def _load_rows_from_seed(self, seed_name: str) -> List[Dict[str, Any]]: package_name = self.project.project_name - try: - seed_node = self.manifest.ref_lookup.perform_lookup( - f"seed.{package_name}.{seed_name}", self.manifest - ) - except DbtInternalError: + seed_node = self.manifest.ref_lookup.find(seed_name, package_name, None, self.manifest) + + if not seed_node: raise ParsingError( f"Unable to find seed '{package_name}.{seed_name}' for unit tests in directories: {self.project.seed_paths}" ) From 3191497d9f83e4c10405c564428a5c66154328cd Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 14 Nov 2023 19:28:43 +0000 Subject: [PATCH 10/15] Add resource_type check --- core/dbt/parser/unit_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 345d2b59662..14291602c5d 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -218,7 +218,7 @@ def _load_rows_from_seed(self, seed_name: str) -> List[Dict[str, Any]]: seed_node = self.manifest.ref_lookup.find(seed_name, package_name, None, self.manifest) - if not seed_node: + if not seed_node or seed_node.resource_type != NodeType.Seed: raise ParsingError( f"Unable to find seed '{package_name}.{seed_name}' for unit tests in directories: {self.project.seed_paths}" ) From 1a2c2ed4c43bda343d911784512742dc55916b72 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 15 Nov 2023 15:22:19 +0000 Subject: [PATCH 11/15] Check if fixture is null --- core/dbt/parser/unit_tests.py | 2 +- tests/functional/unit_testing/test_unit_testing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index ba61b030fd5..031ae38dbfc 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -243,7 +243,7 @@ def parse(self) -> ParseResult: # Check that format and type of rows matches for each given input for input in unit_test.given: - if input.rows is None: + if input.rows is None and input.fixture is None: seed_name = self._get_seed_name_from_ref(input.input) input.rows = self._load_rows_from_seed(seed_name) input.validate_fixture("input", unit_test.name) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 15f23f2a3c6..815881da12b 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -5,7 +5,7 @@ get_manifest, get_artifact, ) -from dbt.exceptions import DuplicateResourceNameError +from dbt.exceptions import DuplicateResourceNameError, ParsingError from fixtures import ( my_model_vars_sql, my_model_a_sql, From 39af105d2f807b3d298fc8db30abd1888df10833 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 15 Nov 2023 15:45:07 +0000 Subject: [PATCH 12/15] Allow for 2 arg ref for seed --- core/dbt/parser/unit_tests.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 031ae38dbfc..28d828e1675 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -206,22 +206,25 @@ def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: self.schema_parser = schema_parser self.yaml = yaml - def _get_seed_name_from_ref(self, ref: str) -> str: - """Extract seed name from ref string.""" - return py_extract_from_source("{{ " + ref + " }}")["refs"][0]["name"] - - def _load_rows_from_seed(self, seed_name: str) -> List[Dict[str, Any]]: + def _load_rows_from_seed(self, ref: Dict[str, str]) -> List[Dict[str, Any]]: """Read rows from seed file on disk if not specified in YAML config. If seed file doesn't exist, return empty list.""" rows: List[Dict[str, Any]] = [] - package_name = self.project.project_name + seed_name = ref["name"] + package_name = ref.get("package", self.project.project_name) seed_node = self.manifest.ref_lookup.find(seed_name, package_name, None, self.manifest) if not seed_node or seed_node.resource_type != NodeType.Seed: - raise ParsingError( - f"Unable to find seed '{package_name}.{seed_name}' for unit tests in directories: {self.project.seed_paths}" - ) + # Seed not found in custom package specified + if package_name != self.project.project_name: + raise ParsingError( + f"Unable to find seed '{package_name}.{seed_name}' for unit tests in '{package_name}' package" + ) + else: + raise ParsingError( + f"Unable to find seed '{package_name}.{seed_name}' for unit tests in directories: {self.project.seed_paths}" + ) seed_path = Path(seed_node.root_path) / seed_node.original_file_path with open(seed_path, "r") as f: @@ -244,8 +247,8 @@ def parse(self) -> ParseResult: # Check that format and type of rows matches for each given input for input in unit_test.given: if input.rows is None and input.fixture is None: - seed_name = self._get_seed_name_from_ref(input.input) - input.rows = self._load_rows_from_seed(seed_name) + ref = py_extract_from_source("{{ " + input.input + " }}")["refs"][0] + input.rows = self._load_rows_from_seed(ref) input.validate_fixture("input", unit_test.name) unit_test.expect.validate_fixture("expected", unit_test.name) From 31196b78a72a4c5eeaa115541e0dba4fe013c971 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 15 Nov 2023 16:20:40 +0000 Subject: [PATCH 13/15] add seed to nodetype list --- core/dbt/parser/unit_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index aeddd58cadd..e58935c9e34 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -133,7 +133,7 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): ), } - if original_input_node.resource_type == NodeType.Model: + if original_input_node.resource_type in (NodeType.Model, NodeType.Seed): input_name = f"{unit_test_node.name}__{original_input_node.name}" input_node = ModelNode( **common_fields, From e9ad3d277f99763e9e33e927f42da65a0797bcf8 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Thu, 16 Nov 2023 14:53:52 +0000 Subject: [PATCH 14/15] quick refactor --- core/dbt/parser/unit_tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index e58935c9e34..c0182ef017c 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -222,8 +222,10 @@ def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: self.schema_parser = schema_parser self.yaml = yaml - def _load_rows_from_seed(self, ref: Dict[str, str]) -> List[Dict[str, Any]]: + def _load_rows_from_seed(self, ref_str: str) -> List[Dict[str, Any]]: """Read rows from seed file on disk if not specified in YAML config. If seed file doesn't exist, return empty list.""" + ref = py_extract_from_source("{{ " + ref_str + " }}")["refs"][0] + rows: List[Dict[str, Any]] = [] seed_name = ref["name"] @@ -263,8 +265,7 @@ def parse(self) -> ParseResult: # Check that format and type of rows matches for each given input for input in unit_test.given: if input.rows is None and input.fixture is None: - ref = py_extract_from_source("{{ " + input.input + " }}")["refs"][0] - input.rows = self._load_rows_from_seed(ref) + input.rows = self._load_rows_from_seed(input.input) input.validate_fixture("input", unit_test.name) unit_test.expect.validate_fixture("expected", unit_test.name) From 6d4f4a6ce06b269ad4f6cf028e2d6f909b95f3be Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 15 Nov 2023 17:18:56 -0500 Subject: [PATCH 15/15] test pinning ddtrace (#9090) --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3f0aba49444..6270928d95d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ black==23.3.0 bumpversion -ddtrace +ddtrace==2.1.7 docutils flake8 flaky