diff --git a/test/integration/005_simple_seed_tests/models-exist/model.sql b/test/integration/005_simple_seed_tests/models-exist/model.sql deleted file mode 100644 index 809a05ba8f9..00000000000 --- a/test/integration/005_simple_seed_tests/models-exist/model.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ this.schema }}.seed_expected diff --git a/test/integration/005_simple_seed_tests/test_simple_seed.py b/test/integration/005_simple_seed_tests/test_simple_seed.py deleted file mode 100644 index e0d91f76a51..00000000000 --- a/test/integration/005_simple_seed_tests/test_simple_seed.py +++ /dev/null @@ -1,349 +0,0 @@ -import os -import csv -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestSimpleSeed(DBTIntegrationTest): - - def setUp(self): - DBTIntegrationTest.setUp(self) - self.run_sql_file("seed.sql") - - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models-downstream-seed" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'seeds': { - 'quote_columns': False, - } - } - - def use_full_refresh_project(self, full_refresh: bool): - overrides = { - 'seeds': { - 'quote_columns': False, - 'full_refresh': full_refresh, - } - } - self.use_default_project(overrides) - - def _seed_and_run(self): - assert len(self.run_dbt(['seed'])) == 1 - self.assertTablesEqual('seed_actual', 'seed_expected') - - assert len(self.run_dbt(['run'])) == 1 - self.assertTablesEqual('model', 'seed_expected') - - def _after_seed_model_state(self, cmd, exists: bool): - assert len(self.run_dbt(cmd)) == 1 - self.assertTablesEqual('seed_actual', 'seed_expected') - if exists: - self.assertTableDoesExist('model') - else: - self.assertTableDoesNotExist('model') - - @use_profile('postgres') - def test_postgres_simple_seed(self): - self._seed_and_run() - - # this should truncate the seed_actual table, then re-insert. - self._after_seed_model_state(['seed'], exists=True) - - @use_profile('postgres') - def test_postgres_simple_seed_full_refresh_flag(self): - self._seed_and_run() - - # this should drop the seed_actual table, then re-create it, so the - # model won't exist. - self._after_seed_model_state(['seed', '--full-refresh'], exists=False) - - @use_profile('postgres') - def test_postgres_simple_seed_full_refresh_config(self): - self._seed_and_run() - - # set the full_refresh config to False - self.use_full_refresh_project(False) - - self._after_seed_model_state(['seed'], exists=True) - # make sure we ignore the full-refresh flag (the config is higher - # priority than the flag) - self._after_seed_model_state(['seed', '--full-refresh'], exists=True) - - # this should drop the seed_actual table, then re-create it, so the - # model won't exist. - self.use_full_refresh_project(True) - self._after_seed_model_state(['seed'], exists=False) - - -class TestSimpleSeedCustomSchema(DBTIntegrationTest): - - def setUp(self): - super().setUp() - self.run_sql_file("seed.sql") - self._created_schemas.add( - self._get_schema_fqn(self.default_database, self.custom_schema_name()) - ) - - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'seeds': { - "schema": "custom_schema", - 'quote_columns': False, - }, - } - - def custom_schema_name(self): - return "{}_{}".format(self.unique_schema(), 'custom_schema') - - @use_profile('postgres') - def test_postgres_simple_seed_with_schema(self): - schema_name = self.custom_schema_name() - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name) - - # this should truncate the seed_actual table, then re-insert - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name) - - @use_profile('postgres') - def test_postgres_simple_seed_with_drop_and_schema(self): - schema_name = "{}_{}".format(self.unique_schema(), 'custom_schema') - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name) - - # this should drop the seed table, then re-create - results = self.run_dbt(["seed", "--full-refresh"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_actual","seed_expected", table_a_schema=schema_name) - - -class TestSimpleSeedDisabled(DBTIntegrationTest): - - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds-config'], - 'seeds': { - "test": { - "seed_enabled": { - "enabled": True - }, - "seed_disabled": { - "enabled": False - } - }, - 'quote_columns': False, - }, - } - - @use_profile('postgres') - def test_postgres_simple_seed_with_disabled(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 2) - self.assertTableDoesExist('seed_enabled') - self.assertTableDoesNotExist('seed_disabled') - - @use_profile('postgres') - def test_postgres_simple_seed_selection(self): - results = self.run_dbt(['seed', '--select', 'seed_enabled']) - self.assertEqual(len(results), 1) - self.assertTableDoesExist('seed_enabled') - self.assertTableDoesNotExist('seed_disabled') - self.assertTableDoesNotExist('seed_tricky') - - @use_profile('postgres') - def test_postgres_simple_seed_exclude(self): - results = self.run_dbt(['seed', '--exclude', 'seed_enabled']) - self.assertEqual(len(results), 1) - self.assertTableDoesNotExist('seed_enabled') - self.assertTableDoesNotExist('seed_disabled') - self.assertTableDoesExist('seed_tricky') - - -class TestSeedParsing(DBTIntegrationTest): - def setUp(self): - super().setUp() - self.run_sql_file("seed.sql") - - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models-exist" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds-bad'], - 'seeds': { - 'quote_columns': False, - }, - } - - @use_profile('postgres') - def test_postgres_dbt_run_skips_seeds(self): - # run does not try to parse the seed files - self.assertEqual(len(self.run_dbt(['run'])), 1) - - # make sure 'dbt seed' fails, otherwise our test is invalid! - self.run_dbt(['seed'], expect_pass=False) - - -class TestSimpleSeedWithBOM(DBTIntegrationTest): - - def setUp(self): - DBTIntegrationTest.setUp(self) - self.run_sql_file("seed.sql") - - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds-bom'], - 'seeds': { - 'quote_columns': False, - }, - } - - @use_profile('postgres') - def test_postgres_simple_seed(self): - # first make sure nobody "fixed" the file by accident - seed_path = os.path.join(self.config.seed_paths[0], 'seed_bom.csv') - # 'data-bom/seed_bom.csv' - with open(seed_path, encoding='utf-8') as fp: - self.assertEqual(fp.read(1), u'\ufeff') - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - self.assertTablesEqual("seed_bom", "seed_expected") - - -class TestSimpleSeedWithUnicode(DBTIntegrationTest): - - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds-unicode'], - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('postgres') - def test_postgres_simple_seed(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - -class TestSimpleSeedWithDots(DBTIntegrationTest): - - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds-dottedseed'], - 'seeds': { - 'quote_columns': False, - } - } - - @use_profile('postgres') - def test_postgres_simple_seed(self): - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - -class TestSimpleBigSeedBatched(DBTIntegrationTest): - @property - def schema(self): - return "simple_seed_005" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds-big'], - 'seeds': { - 'quote_columns': False, - } - } - - def test_big_batched_seed(self): - with open('seeds-big/my_seed.csv', 'w') as f: - writer = csv.writer(f) - writer.writerow(['id']) - for i in range(0, 20000): - writer.writerow([i]) - - results = self.run_dbt(["seed"]) - self.assertEqual(len(results), 1) - - - @use_profile('postgres') - def test_postgres_big_batched_seed(self): - self.test_big_batched_seed() - \ No newline at end of file diff --git a/test/integration/005_simple_seed_tests/seeds-dottedseed/seed.with.dots.csv b/tests/functional/simple_seed/data/seed.with.dots.csv similarity index 100% rename from test/integration/005_simple_seed_tests/seeds-dottedseed/seed.with.dots.csv rename to tests/functional/simple_seed/data/seed.with.dots.csv diff --git a/test/integration/005_simple_seed_tests/seeds-bom/seed_bom.csv b/tests/functional/simple_seed/data/seed_bom.csv similarity index 100% rename from test/integration/005_simple_seed_tests/seeds-bom/seed_bom.csv rename to tests/functional/simple_seed/data/seed_bom.csv diff --git a/test/integration/005_simple_seed_tests/seeds-unicode/seed_unicode.csv b/tests/functional/simple_seed/data/seed_unicode.csv similarity index 100% rename from test/integration/005_simple_seed_tests/seeds-unicode/seed_unicode.csv rename to tests/functional/simple_seed/data/seed_unicode.csv diff --git a/tests/functional/simple_seed/test_seed.py b/tests/functional/simple_seed/test_seed.py index 967bcafa755..41f97686c9f 100644 --- a/tests/functional/simple_seed/test_seed.py +++ b/tests/functional/simple_seed/test_seed.py @@ -1,6 +1,9 @@ +import csv +import os import pytest + from pathlib import Path -import os + from dbt.tests.util import ( run_dbt, read_file, @@ -8,12 +11,30 @@ check_table_does_exist, check_table_does_not_exist, ) -from tests.functional.simple_seed.fixtures import models__downstream_from_seed_actual +from tests.functional.simple_seed.fixtures import ( + models__downstream_from_seed_actual, + models__from_basic_seed, + seeds__disabled_in_config, + seeds__enabled_in_config, + seeds__tricky, + seeds__wont_parse, +) # from `test/integration/test_simple_seed`, test_postgres_simple_seed -class SeedTestBase(object): +class SeedConfigBase(object): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seeds": { + "quote_columns": False, + }, + } + + +class SeedTestBase(SeedConfigBase): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): """Create table for ensuring seeds and models used in tests build correctly""" @@ -23,10 +44,7 @@ def setUp(self, project): def seeds(self): dir_path = os.path.dirname(os.path.realpath(__file__)) seed_actual_csv = read_file(dir_path, "seeds", "seed_actual.csv") - return { - # "model.sql": model, - "seed_actual.csv": seed_actual_csv - } + return {"seed_actual.csv": seed_actual_csv} @pytest.fixture(scope="class") def models(self): @@ -56,16 +74,6 @@ def _check_relation_end_state(self, run_result, project, exists: bool): class TestBasicSeedTests(SeedTestBase): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "seed-paths": ["seeds"], - "seeds": { - "quote_columns": False, - }, - } - def test_postgres_simple_seed(self, project): """Build models and observe that run truncates a seed and re-inserts rows""" self._build_relations_for_test(project) @@ -85,7 +93,6 @@ class TestSeedConfigFullRefreshOn(SeedTestBase): def project_config_update(self): return { "config-version": 2, - "seed-paths": ["seeds"], "seeds": {"quote_columns": False, "full_refresh": True}, } @@ -100,7 +107,6 @@ class TestSeedConfigFullRefreshOff(SeedTestBase): def project_config_update(self): return { "config-version": 2, - "seed-paths": ["seeds"], "seeds": {"quote_columns": False, "full_refresh": False}, } @@ -111,3 +117,174 @@ def test_postgres_simple_seed_full_refresh_config(self, project): self._check_relation_end_state( run_result=run_dbt(["seed", "--full-refresh"]), project=project, exists=True ) + + +class TestSeedCustomSchema(SeedTestBase): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql_file(project.test_data_dir / Path("seed_expected.sql")) + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seeds": { + "schema": "custom_schema", + "quote_columns": False, + }, + } + + def test_postgres_simple_seed_with_schema(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + # TODO: 1/4 custom schema work; but unsure how to provide these lines with custom schema on seed_actual + # check_relations_equal(project.adapter, ["seed_actual", "seed_expected"]) + + # this should truncate the seed_actual table, then re-insert + results = run_dbt(["seed"]) + assert len(results) == 1 + # TODO: 2/4 + # check_relations_equal(project.adapter, ["seed_actual", "seed_expected"]) + + def test_postgres_simple_seed_with_drop_and_schema(self): + results = run_dbt(["seed"]) + assert len(results) == 1 + # TODO: 3/4 + # check_relations_equal(project.adapter, ["seed_actual", "seed_expected"]) + + # this should drop the seed table, then re-create + results = run_dbt(["seed", "--full-refresh"]) + results = run_dbt(["seed"]) + # TODO: 4/4 + # check_relations_equal(project.adapter, ["seed_actual", "seed_expected"]) + + +class TestSimpleSeedEnabledViaConfig(object): + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed_enabled.csv": seeds__enabled_in_config, + "seed_disabled.csv": seeds__disabled_in_config, + "seed_tricky.csv": seeds__tricky, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seeds": { + "test": {"seed_enabled": {"enabled": True}, "seed_disabled": {"enabled": False}}, + "quote_columns": False, + }, + } + + def test_postgres_simple_seed_with_disabled(self, project): + results = run_dbt(["seed"]) + len(results) == 2 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") + # TODO: 1/3 is there a better way to enforce cleanup + project.run_sql(f"drop table {project.test_schema}.seed_enabled") + project.run_sql(f"drop table {project.test_schema}.seed_tricky") + + def test_postgres_simple_seed_selection(self, project): + results = run_dbt(["seed", "--select", "seed_enabled"]) + len(results) == 1 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_not_exist(project.adapter, "seed_tricky") + # TODO: 2/3 + project.run_sql(f"drop table {project.test_schema}.seed_enabled") + + def test_postgres_simple_seed_exclude(self, project): + results = run_dbt(["seed", "--exclude", "seed_enabled"]) + len(results) == 1 + check_table_does_not_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") + # TODO: 3/3 + project.run_sql(f"drop table {project.test_schema}.seed_tricky") + + +class TestSeedParsing(SeedConfigBase): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql_file(project.test_data_dir / Path("seed_expected.sql")) + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": seeds__wont_parse} + + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": models__from_basic_seed} + + def test_postgres_dbt_run_skips_seeds(self, project): + # run does not try to parse the seed files + len(run_dbt()) == 1 + + # make sure 'dbt seed' fails, otherwise our test is invalid! + run_dbt(["seed"], expect_pass=False) + + +class TestSimpleSeedWithBOM(SeedConfigBase): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql_file(project.test_data_dir / Path("seed_expected.sql")) + + @pytest.fixture(scope="class") + def seeds(self): + dir_path = os.path.dirname(os.path.realpath(__file__)) + seed_bom = read_file(dir_path, "data", "seed_bom.csv") + return {"seed_bom.csv": seed_bom} + + def test_postgres_simple_seed(self, project): + # first make sure nobody "fixed" the file by accident + seed_path = project.test_data_dir / Path("seed_bom.csv") + with open(seed_path, encoding="utf-8") as fp: + assert fp.read(1) == "\ufeff" + + results = run_dbt(["seed"]) + len(results) == 1 + check_relations_equal(project.adapter, ["seed_expected", "seed_bom"]) + + +class TestSimpleSeedWithUnicode(SeedConfigBase): + @pytest.fixture(scope="class") + def seeds(self): + dir_path = os.path.dirname(os.path.realpath(__file__)) + seed_unicode = read_file(dir_path, "data", "seed_unicode.csv") + return {"seed_unicode.csv": seed_unicode} + + def test_postgres_simple_seed(self, project): + results = run_dbt(["seed"]) + len(results) == 1 + + +class TestSimpleSeedWithDots(SeedConfigBase): + @pytest.fixture(scope="class") + def seeds(self): + dir_path = os.path.dirname(os.path.realpath(__file__)) + dotted_seed = read_file(dir_path, "data", "seed.with.dots.csv") + return {"seed.with.dots.csv": dotted_seed} + + def test_postgres_simple_seed(self, project): + results = run_dbt(["seed"]) + len(results) == 1 + + +class TestSimpleBigSeedBatched(SeedConfigBase): + def test_postgres_big_batched_seed(self, project): + big_seed = project.test_data_dir / Path("big-seed.csv") + with open(big_seed, "w") as f: + writer = csv.writer(f) + writer.writerow(["id"]) + for i in range(0, 20000): + writer.writerow([i]) + + results = run_dbt(["seed"]) + len(results) == 1