diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 5395247a74c..daacef0f553 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -282,8 +282,8 @@ def adapter( project_root, profiles_root, profiles_yml, - dbt_project_yml, clean_up_logging, + dbt_project_yml, ): # The profiles.yml and dbt_project.yml should already be written out args = Namespace( @@ -385,7 +385,20 @@ def analyses(): # Write out the files provided by models, macros, properties, snapshots, seeds, tests, analyses @pytest.fixture(scope="class") -def project_files(project_root, models, macros, snapshots, properties, seeds, tests, analyses): +def project_files( + project_root, + models, + macros, + snapshots, + properties, + seeds, + tests, + analyses, + selectors_yml, + dependencies_yml, + packages_yml, + dbt_project_yml, +): write_project_files(project_root, "models", {**models, **properties}) write_project_files(project_root, "macros", macros) write_project_files(project_root, "snapshots", snapshots) @@ -515,12 +528,8 @@ def initialization(environment) -> None: enable_test_caching() -# This is the main fixture that is used in all functional tests. It pulls in the other -# fixtures that are necessary to set up a dbt project, and saves some of the information -# in a TestProjInfo class, which it returns, so that individual test cases do not have -# to pull in the other fixtures individually to access their information. @pytest.fixture(scope="class") -def project( +def project_setup( initialization, clean_up_logging, project_root, @@ -528,12 +537,7 @@ def project( request, unique_schema, profiles_yml, - dbt_project_yml, - packages_yml, - dependencies_yml, - selectors_yml, adapter, - project_files, shared_data_dir, test_data_dir, logs_dir, @@ -587,3 +591,16 @@ def project( pass os.chdir(orig_cwd) cleanup_event_logger() + + +# This is the main fixture that is used in all functional tests. It pulls in the other +# fixtures that are necessary to set up a dbt project, and saves some of the information +# in a TestProjInfo class, which it returns, so that individual test cases do not have +# to pull in the other fixtures individually to access their information. +# The order of arguments here determine which steps runs first. +@pytest.fixture(scope="class") +def project( + project_setup: TestProjInfo, + project_files, +): + return project_setup diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index d5c5e49d2aa..cb2fbeccca8 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -90,7 +90,6 @@ def run_dbt( args.extend(["--project-dir", project_dir]) if profiles_dir and "--profiles-dir" not in args: args.extend(["--profiles-dir", profiles_dir]) - dbt = dbtRunner() res = dbt.invoke(args) diff --git a/tests/functional/adapter/basic/test_adapter_methods.py b/tests/functional/adapter/basic/test_adapter_methods.py index 51d987f07de..62f70b4f0f1 100644 --- a/tests/functional/adapter/basic/test_adapter_methods.py +++ b/tests/functional/adapter/basic/test_adapter_methods.py @@ -1,6 +1,5 @@ import pytest -from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import check_relations_equal, run_dbt tests__get_columns_in_relation_sql = """ @@ -73,16 +72,6 @@ def models(self): "model.sql": models__model_sql, } - @pytest.fixture(scope="class") - def project_files( - self, - project_root, - tests, - models, - ): - write_project_files(project_root, "tests", tests) - write_project_files(project_root, "models", models) - @pytest.fixture(scope="class") def project_config_update(self): return { diff --git a/tests/functional/fixtures/__init__.py b/tests/functional/fixtures/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/fixtures/happy_path_fixture.py b/tests/functional/fixtures/happy_path_fixture.py new file mode 100644 index 00000000000..3f365f36413 --- /dev/null +++ b/tests/functional/fixtures/happy_path_fixture.py @@ -0,0 +1,32 @@ +import os +from distutils.dir_util import copy_tree + +import pytest + + +def delete_files_in_directory(directory_path): + try: + with os.scandir(directory_path) as entries: + for entry in entries: + if entry.is_file(): + os.unlink(entry.path) + print("All files deleted successfully.") + except OSError: + print("Error occurred while deleting files.") + + +@pytest.fixture(scope="class") +def happy_path_project_files(project_root): + # copy fixture files to the project root + delete_files_in_directory(project_root) + copy_tree( + os.path.dirname(os.path.realpath(__file__)) + "/happy_path_project", str(project_root) + ) + + +# We do project_setup first because it will write out a dbt_project.yml. +# This file will be overwritten by the files in happy_path_project later on. +@pytest.fixture(scope="class") +def happy_path_project(project_setup, happy_path_project_files): + # A fixture that gives functional test the project living in happy_path_project + return project_setup diff --git a/tests/functional/fixtures/happy_path_project/analyses/a.sql b/tests/functional/fixtures/happy_path_project/analyses/a.sql new file mode 100644 index 00000000000..1f7d87c55a9 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/analyses/a.sql @@ -0,0 +1 @@ +select 4 as id diff --git a/tests/functional/fixtures/happy_path_project/dbt_project.yml b/tests/functional/fixtures/happy_path_project/dbt_project.yml new file mode 100644 index 00000000000..716ab611ef3 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/dbt_project.yml @@ -0,0 +1,17 @@ +analysis-paths: +- analyses +config-version: 2 +flags: + send_anonymous_usage_stats: false +macro-paths: +- macros +name: test +profile: test +seed-paths: +- seeds +seeds: + quote_columns: false +snapshot-paths: +- snapshots +test-paths: +- tests diff --git a/tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql b/tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql new file mode 100644 index 00000000000..2ac8c3b572f --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql @@ -0,0 +1,7 @@ +{% macro cool_macro() %} + wow! +{% endmacro %} + +{% macro other_cool_macro(a, b) %} + cool! +{% endmacro %} diff --git a/tests/functional/fixtures/happy_path_project/models/docs.md b/tests/functional/fixtures/happy_path_project/models/docs.md new file mode 100644 index 00000000000..3f547f37fd9 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/docs.md @@ -0,0 +1,3 @@ +{% docs my_docs %} + some docs +{% enddocs %} diff --git a/tests/functional/fixtures/happy_path_project/models/ephemeral.sql b/tests/functional/fixtures/happy_path_project/models/ephemeral.sql new file mode 100644 index 00000000000..4f9208f28a2 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/ephemeral.sql @@ -0,0 +1,5 @@ +{{ config(materialized='ephemeral') }} + +select + 1 as id, + {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as created_at diff --git a/tests/functional/fixtures/happy_path_project/models/incremental.sql b/tests/functional/fixtures/happy_path_project/models/incremental.sql new file mode 100644 index 00000000000..2bb15542da9 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/incremental.sql @@ -0,0 +1,12 @@ +{{ + config( + materialized = "incremental", + incremental_strategy = "delete+insert", + ) +}} + +select * from {{ ref('seed') }} + +{% if is_incremental() %} + where a > (select max(a) from {{this}}) +{% endif %} diff --git a/tests/functional/fixtures/happy_path_project/models/m.yml b/tests/functional/fixtures/happy_path_project/models/m.yml new file mode 100644 index 00000000000..38e5be96af2 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/m.yml @@ -0,0 +1,7 @@ +metrics: + - name: total_outer + type: simple + description: The total count of outer + label: Total Outer + type_params: + measure: total_outer_count diff --git a/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql b/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql new file mode 100644 index 00000000000..b672fabcad2 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql @@ -0,0 +1,2 @@ +select + {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date_day diff --git a/tests/functional/fixtures/happy_path_project/models/outer.sql b/tests/functional/fixtures/happy_path_project/models/outer.sql new file mode 100644 index 00000000000..85bbabe21e3 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/outer.sql @@ -0,0 +1 @@ +select * from {{ ref('ephemeral') }} diff --git a/tests/functional/fixtures/happy_path_project/models/schema.yml b/tests/functional/fixtures/happy_path_project/models/schema.yml new file mode 100644 index 00000000000..ef6addf5b91 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/schema.yml @@ -0,0 +1,15 @@ +version: 2 +models: + - name: outer + description: The outer table + columns: + - name: id + description: The id value + data_tests: + - unique + - not_null + +sources: + - name: my_source + tables: + - name: my_table diff --git a/tests/functional/fixtures/happy_path_project/models/sm.yml b/tests/functional/fixtures/happy_path_project/models/sm.yml new file mode 100644 index 00000000000..6db38e3ae4c --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/sm.yml @@ -0,0 +1,18 @@ +semantic_models: + - name: my_sm + model: ref('outer') + defaults: + agg_time_dimension: created_at + entities: + - name: my_entity + type: primary + expr: id + dimensions: + - name: created_at + type: time + type_params: + time_granularity: day + measures: + - name: total_outer_count + agg: count + expr: 1 diff --git a/tests/functional/fixtures/happy_path_project/models/sq.yml b/tests/functional/fixtures/happy_path_project/models/sq.yml new file mode 100644 index 00000000000..e28e3805a3f --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/sq.yml @@ -0,0 +1,14 @@ +saved_queries: + - name: my_saved_query + label: My Saved Query + query_params: + metrics: + - total_outer + group_by: + - "Dimension('my_entity__created_at')" + exports: + - name: my_export + config: + alias: my_export_alias + export_as: table + schema: my_export_schema_name diff --git a/tests/functional/fixtures/happy_path_project/models/sub/inner.sql b/tests/functional/fixtures/happy_path_project/models/sub/inner.sql new file mode 100644 index 00000000000..a90004d480d --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/sub/inner.sql @@ -0,0 +1 @@ +select * from {{ ref('outer') }} diff --git a/tests/functional/fixtures/happy_path_project/seeds/seed.csv b/tests/functional/fixtures/happy_path_project/seeds/seed.csv new file mode 100644 index 00000000000..cfa20f81071 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/seeds/seed.csv @@ -0,0 +1,2 @@ +a,b +1,2 diff --git a/tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql b/tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql new file mode 100644 index 00000000000..60d803dfbb4 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql @@ -0,0 +1,12 @@ +{% snapshot my_snapshot %} + {{ + config( + target_database=var('target_database', database), + target_schema=schema, + unique_key='id', + strategy='timestamp', + updated_at='updated_at', + ) + }} + select * from {{database}}.{{schema}}.seed +{% endsnapshot %} diff --git a/tests/functional/fixtures/happy_path_project/tests/t.sql b/tests/functional/fixtures/happy_path_project/tests/t.sql new file mode 100644 index 00000000000..9757b1b7ae9 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/tests/t.sql @@ -0,0 +1 @@ +select 1 as id limit 0 diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index 9b51b6798c9..8d462e258b0 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -1,20 +1,10 @@ import json import os -import pytest - from dbt.tests.util import run_dbt -from tests.functional.list.fixtures import ( # noqa: F401 - analyses, - macros, - metrics, - models, - project_files, - saved_queries, - seeds, - semantic_models, - snapshots, - tests, +from tests.functional.fixtures.happy_path_fixture import ( # noqa: F401 + happy_path_project, + happy_path_project_files, ) @@ -22,21 +12,7 @@ class TestList: def dir(self, value): return os.path.normpath(value) - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, - "analysis-paths": [self.dir("analyses")], - "snapshot-paths": [self.dir("snapshots")], - "macro-paths": [self.dir("macros")], - "seed-paths": [self.dir("seeds")], - "test-paths": [self.dir("tests")], - "seeds": { - "quote_columns": False, - }, - } - - def test_packages_install_path_does_not_exist(self, project): + def test_packages_install_path_does_not_exist(self, happy_path_project): # noqa: F811 run_dbt(["list"]) packages_install_path = "dbt_packages" @@ -47,7 +23,6 @@ def run_dbt_ls(self, args=None, expect_pass=True): full_args = ["ls"] if args is not None: full_args += args - result = run_dbt(args=full_args, expect_pass=expect_pass) return result @@ -67,7 +42,7 @@ def expect_given_output(self, args, expectations): else: assert got == expected - def expect_snapshot_output(self, project): + def expect_snapshot_output(self, happy_path_project): # noqa: F811 expectations = { "name": "my_snapshot", "selector": "test.snapshot.my_snapshot", @@ -86,8 +61,8 @@ def expect_snapshot_output(self, project): "quoting": {}, "column_types": {}, "persist_docs": {}, - "target_database": project.database, - "target_schema": project.test_schema, + "target_database": happy_path_project.database, + "target_schema": happy_path_project.test_schema, "unique_key": "id", "strategy": "timestamp", "updated_at": "updated_at", @@ -735,10 +710,14 @@ def expect_resource_type_env_var(self): } del os.environ["DBT_EXCLUDE_RESOURCE_TYPES"] - def expect_selected_keys(self, project): + def expect_selected_keys(self, happy_path_project): # noqa: F811 """Expect selected fields of the the selected model""" expectations = [ - {"database": project.database, "schema": project.test_schema, "alias": "inner"} + { + "database": happy_path_project.database, + "schema": happy_path_project.test_schema, + "alias": "inner", + } ] results = self.run_dbt_ls( [ @@ -759,7 +738,9 @@ def expect_selected_keys(self, project): """Expect selected fields when --output-keys given multiple times """ - expectations = [{"database": project.database, "schema": project.test_schema}] + expectations = [ + {"database": happy_path_project.database, "schema": happy_path_project.test_schema} + ] results = self.run_dbt_ls( [ "--model", @@ -821,8 +802,8 @@ def expect_selected_keys(self, project): for got, expected in zip(results, expectations): self.assert_json_equal(got, expected) - def test_ls(self, project): - self.expect_snapshot_output(project) + def test_ls(self, happy_path_project): # noqa: F811 + self.expect_snapshot_output(happy_path_project) self.expect_analyses_output() self.expect_model_output() self.expect_source_output() @@ -832,7 +813,7 @@ def test_ls(self, project): self.expect_resource_type_multiple() self.expect_resource_type_env_var() self.expect_all_output() - self.expect_selected_keys(project) + self.expect_selected_keys(happy_path_project) def normalize(path): diff --git a/tests/functional/test_selection/fixtures.py b/tests/functional/test_selection/fixtures.py index 37cce1bbae9..84f4c532e5f 100644 --- a/tests/functional/test_selection/fixtures.py +++ b/tests/functional/test_selection/fixtures.py @@ -1,7 +1,5 @@ import pytest -from dbt.tests.fixtures.project import write_project_files - tests__cf_a_b_sql = """ select * from {{ ref('model_a') }} cross join {{ ref('model_b') }} @@ -84,13 +82,3 @@ def models(): "model_b.sql": models__model_b_sql, "model_a.sql": models__model_a_sql, } - - -@pytest.fixture(scope="class") -def project_files( - project_root, - tests, - models, -): - write_project_files(project_root, "tests", tests) - write_project_files(project_root, "models", models) diff --git a/tests/functional/test_selection/test_selection_expansion.py b/tests/functional/test_selection/test_selection_expansion.py index e42d6196328..d163588303c 100644 --- a/tests/functional/test_selection/test_selection_expansion.py +++ b/tests/functional/test_selection/test_selection_expansion.py @@ -1,11 +1,7 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.test_selection.fixtures import ( # noqa: F401 - models, - project_files, - tests, -) +from tests.functional.test_selection.fixtures import models, tests # noqa: F401 class TestSelectionExpansion: