diff --git a/CHANGELOG.md b/CHANGELOG.md index b48ea496..1f280522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# dbt-dremio 1.X.X - release TBD +# dbt-dremio 1.5.0 - release TBD ## Features @@ -6,9 +6,20 @@ ## Under the Hood +- [#179](https://github.com/dremio/dbt-dremio/issues/179) Upgrade to support dbt-core v1.5.0. + - Add support for Python 3.11. + - Add support for relevant Tests: + - caching + - hooks + - simple_copy +- Add support for model contracts (Stub the feature to let users know the feature is not supported). + ## Dependency - Upgrade sqlparse to 0.4.4 [#180](https://github.com/dremio/dbt-dremio/issues/180). +- Upgrade dbt-core to 1.5.0. +- Upgrade dbt-tests-adapter to 1.5.0. +- Upgrade Requests to 2.31.0. [#183](https://github.com/dremio/dbt-dremio/issues/183). # dbt-dremio 1.4.5 - release March 23, 2023 diff --git a/README.md b/README.md index 95300f72..8688fa1c 100644 --- a/README.md +++ b/README.md @@ -6,20 +6,20 @@ dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. -## dbt-dremio version 1.4.5 +## dbt-dremio version 1.5.0 The `dbt-dremio` package contains all of the code enabling dbt to work with [Dremio](https://www.dremio.com/). For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile). The dbt-dremio package supports both Dremio Cloud and Dremio Software (versions 22.0 and later). -Version 1.4.5 of the dbt-dremio adapter is compatible with dbt-core versions 1.2.0 to 1.4.5. +Version 1.5.0 of the dbt-dremio adapter is compatible with dbt-core versions 1.2.0 to 1.5.0. > Prior to version 1.1.0b, dbt-dremio was created and maintained by [Fabrice Etanchaud](https://github.com/fabrice-etanchaud) on [their GitHub repo](https://github.com/fabrice-etanchaud/dbt-dremio). Code for using Dremio REST APIs was originally authored by [Ryan Murray](https://github.com/rymurr). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 1.0.6.5. The dbt-dremio adapter is maintained and distributed by Dremio starting with version 1.1.0b. ## Getting started - [Install dbt-dremio](https://docs.getdbt.com/reference/warehouse-setups/dremio-setup) - - Version 1.4.5 of dbt-dremio requires dbt-core >= 1.2.0 and <= 1.4.5. Installing dbt-dremio will automatically upgrade existing dbt-core versions earlier than 1.2.0 to 1.4.5, or install dbt-core v1.4.5 if no version of dbt-core is found. + - Version 1.5.0 of dbt-dremio requires dbt-core >= 1.2.0 and <= 1.5.0. Installing dbt-dremio will automatically upgrade existing dbt-core versions earlier than 1.2.0 to 1.5.0, or install dbt-core v1.5.0 if no version of dbt-core is found. - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) ## Join the dbt Community diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index f793d11a..6c07909c 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -13,10 +13,10 @@ charset-normalizer 3.1.0 MIT License click 8.1.3 BSD License colorama 0.4.6 BSD License - dbt-core 1.4.5 Apache Software License - dbt-dremio 1.4.5 Apache Software License + dbt-core 1.5.0 Apache Software License + dbt-dremio 1.5.0 Apache Software License dbt-extractor 0.4.1 Apache Software License - dbt-tests-adapter 1.4.5 Apache Software License + dbt-tests-adapter 1.5.0 Apache Software License exceptiongroup 1.1.1 MIT License future 0.18.3 MIT License grpclib 0.4.3 BSD License @@ -45,7 +45,7 @@ python-slugify 8.0.1 MIT License pytimeparse 1.1.8 MIT License pytz 2022.7.1 MIT License - requests 2.28.2 Apache Software License + requests 2.31.0 Apache Software License six 1.16.0 MIT License sqlparse 0.4.4 BSD License stringcase 1.2.0 MIT diff --git a/dbt/adapters/dremio/__version__.py b/dbt/adapters/dremio/__version__.py index 4424d68f..6caab8c8 100644 --- a/dbt/adapters/dremio/__version__.py +++ b/dbt/adapters/dremio/__version__.py @@ -9,4 +9,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -version = "1.4.5" +version = "1.5.0" diff --git a/dbt/adapters/dremio/credentials.py b/dbt/adapters/dremio/credentials.py index ee0d3498..b851ad39 100644 --- a/dbt/adapters/dremio/credentials.py +++ b/dbt/adapters/dremio/credentials.py @@ -53,6 +53,14 @@ class DremioCredentials(Credentials): def type(self): return "dremio" + @property + def unique_field(self): + """ + Hashed and included in anonymous telemetry to track adapter adoption. + Pick a field that can uniquely identify one team/organization building with this adapter + """ + return self.software_host if self.cloud_host is None else self.cloud_host + @property def aliases(self): return self._ALIASES diff --git a/dbt/include/dremio/macros/materializations/table/create_table_as.sql b/dbt/include/dremio/macros/materializations/table/create_table_as.sql index 381c7d7f..1aa8e909 100644 --- a/dbt/include/dremio/macros/materializations/table/create_table_as.sql +++ b/dbt/include/dremio/macros/materializations/table/create_table_as.sql @@ -26,6 +26,11 @@ limitations under the License.*/ #} {% macro dremio__create_table_as(temporary, relation, sql) -%} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{exceptions.warn("Model contracts are not enforced by dbt-dremio!")}} + {% endif %} + {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} diff --git a/dbt/include/dremio/macros/materializations/view/create_view_as.sql b/dbt/include/dremio/macros/materializations/view/create_view_as.sql index 4095b7e9..0dc162e4 100644 --- a/dbt/include/dremio/macros/materializations/view/create_view_as.sql +++ b/dbt/include/dremio/macros/materializations/view/create_view_as.sql @@ -13,6 +13,11 @@ See the License for the specific language governing permissions and limitations under the License.*/ {% macro dremio__create_view_as(relation, sql) -%} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{exceptions.warn("Model contracts are not enforced by dbt-dremio!")}} + {% endif %} + {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} diff --git a/dev_requirements.txt b/dev_requirements.txt index c22dacdc..5c84a6e6 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,39 +1,22 @@ -agate==1.7.0 attrs==22.2.0 Babel==2.12.1 betterproto==1.2.5 certifi==2022.12.7 -cffi==1.15.1 charset-normalizer==3.1.0 -click==8.1.3 -colorama==0.4.6 -dbt-core==1.4.5 -dbt-dremio==1.4.5 -dbt-extractor==0.4.1 -dbt-tests-adapter==1.4.5 +dbt-tests-adapter==1.5.0 exceptiongroup==1.1.1 future==0.18.3 grpclib==0.4.3 h2==4.1.0 -hologram==0.0.15 hpack==4.0.0 hyperframe==6.0.1 -idna==3.4 iniconfig==2.0.0 -isodate==0.6.1 -Jinja2==3.1.2 jsonschema==4.17.3 leather==0.3.4 -Logbook==1.5.3 MarkupSafe==2.1.2 -mashumaro==3.5 -minimal-snowplow-tracker==0.0.2 msgpack==1.0.5 multidict==6.0.4 -networkx==3.0 -packaging==23.0 -parsedatetime==2.6 -pathspec==0.11.1 +parsedatetime==2.4 pip-licenses==4.1.0 pluggy==1.0.0 prettytable==3.6.0 @@ -43,15 +26,9 @@ pytest==7.2.2 python-dateutil==2.8.2 python-slugify==8.0.1 pytimeparse==1.1.8 -pytz==2022.7.1 -PyYAML==6.0 -requests==2.28.2 six==1.16.0 -sqlparse==0.4.4 stringcase==1.2.0 text-unidecode==1.3 tomli==2.0.1 -typing_extensions==4.5.0 urllib3==1.26.15 wcwidth==0.2.6 -Werkzeug==2.2.3 diff --git a/setup.py b/setup.py index 33fb49b8..65ebdf04 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ from setuptools import find_namespace_packages, setup package_name = "dbt-dremio" -package_version = "1.4.5" +package_version = "1.5.0" description = """The Dremio adapter plugin for dbt""" setup( @@ -27,13 +27,14 @@ packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core>=1.2, <=1.4.5", + "dbt-core>=1.2, <1.6.0", + "requests>=2.31.0", ], classifiers=[ "License :: OSI Approved :: Apache Software License", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], ) diff --git a/tests/caching/test_caching.py b/tests/caching/test_caching.py new file mode 100644 index 00000000..07907963 --- /dev/null +++ b/tests/caching/test_caching.py @@ -0,0 +1,60 @@ +import pytest +from tests.utils.util import BUCKET +from dbt.tests.adapter.caching.test_caching import ( + TestNoPopulateCache, + BaseCachingSelectedSchemaOnly, + BaseCachingLowercaseModel, + BaseCachingUppercaseModel, + model_sql, +) + + +# This ensures the schema works with our datalake +@pytest.fixture(scope="class") +def unique_schema(request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{BUCKET}.{prefix}_{test_file}" + return unique_schema + + +# Override this fixture to set root_path=schema +@pytest.fixture(scope="class") +def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = f"{unique_schema}" + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + + +class TestNoPopulateCacheDremio(TestNoPopulateCache): + pass + + +class TestCachingLowerCaseModelDremio(BaseCachingLowercaseModel): + pass + + +@pytest.mark.skip( + reason="Dremio does not support case-sensitive data file names or table names." +) +class TestCachingUppercaseModelDremio(BaseCachingUppercaseModel): + pass + + +class TestCachingSelectedSchemaOnlyDremio(BaseCachingSelectedSchemaOnly): + pass diff --git a/tests/component/test_profile_template.py b/tests/component/test_profile_template.py index a8b472a3..ff1c7b2f 100644 --- a/tests/component/test_profile_template.py +++ b/tests/component/test_profile_template.py @@ -60,7 +60,6 @@ class TestProfileTemplate: "dremio_space": "@user", "dremio_space_folder": "no_schema", "threads": 1, - "verify_ssl": True, } _DREMIO_CLOUD_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS = { "cloud_host": "api.dremio.cloud", @@ -137,7 +136,6 @@ def _get_dbt_test_project_dict(self, dbt_test_project_name: str) -> Dict[str, an def _test_project_profile_options( self, test_project: Dict[str, any], test_options: Dict[str, any] ) -> None: - assert test_project is not None for option in test_options: diff --git a/tests/hooks/data/seed_model.sql b/tests/hooks/data/seed_model.sql new file mode 100644 index 00000000..a4a5e15a --- /dev/null +++ b/tests/hooks/data/seed_model.sql @@ -0,0 +1,16 @@ + +drop table if exists dbt_test_source.{schema}.on_model_hook; + +create table dbt_test_source.{schema}.on_model_hook ( + test_state VARCHAR, -- start|end + target_dbname VARCHAR, + target_host VARCHAR, + target_name VARCHAR, + target_schema VARCHAR, + target_type VARCHAR, + target_user VARCHAR, + target_pass VARCHAR, + target_threads INT, + run_started_at VARCHAR, + invocation_id VARCHAR +); diff --git a/tests/hooks/data/seed_run.sql b/tests/hooks/data/seed_run.sql new file mode 100644 index 00000000..3be08c89 --- /dev/null +++ b/tests/hooks/data/seed_run.sql @@ -0,0 +1,16 @@ + +drop table if exists dbt_test_source.{schema}.on_run_hook; + +create table dbt_test_source.{schema}.on_run_hook ( + test_state VARCHAR, -- start|end + target_dbname VARCHAR, + target_host VARCHAR, + target_name VARCHAR, + target_schema VARCHAR, + target_type VARCHAR, + target_user VARCHAR, + target_pass VARCHAR, + target_threads INT, + run_started_at VARCHAR, + invocation_id VARCHAR +); diff --git a/tests/hooks/fixtures.py b/tests/hooks/fixtures.py new file mode 100644 index 00000000..a0573c80 --- /dev/null +++ b/tests/hooks/fixtures.py @@ -0,0 +1,402 @@ +from tests.utils.util import SOURCE + +MODEL_PRE_HOOK = f""" + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id + ) VALUES ( + 'start', + '{{{{ target.dbname }}}}', + '{{{{ target.host }}}}', + '{{{{ target.name }}}}', + '{SOURCE}.{{{{ target.schema }}}}', + '{{{{ target.type }}}}', + '{{{{ target.user }}}}', + '{{{{ target.get("pass", "") }}}}', + {{{{ target.threads }}}}, + '{{{{ run_started_at }}}}', + '{{{{ invocation_id }}}}' + ) +""" + +MODEL_POST_HOOK = f""" + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id + ) VALUES ( + 'end', + '{{{{ target.dbname }}}}', + '{{{{ target.host }}}}', + '{{{{ target.name }}}}', + '{SOURCE}.{{{{ target.schema }}}}', + '{{{{ target.type }}}}', + '{{{{ target.user }}}}', + '{{{{ target.get("pass", "") }}}}', + {{{{ target.threads }}}}, + '{{{{ run_started_at }}}}', + '{{{{ invocation_id }}}}' + ) +""" + +macros_missing_column = """ +{% macro export_table_check() %} + + {% set table = 'test_column' %} + + {% set query %} + SELECT column_name + FROM {{ref(table)}} + LIMIT 1 + {% endset %} + + {%- if flags.WHICH in ('run', 'build') -%} + {% set results = run_query(query) %} + {% if execute %} + {%- if results.rows -%} + {{ exceptions.raise_compiler_error("ON_RUN_START_CHECK_NOT_PASSED: Data already exported. DBT Run aborted.") }} + {% else -%} + {{ log("No data found in " ~ table ~ " for current day and runtime region. Proceeding...", true) }} + {%- endif -%} + {%- endif -%} + {%- endif -%} +{% endmacro %} +""" + +models__missing_column = """ +select 1 as col +""" + +macros__before_and_after = f""" +{{% macro custom_run_hook(state, target, run_started_at, invocation_id) %}} + + insert into {SOURCE}.{{{{ target.schema }}}}.on_run_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id + ) VALUES ( + '{{{{ state }}}}', + '{{{{ target.dbname }}}}', + '{{{{ target.host }}}}', + '{{{{ target.name }}}}', + '{SOURCE}.{{{{ target.schema }}}}', + '{{{{ target.type }}}}', + '{{{{ target.user }}}}', + '{{{{ target.get("pass", "") }}}}', + {{{{ target.threads }}}}, + '{{{{ run_started_at }}}}', + '{{{{ invocation_id }}}}' + ) + +{{% endmacro %}} +""" + +macros__hook = """ +{% macro hook() %} + select 1 +{% endmacro %} +""" + +models__hooks = """ +select 1 as id +""" + +models__hooks_configured = f""" +{{{{ + config({{ + "pre_hook": "\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook (\ + test_state,\ + target_dbname,\ + target_host,\ + target_name,\ + target_schema,\ + target_type,\ + target_user,\ + target_pass,\ + target_threads,\ + run_started_at,\ + invocation_id + ) VALUES (\ + 'start',\ + '{{{{ target.dbname }}}}',\ + '{{{{ target.host }}}}',\ + '{{{{ target.name }}}}',\ + '{SOURCE}.{{{{ target.schema }}}}',\ + '{{{{ target.type }}}}',\ + '{{{{ target.user }}}}',\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}',\ + {{{{ target.threads }}}},\ + '{{{{ run_started_at }}}}',\ + '{{{{ invocation_id }}}}'\ + )", + "post-hook": "\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook (\ + test_state,\ + target_dbname,\ + target_host,\ + target_name,\ + target_schema,\ + target_type,\ + target_user,\ + target_pass,\ + target_threads,\ + run_started_at,\ + invocation_id + ) VALUES (\ + 'end',\ + '{{{{ target.dbname }}}}',\ + '{{{{ target.host }}}}',\ + '{{{{ target.name }}}}',\ + '{SOURCE}.{{{{ target.schema }}}}',\ + '{{{{ target.type }}}}',\ + '{{{{ target.user }}}}',\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}',\ + {{{{ target.threads }}}},\ + '{{{{ run_started_at }}}}',\ + '{{{{ invocation_id }}}}'\ + )" + }}) +}}}} + +select 3 as id +""" + +models__hooks_error = f""" +{{{{ + config({{ + "pre_hook": "\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook (\ + test_state,\ + target_dbname,\ + target_host,\ + target_name,\ + target_schema,\ + target_type,\ + target_user,\ + target_pass,\ + target_threads,\ + run_started_at,\ + invocation_id + ) VALUES (\ + 'start',\ + '{{{{ target.dbname }}}}',\ + '{{{{ target.host }}}}',\ + '{{{{ target.name }}}}',\ + '{SOURCE}.{{{{ target.schema }}}}',\ + '{{{{ target.type }}}}',\ + '{{{{ target.user }}}}',\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}',\ + {{{{ target.threads }}}},\ + '{{{{ run_started_at }}}}',\ + '{{{{ invocation_id }}}}' + )", + "pre-hook": "\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook (\ + test_state,\ + target_dbname,\ + target_host,\ + target_name,\ + target_schema,\ + target_type,\ + target_user,\ + target_pass,\ + target_threads,\ + run_started_at,\ + invocation_id + ) VALUES (\ + 'start',\ + '{{{{ target.dbname }}}}',\ + '{{{{ target.host }}}}',\ + '{{{{ target.name }}}}',\ + '{SOURCE}.{{{{ target.schema }}}}',\ + '{{{{ target.type }}}}',\ + '{{{{ target.user }}}}',\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}',\ + {{{{ target.threads }}}},\ + '{{{{ run_started_at }}}}',\ + '{{{{ invocation_id }}}}' + )", + "post-hook": "\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook (\ + test_state,\ + target_dbname,\ + target_host,\ + target_name,\ + target_schema,\ + target_type,\ + target_user,\ + target_pass,\ + target_threads,\ + run_started_at,\ + invocation_id + ) VALUES (\ + 'end',\ + '{{{{ target.dbname }}}}',\ + '{{{{ target.host }}}}',\ + '{{{{ target.name }}}}',\ + '{SOURCE}.{{{{ target.schema }}}}',\ + '{{{{ target.type }}}}',\ + '{{{{ target.user }}}}',\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}',\ + {{{{ target.threads }}}},\ + '{{{{ run_started_at }}}}',\ + '{{{{ invocation_id }}}}'\ + )" + }}) +}}}} + +select 3 as id +""" + +models__hooks_kwargs = f""" +{{{{ + config( + pre_hook="\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook (\ + test_state,\ + target_dbname,\ + target_host,\ + target_name,\ + target_schema,\ + target_type,\ + target_user,\ + target_pass,\ + target_threads,\ + run_started_at,\ + invocation_id + ) VALUES (\ + 'start',\ + '{{{{ target.dbname }}}}',\ + '{{{{ target.host }}}}',\ + '{{{{ target.name }}}}',\ + '{SOURCE}.{{{{ target.schema }}}}',\ + '{{{{ target.type }}}}',\ + '{{{{ target.user }}}}',\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}',\ + {{{{ target.threads }}}},\ + '{{{{ run_started_at }}}}',\ + '{{{{ invocation_id }}}}'\ + )", + post_hook="\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook (\ + test_state,\ + target_dbname,\ + target_host,\ + target_name,\ + target_schema,\ + target_type,\ + target_user,\ + target_pass,\ + target_threads,\ + run_started_at,\ + invocation_id\ + ) VALUES (\ + 'end',\ + '{{{{ target.dbname }}}}',\ + '{{{{ target.host }}}}',\ + '{{{{ target.name }}}}',\ + '{SOURCE}.{{{{ target.schema }}}}',\ + '{{{{ target.type }}}}',\ + '{{{{ target.user }}}}',\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}',\ + {{{{ target.threads }}}},\ + '{{{{ run_started_at }}}}',\ + '{{{{ invocation_id }}}}'\ + )" + ) +}}}} + +select 3 as id +""" + +models__hooked = f""" +{{{{ + config({{ + "pre_hook": "\ + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook select + test_state, + '{{{{ target.dbname }}}}' as target_dbname,\ + '{{{{ target.host }}}}' as target_host,\ + '{{{{ target.name }}}}' as target_name,\ + '{SOURCE}.{{{{ target.schema }}}}' as target_schema,\ + '{{{{ target.type }}}}' as target_type,\ + '{{{{ target.user }}}}' as target_user,\ + '{{{{ target.get(\\"pass\\", \\"\\") }}}}' as target_pass,\ + {{{{ target.threads }}}} as target_threads,\ + '{{{{ run_started_at }}}}' as run_started_at,\ + '{{{{ invocation_id }}}}' as invocation_id + from {{{{ ref('pre') }}}}\ + " + }}) +}}}} +select 1 as id +""" + +models__post = """ +select 'end' as test_state +""" + +models__pre = """ +select 'start' as test_state +""" + +snapshots__test_snapshot = f""" +{{% snapshot example_snapshot %}} +{{{{ + config(target_schema=schema, unique_key='a', strategy='check', check_cols='all') +}}}} +select * from {{{{ ref('example_seed') }}}} +{{% endsnapshot %}} +""" + +properties__seed_models = """ +version: 2 +seeds: +- name: example_seed + columns: + - name: new_col + tests: + - not_null +""" + +properties__test_snapshot_models = """ +version: 2 +snapshots: +- name: example_snapshot + columns: + - name: new_col + tests: + - not_null +""" + +seeds__example_seed_csv = """a,b,c +1,2,3 +4,5,6 +7,8,9 +""" diff --git a/tests/hooks/test_model_hooks.py b/tests/hooks/test_model_hooks.py new file mode 100644 index 00000000..ea897d7f --- /dev/null +++ b/tests/hooks/test_model_hooks.py @@ -0,0 +1,456 @@ +import os +import pytest + +from pathlib import Path + +from dbt.exceptions import CompilationError, ParsingError + +from dbt.tests.util import ( + run_dbt, + write_file, +) + +from dbt.tests.adapter.hooks.fixtures import ( + properties__seed_models, + properties__test_snapshot_models, + seeds__example_seed_csv, +) + +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestHooksRefsOnSeeds, + TestPrePostModelHooksOnSeeds, + TestPrePostModelHooksOnSnapshots, + TestDuplicateHooksInConfigs, +) + +from tests.hooks.fixtures import ( + MODEL_PRE_HOOK, + MODEL_POST_HOOK, + models__hooked, + models__hooks, + models__hooks_configured, + models__hooks_error, + models__hooks_kwargs, + models__post, + models__pre, + snapshots__test_snapshot, +) + +from tests.utils.util import BUCKET, SOURCE + + +# This ensures the schema works with our datalake +@pytest.fixture(scope="class") +def unique_schema(request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{BUCKET}.{test_file}" + return unique_schema + + +# Override this fixture to set root_path=schema +@pytest.fixture(scope="class") +def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = f"{unique_schema}" + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + + +class BaseTestPrePost(object): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql_file(project.test_data_dir / Path("seed_model.sql")) + + def get_ctx_vars(self, state, count, project): + fields = [ + "test_state", + "target_dbname", + "target_host", + "target_name", + "target_schema", + "target_threads", + "target_type", + "target_user", + "target_pass", + "run_started_at", + "invocation_id", + ] + field_list = ", ".join(['"{}"'.format(f) for f in fields]) + query = f"select {field_list} from {SOURCE}.{project.test_schema}.on_model_hook where test_state = '{state}'" + + vals = project.run_sql(query, fetch="all") + assert len(vals) != 0, "nothing inserted into hooks table" + assert len(vals) >= count, "too few rows in hooks table" + assert len(vals) <= count, "too many rows in hooks table" + return [{k: v for k, v in zip(fields, val)} for val in vals] + + def check_hooks(self, state, project, host, count=1): + ctxs = self.get_ctx_vars(state, count=count, project=project) + for ctx in ctxs: + assert ctx["test_state"] == state + assert ctx["target_dbname"] is None + assert ctx["target_host"] is None + assert ctx["target_name"] == "default" + assert ctx["target_schema"] == f"{SOURCE}.{project.test_schema}" + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "dremio" + assert ctx["target_user"] == os.getenv("DREMIO_CLOUD_USERNAME") + assert ctx["target_pass"] is None + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + + +class TestPrePostModelHooksDremio(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [ + # inside transaction (runs second) + MODEL_PRE_HOOK, + # outside transaction (runs first) + { + "sql": "vacuum {{ this.schema }}.on_model_hook", + "transaction": False, + }, + ], + "post-hook": [ + # outside transaction (runs second) + { + "sql": "vacuum {{ this.schema }}.on_model_hook", + "transaction": False, + }, + # inside transaction (runs first) + MODEL_POST_HOOK, + ], + } + } + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks} + + def test_pre_and_post_run_hooks(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksUnderscoresDremio(TestPrePostModelHooksDremio): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre_hook": [ + # inside transaction (runs second) + MODEL_PRE_HOOK, + # outside transaction (runs first) + { + "sql": "vacuum {{ this.schema }}.on_model_hook", + "transaction": False, + }, + ], + "post_hook": [ + # outside transaction (runs second) + { + "sql": "vacuum {{ this.schema }}.on_model_hook", + "transaction": False, + }, + # inside transaction (runs first) + MODEL_POST_HOOK, + ], + } + } + } + + +class TestHookRefsDremio(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "hooked": { + "post-hook": [ + f""" + insert into {SOURCE}.{{{{this.schema}}}}.on_model_hook select + test_state, + '{{{{ target.dbname }}}}' as target_dbname, + '{{{{ target.host }}}}' as target_host, + '{{{{ target.name }}}}' as target_name, + '{SOURCE}.{{{{ target.schema }}}}' as target_schema, + '{{{{ target.type }}}}' as target_type, + '{{{{ target.user }}}}' as target_user, + '{{{{ target.get(pass, "") }}}}' as target_pass, + {{{{ target.threads }}}} as target_threads, + '{{{{ run_started_at }}}}' as run_started_at, + '{{{{ invocation_id }}}}' as invocation_id + from {{{{ ref('post') }}}}""".strip() + ], + } + }, + } + } + + @pytest.fixture(scope="class") + def models(self): + return { + "hooked.sql": models__hooked, + "post.sql": models__post, + "pre.sql": models__pre, + } + + def test_pre_post_model_hooks_refed(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksOnSeedsDremio(TestPrePostModelHooksOnSeeds): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "post-hook": [ + "alter table {{ this }} add columns (new_col int)", + "update {{ this }} set new_col = 1", + # call any macro to track dependency: https://github.com/dbt-labs/dbt-core/issues/6806 + "select CAST(1 AS {{ dbt.type_int() }}) as id", + ], + "quote_columns": False, + }, + } + + +class TestHooksRefsOnSeedsDremio(TestHooksRefsOnSeeds): + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": properties__seed_models, "post.sql": models__post} + + +class TestPrePostModelHooksOnSeedsPlusPrefixedDremio( + TestPrePostModelHooksOnSeedsDremio +): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add columns (new_col int)", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSeedsPlusPrefixedWhitespaceDremio( + TestPrePostModelHooksOnSeedsDremio +): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add columns (new_col int)", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSnapshotsDremio(TestPrePostModelHooksOnSnapshots): + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{BUCKET}.{prefix}_{test_file}" + return unique_schema + + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + target["database"] = target["datalake"] + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + path = Path(project.project_root) / "test-snapshots" + Path.mkdir(path) + write_file(snapshots__test_snapshot, path, "snapshot.sql") + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add columns (new_col int)", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": {"quote_columns": False, "+twin_strategy": "prevent"}, + } + + def test_hooks_on_snapshots(self, project): + res = run_dbt(["seed"]) + assert len(res) == 1, "Expected exactly one item" + res = run_dbt(["snapshot"]) + assert len(res) == 1, "Expected exactly one item" + res = run_dbt(["test"]) + assert len(res) == 1, "Expected exactly one item" + + +class PrePostModelHooksInConfigSetup(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "macro-paths": ["macros"], + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks_configured} + + +class TestPrePostModelHooksInConfigDremio(PrePostModelHooksInConfigSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"seeds": {"+twin_strategy": "prevent"}} + + def test_pre_and_post_model_hooks_model(self, project, dbt_profile_target): + run_dbt() + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksInConfigWithCountDremio(PrePostModelHooksInConfigSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [ + # inside transaction (runs second) + MODEL_PRE_HOOK, + # outside transaction (runs first) + { + "sql": "vacuum {{ this.schema }}.on_model_hook", + "transaction": False, + }, + ], + "post-hook": [ + # outside transaction (runs second) + { + "sql": "vacuum {{ this.schema }}.on_model_hook", + "transaction": False, + }, + # inside transaction (runs first) + MODEL_POST_HOOK, + ], + } + }, + "seeds": {"+twin_strategy": "prevent"}, + } + + def test_pre_and_post_model_hooks_model_and_project( + self, project, dbt_profile_target + ): + run_dbt() + + self.check_hooks( + "start", project, dbt_profile_target.get("host", None), count=2 + ) + self.check_hooks("end", project, dbt_profile_target.get("host", None), count=2) + + +class TestPrePostModelHooksInConfigKwargsDremio(TestPrePostModelHooksInConfigDremio): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"seeds": {"+twin_strategy": "prevent"}} + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks_kwargs} + + +class TestPrePostSnapshotHooksInConfigKwargsDremio( + TestPrePostModelHooksOnSnapshotsDremio +): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + path = Path(project.project_root) / "test-kwargs-snapshots" + Path.mkdir(path) + write_file(snapshots__test_snapshot, path, "snapshot.sql") + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-kwargs-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add columns (new_col int)", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": {"quote_columns": False, "+twin_strategy": "prevent"}, + } + + +class TestDuplicateHooksInConfigsDremio(TestDuplicateHooksInConfigs): + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks_error} diff --git a/tests/hooks/test_run_hooks.py b/tests/hooks/test_run_hooks.py new file mode 100644 index 00000000..7233ba22 --- /dev/null +++ b/tests/hooks/test_run_hooks.py @@ -0,0 +1,196 @@ +import os +import pytest + +from pathlib import Path + +from tests.hooks.fixtures import ( + macros__hook, + macros__before_and_after, + models__hooks, + seeds__example_seed_csv, + macros_missing_column, + models__missing_column, +) + +from dbt.tests.util import ( + check_table_does_not_exist, + run_dbt, +) + +from dbt.tests.adapter.hooks.test_run_hooks import TestAfterRunHooks + +from tests.utils.util import BUCKET, SOURCE + + +# This ensures the schema works with our datalake +@pytest.fixture(scope="class") +def unique_schema(request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{BUCKET}.{prefix}_{test_file}" + return unique_schema + + +# Override this fixture to set root_path=schema +@pytest.fixture(scope="class") +def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = f"{unique_schema}" + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + + +class TestPrePostRunHooksDremio(object): + @pytest.fixture(scope="function") + def setUp(self, project): + project.run_sql_file(project.test_data_dir / Path("seed_run.sql")) + project.run_sql( + f"drop table if exists {SOURCE}.{ project.test_schema }.run_hooks_schemas" + ) + project.run_sql( + f"drop table if exists {SOURCE}.{ project.test_schema }.db_schemas" + ) + os.environ["TERM_TEST"] = "TESTING" + + @pytest.fixture(scope="class") + def macros(self): + return { + "hook.sql": macros__hook, + "before-and-after.sql": macros__before_and_after, + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks} + + @pytest.fixture(scope="class") + def seeds(self): + return {"example_seed.csv": seeds__example_seed_csv} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + # The create and drop table statements here validate that these hooks run + # in the same order that they are defined. Drop before create is an error. + # Also check that the table does not exist below. + "on-run-start": [ + "{{ custom_run_hook('start', target, run_started_at, invocation_id) }}", + f"create table {SOURCE}.{{{{ target.schema }}}}.start_hook_order_test ( id int )", + f"drop table {SOURCE}.{{{{ target.schema }}}}.start_hook_order_test", + "{{ log(env_var('TERM_TEST'), info=True) }}", + ], + "on-run-end": [ + "{{ custom_run_hook('end', target, run_started_at, invocation_id) }}", + f"create table {SOURCE}.{{{{ target.schema }}}}.end_hook_order_test ( id int )", + f"drop table {SOURCE}.{{{{ target.schema }}}}.end_hook_order_test", + f"create table {SOURCE}.{{{{ target.schema }}}}.run_hooks_schemas ( schema VARCHAR )", + f"insert into {SOURCE}.{{{{ target.schema }}}}.run_hooks_schemas (schema) values {{% for schema in schemas %}}( '{{{{ schema }}}}' ){{% if not loop.last %}},{{% endif %}}{{% endfor %}}", + f"create table {SOURCE}.{{{{ target.schema }}}}.db_schemas ( db VARCHAR, schema VARCHAR )", + f"insert into {SOURCE}.{{{{ target.schema }}}}.db_schemas (db, schema) values {{% for db, schema in database_schemas %}}('{{{{ db }}}}', '{{{{ schema }}}}' ){{% if not loop.last %}},{{% endif %}}{{% endfor %}}", + ], + "seeds": { + "quote_columns": False, + }, + } + + def get_ctx_vars(self, state, project): + fields = [ + "test_state", + "target_dbname", + "target_host", + "target_name", + "target_schema", + "target_threads", + "target_type", + "target_user", + "target_pass", + "run_started_at", + "invocation_id", + ] + field_list = ", ".join(['"{}"'.format(f) for f in fields]) + query = f"select {field_list} from {SOURCE}.{project.test_schema}.on_run_hook where test_state = '{state}'" + + vals = project.run_sql(query, fetch="all") + assert len(vals) != 0, "nothing inserted into on_run_hook table" + assert len(vals) == 1, "too many rows in hooks table" + ctx = dict([(k, v) for (k, v) in zip(fields, vals[0])]) + + return ctx + + def assert_used_schemas(self, project): + schemas_query = ( + f"select * from {SOURCE}.{project.test_schema}.run_hooks_schemas" + ) + results = project.run_sql(schemas_query, fetch="all") + assert len(results) == 1 + assert results[0][0] == project.test_schema + + db_schemas_query = f"select * from {SOURCE}.{project.test_schema}.db_schemas" + results = project.run_sql(db_schemas_query, fetch="all") + assert len(results) == 1 + assert results[0][0] == project.database or results[0][0] == SOURCE + assert results[0][1] == project.test_schema + + def check_hooks(self, state, project, host): + ctx = self.get_ctx_vars(state, project) + + assert ctx["test_state"] == state + assert ctx["target_dbname"] is None + assert ctx["target_host"] is None + assert ctx["target_name"] == "default" + assert ctx["target_schema"] == f"{SOURCE}.{project.test_schema}" + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "dremio" + assert ctx["target_user"] == os.getenv("DREMIO_CLOUD_USERNAME") + assert ctx["target_pass"] is None + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + + def test_pre_and_post_run_hooks(self, setUp, project, dbt_profile_target): + run_dbt(["run"]) + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + check_table_does_not_exist(project.adapter, "start_hook_order_test") + check_table_does_not_exist(project.adapter, "end_hook_order_test") + self.assert_used_schemas(project) + + def test_pre_and_post_seed_hooks(self, setUp, project, dbt_profile_target): + run_dbt(["seed"]) + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + check_table_does_not_exist(project.adapter, "start_hook_order_test") + check_table_does_not_exist(project.adapter, "end_hook_order_test") + self.assert_used_schemas(project) + + +class TestAfterRunHooksDremio(TestAfterRunHooks): + @pytest.fixture(scope="class") + def macros(self): + return {"temp_macro.sql": macros_missing_column} + + @pytest.fixture(scope="class") + def models(self): + return {"test_column.sql": models__missing_column} diff --git a/tests/simple_copy/test_copy_uppercase.py b/tests/simple_copy/test_copy_uppercase.py new file mode 100644 index 00000000..10953aa5 --- /dev/null +++ b/tests/simple_copy/test_copy_uppercase.py @@ -0,0 +1,58 @@ +import pytest +from dbt.tests.util import run_dbt, check_relations_equal + +from dbt.tests.adapter.simple_copy.fixtures import ( + _PROPERTIES__SCHEMA_YML, + _SEEDS__SEED_INITIAL, + _MODELS__ADVANCED_INCREMENTAL, + _MODELS__COMPOUND_SORT, + _MODELS__DISABLED, + _MODELS__EMPTY, + _MODELS_GET_AND_REF_UPPERCASE, + _MODELS__INCREMENTAL, + _MODELS__INTERLEAVED_SORT, + _MODELS__MATERIALIZED, + _MODELS__VIEW_MODEL, +) + +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestSimpleCopyUppercaseDremio: + @pytest.fixture(scope="class") + def models(self): + return { + "ADVANCED_INCREMENTAL.sql": _MODELS__ADVANCED_INCREMENTAL, + "COMPOUND_SORT.sql": _MODELS__COMPOUND_SORT, + "DISABLED.sql": _MODELS__DISABLED, + "EMPTY.sql": _MODELS__EMPTY, + "GET_AND_REF.sql": _MODELS_GET_AND_REF_UPPERCASE, + "INCREMENTAL.sql": _MODELS__INCREMENTAL, + "INTERLEAVED_SORT.sql": _MODELS__INTERLEAVED_SORT, + "MATERIALIZED.sql": _MODELS__MATERIALIZED, + "VIEW_MODEL.sql": _MODELS__VIEW_MODEL, + } + + @pytest.fixture(scope="class") + def properties(self): + return { + "schema.yml": _PROPERTIES__SCHEMA_YML, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": _SEEDS__SEED_INITIAL} + + def test_simple_copy_uppercase(self, project): + # Load the seed file and check that it worked + results = run_dbt(["seed"]) + assert len(results) == 1 + + # Run the project and ensure that all the models loaded + results = run_dbt() + assert len(results) == 7 + + check_relations_equal( + project.adapter, + ["seed", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"], + ) diff --git a/tests/simple_copy/test_simple_copy.py b/tests/simple_copy/test_simple_copy.py new file mode 100644 index 00000000..456a8ee3 --- /dev/null +++ b/tests/simple_copy/test_simple_copy.py @@ -0,0 +1,142 @@ +# mix in biguery +# mix in snowflake + +import pytest + +from pathlib import Path + +from dbt.tests.util import run_dbt, rm_file, write_file, check_relations_equal + +from dbt.tests.adapter.simple_copy.fixtures import ( + _PROPERTIES__SCHEMA_YML, + _SEEDS__SEED_INITIAL, + _SEEDS__SEED_UPDATE, + _MODELS__ADVANCED_INCREMENTAL, + _MODELS__COMPOUND_SORT, + _MODELS__DISABLED, + _MODELS__EMPTY, + _MODELS__GET_AND_REF, + _MODELS__INCREMENTAL, + _MODELS__INTERLEAVED_SORT, + _MODELS__MATERIALIZED, + _MODELS__VIEW_MODEL, +) + +from tests.fixtures.profiles import unique_schema, dbt_profile_data +from tests.utils.util import BUCKET, SOURCE + + +class SimpleCopySetup: + @pytest.fixture(scope="class") + def models(self): + return { + "advanced_incremental.sql": _MODELS__ADVANCED_INCREMENTAL, + "compound_sort.sql": _MODELS__COMPOUND_SORT, + "disabled.sql": _MODELS__DISABLED, + "empty.sql": _MODELS__EMPTY, + "get_and_ref.sql": _MODELS__GET_AND_REF, + "incremental.sql": _MODELS__INCREMENTAL, + "interleaved_sort.sql": _MODELS__INTERLEAVED_SORT, + "materialized.sql": _MODELS__MATERIALIZED, + "view_model.sql": _MODELS__VIEW_MODEL, + } + + @pytest.fixture(scope="class") + def properties(self): + return { + "schema.yml": _PROPERTIES__SCHEMA_YML, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": _SEEDS__SEED_INITIAL} + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"seeds": {"quote_columns": False}} + + +class SimpleCopyBase(SimpleCopySetup): + def test_simple_copy(self, project): + # Load the seed file and check that it worked + results = run_dbt(["seed"]) + assert len(results) == 1 + + # Run the project and ensure that all the models loaded + results = run_dbt() + assert len(results) == 7 + check_relations_equal( + project.adapter, + ["seed", "view_model", "incremental", "materialized", "get_and_ref"], + ) + + # Change the seed.csv file and see if everything is the same, i.e. everything has been updated + main_seed_file = project.project_root / Path("seeds") / Path("seed.csv") + rm_file(main_seed_file) + write_file(_SEEDS__SEED_UPDATE, main_seed_file) + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + check_relations_equal( + project.adapter, + ["seed", "view_model", "incremental", "materialized", "get_and_ref"], + ) + + def test_simple_copy_with_materialized_views(self, project): + project.run_sql( + f"create table {SOURCE}.{BUCKET}.{project.test_schema}.unrelated_table (id int)" + ) + sql = f""" + create view "dbt_test".{project.test_schema}.unrelated_materialized_view as ( + select * from {SOURCE}.{BUCKET}.{project.test_schema}.unrelated_table + ) + """ + project.run_sql(sql) + sql = f""" + create view "dbt_test".{project.test_schema}.unrelated_view as ( + select * from "dbt_test".{project.test_schema}.unrelated_materialized_view + ) + """ + project.run_sql(sql) + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + +def _dremio_get_tables_in_schema(self): + sql = """ + select table_name, + case when table_type = 'BASE TABLE' then 'table' + when table_type = 'VIEW' then 'view' + else table_type + end as materialization + from information_schema."tables" + where {} + order by table_name + """ + sql = sql.format("{} LIKE '{}'".format("table_schema", self.test_schema)) + result = self.run_sql(sql, fetch="all") + return {model_name: materialization for (model_name, materialization) in result} + + +class EmptyModelsArentRunBase(SimpleCopySetup): + def test_dbt_doesnt_run_empty_models(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + tables = _dremio_get_tables_in_schema(self=project) + + assert "empty" not in tables.keys() + assert "disabled" not in tables.keys() + + +class TestSimpleCopyBaseDremio(SimpleCopyBase): + pass + + +class TestEmptyModelsArentRunDremio(EmptyModelsArentRunBase): + pass