Skip to content

Commit

Permalink
mv on_schema_change tests -> "adapter zone" (#6618) (#6686)
Browse files Browse the repository at this point in the history 
* mv `on_schema_change` tests -> "adapter zone" (#6618)

* Mv incremental on_schema_change tests to 'adapter zone'

* Use type_string()

* Cleanup

* mv `on_schema_change` tests -> "adapter zone" (#6618)

* Mv incremental on_schema_change tests to 'adapter zone'

* Use type_string()

* Cleanup

Co-authored-by: Jeremy Cohen <jeremy@dbtlabs.com>
  • Loading branch information
@colin-rogers-dbt @jtcohen6
colin-rogers-dbt and jtcohen6 authored Jan 20, 2023
1 parent d934e71 commit e61a39f
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 142 deletions.
106 changes: 8 additions & 98 deletions ...onal/incremental_schema_tests/fixtures.py → ...dbt/tests/adapter/incremental/fixtures.py
View file
Original file line number Diff line number Diff line change
@@ -1,61 +1,3 @@

#
# Properties
#
_PROPERTIES__SCHEMA = """
version: 2
models:
- name: model_a
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_ignore
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_ignore_target
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_append_new_columns
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_append_new_columns_target
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_sync_all_columns
columns:
- name: id
tags: [column_level_tag]
tests:
- unique
- name: incremental_sync_all_columns_target
columns:
- name: id
tags: [column_leveL_tag]
tests:
- unique
"""

#
# Models
#
Expand All @@ -71,7 +13,7 @@
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
{% if is_incremental() %}
Expand Down Expand Up @@ -124,7 +66,7 @@
)
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
select id
,cast(field1 as {{string_type}}) as field1
Expand Down Expand Up @@ -185,7 +127,7 @@
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
{% if is_incremental() %}
Expand Down Expand Up @@ -216,7 +158,7 @@
)
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
Expand Down Expand Up @@ -268,7 +210,7 @@
config(materialized='table')
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
with source_data as (
Expand All @@ -294,7 +236,7 @@
)
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
Expand Down Expand Up @@ -328,7 +270,7 @@
)
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
select id
,cast(field1 as {{string_type}}) as field1
Expand All @@ -345,7 +287,7 @@
config(materialized='table')
}}
{% set string_type = 'varchar(10)' %}
{% set string_type = dbt.type_string() %}
with source_data as (
Expand All @@ -361,35 +303,3 @@
from source_data
"""

#
# Tests
#

_TESTS__SELECT_FROM_INCREMENTAL_IGNORE = """
select * from {{ ref('incremental_ignore') }} where false
"""

_TESTS__SELECT_FROM_A = """
select * from {{ ref('model_a') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET = """
select * from {{ ref('incremental_append_new_columns_target') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS = """
select * from {{ ref('incremental_sync_all_columns') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """
select * from {{ ref('incremental_sync_all_columns_target') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET = """
select * from {{ ref('incremental_ignore_target') }} where false
"""

_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS = """
select * from {{ ref('incremental_append_new_columns') }} where false
"""
59 changes: 15 additions & 44 deletions ...l_schema_tests/test_incremental_schema.py → ...ntal/test_incremental_on_schema_change.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@
run_dbt,
)

from tests.functional.incremental_schema_tests.fixtures import (
_PROPERTIES__SCHEMA,
from dbt.tests.adapter.incremental.fixtures import (
_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY,
_MODELS__INCREMENTAL_IGNORE,
_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET,
Expand All @@ -19,23 +18,10 @@
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS,
_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_IGNORE,
_TESTS__SELECT_FROM_A,
_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS,
_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET,
_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS,
)


class TestIncrementalSchemaChange:
@pytest.fixture(scope="class")
def properties(self):
return {
"schema.yml": _PROPERTIES__SCHEMA,
}

class BaseIncrementalOnSchemaChangeSetup:
@pytest.fixture(scope="class")
def models(self):
return {
Expand All @@ -52,32 +38,11 @@ def models(self):
"incremental_append_new_columns_target.sql":
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET,
"incremental_append_new_columns.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS,
"incremental_sync_all_columns_target.sql":
_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
"incremental_append_new_columns_remove_one_target.sql":
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET,
"incremental_sync_all_columns_target.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
"incremental_append_new_columns_remove_one_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET,
}

@pytest.fixture(scope="class")
def tests(self):
return {
"select_from_incremental.sql": _TESTS__SELECT_FROM_INCREMENTAL_IGNORE,
"select_from_a.sql": _TESTS__SELECT_FROM_A,
"select_from_incremental_append_new_columns_target.sql":
_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS_TARGET,
"select_from_incremental_sync_all_columns.sql":
_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS,
"select_from_incremental_sync_all_columns_target.sql":
_TESTS__SELECT_FROM_INCREMENTAL_SYNC_ALL_COLUMNS_TARGET,
"select_from_incremental_ignore_target.sql":
_TESTS__SELECT_FROM_INCREMENTAL_IGNORE_TARGET,
"select_from_incremental_append_new_columns.sql":
_TESTS__SELECT_FROM_INCREMENTAL_APPEND_NEW_COLUMNS,
}

def run_twice_and_assert(
self, include, compare_source, compare_target, project
):
def run_twice_and_assert(self, include, compare_source, compare_target, project):

# dbt run (twice)
run_args = ['run']
Expand Down Expand Up @@ -115,6 +80,8 @@ def run_incremental_sync_remove_only(self, project):
compare_target = 'incremental_sync_remove_only_target'
self.run_twice_and_assert(select, compare_source, compare_target, project)


class BaseIncrementalOnSchemaChange(BaseIncrementalOnSchemaChangeSetup):
def test_run_incremental_ignore(self, project):
select = 'model_a incremental_ignore incremental_ignore_target'
compare_source = 'incremental_ignore'
Expand All @@ -130,7 +97,11 @@ def test_run_incremental_sync_all_columns(self, project):
self.run_incremental_sync_remove_only(project)

def test_run_incremental_fail_on_schema_change(self, project):
select = 'model_a incremental_fail'
run_dbt(['run', '--models', select, '--full-refresh'])
results_two = run_dbt(['run', '--models', select], expect_pass=False)
assert 'Compilation Error' in results_two[1].message
select = "model_a incremental_fail"
run_dbt(["run", "--models", select, "--full-refresh"])
results_two = run_dbt(["run", "--models", select], expect_pass=False)
assert "Compilation Error" in results_two[1].message


class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange):
pass

0 comments on commit e61a39f

Please sign in to comment.