diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index eafb49efe76..d6afe223278 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -8,6 +8,7 @@ parse_file_type_to_parser, ) from dbt.events.functions import fire_event +from dbt.events.base_types import EventLevel from dbt.events.types import ( PartialParsingEnabled, PartialParsingFile, @@ -155,7 +156,11 @@ def build_file_diff(self): self.macro_child_map = self.saved_manifest.build_macro_child_map() deleted = len(deleted) + len(deleted_schema_files) changed = len(changed) + len(changed_schema_files) - fire_event(PartialParsingEnabled(deleted=deleted, added=len(added), changed=changed)) + event = PartialParsingEnabled(deleted=deleted, added=len(added), changed=changed) + if os.environ.get("DBT_PP_TEST"): + fire_event(event, level=EventLevel.INFO) + else: + fire_event(event) self.file_diff = file_diff # generate the list of files that need parsing diff --git a/test/integration/068_partial_parsing_tests/local_dependency/dbt_project.yml b/test/integration/068_partial_parsing_tests/local_dependency/dbt_project.yml deleted file mode 100644 index d56280a5577..00000000000 --- a/test/integration/068_partial_parsing_tests/local_dependency/dbt_project.yml +++ /dev/null @@ -1,23 +0,0 @@ - -name: 'local_dep' -version: '1.0' -config-version: 2 - -profile: 'default' - -model-paths: ["models"] -analysis-paths: ["analyses"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] - -require-dbt-version: '>=0.1.0' - -target-path: "target" # directory which will store compiled SQL files -clean-targets: # directories to be removed by `dbt clean` - - "target" - - "dbt_packages" - - -seeds: - quote_columns: False diff --git a/test/integration/068_partial_parsing_tests/local_dependency/macros/dep_macro.sql b/test/integration/068_partial_parsing_tests/local_dependency/macros/dep_macro.sql deleted file mode 100644 index 81e9a0faeef..00000000000 --- a/test/integration/068_partial_parsing_tests/local_dependency/macros/dep_macro.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro some_overridden_macro() -%} -100 -{%- endmacro %} diff --git a/test/integration/068_partial_parsing_tests/local_dependency/models/model_to_import.sql b/test/integration/068_partial_parsing_tests/local_dependency/models/model_to_import.sql deleted file mode 100644 index 4b91aa0f2fa..00000000000 --- a/test/integration/068_partial_parsing_tests/local_dependency/models/model_to_import.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('seed') }} diff --git a/test/integration/068_partial_parsing_tests/local_dependency/models/schema.yml b/test/integration/068_partial_parsing_tests/local_dependency/models/schema.yml deleted file mode 100644 index 3d804a7c153..00000000000 --- a/test/integration/068_partial_parsing_tests/local_dependency/models/schema.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -sources: - - name: seed_source - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - tests: - - unique diff --git a/test/integration/068_partial_parsing_tests/local_dependency/seeds/seed.csv b/test/integration/068_partial_parsing_tests/local_dependency/seeds/seed.csv deleted file mode 100644 index 3ff3deb87eb..00000000000 --- a/test/integration/068_partial_parsing_tests/local_dependency/seeds/seed.csv +++ /dev/null @@ -1,2 +0,0 @@ -id -1 diff --git a/test/integration/068_partial_parsing_tests/test-files/custom_schema_tests1.sql b/test/integration/068_partial_parsing_tests/test-files/custom_schema_tests1.sql deleted file mode 100644 index 0f64eb17c0d..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/custom_schema_tests1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "WARN") }} - - select * from {{ model }} - -{% endtest %} diff --git a/test/integration/068_partial_parsing_tests/test-files/custom_schema_tests2.sql b/test/integration/068_partial_parsing_tests/test-files/custom_schema_tests2.sql deleted file mode 100644 index ba5b53fa5a9..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/custom_schema_tests2.sql +++ /dev/null @@ -1,19 +0,0 @@ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "ERROR") }} - - select * from {{ model }} - -{% endtest %} diff --git a/test/integration/068_partial_parsing_tests/test-files/customers.sql b/test/integration/068_partial_parsing_tests/test-files/customers.sql deleted file mode 100644 index 98e19b557eb..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/customers.sql +++ /dev/null @@ -1,19 +0,0 @@ -with source as ( - - select * from {{ source('seed_sources', 'raw_customers') }} - -), - -renamed as ( - - select - id as customer_id, - first_name, - last_name, - email - - from source - -) - -select * from renamed diff --git a/test/integration/068_partial_parsing_tests/test-files/customers1.md b/test/integration/068_partial_parsing_tests/test-files/customers1.md deleted file mode 100644 index bba48335825..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/customers1.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs customer_table %} - -This table contains customer data - -{% enddocs %} diff --git a/test/integration/068_partial_parsing_tests/test-files/customers2.md b/test/integration/068_partial_parsing_tests/test-files/customers2.md deleted file mode 100644 index f8306f34e49..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/customers2.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs customer_table %} - -LOTS of customer data - -{% enddocs %} diff --git a/test/integration/068_partial_parsing_tests/test-files/empty_schema.yml b/test/integration/068_partial_parsing_tests/test-files/empty_schema.yml deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/test/integration/068_partial_parsing_tests/test-files/empty_schema_with_version.yml b/test/integration/068_partial_parsing_tests/test-files/empty_schema_with_version.yml deleted file mode 100644 index 22817d2a9c7..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/empty_schema_with_version.yml +++ /dev/null @@ -1 +0,0 @@ -version: 2 diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var-sources.yml b/test/integration/068_partial_parsing_tests/test-files/env_var-sources.yml deleted file mode 100644 index 2b5809b1cb9..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var-sources.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 2 -sources: - - name: seed_sources - schema: "{{ target.schema }}" - database: "{{ env_var('ENV_VAR_DATABASE') }}" - tables: - - name: raw_customers - columns: - - name: id - tests: - - not_null: - severity: "{{ env_var('ENV_VAR_SEVERITY') }}" - - unique - - name: first_name - - name: last_name - - name: email - - diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_macro.sql b/test/integration/068_partial_parsing_tests/test-files/env_var_macro.sql deleted file mode 100644 index 0bf3eda6c07..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_macro.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro do_something(foo2, bar2) %} - - select - '{{ foo2 }}' as foo2, - '{{ bar2 }}' as bar2 - -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_macros.yml b/test/integration/068_partial_parsing_tests/test-files/env_var_macros.yml deleted file mode 100644 index 8888f65237d..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_macros.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 -macros: - - name: do_something - description: "This is a test macro" - meta: - some_key: "{{ env_var('ENV_VAR_SOME_KEY') }}" - diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_metrics.yml b/test/integration/068_partial_parsing_tests/test-files/env_var_metrics.yml deleted file mode 100644 index b8112fea010..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_metrics.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 - -metrics: - - - model: "ref('people')" - name: number_of_people - description: Total count of people - label: "Number of people" - calculation_method: count - expression: "*" - timestamp: created_at - time_grains: [day, week, month] - dimensions: - - favorite_color - - loves_dbt - meta: - my_meta: '{{ env_var("ENV_VAR_METRICS") }}' - - - model: "ref('people')" - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - calculation_method: sum - expression: tenure - timestamp: created_at - time_grains: [day] - filters: - - field: loves_dbt - operator: is - value: 'true' diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_model.sql b/test/integration/068_partial_parsing_tests/test-files/env_var_model.sql deleted file mode 100644 index a926d16d9d8..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_model.sql +++ /dev/null @@ -1 +0,0 @@ -select '{{ env_var('ENV_VAR_TEST') }}' as vartest diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_model_one.sql b/test/integration/068_partial_parsing_tests/test-files/env_var_model_one.sql deleted file mode 100644 index e1875231d2e..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_model_one.sql +++ /dev/null @@ -1 +0,0 @@ -select 'blue' as fun diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_model_test.yml b/test/integration/068_partial_parsing_tests/test-files/env_var_model_test.yml deleted file mode 100644 index 147b96de1b6..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_model_test.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 -models: - - name: model_color - columns: - - name: fun - tests: - - unique: - enabled: "{{ env_var('ENV_VAR_ENABLED', True) }}" diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_schema.yml b/test/integration/068_partial_parsing_tests/test-files/env_var_schema.yml deleted file mode 100644 index f8cf1ed9d67..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_schema.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_schema2.yml b/test/integration/068_partial_parsing_tests/test-files/env_var_schema2.yml deleted file mode 100644 index b1f3f079f6a..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_schema2.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - tests: - - check_color: - column_name: fun - color: "env_var('ENV_VAR_COLOR')" - diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_schema3.yml b/test/integration/068_partial_parsing_tests/test-files/env_var_schema3.yml deleted file mode 100644 index 3b0409637db..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/env_var_schema3.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 2 - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - tests: - - check_color: - column_name: fun - color: "env_var('ENV_VAR_COLOR')" - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "{{ env_var('ENV_VAR_OWNER') }}" - email: "tester@dashboard.com" - depends_on: - - ref("model_color") - - source("seed_sources", "raw_customers") diff --git a/test/integration/068_partial_parsing_tests/test-files/generic_schema.yml b/test/integration/068_partial_parsing_tests/test-files/generic_schema.yml deleted file mode 100644 index 9a44074728a..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/generic_schema.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: 2 - -models: - - name: orders - description: "Some order data" - columns: - - name: id - tests: - - unique diff --git a/test/integration/068_partial_parsing_tests/test-files/generic_test.sql b/test/integration/068_partial_parsing_tests/test-files/generic_test.sql deleted file mode 100644 index ca09beaadb7..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/generic_test.sql +++ /dev/null @@ -1,26 +0,0 @@ -{% test is_odd(model, column_name) %} - -with validation as ( - - select - {{ column_name }} as odd_field - - from {{ model }} - -), - -validation_errors as ( - - select - odd_field - - from validation - -- if this is true, then odd_field is actually even! - where (odd_field % 2) = 0 - -) - -select * -from validation_errors - -{% endtest %} \ No newline at end of file diff --git a/test/integration/068_partial_parsing_tests/test-files/generic_test_edited.sql b/test/integration/068_partial_parsing_tests/test-files/generic_test_edited.sql deleted file mode 100644 index 5a3b611ff7a..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/generic_test_edited.sql +++ /dev/null @@ -1,26 +0,0 @@ -{% test is_odd(model, column_name) %} - -with validation as ( - - select - {{ column_name }} as odd_field2 - - from {{ model }} - -), - -validation_errors as ( - - select - odd_field2 - - from validation - -- if this is true, then odd_field is actually even! - where (odd_field2 % 2) = 0 - -) - -select * -from validation_errors - -{% endtest %} \ No newline at end of file diff --git a/test/integration/068_partial_parsing_tests/test-files/generic_test_schema.yml b/test/integration/068_partial_parsing_tests/test-files/generic_test_schema.yml deleted file mode 100644 index c8307bc1021..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/generic_test_schema.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 - -models: - - name: orders - description: "Some order data" - columns: - - name: id - tests: - - unique - - is_odd diff --git a/test/integration/068_partial_parsing_tests/test-files/gsm_override.sql b/test/integration/068_partial_parsing_tests/test-files/gsm_override.sql deleted file mode 100644 index 46c7a39ddaa..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/gsm_override.sql +++ /dev/null @@ -1,6 +0,0 @@ -- custom macro -{% macro generate_schema_name(schema_name, node) %} - - {{ schema_name }}_{{ target.schema }} - -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/gsm_override2.sql b/test/integration/068_partial_parsing_tests/test-files/gsm_override2.sql deleted file mode 100644 index 1bfddb9dadb..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/gsm_override2.sql +++ /dev/null @@ -1,6 +0,0 @@ -- custom macro xxxx -{% macro generate_schema_name(schema_name, node) %} - - {{ schema_name }}_{{ target.schema }} - -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/macros-schema.yml b/test/integration/068_partial_parsing_tests/test-files/macros-schema.yml deleted file mode 100644 index cf221dec670..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/macros-schema.yml +++ /dev/null @@ -1,8 +0,0 @@ - -version: 2 - -models: - - name: model_a - tests: - - type_one - - type_two diff --git a/test/integration/068_partial_parsing_tests/test-files/macros.yml b/test/integration/068_partial_parsing_tests/test-files/macros.yml deleted file mode 100644 index 9ee72fad0ea..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/macros.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: 2 -macros: - - name: do_something - description: "This is a test macro" diff --git a/test/integration/068_partial_parsing_tests/test-files/metric_model_a.sql b/test/integration/068_partial_parsing_tests/test-files/metric_model_a.sql deleted file mode 100644 index 010a0c29a02..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/metric_model_a.sql +++ /dev/null @@ -1,21 +0,0 @@ -{% - set metric_list = [ - metric('number_of_people'), - metric('collective_tenure') - ] -%} - -{% if not execute %} - - {% set metric_names = [] %} - {% for m in metric_list %} - {% do metric_names.append(m.metric_name) %} - {% endfor %} - - -- this config does nothing, but it lets us check these values - {{ config(metric_names = metric_names) }} - -{% endif %} - - -select 1 as fun diff --git a/test/integration/068_partial_parsing_tests/test-files/model_a.sql b/test/integration/068_partial_parsing_tests/test-files/model_a.sql deleted file mode 100644 index 3bd54a4c1b6..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_a.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as fun diff --git a/test/integration/068_partial_parsing_tests/test-files/model_b.sql b/test/integration/068_partial_parsing_tests/test-files/model_b.sql deleted file mode 100644 index 01f38b0698e..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_b.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as notfun diff --git a/test/integration/068_partial_parsing_tests/test-files/model_color.sql b/test/integration/068_partial_parsing_tests/test-files/model_color.sql deleted file mode 100644 index e1875231d2e..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_color.sql +++ /dev/null @@ -1 +0,0 @@ -select 'blue' as fun diff --git a/test/integration/068_partial_parsing_tests/test-files/model_four1.sql b/test/integration/068_partial_parsing_tests/test-files/model_four1.sql deleted file mode 100644 index 97c5b226d8c..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_four1.sql +++ /dev/null @@ -1 +0,0 @@ -select * from {{ ref('model_three') }} diff --git a/test/integration/068_partial_parsing_tests/test-files/model_four2.sql b/test/integration/068_partial_parsing_tests/test-files/model_four2.sql deleted file mode 100644 index c38a4c9194f..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_four2.sql +++ /dev/null @@ -1 +0,0 @@ -select fun from {{ ref('model_one') }} diff --git a/test/integration/068_partial_parsing_tests/test-files/model_one.sql b/test/integration/068_partial_parsing_tests/test-files/model_one.sql deleted file mode 100644 index 3bd54a4c1b6..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_one.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as fun diff --git a/test/integration/068_partial_parsing_tests/test-files/model_three.sql b/test/integration/068_partial_parsing_tests/test-files/model_three.sql deleted file mode 100644 index 45aa2b750f7..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_three.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(materialized='table') }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data diff --git a/test/integration/068_partial_parsing_tests/test-files/model_three_disabled.sql b/test/integration/068_partial_parsing_tests/test-files/model_three_disabled.sql deleted file mode 100644 index a338a2ef4d2..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_three_disabled.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(materialized='table', enabled=False) }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data diff --git a/test/integration/068_partial_parsing_tests/test-files/model_three_disabled2.sql b/test/integration/068_partial_parsing_tests/test-files/model_three_disabled2.sql deleted file mode 100644 index 4d416ab516e..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_three_disabled2.sql +++ /dev/null @@ -1,13 +0,0 @@ -- Disabled model -{{ config(materialized='table', enabled=False) }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data diff --git a/test/integration/068_partial_parsing_tests/test-files/model_three_modified.sql b/test/integration/068_partial_parsing_tests/test-files/model_three_modified.sql deleted file mode 100644 index e2d2df486c5..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_three_modified.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ config(materialized='table') }} - -with source_data as ( - - {#- This is model three #} - - select 1 as id - union all - select null as id - -) - -select * -from source_data diff --git a/test/integration/068_partial_parsing_tests/test-files/model_two.sql b/test/integration/068_partial_parsing_tests/test-files/model_two.sql deleted file mode 100644 index 01f38b0698e..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/model_two.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as notfun diff --git a/test/integration/068_partial_parsing_tests/test-files/models-schema1.yml b/test/integration/068_partial_parsing_tests/test-files/models-schema1.yml deleted file mode 100644 index 36e5ce68a6e..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/models-schema1.yml +++ /dev/null @@ -1,5 +0,0 @@ -version: 2 - -models: - - name: model_one - description: "The first model" diff --git a/test/integration/068_partial_parsing_tests/test-files/models-schema2.yml b/test/integration/068_partial_parsing_tests/test-files/models-schema2.yml deleted file mode 100644 index 7c9a890a481..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/models-schema2.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - columns: - - name: id - tests: - - unique diff --git a/test/integration/068_partial_parsing_tests/test-files/models-schema2b.yml b/test/integration/068_partial_parsing_tests/test-files/models-schema2b.yml deleted file mode 100644 index c9369126ffc..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/models-schema2b.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - columns: - - name: id - tests: - - not_null diff --git a/test/integration/068_partial_parsing_tests/test-files/models-schema3.yml b/test/integration/068_partial_parsing_tests/test-files/models-schema3.yml deleted file mode 100644 index 11e4468d248..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/models-schema3.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - tests: - - unique -macros: - - name: do_something - description: "This is a test macro" diff --git a/test/integration/068_partial_parsing_tests/test-files/models-schema4.yml b/test/integration/068_partial_parsing_tests/test-files/models-schema4.yml deleted file mode 100644 index 8087615fe49..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/models-schema4.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: 2 - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - config: - enabled: false - columns: - - name: id - tests: - - unique diff --git a/test/integration/068_partial_parsing_tests/test-files/models-schema4b.yml b/test/integration/068_partial_parsing_tests/test-files/models-schema4b.yml deleted file mode 100644 index e73ffcef1de..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/models-schema4b.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: 2 - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - config: - enabled: true - columns: - - name: id - tests: - - unique diff --git a/test/integration/068_partial_parsing_tests/test-files/my_analysis.sql b/test/integration/068_partial_parsing_tests/test-files/my_analysis.sql deleted file mode 100644 index ec6959e9a68..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/my_analysis.sql +++ /dev/null @@ -1 +0,0 @@ -select * from customers diff --git a/test/integration/068_partial_parsing_tests/test-files/my_macro.sql b/test/integration/068_partial_parsing_tests/test-files/my_macro.sql deleted file mode 100644 index 0bf3eda6c07..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/my_macro.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro do_something(foo2, bar2) %} - - select - '{{ foo2 }}' as foo2, - '{{ bar2 }}' as bar2 - -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/my_macro2.sql b/test/integration/068_partial_parsing_tests/test-files/my_macro2.sql deleted file mode 100644 index e64aafa5ab5..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/my_macro2.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro do_something(foo2, bar2) %} - - select - 'foo' as foo2, - 'var' as bar2 - -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/my_metric.yml b/test/integration/068_partial_parsing_tests/test-files/my_metric.yml deleted file mode 100644 index 521bc92290f..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/my_metric.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: 2 -metrics: - - name: new_customers - label: New Customers - model: customers - description: "The number of paid customers who are using the product" - calculation_method: count - expression: user_id - timestamp: signup_date - time_grains: [day, week, month] - dimensions: - - plan - - country - filters: - - field: is_paying - value: True - operator: '=' - +meta: - is_okr: True - tags: - - okrs - - diff --git a/test/integration/068_partial_parsing_tests/test-files/my_test.sql b/test/integration/068_partial_parsing_tests/test-files/my_test.sql deleted file mode 100644 index fbfb738bc9a..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/my_test.sql +++ /dev/null @@ -1,2 +0,0 @@ -select - * from {{ ref('customers') }} where first_name = '{{ macro_something() }}' diff --git a/test/integration/068_partial_parsing_tests/test-files/orders.sql b/test/integration/068_partial_parsing_tests/test-files/orders.sql deleted file mode 100644 index ef61d616cc1..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/orders.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id, 101 as user_id, 'pending' as status diff --git a/test/integration/068_partial_parsing_tests/test-files/people.sql b/test/integration/068_partial_parsing_tests/test-files/people.sql deleted file mode 100644 index ce58d41a599..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/people.sql +++ /dev/null @@ -1,3 +0,0 @@ -select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at -union all -select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at diff --git a/test/integration/068_partial_parsing_tests/test-files/people_metrics.yml b/test/integration/068_partial_parsing_tests/test-files/people_metrics.yml deleted file mode 100644 index 99d31a4e632..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/people_metrics.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 - -metrics: - - - model: "ref('people')" - name: number_of_people - description: Total count of people - label: "Number of people" - calculation_method: count - expression: "*" - timestamp: created_at - time_grains: [day, week, month] - dimensions: - - favorite_color - - loves_dbt - meta: - my_meta: 'testing' - - - model: "ref('people')" - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - calculation_method: sum - expression: tenure - timestamp: created_at - time_grains: [day] - filters: - - field: loves_dbt - operator: is - value: 'true' diff --git a/test/integration/068_partial_parsing_tests/test-files/people_metrics2.yml b/test/integration/068_partial_parsing_tests/test-files/people_metrics2.yml deleted file mode 100644 index 5f826e66e85..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/people_metrics2.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 - -metrics: - - - model: "ref('people')" - name: number_of_people - description: Total count of people - label: "Number of people" - calculation_method: count - expression: "*" - timestamp: created_at - time_grains: [day, week, month] - dimensions: - - favorite_color - - loves_dbt - meta: - my_meta: 'replaced' - - - model: "ref('people')" - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - calculation_method: sum - expression: tenure - timestamp: created_at - time_grains: [day] - filters: - - field: loves_dbt - operator: is - value: 'true' diff --git a/test/integration/068_partial_parsing_tests/test-files/people_metrics3.yml b/test/integration/068_partial_parsing_tests/test-files/people_metrics3.yml deleted file mode 100644 index b9c640591fc..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/people_metrics3.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 2 - -metrics: - - - model: "ref('people')" - name: number_of_people - description: Total count of people - label: "Number of people" - calculation_method: count - expression: "*" - timestamp: created_at - time_grains: [day, week, month] - dimensions: - - favorite_color - - loves_dbt - meta: - my_meta: 'replaced' diff --git a/test/integration/068_partial_parsing_tests/test-files/raw_customers.csv b/test/integration/068_partial_parsing_tests/test-files/raw_customers.csv deleted file mode 100644 index 2315be73844..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/raw_customers.csv +++ /dev/null @@ -1,11 +0,0 @@ -id,first_name,last_name,email -1,Michael,Perez,mperez0@chronoengine.com -2,Shawn,Mccoy,smccoy1@reddit.com -3,Kathleen,Payne,kpayne2@cargocollective.com -4,Jimmy,Cooper,jcooper3@cargocollective.com -5,Katherine,Rice,krice4@typepad.com -6,Sarah,Ryan,sryan5@gnu.org -7,Martin,Mcdonald,mmcdonald6@opera.com -8,Frank,Robinson,frobinson7@wunderground.com -9,Jennifer,Franklin,jfranklin8@mail.ru -10,Henry,Welch,hwelch9@list-manage.com diff --git a/test/integration/068_partial_parsing_tests/test-files/ref_override.sql b/test/integration/068_partial_parsing_tests/test-files/ref_override.sql deleted file mode 100644 index cd16793d3c4..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/ref_override.sql +++ /dev/null @@ -1,4 +0,0 @@ -- Macro to override ref -{% macro ref(modelname) %} -{% do return(builtins.ref(modelname)) %} -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/ref_override2.sql b/test/integration/068_partial_parsing_tests/test-files/ref_override2.sql deleted file mode 100644 index 2e8027d8e80..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/ref_override2.sql +++ /dev/null @@ -1,4 +0,0 @@ -- Macro to override ref xxxx -{% macro ref(modelname) %} -{% do return(builtins.ref(modelname)) %} -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/schema-models-c.yml b/test/integration/068_partial_parsing_tests/test-files/schema-models-c.yml deleted file mode 100644 index 432b5e0efe3..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/schema-models-c.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 - -sources: - - name: seed_source - description: "This is a source override" - overrides: local_dep - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - tests: - - unique - - not_null diff --git a/test/integration/068_partial_parsing_tests/test-files/schema-sources1.yml b/test/integration/068_partial_parsing_tests/test-files/schema-sources1.yml deleted file mode 100644 index 30363115e09..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/schema-sources1.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 2 -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - - diff --git a/test/integration/068_partial_parsing_tests/test-files/schema-sources2.yml b/test/integration/068_partial_parsing_tests/test-files/schema-sources2.yml deleted file mode 100644 index 5927952917f..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/schema-sources2.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: 2 - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - ref("raw_customers") - - source("seed_sources", "raw_customers") - diff --git a/test/integration/068_partial_parsing_tests/test-files/schema-sources3.yml b/test/integration/068_partial_parsing_tests/test-files/schema-sources3.yml deleted file mode 100644 index 54133a9a2f5..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/schema-sources3.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: 2 - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - source("seed_sources", "raw_customers") - diff --git a/test/integration/068_partial_parsing_tests/test-files/schema-sources4.yml b/test/integration/068_partial_parsing_tests/test-files/schema-sources4.yml deleted file mode 100644 index af76a0f315a..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/schema-sources4.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - every_value_is_blue - - name: first_name - - name: last_name - - name: email - -seeds: - - name: raw_customers - description: "Raw customer data" - columns: - - name: id - tests: - - unique - - not_null - - name: first_name - - name: last_name - - name: email - diff --git a/test/integration/068_partial_parsing_tests/test-files/schema-sources5.yml b/test/integration/068_partial_parsing_tests/test-files/schema-sources5.yml deleted file mode 100644 index 57818771b71..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/schema-sources5.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: 2 - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -seeds: - - name: rad_customers - description: "Raw customer data" - columns: - - name: id - tests: - - unique - - not_null - - name: first_name - - name: last_name - - name: email - diff --git a/test/integration/068_partial_parsing_tests/test-files/snapshot.sql b/test/integration/068_partial_parsing_tests/test-files/snapshot.sql deleted file mode 100644 index c82a2fa5906..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/snapshot.sql +++ /dev/null @@ -1,29 +0,0 @@ -{% snapshot orders_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['status'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -{% snapshot orders2_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['order_date'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} diff --git a/test/integration/068_partial_parsing_tests/test-files/snapshot2.sql b/test/integration/068_partial_parsing_tests/test-files/snapshot2.sql deleted file mode 100644 index 27d320618c9..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/snapshot2.sql +++ /dev/null @@ -1,30 +0,0 @@ -- add a comment -{% snapshot orders_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['status'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -{% snapshot orders2_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['order_date'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} diff --git a/test/integration/068_partial_parsing_tests/test-files/sources-tests1.sql b/test/integration/068_partial_parsing_tests/test-files/sources-tests1.sql deleted file mode 100644 index dd8710f0556..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/sources-tests1.sql +++ /dev/null @@ -1,9 +0,0 @@ - -{% test every_value_is_blue(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} = 9999 - -{% endtest %} - diff --git a/test/integration/068_partial_parsing_tests/test-files/sources-tests2.sql b/test/integration/068_partial_parsing_tests/test-files/sources-tests2.sql deleted file mode 100644 index 3abcf30a658..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/sources-tests2.sql +++ /dev/null @@ -1,9 +0,0 @@ - -{% test every_value_is_blue(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} != 99 - -{% endtest %} - diff --git a/test/integration/068_partial_parsing_tests/test-files/test-macro.sql b/test/integration/068_partial_parsing_tests/test-files/test-macro.sql deleted file mode 100644 index f2b1ecfc86b..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/test-macro.sql +++ /dev/null @@ -1,5 +0,0 @@ -{% macro macro_something() %} - - {% do return('macro_something') %} - -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/test-macro2.sql b/test/integration/068_partial_parsing_tests/test-files/test-macro2.sql deleted file mode 100644 index 52b4469cd01..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/test-macro2.sql +++ /dev/null @@ -1,5 +0,0 @@ -{% macro macro_something() %} - - {% do return('some_name') %} - -{% endmacro %} diff --git a/test/integration/068_partial_parsing_tests/test-files/test_color.sql b/test/integration/068_partial_parsing_tests/test-files/test_color.sql deleted file mode 100644 index 0bb1cdcd96c..00000000000 --- a/test/integration/068_partial_parsing_tests/test-files/test_color.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% test check_color(model, column_name, color) %} - - select * - from {{ model }} - where {{ column_name }} = '{{ color }}' - -{% endtest %} diff --git a/test/integration/068_partial_parsing_tests/test_partial_parsing.py b/test/integration/068_partial_parsing_tests/test_partial_parsing.py deleted file mode 100644 index d411a738602..00000000000 --- a/test/integration/068_partial_parsing_tests/test_partial_parsing.py +++ /dev/null @@ -1,580 +0,0 @@ -from dbt.exceptions import CompilationError -from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.files import ParseFileType -from dbt.contracts.results import TestStatus -from dbt.parser.partial import special_override_macros -from test.integration.base import DBTIntegrationTest, use_profile, normalize, get_manifest -import shutil -import os - - -# Note: every test case needs to have separate directories, otherwise -# they will interfere with each other when tests are multi-threaded - -class BasePPTest(DBTIntegrationTest): - - @property - def schema(self): - return "test_068A" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'test-paths': ['tests'], - 'macro-paths': ['macros'], - 'analysis-paths': ['analyses'], - 'snapshot-paths': ['snapshots'], - 'seeds': { - 'quote_columns': False, - }, - } - - def setup_directories(self): - # Create the directories for the test in the `self.test_root_dir` - # directory after everything else is symlinked. We can copy to and - # delete files in this directory without tests interfering with each other. - os.mkdir(os.path.join(self.test_root_dir, 'models')) - os.mkdir(os.path.join(self.test_root_dir, 'tests')) - os.mkdir(os.path.join(self.test_root_dir, 'tests', 'generic')) - os.mkdir(os.path.join(self.test_root_dir, 'seeds')) - os.mkdir(os.path.join(self.test_root_dir, 'macros')) - os.mkdir(os.path.join(self.test_root_dir, 'analyses')) - os.mkdir(os.path.join(self.test_root_dir, 'snapshots')) - os.environ['DBT_PP_TEST'] = 'true' - - - -class ModelTest(BasePPTest): - - @use_profile('postgres') - def test_postgres_pp_models(self): - self.setup_directories() - self.copy_file('test-files/model_one.sql', 'models/model_one.sql') - # initial run - self.run_dbt(['clean']) - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - - # add a model file - self.copy_file('test-files/model_two.sql', 'models/model_two.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - - # add a schema file - self.copy_file('test-files/models-schema1.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - manifest = get_manifest() - self.assertIn('model.test.model_one', manifest.nodes) - model_one_node = manifest.nodes['model.test.model_one'] - self.assertEqual(model_one_node.description, 'The first model') - self.assertEqual(model_one_node.patch_path, 'test://' + normalize('models/schema.yml')) - - # add a model and a schema file (with a test) at the same time - self.copy_file('test-files/models-schema2.yml', 'models/schema.yml') - self.copy_file('test-files/model_three.sql', 'models/model_three.sql') - results = self.run_dbt(["--partial-parse", "test"], expect_pass=False) - self.assertEqual(len(results), 1) - manifest = get_manifest() - project_files = [f for f in manifest.files if f.startswith('test://')] - self.assertEqual(len(project_files), 4) - model_3_file_id = 'test://' + normalize('models/model_three.sql') - self.assertIn(model_3_file_id, manifest.files) - model_three_file = manifest.files[model_3_file_id] - self.assertEqual(model_three_file.parse_file_type, ParseFileType.Model) - self.assertEqual(type(model_three_file).__name__, 'SourceFile') - model_three_node = manifest.nodes[model_three_file.nodes[0]] - schema_file_id = 'test://' + normalize('models/schema.yml') - self.assertEqual(model_three_node.patch_path, schema_file_id) - self.assertEqual(model_three_node.description, 'The third model') - schema_file = manifest.files[schema_file_id] - self.assertEqual(type(schema_file).__name__, 'SchemaSourceFile') - self.assertEqual(len(schema_file.tests), 1) - tests = schema_file.get_all_test_ids() - self.assertEqual(tests, ['test.test.unique_model_three_id.6776ac8160']) - unique_test_id = tests[0] - self.assertIn(unique_test_id, manifest.nodes) - - # modify model sql file, ensure description still there - self.copy_file('test-files/model_three_modified.sql', 'models/model_three.sql') - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - model_id = 'model.test.model_three' - self.assertIn(model_id, manifest.nodes) - model_three_node = manifest.nodes[model_id] - self.assertEqual(model_three_node.description, 'The third model') - - # Change the model 3 test from unique to not_null - self.copy_file('test-files/models-schema2b.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "test"], expect_pass=False) - manifest = get_manifest() - schema_file_id = 'test://' + normalize('models/schema.yml') - schema_file = manifest.files[schema_file_id] - tests = schema_file.get_all_test_ids() - self.assertEqual(tests, ['test.test.not_null_model_three_id.3162ce0a6f']) - not_null_test_id = tests[0] - self.assertIn(not_null_test_id, manifest.nodes.keys()) - self.assertNotIn(unique_test_id, manifest.nodes.keys()) - self.assertEqual(len(results), 1) - - # go back to previous version of schema file, removing patch, test, and model for model three - self.copy_file('test-files/models-schema1.yml', 'models/schema.yml') - self.rm_file(normalize('models/model_three.sql')) - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - - # remove schema file, still have 3 models - self.copy_file('test-files/model_three.sql', 'models/model_three.sql') - self.rm_file(normalize('models/schema.yml')) - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - manifest = get_manifest() - schema_file_id = 'test://' + normalize('models/schema.yml') - self.assertNotIn(schema_file_id, manifest.files) - project_files = [f for f in manifest.files if f.startswith('test://')] - self.assertEqual(len(project_files), 3) - - # Put schema file back and remove a model - # referred to in schema file - self.copy_file('test-files/models-schema2.yml', 'models/schema.yml') - self.rm_file('models/model_three.sql') - with self.assertRaises(CompilationError): - results = self.run_dbt(["--partial-parse", "--warn-error", "run"]) - - # Put model back again - self.copy_file('test-files/model_three.sql', 'models/model_three.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Add model four refing model three - self.copy_file('test-files/model_four1.sql', 'models/model_four.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 4) - - # Remove model_three and change model_four to ref model_one - # and change schema file to remove model_three - self.rm_file('models/model_three.sql') - self.copy_file('test-files/model_four2.sql', 'models/model_four.sql') - self.copy_file('test-files/models-schema1.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Remove model four, put back model three, put back schema file - self.copy_file('test-files/model_three.sql', 'models/model_three.sql') - self.copy_file('test-files/models-schema2.yml', 'models/schema.yml') - self.rm_file('models/model_four.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # disable model three in the schema file - self.copy_file('test-files/models-schema4.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - - # update enabled config to be true for model three in the schema file - self.copy_file('test-files/models-schema4b.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # disable model three in the schema file again - self.copy_file('test-files/models-schema4.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - - # remove disabled config for model three in the schema file to check it gets enabled - self.copy_file('test-files/models-schema3.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Add a macro - self.copy_file('test-files/my_macro.sql', 'macros/my_macro.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - manifest = get_manifest() - macro_id = 'macro.test.do_something' - self.assertIn(macro_id, manifest.macros) - - # Modify the macro - self.copy_file('test-files/my_macro2.sql', 'macros/my_macro.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Add a macro patch - self.copy_file('test-files/models-schema3.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Remove the macro - self.rm_file('macros/my_macro.sql') - with self.assertRaises(CompilationError): - results = self.run_dbt(["--partial-parse", "--warn-error", "run"]) - - # put back macro file, got back to schema file with no macro - # add separate macro patch schema file - self.copy_file('test-files/models-schema2.yml', 'models/schema.yml') - self.copy_file('test-files/my_macro.sql', 'macros/my_macro.sql') - self.copy_file('test-files/macros.yml', 'macros/macros.yml') - results = self.run_dbt(["--partial-parse", "run"]) - - # delete macro and schema file - self.rm_file('macros/my_macro.sql') - self.rm_file('macros/macros.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Add an empty schema file - self.copy_file('test-files/empty_schema.yml', 'models/eschema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Add version to empty schema file - self.copy_file('test-files/empty_schema_with_version.yml', 'models/eschema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - - # Disable model_three - self.copy_file('test-files/model_three_disabled.sql', 'models/model_three.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - manifest = get_manifest() - model_id = 'model.test.model_three' - self.assertIn(model_id, manifest.disabled) - self.assertNotIn(model_id, manifest.nodes) - - # Edit disabled model three - self.copy_file('test-files/model_three_disabled2.sql', 'models/model_three.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - manifest = get_manifest() - model_id = 'model.test.model_three' - self.assertIn(model_id, manifest.disabled) - self.assertNotIn(model_id, manifest.nodes) - - # Remove disabled from model three - self.copy_file('test-files/model_three.sql', 'models/model_three.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - manifest = get_manifest() - model_id = 'model.test.model_three' - self.assertIn(model_id, manifest.nodes) - self.assertNotIn(model_id, manifest.disabled) - - -class TestSources(BasePPTest): - - @use_profile('postgres') - def test_postgres_pp_sources(self): - self.setup_directories() - # initial run - self.copy_file('test-files/model_one.sql', 'models/model_one.sql') - self.run_dbt(['clean']) - self.copy_file('test-files/raw_customers.csv', 'seeds/raw_customers.csv') - self.copy_file('test-files/sources-tests1.sql', 'macros/tests.sql') - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - - # Partial parse running 'seed' - self.run_dbt(['--partial-parse', 'seed']) - manifest = get_manifest() - seed_file_id = 'test://' + normalize('seeds/raw_customers.csv') - self.assertIn(seed_file_id, manifest.files) - - # Add another seed file - self.copy_file('test-files/raw_customers.csv', 'seeds/more_customers.csv') - self.run_dbt(['--partial-parse', 'run']) - seed_file_id = 'test://' + normalize('seeds/more_customers.csv') - manifest = get_manifest() - self.assertIn(seed_file_id, manifest.files) - seed_id = 'seed.test.more_customers' - self.assertIn(seed_id, manifest.nodes) - - # Remove seed file and add a schema files with a source referring to raw_customers - self.rm_file(normalize('seeds/more_customers.csv')) - self.copy_file('test-files/schema-sources1.yml', 'models/sources.yml') - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - self.assertEqual(len(manifest.sources), 1) - file_id = 'test://' + normalize('models/sources.yml') - self.assertIn(file_id, manifest.files) - - # add a model referring to raw_customers source - self.copy_file('test-files/customers.sql', 'models/customers.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - - # remove sources schema file - self.rm_file(normalize('models/sources.yml')) - with self.assertRaises(CompilationError): - results = self.run_dbt(["--partial-parse", "run"]) - - # put back sources and add an exposures file - self.copy_file('test-files/schema-sources2.yml', 'models/sources.yml') - results = self.run_dbt(["--partial-parse", "run"]) - - # remove seed referenced in exposures file - self.rm_file(normalize('seeds/raw_customers.csv')) - with self.assertRaises(CompilationError): - results = self.run_dbt(["--partial-parse", "run"]) - - # put back seed and remove depends_on from exposure - self.copy_file('test-files/raw_customers.csv', 'seeds/raw_customers.csv') - self.copy_file('test-files/schema-sources3.yml', 'models/sources.yml') - results = self.run_dbt(["--partial-parse", "run"]) - - # Add seed config with test to schema.yml, remove exposure - self.copy_file('test-files/schema-sources4.yml', 'models/sources.yml') - results = self.run_dbt(["--partial-parse", "run"]) - - # Change seed name to wrong name - self.copy_file('test-files/schema-sources5.yml', 'models/sources.yml') - with self.assertRaises(CompilationError): - results = self.run_dbt(["--partial-parse", "--warn-error", "run"]) - - # Put back seed name to right name - self.copy_file('test-files/schema-sources4.yml', 'models/sources.yml') - results = self.run_dbt(["--partial-parse", "run"]) - - # Add docs file customers.md - self.copy_file('test-files/customers1.md', 'models/customers.md') - results = self.run_dbt(["--partial-parse", "run"]) - - # Change docs file customers.md - self.copy_file('test-files/customers2.md', 'models/customers.md') - results = self.run_dbt(["--partial-parse", "run"]) - - # Delete docs file - self.rm_file(normalize('models/customers.md')) - results = self.run_dbt(["--partial-parse", "run"]) - - # Add a data test - self.copy_file('test-files/test-macro.sql', 'macros/test-macro.sql') - self.copy_file('test-files/my_test.sql', 'tests/my_test.sql') - results = self.run_dbt(["--partial-parse", "test"]) - manifest = get_manifest() - self.assertEqual(len(manifest.nodes), 9) - test_id = 'test.test.my_test' - self.assertIn(test_id, manifest.nodes) - - # Change macro that data test depends on - self.copy_file('test-files/test-macro2.sql', 'macros/test-macro.sql') - results = self.run_dbt(["--partial-parse", "test"]) - manifest = get_manifest() - - # Add an analysis - self.copy_file('test-files/my_analysis.sql', 'analyses/my_analysis.sql') - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - - # Remove data test - self.rm_file(normalize('tests/my_test.sql')) - results = self.run_dbt(["--partial-parse", "test"]) - manifest = get_manifest() - self.assertEqual(len(manifest.nodes), 9) - - # Remove analysis - self.rm_file(normalize('analyses/my_analysis.sql')) - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - self.assertEqual(len(manifest.nodes), 8) - - # Change source test - self.copy_file('test-files/sources-tests2.sql', 'macros/tests.sql') - results = self.run_dbt(["--partial-parse", "run"]) - - -class TestPartialParsingDependency(BasePPTest): - - @property - def packages_config(self): - return { - "packages": [ - { - 'local': 'local_dependency' - } - ] - } - - @use_profile("postgres") - def test_postgres_parsing_with_dependency(self): - self.setup_directories() - self.copy_file('test-files/model_one.sql', 'models/model_one.sql') - self.run_dbt(["clean"]) - self.run_dbt(["deps"]) - self.run_dbt(["seed"]) - self.run_dbt(["run"]) - - # Add a source override - self.copy_file('test-files/schema-models-c.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - manifest = get_manifest() - self.assertEqual(len(manifest.sources), 1) - source_id = 'source.local_dep.seed_source.seed' - self.assertIn(source_id, manifest.sources) - # We have 1 root model, 1 local_dep model, 1 local_dep seed, 1 local_dep source test, 2 root source tests - self.assertEqual(len(manifest.nodes), 5) - test_id = 'test.local_dep.source_unique_seed_source_seed_id.afa94935ed' - test_node = manifest.nodes[test_id] - - - # Remove a source override - self.rm_file(normalize('models/schema.yml')) - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - self.assertEqual(len(manifest.sources), 1) - - -class TestMacros(BasePPTest): - - @use_profile('postgres') - def test_postgres_nested_macros(self): - self.setup_directories() - self.copy_file('test-files/model_a.sql', 'models/model_a.sql') - self.copy_file('test-files/model_b.sql', 'models/model_b.sql') - self.copy_file('test-files/macros-schema.yml', 'models/schema.yml') - self.copy_file('test-files/custom_schema_tests1.sql', 'macros/custom_schema_tests.sql') - results = self.run_dbt() - self.assertEqual(len(results), 2) - manifest = get_manifest() - macro_child_map = manifest.build_macro_child_map() - macro_unique_id = 'macro.test.test_type_two' - - results = self.run_dbt(['test'], expect_pass=False) - results = sorted(results, key=lambda r: r.node.name) - self.assertEqual(len(results), 2) - # type_one_model_a_ - self.assertEqual(results[0].status, TestStatus.Fail) - self.assertRegex(results[0].node.compiled_code, r'union all') - # type_two_model_a_ - self.assertEqual(results[1].status, TestStatus.Warn) - self.assertEqual(results[1].node.config.severity, 'WARN') - - self.copy_file('test-files/custom_schema_tests2.sql', 'macros/custom_schema_tests.sql') - results = self.run_dbt(["--partial-parse", "test"], expect_pass=False) - manifest = get_manifest() - test_node_id = 'test.test.type_two_model_a_.842bc6c2a7' - self.assertIn(test_node_id, manifest.nodes) - results = sorted(results, key=lambda r: r.node.name) - self.assertEqual(len(results), 2) - # type_two_model_a_ - self.assertEqual(results[1].status, TestStatus.Fail) - self.assertEqual(results[1].node.config.severity, 'ERROR') - - @use_profile('postgres') - def test_postgres_skip_macros(self): - expected_special_override_macros = [ - 'ref', 'source', 'config', 'generate_schema_name', - 'generate_database_name', 'generate_alias_name' - ] - self.assertEqual(special_override_macros, expected_special_override_macros) - - # initial run so we have a msgpack file - self.setup_directories() - self.copy_file('test-files/model_one.sql', 'models/model_one.sql') - # use empty_schema file for bug #4850 - self.copy_file('test-files/empty_schema.yml', 'models/eschema.yml') - results = self.run_dbt() - - # add a new ref override macro - self.copy_file('test-files/ref_override.sql', 'macros/ref_override.sql') - results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run']) - self.assertTrue('Starting full parse.' in log_output) - - # modify a ref override macro - self.copy_file('test-files/ref_override2.sql', 'macros/ref_override.sql') - results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run']) - self.assertTrue('Starting full parse.' in log_output) - - # remove a ref override macro - self.rm_file(normalize('macros/ref_override.sql')) - results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run']) - self.assertTrue('Starting full parse.' in log_output) - - # custom generate_schema_name macro - self.copy_file('test-files/gsm_override.sql', 'macros/gsm_override.sql') - results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run']) - self.assertTrue('Starting full parse.' in log_output) - - # change generate_schema_name macro - self.copy_file('test-files/gsm_override2.sql', 'macros/gsm_override.sql') - results, log_output = self.run_dbt_and_capture(['--partial-parse', 'run']) - self.assertTrue('Starting full parse.' in log_output) - - -class TestSnapshots(BasePPTest): - - @use_profile('postgres') - def test_postgres_pp_snapshots(self): - - # initial run - self.setup_directories() - self.copy_file('test-files/orders.sql', 'models/orders.sql') - results = self.run_dbt() - self.assertEqual(len(results), 1) - - # add snapshot - self.copy_file('test-files/snapshot.sql', 'snapshots/snapshot.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - snapshot_id = 'snapshot.test.orders_snapshot' - self.assertIn(snapshot_id, manifest.nodes) - snapshot2_id = 'snapshot.test.orders2_snapshot' - self.assertIn(snapshot2_id, manifest.nodes) - - # run snapshot - results = self.run_dbt(["--partial-parse", "snapshot"]) - self.assertEqual(len(results), 2) - - # modify snapshot - self.copy_file('test-files/snapshot2.sql', 'snapshots/snapshot.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 1) - - # delete snapshot - self.rm_file(normalize('snapshots/snapshot.sql')) - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 1) - - -class TestTests(BasePPTest): - - @use_profile('postgres') - def test_postgres_pp_generic_tests(self): - - # initial run - self.setup_directories() - self.copy_file('test-files/orders.sql', 'models/orders.sql') - self.copy_file('test-files/generic_schema.yml', 'models/schema.yml') - results = self.run_dbt() - self.assertEqual(len(results), 1) - manifest = get_manifest() - expected_nodes = ['model.test.orders', 'test.test.unique_orders_id.1360ecc70e'] - self.assertCountEqual(expected_nodes, list(manifest.nodes.keys())) - - # add generic test in test-path - self.copy_file('test-files/generic_test.sql', 'tests/generic/generic_test.sql') - self.copy_file('test-files/generic_test_schema.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - test_id = 'test.test.is_odd_orders_id.82834fdc5b' - self.assertIn(test_id, manifest.nodes) - expected_nodes = ['model.test.orders', 'test.test.unique_orders_id.1360ecc70e', 'test.test.is_odd_orders_id.82834fdc5b'] - self.assertCountEqual(expected_nodes, list(manifest.nodes.keys())) - - # edit generic test in test-path - self.copy_file('test-files/generic_test_edited.sql', 'tests/generic/generic_test.sql') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - test_id = 'test.test.is_odd_orders_id.82834fdc5b' - self.assertIn(test_id, manifest.nodes) - expected_nodes = ['model.test.orders', 'test.test.unique_orders_id.1360ecc70e', 'test.test.is_odd_orders_id.82834fdc5b'] - self.assertCountEqual(expected_nodes, list(manifest.nodes.keys())) diff --git a/test/integration/068_partial_parsing_tests/test_pp_metrics.py b/test/integration/068_partial_parsing_tests/test_pp_metrics.py deleted file mode 100644 index 5debe6d2b85..00000000000 --- a/test/integration/068_partial_parsing_tests/test_pp_metrics.py +++ /dev/null @@ -1,106 +0,0 @@ -from dbt.exceptions import CompilationError -from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.files import ParseFileType -from dbt.contracts.results import TestStatus -from dbt.parser.partial import special_override_macros -from test.integration.base import DBTIntegrationTest, use_profile, normalize, get_manifest -import shutil -import os - - -# Note: every test case needs to have separate directories, otherwise -# they will interfere with each other when tests are multi-threaded - -class BasePPTest(DBTIntegrationTest): - - @property - def schema(self): - return "test_068A" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'data-paths': ['seeds'], - 'test-paths': ['tests'], - 'macro-paths': ['macros'], - 'analysis-paths': ['analyses'], - 'snapshot-paths': ['snapshots'], - 'seeds': { - 'quote_columns': False, - }, - } - - def setup_directories(self): - # Create the directories for the test in the `self.test_root_dir` - # directory after everything else is symlinked. We can copy to and - # delete files in this directory without tests interfering with each other. - os.mkdir(os.path.join(self.test_root_dir, 'models')) - os.mkdir(os.path.join(self.test_root_dir, 'tests')) - os.mkdir(os.path.join(self.test_root_dir, 'seeds')) - os.mkdir(os.path.join(self.test_root_dir, 'macros')) - os.mkdir(os.path.join(self.test_root_dir, 'analyses')) - os.mkdir(os.path.join(self.test_root_dir, 'snapshots')) - os.environ['DBT_PP_TEST'] = 'true' - - - -class MetricsTest(BasePPTest): - - @use_profile('postgres') - def test_postgres_metrics(self): - self.setup_directories() - # initial run - self.copy_file('test-files/people.sql', 'models/people.sql') - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - self.assertEqual(len(manifest.nodes), 1) - - # Add metrics yaml file - self.copy_file('test-files/people_metrics.yml', 'models/people_metrics.yml') - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - self.assertEqual(len(manifest.metrics), 2) - metric_people_id = 'metric.test.number_of_people' - metric_tenure_id = 'metric.test.collective_tenure' - metric_people = manifest.metrics[metric_people_id] - metric_tenure = manifest.metrics[metric_tenure_id] - expected_meta = {'my_meta': 'testing'} - self.assertEqual(metric_people.meta, expected_meta) - self.assertEqual(metric_people.refs, [['people']]) - self.assertEqual(metric_tenure.refs, [['people']]) - expected_depends_on_nodes = ['model.test.people'] - self.assertEqual(metric_people.depends_on.nodes, expected_depends_on_nodes) - - # Change metrics yaml files - self.copy_file('test-files/people_metrics2.yml', 'models/people_metrics.yml') - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - metric_people = manifest.metrics[metric_people_id] - expected_meta = {'my_meta': 'replaced'} - self.assertEqual(metric_people.meta, expected_meta) - expected_depends_on_nodes = ['model.test.people'] - self.assertEqual(metric_people.depends_on.nodes, expected_depends_on_nodes) - - # Add model referring to metric - self.copy_file('test-files/metric_model_a.sql', 'models/metric_model_a.sql') - results = self.run_dbt(["run"]) - manifest = get_manifest() - model_a = manifest.nodes['model.test.metric_model_a'] - expected_depends_on_nodes = ['metric.test.number_of_people', 'metric.test.collective_tenure'] - self.assertEqual(model_a.depends_on.nodes, expected_depends_on_nodes) - - # Then delete a metric - self.copy_file('test-files/people_metrics3.yml', 'models/people_metrics.yml') - with self.assertRaises(CompilationError): - # We use "parse" here and not "run" because we're checking that the CompilationError - # occurs at parse time, not compilation - results = self.run_dbt(["parse"]) - diff --git a/test/integration/068_partial_parsing_tests/test_pp_vars.py b/test/integration/068_partial_parsing_tests/test_pp_vars.py deleted file mode 100644 index a73bfc43fa3..00000000000 --- a/test/integration/068_partial_parsing_tests/test_pp_vars.py +++ /dev/null @@ -1,416 +0,0 @@ -from dbt.exceptions import ParsingError -from dbt.constants import SECRET_ENV_PREFIX -from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.files import ParseFileType -from dbt.contracts.results import TestStatus -from dbt.parser.partial import special_override_macros -from test.integration.base import DBTIntegrationTest, use_profile, normalize, get_manifest -import shutil -import os - - -# Note: every test case needs to have separate directories, otherwise -# they will interfere with each other when tests are multi-threaded - -class BasePPTest(DBTIntegrationTest): - - @property - def schema(self): - return "test_068A" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'test-paths': ['tests'], - 'macro-paths': ['macros'], - 'seeds': { - 'quote_columns': False, - }, - } - - def setup_directories(self): - # Create the directories for the test in the `self.test_root_dir` - # directory after everything else is symlinked. We can copy to and - # delete files in this directory without tests interfering with each other. - os.mkdir(os.path.join(self.test_root_dir, 'models')) - os.mkdir(os.path.join(self.test_root_dir, 'tests')) - os.mkdir(os.path.join(self.test_root_dir, 'macros')) - os.mkdir(os.path.join(self.test_root_dir, 'seeds')) - os.environ['DBT_PP_TEST'] = 'true' - - -class EnvVarTest(BasePPTest): - - @use_profile('postgres') - def test_postgres_env_vars_models(self): - self.setup_directories() - self.copy_file('test-files/model_color.sql', 'models/model_color.sql') - # initial run - self.run_dbt(['clean']) - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - - # copy a file with an env_var call without an env_var - self.copy_file('test-files/env_var_model.sql', 'models/env_var_model.sql') - with self.assertRaises(ParsingError): - results = self.run_dbt(["--partial-parse", "run"]) - - # set the env var - os.environ['ENV_VAR_TEST'] = 'TestingEnvVars' - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - manifest = get_manifest() - expected_env_vars = {"ENV_VAR_TEST": "TestingEnvVars"} - self.assertEqual(expected_env_vars, manifest.env_vars) - model_id = 'model.test.env_var_model' - model = manifest.nodes[model_id] - model_created_at = model.created_at - - # change the env var - os.environ['ENV_VAR_TEST'] = 'second' - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 2) - manifest = get_manifest() - expected_env_vars = {"ENV_VAR_TEST": "second"} - self.assertEqual(expected_env_vars, manifest.env_vars) - self.assertNotEqual(model_created_at, manifest.nodes[model_id].created_at) - - # set an env_var in a schema file - self.copy_file('test-files/env_var_schema.yml', 'models/schema.yml') - self.copy_file('test-files/env_var_model_one.sql', 'models/model_one.sql') - with self.assertRaises(ParsingError): - results = self.run_dbt(["--partial-parse", "run"]) - - # actually set the env_var - os.environ['TEST_SCHEMA_VAR'] = 'view' - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view"} - self.assertEqual(expected_env_vars, manifest.env_vars) - - # env vars in a source - os.environ['ENV_VAR_DATABASE'] = 'dbt' - os.environ['ENV_VAR_SEVERITY'] = 'warn' - self.copy_file('test-files/raw_customers.csv', 'seeds/raw_customers.csv') - self.copy_file('test-files/env_var-sources.yml', 'models/sources.yml') - self.run_dbt(['--partial-parse', 'seed']) - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - manifest = get_manifest() - expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "dbt", "ENV_VAR_SEVERITY": "warn"} - self.assertEqual(expected_env_vars, manifest.env_vars) - self.assertEqual(len(manifest.sources), 1) - source_id = 'source.test.seed_sources.raw_customers' - source = manifest.sources[source_id] - self.assertEqual(source.database, 'dbt') - schema_file = manifest.files[source.file_id] - test_id = 'test.test.source_not_null_seed_sources_raw_customers_id.e39ee7bf0d' - test_node = manifest.nodes[test_id] - self.assertEqual(test_node.config.severity, 'WARN') - - # Change severity env var - os.environ['ENV_VAR_SEVERITY'] = 'error' - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "dbt", "ENV_VAR_SEVERITY": "error"} - self.assertEqual(expected_env_vars, manifest.env_vars) - source_id = 'source.test.seed_sources.raw_customers' - source = manifest.sources[source_id] - schema_file = manifest.files[source.file_id] - expected_schema_file_env_vars = {'sources': {'seed_sources': ['ENV_VAR_DATABASE', 'ENV_VAR_SEVERITY']}} - self.assertEqual(expected_schema_file_env_vars, schema_file.env_vars) - test_node = manifest.nodes[test_id] - self.assertEqual(test_node.config.severity, 'ERROR') - - # Change database env var - os.environ['ENV_VAR_DATABASE'] = 'test_dbt' - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view", "ENV_VAR_DATABASE": "test_dbt", "ENV_VAR_SEVERITY": "error"} - self.assertEqual(expected_env_vars, manifest.env_vars) - source = manifest.sources[source_id] - self.assertEqual(source.database, 'test_dbt') - - # Delete database env var - del os.environ['ENV_VAR_DATABASE'] - with self.assertRaises(ParsingError): - results = self.run_dbt(["--partial-parse", "run"]) - os.environ['ENV_VAR_DATABASE'] = 'test_dbt' - - # Add generic test with test kwarg that's rendered late (no curly brackets) - os.environ['ENV_VAR_DATABASE'] = 'dbt' - self.copy_file('test-files/test_color.sql', 'macros/test_color.sql') - results = self.run_dbt(["--partial-parse", "run"]) - # Add source test using test_color and an env_var for color - self.copy_file('test-files/env_var_schema2.yml', 'models/schema.yml') - with self.assertRaises(ParsingError): - results = self.run_dbt(["--partial-parse", "run"]) - os.environ['ENV_VAR_COLOR'] = 'green' - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - test_color_id = 'test.test.check_color_model_one_env_var_ENV_VAR_COLOR___fun.89638de387' - test_node = manifest.nodes[test_color_id] - # kwarg was rendered but not changed (it will be rendered again when compiled) - self.assertEqual(test_node.test_metadata.kwargs['color'], "env_var('ENV_VAR_COLOR')") - results = self.run_dbt(["--partial-parse", "test"]) - - # Add an exposure with an env_var - os.environ['ENV_VAR_OWNER'] = "John Doe" - self.copy_file('test-files/env_var_schema3.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - "ENV_VAR_COLOR": 'green', - "ENV_VAR_OWNER": "John Doe", - } - self.assertEqual(expected_env_vars, manifest.env_vars) - exposure = list(manifest.exposures.values())[0] - schema_file = manifest.files[exposure.file_id] - expected_sf_env_vars = { - 'models': { - 'model_one': ['TEST_SCHEMA_VAR', 'ENV_VAR_COLOR'] - }, - 'exposures': { - 'proxy_for_dashboard': ['ENV_VAR_OWNER'] - } - } - self.assertEqual(expected_sf_env_vars, schema_file.env_vars) - - # add a macro and a macro schema file - os.environ['ENV_VAR_SOME_KEY'] = 'toodles' - self.copy_file('test-files/env_var_macro.sql', 'macros/env_var_macro.sql') - self.copy_file('test-files/env_var_macros.yml', 'macros/env_var_macros.yml') - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - "ENV_VAR_COLOR": 'green', - "ENV_VAR_OWNER": "John Doe", - "ENV_VAR_SOME_KEY": "toodles", - } - self.assertEqual(expected_env_vars, manifest.env_vars) - macro_id = 'macro.test.do_something' - macro = manifest.macros[macro_id] - self.assertEqual(macro.meta, {"some_key": "toodles"}) - # change the env var - os.environ['ENV_VAR_SOME_KEY'] = 'dumdedum' - results = self.run_dbt(["--partial-parse", "run"]) - manifest = get_manifest() - macro = manifest.macros[macro_id] - self.assertEqual(macro.meta, {"some_key": "dumdedum"}) - - # Add a schema file with a test on model_color and env_var in test enabled config - self.copy_file('test-files/env_var_model_test.yml', 'models/schema.yml') - results = self.run_dbt(["--partial-parse", "run"]) - self.assertEqual(len(results), 3) - manifest = get_manifest() - model_color = manifest.nodes['model.test.model_color'] - schema_file = manifest.files[model_color.patch_path] - expected_env_vars = {'models': {'model_one': ['TEST_SCHEMA_VAR', 'ENV_VAR_COLOR'], 'model_color': ['ENV_VAR_ENABLED']}, 'exposures': {'proxy_for_dashboard': ['ENV_VAR_OWNER']}} - self.assertEqual(expected_env_vars, schema_file.env_vars) - - # Add a metrics file with env_vars - os.environ['ENV_VAR_METRICS'] = 'TeStInG' - self.copy_file('test-files/people.sql', 'models/people.sql') - self.copy_file('test-files/env_var_metrics.yml', 'models/metrics.yml') - results = self.run_dbt(["run"]) - manifest = get_manifest() - self.assertIn('ENV_VAR_METRICS', manifest.env_vars) - self.assertEqual(manifest.env_vars['ENV_VAR_METRICS'], 'TeStInG') - metric_node = manifest.metrics['metric.test.number_of_people'] - self.assertEqual(metric_node.meta, {'my_meta': 'TeStInG'}) - - # Change metrics env var - os.environ['ENV_VAR_METRICS'] = 'Changed!' - results = self.run_dbt(["run"]) - manifest = get_manifest() - metric_node = manifest.metrics['metric.test.number_of_people'] - self.assertEqual(metric_node.meta, {'my_meta': 'Changed!'}) - - # delete the env vars to cleanup - del os.environ['ENV_VAR_TEST'] - del os.environ['ENV_VAR_SEVERITY'] - del os.environ['ENV_VAR_DATABASE'] - del os.environ['TEST_SCHEMA_VAR'] - del os.environ['ENV_VAR_COLOR'] - del os.environ['ENV_VAR_SOME_KEY'] - del os.environ['ENV_VAR_OWNER'] - del os.environ['ENV_VAR_METRICS'] - - -class ProjectEnvVarTest(BasePPTest): - - @property - def project_config(self): - # Need to set the environment variable here initially because - # the unittest setup does a load_config. - os.environ['ENV_VAR_NAME'] = "Jane Smith" - return { - 'config-version': 2, - 'seed-paths': ['seeds'], - 'test-paths': ['tests'], - 'macro-paths': ['macros'], - 'seeds': { - 'quote_columns': False, - }, - 'models': { - '+meta': { - 'meta_name': "{{ env_var('ENV_VAR_NAME') }}" - } - } - } - - @use_profile('postgres') - def test_postgres_project_env_vars(self): - - # Initial run - self.setup_directories() - self.copy_file('test-files/model_one.sql', 'models/model_one.sql') - self.run_dbt(['clean']) - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - state_check = manifest.state_check - model_id = 'model.test.model_one' - model = manifest.nodes[model_id] - self.assertEqual(model.config.meta['meta_name'], 'Jane Smith') - env_vars_hash_checksum = state_check.project_env_vars_hash.checksum - - # Change the environment variable - os.environ['ENV_VAR_NAME'] = "Jane Doe" - results = self.run_dbt(["run"]) - self.assertEqual(len(results), 1) - manifest = get_manifest() - model = manifest.nodes[model_id] - self.assertEqual(model.config.meta['meta_name'], 'Jane Doe') - self.assertNotEqual(env_vars_hash_checksum, manifest.state_check.project_env_vars_hash.checksum) - - # cleanup - del os.environ['ENV_VAR_NAME'] - - -class ProfileEnvVarTest(BasePPTest): - - @property - def profile_config(self): - # Need to set these here because the base integration test class - # calls 'load_config' before the tests are run. - # Note: only the specified profile is rendered, so there's no - # point it setting env_vars in non-used profiles. - os.environ['ENV_VAR_USER'] = 'root' - os.environ['ENV_VAR_PASS'] = 'password' - return { - 'config': { - 'send_anonymous_usage_stats': False - }, - 'test': { - 'outputs': { - 'dev': { - 'type': 'postgres', - 'threads': 1, - 'host': self.database_host, - 'port': 5432, - 'user': "root", - 'pass': "password", - 'user': "{{ env_var('ENV_VAR_USER') }}", - 'pass': "{{ env_var('ENV_VAR_PASS') }}", - 'dbname': 'dbt', - 'schema': self.unique_schema() - }, - }, - 'target': 'dev' - } - } - - @use_profile('postgres') - def test_postgres_profile_env_vars(self): - - # Initial run - os.environ['ENV_VAR_USER'] = 'root' - os.environ['ENV_VAR_PASS'] = 'password' - self.setup_directories() - self.copy_file('test-files/model_one.sql', 'models/model_one.sql') - results = self.run_dbt(["run"]) - manifest = get_manifest() - env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum - - # Change env_vars, the user doesn't exist, this should fail - os.environ['ENV_VAR_USER'] = 'fake_user' - (results, log_output) = self.run_dbt_and_capture(["run"], expect_pass=False) - self.assertTrue('env vars used in profiles.yml have changed' in log_output) - manifest = get_manifest() - self.assertNotEqual(env_vars_checksum, manifest.state_check.profile_env_vars_hash.checksum) - - -class ProfileSecretEnvVarTest(BasePPTest): - - @property - def profile_config(self): - # Need to set these here because the base integration test class - # calls 'load_config' before the tests are run. - # Note: only the specified profile is rendered, so there's no - # point it setting env_vars in non-used profiles. - - # user is secret and password is not. postgres on macos doesn't care if the password - # changes so we have to change the user. related: https://github.com/dbt-labs/dbt-core/pull/4250 - os.environ[SECRET_ENV_PREFIX + 'USER'] = 'root' - os.environ['ENV_VAR_PASS'] = 'password' - return { - 'config': { - 'send_anonymous_usage_stats': False - }, - 'test': { - 'outputs': { - 'dev': { - 'type': 'postgres', - 'threads': 1, - 'host': self.database_host, - 'port': 5432, - 'user': "root", - 'pass': "password", - 'user': "{{ env_var('DBT_ENV_SECRET_USER') }}", - 'pass': "{{ env_var('ENV_VAR_PASS') }}", - 'dbname': 'dbt', - 'schema': self.unique_schema() - }, - }, - 'target': 'dev' - } - } - - @use_profile('postgres') - def test_postgres_profile_secret_env_vars(self): - - # Initial run - os.environ[SECRET_ENV_PREFIX + 'USER'] = 'root' - os.environ['ENV_VAR_PASS'] = 'password' - self.setup_directories() - self.copy_file('test-files/model_one.sql', 'models/model_one.sql') - results = self.run_dbt(["run"]) - manifest = get_manifest() - env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum - - # Change a secret var, it shouldn't register because we shouldn't save secrets. - os.environ[SECRET_ENV_PREFIX + 'USER'] = 'boop' - # this dbt run is going to fail because the password isn't actually the right one, - # but that doesn't matter because we just want to see if the manifest has included - # the secret in the hash of environment variables. - (results, log_output) = self.run_dbt_and_capture(["run"], expect_pass=False) - # I020 is the event code for "env vars used in profiles.yml have changed" - self.assertFalse('I020' in log_output) - manifest = get_manifest() - self.assertEqual(env_vars_checksum, manifest.state_check.profile_env_vars_hash.checksum) - diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py new file mode 100644 index 00000000000..7681b9dcb8c --- /dev/null +++ b/tests/functional/partial_parsing/fixtures.py @@ -0,0 +1,1126 @@ +local_dependency__dbt_project_yml = """ + +name: 'local_dep' +version: '1.0' +config-version: 2 + +profile: 'default' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] + +require-dbt-version: '>=0.1.0' + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +seeds: + quote_columns: False + +""" + +local_dependency__models__schema_yml = """ +version: 2 +sources: + - name: seed_source + schema: "{{ var('schema_override', target.schema) }}" + tables: + - name: "seed" + columns: + - name: id + tests: + - unique + +""" + +local_dependency__models__model_to_import_sql = """ +select * from {{ ref('seed') }} + +""" + +local_dependency__macros__dep_macro_sql = """ +{% macro some_overridden_macro() -%} +100 +{%- endmacro %} + +""" + +local_dependency__seeds__seed_csv = """id +1 +""" + +empty_schema_with_version_yml = """ +version: 2 + +""" + +schema_sources5_yml = """ +version: 2 + +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_customers + columns: + - name: id + tests: + - not_null: + severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" + - unique + - name: first_name + - name: last_name + - name: email + +seeds: + - name: rad_customers + description: "Raw customer data" + columns: + - name: id + tests: + - unique + - not_null + - name: first_name + - name: last_name + - name: email + + +""" + +my_macro2_sql = """ +{% macro do_something(foo2, bar2) %} + + select + 'foo' as foo2, + 'var' as bar2 + +{% endmacro %} + +""" + +raw_customers_csv = """id,first_name,last_name,email +1,Michael,Perez,mperez0@chronoengine.com +2,Shawn,Mccoy,smccoy1@reddit.com +3,Kathleen,Payne,kpayne2@cargocollective.com +4,Jimmy,Cooper,jcooper3@cargocollective.com +5,Katherine,Rice,krice4@typepad.com +6,Sarah,Ryan,sryan5@gnu.org +7,Martin,Mcdonald,mmcdonald6@opera.com +8,Frank,Robinson,frobinson7@wunderground.com +9,Jennifer,Franklin,jfranklin8@mail.ru +10,Henry,Welch,hwelch9@list-manage.com +""" + +model_three_disabled2_sql = """ +- Disabled model +{{ config(materialized='table', enabled=False) }} + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +""" + +schema_sources4_yml = """ +version: 2 + +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_customers + columns: + - name: id + tests: + - not_null: + severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" + - unique + - every_value_is_blue + - name: first_name + - name: last_name + - name: email + +seeds: + - name: raw_customers + description: "Raw customer data" + columns: + - name: id + tests: + - unique + - not_null + - name: first_name + - name: last_name + - name: email + + +""" + +env_var_schema_yml = """ +version: 2 + +models: + - name: model_one + config: + materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" + +""" + +my_test_sql = """ +select + * from {{ ref('customers') }} where first_name = '{{ macro_something() }}' + +""" + +empty_schema_yml = """ + +""" + +schema_models_c_yml = """ +version: 2 + +sources: + - name: seed_source + description: "This is a source override" + overrides: local_dep + schema: "{{ var('schema_override', target.schema) }}" + tables: + - name: "seed" + columns: + - name: id + tests: + - unique + - not_null + +""" + +env_var_sources_yml = """ +version: 2 +sources: + - name: seed_sources + schema: "{{ target.schema }}" + database: "{{ env_var('ENV_VAR_DATABASE') }}" + tables: + - name: raw_customers + columns: + - name: id + tests: + - not_null: + severity: "{{ env_var('ENV_VAR_SEVERITY') }}" + - unique + - name: first_name + - name: last_name + - name: email + + + +""" + +generic_test_edited_sql = """ +{% test is_odd(model, column_name) %} + +with validation as ( + + select + {{ column_name }} as odd_field2 + + from {{ model }} + +), + +validation_errors as ( + + select + odd_field2 + + from validation + -- if this is true, then odd_field is actually even! + where (odd_field2 % 2) = 0 + +) + +select * +from validation_errors + +{% endtest %} +""" + +schema_sources1_yml = """ +version: 2 +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_customers + columns: + - name: id + tests: + - not_null: + severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" + - unique + - name: first_name + - name: last_name + - name: email + + + +""" + +schema_sources3_yml = """ +version: 2 + +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_customers + columns: + - name: id + tests: + - not_null: + severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" + - unique + - name: first_name + - name: last_name + - name: email + +exposures: + - name: proxy_for_dashboard + description: "This is for the XXX dashboard" + type: "dashboard" + owner: + name: "Dashboard Tester" + email: "tester@dashboard.com" + depends_on: + - ref("model_one") + - source("seed_sources", "raw_customers") + + +""" + +my_analysis_sql = """ +select * from customers + +""" + +schema_sources2_yml = """ +version: 2 + +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_customers + columns: + - name: id + tests: + - not_null: + severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" + - unique + - name: first_name + - name: last_name + - name: email + +exposures: + - name: proxy_for_dashboard + description: "This is for the XXX dashboard" + type: "dashboard" + owner: + name: "Dashboard Tester" + email: "tester@dashboard.com" + depends_on: + - ref("model_one") + - ref("raw_customers") + - source("seed_sources", "raw_customers") + + +""" + +model_color_sql = """ +select 'blue' as fun + +""" + +my_metric_yml = """ +version: 2 +metrics: + - name: new_customers + label: New Customers + model: customers + description: "The number of paid customers who are using the product" + calculation_method: count + expression: user_id + timestamp: signup_date + time_grains: [day, week, month] + dimensions: + - plan + - country + filters: + - field: is_paying + value: True + operator: '=' + +meta: + is_okr: True + tags: + - okrs + + + +""" + +env_var_schema2_yml = """ +version: 2 + +models: + - name: model_one + config: + materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" + tests: + - check_color: + column_name: fun + color: "env_var('ENV_VAR_COLOR')" + + +""" + +gsm_override_sql = """ +- custom macro +{% macro generate_schema_name(schema_name, node) %} + + {{ schema_name }}_{{ target.schema }} + +{% endmacro %} + +""" + +model_four1_sql = """ +select * from {{ ref('model_three') }} + +""" + +model_one_sql = """ +select 1 as fun + +""" + +env_var_schema3_yml = """ +version: 2 + +models: + - name: model_one + config: + materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" + tests: + - check_color: + column_name: fun + color: "env_var('ENV_VAR_COLOR')" + +exposures: + - name: proxy_for_dashboard + description: "This is for the XXX dashboard" + type: "dashboard" + owner: + name: "{{ env_var('ENV_VAR_OWNER') }}" + email: "tester@dashboard.com" + depends_on: + - ref("model_color") + - source("seed_sources", "raw_customers") + +""" + +env_var_metrics_yml = """ +version: 2 + +metrics: + + - model: "ref('people')" + name: number_of_people + description: Total count of people + label: "Number of people" + calculation_method: count + expression: "*" + timestamp: created_at + time_grains: [day, week, month] + dimensions: + - favorite_color + - loves_dbt + meta: + my_meta: '{{ env_var("ENV_VAR_METRICS") }}' + + - model: "ref('people')" + name: collective_tenure + description: Total number of years of team experience + label: "Collective tenure" + calculation_method: sum + expression: tenure + timestamp: created_at + time_grains: [day] + filters: + - field: loves_dbt + operator: is + value: 'true' + +""" + +customers_sql = """ +with source as ( + + select * from {{ source('seed_sources', 'raw_customers') }} + +), + +renamed as ( + + select + id as customer_id, + first_name, + last_name, + email + + from source + +) + +select * from renamed + +""" + +model_four2_sql = """ +select fun from {{ ref('model_one') }} + +""" + +env_var_model_sql = """ +select '{{ env_var('ENV_VAR_TEST') }}' as vartest + +""" + +env_var_model_one_sql = """ +select 'blue' as fun + +""" + +custom_schema_tests2_sql = """ +{% test type_one(model) %} + + select * from ( + + select * from {{ model }} + union all + select * from {{ ref('model_b') }} + + ) as Foo + +{% endtest %} + +{% test type_two(model) %} + + {{ config(severity = "ERROR") }} + + select * from {{ model }} + +{% endtest %} + +""" + +metric_model_a_sql = """ +{% + set metric_list = [ + metric('number_of_people'), + metric('collective_tenure') + ] +%} + +{% if not execute %} + + {% set metric_names = [] %} + {% for m in metric_list %} + {% do metric_names.append(m.metric_name) %} + {% endfor %} + + -- this config does nothing, but it lets us check these values + {{ config(metric_names = metric_names) }} + +{% endif %} + + +select 1 as fun + +""" + +model_b_sql = """ +select 1 as notfun + +""" + +customers2_md = """ +{% docs customer_table %} + +LOTS of customer data + +{% enddocs %} + +""" + +custom_schema_tests1_sql = """ +{% test type_one(model) %} + + select * from ( + + select * from {{ model }} + union all + select * from {{ ref('model_b') }} + + ) as Foo + +{% endtest %} + +{% test type_two(model) %} + + {{ config(severity = "WARN") }} + + select * from {{ model }} + +{% endtest %} + +""" + +people_metrics_yml = """ +version: 2 + +metrics: + + - model: "ref('people')" + name: number_of_people + description: Total count of people + label: "Number of people" + calculation_method: count + expression: "*" + timestamp: created_at + time_grains: [day, week, month] + dimensions: + - favorite_color + - loves_dbt + meta: + my_meta: 'testing' + + - model: "ref('people')" + name: collective_tenure + description: Total number of years of team experience + label: "Collective tenure" + calculation_method: sum + expression: tenure + timestamp: created_at + time_grains: [day] + filters: + - field: loves_dbt + operator: is + value: 'true' + +""" + +people_sql = """ +select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at +union all +select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at + +""" + +orders_sql = """ +select 1 as id, 101 as user_id, 'pending' as status + +""" + +model_a_sql = """ +select 1 as fun + +""" + +model_three_disabled_sql = """ +{{ config(materialized='table', enabled=False) }} + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +""" + +models_schema2b_yml = """ +version: 2 + +models: + - name: model_one + description: "The first model" + - name: model_three + description: "The third model" + columns: + - name: id + tests: + - not_null + +""" + +env_var_macros_yml = """ +version: 2 +macros: + - name: do_something + description: "This is a test macro" + meta: + some_key: "{{ env_var('ENV_VAR_SOME_KEY') }}" + + +""" + +models_schema4_yml = """ +version: 2 + +models: + - name: model_one + description: "The first model" + - name: model_three + description: "The third model" + config: + enabled: false + columns: + - name: id + tests: + - unique + +""" + +model_two_sql = """ +select 1 as notfun + +""" + +generic_test_schema_yml = """ +version: 2 + +models: + - name: orders + description: "Some order data" + columns: + - name: id + tests: + - unique + - is_odd + +""" + +customers1_md = """ +{% docs customer_table %} + +This table contains customer data + +{% enddocs %} + +""" + +model_three_modified_sql = """ +{{ config(materialized='table') }} + +with source_data as ( + + {#- This is model three #} + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +""" + +macros_yml = """ +version: 2 +macros: + - name: do_something + description: "This is a test macro" + +""" + +test_color_sql = """ +{% test check_color(model, column_name, color) %} + + select * + from {{ model }} + where {{ column_name }} = '{{ color }}' + +{% endtest %} + +""" + +models_schema2_yml = """ +version: 2 + +models: + - name: model_one + description: "The first model" + - name: model_three + description: "The third model" + columns: + - name: id + tests: + - unique + +""" + +gsm_override2_sql = """ +- custom macro xxxx +{% macro generate_schema_name(schema_name, node) %} + + {{ schema_name }}_{{ target.schema }} + +{% endmacro %} + +""" + +models_schema3_yml = """ +version: 2 + +models: + - name: model_one + description: "The first model" + - name: model_three + description: "The third model" + tests: + - unique +macros: + - name: do_something + description: "This is a test macro" + +""" + +generic_test_sql = """ +{% test is_odd(model, column_name) %} + +with validation as ( + + select + {{ column_name }} as odd_field + + from {{ model }} + +), + +validation_errors as ( + + select + odd_field + + from validation + -- if this is true, then odd_field is actually even! + where (odd_field % 2) = 0 + +) + +select * +from validation_errors + +{% endtest %} +""" + +env_var_model_test_yml = """ +version: 2 +models: + - name: model_color + columns: + - name: fun + tests: + - unique: + enabled: "{{ env_var('ENV_VAR_ENABLED', True) }}" + +""" + +model_three_sql = """ +{{ config(materialized='table') }} + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +""" + +ref_override2_sql = """ +- Macro to override ref xxxx +{% macro ref(modelname) %} +{% do return(builtins.ref(modelname)) %} +{% endmacro %} + +""" + +models_schema1_yml = """ +version: 2 + +models: + - name: model_one + description: "The first model" + +""" + +macros_schema_yml = """ + +version: 2 + +models: + - name: model_a + tests: + - type_one + - type_two + +""" + +my_macro_sql = """ +{% macro do_something(foo2, bar2) %} + + select + '{{ foo2 }}' as foo2, + '{{ bar2 }}' as bar2 + +{% endmacro %} + +""" + +snapshot_sql = """ +{% snapshot orders_snapshot %} + +{{ + config( + target_schema=schema, + strategy='check', + unique_key='id', + check_cols=['status'], + ) +}} + +select * from {{ ref('orders') }} + +{% endsnapshot %} + +{% snapshot orders2_snapshot %} + +{{ + config( + target_schema=schema, + strategy='check', + unique_key='id', + check_cols=['order_date'], + ) +}} + +select * from {{ ref('orders') }} + +{% endsnapshot %} + +""" + +models_schema4b_yml = """ +version: 2 + +models: + - name: model_one + description: "The first model" + - name: model_three + description: "The third model" + config: + enabled: true + columns: + - name: id + tests: + - unique + +""" + +test_macro_sql = """ +{% macro macro_something() %} + + {% do return('macro_something') %} + +{% endmacro %} + +""" + +people_metrics2_yml = """ +version: 2 + +metrics: + + - model: "ref('people')" + name: number_of_people + description: Total count of people + label: "Number of people" + calculation_method: count + expression: "*" + timestamp: created_at + time_grains: [day, week, month] + dimensions: + - favorite_color + - loves_dbt + meta: + my_meta: 'replaced' + + - model: "ref('people')" + name: collective_tenure + description: Total number of years of team experience + label: "Collective tenure" + calculation_method: sum + expression: tenure + timestamp: created_at + time_grains: [day] + filters: + - field: loves_dbt + operator: is + value: 'true' + +""" + +generic_schema_yml = """ +version: 2 + +models: + - name: orders + description: "Some order data" + columns: + - name: id + tests: + - unique + +""" + +snapshot2_sql = """ +- add a comment +{% snapshot orders_snapshot %} + +{{ + config( + target_schema=schema, + strategy='check', + unique_key='id', + check_cols=['status'], + ) +}} + +select * from {{ ref('orders') }} + +{% endsnapshot %} + +{% snapshot orders2_snapshot %} + +{{ + config( + target_schema=schema, + strategy='check', + unique_key='id', + check_cols=['order_date'], + ) +}} + +select * from {{ ref('orders') }} + +{% endsnapshot %} + +""" + +sources_tests2_sql = """ + +{% test every_value_is_blue(model, column_name) %} + + select * + from {{ model }} + where {{ column_name }} != 99 + +{% endtest %} + + +""" + +people_metrics3_yml = """ +version: 2 + +metrics: + + - model: "ref('people')" + name: number_of_people + description: Total count of people + label: "Number of people" + calculation_method: count + expression: "*" + timestamp: created_at + time_grains: [day, week, month] + dimensions: + - favorite_color + - loves_dbt + meta: + my_meta: 'replaced' + +""" + +ref_override_sql = """ +- Macro to override ref +{% macro ref(modelname) %} +{% do return(builtins.ref(modelname)) %} +{% endmacro %} + +""" + +test_macro2_sql = """ +{% macro macro_something() %} + + {% do return('some_name') %} + +{% endmacro %} + +""" + +env_var_macro_sql = """ +{% macro do_something(foo2, bar2) %} + + select + '{{ foo2 }}' as foo2, + '{{ bar2 }}' as bar2 + +{% endmacro %} + +""" + +sources_tests1_sql = """ + +{% test every_value_is_blue(model, column_name) %} + + select * + from {{ model }} + where {{ column_name }} = 9999 + +{% endtest %} + + +""" diff --git a/tests/functional/partial_parsing/test_partial_parsing.py b/tests/functional/partial_parsing/test_partial_parsing.py new file mode 100644 index 00000000000..f70b2e0f9fa --- /dev/null +++ b/tests/functional/partial_parsing/test_partial_parsing.py @@ -0,0 +1,643 @@ +import pytest + +from dbt.tests.util import run_dbt, get_manifest, write_file, rm_file, run_dbt_and_capture +from dbt.tests.fixtures.project import write_project_files +from tests.functional.partial_parsing.fixtures import ( + model_one_sql, + model_two_sql, + models_schema1_yml, + models_schema2_yml, + models_schema2b_yml, + model_three_sql, + model_three_modified_sql, + model_four1_sql, + model_four2_sql, + models_schema4_yml, + models_schema4b_yml, + models_schema3_yml, + my_macro_sql, + my_macro2_sql, + macros_yml, + empty_schema_yml, + empty_schema_with_version_yml, + model_three_disabled_sql, + model_three_disabled2_sql, + raw_customers_csv, + customers_sql, + sources_tests1_sql, + schema_sources1_yml, + schema_sources2_yml, + schema_sources3_yml, + schema_sources4_yml, + schema_sources5_yml, + customers1_md, + customers2_md, + test_macro_sql, + my_test_sql, + test_macro2_sql, + my_analysis_sql, + sources_tests2_sql, + local_dependency__dbt_project_yml, + local_dependency__models__schema_yml, + local_dependency__models__model_to_import_sql, + local_dependency__macros__dep_macro_sql, + local_dependency__seeds__seed_csv, + schema_models_c_yml, + model_a_sql, + model_b_sql, + macros_schema_yml, + custom_schema_tests1_sql, + custom_schema_tests2_sql, + ref_override_sql, + ref_override2_sql, + gsm_override_sql, + gsm_override2_sql, + orders_sql, + snapshot_sql, + snapshot2_sql, + generic_schema_yml, + generic_test_sql, + generic_test_schema_yml, + generic_test_edited_sql, +) + +from dbt.exceptions import CompilationError +from dbt.contracts.files import ParseFileType +from dbt.contracts.results import TestStatus +import re +import os + +os.environ["DBT_PP_TEST"] = "true" + + +def normalize(path): + return os.path.normcase(os.path.normpath(path)) + + +class TestModels: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + def test_pp_models(self, project): + # initial run + # run_dbt(['clean']) + results = run_dbt(["run"]) + assert len(results) == 1 + + # add a model file + write_file(model_two_sql, project.project_root, "models", "model_two.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + + # add a schema file + write_file(models_schema1_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + manifest = get_manifest(project.project_root) + assert "model.test.model_one" in manifest.nodes + model_one_node = manifest.nodes["model.test.model_one"] + assert model_one_node.description == "The first model" + assert model_one_node.patch_path == "test://" + normalize("models/schema.yml") + + # add a model and a schema file (with a test) at the same time + write_file(models_schema2_yml, project.project_root, "models", "schema.yml") + write_file(model_three_sql, project.project_root, "models", "model_three.sql") + results = run_dbt(["--partial-parse", "test"], expect_pass=False) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + project_files = [f for f in manifest.files if f.startswith("test://")] + assert len(project_files) == 4 + model_3_file_id = "test://" + normalize("models/model_three.sql") + assert model_3_file_id in manifest.files + model_three_file = manifest.files[model_3_file_id] + assert model_three_file.parse_file_type == ParseFileType.Model + assert type(model_three_file).__name__ == "SourceFile" + model_three_node = manifest.nodes[model_three_file.nodes[0]] + schema_file_id = "test://" + normalize("models/schema.yml") + assert model_three_node.patch_path == schema_file_id + assert model_three_node.description == "The third model" + schema_file = manifest.files[schema_file_id] + assert type(schema_file).__name__ == "SchemaSourceFile" + assert len(schema_file.tests) == 1 + tests = schema_file.get_all_test_ids() + assert tests == ["test.test.unique_model_three_id.6776ac8160"] + unique_test_id = tests[0] + assert unique_test_id in manifest.nodes + + # modify model sql file, ensure description still there + write_file(model_three_modified_sql, project.project_root, "models", "model_three.sql") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + model_id = "model.test.model_three" + assert model_id in manifest.nodes + model_three_node = manifest.nodes[model_id] + assert model_three_node.description == "The third model" + + # Change the model 3 test from unique to not_null + write_file(models_schema2b_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "test"], expect_pass=False) + manifest = get_manifest(project.project_root) + schema_file_id = "test://" + normalize("models/schema.yml") + schema_file = manifest.files[schema_file_id] + tests = schema_file.get_all_test_ids() + assert tests == ["test.test.not_null_model_three_id.3162ce0a6f"] + not_null_test_id = tests[0] + assert not_null_test_id in manifest.nodes.keys() + assert unique_test_id not in manifest.nodes.keys() + assert len(results) == 1 + + # go back to previous version of schema file, removing patch, test, and model for model three + write_file(models_schema1_yml, project.project_root, "models", "schema.yml") + rm_file(project.project_root, "models", "model_three.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + + # remove schema file, still have 3 models + write_file(model_three_sql, project.project_root, "models", "model_three.sql") + rm_file(project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + manifest = get_manifest(project.project_root) + schema_file_id = "test://" + normalize("models/schema.yml") + assert schema_file_id not in manifest.files + project_files = [f for f in manifest.files if f.startswith("test://")] + assert len(project_files) == 3 + + # Put schema file back and remove a model + # referred to in schema file + write_file(models_schema2_yml, project.project_root, "models", "schema.yml") + rm_file(project.project_root, "models", "model_three.sql") + with pytest.raises(CompilationError): + results = run_dbt(["--partial-parse", "--warn-error", "run"]) + + # Put model back again + write_file(model_three_sql, project.project_root, "models", "model_three.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Add model four refing model three + write_file(model_four1_sql, project.project_root, "models", "model_four.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 4 + + # Remove model_three and change model_four to ref model_one + # and change schema file to remove model_three + rm_file(project.project_root, "models", "model_three.sql") + write_file(model_four2_sql, project.project_root, "models", "model_four.sql") + write_file(models_schema1_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Remove model four, put back model three, put back schema file + write_file(model_three_sql, project.project_root, "models", "model_three.sql") + write_file(models_schema2_yml, project.project_root, "models", "schema.yml") + rm_file(project.project_root, "models", "model_four.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # disable model three in the schema file + write_file(models_schema4_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + + # update enabled config to be true for model three in the schema file + write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # disable model three in the schema file again + write_file(models_schema4_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + + # remove disabled config for model three in the schema file to check it gets enabled + write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Add a macro + write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + manifest = get_manifest(project.project_root) + macro_id = "macro.test.do_something" + assert macro_id in manifest.macros + + # Modify the macro + write_file(my_macro2_sql, project.project_root, "macros", "my_macro.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Add a macro patch + write_file(models_schema3_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Remove the macro + rm_file(project.project_root, "macros", "my_macro.sql") + with pytest.raises(CompilationError): + results = run_dbt(["--partial-parse", "--warn-error", "run"]) + + # put back macro file, got back to schema file with no macro + # add separate macro patch schema file + write_file(models_schema2_yml, project.project_root, "models", "schema.yml") + write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") + write_file(macros_yml, project.project_root, "macros", "macros.yml") + results = run_dbt(["--partial-parse", "run"]) + + # delete macro and schema file + rm_file(project.project_root, "macros", "my_macro.sql") + rm_file(project.project_root, "macros", "macros.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Add an empty schema file + write_file(empty_schema_yml, project.project_root, "models", "eschema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Add version to empty schema file + write_file(empty_schema_with_version_yml, project.project_root, "models", "eschema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + + # Disable model_three + write_file(model_three_disabled_sql, project.project_root, "models", "model_three.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + manifest = get_manifest(project.project_root) + model_id = "model.test.model_three" + assert model_id in manifest.disabled + assert model_id not in manifest.nodes + + # Edit disabled model three + write_file(model_three_disabled2_sql, project.project_root, "models", "model_three.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + manifest = get_manifest(project.project_root) + model_id = "model.test.model_three" + assert model_id in manifest.disabled + assert model_id not in manifest.nodes + + # Remove disabled from model three + write_file(model_three_sql, project.project_root, "models", "model_three.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + manifest = get_manifest(project.project_root) + model_id = "model.test.model_three" + assert model_id in manifest.nodes + assert model_id not in manifest.disabled + + +class TestSources: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + def test_pp_sources(self, project): + # initial run + write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") + write_file(sources_tests1_sql, project.project_root, "macros", "tests.sql") + results = run_dbt(["run"]) + assert len(results) == 1 + + # Partial parse running 'seed' + run_dbt(["--partial-parse", "seed"]) + manifest = get_manifest(project.project_root) + seed_file_id = "test://" + normalize("seeds/raw_customers.csv") + assert seed_file_id in manifest.files + + # Add another seed file + write_file(raw_customers_csv, project.project_root, "seeds", "more_customers.csv") + run_dbt(["--partial-parse", "run"]) + seed_file_id = "test://" + normalize("seeds/more_customers.csv") + manifest = get_manifest(project.project_root) + assert seed_file_id in manifest.files + seed_id = "seed.test.more_customers" + assert seed_id in manifest.nodes + + # Remove seed file and add a schema files with a source referring to raw_customers + rm_file(project.project_root, "seeds", "more_customers.csv") + write_file(schema_sources1_yml, project.project_root, "models", "sources.yml") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + assert len(manifest.sources) == 1 + file_id = "test://" + normalize("models/sources.yml") + assert file_id in manifest.files + + # add a model referring to raw_customers source + write_file(customers_sql, project.project_root, "models", "customers.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + + # remove sources schema file + rm_file(project.project_root, "models", "sources.yml") + with pytest.raises(CompilationError): + results = run_dbt(["--partial-parse", "run"]) + + # put back sources and add an exposures file + write_file(schema_sources2_yml, project.project_root, "models", "sources.yml") + results = run_dbt(["--partial-parse", "run"]) + + # remove seed referenced in exposures file + rm_file(project.project_root, "seeds", "raw_customers.csv") + with pytest.raises(CompilationError): + results = run_dbt(["--partial-parse", "run"]) + + # put back seed and remove depends_on from exposure + write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") + write_file(schema_sources3_yml, project.project_root, "models", "sources.yml") + results = run_dbt(["--partial-parse", "run"]) + + # Add seed config with test to schema.yml, remove exposure + write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") + results = run_dbt(["--partial-parse", "run"]) + + # Change seed name to wrong name + write_file(schema_sources5_yml, project.project_root, "models", "sources.yml") + with pytest.raises(CompilationError): + results = run_dbt(["--partial-parse", "--warn-error", "run"]) + + # Put back seed name to right name + write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") + results = run_dbt(["--partial-parse", "run"]) + + # Add docs file customers.md + write_file(customers1_md, project.project_root, "models", "customers.md") + results = run_dbt(["--partial-parse", "run"]) + + # Change docs file customers.md + write_file(customers2_md, project.project_root, "models", "customers.md") + results = run_dbt(["--partial-parse", "run"]) + + # Delete docs file + rm_file(project.project_root, "models", "customers.md") + results = run_dbt(["--partial-parse", "run"]) + + # Add a data test + write_file(test_macro_sql, project.project_root, "macros", "test-macro.sql") + write_file(my_test_sql, project.project_root, "tests", "my_test.sql") + results = run_dbt(["--partial-parse", "test"]) + manifest = get_manifest(project.project_root) + assert len(manifest.nodes) == 9 + test_id = "test.test.my_test" + assert test_id in manifest.nodes + + # Change macro that data test depends on + write_file(test_macro2_sql, project.project_root, "macros", "test-macro.sql") + results = run_dbt(["--partial-parse", "test"]) + manifest = get_manifest(project.project_root) + + # Add an analysis + write_file(my_analysis_sql, project.project_root, "analyses", "my_analysis.sql") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + + # Remove data test + rm_file(project.project_root, "tests", "my_test.sql") + results = run_dbt(["--partial-parse", "test"]) + manifest = get_manifest(project.project_root) + assert len(manifest.nodes) == 9 + + # Remove analysis + rm_file(project.project_root, "analyses", "my_analysis.sql") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + assert len(manifest.nodes) == 8 + + # Change source test + write_file(sources_tests2_sql, project.project_root, "macros", "tests.sql") + results = run_dbt(["--partial-parse", "run"]) + + +class TestPartialParsingDependency: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project_root): + local_dependency_files = { + "dbt_project.yml": local_dependency__dbt_project_yml, + "models": { + "schema.yml": local_dependency__models__schema_yml, + "model_to_import.sql": local_dependency__models__model_to_import_sql, + }, + "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, + "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, + } + write_project_files(project_root, "local_dependency", local_dependency_files) + + @pytest.fixture(scope="class") + def packages(self): + return {"packages": [{"local": "local_dependency"}]} + + def test_parsing_with_dependency(self, project): + run_dbt(["clean"]) + run_dbt(["deps"]) + run_dbt(["seed"]) + run_dbt(["run"]) + + # Add a source override + write_file(schema_models_c_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + manifest = get_manifest(project.project_root) + assert len(manifest.sources) == 1 + source_id = "source.local_dep.seed_source.seed" + assert source_id in manifest.sources + # We have 1 root model, 1 local_dep model, 1 local_dep seed, 1 local_dep source test, 2 root source tests + assert len(manifest.nodes) == 5 + test_id = "test.local_dep.source_unique_seed_source_seed_id.afa94935ed" + assert test_id in manifest.nodes + + # Remove a source override + rm_file(project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + assert len(manifest.sources) == 1 + + +class TestNestedMacros: + @pytest.fixture(scope="class") + def models(self): + return { + "model_a.sql": model_a_sql, + "model_b.sql": model_b_sql, + "schema.yml": macros_schema_yml, + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "custom_schema_tests.sql": custom_schema_tests1_sql, + } + + def test_nested_macros(self, project): + results = run_dbt() + assert len(results) == 2 + manifest = get_manifest(project.project_root) + macro_child_map = manifest.build_macro_child_map() + macro_unique_id = "macro.test.test_type_two" + assert macro_unique_id in macro_child_map + + results = run_dbt(["test"], expect_pass=False) + results = sorted(results, key=lambda r: r.node.name) + assert len(results) == 2 + # type_one_model_a_ + assert results[0].status == TestStatus.Fail + assert re.search(r"union all", results[0].node.compiled_code) + # type_two_model_a_ + assert results[1].status == TestStatus.Warn + assert results[1].node.config.severity == "WARN" + + write_file( + custom_schema_tests2_sql, project.project_root, "macros", "custom_schema_tests.sql" + ) + results = run_dbt(["--partial-parse", "test"], expect_pass=False) + manifest = get_manifest(project.project_root) + test_node_id = "test.test.type_two_model_a_.842bc6c2a7" + assert test_node_id in manifest.nodes + results = sorted(results, key=lambda r: r.node.name) + assert len(results) == 2 + # type_two_model_a_ + assert results[1].status == TestStatus.Fail + assert results[1].node.config.severity == "ERROR" + + +class TestSkipMacros: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + "eschema.yml": empty_schema_yml, + } + + def test_skip_macros(self, project): + # initial run so we have a msgpack file + # includes empty_schema file for bug #4850 + results = run_dbt() + + # add a new ref override macro + write_file(ref_override_sql, project.project_root, "macros", "ref_override.sql") + results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) + assert "Starting full parse." in log_output + + # modify a ref override macro + write_file(ref_override2_sql, project.project_root, "macros", "ref_override.sql") + results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) + assert "Starting full parse." in log_output + + # remove a ref override macro + rm_file(project.project_root, "macros", "ref_override.sql") + results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) + assert "Starting full parse." in log_output + + # custom generate_schema_name macro + write_file(gsm_override_sql, project.project_root, "macros", "gsm_override.sql") + results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) + assert "Starting full parse." in log_output + + # change generate_schema_name macro + write_file(gsm_override2_sql, project.project_root, "macros", "gsm_override.sql") + results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) + assert "Starting full parse." in log_output + + +class TestSnapshots: + @pytest.fixture(scope="class") + def models(self): + return { + "orders.sql": orders_sql, + } + + def test_pp_snapshots(self, project): + + # initial run + results = run_dbt() + assert len(results) == 1 + + # add snapshot + write_file(snapshot_sql, project.project_root, "snapshots", "snapshot.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + snapshot_id = "snapshot.test.orders_snapshot" + assert snapshot_id in manifest.nodes + snapshot2_id = "snapshot.test.orders2_snapshot" + assert snapshot2_id in manifest.nodes + + # run snapshot + results = run_dbt(["--partial-parse", "snapshot"]) + assert len(results) == 2 + + # modify snapshot + write_file(snapshot2_sql, project.project_root, "snapshots", "snapshot.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 1 + + # delete snapshot + rm_file(project.project_root, "snapshots", "snapshot.sql") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 1 + + +class TestTests: + @pytest.fixture(scope="class") + def models(self): + return { + "orders.sql": orders_sql, + "schema.yml": generic_schema_yml, + } + + @pytest.fixture(scope="class") + def tests(self): + # Make sure "generic" directory is created + return {"generic": {"readme.md": ""}} + + def test_pp_generic_tests(self, project): + + # initial run + results = run_dbt() + assert len(results) == 1 + manifest = get_manifest(project.project_root) + expected_nodes = ["model.test.orders", "test.test.unique_orders_id.1360ecc70e"] + assert expected_nodes == list(manifest.nodes.keys()) + + # add generic test in test-path + write_file(generic_test_sql, project.project_root, "tests", "generic", "generic_test.sql") + write_file(generic_test_schema_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + test_id = "test.test.is_odd_orders_id.82834fdc5b" + assert test_id in manifest.nodes + expected_nodes = [ + "model.test.orders", + "test.test.unique_orders_id.1360ecc70e", + "test.test.is_odd_orders_id.82834fdc5b", + ] + assert expected_nodes == list(manifest.nodes.keys()) + + # edit generic test in test-path + write_file( + generic_test_edited_sql, project.project_root, "tests", "generic", "generic_test.sql" + ) + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + test_id = "test.test.is_odd_orders_id.82834fdc5b" + assert test_id in manifest.nodes + expected_nodes = [ + "model.test.orders", + "test.test.unique_orders_id.1360ecc70e", + "test.test.is_odd_orders_id.82834fdc5b", + ] + assert expected_nodes == list(manifest.nodes.keys()) diff --git a/tests/functional/partial_parsing/test_pp_metrics.py b/tests/functional/partial_parsing/test_pp_metrics.py new file mode 100644 index 00000000000..575c5ca613e --- /dev/null +++ b/tests/functional/partial_parsing/test_pp_metrics.py @@ -0,0 +1,73 @@ +import pytest + +from dbt.tests.util import run_dbt, write_file, get_manifest +from tests.functional.partial_parsing.fixtures import ( + people_sql, + people_metrics_yml, + people_metrics2_yml, + metric_model_a_sql, + people_metrics3_yml, +) + +from dbt.exceptions import CompilationError + + +class TestMetrics: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": people_sql, + } + + def test_metrics(self, project): + # initial run + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + assert len(manifest.nodes) == 1 + + # Add metrics yaml file + write_file(people_metrics_yml, project.project_root, "models", "people_metrics.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + assert len(manifest.metrics) == 2 + metric_people_id = "metric.test.number_of_people" + metric_tenure_id = "metric.test.collective_tenure" + metric_people = manifest.metrics[metric_people_id] + metric_tenure = manifest.metrics[metric_tenure_id] + expected_meta = {"my_meta": "testing"} + assert metric_people.meta == expected_meta + assert metric_people.refs == [["people"]] + assert metric_tenure.refs == [["people"]] + expected_depends_on_nodes = ["model.test.people"] + assert metric_people.depends_on.nodes == expected_depends_on_nodes + + # Change metrics yaml files + write_file(people_metrics2_yml, project.project_root, "models", "people_metrics.yml") + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + metric_people = manifest.metrics[metric_people_id] + expected_meta = {"my_meta": "replaced"} + assert metric_people.meta == expected_meta + expected_depends_on_nodes = ["model.test.people"] + assert metric_people.depends_on.nodes == expected_depends_on_nodes + + # Add model referring to metric + write_file(metric_model_a_sql, project.project_root, "models", "metric_model_a.sql") + results = run_dbt(["run"]) + manifest = get_manifest(project.project_root) + model_a = manifest.nodes["model.test.metric_model_a"] + expected_depends_on_nodes = [ + "metric.test.number_of_people", + "metric.test.collective_tenure", + ] + assert model_a.depends_on.nodes == expected_depends_on_nodes + + # Then delete a metric + write_file(people_metrics3_yml, project.project_root, "models", "people_metrics.yml") + with pytest.raises(CompilationError): + # We use "parse" here and not "run" because we're checking that the CompilationError + # occurs at parse time, not compilation + results = run_dbt(["parse"]) diff --git a/tests/functional/partial_parsing/test_pp_vars.py b/tests/functional/partial_parsing/test_pp_vars.py new file mode 100644 index 00000000000..19b3c7db849 --- /dev/null +++ b/tests/functional/partial_parsing/test_pp_vars.py @@ -0,0 +1,386 @@ +import pytest + +from dbt.tests.util import run_dbt, write_file, run_dbt_and_capture, get_manifest + +from tests.functional.partial_parsing.fixtures import ( + model_color_sql, + env_var_model_sql, + env_var_schema_yml, + env_var_model_one_sql, + raw_customers_csv, + env_var_sources_yml, + test_color_sql, + env_var_schema2_yml, + env_var_schema3_yml, + env_var_macro_sql, + env_var_macros_yml, + env_var_model_test_yml, + people_sql, + env_var_metrics_yml, + model_one_sql, +) + + +from dbt.exceptions import ParsingError +from dbt.constants import SECRET_ENV_PREFIX +import os + + +os.environ["DBT_PP_TEST"] = "true" + + +class TestEnvVars: + @pytest.fixture(scope="class") + def models(self): + return { + "model_color.sql": model_color_sql, + } + + def test_env_vars_models(self, project): + + # initial run + results = run_dbt(["run"]) + assert len(results) == 1 + + # copy a file with an env_var call without an env_var + write_file(env_var_model_sql, project.project_root, "models", "env_var_model.sql") + with pytest.raises(ParsingError): + results = run_dbt(["--partial-parse", "run"]) + + # set the env var + os.environ["ENV_VAR_TEST"] = "TestingEnvVars" + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + manifest = get_manifest(project.project_root) + expected_env_vars = {"ENV_VAR_TEST": "TestingEnvVars"} + assert expected_env_vars == manifest.env_vars + model_id = "model.test.env_var_model" + model = manifest.nodes[model_id] + model_created_at = model.created_at + + # change the env var + os.environ["ENV_VAR_TEST"] = "second" + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 2 + manifest = get_manifest(project.project_root) + expected_env_vars = {"ENV_VAR_TEST": "second"} + assert expected_env_vars == manifest.env_vars + assert model_created_at != manifest.nodes[model_id].created_at + + # set an env_var in a schema file + write_file(env_var_schema_yml, project.project_root, "models", "schema.yml") + write_file(env_var_model_one_sql, project.project_root, "models", "model_one.sql") + with pytest.raises(ParsingError): + results = run_dbt(["--partial-parse", "run"]) + + # actually set the env_var + os.environ["TEST_SCHEMA_VAR"] = "view" + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view"} + assert expected_env_vars == manifest.env_vars + + # env vars in a source + os.environ["ENV_VAR_DATABASE"] = "dbt" + os.environ["ENV_VAR_SEVERITY"] = "warn" + write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") + write_file(env_var_sources_yml, project.project_root, "models", "sources.yml") + run_dbt(["--partial-parse", "seed"]) + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + manifest = get_manifest(project.project_root) + expected_env_vars = { + "ENV_VAR_TEST": "second", + "TEST_SCHEMA_VAR": "view", + "ENV_VAR_DATABASE": "dbt", + "ENV_VAR_SEVERITY": "warn", + } + assert expected_env_vars == manifest.env_vars + assert len(manifest.sources) == 1 + source_id = "source.test.seed_sources.raw_customers" + source = manifest.sources[source_id] + assert source.database == "dbt" + schema_file = manifest.files[source.file_id] + test_id = "test.test.source_not_null_seed_sources_raw_customers_id.e39ee7bf0d" + test_node = manifest.nodes[test_id] + assert test_node.config.severity == "WARN" + + # Change severity env var + os.environ["ENV_VAR_SEVERITY"] = "error" + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + expected_env_vars = { + "ENV_VAR_TEST": "second", + "TEST_SCHEMA_VAR": "view", + "ENV_VAR_DATABASE": "dbt", + "ENV_VAR_SEVERITY": "error", + } + assert expected_env_vars == manifest.env_vars + source_id = "source.test.seed_sources.raw_customers" + source = manifest.sources[source_id] + schema_file = manifest.files[source.file_id] + expected_schema_file_env_vars = { + "sources": {"seed_sources": ["ENV_VAR_DATABASE", "ENV_VAR_SEVERITY"]} + } + assert expected_schema_file_env_vars == schema_file.env_vars + test_node = manifest.nodes[test_id] + assert test_node.config.severity == "ERROR" + + # Change database env var + os.environ["ENV_VAR_DATABASE"] = "test_dbt" + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + expected_env_vars = { + "ENV_VAR_TEST": "second", + "TEST_SCHEMA_VAR": "view", + "ENV_VAR_DATABASE": "test_dbt", + "ENV_VAR_SEVERITY": "error", + } + assert expected_env_vars == manifest.env_vars + source = manifest.sources[source_id] + assert source.database == "test_dbt" + + # Delete database env var + del os.environ["ENV_VAR_DATABASE"] + with pytest.raises(ParsingError): + results = run_dbt(["--partial-parse", "run"]) + os.environ["ENV_VAR_DATABASE"] = "test_dbt" + + # Add generic test with test kwarg that's rendered late (no curly brackets) + os.environ["ENV_VAR_DATABASE"] = "dbt" + write_file(test_color_sql, project.project_root, "macros", "test_color.sql") + results = run_dbt(["--partial-parse", "run"]) + # Add source test using test_color and an env_var for color + write_file(env_var_schema2_yml, project.project_root, "models/schema.yml") + with pytest.raises(ParsingError): + results = run_dbt(["--partial-parse", "run"]) + os.environ["ENV_VAR_COLOR"] = "green" + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + test_color_id = "test.test.check_color_model_one_env_var_ENV_VAR_COLOR___fun.89638de387" + test_node = manifest.nodes[test_color_id] + # kwarg was rendered but not changed (it will be rendered again when compiled) + assert test_node.test_metadata.kwargs["color"] == "env_var('ENV_VAR_COLOR')" + results = run_dbt(["--partial-parse", "test"]) + + # Add an exposure with an env_var + os.environ["ENV_VAR_OWNER"] = "John Doe" + write_file(env_var_schema3_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + expected_env_vars = { + "ENV_VAR_TEST": "second", + "TEST_SCHEMA_VAR": "view", + "ENV_VAR_DATABASE": "dbt", + "ENV_VAR_SEVERITY": "error", + "ENV_VAR_COLOR": "green", + "ENV_VAR_OWNER": "John Doe", + } + assert expected_env_vars == manifest.env_vars + exposure = list(manifest.exposures.values())[0] + schema_file = manifest.files[exposure.file_id] + expected_sf_env_vars = { + "models": {"model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"]}, + "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, + } + assert expected_sf_env_vars == schema_file.env_vars + + # add a macro and a macro schema file + os.environ["ENV_VAR_SOME_KEY"] = "toodles" + write_file(env_var_macro_sql, project.project_root, "macros", "env_var_macro.sql") + write_file(env_var_macros_yml, project.project_root, "macros", "env_var_macros.yml") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + expected_env_vars = { + "ENV_VAR_TEST": "second", + "TEST_SCHEMA_VAR": "view", + "ENV_VAR_DATABASE": "dbt", + "ENV_VAR_SEVERITY": "error", + "ENV_VAR_COLOR": "green", + "ENV_VAR_OWNER": "John Doe", + "ENV_VAR_SOME_KEY": "toodles", + } + assert expected_env_vars == manifest.env_vars + macro_id = "macro.test.do_something" + macro = manifest.macros[macro_id] + assert macro.meta == {"some_key": "toodles"} + # change the env var + os.environ["ENV_VAR_SOME_KEY"] = "dumdedum" + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + macro = manifest.macros[macro_id] + assert macro.meta == {"some_key": "dumdedum"} + + # Add a schema file with a test on model_color and env_var in test enabled config + write_file(env_var_model_test_yml, project.project_root, "models", "schema.yml") + results = run_dbt(["--partial-parse", "run"]) + assert len(results) == 3 + manifest = get_manifest(project.project_root) + model_color = manifest.nodes["model.test.model_color"] + schema_file = manifest.files[model_color.patch_path] + expected_env_vars = { + "models": { + "model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"], + "model_color": ["ENV_VAR_ENABLED"], + }, + "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, + } + assert expected_env_vars == schema_file.env_vars + + # Add a metrics file with env_vars + os.environ["ENV_VAR_METRICS"] = "TeStInG" + write_file(people_sql, project.project_root, "models", "people.sql") + write_file(env_var_metrics_yml, project.project_root, "models", "metrics.yml") + results = run_dbt(["run"]) + manifest = get_manifest(project.project_root) + assert "ENV_VAR_METRICS" in manifest.env_vars + assert manifest.env_vars["ENV_VAR_METRICS"] == "TeStInG" + metric_node = manifest.metrics["metric.test.number_of_people"] + assert metric_node.meta == {"my_meta": "TeStInG"} + + # Change metrics env var + os.environ["ENV_VAR_METRICS"] = "Changed!" + results = run_dbt(["run"]) + manifest = get_manifest(project.project_root) + metric_node = manifest.metrics["metric.test.number_of_people"] + assert metric_node.meta == {"my_meta": "Changed!"} + + # delete the env vars to cleanup + del os.environ["ENV_VAR_TEST"] + del os.environ["ENV_VAR_SEVERITY"] + del os.environ["ENV_VAR_DATABASE"] + del os.environ["TEST_SCHEMA_VAR"] + del os.environ["ENV_VAR_COLOR"] + del os.environ["ENV_VAR_SOME_KEY"] + del os.environ["ENV_VAR_OWNER"] + del os.environ["ENV_VAR_METRICS"] + + +class TestProjectEnvVars: + @pytest.fixture(scope="class") + def project_config_update(self): + # Need to set the environment variable here initially because + # the project fixture loads the config. + os.environ["ENV_VAR_NAME"] = "Jane Smith" + return {"models": {"+meta": {"meta_name": "{{ env_var('ENV_VAR_NAME') }}"}}} + + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + def test_project_env_vars(self, project): + # Initial run + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + state_check = manifest.state_check + model_id = "model.test.model_one" + model = manifest.nodes[model_id] + assert model.config.meta["meta_name"] == "Jane Smith" + env_vars_hash_checksum = state_check.project_env_vars_hash.checksum + + # Change the environment variable + os.environ["ENV_VAR_NAME"] = "Jane Doe" + results = run_dbt(["run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.meta["meta_name"] == "Jane Doe" + assert env_vars_hash_checksum != manifest.state_check.project_env_vars_hash.checksum + + # cleanup + del os.environ["ENV_VAR_NAME"] + + +class TestProfileEnvVars: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + @pytest.fixture(scope="class") + def dbt_profile_target(self): + # Need to set these here because the base integration test class + # calls 'load_config' before the tests are run. + # Note: only the specified profile is rendered, so there's no + # point it setting env_vars in non-used profiles. + os.environ["ENV_VAR_USER"] = "root" + os.environ["ENV_VAR_PASS"] = "password" + return { + "type": "postgres", + "threads": 4, + "host": "localhost", + "port": 5432, + "user": "{{ env_var('ENV_VAR_USER') }}", + "pass": "{{ env_var('ENV_VAR_PASS') }}", + "dbname": "dbt", + } + + def test_profile_env_vars(self, project): + + # Initial run + os.environ["ENV_VAR_USER"] = "root" + os.environ["ENV_VAR_PASS"] = "password" + + results = run_dbt(["run"]) + manifest = get_manifest(project.project_root) + env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum + + # Change env_vars, the user doesn't exist, this should fail + os.environ["ENV_VAR_USER"] = "fake_user" + (results, log_output) = run_dbt_and_capture(["run"], expect_pass=False) + assert "env vars used in profiles.yml have changed" in log_output + manifest = get_manifest(project.project_root) + assert env_vars_checksum != manifest.state_check.profile_env_vars_hash.checksum + + +class TestProfileSecretEnvVars: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + } + + @property + def dbt_profile_target(self): + # Need to set these here because the base integration test class + # calls 'load_config' before the tests are run. + # Note: only the specified profile is rendered, so there's no + # point in setting env_vars in non-used profiles. + + # user is secret and password is not. postgres on macos doesn't care if the password + # changes so we have to change the user. related: https://github.com/dbt-labs/dbt-core/pull/4250 + os.environ[SECRET_ENV_PREFIX + "USER"] = "root" + os.environ["ENV_VAR_PASS"] = "password" + return { + "type": "postgres", + "threads": 4, + "host": "localhost", + "port": 5432, + "user": "{{ env_var('DBT_ENV_SECRET_USER') }}", + "pass": "{{ env_var('ENV_VAR_PASS') }}", + "dbname": "dbt", + } + + def test_profile_secret_env_vars(self, project): + + # Initial run + os.environ[SECRET_ENV_PREFIX + "USER"] = "root" + os.environ["ENV_VAR_PASS"] = "password" + + results = run_dbt(["run"]) + manifest = get_manifest(project.project_root) + env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum + + # Change a secret var, it shouldn't register because we shouldn't save secrets. + os.environ[SECRET_ENV_PREFIX + "USER"] = "fake_user" + # we just want to see if the manifest has included + # the secret in the hash of environment variables. + (results, log_output) = run_dbt_and_capture(["run"], expect_pass=True) + # I020 is the event code for "env vars used in profiles.yml have changed" + assert not ("I020" in log_output) + manifest = get_manifest(project.project_root) + assert env_vars_checksum == manifest.state_check.profile_env_vars_hash.checksum