From f1a9958ff157b41da2fd8fec311b862817e5287e Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Apr 2022 18:23:13 -0400 Subject: [PATCH 1/8] Convert docs_generate_tests to new framework --- core/dbt/tests/fixtures/project.py | 27 +- core/dbt/tests/util.py | 15 +- .../assets/lorem-ipsum.txt | 1 - .../fail_macros/failure.sql | 3 - .../macros/dummy_test.sql | 9 - .../029_docs_generate_tests/macros/macro.md | 8 - .../029_docs_generate_tests/macros/schema.yml | 10 - .../029_docs_generate_tests/models/model.sql | 8 - .../029_docs_generate_tests/models/readme.md | 1 - .../029_docs_generate_tests/models/schema.yml | 82 - .../models/second_model.sql | 8 - .../ref_models/docs.md | 31 - .../ref_models/ephemeral_copy.sql | 7 - .../ref_models/ephemeral_summary.sql | 9 - .../ref_models/schema.yml | 48 - .../ref_models/view_summary.sql | 8 - .../029_docs_generate_tests/seed/schema.yml | 15 - .../029_docs_generate_tests/seed/seed.csv | 2 - .../snapshot/snapshot_seed.sql | 11 - .../test_docs_generate.py | 2109 ----------------- .../trivial_models/model.sql | 1 - .../docs_generate/expected_catalog.py | 244 ++ .../docs_generate/expected_manifest.py | 1321 +++++++++++ .../docs_generate/expected_run_results.py | 132 ++ .../docs_generate/test_docs_generate.py | 545 +++++ .../functional/docs_generate/test_escapes.py | 26 + .../functional/docs_generate/test_override.py | 35 + 27 files changed, 2331 insertions(+), 2385 deletions(-) delete mode 100644 test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt delete mode 100644 test/integration/029_docs_generate_tests/fail_macros/failure.sql delete mode 100644 test/integration/029_docs_generate_tests/macros/dummy_test.sql delete mode 100644 test/integration/029_docs_generate_tests/macros/macro.md delete mode 100644 test/integration/029_docs_generate_tests/macros/schema.yml delete mode 100644 test/integration/029_docs_generate_tests/models/model.sql delete mode 100644 test/integration/029_docs_generate_tests/models/readme.md delete mode 100644 test/integration/029_docs_generate_tests/models/schema.yml delete mode 100644 test/integration/029_docs_generate_tests/models/second_model.sql delete mode 100644 test/integration/029_docs_generate_tests/ref_models/docs.md delete mode 100644 test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql delete mode 100644 test/integration/029_docs_generate_tests/ref_models/ephemeral_summary.sql delete mode 100644 test/integration/029_docs_generate_tests/ref_models/schema.yml delete mode 100644 test/integration/029_docs_generate_tests/ref_models/view_summary.sql delete mode 100644 test/integration/029_docs_generate_tests/seed/schema.yml delete mode 100644 test/integration/029_docs_generate_tests/seed/seed.csv delete mode 100644 test/integration/029_docs_generate_tests/snapshot/snapshot_seed.sql delete mode 100644 test/integration/029_docs_generate_tests/test_docs_generate.py delete mode 100644 test/integration/029_docs_generate_tests/trivial_models/model.sql create mode 100644 tests/functional/docs_generate/expected_catalog.py create mode 100644 tests/functional/docs_generate/expected_manifest.py create mode 100644 tests/functional/docs_generate/expected_run_results.py create mode 100644 tests/functional/docs_generate/test_docs_generate.py create mode 100644 tests/functional/docs_generate/test_escapes.py create mode 100644 tests/functional/docs_generate/test_override.py diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index bb42a00ab3a..56756ac0e22 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -253,16 +253,17 @@ def write_project_files(project_root, dir_name, file_dict): # Write files out from file_dict. Can be nested directories... def write_project_files_recursively(path, file_dict): if type(file_dict) is not dict: - raise TestProcessingException(f"Error creating {path}. Did you forget the file extension?") + raise TestProcessingException(f"File dict is not a dict: '{file_dict}' for path '{path}'") + suffix_list = [".sql", ".csv", ".md", ".txt"] for name, value in file_dict.items(): - if name.endswith(".sql") or name.endswith(".csv") or name.endswith(".md"): - write_file(value, path, name) - elif name.endswith(".yml") or name.endswith(".yaml"): + if name.endswith(".yml") or name.endswith(".yaml"): if isinstance(value, str): data = value else: data = yaml.safe_dump(value) write_file(data, path, name) + elif name.endswith(tuple(suffix_list)): + write_file(value, path, name) else: write_project_files_recursively(path.mkdir(name), value) @@ -356,6 +357,7 @@ def __init__( self.test_schema = test_schema self.database = database self.test_config = test_config + self.created_schemas = [] @property def adapter(self): @@ -377,20 +379,21 @@ def run_sql(self, sql, fetch=None): # Create the unique test schema. Used in test setup, so that we're # ready for initial sql prior to a run_dbt command. - def create_test_schema(self): + def create_test_schema(self, schema_name=None): + if schema_name is None: + schema_name = self.test_schema with get_connection(self.adapter): - relation = self.adapter.Relation.create( - database=self.database, schema=self.test_schema - ) + relation = self.adapter.Relation.create(database=self.database, schema=schema_name) self.adapter.create_schema(relation) + self.created_schemas.append(schema_name) # Drop the unique test schema, usually called in test cleanup def drop_test_schema(self): with get_connection(self.adapter): - relation = self.adapter.Relation.create( - database=self.database, schema=self.test_schema - ) - self.adapter.drop_schema(relation) + for schema_name in self.created_schemas: + relation = self.adapter.Relation.create(database=self.database, schema=schema_name) + self.adapter.drop_schema(relation) + self.created_schemas = [] # This return a dictionary of table names to 'view' or 'table' values. def get_tables_in_schema(self): diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index f5d8cb98b88..b87f2e11b39 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -3,6 +3,7 @@ import yaml import json import warnings +from datetime import datetime from typing import List from contextlib import contextmanager @@ -104,9 +105,9 @@ def copy_file(src_path, src, dest_path, dest) -> None: # Used in tests when you want to remove a file from the project directory -def rm_file(src_path, src) -> None: +def rm_file(*paths) -> None: # remove files from proj_path - os.remove(os.path.join(src_path, src)) + os.remove(os.path.join(*paths)) # Used in tests to write out the string contents of a file to a @@ -167,6 +168,16 @@ def check_result_nodes_by_unique_id(results, unique_ids): assert set(unique_ids) == set(result_unique_ids) +# Check datetime is between start and end/now +def check_datetime_between(timestr, start, end=None): + datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" + if end is None: + end = datetime.utcnow() + parsed = datetime.strptime(timestr, datefmt) + assert start <= parsed + assert end >= parsed + + class TestProcessingException(Exception): pass diff --git a/test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt b/test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt deleted file mode 100644 index cee7a927c5f..00000000000 --- a/test/integration/029_docs_generate_tests/assets/lorem-ipsum.txt +++ /dev/null @@ -1 +0,0 @@ -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. diff --git a/test/integration/029_docs_generate_tests/fail_macros/failure.sql b/test/integration/029_docs_generate_tests/fail_macros/failure.sql deleted file mode 100644 index f0519ed245a..00000000000 --- a/test/integration/029_docs_generate_tests/fail_macros/failure.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro get_catalog(information_schema, schemas) %} - {% do exceptions.raise_compiler_error('rejected: no catalogs for you') %} -{% endmacro %} diff --git a/test/integration/029_docs_generate_tests/macros/dummy_test.sql b/test/integration/029_docs_generate_tests/macros/dummy_test.sql deleted file mode 100644 index adf4d7d930c..00000000000 --- a/test/integration/029_docs_generate_tests/macros/dummy_test.sql +++ /dev/null @@ -1,9 +0,0 @@ - -{% test nothing(model) %} - --- a silly test to make sure that table-level tests show up in the manifest --- without a column_name field -select 0 - -{% endtest %} - diff --git a/test/integration/029_docs_generate_tests/macros/macro.md b/test/integration/029_docs_generate_tests/macros/macro.md deleted file mode 100644 index 6a48ffc1b37..00000000000 --- a/test/integration/029_docs_generate_tests/macros/macro.md +++ /dev/null @@ -1,8 +0,0 @@ - -{% docs macro_info %} -My custom test that I wrote that does nothing -{% enddocs %} - -{% docs macro_arg_info %} -The model for my custom test -{% enddocs %} diff --git a/test/integration/029_docs_generate_tests/macros/schema.yml b/test/integration/029_docs_generate_tests/macros/schema.yml deleted file mode 100644 index 6b33f47307e..00000000000 --- a/test/integration/029_docs_generate_tests/macros/schema.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -macros: - - name: test_nothing - description: "{{ doc('macro_info') }}" - meta: - some_key: 100 - arguments: - - name: model - type: Relation - description: "{{ doc('macro_arg_info') }}" diff --git a/test/integration/029_docs_generate_tests/models/model.sql b/test/integration/029_docs_generate_tests/models/model.sql deleted file mode 100644 index 0759a4572a8..00000000000 --- a/test/integration/029_docs_generate_tests/models/model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized='view', - database=var('alternate_db') - ) -}} - -select * from {{ ref('seed') }} diff --git a/test/integration/029_docs_generate_tests/models/readme.md b/test/integration/029_docs_generate_tests/models/readme.md deleted file mode 100644 index d59a7f44724..00000000000 --- a/test/integration/029_docs_generate_tests/models/readme.md +++ /dev/null @@ -1 +0,0 @@ -This is a readme.md file with {{ invalid-ish jinja }} in it diff --git a/test/integration/029_docs_generate_tests/models/schema.yml b/test/integration/029_docs_generate_tests/models/schema.yml deleted file mode 100644 index 600be50eaa0..00000000000 --- a/test/integration/029_docs_generate_tests/models/schema.yml +++ /dev/null @@ -1,82 +0,0 @@ -version: 2 - -models: - - name: model - description: "The test model" - docs: - show: false - columns: - - name: id - description: The user ID number - tests: - - unique - - not_null - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated - tests: - - test.nothing - - - name: second_model - description: "The second test model" - docs: - show: false - columns: - - name: id - description: The user ID number - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated - - -sources: - - name: my_source - description: "My source" - loader: a_loader - schema: "{{ var('test_schema') }}" - tables: - - name: my_table - description: "My table" - identifier: seed - quoting: - identifier: True - columns: - - name: id - description: "An ID field" - - -exposures: - - name: simple_exposure - type: dashboard - depends_on: - - ref('model') - - source('my_source', 'my_table') - owner: - email: something@example.com - - name: notebook_exposure - type: notebook - depends_on: - - ref('model') - - ref('second_model') - owner: - email: something@example.com - name: Some name - description: > - A description of the complex exposure - maturity: medium - meta: - tool: 'my_tool' - languages: - - python - tags: ['my_department'] - url: http://example.com/notebook/1 diff --git a/test/integration/029_docs_generate_tests/models/second_model.sql b/test/integration/029_docs_generate_tests/models/second_model.sql deleted file mode 100644 index 37f83155ce9..00000000000 --- a/test/integration/029_docs_generate_tests/models/second_model.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized='view', - schema='test', - ) -}} - -select * from {{ ref('seed') }} diff --git a/test/integration/029_docs_generate_tests/ref_models/docs.md b/test/integration/029_docs_generate_tests/ref_models/docs.md deleted file mode 100644 index c5ad96862c6..00000000000 --- a/test/integration/029_docs_generate_tests/ref_models/docs.md +++ /dev/null @@ -1,31 +0,0 @@ -{% docs ephemeral_summary %} -A summmary table of the ephemeral copy of the seed data -{% enddocs %} - -{% docs summary_first_name %} -The first name being summarized -{% enddocs %} - -{% docs summary_count %} -The number of instances of the first name -{% enddocs %} - -{% docs view_summary %} -A view of the summary of the ephemeral copy of the seed data -{% enddocs %} - -{% docs source_info %} -My source -{% enddocs %} - -{% docs table_info %} -My table -{% enddocs %} - -{% docs column_info %} -An ID field -{% enddocs %} - -{% docs notebook_info %} -A description of the complex exposure -{% enddocs %} diff --git a/test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql b/test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql deleted file mode 100644 index 3f7e698ce4d..00000000000 --- a/test/integration/029_docs_generate_tests/ref_models/ephemeral_copy.sql +++ /dev/null @@ -1,7 +0,0 @@ -{{ - config( - materialized = "ephemeral" - ) -}} - -select * from {{ source("my_source", "my_table") }} diff --git a/test/integration/029_docs_generate_tests/ref_models/ephemeral_summary.sql b/test/integration/029_docs_generate_tests/ref_models/ephemeral_summary.sql deleted file mode 100644 index 4dba8a56470..00000000000 --- a/test/integration/029_docs_generate_tests/ref_models/ephemeral_summary.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ - config( - materialized = "table" - ) -}} - -select first_name, count(*) as ct from {{ref('ephemeral_copy')}} -group by first_name -order by first_name asc diff --git a/test/integration/029_docs_generate_tests/ref_models/schema.yml b/test/integration/029_docs_generate_tests/ref_models/schema.yml deleted file mode 100644 index 6d7dcea05f1..00000000000 --- a/test/integration/029_docs_generate_tests/ref_models/schema.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: 2 - -models: - - name: ephemeral_summary - description: "{{ doc('ephemeral_summary') }}" - columns: &summary_columns - - name: first_name - description: "{{ doc('summary_first_name') }}" - - name: ct - description: "{{ doc('summary_count') }}" - - name: view_summary - description: "{{ doc('view_summary') }}" - columns: *summary_columns - -sources: - - name: my_source - description: "{{ doc('source_info') }}" - loader: a_loader - schema: "{{ var('test_schema') }}" - quoting: - database: False - identifier: False - tables: - - name: my_table - description: "{{ doc('table_info') }}" - identifier: seed - quoting: - identifier: True - columns: - - name: id - description: "{{ doc('column_info') }}" - -exposures: - - name: notebook_exposure - type: notebook - depends_on: - - ref('view_summary') - owner: - email: something@example.com - name: Some name - description: "{{ doc('notebook_info') }}" - maturity: medium - url: http://example.com/notebook/1 - meta: - tool: 'my_tool' - languages: - - python - tags: ['my_department'] diff --git a/test/integration/029_docs_generate_tests/ref_models/view_summary.sql b/test/integration/029_docs_generate_tests/ref_models/view_summary.sql deleted file mode 100644 index 6ad6c3dd01f..00000000000 --- a/test/integration/029_docs_generate_tests/ref_models/view_summary.sql +++ /dev/null @@ -1,8 +0,0 @@ -{{ - config( - materialized = "view" - ) -}} - -select first_name, ct from {{ref('ephemeral_summary')}} -order by ct asc diff --git a/test/integration/029_docs_generate_tests/seed/schema.yml b/test/integration/029_docs_generate_tests/seed/schema.yml deleted file mode 100644 index ef5e7dc9efb..00000000000 --- a/test/integration/029_docs_generate_tests/seed/schema.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: 2 -seeds: - - name: seed - description: "The test seed" - columns: - - name: id - description: The user ID number - - name: first_name - description: The user's first name - - name: email - description: The user's email - - name: ip_address - description: The user's IP address - - name: updated_at - description: The last time this user's email was updated diff --git a/test/integration/029_docs_generate_tests/seed/seed.csv b/test/integration/029_docs_generate_tests/seed/seed.csv deleted file mode 100644 index ef154f552c9..00000000000 --- a/test/integration/029_docs_generate_tests/seed/seed.csv +++ /dev/null @@ -1,2 +0,0 @@ -id,first_name,email,ip_address,updated_at -1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 diff --git a/test/integration/029_docs_generate_tests/snapshot/snapshot_seed.sql b/test/integration/029_docs_generate_tests/snapshot/snapshot_seed.sql deleted file mode 100644 index 83bc54fe784..00000000000 --- a/test/integration/029_docs_generate_tests/snapshot/snapshot_seed.sql +++ /dev/null @@ -1,11 +0,0 @@ -{% snapshot snapshot_seed %} -{{ - config( - unique_key='id', - strategy='check', - check_cols='all', - target_schema=var('alternate_schema') - ) -}} -select * from {{ ref('seed') }} -{% endsnapshot %} \ No newline at end of file diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py deleted file mode 100644 index 7075fe795b0..00000000000 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ /dev/null @@ -1,2109 +0,0 @@ -import hashlib -import json -import os -import random -import shutil -import tempfile -import time -from datetime import datetime -from unittest.mock import ANY, patch - -from pytest import mark -from test.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ - AnyString, AnyStringWith, normalize, Normalized - -import dbt.tracking -import dbt.version -from dbt.exceptions import CompilationException - - -def _read_file(path): - with open(path, 'r') as fp: - return fp.read().replace('\r', '').replace('\\r', '') - - -class LineIndifferent: - def __init__(self, expected): - self.expected = expected.replace('\r', '') - - def __eq__(self, other): - return self.expected == other.replace('\r', '') - - def __repr__(self): - return 'LineIndifferent({!r})'.format(self.expected) - - def __str__(self): - return self.__repr__() - - -class OneOf: - def __init__(self, *options): - self.options = options - - def __eq__(self, other): - return any(o == other for o in self.options) - - def __repr__(self): - return 'OneOf(*{!r})'.format(self.options) - - def __str__(self): - return self.__repr__() - - -def _read_json(path): - # read json generated by dbt. - with open(path) as fp: - return json.load(fp) - - -def walk_files(path): - for root, dirs, files in os.walk(path): - for basename in files: - yield os.path.join(root, basename) - - -class TestDocsGenerateEscapes(DBTIntegrationTest): - prefix = "pgtest{}{:04}".format(int(time.time()), random.randint(0, 9999)) - - @property - def schema(self): - return 'docs_generate_029' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("trivial_models") - - def run_and_generate(self): - self.assertEqual(len(self.run_dbt(['run'])), 1) - os.remove(normalize('target/manifest.json')) - os.remove(normalize('target/run_results.json')) - self.run_dbt(['docs', 'generate']) - - @use_profile('postgres') - def test_postgres_include_schema(self): - self.run_and_generate() - manifest = _read_json('./target/manifest.json') - self.assertIn('nodes', manifest) - self.assertEqual(len(manifest['nodes']), 1) - self.assertIn('model.test.model', manifest['nodes']) - self.assertIn('schema', manifest['nodes']['model.test.model']) - self.assertEqual('pg', manifest['nodes'] - ['model.test.model']['schema'][:2]) - - -class TestDocsGenerate(DBTIntegrationTest): - setup_alternate_db = True - - def adapter_case(self, value): - return value.lower() - - def setUp(self): - super().setUp() - self.maxDiff = None - self.alternate_schema = self.unique_schema() + '_test' - - self._created_schemas.add(self.alternate_schema) - os.environ['DBT_ENV_CUSTOM_ENV_env_key'] = 'env_value' - - def tearDown(self): - super().tearDown() - del os.environ['DBT_ENV_CUSTOM_ENV_env_key'] - - @property - def schema(self): - return 'docs_generate_029' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("models") - - @property - def project_config(self): - return { - 'config-version': 2, - 'quoting': { - 'identifier': False - } - } - - def run_and_generate(self, extra=None, seed_count=1, model_count=2, alternate_db=None, args=None): - if alternate_db is None: - alternate_db = self.alternative_database - project = { - "seed-paths": [self.dir("seed")], - 'macro-paths': [self.dir('macros')], - 'snapshot-paths': [self.dir('snapshot')], - 'vars': { - 'alternate_db': alternate_db, - 'alternate_schema': self.alternate_schema, - }, - 'seeds': { - 'quote_columns': True, - }, - } - if extra: - project.update(extra) - self.use_default_project(project) - - vars_arg = '--vars={{test_schema: {}}}'.format(self.unique_schema()) - - self.assertEqual(len(self.run_dbt(["seed", vars_arg])), seed_count) - self.assertEqual(len(self.run_dbt(['run', vars_arg])), model_count) - os.remove(normalize('target/manifest.json')) - os.remove(normalize('target/run_results.json')) - self.generate_start_time = datetime.utcnow() - base_args = ['docs', 'generate', vars_arg] - if args: - base_args.extend(args) - self.run_dbt(base_args) - - def _no_stats(self): - return { - 'has_stats': { - 'id': 'has_stats', - 'label': 'Has Stats?', - 'value': False, - 'description': 'Indicates whether there are statistics for this table', - 'include': False, - }, - } - - def _expected_catalog(self, id_type, text_type, time_type, view_type, - table_type, model_stats, seed_stats=None, case=None, - case_columns=False, model_database=None): - if case is None: - def case(x): return x - col_case = case if case_columns else lambda x: x - - if seed_stats is None: - seed_stats = model_stats - - if model_database is None: - model_database = self.default_database - my_schema_name = self.unique_schema() - role = self.get_role() - expected_cols = { - col_case('id'): { - 'name': col_case('id'), - 'index': 1, - 'type': id_type, - 'comment': None, - }, - col_case('first_name'): { - 'name': col_case('first_name'), - 'index': 2, - 'type': text_type, - 'comment': None, - }, - col_case('email'): { - 'name': col_case('email'), - 'index': 3, - 'type': text_type, - 'comment': None, - }, - col_case('ip_address'): { - 'name': col_case('ip_address'), - 'index': 4, - 'type': text_type, - 'comment': None, - }, - col_case('updated_at'): { - 'name': col_case('updated_at'), - 'index': 5, - 'type': time_type, - 'comment': None, - }, - } - return { - 'nodes': { - 'model.test.model': { - 'unique_id': 'model.test.model', - 'metadata': { - 'schema': my_schema_name, - 'database': model_database, - 'name': case('model'), - 'type': view_type, - 'comment': None, - 'owner': role, - }, - 'stats': model_stats, - 'columns': expected_cols, - }, - 'model.test.second_model': { - 'unique_id': 'model.test.second_model', - 'metadata': { - 'schema': self.alternate_schema, - 'database': self.default_database, - 'name': case('second_model'), - 'type': view_type, - 'comment': None, - 'owner': role, - }, - 'stats': model_stats, - 'columns': expected_cols, - }, - 'seed.test.seed': { - 'unique_id': 'seed.test.seed', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': case('seed'), - 'type': table_type, - 'comment': None, - 'owner': role, - }, - 'stats': seed_stats, - 'columns': expected_cols, - }, - }, - 'sources': { - 'source.test.my_source.my_table': { - 'unique_id': 'source.test.my_source.my_table', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': case('seed'), - 'type': table_type, - 'comment': None, - 'owner': role, - }, - 'stats': seed_stats, - 'columns': expected_cols, - }, - }, - } - - def expected_postgres_catalog(self): - return self._expected_catalog( - id_type='integer', - text_type='text', - time_type='timestamp without time zone', - view_type='VIEW', - table_type='BASE TABLE', - model_stats=self._no_stats() - ) - - def get_role(self): - if self.adapter_type in {'postgres'}: - profile = self.get_profile(self.adapter_type) - target_name = profile['test']['target'] - return profile['test']['outputs'][target_name]['user'] - return None - - def expected_postgres_references_catalog(self): - model_database = self.default_database - my_schema_name = self.unique_schema() - role = self.get_role() - stats = self._no_stats() - summary_columns = { - 'first_name': { - 'name': 'first_name', - 'index': 1, - 'type': 'text', - 'comment': None, - }, - 'ct': { - 'name': 'ct', - 'index': 2, - 'type': 'bigint', - 'comment': None, - }, - } - - seed_columns = { - 'id': { - 'name': 'id', - 'index': 1, - 'type': 'integer', - 'comment': None, - }, - 'first_name': { - 'name': 'first_name', - 'index': 2, - 'type': 'text', - 'comment': None, - }, - 'email': { - 'name': 'email', - 'index': 3, - 'type': 'text', - 'comment': None, - }, - 'ip_address': { - 'name': 'ip_address', - 'index': 4, - 'type': 'text', - 'comment': None, - }, - 'updated_at': { - 'name': 'updated_at', - 'index': 5, - 'type': 'timestamp without time zone', - 'comment': None, - }, - } - return { - 'nodes': { - 'seed.test.seed': { - 'unique_id': 'seed.test.seed', - 'metadata': { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'seed', - 'type': 'BASE TABLE', - 'comment': None, - 'owner': role, - }, - 'stats': stats, - 'columns': seed_columns - }, - 'model.test.ephemeral_summary': { - 'unique_id': 'model.test.ephemeral_summary', - 'metadata': { - 'schema': my_schema_name, - 'database': model_database, - 'name': 'ephemeral_summary', - 'type': 'BASE TABLE', - 'comment': None, - 'owner': role, - }, - 'stats': stats, - 'columns': summary_columns, - }, - 'model.test.view_summary': { - 'unique_id': 'model.test.view_summary', - 'metadata': { - 'schema': my_schema_name, - 'database': model_database, - 'name': 'view_summary', - 'type': 'VIEW', - 'comment': None, - 'owner': role, - }, - 'stats': stats, - 'columns': summary_columns, - }, - }, - 'sources': { - "source.test.my_source.my_table": { - "unique_id": "source.test.my_source.my_table", - "metadata": { - 'schema': my_schema_name, - 'database': self.default_database, - 'name': 'seed', - 'type': 'BASE TABLE', - 'comment': None, - 'owner': role, - }, - "stats": stats, - 'columns': seed_columns, - }, - }, - } - - def expected_presto_catalog(self): - return self._expected_catalog( - id_type='integer', - text_type='varchar', - time_type='timestamp', - view_type='VIEW', - table_type='BASE TABLE', - model_stats=self._no_stats(), - seed_stats=self._no_stats(), - model_database=self.default_database, - ) - - def verify_catalog(self, expected): - self.assertTrue(os.path.exists('./target/catalog.json')) - - catalog = _read_json('./target/catalog.json') - - assert set(catalog) == {'errors', 'metadata', 'nodes', 'sources'} - - self.verify_metadata( - catalog['metadata'], 'https://schemas.getdbt.com/dbt/catalog/v1.json') - assert not catalog['errors'] - - for key in 'nodes', 'sources': - assert catalog[key] == expected[key] - - def verify_manifest_macros(self, manifest, expected=None): - self.assertIn('macros', manifest) - if expected is None: - self._verify_generic_macro_structure(manifest) - return - for unique_id, expected_macro in expected.items(): - self.assertIn(unique_id, manifest['macros']) - actual_macro = manifest['macros'][unique_id] - self.assertEqual(expected_macro, actual_macro) - - def rendered_model_config(self, **updates): - result = { - 'database': None, - 'schema': None, - 'alias': None, - 'enabled': True, - 'materialized': 'view', - 'pre-hook': [], - 'post-hook': [], - 'column_types': {}, - 'quoting': {}, - 'tags': [], - 'persist_docs': {}, - 'full_refresh': None, - 'on_schema_change': 'ignore', - 'meta': {}, - 'unique_key': None, - } - result.update(updates) - return result - - def unrendered_model_config(self, **updates): - return updates - - def rendered_seed_config(self, **updates): - result = { - 'enabled': True, - 'materialized': 'seed', - 'persist_docs': {}, - 'pre-hook': [], - 'post-hook': [], - 'column_types': {}, - 'quoting': {}, - 'tags': [], - 'quote_columns': True, - 'full_refresh': None, - 'on_schema_change': 'ignore', - 'database': None, - 'schema': None, - 'alias': None, - 'meta': {}, - 'unique_key': None, - } - result.update(updates) - return result - - def unrendered_seed_config(self, **updates): - result = {'quote_columns': True} - result.update(updates) - return result - - def rendered_snapshot_config(self, **updates): - result = { - 'database': None, - 'schema': None, - 'alias': None, - 'enabled': True, - 'materialized': 'snapshot', - 'pre-hook': [], - 'post-hook': [], - 'column_types': {}, - 'quoting': {}, - 'tags': [], - 'persist_docs': {}, - 'full_refresh': None, - 'on_schema_change': 'ignore', - 'strategy': 'check', - 'check_cols': 'all', - 'unique_key': 'id', - 'target_schema': None, - 'meta': {}, - } - result.update(updates) - return result - - def unrendered_snapshot_config(self, **updates): - result = { - 'check_cols': 'all', - 'strategy': 'check', - 'target_schema': None, - 'unique_key': 'id' - } - result.update(updates) - return result - - def rendered_tst_config(self, **updates): - result = { - 'enabled': True, - 'materialized': 'test', - 'tags': [], - 'severity': 'ERROR', - 'store_failures': None, - 'warn_if': '!= 0', - 'error_if': '!= 0', - 'fail_calc': 'count(*)', - 'where': None, - 'limit': None, - 'database': None, - 'schema': 'dbt_test__audit', - 'alias': None, - 'meta': {}, - } - result.update(updates) - return result - - def unrendered_tst_config(self, **updates): - result = {} - result.update(updates) - return result - - def _verify_generic_macro_structure(self, manifest): - # just test a known global macro to avoid having to update this every - # time they change. - self.assertIn('macro.dbt.get_quoted_csv', manifest['macros']) - macro = manifest['macros']['macro.dbt.get_quoted_csv'] - self.assertEqual( - set(macro), - { - 'path', 'original_file_path', 'package_name', - 'root_path', 'name', 'unique_id', 'tags', 'resource_type', - 'depends_on', 'meta', 'description', 'patch_path', 'arguments', - 'macro_sql', 'docs', 'created_at', - } - ) - # Don't compare the sql, just make sure it exists - self.assertTrue(len(macro['macro_sql']) > 10) - without_sql = { - k: v for k, v in macro.items() - if k not in {'macro_sql'} - } - # Windows means we can't hard-code these. - helpers_path = Normalized('macros/materializations/models/incremental/column_helpers.sql') - root_path = Normalized(os.path.join( - self.dbt_core_install_root, 'include', 'global_project' - )) - self.assertEqual( - { - 'path': helpers_path, - 'original_file_path': helpers_path, - 'package_name': 'dbt', - 'root_path': root_path, - 'name': 'get_quoted_csv', - 'unique_id': 'macro.dbt.get_quoted_csv', - 'created_at': ANY, - 'tags': [], - 'resource_type': 'macro', - 'depends_on': {'macros': []}, - 'description': '', - 'docs': {'show': True}, - 'patch_path': None, - 'meta': {}, - 'arguments': [], - }, - without_sql, - ) - - def expected_seeded_manifest(self, model_database=None, quote_model=False): - models_path = self.dir('models') - model_sql_path = os.path.join(models_path, 'model.sql') - second_model_sql_path = os.path.join(models_path, 'second_model.sql') - model_schema_yml_path = os.path.join(models_path, 'schema.yml') - seed_schema_yml_path = os.path.join(self.dir('seed'), 'schema.yml') - seed_path = self.dir(os.path.join('seed', 'seed.csv')) - snapshot_path = self.dir(os.path.join('snapshot', 'snapshot_seed.sql')) - - my_schema_name = self.unique_schema() - test_audit_schema = my_schema_name + '_dbt_test__audit' - - if model_database is None: - model_database = self.alternative_database - - model_config = self.rendered_model_config(database=model_database) - second_config = self.rendered_model_config( - schema=self.alternate_schema[-4:]) - - unrendered_model_config = self.unrendered_model_config( - database=model_database, materialized='view') - unrendered_second_config = self.unrendered_model_config( - schema=self.alternate_schema[-4:], materialized='view') - - seed_config = self.rendered_seed_config() - unrendered_seed_config = self.unrendered_seed_config() - - test_config = self.rendered_tst_config() - unrendered_test_config = self.unrendered_tst_config() - - snapshot_config = self.rendered_snapshot_config( - target_schema=self.alternate_schema) - unrendered_snapshot_config = self.unrendered_snapshot_config( - target_schema=self.alternate_schema - ) - - quote_database = quote_schema = True - relation_name_node_format = self._relation_name_format( - quote_database, quote_schema, quote_model - ) - relation_name_source_format = self._relation_name_format( - quote_database, quote_schema, quote_identifier=True - ) - - return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v5.json', - 'dbt_version': dbt.version.__version__, - 'nodes': { - 'model.test.model': { - 'compiled_path': Normalized('target/compiled/test/models/model.sql'), - 'build_path': None, - 'created_at': ANY, - 'name': 'model', - 'root_path': self.test_root_realpath, - 'relation_name': relation_name_node_format.format( - model_database, my_schema_name, 'model' - ), - 'resource_type': 'model', - 'path': 'model.sql', - 'original_file_path': model_sql_path, - 'package_name': 'test', - 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'sources': [], - 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, - 'unique_id': 'model.test.model', - 'fqn': ['test', 'model'], - 'tags': [], - 'meta': {}, - 'config': model_config, - 'schema': my_schema_name, - 'database': model_database, - 'deferred': False, - 'alias': 'model', - 'description': 'The test model', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'patch_path': 'test://' + model_schema_yml_path, - 'docs': {'show': False}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(model_sql_path), - 'unrendered_config': unrendered_model_config, - }, - 'model.test.second_model': { - 'compiled_path': Normalized('target/compiled/test/models/second_model.sql'), - 'build_path': None, - 'created_at': ANY, - 'name': 'second_model', - 'root_path': self.test_root_realpath, - 'relation_name': relation_name_node_format.format( - self.default_database, self.alternate_schema, - 'second_model' - ), - 'resource_type': 'model', - 'path': 'second_model.sql', - 'original_file_path': second_model_sql_path, - 'package_name': 'test', - 'raw_sql': LineIndifferent(_read_file(second_model_sql_path).rstrip('\r\n')), - 'refs': [['seed']], - 'sources': [], - 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, - 'unique_id': 'model.test.second_model', - 'fqn': ['test', 'second_model'], - 'tags': [], - 'meta': {}, - 'config': second_config, - 'schema': self.alternate_schema, - 'database': self.default_database, - 'deferred': False, - 'alias': 'second_model', - 'description': 'The second test model', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'patch_path': 'test://' + model_schema_yml_path, - 'docs': {'show': False}, - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(second_model_sql_path), - 'unrendered_config': unrendered_second_config - }, - 'seed.test.seed': { - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'compiled': True, - 'compiled_sql': '', - 'config': seed_config, - 'patch_path': 'test://' + seed_schema_yml_path, - 'path': 'seed.csv', - 'name': 'seed', - 'root_path': self.test_root_realpath, - 'relation_name': relation_name_node_format.format( - self.default_database, my_schema_name, 'seed' - ), - 'resource_type': 'seed', - 'raw_sql': '', - 'package_name': 'test', - 'original_file_path': seed_path, - 'refs': [], - 'sources': [], - 'depends_on': {'nodes': [], 'macros': []}, - 'unique_id': 'seed.test.seed', - 'fqn': ['test', 'seed'], - 'tags': [], - 'meta': {}, - 'schema': my_schema_name, - 'database': self.default_database, - 'alias': 'seed', - 'deferred': False, - 'description': 'The test seed', - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': '', - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(seed_path), - 'unrendered_config': unrendered_seed_config, - }, - 'test.test.not_null_model_id.d01cc630e6': { - 'alias': 'not_null_model_id', - 'compiled_path': Normalized('target/compiled/test/models/schema.yml/not_null_model_id.sql'), - 'build_path': None, - 'created_at': ANY, - 'column_name': 'id', - 'columns': {}, - 'config': test_config, - 'sources': [], - 'depends_on': { - 'macros': ['macro.dbt.test_not_null', 'macro.dbt.get_where_subquery'], - 'nodes': ['model.test.model'], - }, - 'deferred': False, - 'description': '', - 'file_key_name': 'models.model', - 'fqn': ['test', 'not_null_model_id'], - 'name': 'not_null_model_id', - 'original_file_path': model_schema_yml_path, - 'package_name': 'test', - 'patch_path': None, - 'path': Normalized('not_null_model_id.sql'), - 'raw_sql': "{{ test_not_null(**_dbt_generic_test_kwargs) }}", - 'refs': [['model']], - 'relation_name': None, - 'resource_type': 'test', - 'root_path': self.test_root_realpath, - 'schema': test_audit_schema, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'test.test.not_null_model_id.d01cc630e6', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': AnyStringWith('where id is null'), - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'test_metadata': { - 'namespace': None, - 'name': 'not_null', - 'kwargs': { - 'column_name': 'id', - 'model': "{{ get_where_subquery(ref('model')) }}", - }, - }, - 'checksum': {'name': 'none', 'checksum': ''}, - 'unrendered_config': unrendered_test_config, - }, - 'snapshot.test.snapshot_seed': { - 'alias': 'snapshot_seed', - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'checksum': self._checksum_file(snapshot_path), - 'columns': {}, - 'compiled': True, - 'compiled_sql': ANY, - 'config': snapshot_config, - 'database': self.default_database, - 'deferred': False, - 'depends_on': { - 'macros': [], - 'nodes': ['seed.test.seed'], - }, - 'description': '', - 'docs': {'show': True}, - 'extra_ctes': [], - 'extra_ctes_injected': True, - 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], - 'meta': {}, - 'name': 'snapshot_seed', - 'original_file_path': snapshot_path, - 'package_name': 'test', - 'patch_path': None, - 'path': normalize('snapshot_seed.sql'), - 'raw_sql': LineIndifferent( - _read_file(snapshot_path) - .replace('{% snapshot snapshot_seed %}', '') - .replace('{% endsnapshot %}', '')), - 'refs': [['seed']], - 'relation_name': relation_name_node_format.format( - self.default_database, self.alternate_schema, - 'snapshot_seed' - ), - 'resource_type': 'snapshot', - 'root_path': self.test_root_realpath, - 'schema': self.alternate_schema, - 'sources': [], - 'tags': [], - 'unique_id': 'snapshot.test.snapshot_seed', - 'unrendered_config': unrendered_snapshot_config, - }, - 'test.test.test_nothing_model_.5d38568946': { - 'alias': 'test_nothing_model_', - 'compiled_path': Normalized('target/compiled/test/models/schema.yml/test_nothing_model_.sql'), - 'build_path': None, - 'created_at': ANY, - 'column_name': None, - 'columns': {}, - 'config': test_config, - 'sources': [], - 'depends_on': { - 'macros': ['macro.test.test_nothing', 'macro.dbt.get_where_subquery'], - 'nodes': ['model.test.model'], - }, - 'deferred': False, - 'description': '', - 'file_key_name': 'models.model', - 'fqn': ['test', 'test_nothing_model_'], - 'name': 'test_nothing_model_', - 'original_file_path': model_schema_yml_path, - 'package_name': 'test', - 'patch_path': None, - 'path': normalize('test_nothing_model_.sql'), - 'raw_sql': "{{ test.test_nothing(**_dbt_generic_test_kwargs) }}", - 'refs': [['model']], - 'relation_name': None, - 'resource_type': 'test', - 'root_path': self.test_root_realpath, - 'schema': test_audit_schema, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'test.test.test_nothing_model_.5d38568946', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': AnyStringWith('select 0'), - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'test_metadata': { - 'namespace': 'test', - 'name': 'nothing', - 'kwargs': { - 'model': "{{ get_where_subquery(ref('model')) }}", - }, - }, - 'checksum': {'name': 'none', 'checksum': ''}, - 'unrendered_config': unrendered_test_config, - }, - 'test.test.unique_model_id.67b76558ff': { - 'alias': 'unique_model_id', - 'compiled_path': Normalized('target/compiled/test/models/schema.yml/unique_model_id.sql'), - 'build_path': None, - 'created_at': ANY, - 'column_name': 'id', - 'columns': {}, - 'config': test_config, - 'sources': [], - 'depends_on': { - 'macros': ['macro.dbt.test_unique', 'macro.dbt.get_where_subquery'], - 'nodes': ['model.test.model'], - }, - 'deferred': False, - 'description': '', - 'file_key_name': 'models.model', - 'fqn': ['test', 'unique_model_id'], - 'name': 'unique_model_id', - 'original_file_path': model_schema_yml_path, - 'package_name': 'test', - 'patch_path': None, - 'path': normalize('unique_model_id.sql'), - 'raw_sql': "{{ test_unique(**_dbt_generic_test_kwargs) }}", - 'refs': [['model']], - 'relation_name': None, - 'resource_type': 'test', - 'root_path': self.test_root_realpath, - 'schema': test_audit_schema, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'test.test.unique_model_id.67b76558ff', - 'docs': {'show': True}, - 'compiled': True, - 'compiled_sql': AnyStringWith('count(*)'), - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'test_metadata': { - 'namespace': None, - 'name': 'unique', - 'kwargs': { - 'column_name': 'id', - 'model': "{{ get_where_subquery(ref('model')) }}", - }, - }, - 'checksum': {'name': 'none', 'checksum': ''}, - 'unrendered_config': unrendered_test_config, - }, - }, - 'sources': { - 'source.test.my_source.my_table': { - 'created_at': ANY, - 'columns': { - 'id': { - 'description': 'An ID field', - 'name': 'id', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - } - }, - 'config': { - 'enabled': True, - }, - 'quoting': { - 'database': None, - 'schema': None, - 'identifier': True, - 'column': None, - }, - 'database': self.default_database, - 'description': 'My table', - 'external': None, - 'freshness': {'error_after': {'count': None, 'period': None}, 'warn_after': {'count': None, 'period': None}, 'filter': None}, - 'identifier': 'seed', - 'loaded_at_field': None, - 'loader': 'a_loader', - 'meta': {}, - 'name': 'my_table', - 'original_file_path': self.dir('models/schema.yml'), - 'package_name': 'test', - 'path': self.dir('models/schema.yml'), - 'patch_path': None, - 'relation_name': relation_name_source_format.format( - self.default_database, my_schema_name, 'seed' - ), - 'resource_type': 'source', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'source_description': 'My source', - 'source_name': 'my_source', - 'source_meta': {}, - 'tags': [], - 'unique_id': 'source.test.my_source.my_table', - 'fqn': ['test', 'my_source', 'my_table'], - 'unrendered_config': {}, - }, - }, - 'exposures': { - 'exposure.test.notebook_exposure': { - 'created_at': ANY, - 'depends_on': { - 'macros': [], - 'nodes': ['model.test.model', 'model.test.second_model'] - }, - 'description': 'A description of the complex exposure\n', - 'fqn': ['test', 'notebook_exposure'], - 'maturity': 'medium', - 'meta': {'tool': 'my_tool', 'languages': ['python']}, - 'tags': ['my_department'], - 'name': 'notebook_exposure', - 'original_file_path': self.dir('models/schema.yml'), - 'owner': { - 'email': 'something@example.com', - 'name': 'Some name' - }, - 'package_name': 'test', - 'path': 'schema.yml', - 'refs': [['model'], ['second_model']], - 'resource_type': 'exposure', - 'root_path': self.test_root_realpath, - 'sources': [], - 'type': 'notebook', - 'unique_id': 'exposure.test.notebook_exposure', - 'url': 'http://example.com/notebook/1' - }, - 'exposure.test.simple_exposure': { - 'created_at': ANY, - 'depends_on': { - 'macros': [], - 'nodes': [ - 'source.test.my_source.my_table', - 'model.test.model' - ], - }, - 'description': '', - 'fqn': ['test', 'simple_exposure'], - 'name': 'simple_exposure', - 'original_file_path': self.dir('models/schema.yml'), - 'owner': { - 'email': 'something@example.com', - 'name': None, - }, - 'package_name': 'test', - 'path': 'schema.yml', - 'refs': [['model']], - 'resource_type': 'exposure', - 'root_path': self.test_root_realpath, - 'sources': [['my_source', 'my_table']], - 'type': 'dashboard', - 'unique_id': 'exposure.test.simple_exposure', - 'url': None, - 'maturity': None, - 'meta': {}, - 'tags': [] - } - }, - 'metrics': {}, - 'selectors': {}, - 'parent_map': { - 'model.test.model': ['seed.test.seed'], - 'model.test.second_model': ['seed.test.seed'], - 'exposure.test.notebook_exposure': ['model.test.model', 'model.test.second_model'], - 'exposure.test.simple_exposure': ['model.test.model', 'source.test.my_source.my_table'], - 'seed.test.seed': [], - 'snapshot.test.snapshot_seed': ['seed.test.seed'], - 'source.test.my_source.my_table': [], - 'test.test.not_null_model_id.d01cc630e6': ['model.test.model'], - 'test.test.test_nothing_model_.5d38568946': ['model.test.model'], - 'test.test.unique_model_id.67b76558ff': ['model.test.model'], - }, - 'child_map': { - 'model.test.model': [ - 'exposure.test.notebook_exposure', - 'exposure.test.simple_exposure', - 'test.test.not_null_model_id.d01cc630e6', - 'test.test.test_nothing_model_.5d38568946', - 'test.test.unique_model_id.67b76558ff', - ], - 'model.test.second_model': ['exposure.test.notebook_exposure'], - 'exposure.test.notebook_exposure': [], - 'exposure.test.simple_exposure': [], - 'seed.test.seed': ['model.test.model', - 'model.test.second_model', - 'snapshot.test.snapshot_seed'], - 'snapshot.test.snapshot_seed': [], - 'source.test.my_source.my_table': ['exposure.test.simple_exposure'], - 'test.test.not_null_model_id.d01cc630e6': [], - 'test.test.test_nothing_model_.5d38568946': [], - 'test.test.unique_model_id.67b76558ff': [], - }, - 'docs': { - 'dbt.__overview__': ANY, - 'test.macro_info': ANY, - 'test.macro_arg_info': ANY, - }, - 'disabled': {}, - } - - def expected_postgres_references_manifest(self, model_database=None): - if model_database is None: - model_database = self.default_database - my_schema_name = self.unique_schema() - docs_path = self.dir('ref_models/docs.md') - ephemeral_copy_path = self.dir('ref_models/ephemeral_copy.sql') - ephemeral_summary_path = self.dir('ref_models/ephemeral_summary.sql') - view_summary_path = self.dir('ref_models/view_summary.sql') - seed_path = self.dir('seed/seed.csv') - snapshot_path = self.dir('snapshot/snapshot_seed.sql') - - return { - 'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v5.json', - 'dbt_version': dbt.version.__version__, - 'nodes': { - 'model.test.ephemeral_copy': { - 'alias': 'ephemeral_copy', - 'compiled_path': Normalized('target/compiled/test/ref_models/ephemeral_copy.sql'), - 'build_path': None, - 'created_at': ANY, - 'columns': {}, - 'config': self.rendered_model_config(materialized='ephemeral'), - 'sources': [['my_source', 'my_table']], - 'depends_on': { - 'macros': [], - 'nodes': ['source.test.my_source.my_table'] - }, - 'deferred': False, - 'description': '', - 'docs': {'show': True}, - 'fqn': ['test', 'ephemeral_copy'], - 'name': 'ephemeral_copy', - 'original_file_path': ephemeral_copy_path, - 'package_name': 'test', - 'patch_path': None, - 'path': 'ephemeral_copy.sql', - 'raw_sql': LineIndifferent( - '{{\n config(\n materialized = "ephemeral"\n )\n}}' - '\n\nselect * from {{ source("my_source", "my_table") }}' - ), - 'refs': [], - 'relation_name': None, - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.ephemeral_copy', - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(ephemeral_copy_path), - 'unrendered_config': self.unrendered_model_config(materialized='ephemeral'), - }, - 'model.test.ephemeral_summary': { - 'alias': 'ephemeral_summary', - 'compiled_path': Normalized('target/compiled/test/ref_models/ephemeral_summary.sql'), - 'build_path': None, - 'created_at': ANY, - 'columns': { - 'first_name': { - 'description': 'The first name being summarized', - 'name': 'first_name', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ct': { - 'description': 'The number of instances of the first name', - 'name': 'ct', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'config': self.rendered_model_config(materialized='table'), - 'sources': [], - 'depends_on': { - 'macros': [], - 'nodes': ['model.test.ephemeral_copy'] - }, - 'deferred': False, - 'description': 'A summmary table of the ephemeral copy of the seed data', - 'docs': {'show': True}, - 'fqn': ['test', 'ephemeral_summary'], - 'name': 'ephemeral_summary', - 'original_file_path': ephemeral_summary_path, - 'package_name': 'test', - 'patch_path': 'test://' + self.dir('ref_models/schema.yml'), - 'path': 'ephemeral_summary.sql', - 'raw_sql': LineIndifferent( - '{{\n config(\n materialized = "table"\n )\n}}\n\n' - 'select first_name, count(*) as ct from ' - "{{ref('ephemeral_copy')}}\ngroup by first_name\n" - 'order by first_name asc' - ), - 'refs': [['ephemeral_copy']], - 'relation_name': '"{0}"."{1}".ephemeral_summary'.format( - model_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.ephemeral_summary', - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [ANY], - 'checksum': self._checksum_file(ephemeral_summary_path), - 'unrendered_config': self.unrendered_model_config(materialized='table'), - }, - 'model.test.view_summary': { - 'alias': 'view_summary', - 'compiled_path': Normalized('target/compiled/test/ref_models/view_summary.sql'), - 'build_path': None, - 'created_at': ANY, - 'columns': { - 'first_name': { - 'description': 'The first name being summarized', - 'name': 'first_name', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ct': { - 'description': 'The number of instances of the first name', - 'name': 'ct', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'config': self.rendered_model_config(), - 'database': self.default_database, - 'depends_on': { - 'macros': [], - 'nodes': ['model.test.ephemeral_summary'] - }, - 'deferred': False, - 'description': 'A view of the summary of the ephemeral copy of the seed data', - 'docs': {'show': True}, - 'fqn': ['test', 'view_summary'], - 'name': 'view_summary', - 'original_file_path': view_summary_path, - 'package_name': 'test', - 'patch_path': 'test://' + self.dir('ref_models/schema.yml'), - 'path': 'view_summary.sql', - 'raw_sql': LineIndifferent( - '{{\n config(\n materialized = "view"\n )\n}}\n\n' - 'select first_name, ct from ' - "{{ref('ephemeral_summary')}}\norder by ct asc" - ), - 'refs': [['ephemeral_summary']], - 'relation_name': '"{0}"."{1}".view_summary'.format( - model_database, my_schema_name - ), - 'resource_type': 'model', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'sources': [], - 'tags': [], - 'meta': {}, - 'unique_id': 'model.test.view_summary', - 'compiled': True, - 'compiled_sql': ANY, - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(view_summary_path), - 'unrendered_config': self.unrendered_model_config(materialized='view'), - }, - 'seed.test.seed': { - 'alias': 'seed', - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'columns': { - 'id': { - 'name': 'id', - 'description': 'The user ID number', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'first_name': { - 'name': 'first_name', - 'description': "The user's first name", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'email': { - 'name': 'email', - 'description': "The user's email", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'ip_address': { - 'name': 'ip_address', - 'description': "The user's IP address", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - 'updated_at': { - 'name': 'updated_at', - 'description': "The last time this user's email was updated", - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - }, - }, - 'config': self.rendered_seed_config(), - 'sources': [], - 'depends_on': {'macros': [], 'nodes': []}, - 'deferred': False, - 'description': 'The test seed', - 'docs': {'show': True}, - 'fqn': ['test', 'seed'], - 'name': 'seed', - 'original_file_path': seed_path, - 'package_name': 'test', - 'patch_path': 'test://' + self.dir('seed/schema.yml'), - 'path': 'seed.csv', - 'raw_sql': '', - 'refs': [], - 'relation_name': '"{0}"."{1}".seed'.format( - model_database, my_schema_name - ), - 'resource_type': 'seed', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'database': self.default_database, - 'tags': [], - 'meta': {}, - 'unique_id': 'seed.test.seed', - 'compiled': True, - 'compiled_sql': '', - 'extra_ctes_injected': True, - 'extra_ctes': [], - 'checksum': self._checksum_file(seed_path), - 'unrendered_config': self.unrendered_seed_config(), - }, - 'snapshot.test.snapshot_seed': { - 'alias': 'snapshot_seed', - 'compiled_path': None, - 'build_path': None, - 'created_at': ANY, - 'checksum': self._checksum_file(snapshot_path), - 'columns': {}, - 'compiled': True, - 'compiled_sql': ANY, - 'config': self.rendered_snapshot_config( - target_schema=self.alternate_schema - ), - 'database': model_database, - 'deferred': False, - 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, - 'description': '', - 'docs': {'show': True}, - 'extra_ctes': [], - 'extra_ctes_injected': True, - 'fqn': ['test', 'snapshot_seed', 'snapshot_seed'], - 'meta': {}, - 'name': 'snapshot_seed', - 'original_file_path': snapshot_path, - 'package_name': 'test', - 'patch_path': None, - 'path': 'snapshot_seed.sql', - 'raw_sql': ANY, - 'refs': [['seed']], - 'relation_name': '"{0}"."{1}".snapshot_seed'.format( - model_database, self.alternate_schema - ), - 'resource_type': 'snapshot', - 'root_path': self.test_root_realpath, - 'schema': self.alternate_schema, - 'sources': [], - 'tags': [], - 'unique_id': 'snapshot.test.snapshot_seed', - 'unrendered_config': self.unrendered_snapshot_config( - target_schema=self.alternate_schema - )} - }, - 'sources': { - 'source.test.my_source.my_table': { - 'columns': { - 'id': { - 'description': 'An ID field', - 'name': 'id', - 'data_type': None, - 'meta': {}, - 'quote': None, - 'tags': [], - } - }, - 'config': { - 'enabled': True, - }, - 'quoting': { - 'database': False, - 'schema': None, - 'identifier': True, - 'column': None, - }, - 'created_at': ANY, - 'database': self.default_database, - 'description': 'My table', - 'external': None, - 'freshness': {'error_after': {'count': None, 'period': None}, 'warn_after': {'count': None, 'period': None}, 'filter': None}, - 'identifier': 'seed', - 'loaded_at_field': None, - 'loader': 'a_loader', - 'meta': {}, - 'name': 'my_table', - 'original_file_path': self.dir('ref_models/schema.yml'), - 'package_name': 'test', - 'path': self.dir('ref_models/schema.yml'), - 'patch_path': None, - 'relation_name': '{0}."{1}"."seed"'.format( - self.default_database, my_schema_name - ), - 'resource_type': 'source', - 'root_path': self.test_root_realpath, - 'schema': my_schema_name, - 'source_description': 'My source', - 'source_name': 'my_source', - 'source_meta': {}, - 'tags': [], - 'unique_id': 'source.test.my_source.my_table', - 'fqn': ['test', 'my_source', 'my_table'], - 'unrendered_config': {} - }, - }, - 'exposures': { - 'exposure.test.notebook_exposure': { - 'created_at': ANY, - 'depends_on': { - 'macros': [], - 'nodes': ['model.test.view_summary'] - }, - 'description': 'A description of the complex exposure', - 'fqn': ['test', 'notebook_exposure'], - 'maturity': 'medium', - 'meta': {'tool': 'my_tool', 'languages': ['python']}, - 'tags': ['my_department'], - 'name': 'notebook_exposure', - 'original_file_path': self.dir('ref_models/schema.yml'), - 'owner': { - 'email': 'something@example.com', - 'name': 'Some name' - }, - 'package_name': 'test', - 'path': 'schema.yml', - 'refs': [['view_summary']], - 'resource_type': 'exposure', - 'root_path': self.test_root_realpath, - 'sources': [], - 'type': 'notebook', - 'unique_id': 'exposure.test.notebook_exposure', - 'url': 'http://example.com/notebook/1' - }, - }, - 'metrics': {}, - 'selectors': {}, - 'docs': { - 'dbt.__overview__': ANY, - 'test.column_info': { - 'block_contents': 'An ID field', - 'name': 'column_info', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.column_info', - }, - 'test.ephemeral_summary': { - 'block_contents': ( - 'A summmary table of the ephemeral copy of the seed data' - ), - 'name': 'ephemeral_summary', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.ephemeral_summary', - }, - 'test.source_info': { - 'block_contents': 'My source', - 'name': 'source_info', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.source_info', - }, - 'test.summary_count': { - 'block_contents': 'The number of instances of the first name', - 'name': 'summary_count', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.summary_count', - }, - 'test.summary_first_name': { - 'block_contents': 'The first name being summarized', - 'name': 'summary_first_name', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.summary_first_name', - }, - 'test.table_info': { - 'block_contents': 'My table', - 'name': 'table_info', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.table_info', - }, - 'test.view_summary': { - 'block_contents': ( - 'A view of the summary of the ephemeral copy of the ' - 'seed data' - ), - 'name': 'view_summary', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.view_summary', - }, - 'test.macro_info': { - 'block_contents': 'My custom test that I wrote that does nothing', - 'name': 'macro_info', - 'original_file_path': self.dir('macros/macro.md'), - 'package_name': 'test', - 'path': 'macro.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.macro_info', - }, - 'test.notebook_info': { - 'block_contents': 'A description of the complex exposure', - 'name': 'notebook_info', - 'original_file_path': docs_path, - 'package_name': 'test', - 'path': 'docs.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.notebook_info' - }, - 'test.macro_arg_info': { - 'block_contents': 'The model for my custom test', - 'name': 'macro_arg_info', - 'original_file_path': self.dir('macros/macro.md'), - 'package_name': 'test', - 'path': 'macro.md', - 'root_path': self.test_root_realpath, - 'unique_id': 'test.macro_arg_info', - }, - }, - 'child_map': { - 'model.test.ephemeral_copy': ['model.test.ephemeral_summary'], - 'exposure.test.notebook_exposure': [], - 'model.test.ephemeral_summary': ['model.test.view_summary'], - 'model.test.view_summary': ['exposure.test.notebook_exposure'], - 'seed.test.seed': ['snapshot.test.snapshot_seed'], - 'snapshot.test.snapshot_seed': [], - 'source.test.my_source.my_table': ['model.test.ephemeral_copy'], - }, - 'parent_map': { - 'model.test.ephemeral_copy': ['source.test.my_source.my_table'], - 'model.test.ephemeral_summary': ['model.test.ephemeral_copy'], - 'model.test.view_summary': ['model.test.ephemeral_summary'], - 'exposure.test.notebook_exposure': ['model.test.view_summary'], - 'seed.test.seed': [], - 'snapshot.test.snapshot_seed': ['seed.test.seed'], - 'source.test.my_source.my_table': [], - }, - 'disabled': {}, - 'macros': { - 'macro.test.test_nothing': { - 'name': 'test_nothing', - 'depends_on': {'macros': []}, - 'created_at': ANY, - 'description': 'My custom test that I wrote that does nothing', - 'docs': {'show': True}, - 'macro_sql': AnyStringWith('test nothing'), - 'original_file_path': self.dir('macros/dummy_test.sql'), - 'path': self.dir('macros/dummy_test.sql'), - 'package_name': 'test', - 'meta': { - 'some_key': 100, - }, - 'patch_path': 'test://' + self.dir('macros/schema.yml'), - 'resource_type': 'macro', - 'unique_id': 'macro.test.test_nothing', - 'tags': [], - 'root_path': self.test_root_realpath, - 'arguments': [ - { - 'name': 'model', - 'type': 'Relation', - 'description': 'The model for my custom test', - }, - ], - } - } - } - - def _checksum_file(self, path): - """windows has silly git behavior that adds newlines, and python does - silly things if we just open(..., 'r').encode('utf-8'). - """ - with open(self.dir(path), 'rb') as fp: - hashed = hashlib.sha256(fp.read()).hexdigest() - return { - 'name': 'sha256', - 'checksum': hashed, - } - - def _path_to(self, searched_path: str, relative_path: str): - return { - 'searched_path': normalize(searched_path), - 'relative_path': normalize(relative_path), - 'project_root': normalize(self.test_root_dir), - } - - def _absolute_path_to(self, searched_path: str, relative_path: str): - return os.path.join( - normalize(self.test_root_dir), - normalize(searched_path), - normalize(relative_path) - ) - - def _relation_name_format(self, quote_database: bool, quote_schema: bool, - quote_identifier: bool): - return ".".join(( - self._quote("{0}") if quote_database else '{0}', - self._quote("{1}") if quote_schema else '{1}', - self._quote("{2}") if quote_identifier else '{2}', - )) - - def verify_metadata(self, metadata, dbt_schema_version): - assert 'generated_at' in metadata - self.assertBetween(metadata['generated_at'], - start=self.generate_start_time) - assert 'dbt_version' in metadata - assert metadata['dbt_version'] == dbt.version.__version__ - assert 'dbt_schema_version' in metadata - assert metadata['dbt_schema_version'] == dbt_schema_version - assert metadata['invocation_id'] == dbt.tracking.active_user.invocation_id - key = 'env_key' - if os.name == 'nt': - key = key.upper() - assert metadata['env'] == { - key: 'env_value' - } - - def verify_manifest(self, expected_manifest): - self.assertTrue(os.path.exists('./target/manifest.json')) - - manifest = _read_json('./target/manifest.json') - - manifest_keys = frozenset({ - 'nodes', 'sources', 'macros', 'parent_map', 'child_map', 'metrics', - 'docs', 'metadata', 'docs', 'disabled', 'exposures', 'selectors', - }) - - self.assertEqual(frozenset(manifest), manifest_keys) - - for key in manifest_keys: - if key == 'macros': - self.verify_manifest_macros( - manifest, expected_manifest.get('macros')) - elif key == 'metadata': - metadata = manifest['metadata'] - self.verify_metadata( - metadata, 'https://schemas.getdbt.com/dbt/manifest/v5.json') - assert 'project_id' in metadata and metadata[ - 'project_id'] == '098f6bcd4621d373cade4e832627b4f6' - assert 'send_anonymous_usage_stats' in metadata and metadata[ - 'send_anonymous_usage_stats'] is False - assert 'adapter_type' in metadata and metadata['adapter_type'] == self.adapter_type - else: - self.assertIn(key, expected_manifest) # sanity check - self.assertEqual(manifest[key], expected_manifest[key]) - - def _quote(self, value): - quote_char = '"' - return '{0}{1}{0}'.format(quote_char, value) - - def expected_run_results(self): - """ - The expected results of this run. - """ - - return [ - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'model.test.model', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'model.test.second_model', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'seed.test.seed', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'snapshot.test.snapshot_seed', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'test.test.not_null_model_id.d01cc630e6', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'test.test.test_nothing_model_.5d38568946', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'test.test.unique_model_id.67b76558ff', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - ] - - def expected_postgres_references_run_results(self): - return [ - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'model.test.ephemeral_summary', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'model.test.view_summary', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'seed.test.seed', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - { - 'status': 'success', - 'message': None, - 'execution_time': AnyFloat(), - 'unique_id': 'snapshot.test.snapshot_seed', - 'adapter_response': ANY, - 'thread_id': ANY, - 'timing': [ANY, ANY], - 'failures': ANY, - }, - ] - - def verify_run_results(self, expected_run_results): - run_results = _read_json('./target/run_results.json') - assert 'metadata' in run_results - self.verify_metadata( - run_results['metadata'], 'https://schemas.getdbt.com/dbt/run-results/v4.json') - self.assertIn('elapsed_time', run_results) - self.assertGreater(run_results['elapsed_time'], 0) - self.assertTrue( - isinstance(run_results['elapsed_time'], float), - "run_results['elapsed_time'] is of type {}, expected float".format( - str(type(run_results['elapsed_time']))) - ) - - assert 'args' in run_results - # sort the results so we can make reasonable assertions - run_results['results'].sort(key=lambda r: r['unique_id']) - assert run_results['results'] == expected_run_results - set(run_results) == {'elapsed_time', 'results', 'metadata'} - - @use_profile('postgres') - def test__postgres__run_and_generate_no_compile(self): - self.run_and_generate( - alternate_db=self.default_database, args=['--no-compile']) - self.verify_catalog(self.expected_postgres_catalog()) - self.assertFalse(os.path.exists('./target/manifest.json')) - - @use_profile('postgres') - def test__postgres__run_and_generate(self): - self.run_and_generate(alternate_db=self.default_database) - self.verify_catalog(self.expected_postgres_catalog()) - self.verify_manifest(self.expected_seeded_manifest( - model_database=self.default_database - )) - self.verify_run_results(self.expected_run_results()) - - @use_profile('postgres') - def test__postgres_references(self): - self.run_and_generate( - {'model-paths': [self.dir('ref_models')]}, - model_count=2, - alternate_db=self.default_database - ) - - self.verify_catalog(self.expected_postgres_references_catalog()) - self.verify_manifest(self.expected_postgres_references_manifest()) - self.verify_run_results( - self.expected_postgres_references_run_results()) - - @use_profile('postgres') - def test_postgres_asset_paths_copied(self): - self.run_and_generate( - {'asset-paths': [self.dir('assets'), - self.dir('non-existent-assets')]}, - ) - - assert os.path.exists('./target/assets') - assert os.path.exists('./target/assets/lorem-ipsum.txt') - - assert not os.path.exists('./target/non-existent-assets') - - -class TestDocsGenerateOverride(DBTIntegrationTest): - - @property - def schema(self): - return 'docs_generate_029' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("trivial_models") - - @property - def project_config(self): - return { - 'config-version': 2, - 'macro-paths': [self.dir('fail_macros')], - } - - @use_profile('postgres') - def test_postgres_override_used(self): - self.assertEqual(len(self.run_dbt(['run'])), 1) - # this should pick up our failure macro and raise a compilation exception - with self.assertRaises(CompilationException) as exc: - self.run_dbt(['--warn-error', 'docs', 'generate']) - - self.assertIn('rejected: no catalogs for you', str(exc.exception)) - - -@mark.skipif(os.name != 'nt', reason='This is only relevant on windows') -class TestDocsGenerateLongWindowsPaths(DBTIntegrationTest): - def _generate_test_root_dir(self): - assert os.name == 'nt' - magic_prefix = '\\\\?\\' - - # tempfile.mkdtemp doesn't use `\\?\` by default so we have to - # get a tiny bit creative. - temp_dir = tempfile.gettempdir() - if not temp_dir.startswith(magic_prefix): - temp_dir = magic_prefix + temp_dir - outer = tempfile.mkdtemp(prefix='dbt-int-test-', dir=temp_dir) - # then inside _that_ directory make a new one that gets us to just - # barely 260 total. I picked 250 to account for the '\' and anything - # else. The key is that len(inner) + len('target\\compiled\\...') will - # be >260 chars - new_length = 250 - len(outer) - inner = os.path.join(outer, 'a'*new_length) - os.mkdir(inner) - return normalize(inner) - - def _symlink_test_folders(self): - # dbt's normal symlink behavior breaks this test, so special-case it - for entry in os.listdir(self.test_original_source_path): - src = os.path.join(self.test_original_source_path, entry) - tst = os.path.join(self.test_root_dir, entry) - if entry == 'trivial_models': - shutil.copytree(src, tst) - elif entry == 'local_dependency': - continue - elif os.path.isdir(entry) or entry.endswith('.sql'): - os.symlink(src, tst) - - @property - def schema(self): - return 'docs_generate_029' - - @staticmethod - def dir(path): - return normalize(path) - - @property - def models(self): - return self.dir("trivial_models") - - def run_and_generate(self): - self.assertEqual(len(self.run_dbt(['run'])), 1) - os.remove(normalize('target/manifest.json')) - os.remove(normalize('target/run_results.json')) - self.run_dbt(['docs', 'generate']) - - @use_profile('postgres') - def test_postgres_long_paths(self): - self.run_and_generate() - # this doesn't use abspath, so all should be well here - manifest = _read_json('./target/manifest.json') - self.assertIn('nodes', manifest) - assert os.path.exists('./target/run/test/trivial_models/model.sql') - self.run_dbt(['clean']) - assert not os.path.exists('./target/run') diff --git a/test/integration/029_docs_generate_tests/trivial_models/model.sql b/test/integration/029_docs_generate_tests/trivial_models/model.sql deleted file mode 100644 index 43258a71464..00000000000 --- a/test/integration/029_docs_generate_tests/trivial_models/model.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id diff --git a/tests/functional/docs_generate/expected_catalog.py b/tests/functional/docs_generate/expected_catalog.py new file mode 100644 index 00000000000..43937f48c9a --- /dev/null +++ b/tests/functional/docs_generate/expected_catalog.py @@ -0,0 +1,244 @@ +def no_stats(): + return { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": False, + "description": "Indicates whether there are statistics for this table", + "include": False, + }, + } + + +def base_expected_catalog( + project, + id_type, + text_type, + time_type, + view_type, + table_type, + model_stats, + seed_stats=None, + case=None, + case_columns=False, + model_database=None, +): + + if case is None: + + def case(x): + return x + + col_case = case if case_columns else lambda x: x + + if seed_stats is None: + seed_stats = model_stats + + if model_database is None: + model_database = project.database + my_schema_name = project.test_schema + role = "root" + alternate_schema = project.test_schema + "_test" + + expected_cols = { + col_case("id"): { + "name": col_case("id"), + "index": 1, + "type": id_type, + "comment": None, + }, + col_case("first_name"): { + "name": col_case("first_name"), + "index": 2, + "type": text_type, + "comment": None, + }, + col_case("email"): { + "name": col_case("email"), + "index": 3, + "type": text_type, + "comment": None, + }, + col_case("ip_address"): { + "name": col_case("ip_address"), + "index": 4, + "type": text_type, + "comment": None, + }, + col_case("updated_at"): { + "name": col_case("updated_at"), + "index": 5, + "type": time_type, + "comment": None, + }, + } + return { + "nodes": { + "model.test.model": { + "unique_id": "model.test.model", + "metadata": { + "schema": my_schema_name, + "database": model_database, + "name": case("model"), + "type": view_type, + "comment": None, + "owner": role, + }, + "stats": model_stats, + "columns": expected_cols, + }, + "model.test.second_model": { + "unique_id": "model.test.second_model", + "metadata": { + "schema": alternate_schema, + "database": project.database, + "name": case("second_model"), + "type": view_type, + "comment": None, + "owner": role, + }, + "stats": model_stats, + "columns": expected_cols, + }, + "seed.test.seed": { + "unique_id": "seed.test.seed", + "metadata": { + "schema": my_schema_name, + "database": project.database, + "name": case("seed"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": seed_stats, + "columns": expected_cols, + }, + }, + "sources": { + "source.test.my_source.my_table": { + "unique_id": "source.test.my_source.my_table", + "metadata": { + "schema": my_schema_name, + "database": project.database, + "name": case("seed"), + "type": table_type, + "comment": None, + "owner": role, + }, + "stats": seed_stats, + "columns": expected_cols, + }, + }, + } + + +def expected_references_catalog(project): + model_database = project.database + my_schema_name = project.test_schema + role = "root" + stats = no_stats() + summary_columns = { + "first_name": { + "name": "first_name", + "index": 1, + "type": "text", + "comment": None, + }, + "ct": { + "name": "ct", + "index": 2, + "type": "bigint", + "comment": None, + }, + } + + seed_columns = { + "id": { + "name": "id", + "index": 1, + "type": "integer", + "comment": None, + }, + "first_name": { + "name": "first_name", + "index": 2, + "type": "text", + "comment": None, + }, + "email": { + "name": "email", + "index": 3, + "type": "text", + "comment": None, + }, + "ip_address": { + "name": "ip_address", + "index": 4, + "type": "text", + "comment": None, + }, + "updated_at": { + "name": "updated_at", + "index": 5, + "type": "timestamp without time zone", + "comment": None, + }, + } + return { + "nodes": { + "seed.test.seed": { + "unique_id": "seed.test.seed", + "metadata": { + "schema": my_schema_name, + "database": project.database, + "name": "seed", + "type": "BASE TABLE", + "comment": None, + "owner": role, + }, + "stats": stats, + "columns": seed_columns, + }, + "model.test.ephemeral_summary": { + "unique_id": "model.test.ephemeral_summary", + "metadata": { + "schema": my_schema_name, + "database": model_database, + "name": "ephemeral_summary", + "type": "BASE TABLE", + "comment": None, + "owner": role, + }, + "stats": stats, + "columns": summary_columns, + }, + "model.test.view_summary": { + "unique_id": "model.test.view_summary", + "metadata": { + "schema": my_schema_name, + "database": model_database, + "name": "view_summary", + "type": "VIEW", + "comment": None, + "owner": role, + }, + "stats": stats, + "columns": summary_columns, + }, + }, + "sources": { + "source.test.my_source.my_table": { + "unique_id": "source.test.my_source.my_table", + "metadata": { + "schema": my_schema_name, + "database": project.database, + "name": "seed", + "type": "BASE TABLE", + "comment": None, + "owner": role, + }, + "stats": stats, + "columns": seed_columns, + }, + }, + } diff --git a/tests/functional/docs_generate/expected_manifest.py b/tests/functional/docs_generate/expected_manifest.py new file mode 100644 index 00000000000..aeadbec10d1 --- /dev/null +++ b/tests/functional/docs_generate/expected_manifest.py @@ -0,0 +1,1321 @@ +import hashlib +import dbt +import os +from unittest.mock import ANY + +# This produces an "expected manifest", with a number of the fields +# modified to avoid ephemeral changes. +# ANY +# AnyStringWith +# LineIndifferent +# It also uses some convenience methods to generate the +# various config dictionaries. + + +def get_rendered_model_config(**updates): + result = { + "database": None, + "schema": None, + "alias": None, + "enabled": True, + "materialized": "view", + "pre-hook": [], + "post-hook": [], + "column_types": {}, + "quoting": {}, + "tags": [], + "persist_docs": {}, + "full_refresh": None, + "on_schema_change": "ignore", + "meta": {}, + "unique_key": None, + } + result.update(updates) + return result + + +def get_unrendered_model_config(**updates): + return updates + + +def get_rendered_seed_config(**updates): + result = { + "enabled": True, + "materialized": "seed", + "persist_docs": {}, + "pre-hook": [], + "post-hook": [], + "column_types": {}, + "quoting": {}, + "tags": [], + "quote_columns": True, + "full_refresh": None, + "on_schema_change": "ignore", + "database": None, + "schema": None, + "alias": None, + "meta": {}, + "unique_key": None, + } + result.update(updates) + return result + + +def get_unrendered_seed_config(**updates): + result = {"quote_columns": True} + result.update(updates) + return result + + +def get_rendered_snapshot_config(**updates): + result = { + "database": None, + "schema": None, + "alias": None, + "enabled": True, + "materialized": "snapshot", + "pre-hook": [], + "post-hook": [], + "column_types": {}, + "quoting": {}, + "tags": [], + "persist_docs": {}, + "full_refresh": None, + "on_schema_change": "ignore", + "strategy": "check", + "check_cols": "all", + "unique_key": "id", + "target_schema": None, + "meta": {}, + } + result.update(updates) + return result + + +def get_unrendered_snapshot_config(**updates): + result = {"check_cols": "all", "strategy": "check", "target_schema": None, "unique_key": "id"} + result.update(updates) + return result + + +def get_rendered_tst_config(**updates): + result = { + "enabled": True, + "materialized": "test", + "tags": [], + "severity": "ERROR", + "store_failures": None, + "warn_if": "!= 0", + "error_if": "!= 0", + "fail_calc": "count(*)", + "where": None, + "limit": None, + "database": None, + "schema": "dbt_test__audit", + "alias": None, + "meta": {}, + } + result.update(updates) + return result + + +def get_unrendered_tst_config(**updates): + result = {} + result.update(updates) + return result + + +def quote(value): + quote_char = '"' + return "{0}{1}{0}".format(quote_char, value) + + +def relation_name_format(quote_database: bool, quote_schema: bool, quote_identifier: bool): + return ".".join( + ( + quote("{0}") if quote_database else "{0}", + quote("{1}") if quote_schema else "{1}", + quote("{2}") if quote_identifier else "{2}", + ) + ) + + +def checksum_file(path): + """windows has silly git behavior that adds newlines, and python does + silly things if we just open(..., 'r').encode('utf-8'). + """ + with open(path, "rb") as fp: + hashed = hashlib.sha256(fp.read()).hexdigest() + return { + "name": "sha256", + "checksum": hashed, + } + + +def read_file_replace_returns(path): + with open(path, "r") as fp: + return fp.read().replace("\r", "").replace("\n", "") + + +class LineIndifferent: + def __init__(self, expected): + self.expected = expected.replace("\r", "") + + def __eq__(self, other): + got = other.replace("\r", "").replace("\n", "") + return self.expected == got + + def __repr__(self): + return "LineIndifferent({!r})".format(self.expected) + + def __str__(self): + return self.__repr__() + + +class AnyStringWith: + def __init__(self, contains=None): + self.contains = contains + + def __eq__(self, other): + if not isinstance(other, str): + return False + + if self.contains is None: + return True + + return self.contains in other + + def __repr__(self): + return "AnyStringWith<{!r}>".format(self.contains) + + +def expected_seeded_manifest(project, model_database=None, quote_model=False): + + model_sql_path = os.path.join("models", "model.sql") + second_model_sql_path = os.path.join("models", "second_model.sql") + model_schema_yml_path = os.path.join("models", "schema.yml") + seed_schema_yml_path = os.path.join("seeds", "schema.yml") + seed_path = os.path.join("seeds", "seed.csv") + snapshot_path = os.path.join("snapshots", "snapshot_seed.sql") + + my_schema_name = project.test_schema + alternate_schema = project.test_schema + "_test" + test_audit_schema = my_schema_name + "_dbt_test__audit" + + model_database = project.database + + model_config = get_rendered_model_config() + second_config = get_rendered_model_config(schema="test") + + unrendered_model_config = get_unrendered_model_config(materialized="view") + + unrendered_second_config = get_unrendered_model_config(schema="test", materialized="view") + + seed_config = get_rendered_seed_config() + unrendered_seed_config = get_unrendered_seed_config() + + test_config = get_rendered_tst_config() + unrendered_test_config = get_unrendered_tst_config() + + snapshot_config = get_rendered_snapshot_config(target_schema=alternate_schema) + unrendered_snapshot_config = get_unrendered_snapshot_config(target_schema=alternate_schema) + + quote_database = quote_schema = True + relation_name_node_format = relation_name_format(quote_database, quote_schema, quote_model) + relation_name_source_format = relation_name_format( + quote_database, quote_schema, quote_identifier=True + ) + + compiled_model_path = os.path.join("target", "compiled", "test", "models") + + model_raw_sql = read_file_replace_returns(model_sql_path).rstrip("\r\n") + + return { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "dbt_version": dbt.version.__version__, + "nodes": { + "model.test.model": { + "compiled_path": os.path.join(compiled_model_path, "model.sql"), + "build_path": None, + "created_at": ANY, + "name": "model", + "root_path": project.project_root, + "relation_name": relation_name_node_format.format( + model_database, my_schema_name, "model" + ), + "resource_type": "model", + "path": "model.sql", + "original_file_path": model_sql_path, + "package_name": "test", + "raw_sql": LineIndifferent(model_raw_sql), + "refs": [["seed"]], + "sources": [], + "depends_on": {"nodes": ["seed.test.seed"], "macros": []}, + "unique_id": "model.test.model", + "fqn": ["test", "model"], + "tags": [], + "meta": {}, + "config": model_config, + "schema": my_schema_name, + "database": model_database, + "deferred": False, + "alias": "model", + "description": "The test model", + "columns": { + "id": { + "name": "id", + "description": "The user ID number", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "first_name": { + "name": "first_name", + "description": "The user's first name", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "email": { + "name": "email", + "description": "The user's email", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "ip_address": { + "name": "ip_address", + "description": "The user's IP address", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "updated_at": { + "name": "updated_at", + "description": "The last time this user's email was updated", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + }, + "patch_path": "test://" + model_schema_yml_path, + "docs": {"show": False}, + "compiled": True, + "compiled_sql": ANY, + "extra_ctes_injected": True, + "extra_ctes": [], + "checksum": checksum_file(model_sql_path), + "unrendered_config": unrendered_model_config, + }, + "model.test.second_model": { + "compiled_path": os.path.join(compiled_model_path, "second_model.sql"), + "build_path": None, + "created_at": ANY, + "name": "second_model", + "root_path": project.project_root, + "relation_name": relation_name_node_format.format( + project.database, alternate_schema, "second_model" + ), + "resource_type": "model", + "path": "second_model.sql", + "original_file_path": second_model_sql_path, + "package_name": "test", + "raw_sql": LineIndifferent( + read_file_replace_returns(second_model_sql_path).rstrip("\r\n") + ), + "refs": [["seed"]], + "sources": [], + "depends_on": {"nodes": ["seed.test.seed"], "macros": []}, + "unique_id": "model.test.second_model", + "fqn": ["test", "second_model"], + "tags": [], + "meta": {}, + "config": second_config, + "schema": alternate_schema, + "database": project.database, + "deferred": False, + "alias": "second_model", + "description": "The second test model", + "columns": { + "id": { + "name": "id", + "description": "The user ID number", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "first_name": { + "name": "first_name", + "description": "The user's first name", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "email": { + "name": "email", + "description": "The user's email", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "ip_address": { + "name": "ip_address", + "description": "The user's IP address", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "updated_at": { + "name": "updated_at", + "description": "The last time this user's email was updated", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + }, + "patch_path": "test://" + model_schema_yml_path, + "docs": {"show": False}, + "compiled": True, + "compiled_sql": ANY, + "extra_ctes_injected": True, + "extra_ctes": [], + "checksum": checksum_file(second_model_sql_path), + "unrendered_config": unrendered_second_config, + }, + "seed.test.seed": { + "compiled_path": None, + "build_path": None, + "created_at": ANY, + "compiled": True, + "compiled_sql": "", + "config": seed_config, + "patch_path": "test://" + seed_schema_yml_path, + "path": "seed.csv", + "name": "seed", + "root_path": project.project_root, + "relation_name": relation_name_node_format.format( + project.database, my_schema_name, "seed" + ), + "resource_type": "seed", + "raw_sql": "", + "package_name": "test", + "original_file_path": seed_path, + "refs": [], + "sources": [], + "depends_on": {"nodes": [], "macros": []}, + "unique_id": "seed.test.seed", + "fqn": ["test", "seed"], + "tags": [], + "meta": {}, + "schema": my_schema_name, + "database": project.database, + "alias": "seed", + "deferred": False, + "description": "The test seed", + "columns": { + "id": { + "name": "id", + "description": "The user ID number", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "first_name": { + "name": "first_name", + "description": "The user's first name", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "email": { + "name": "email", + "description": "The user's email", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "ip_address": { + "name": "ip_address", + "description": "The user's IP address", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "updated_at": { + "name": "updated_at", + "description": "The last time this user's email was updated", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + }, + "docs": {"show": True}, + "compiled": True, + "compiled_sql": "", + "extra_ctes_injected": True, + "extra_ctes": [], + "checksum": checksum_file(seed_path), + "unrendered_config": unrendered_seed_config, + }, + "test.test.not_null_model_id.d01cc630e6": { + "alias": "not_null_model_id", + "compiled_path": os.path.join( + compiled_model_path, "schema.yml", "not_null_model_id.sql" + ), + "build_path": None, + "created_at": ANY, + "column_name": "id", + "columns": {}, + "config": test_config, + "sources": [], + "depends_on": { + "macros": ["macro.dbt.test_not_null", "macro.dbt.get_where_subquery"], + "nodes": ["model.test.model"], + }, + "deferred": False, + "description": "", + "file_key_name": "models.model", + "fqn": ["test", "not_null_model_id"], + "name": "not_null_model_id", + "original_file_path": model_schema_yml_path, + "package_name": "test", + "patch_path": None, + "path": "not_null_model_id.sql", + "raw_sql": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [["model"]], + "relation_name": None, + "resource_type": "test", + "root_path": project.project_root, + "schema": test_audit_schema, + "database": project.database, + "tags": [], + "meta": {}, + "unique_id": "test.test.not_null_model_id.d01cc630e6", + "docs": {"show": True}, + "compiled": True, + "compiled_sql": AnyStringWith("where id is null"), + "extra_ctes_injected": True, + "extra_ctes": [], + "test_metadata": { + "namespace": None, + "name": "not_null", + "kwargs": { + "column_name": "id", + "model": "{{ get_where_subquery(ref('model')) }}", + }, + }, + "checksum": {"name": "none", "checksum": ""}, + "unrendered_config": unrendered_test_config, + }, + "snapshot.test.snapshot_seed": { + "alias": "snapshot_seed", + "compiled_path": None, + "build_path": None, + "created_at": ANY, + "checksum": checksum_file(snapshot_path), + "columns": {}, + "compiled": True, + "compiled_sql": ANY, + "config": snapshot_config, + "database": project.database, + "deferred": False, + "depends_on": { + "macros": [], + "nodes": ["seed.test.seed"], + }, + "description": "", + "docs": {"show": True}, + "extra_ctes": [], + "extra_ctes_injected": True, + "fqn": ["test", "snapshot_seed", "snapshot_seed"], + "meta": {}, + "name": "snapshot_seed", + "original_file_path": snapshot_path, + "package_name": "test", + "patch_path": None, + "path": "snapshot_seed.sql", + "raw_sql": LineIndifferent( + read_file_replace_returns(snapshot_path) + .replace("{% snapshot snapshot_seed %}", "") + .replace("{% endsnapshot %}", "") + ), + "refs": [["seed"]], + "relation_name": relation_name_node_format.format( + project.database, alternate_schema, "snapshot_seed" + ), + "resource_type": "snapshot", + "root_path": project.project_root, + "schema": alternate_schema, + "sources": [], + "tags": [], + "unique_id": "snapshot.test.snapshot_seed", + "unrendered_config": unrendered_snapshot_config, + }, + "test.test.test_nothing_model_.5d38568946": { + "alias": "test_nothing_model_", + "compiled_path": os.path.join( + compiled_model_path, "schema.yml", "test_nothing_model_.sql" + ), + "build_path": None, + "created_at": ANY, + "column_name": None, + "columns": {}, + "config": test_config, + "sources": [], + "depends_on": { + "macros": ["macro.test.test_nothing", "macro.dbt.get_where_subquery"], + "nodes": ["model.test.model"], + }, + "deferred": False, + "description": "", + "file_key_name": "models.model", + "fqn": ["test", "test_nothing_model_"], + "name": "test_nothing_model_", + "original_file_path": model_schema_yml_path, + "package_name": "test", + "patch_path": None, + "path": "test_nothing_model_.sql", + "raw_sql": "{{ test.test_nothing(**_dbt_generic_test_kwargs) }}", + "refs": [["model"]], + "relation_name": None, + "resource_type": "test", + "root_path": project.project_root, + "schema": test_audit_schema, + "database": project.database, + "tags": [], + "meta": {}, + "unique_id": "test.test.test_nothing_model_.5d38568946", + "docs": {"show": True}, + "compiled": True, + "compiled_sql": AnyStringWith("select 0"), + "extra_ctes_injected": True, + "extra_ctes": [], + "test_metadata": { + "namespace": "test", + "name": "nothing", + "kwargs": { + "model": "{{ get_where_subquery(ref('model')) }}", + }, + }, + "checksum": {"name": "none", "checksum": ""}, + "unrendered_config": unrendered_test_config, + }, + "test.test.unique_model_id.67b76558ff": { + "alias": "unique_model_id", + "compiled_path": os.path.join( + compiled_model_path, "schema.yml", "unique_model_id.sql" + ), + "build_path": None, + "created_at": ANY, + "column_name": "id", + "columns": {}, + "config": test_config, + "sources": [], + "depends_on": { + "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], + "nodes": ["model.test.model"], + }, + "deferred": False, + "description": "", + "file_key_name": "models.model", + "fqn": ["test", "unique_model_id"], + "name": "unique_model_id", + "original_file_path": model_schema_yml_path, + "package_name": "test", + "patch_path": None, + "path": "unique_model_id.sql", + "raw_sql": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [["model"]], + "relation_name": None, + "resource_type": "test", + "root_path": project.project_root, + "schema": test_audit_schema, + "database": project.database, + "tags": [], + "meta": {}, + "unique_id": "test.test.unique_model_id.67b76558ff", + "docs": {"show": True}, + "compiled": True, + "compiled_sql": AnyStringWith("count(*)"), + "extra_ctes_injected": True, + "extra_ctes": [], + "test_metadata": { + "namespace": None, + "name": "unique", + "kwargs": { + "column_name": "id", + "model": "{{ get_where_subquery(ref('model')) }}", + }, + }, + "checksum": {"name": "none", "checksum": ""}, + "unrendered_config": unrendered_test_config, + }, + }, + "sources": { + "source.test.my_source.my_table": { + "created_at": ANY, + "columns": { + "id": { + "description": "An ID field", + "name": "id", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + } + }, + "config": { + "enabled": True, + }, + "quoting": { + "database": None, + "schema": None, + "identifier": True, + "column": None, + }, + "database": project.database, + "description": "My table", + "external": None, + "freshness": { + "error_after": {"count": None, "period": None}, + "warn_after": {"count": None, "period": None}, + "filter": None, + }, + "identifier": "seed", + "loaded_at_field": None, + "loader": "a_loader", + "meta": {}, + "name": "my_table", + "original_file_path": os.path.join("models", "schema.yml"), + "package_name": "test", + "path": os.path.join("models", "schema.yml"), + "patch_path": None, + "relation_name": relation_name_source_format.format( + project.database, my_schema_name, "seed" + ), + "resource_type": "source", + "root_path": project.project_root, + "schema": my_schema_name, + "source_description": "My source", + "source_name": "my_source", + "source_meta": {}, + "tags": [], + "unique_id": "source.test.my_source.my_table", + "fqn": ["test", "my_source", "my_table"], + "unrendered_config": {}, + }, + }, + "exposures": { + "exposure.test.notebook_exposure": { + "created_at": ANY, + "depends_on": { + "macros": [], + "nodes": ["model.test.model", "model.test.second_model"], + }, + "description": "A description of the complex exposure\n", + "fqn": ["test", "notebook_exposure"], + "maturity": "medium", + "meta": {"tool": "my_tool", "languages": ["python"]}, + "tags": ["my_department"], + "name": "notebook_exposure", + "original_file_path": os.path.join("models", "schema.yml"), + "owner": {"email": "something@example.com", "name": "Some name"}, + "package_name": "test", + "path": "schema.yml", + "refs": [["model"], ["second_model"]], + "resource_type": "exposure", + "root_path": project.project_root, + "sources": [], + "type": "notebook", + "unique_id": "exposure.test.notebook_exposure", + "url": "http://example.com/notebook/1", + }, + "exposure.test.simple_exposure": { + "created_at": ANY, + "depends_on": { + "macros": [], + "nodes": ["source.test.my_source.my_table", "model.test.model"], + }, + "description": "", + "fqn": ["test", "simple_exposure"], + "name": "simple_exposure", + "original_file_path": os.path.join("models", "schema.yml"), + "owner": { + "email": "something@example.com", + "name": None, + }, + "package_name": "test", + "path": "schema.yml", + "refs": [["model"]], + "resource_type": "exposure", + "root_path": project.project_root, + "sources": [["my_source", "my_table"]], + "type": "dashboard", + "unique_id": "exposure.test.simple_exposure", + "url": None, + "maturity": None, + "meta": {}, + "tags": [], + }, + }, + "metrics": {}, + "selectors": {}, + "parent_map": { + "model.test.model": ["seed.test.seed"], + "model.test.second_model": ["seed.test.seed"], + "exposure.test.notebook_exposure": ["model.test.model", "model.test.second_model"], + "exposure.test.simple_exposure": [ + "model.test.model", + "source.test.my_source.my_table", + ], + "seed.test.seed": [], + "snapshot.test.snapshot_seed": ["seed.test.seed"], + "source.test.my_source.my_table": [], + "test.test.not_null_model_id.d01cc630e6": ["model.test.model"], + "test.test.test_nothing_model_.5d38568946": ["model.test.model"], + "test.test.unique_model_id.67b76558ff": ["model.test.model"], + }, + "child_map": { + "model.test.model": [ + "exposure.test.notebook_exposure", + "exposure.test.simple_exposure", + "test.test.not_null_model_id.d01cc630e6", + "test.test.test_nothing_model_.5d38568946", + "test.test.unique_model_id.67b76558ff", + ], + "model.test.second_model": ["exposure.test.notebook_exposure"], + "exposure.test.notebook_exposure": [], + "exposure.test.simple_exposure": [], + "seed.test.seed": [ + "model.test.model", + "model.test.second_model", + "snapshot.test.snapshot_seed", + ], + "snapshot.test.snapshot_seed": [], + "source.test.my_source.my_table": ["exposure.test.simple_exposure"], + "test.test.not_null_model_id.d01cc630e6": [], + "test.test.test_nothing_model_.5d38568946": [], + "test.test.unique_model_id.67b76558ff": [], + }, + "docs": { + "dbt.__overview__": ANY, + "test.macro_info": ANY, + "test.macro_arg_info": ANY, + }, + "disabled": {}, + } + + +def expected_references_manifest(project): + model_database = project.database + my_schema_name = project.test_schema + docs_path = os.path.join("models", "docs.md") + ephemeral_copy_path = os.path.join("models", "ephemeral_copy.sql") + ephemeral_summary_path = os.path.join("models", "ephemeral_summary.sql") + view_summary_path = os.path.join("models", "view_summary.sql") + seed_path = os.path.join("seeds", "seed.csv") + snapshot_path = os.path.join("snapshots", "snapshot_seed.sql") + compiled_model_path = os.path.join("target", "compiled", "test", "models") + schema_yml_path = os.path.join("models", "schema.yml") + + ephemeral_copy_sql = read_file_replace_returns(ephemeral_copy_path).rstrip("\r\n") + ephemeral_summary_sql = read_file_replace_returns(ephemeral_summary_path).rstrip("\r\n") + view_summary_sql = read_file_replace_returns(view_summary_path).rstrip("\r\n") + alternate_schema = project.test_schema + "_test" + + return { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "dbt_version": dbt.version.__version__, + "nodes": { + "model.test.ephemeral_copy": { + "alias": "ephemeral_copy", + "compiled_path": os.path.join(compiled_model_path, "ephemeral_copy.sql"), + "build_path": None, + "created_at": ANY, + "columns": {}, + "config": get_rendered_model_config(materialized="ephemeral"), + "sources": [["my_source", "my_table"]], + "depends_on": {"macros": [], "nodes": ["source.test.my_source.my_table"]}, + "deferred": False, + "description": "", + "docs": {"show": True}, + "fqn": ["test", "ephemeral_copy"], + "name": "ephemeral_copy", + "original_file_path": ephemeral_copy_path, + "package_name": "test", + "patch_path": None, + "path": "ephemeral_copy.sql", + "raw_sql": LineIndifferent(ephemeral_copy_sql), + "refs": [], + "relation_name": None, + "resource_type": "model", + "root_path": project.project_root, + "schema": my_schema_name, + "database": project.database, + "tags": [], + "meta": {}, + "unique_id": "model.test.ephemeral_copy", + "compiled": True, + "compiled_sql": ANY, + "extra_ctes_injected": True, + "extra_ctes": [], + "checksum": checksum_file(ephemeral_copy_path), + "unrendered_config": get_unrendered_model_config(materialized="ephemeral"), + }, + "model.test.ephemeral_summary": { + "alias": "ephemeral_summary", + "compiled_path": os.path.join(compiled_model_path, "ephemeral_summary.sql"), + "build_path": None, + "created_at": ANY, + "columns": { + "first_name": { + "description": "The first name being summarized", + "name": "first_name", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "ct": { + "description": "The number of instances of the first name", + "name": "ct", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + }, + "config": get_rendered_model_config(materialized="table"), + "sources": [], + "depends_on": {"macros": [], "nodes": ["model.test.ephemeral_copy"]}, + "deferred": False, + "description": "A summmary table of the ephemeral copy of the seed data", + "docs": {"show": True}, + "fqn": ["test", "ephemeral_summary"], + "name": "ephemeral_summary", + "original_file_path": ephemeral_summary_path, + "package_name": "test", + "patch_path": "test://" + os.path.join("models", "schema.yml"), + "path": "ephemeral_summary.sql", + "raw_sql": LineIndifferent(ephemeral_summary_sql), + "refs": [["ephemeral_copy"]], + "relation_name": '"{0}"."{1}".ephemeral_summary'.format( + model_database, my_schema_name + ), + "resource_type": "model", + "root_path": project.project_root, + "schema": my_schema_name, + "database": project.database, + "tags": [], + "meta": {}, + "unique_id": "model.test.ephemeral_summary", + "compiled": True, + "compiled_sql": ANY, + "extra_ctes_injected": True, + "extra_ctes": [ANY], + "checksum": checksum_file(ephemeral_summary_path), + "unrendered_config": get_unrendered_model_config(materialized="table"), + }, + "model.test.view_summary": { + "alias": "view_summary", + "compiled_path": os.path.join(compiled_model_path, "view_summary.sql"), + "build_path": None, + "created_at": ANY, + "columns": { + "first_name": { + "description": "The first name being summarized", + "name": "first_name", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "ct": { + "description": "The number of instances of the first name", + "name": "ct", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + }, + "config": get_rendered_model_config(), + "database": project.database, + "depends_on": {"macros": [], "nodes": ["model.test.ephemeral_summary"]}, + "deferred": False, + "description": "A view of the summary of the ephemeral copy of the seed data", + "docs": {"show": True}, + "fqn": ["test", "view_summary"], + "name": "view_summary", + "original_file_path": view_summary_path, + "package_name": "test", + "patch_path": "test://" + schema_yml_path, + "path": "view_summary.sql", + "raw_sql": LineIndifferent(view_summary_sql), + "refs": [["ephemeral_summary"]], + "relation_name": '"{0}"."{1}".view_summary'.format(model_database, my_schema_name), + "resource_type": "model", + "root_path": project.project_root, + "schema": my_schema_name, + "sources": [], + "tags": [], + "meta": {}, + "unique_id": "model.test.view_summary", + "compiled": True, + "compiled_sql": ANY, + "extra_ctes_injected": True, + "extra_ctes": [], + "checksum": checksum_file(view_summary_path), + "unrendered_config": get_unrendered_model_config(materialized="view"), + }, + "seed.test.seed": { + "alias": "seed", + "compiled_path": None, + "build_path": None, + "created_at": ANY, + "columns": { + "id": { + "name": "id", + "description": "The user ID number", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "first_name": { + "name": "first_name", + "description": "The user's first name", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "email": { + "name": "email", + "description": "The user's email", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "ip_address": { + "name": "ip_address", + "description": "The user's IP address", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + "updated_at": { + "name": "updated_at", + "description": "The last time this user's email was updated", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + }, + }, + "config": get_rendered_seed_config(), + "sources": [], + "depends_on": {"macros": [], "nodes": []}, + "deferred": False, + "description": "The test seed", + "docs": {"show": True}, + "fqn": ["test", "seed"], + "name": "seed", + "original_file_path": seed_path, + "package_name": "test", + "patch_path": "test://" + os.path.join("seeds", "schema.yml"), + "path": "seed.csv", + "raw_sql": "", + "refs": [], + "relation_name": '"{0}"."{1}".seed'.format(model_database, my_schema_name), + "resource_type": "seed", + "root_path": project.project_root, + "schema": my_schema_name, + "database": project.database, + "tags": [], + "meta": {}, + "unique_id": "seed.test.seed", + "compiled": True, + "compiled_sql": "", + "extra_ctes_injected": True, + "extra_ctes": [], + "checksum": checksum_file(seed_path), + "unrendered_config": get_unrendered_seed_config(), + }, + "snapshot.test.snapshot_seed": { + "alias": "snapshot_seed", + "compiled_path": None, + "build_path": None, + "created_at": ANY, + "checksum": checksum_file(snapshot_path), + "columns": {}, + "compiled": True, + "compiled_sql": ANY, + "config": get_rendered_snapshot_config(target_schema=alternate_schema), + "database": model_database, + "deferred": False, + "depends_on": {"macros": [], "nodes": ["seed.test.seed"]}, + "description": "", + "docs": {"show": True}, + "extra_ctes": [], + "extra_ctes_injected": True, + "fqn": ["test", "snapshot_seed", "snapshot_seed"], + "meta": {}, + "name": "snapshot_seed", + "original_file_path": snapshot_path, + "package_name": "test", + "patch_path": None, + "path": "snapshot_seed.sql", + "raw_sql": ANY, + "refs": [["seed"]], + "relation_name": '"{0}"."{1}".snapshot_seed'.format( + model_database, alternate_schema + ), + "resource_type": "snapshot", + "root_path": project.project_root, + "schema": alternate_schema, + "sources": [], + "tags": [], + "unique_id": "snapshot.test.snapshot_seed", + "unrendered_config": get_unrendered_snapshot_config( + target_schema=alternate_schema + ), + }, + }, + "sources": { + "source.test.my_source.my_table": { + "columns": { + "id": { + "description": "An ID field", + "name": "id", + "data_type": None, + "meta": {}, + "quote": None, + "tags": [], + } + }, + "config": { + "enabled": True, + }, + "quoting": { + "database": False, + "schema": None, + "identifier": True, + "column": None, + }, + "created_at": ANY, + "database": project.database, + "description": "My table", + "external": None, + "freshness": { + "error_after": {"count": None, "period": None}, + "warn_after": {"count": None, "period": None}, + "filter": None, + }, + "identifier": "seed", + "loaded_at_field": None, + "loader": "a_loader", + "meta": {}, + "name": "my_table", + "original_file_path": os.path.join("models", "schema.yml"), + "package_name": "test", + "path": os.path.join("models", "schema.yml"), + "patch_path": None, + "relation_name": '{0}."{1}"."seed"'.format(project.database, my_schema_name), + "resource_type": "source", + "root_path": project.project_root, + "schema": my_schema_name, + "source_description": "My source", + "source_name": "my_source", + "source_meta": {}, + "tags": [], + "unique_id": "source.test.my_source.my_table", + "fqn": ["test", "my_source", "my_table"], + "unrendered_config": {}, + }, + }, + "exposures": { + "exposure.test.notebook_exposure": { + "created_at": ANY, + "depends_on": {"macros": [], "nodes": ["model.test.view_summary"]}, + "description": "A description of the complex exposure", + "fqn": ["test", "notebook_exposure"], + "maturity": "medium", + "meta": {"tool": "my_tool", "languages": ["python"]}, + "tags": ["my_department"], + "name": "notebook_exposure", + "original_file_path": os.path.join("models", "schema.yml"), + "owner": {"email": "something@example.com", "name": "Some name"}, + "package_name": "test", + "path": "schema.yml", + "refs": [["view_summary"]], + "resource_type": "exposure", + "root_path": project.project_root, + "sources": [], + "type": "notebook", + "unique_id": "exposure.test.notebook_exposure", + "url": "http://example.com/notebook/1", + }, + }, + "metrics": {}, + "selectors": {}, + "docs": { + "dbt.__overview__": ANY, + "test.column_info": { + "block_contents": "An ID field", + "name": "column_info", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.column_info", + }, + "test.ephemeral_summary": { + "block_contents": ("A summmary table of the ephemeral copy of the seed data"), + "name": "ephemeral_summary", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.ephemeral_summary", + }, + "test.source_info": { + "block_contents": "My source", + "name": "source_info", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.source_info", + }, + "test.summary_count": { + "block_contents": "The number of instances of the first name", + "name": "summary_count", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.summary_count", + }, + "test.summary_first_name": { + "block_contents": "The first name being summarized", + "name": "summary_first_name", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.summary_first_name", + }, + "test.table_info": { + "block_contents": "My table", + "name": "table_info", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.table_info", + }, + "test.view_summary": { + "block_contents": ( + "A view of the summary of the ephemeral copy of the " "seed data" + ), + "name": "view_summary", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.view_summary", + }, + "test.macro_info": { + "block_contents": "My custom test that I wrote that does nothing", + "name": "macro_info", + "original_file_path": os.path.join("macros", "macro.md"), + "package_name": "test", + "path": "macro.md", + "root_path": project.project_root, + "unique_id": "test.macro_info", + }, + "test.notebook_info": { + "block_contents": "A description of the complex exposure", + "name": "notebook_info", + "original_file_path": docs_path, + "package_name": "test", + "path": "docs.md", + "root_path": project.project_root, + "unique_id": "test.notebook_info", + }, + "test.macro_arg_info": { + "block_contents": "The model for my custom test", + "name": "macro_arg_info", + "original_file_path": os.path.join("macros", "macro.md"), + "package_name": "test", + "path": "macro.md", + "root_path": project.project_root, + "unique_id": "test.macro_arg_info", + }, + }, + "child_map": { + "model.test.ephemeral_copy": ["model.test.ephemeral_summary"], + "exposure.test.notebook_exposure": [], + "model.test.ephemeral_summary": ["model.test.view_summary"], + "model.test.view_summary": ["exposure.test.notebook_exposure"], + "seed.test.seed": ["snapshot.test.snapshot_seed"], + "snapshot.test.snapshot_seed": [], + "source.test.my_source.my_table": ["model.test.ephemeral_copy"], + }, + "parent_map": { + "model.test.ephemeral_copy": ["source.test.my_source.my_table"], + "model.test.ephemeral_summary": ["model.test.ephemeral_copy"], + "model.test.view_summary": ["model.test.ephemeral_summary"], + "exposure.test.notebook_exposure": ["model.test.view_summary"], + "seed.test.seed": [], + "snapshot.test.snapshot_seed": ["seed.test.seed"], + "source.test.my_source.my_table": [], + }, + "disabled": {}, + "macros": { + "macro.test.test_nothing": { + "name": "test_nothing", + "depends_on": {"macros": []}, + "created_at": ANY, + "description": "My custom test that I wrote that does nothing", + "docs": {"show": True}, + "macro_sql": AnyStringWith("test nothing"), + "original_file_path": os.path.join("macros", "dummy_test.sql"), + "path": os.path.join("macros", "dummy_test.sql"), + "package_name": "test", + "meta": { + "some_key": 100, + }, + "patch_path": "test://" + os.path.join("macros", "schema.yml"), + "resource_type": "macro", + "unique_id": "macro.test.test_nothing", + "tags": [], + "root_path": project.project_root, + "arguments": [ + { + "name": "model", + "type": "Relation", + "description": "The model for my custom test", + }, + ], + } + }, + } diff --git a/tests/functional/docs_generate/expected_run_results.py b/tests/functional/docs_generate/expected_run_results.py new file mode 100644 index 00000000000..83cf6511124 --- /dev/null +++ b/tests/functional/docs_generate/expected_run_results.py @@ -0,0 +1,132 @@ +from unittest.mock import ANY + + +class AnyFloat: + """Any float. Use this in assert calls to assert that it is a float.""" + + def __eq__(self, other): + return isinstance(other, float) + + +def expected_run_results(): + """ + The expected results of this run. + """ + + return [ + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "model.test.model", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "model.test.second_model", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "seed.test.seed", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "snapshot.test.snapshot_seed", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "test.test.not_null_model_id.d01cc630e6", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "test.test.test_nothing_model_.5d38568946", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "test.test.unique_model_id.67b76558ff", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + ] + + +def expected_references_run_results(): + return [ + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "model.test.ephemeral_summary", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "model.test.view_summary", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "seed.test.seed", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + { + "status": "success", + "message": None, + "execution_time": AnyFloat(), + "unique_id": "snapshot.test.snapshot_seed", + "adapter_response": ANY, + "thread_id": ANY, + "timing": [ANY, ANY], + "failures": ANY, + }, + ] diff --git a/tests/functional/docs_generate/test_docs_generate.py b/tests/functional/docs_generate/test_docs_generate.py new file mode 100644 index 00000000000..c85087de5e3 --- /dev/null +++ b/tests/functional/docs_generate/test_docs_generate.py @@ -0,0 +1,545 @@ +import pytest +import os +from datetime import datetime +import dbt + +from dbt.tests.util import run_dbt, rm_file, get_artifact, check_datetime_between +from dbt.tests.fixtures.project import write_project_files +from tests.functional.docs_generate.expected_catalog import ( + base_expected_catalog, + no_stats, + expected_references_catalog, +) +from tests.functional.docs_generate.expected_manifest import ( + expected_seeded_manifest, + expected_references_manifest, +) +from tests.functional.docs_generate.expected_run_results import ( + expected_run_results, + expected_references_run_results, +) + +models__schema_yml = """ +version: 2 + +models: + - name: model + description: "The test model" + docs: + show: false + columns: + - name: id + description: The user ID number + tests: + - unique + - not_null + - name: first_name + description: The user's first name + - name: email + description: The user's email + - name: ip_address + description: The user's IP address + - name: updated_at + description: The last time this user's email was updated + tests: + - test.nothing + + - name: second_model + description: "The second test model" + docs: + show: false + columns: + - name: id + description: The user ID number + - name: first_name + description: The user's first name + - name: email + description: The user's email + - name: ip_address + description: The user's IP address + - name: updated_at + description: The last time this user's email was updated + + +sources: + - name: my_source + description: "My source" + loader: a_loader + schema: "{{ var('test_schema') }}" + tables: + - name: my_table + description: "My table" + identifier: seed + quoting: + identifier: True + columns: + - name: id + description: "An ID field" + + +exposures: + - name: simple_exposure + type: dashboard + depends_on: + - ref('model') + - source('my_source', 'my_table') + owner: + email: something@example.com + - name: notebook_exposure + type: notebook + depends_on: + - ref('model') + - ref('second_model') + owner: + email: something@example.com + name: Some name + description: > + A description of the complex exposure + maturity: medium + meta: + tool: 'my_tool' + languages: + - python + tags: ['my_department'] + url: http://example.com/notebook/1 +""" + +models__second_model_sql = """ +{{ + config( + materialized='view', + schema='test', + ) +}} + +select * from {{ ref('seed') }} +""" + +models__readme_md = """ +This is a readme.md file with {{ invalid-ish jinja }} in it +""" + +models__model_sql = """ +{{ + config( + materialized='view', + ) +}} + +select * from {{ ref('seed') }} +""" + +seed__schema_yml = """ +version: 2 +seeds: + - name: seed + description: "The test seed" + columns: + - name: id + description: The user ID number + - name: first_name + description: The user's first name + - name: email + description: The user's email + - name: ip_address + description: The user's IP address + - name: updated_at + description: The last time this user's email was updated +""" + +seed__seed_csv = """id,first_name,email,ip_address,updated_at +1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 +""" + +macros__schema_yml = """ +version: 2 +macros: + - name: test_nothing + description: "{{ doc('macro_info') }}" + meta: + some_key: 100 + arguments: + - name: model + type: Relation + description: "{{ doc('macro_arg_info') }}" +""" + +macros__macro_md = """ +{% docs macro_info %} +My custom test that I wrote that does nothing +{% enddocs %} + +{% docs macro_arg_info %} +The model for my custom test +{% enddocs %} +""" + +macros__dummy_test_sql = """ +{% test nothing(model) %} + +-- a silly test to make sure that table-level tests show up in the manifest +-- without a column_name field +select 0 + +{% endtest %} +""" + +snapshot__snapshot_seed_sql = """ +{% snapshot snapshot_seed %} +{{ + config( + unique_key='id', + strategy='check', + check_cols='all', + target_schema=var('alternate_schema') + ) +}} +select * from {{ ref('seed') }} +{% endsnapshot %} +""" + +ref_models__schema_yml = """ +version: 2 + +models: + - name: ephemeral_summary + description: "{{ doc('ephemeral_summary') }}" + columns: &summary_columns + - name: first_name + description: "{{ doc('summary_first_name') }}" + - name: ct + description: "{{ doc('summary_count') }}" + - name: view_summary + description: "{{ doc('view_summary') }}" + columns: *summary_columns + +sources: + - name: my_source + description: "{{ doc('source_info') }}" + loader: a_loader + schema: "{{ var('test_schema') }}" + quoting: + database: False + identifier: False + tables: + - name: my_table + description: "{{ doc('table_info') }}" + identifier: seed + quoting: + identifier: True + columns: + - name: id + description: "{{ doc('column_info') }}" + +exposures: + - name: notebook_exposure + type: notebook + depends_on: + - ref('view_summary') + owner: + email: something@example.com + name: Some name + description: "{{ doc('notebook_info') }}" + maturity: medium + url: http://example.com/notebook/1 + meta: + tool: 'my_tool' + languages: + - python + tags: ['my_department'] + +""" + +ref_models__view_summary_sql = """ +{{ + config( + materialized = "view" + ) +}} + +select first_name, ct from {{ref('ephemeral_summary')}} +order by ct asc + +""" + +ref_models__ephemeral_summary_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select first_name, count(*) as ct from {{ref('ephemeral_copy')}} +group by first_name +order by first_name asc + +""" + +ref_models__ephemeral_copy_sql = """ +{{ + config( + materialized = "ephemeral" + ) +}} + +select * from {{ source("my_source", "my_table") }} + +""" + +ref_models__docs_md = """ +{% docs ephemeral_summary %} +A summmary table of the ephemeral copy of the seed data +{% enddocs %} + +{% docs summary_first_name %} +The first name being summarized +{% enddocs %} + +{% docs summary_count %} +The number of instances of the first name +{% enddocs %} + +{% docs view_summary %} +A view of the summary of the ephemeral copy of the seed data +{% enddocs %} + +{% docs source_info %} +My source +{% enddocs %} + +{% docs table_info %} +My table +{% enddocs %} + +{% docs column_info %} +An ID field +{% enddocs %} + +{% docs notebook_info %} +A description of the complex exposure +{% enddocs %} + +""" + + +def verify_catalog(project, expected_catalog, start_time): + # get the catalog.json + catalog_path = os.path.join(project.project_root, "target", "catalog.json") + assert os.path.exists(catalog_path) + catalog = get_artifact(catalog_path) + + # verify the catalog + assert set(catalog) == {"errors", "metadata", "nodes", "sources"} + verify_metadata( + catalog["metadata"], "https://schemas.getdbt.com/dbt/catalog/v1.json", start_time + ) + assert not catalog["errors"] + for key in "nodes", "sources": + assert catalog[key] == expected_catalog[key] + + +def verify_metadata(metadata, dbt_schema_version, start_time): + assert "generated_at" in metadata + check_datetime_between(metadata["generated_at"], start=start_time) + assert "dbt_version" in metadata + assert metadata["dbt_version"] == dbt.version.__version__ + assert "dbt_schema_version" in metadata + assert metadata["dbt_schema_version"] == dbt_schema_version + assert metadata["invocation_id"] == dbt.tracking.active_user.invocation_id + key = "env_key" + if os.name == "nt": + key = key.upper() + assert metadata["env"] == {key: "env_value"} + + +def verify_manifest(project, expected_manifest, start_time): + manifest_path = os.path.join(project.project_root, "target", "manifest.json") + assert os.path.exists(manifest_path) + + manifest = get_artifact(manifest_path) + + manifest_keys = { + "nodes", + "sources", + "macros", + "parent_map", + "child_map", + "metrics", + "docs", + "metadata", + "docs", + "disabled", + "exposures", + "selectors", + } + + assert set(manifest.keys()) == manifest_keys + + for key in manifest_keys: + if key == "macros": + verify_manifest_macros(manifest, expected_manifest.get("macros")) + elif key == "metadata": + metadata = manifest["metadata"] + verify_metadata( + metadata, "https://schemas.getdbt.com/dbt/manifest/v5.json", start_time + ) + assert ( + "project_id" in metadata + and metadata["project_id"] == "098f6bcd4621d373cade4e832627b4f6" + ) + assert ( + "send_anonymous_usage_stats" in metadata + and metadata["send_anonymous_usage_stats"] is False + ) + assert "adapter_type" in metadata and metadata["adapter_type"] == project.adapter_type + elif key in ["nodes", "sources", "exposures", "metrics", "disabled", "docs"]: + for unique_id, node in expected_manifest[key].items(): + assert unique_id in manifest[key] + assert manifest[key][unique_id] == node + else: # ['docs', 'parent_map', 'child_map', 'selectors'] + assert manifest[key] == expected_manifest[key] + + +def verify_manifest_macros(manifest, expected=None): + assert "macros" in manifest + if expected: + for unique_id, expected_macro in expected.items(): + assert unique_id in manifest["macros"] + actual_macro = manifest["macros"][unique_id] + assert expected_macro == actual_macro + + +def verify_run_results(project, expected_run_results, start_time): + run_results_path = os.path.join(project.project_root, "target", "run_results.json") + run_results = get_artifact(run_results_path) + assert "metadata" in run_results + verify_metadata( + run_results["metadata"], "https://schemas.getdbt.com/dbt/run-results/v5.json", start_time + ) + assert "elapsed_time" in run_results + assert run_results["elapsed_time"] > 0 + assert isinstance(run_results["elapsed_time"], float) + assert "args" in run_results + # sort the results so we can make reasonable assertions + run_results["results"].sort(key=lambda r: r["unique_id"]) + assert run_results["results"] == expected_run_results + set(run_results) == {"elapsed_time", "results", "metadata"} + + +def run_and_generate(project, args=None): + results = run_dbt(["run"]) + assert len(results) == 2 + rm_file(project.project_root, "target", "manifest.json") + rm_file(project.project_root, "target", "run_results.json") + + start_time = datetime.utcnow() + run_args = ["docs", "generate"] + if args: + run_args.extend(args) + catalog = run_dbt(run_args) + assert catalog + return start_time + + +class BaseGenerateProject: + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + alternate_schema_name = project.test_schema + "_test" + project.create_test_schema(schema_name=alternate_schema_name) + os.environ["DBT_ENV_CUSTOM_ENV_env_key"] = "env_value" + assets = {"lorem-ipsum.txt": "Lorem ipsum dolor sit amet"} + write_project_files(project.project_root, "assets", assets) + run_dbt(["seed"]) + yield + del os.environ["DBT_ENV_CUSTOM_ENV_env_key"] + + @pytest.fixture(scope="class") + def seeds(self): + return {"schema.yml": seed__schema_yml, "seed.csv": seed__seed_csv} + + @pytest.fixture(scope="class") + def macros(self): + return { + "schema.yml": macros__schema_yml, + "macro.md": macros__macro_md, + "dummy_test.sql": macros__dummy_test_sql, + } + + @pytest.fixture(scope="class") + def snapshots(self): + return {"snapshot_seed.sql": snapshot__snapshot_seed_sql} + + @pytest.fixture(scope="class") + def project_config_update(self, unique_schema): + alternate_schema = unique_schema + "_test" + return { + "asset-paths": ["assets", "invalid-asset-paths"], + "vars": { + "test_schema": unique_schema, + "alternate_schema": alternate_schema, + }, + "seeds": { + "quote_columns": True, + }, + "quoting": {"identifier": False}, + } + + +class TestDocsGenerate(BaseGenerateProject): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + "second_model.sql": models__second_model_sql, + "readme.md": models__readme_md, + "model.sql": models__model_sql, + } + + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + id_type="integer", + text_type="text", + time_type="timestamp without time zone", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + ) + + # Test "--no-compile" flag works and produces no manifest.json + def test_run_and_generate_no_compile(self, project, expected_catalog): + start_time = run_and_generate(project, ["--no-compile"]) + assert not os.path.exists(os.path.join(project.project_root, "target", "manifest.json")) + verify_catalog(project, expected_catalog, start_time) + + # Test generic "docs generate" command + def test_run_and_generate(self, project, expected_catalog): + start_time = run_and_generate(project) + verify_catalog(project, expected_catalog, start_time) + + verify_manifest(project, expected_seeded_manifest(project, quote_model=False), start_time) + verify_run_results(project, expected_run_results(), start_time) + + # Check that assets have been copied to the target directory for use in the docs html page + assert os.path.exists(os.path.join(".", "target", "assets")) + assert os.path.exists(os.path.join(".", "target", "assets", "lorem-ipsum.txt")) + assert not os.path.exists(os.path.join(".", "target", "non-existent-assets")) + + +class TestReferences(BaseGenerateProject): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": ref_models__schema_yml, + "view_summary.sql": ref_models__view_summary_sql, + "ephemeral_summary.sql": ref_models__ephemeral_summary_sql, + "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, + "docs.md": ref_models__docs_md, + } + + def test_references(self, project): + start_time = run_and_generate(project) + verify_catalog(project, expected_references_catalog(project), start_time) + verify_manifest(project, expected_references_manifest(project), start_time) + verify_run_results(project, expected_references_run_results(), start_time) diff --git a/tests/functional/docs_generate/test_escapes.py b/tests/functional/docs_generate/test_escapes.py new file mode 100644 index 00000000000..ff39871497d --- /dev/null +++ b/tests/functional/docs_generate/test_escapes.py @@ -0,0 +1,26 @@ +import pytest +from dbt.tests.util import run_dbt, get_manifest, rm_file + +model_sql = """ +select 1 as id +""" + + +class TestDocsGenerateEscapes: + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": model_sql} + + def test_include_schema(self, project): + results = run_dbt(["run"]) + assert len(results) == 1 + rm_file(project.project_root, "target", "manifest.json") + rm_file(project.project_root, "target", "run_results.json") + + run_dbt(["docs", "generate"]) + manifest = get_manifest(project.project_root) + assert manifest + assert manifest.nodes + assert len(manifest.nodes) == 1 + assert "model.test.model" in manifest.nodes + assert manifest.nodes["model.test.model"].schema == project.test_schema diff --git a/tests/functional/docs_generate/test_override.py b/tests/functional/docs_generate/test_override.py new file mode 100644 index 00000000000..46a037bdcc5 --- /dev/null +++ b/tests/functional/docs_generate/test_override.py @@ -0,0 +1,35 @@ +import pytest +from dbt.tests.util import run_dbt +from dbt.exceptions import CompilationException + +model_sql = """ +select 1 as id +""" + +fail_macros__failure_sql = """ +{% macro get_catalog(information_schema, schemas) %} + {% do exceptions.raise_compiler_error('rejected: no catalogs for you') %} +{% endmacro %} + +""" + + +class TestDocsGenerateOverride: + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": model_sql} + + @pytest.fixture(scope="class") + def macros(self): + return {"failure.sql": fail_macros__failure_sql} + + def test_override_used( + self, + project, + ): + results = run_dbt(["run"]) + assert len(results) == 1 + # this should pick up our failure macro and raise a compilation exception + with pytest.raises(CompilationException) as excinfo: + run_dbt(["--warn-error", "docs", "generate"]) + assert "rejected: no catalogs for you" in str(excinfo.value) From 5321aa1420ea6ba6be561ab677bb74d3438cb170 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Apr 2022 18:30:25 -0400 Subject: [PATCH 2/8] Changie --- .changes/unreleased/Under the Hood-20220413-183014.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20220413-183014.yaml diff --git a/.changes/unreleased/Under the Hood-20220413-183014.yaml b/.changes/unreleased/Under the Hood-20220413-183014.yaml new file mode 100644 index 00000000000..95b0a4e2294 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20220413-183014.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Convert 029_docs_generate tests to new framework +time: 2022-04-13T18:30:14.706391-04:00 +custom: + Author: gshank + Issue: "5035" + PR: "5058" From abbdf37dc38147ddfce3f98e83e2a63c802afbcd Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Apr 2022 18:47:23 -0400 Subject: [PATCH 3/8] remove unnecessary test --- .../functional/docs_generate/test_escapes.py | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 tests/functional/docs_generate/test_escapes.py diff --git a/tests/functional/docs_generate/test_escapes.py b/tests/functional/docs_generate/test_escapes.py deleted file mode 100644 index ff39871497d..00000000000 --- a/tests/functional/docs_generate/test_escapes.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest -from dbt.tests.util import run_dbt, get_manifest, rm_file - -model_sql = """ -select 1 as id -""" - - -class TestDocsGenerateEscapes: - @pytest.fixture(scope="class") - def models(self): - return {"model.sql": model_sql} - - def test_include_schema(self, project): - results = run_dbt(["run"]) - assert len(results) == 1 - rm_file(project.project_root, "target", "manifest.json") - rm_file(project.project_root, "target", "run_results.json") - - run_dbt(["docs", "generate"]) - manifest = get_manifest(project.project_root) - assert manifest - assert manifest.nodes - assert len(manifest.nodes) == 1 - assert "model.test.model" in manifest.nodes - assert manifest.nodes["model.test.model"].schema == project.test_schema From 5a0fdec26a4664f885efd97f185a17f3b6a082df Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 14 Apr 2022 09:40:21 -0400 Subject: [PATCH 4/8] Tweaks --- tests/functional/docs_generate/expected_manifest.py | 6 +++--- tests/functional/docs_generate/test_docs_generate.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/docs_generate/expected_manifest.py b/tests/functional/docs_generate/expected_manifest.py index aeadbec10d1..ef7a9250331 100644 --- a/tests/functional/docs_generate/expected_manifest.py +++ b/tests/functional/docs_generate/expected_manifest.py @@ -6,7 +6,7 @@ # This produces an "expected manifest", with a number of the fields # modified to avoid ephemeral changes. # ANY -# AnyStringWith +# AnyStringWith # LineIndifferent # It also uses some convenience methods to generate the # various config dictionaries. @@ -231,7 +231,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): model_raw_sql = read_file_replace_returns(model_sql_path).rstrip("\r\n") return { - "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v5.json", "dbt_version": dbt.version.__version__, "nodes": { "model.test.model": { @@ -839,7 +839,7 @@ def expected_references_manifest(project): alternate_schema = project.test_schema + "_test" return { - "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v4.json", + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v5.json", "dbt_version": dbt.version.__version__, "nodes": { "model.test.ephemeral_copy": { diff --git a/tests/functional/docs_generate/test_docs_generate.py b/tests/functional/docs_generate/test_docs_generate.py index c85087de5e3..7821c2b6d37 100644 --- a/tests/functional/docs_generate/test_docs_generate.py +++ b/tests/functional/docs_generate/test_docs_generate.py @@ -414,7 +414,7 @@ def verify_run_results(project, expected_run_results, start_time): run_results = get_artifact(run_results_path) assert "metadata" in run_results verify_metadata( - run_results["metadata"], "https://schemas.getdbt.com/dbt/run-results/v5.json", start_time + run_results["metadata"], "https://schemas.getdbt.com/dbt/run-results/v4.json", start_time ) assert "elapsed_time" in run_results assert run_results["elapsed_time"] > 0 From afcfcbb7f6a36a9cb875f8899a803b27e88efb15 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 14 Apr 2022 19:17:58 -0400 Subject: [PATCH 5/8] Split tests into adapter zone and functional. Add schemas. --- schemas/dbt/catalog/v1.json | 249 + schemas/dbt/manifest/v5.json | 5984 +++++++++++++++++ schemas/dbt/run-results/v4.json | 402 ++ schemas/dbt/sources/v3.json | 281 + .../tests/adapter/basic}/expected_catalog.py | 0 .../adapter/basic}/test_docs_generate.py | 101 +- .../expected_manifest.py | 0 .../expected_run_results.py | 0 tests/functional/artifacts/test_artifacts.py | 484 ++ .../test_override.py | 0 10 files changed, 7411 insertions(+), 90 deletions(-) create mode 100644 schemas/dbt/catalog/v1.json create mode 100644 schemas/dbt/manifest/v5.json create mode 100644 schemas/dbt/run-results/v4.json create mode 100644 schemas/dbt/sources/v3.json rename tests/{functional/docs_generate => adapter/dbt/tests/adapter/basic}/expected_catalog.py (100%) rename tests/{functional/docs_generate => adapter/dbt/tests/adapter/basic}/test_docs_generate.py (76%) rename tests/functional/{docs_generate => artifacts}/expected_manifest.py (100%) rename tests/functional/{docs_generate => artifacts}/expected_run_results.py (100%) create mode 100644 tests/functional/artifacts/test_artifacts.py rename tests/functional/{docs_generate => artifacts}/test_override.py (100%) diff --git a/schemas/dbt/catalog/v1.json b/schemas/dbt/catalog/v1.json new file mode 100644 index 00000000000..ab969e349ac --- /dev/null +++ b/schemas/dbt/catalog/v1.json @@ -0,0 +1,249 @@ +{ + "type": "object", + "required": [ + "metadata", + "nodes", + "sources" + ], + "properties": { + "metadata": { + "$ref": "#/definitions/CatalogMetadata" + }, + "nodes": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/CatalogTable" + } + }, + "sources": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/CatalogTable" + } + }, + "errors": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CatalogArtifact(metadata: dbt.contracts.results.CatalogMetadata, nodes: Dict[str, dbt.contracts.results.CatalogTable], sources: Dict[str, dbt.contracts.results.CatalogTable], errors: Union[List[str], NoneType] = None, _compile_results: Union[Any, NoneType] = None)", + "definitions": { + "CatalogMetadata": { + "type": "object", + "required": [], + "properties": { + "dbt_schema_version": { + "type": "string", + "default": "https://schemas.getdbt.com/dbt/catalog/v1.json" + }, + "dbt_version": { + "type": "string", + "default": "1.2.0a1" + }, + "generated_at": { + "type": "string", + "format": "date-time", + "default": "2022-04-14T19:46:57.674922Z" + }, + "invocation_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "16540468-9312-488f-9721-e3fb46751526" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "default": {} + } + }, + "additionalProperties": false, + "description": "CatalogMetadata(dbt_schema_version: str = , dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" + }, + "CatalogTable": { + "type": "object", + "required": [ + "metadata", + "columns", + "stats" + ], + "properties": { + "metadata": { + "$ref": "#/definitions/TableMetadata" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnMetadata" + } + }, + "stats": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/StatsItem" + } + }, + "unique_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CatalogTable(metadata: dbt.contracts.results.TableMetadata, columns: Dict[str, dbt.contracts.results.ColumnMetadata], stats: Dict[str, dbt.contracts.results.StatsItem], unique_id: Union[str, NoneType] = None)" + }, + "TableMetadata": { + "type": "object", + "required": [ + "type", + "schema", + "name" + ], + "properties": { + "type": { + "type": "string" + }, + "schema": { + "type": "string" + }, + "name": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "comment": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "owner": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "TableMetadata(type: str, schema: str, name: str, database: Union[str, NoneType] = None, comment: Union[str, NoneType] = None, owner: Union[str, NoneType] = None)" + }, + "ColumnMetadata": { + "type": "object", + "required": [ + "type", + "index", + "name" + ], + "properties": { + "type": { + "type": "string" + }, + "index": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "comment": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "ColumnMetadata(type: str, index: int, name: str, comment: Union[str, NoneType] = None)" + }, + "StatsItem": { + "type": "object", + "required": [ + "id", + "label", + "include" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "value": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "include": { + "type": "boolean" + }, + "description": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "StatsItem(id: str, label: str, value: Union[bool, str, float, NoneType], include: bool, description: Union[str, NoneType] = None)" + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://schemas.getdbt.com/dbt/catalog/v1.json" +} diff --git a/schemas/dbt/manifest/v5.json b/schemas/dbt/manifest/v5.json new file mode 100644 index 00000000000..addffd2f342 --- /dev/null +++ b/schemas/dbt/manifest/v5.json @@ -0,0 +1,5984 @@ +{ + "type": "object", + "required": [ + "metadata", + "nodes", + "sources", + "macros", + "docs", + "exposures", + "metrics", + "selectors" + ], + "properties": { + "metadata": { + "$ref": "#/definitions/ManifestMetadata", + "description": "Metadata about the manifest" + }, + "nodes": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "$ref": "#/definitions/CompiledAnalysisNode" + }, + { + "$ref": "#/definitions/CompiledSingularTestNode" + }, + { + "$ref": "#/definitions/CompiledModelNode" + }, + { + "$ref": "#/definitions/CompiledHookNode" + }, + { + "$ref": "#/definitions/CompiledRPCNode" + }, + { + "$ref": "#/definitions/CompiledSqlNode" + }, + { + "$ref": "#/definitions/CompiledGenericTestNode" + }, + { + "$ref": "#/definitions/CompiledSeedNode" + }, + { + "$ref": "#/definitions/CompiledSnapshotNode" + }, + { + "$ref": "#/definitions/ParsedAnalysisNode" + }, + { + "$ref": "#/definitions/ParsedSingularTestNode" + }, + { + "$ref": "#/definitions/ParsedHookNode" + }, + { + "$ref": "#/definitions/ParsedModelNode" + }, + { + "$ref": "#/definitions/ParsedRPCNode" + }, + { + "$ref": "#/definitions/ParsedSqlNode" + }, + { + "$ref": "#/definitions/ParsedGenericTestNode" + }, + { + "$ref": "#/definitions/ParsedSeedNode" + }, + { + "$ref": "#/definitions/ParsedSnapshotNode" + } + ] + }, + "description": "The nodes defined in the dbt project and its dependencies" + }, + "sources": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ParsedSourceDefinition" + }, + "description": "The sources defined in the dbt project and its dependencies" + }, + "macros": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ParsedMacro" + }, + "description": "The macros defined in the dbt project and its dependencies" + }, + "docs": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ParsedDocumentation" + }, + "description": "The docs defined in the dbt project and its dependencies" + }, + "exposures": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ParsedExposure" + }, + "description": "The exposures defined in the dbt project and its dependencies" + }, + "metrics": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ParsedMetric" + }, + "description": "The metrics defined in the dbt project and its dependencies" + }, + "selectors": { + "type": "object", + "description": "The selectors defined in selectors.yml" + }, + "disabled": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/CompiledAnalysisNode" + }, + { + "$ref": "#/definitions/CompiledSingularTestNode" + }, + { + "$ref": "#/definitions/CompiledModelNode" + }, + { + "$ref": "#/definitions/CompiledHookNode" + }, + { + "$ref": "#/definitions/CompiledRPCNode" + }, + { + "$ref": "#/definitions/CompiledSqlNode" + }, + { + "$ref": "#/definitions/CompiledGenericTestNode" + }, + { + "$ref": "#/definitions/CompiledSeedNode" + }, + { + "$ref": "#/definitions/CompiledSnapshotNode" + }, + { + "$ref": "#/definitions/ParsedAnalysisNode" + }, + { + "$ref": "#/definitions/ParsedSingularTestNode" + }, + { + "$ref": "#/definitions/ParsedHookNode" + }, + { + "$ref": "#/definitions/ParsedModelNode" + }, + { + "$ref": "#/definitions/ParsedRPCNode" + }, + { + "$ref": "#/definitions/ParsedSqlNode" + }, + { + "$ref": "#/definitions/ParsedGenericTestNode" + }, + { + "$ref": "#/definitions/ParsedSeedNode" + }, + { + "$ref": "#/definitions/ParsedSnapshotNode" + }, + { + "$ref": "#/definitions/ParsedSourceDefinition" + } + ] + } + } + }, + { + "type": "null" + } + ], + "description": "A mapping of the disabled nodes in the target" + }, + "parent_map": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "type": "null" + } + ], + "description": "A mapping from\u00a0child nodes to their dependencies" + }, + "child_map": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "type": "null" + } + ], + "description": "A mapping from parent nodes to their dependents" + } + }, + "additionalProperties": false, + "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode]], sources: Mapping[str, dbt.contracts.graph.parsed.ParsedSourceDefinition], macros: Mapping[str, dbt.contracts.graph.parsed.ParsedMacro], docs: Mapping[str, dbt.contracts.graph.parsed.ParsedDocumentation], exposures: Mapping[str, dbt.contracts.graph.parsed.ParsedExposure], metrics: Mapping[str, dbt.contracts.graph.parsed.ParsedMetric], selectors: Mapping[str, Any], disabled: Union[Mapping[str, List[Union[dbt.contracts.graph.compiled.CompiledAnalysisNode, dbt.contracts.graph.compiled.CompiledSingularTestNode, dbt.contracts.graph.compiled.CompiledModelNode, dbt.contracts.graph.compiled.CompiledHookNode, dbt.contracts.graph.compiled.CompiledRPCNode, dbt.contracts.graph.compiled.CompiledSqlNode, dbt.contracts.graph.compiled.CompiledGenericTestNode, dbt.contracts.graph.compiled.CompiledSeedNode, dbt.contracts.graph.compiled.CompiledSnapshotNode, dbt.contracts.graph.parsed.ParsedAnalysisNode, dbt.contracts.graph.parsed.ParsedSingularTestNode, dbt.contracts.graph.parsed.ParsedHookNode, dbt.contracts.graph.parsed.ParsedModelNode, dbt.contracts.graph.parsed.ParsedRPCNode, dbt.contracts.graph.parsed.ParsedSqlNode, dbt.contracts.graph.parsed.ParsedGenericTestNode, dbt.contracts.graph.parsed.ParsedSeedNode, dbt.contracts.graph.parsed.ParsedSnapshotNode, dbt.contracts.graph.parsed.ParsedSourceDefinition]]], NoneType], parent_map: Union[Dict[str, List[str]], NoneType], child_map: Union[Dict[str, List[str]], NoneType])", + "definitions": { + "ManifestMetadata": { + "type": "object", + "required": [], + "properties": { + "dbt_schema_version": { + "type": "string", + "default": "https://schemas.getdbt.com/dbt/manifest/v5.json" + }, + "dbt_version": { + "type": "string", + "default": "1.2.0a1" + }, + "generated_at": { + "type": "string", + "format": "date-time", + "default": "2022-04-14T19:46:57.676592Z" + }, + "invocation_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "16540468-9312-488f-9721-e3fb46751526" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "default": {} + }, + "project_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A unique identifier for the project" + }, + "user_id": { + "oneOf": [ + { + "type": "string", + "pattern": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + }, + { + "type": "null" + } + ], + "description": "A unique identifier for the user" + }, + "send_anonymous_usage_stats": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Whether dbt is configured to send anonymous usage statistics" + }, + "adapter_type": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The type name of the adapter" + } + }, + "additionalProperties": false, + "description": "Metadata for the manifest." + }, + "CompiledAnalysisNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "analysis" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.680463 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledAnalysisNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" + }, + "FileHash": { + "type": "object", + "required": [ + "name", + "checksum" + ], + "properties": { + "name": { + "type": "string" + }, + "checksum": { + "type": "string" + } + }, + "additionalProperties": false, + "description": "FileHash(name: str, checksum: str)" + }, + "NodeConfig": { + "type": "object", + "required": [], + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "alias": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "tags": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ], + "default": [] + }, + "meta": { + "type": "object", + "default": {} + }, + "materialized": { + "type": "string", + "default": "view" + }, + "persist_docs": { + "type": "object", + "default": {} + }, + "post-hook": { + "type": "array", + "items": { + "$ref": "#/definitions/Hook" + }, + "default": [] + }, + "pre-hook": { + "type": "array", + "items": { + "$ref": "#/definitions/Hook" + }, + "default": [] + }, + "quoting": { + "type": "object", + "default": {} + }, + "column_types": { + "type": "object", + "default": {} + }, + "full_refresh": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "unique_key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "on_schema_change": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "ignore" + } + }, + "additionalProperties": true, + "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'view', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore')" + }, + "Hook": { + "type": "object", + "required": [ + "sql" + ], + "properties": { + "sql": { + "type": "string" + }, + "transaction": { + "type": "boolean", + "default": true + }, + "index": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "Hook(sql: str, transaction: bool = True, index: Union[int, NoneType] = None)" + }, + "DependsOn": { + "type": "object", + "required": [], + "properties": { + "macros": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + } + }, + "additionalProperties": false, + "description": "DependsOn(macros: List[str] = , nodes: List[str] = )" + }, + "ColumnInfo": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string", + "default": "" + }, + "meta": { + "type": "object", + "default": {} + }, + "data_type": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "quote": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + } + }, + "additionalProperties": true, + "description": "ColumnInfo(name: str, description: str = '', meta: Dict[str, Any] = , data_type: Union[str, NoneType] = None, quote: Union[bool, NoneType] = None, tags: List[str] = , _extra: Dict[str, Any] = )" + }, + "Docs": { + "type": "object", + "required": [], + "properties": { + "show": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false, + "description": "Docs(show: bool = True)" + }, + "InjectedCTE": { + "type": "object", + "required": [ + "id", + "sql" + ], + "properties": { + "id": { + "type": "string" + }, + "sql": { + "type": "string" + } + }, + "additionalProperties": false, + "description": "InjectedCTE(id: str, sql: str)" + }, + "CompiledSingularTestNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "test" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/TestConfig", + "default": { + "enabled": true, + "alias": null, + "schema": "dbt_test__audit", + "database": null, + "tags": [], + "meta": {}, + "materialized": "test", + "severity": "ERROR", + "store_failures": null, + "where": null, + "limit": null, + "fail_calc": "count(*)", + "warn_if": "!= 0", + "error_if": "!= 0" + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.682705 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledSingularTestNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" + }, + "TestConfig": { + "type": "object", + "required": [], + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "alias": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "dbt_test__audit" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "tags": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ], + "default": [] + }, + "meta": { + "type": "object", + "default": {} + }, + "materialized": { + "type": "string", + "default": "test" + }, + "severity": { + "type": "string", + "pattern": "^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$", + "default": "ERROR" + }, + "store_failures": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "where": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "limit": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "fail_calc": { + "type": "string", + "default": "count(*)" + }, + "warn_if": { + "type": "string", + "default": "!= 0" + }, + "error_if": { + "type": "string", + "default": "!= 0" + } + }, + "additionalProperties": true, + "description": "TestConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = 'dbt_test__audit', database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'test', severity: dbt.contracts.graph.model_config.Severity = 'ERROR', store_failures: Union[bool, NoneType] = None, where: Union[str, NoneType] = None, limit: Union[int, NoneType] = None, fail_calc: str = 'count(*)', warn_if: str = '!= 0', error_if: str = '!= 0')" + }, + "CompiledModelNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "model" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.684142 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledModelNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" + }, + "CompiledHookNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "operation" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.685675 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "index": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledHookNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None, index: Union[int, NoneType] = None)" + }, + "CompiledRPCNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "rpc" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.687135 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledRPCNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" + }, + "CompiledSqlNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "sql" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.688512 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledSqlNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" + }, + "CompiledGenericTestNode": { + "type": "object", + "required": [ + "raw_sql", + "test_metadata", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "test_metadata": { + "$ref": "#/definitions/TestMetadata" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "test" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/TestConfig", + "default": { + "enabled": true, + "alias": null, + "schema": "dbt_test__audit", + "database": null, + "tags": [], + "meta": {}, + "materialized": "test", + "severity": "ERROR", + "store_failures": null, + "where": null, + "limit": null, + "fail_calc": "count(*)", + "warn_if": "!= 0", + "error_if": "!= 0" + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.690442 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "column_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "file_key_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledGenericTestNode(raw_sql: str, test_metadata: dbt.contracts.graph.parsed.TestMetadata, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None, column_name: Union[str, NoneType] = None, file_key_name: Union[str, NoneType] = None)" + }, + "TestMetadata": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "kwargs": { + "type": "object", + "default": {} + }, + "namespace": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "TestMetadata(name: str, kwargs: Dict[str, Any] = , namespace: Union[str, NoneType] = None)" + }, + "CompiledSeedNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "seed" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/SeedConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "seed", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "quote_columns": null, + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.693683 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledSeedNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" + }, + "SeedConfig": { + "type": "object", + "required": [], + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "alias": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "tags": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ], + "default": [] + }, + "meta": { + "type": "object", + "default": {} + }, + "materialized": { + "type": "string", + "default": "seed" + }, + "persist_docs": { + "type": "object", + "default": {} + }, + "post-hook": { + "type": "array", + "items": { + "$ref": "#/definitions/Hook" + }, + "default": [] + }, + "pre-hook": { + "type": "array", + "items": { + "$ref": "#/definitions/Hook" + }, + "default": [] + }, + "quoting": { + "type": "object", + "default": {} + }, + "column_types": { + "type": "object", + "default": {} + }, + "full_refresh": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "unique_key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "on_schema_change": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "ignore" + }, + "quote_columns": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'seed', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', quote_columns: Union[bool, NoneType] = None)" + }, + "CompiledSnapshotNode": { + "type": "object", + "required": [ + "raw_sql", + "compiled", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "compiled": { + "type": "boolean" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "snapshot" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.695231 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "compiled_sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "extra_ctes_injected": { + "type": "boolean", + "default": false + }, + "extra_ctes": { + "type": "array", + "items": { + "$ref": "#/definitions/InjectedCTE" + }, + "default": [] + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "CompiledSnapshotNode(raw_sql: str, compiled: bool, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , compiled_sql: Union[str, NoneType] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.compiled.InjectedCTE] = , relation_name: Union[str, NoneType] = None, _pre_injected_sql: Union[str, NoneType] = None)" + }, + "ParsedAnalysisNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "analysis" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.6965492 + }, + "config_call_dict": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "ParsedAnalysisNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" + }, + "ParsedSingularTestNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "test" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/TestConfig", + "default": { + "enabled": true, + "alias": null, + "schema": "dbt_test__audit", + "database": null, + "tags": [], + "meta": {}, + "materialized": "test", + "severity": "ERROR", + "store_failures": null, + "where": null, + "limit": null, + "fail_calc": "count(*)", + "warn_if": "!= 0", + "error_if": "!= 0" + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.697702 + }, + "config_call_dict": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "ParsedSingularTestNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" + }, + "ParsedHookNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "operation" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.6988769 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "index": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "ParsedHookNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , index: Union[int, NoneType] = None)" + }, + "ParsedModelNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "model" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.700115 + }, + "config_call_dict": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "ParsedModelNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" + }, + "ParsedRPCNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "rpc" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.701325 + }, + "config_call_dict": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "ParsedRPCNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" + }, + "ParsedSqlNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "sql" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/NodeConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "view", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.7026129 + }, + "config_call_dict": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "ParsedSqlNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" + }, + "ParsedGenericTestNode": { + "type": "object", + "required": [ + "raw_sql", + "test_metadata", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "test_metadata": { + "$ref": "#/definitions/TestMetadata" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "test" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/TestConfig", + "default": { + "enabled": true, + "alias": null, + "schema": "dbt_test__audit", + "database": null, + "tags": [], + "meta": {}, + "materialized": "test", + "severity": "ERROR", + "store_failures": null, + "where": null, + "limit": null, + "fail_calc": "count(*)", + "warn_if": "!= 0", + "error_if": "!= 0" + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.7037811 + }, + "config_call_dict": { + "type": "object", + "default": {} + }, + "column_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "file_key_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "ParsedGenericTestNode(raw_sql: str, test_metadata: dbt.contracts.graph.parsed.TestMetadata, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , column_name: Union[str, NoneType] = None, file_key_name: Union[str, NoneType] = None)" + }, + "ParsedSeedNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "seed" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/SeedConfig", + "default": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "materialized": "seed", + "persist_docs": {}, + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "quote_columns": null, + "post-hook": [], + "pre-hook": [] + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.705039 + }, + "config_call_dict": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "ParsedSeedNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SeedConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" + }, + "ParsedSnapshotNode": { + "type": "object", + "required": [ + "raw_sql", + "schema", + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "resource_type", + "alias", + "checksum", + "config" + ], + "properties": { + "raw_sql": { + "type": "string" + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "snapshot" + ] + }, + "alias": { + "type": "string" + }, + "checksum": { + "$ref": "#/definitions/FileHash" + }, + "config": { + "$ref": "#/definitions/SnapshotConfig" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "compiled_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "build_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "deferred": { + "type": "boolean", + "default": false + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "created_at": { + "type": "number", + "default": 1649965617.707365 + }, + "config_call_dict": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "ParsedSnapshotNode(raw_sql: str, database: Union[str, NoneType], schema: str, fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, resource_type: dbt.node_types.NodeType, alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , refs: List[List[str]] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, compiled_path: Union[str, NoneType] = None, build_path: Union[str, NoneType] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = )" + }, + "SnapshotConfig": { + "type": "object", + "required": [], + "properties": { + "enabled": { + "type": "boolean", + "default": true + }, + "alias": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "tags": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ], + "default": [] + }, + "meta": { + "type": "object", + "default": {} + }, + "materialized": { + "type": "string", + "default": "snapshot" + }, + "persist_docs": { + "type": "object", + "default": {} + }, + "post-hook": { + "type": "array", + "items": { + "$ref": "#/definitions/Hook" + }, + "default": [] + }, + "pre-hook": { + "type": "array", + "items": { + "$ref": "#/definitions/Hook" + }, + "default": [] + }, + "quoting": { + "type": "object", + "default": {} + }, + "column_types": { + "type": "object", + "default": {} + }, + "full_refresh": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "unique_key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "on_schema_change": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "ignore" + }, + "strategy": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "target_schema": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "target_database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "updated_at": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "check_cols": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Union[str, NoneType] = None, schema: Union[str, NoneType] = None, database: Union[str, NoneType] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'snapshot', persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Union[bool, NoneType] = None, unique_key: Union[str, NoneType] = None, on_schema_change: Union[str, NoneType] = 'ignore', strategy: Union[str, NoneType] = None, target_schema: Union[str, NoneType] = None, target_database: Union[str, NoneType] = None, updated_at: Union[str, NoneType] = None, check_cols: Union[str, List[str], NoneType] = None)" + }, + "ParsedSourceDefinition": { + "type": "object", + "required": [ + "fqn", + "schema", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "source_name", + "source_description", + "loader", + "identifier", + "resource_type" + ], + "properties": { + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "database": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "schema": { + "type": "string" + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "source_name": { + "type": "string" + }, + "source_description": { + "type": "string" + }, + "loader": { + "type": "string" + }, + "identifier": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "source" + ] + }, + "quoting": { + "$ref": "#/definitions/Quoting", + "default": { + "database": null, + "schema": null, + "identifier": null, + "column": null + } + }, + "loaded_at_field": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "freshness": { + "oneOf": [ + { + "$ref": "#/definitions/FreshnessThreshold" + }, + { + "type": "null" + } + ] + }, + "external": { + "oneOf": [ + { + "$ref": "#/definitions/ExternalTable" + }, + { + "type": "null" + } + ] + }, + "description": { + "type": "string", + "default": "" + }, + "columns": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ColumnInfo" + }, + "default": {} + }, + "meta": { + "type": "object", + "default": {} + }, + "source_meta": { + "type": "object", + "default": {} + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "config": { + "$ref": "#/definitions/SourceConfig", + "default": { + "enabled": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "unrendered_config": { + "type": "object", + "default": {} + }, + "relation_name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": "number", + "default": 1649965617.7098281 + } + }, + "additionalProperties": false, + "description": "ParsedSourceDefinition(fqn: List[str], database: Union[str, NoneType], schema: str, unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, source_name: str, source_description: str, loader: str, identifier: str, resource_type: dbt.node_types.NodeType, _event_status: Dict[str, Any] = , quoting: dbt.contracts.graph.unparsed.Quoting = , loaded_at_field: Union[str, NoneType] = None, freshness: Union[dbt.contracts.graph.unparsed.FreshnessThreshold, NoneType] = None, external: Union[dbt.contracts.graph.unparsed.ExternalTable, NoneType] = None, description: str = '', columns: Dict[str, dbt.contracts.graph.parsed.ColumnInfo] = , meta: Dict[str, Any] = , source_meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.SourceConfig = , patch_path: Union[pathlib.Path, NoneType] = None, unrendered_config: Dict[str, Any] = , relation_name: Union[str, NoneType] = None, created_at: float = )" + }, + "Quoting": { + "type": "object", + "required": [], + "properties": { + "database": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "schema": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "identifier": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "column": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "Quoting(database: Union[bool, NoneType] = None, schema: Union[bool, NoneType] = None, identifier: Union[bool, NoneType] = None, column: Union[bool, NoneType] = None)" + }, + "FreshnessThreshold": { + "type": "object", + "required": [], + "properties": { + "warn_after": { + "oneOf": [ + { + "$ref": "#/definitions/Time" + }, + { + "type": "null" + } + ], + "default": { + "count": null, + "period": null + } + }, + "error_after": { + "oneOf": [ + { + "$ref": "#/definitions/Time" + }, + { + "type": "null" + } + ], + "default": { + "count": null, + "period": null + } + }, + "filter": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "FreshnessThreshold(warn_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , error_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , filter: Union[str, NoneType] = None)" + }, + "FreshnessMetadata": { + "type": "object", + "required": [], + "properties": { + "dbt_schema_version": { + "type": "string", + "default": "https://schemas.getdbt.com/dbt/sources/v3.json" + }, + "dbt_version": { + "type": "string", + "default": "1.2.0a1" + }, + "generated_at": { + "type": "string", + "format": "date-time", + "default": "2022-04-14T19:46:57.671562Z" + }, + "invocation_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "16540468-9312-488f-9721-e3fb46751526" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "default": {} + } + }, + "additionalProperties": false, + "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" + }, + "SourceFreshnessRuntimeError": { + "type": "object", + "required": [ + "unique_id", + "status" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "error": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "status": { + "type": "string", + "enum": [ + "runtime error" + ] + } + }, + "additionalProperties": false, + "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" + }, + "SourceFreshnessOutput": { + "type": "object", + "required": [ + "unique_id", + "max_loaded_at", + "snapshotted_at", + "max_loaded_at_time_ago_in_s", + "status", + "criteria", + "adapter_response", + "timing", + "thread_id", + "execution_time" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "max_loaded_at": { + "type": "string", + "format": "date-time" + }, + "snapshotted_at": { + "type": "string", + "format": "date-time" + }, + "max_loaded_at_time_ago_in_s": { + "type": "number" + }, + "status": { + "type": "string", + "enum": [ + "pass", + "warn", + "error", + "runtime error" + ] + }, + "criteria": { + "$ref": "#/definitions/FreshnessThreshold" + }, + "adapter_response": { + "type": "object" + }, + "timing": { + "type": "array", + "items": { + "$ref": "#/definitions/TimingInfo" + } + }, + "thread_id": { + "type": "string" + }, + "execution_time": { + "type": "number" + } + }, + "additionalProperties": false, + "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" + }, + "Time": { + "type": "object", + "required": [], + "properties": { + "count": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "period": { + "oneOf": [ + { + "type": "string", + "enum": [ + "minute", + "hour", + "day" + ] + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "Time(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.TimePeriod, NoneType] = None)" + }, + "TimingInfo": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "started_at": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "completed_at": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "TimingInfo(name: str, started_at: Union[datetime.datetime, NoneType] = None, completed_at: Union[datetime.datetime, NoneType] = None)" + }, + "ExternalTable": { + "type": "object", + "required": [], + "properties": { + "location": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "file_format": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "row_format": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "tbl_properties": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "partitions": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/ExternalPartition" + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": true, + "description": "ExternalTable(_extra: Dict[str, Any] = , location: Union[str, NoneType] = None, file_format: Union[str, NoneType] = None, row_format: Union[str, NoneType] = None, tbl_properties: Union[str, NoneType] = None, partitions: Union[List[dbt.contracts.graph.unparsed.ExternalPartition], NoneType] = None)" + }, + "ExternalPartition": { + "type": "object", + "required": [], + "properties": { + "name": { + "type": "string", + "default": "" + }, + "description": { + "type": "string", + "default": "" + }, + "data_type": { + "type": "string", + "default": "" + }, + "meta": { + "type": "object", + "default": {} + } + }, + "additionalProperties": true, + "description": "ExternalPartition(_extra: Dict[str, Any] = , name: str = '', description: str = '', data_type: str = '', meta: Dict[str, Any] = )" + }, + "SourceConfig": { + "type": "object", + "required": [], + "properties": { + "enabled": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": true, + "description": "SourceConfig(_extra: Dict[str, Any] = , enabled: bool = True)" + }, + "ParsedMacro": { + "type": "object", + "required": [ + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "macro_sql", + "resource_type" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "macro_sql": { + "type": "string" + }, + "resource_type": { + "type": "string", + "enum": [ + "macro" + ] + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/MacroDependsOn", + "default": { + "macros": [] + } + }, + "description": { + "type": "string", + "default": "" + }, + "meta": { + "type": "object", + "default": {} + }, + "docs": { + "$ref": "#/definitions/Docs", + "default": { + "show": true + } + }, + "patch_path": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "arguments": { + "type": "array", + "items": { + "$ref": "#/definitions/MacroArgument" + }, + "default": [] + }, + "created_at": { + "type": "number", + "default": 1649965617.710664 + } + }, + "additionalProperties": false, + "description": "ParsedMacro(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, macro_sql: str, resource_type: dbt.node_types.NodeType, tags: List[str] = , depends_on: dbt.contracts.graph.parsed.MacroDependsOn = , description: str = '', meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Union[str, NoneType] = None, arguments: List[dbt.contracts.graph.unparsed.MacroArgument] = , created_at: float = )" + }, + "MacroDependsOn": { + "type": "object", + "required": [], + "properties": { + "macros": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + } + }, + "additionalProperties": false, + "description": "MacroDependsOn(macros: List[str] = )" + }, + "MacroArgument": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "type": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "description": { + "type": "string", + "default": "" + } + }, + "additionalProperties": false, + "description": "MacroArgument(name: str, type: Union[str, NoneType] = None, description: str = '')" + }, + "ParsedDocumentation": { + "type": "object", + "required": [ + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "block_contents" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "block_contents": { + "type": "string" + } + }, + "additionalProperties": false, + "description": "ParsedDocumentation(unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, block_contents: str)" + }, + "ParsedExposure": { + "type": "object", + "required": [ + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "name", + "type", + "owner" + ], + "properties": { + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "dashboard", + "notebook", + "analysis", + "ml", + "application" + ] + }, + "owner": { + "$ref": "#/definitions/ExposureOwner" + }, + "resource_type": { + "type": "string", + "enum": [ + "model", + "analysis", + "test", + "snapshot", + "operation", + "seed", + "rpc", + "sql", + "docs", + "source", + "macro", + "exposure", + "metric" + ], + "default": "exposure" + }, + "description": { + "type": "string", + "default": "" + }, + "maturity": { + "oneOf": [ + { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + }, + { + "type": "null" + } + ] + }, + "meta": { + "type": "object", + "default": {} + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "url": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "created_at": { + "type": "number", + "default": 1649965617.711809 + } + }, + "additionalProperties": false, + "description": "ParsedExposure(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, name: str, type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.ExposureOwner, resource_type: dbt.node_types.NodeType = , description: str = '', maturity: Union[dbt.contracts.graph.unparsed.MaturityType, NoneType] = None, meta: Dict[str, Any] = , tags: List[str] = , url: Union[str, NoneType] = None, depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , sources: List[List[str]] = , created_at: float = )" + }, + "ExposureOwner": { + "type": "object", + "required": [ + "email" + ], + "properties": { + "email": { + "type": "string" + }, + "name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "ExposureOwner(email: str, name: Union[str, NoneType] = None)" + }, + "ParsedMetric": { + "type": "object", + "required": [ + "fqn", + "unique_id", + "package_name", + "root_path", + "path", + "original_file_path", + "model", + "name", + "description", + "label", + "type", + "filters", + "time_grains", + "dimensions" + ], + "properties": { + "fqn": { + "type": "array", + "items": { + "type": "string" + } + }, + "unique_id": { + "type": "string" + }, + "package_name": { + "type": "string" + }, + "root_path": { + "type": "string" + }, + "path": { + "type": "string" + }, + "original_file_path": { + "type": "string" + }, + "model": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "label": { + "type": "string" + }, + "type": { + "type": "string" + }, + "sql": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "timestamp": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "filters": { + "type": "array", + "items": { + "$ref": "#/definitions/MetricFilter" + } + }, + "time_grains": { + "type": "array", + "items": { + "type": "string" + } + }, + "dimensions": { + "type": "array", + "items": { + "type": "string" + } + }, + "resource_type": { + "type": "string", + "enum": [ + "model", + "analysis", + "test", + "snapshot", + "operation", + "seed", + "rpc", + "sql", + "docs", + "source", + "macro", + "exposure", + "metric" + ], + "default": "metric" + }, + "meta": { + "type": "object", + "default": {} + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "sources": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "depends_on": { + "$ref": "#/definitions/DependsOn", + "default": { + "macros": [], + "nodes": [] + } + }, + "refs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": [] + }, + "created_at": { + "type": "number", + "default": 1649965617.712818 + } + }, + "additionalProperties": false, + "description": "ParsedMetric(fqn: List[str], unique_id: str, package_name: str, root_path: str, path: str, original_file_path: str, model: str, name: str, description: str, label: str, type: str, sql: Union[str, NoneType], timestamp: Union[str, NoneType], filters: List[dbt.contracts.graph.unparsed.MetricFilter], time_grains: List[str], dimensions: List[str], resource_type: dbt.node_types.NodeType = , meta: Dict[str, Any] = , tags: List[str] = , sources: List[List[str]] = , depends_on: dbt.contracts.graph.parsed.DependsOn = , refs: List[List[str]] = , created_at: float = )" + }, + "MetricFilter": { + "type": "object", + "required": [ + "field", + "operator", + "value" + ], + "properties": { + "field": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "additionalProperties": false, + "description": "MetricFilter(field: str, operator: str, value: str)" + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://schemas.getdbt.com/dbt/manifest/v5.json" +} diff --git a/schemas/dbt/run-results/v4.json b/schemas/dbt/run-results/v4.json new file mode 100644 index 00000000000..654a35c4713 --- /dev/null +++ b/schemas/dbt/run-results/v4.json @@ -0,0 +1,402 @@ +{ + "type": "object", + "required": [ + "metadata", + "results", + "elapsed_time" + ], + "properties": { + "metadata": { + "$ref": "#/definitions/BaseArtifactMetadata" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/RunResultOutput" + } + }, + "elapsed_time": { + "type": "number" + }, + "args": { + "type": "object", + "default": {} + } + }, + "additionalProperties": false, + "description": "RunResultsArtifact(metadata: dbt.contracts.util.BaseArtifactMetadata, results: Sequence[dbt.contracts.results.RunResultOutput], elapsed_time: float, args: Dict[str, Any] = )", + "definitions": { + "BaseArtifactMetadata": { + "type": "object", + "required": [ + "dbt_schema_version" + ], + "properties": { + "dbt_schema_version": { + "type": "string" + }, + "dbt_version": { + "type": "string", + "default": "1.2.0a1" + }, + "generated_at": { + "type": "string", + "format": "date-time", + "default": "2022-04-14T19:46:57.673939Z" + }, + "invocation_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "16540468-9312-488f-9721-e3fb46751526" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "default": {} + } + }, + "additionalProperties": false, + "description": "BaseArtifactMetadata(dbt_schema_version: str, dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" + }, + "RunResultOutput": { + "type": "object", + "required": [ + "status", + "timing", + "thread_id", + "execution_time", + "adapter_response", + "unique_id" + ], + "properties": { + "status": { + "oneOf": [ + { + "type": "string", + "enum": [ + "success", + "error", + "skipped" + ] + }, + { + "type": "string", + "enum": [ + "pass", + "error", + "fail", + "warn", + "skipped" + ] + }, + { + "type": "string", + "enum": [ + "pass", + "warn", + "error", + "runtime error" + ] + } + ] + }, + "timing": { + "type": "array", + "items": { + "$ref": "#/definitions/TimingInfo" + } + }, + "thread_id": { + "type": "string" + }, + "execution_time": { + "type": "number" + }, + "adapter_response": { + "type": "object" + }, + "message": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "failures": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "unique_id": { + "type": "string" + } + }, + "additionalProperties": false, + "description": "RunResultOutput(status: Union[dbt.contracts.results.RunStatus, dbt.contracts.results.TestStatus, dbt.contracts.results.FreshnessStatus], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float, adapter_response: Dict[str, Any], message: Union[str, NoneType], failures: Union[int, NoneType], unique_id: str)" + }, + "TimingInfo": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "started_at": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "completed_at": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "TimingInfo(name: str, started_at: Union[datetime.datetime, NoneType] = None, completed_at: Union[datetime.datetime, NoneType] = None)" + }, + "FreshnessMetadata": { + "type": "object", + "required": [], + "properties": { + "dbt_schema_version": { + "type": "string", + "default": "https://schemas.getdbt.com/dbt/sources/v3.json" + }, + "dbt_version": { + "type": "string", + "default": "1.2.0a1" + }, + "generated_at": { + "type": "string", + "format": "date-time", + "default": "2022-04-14T19:46:57.671562Z" + }, + "invocation_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "16540468-9312-488f-9721-e3fb46751526" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "default": {} + } + }, + "additionalProperties": false, + "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" + }, + "SourceFreshnessRuntimeError": { + "type": "object", + "required": [ + "unique_id", + "status" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "error": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "status": { + "type": "string", + "enum": [ + "runtime error" + ] + } + }, + "additionalProperties": false, + "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" + }, + "SourceFreshnessOutput": { + "type": "object", + "required": [ + "unique_id", + "max_loaded_at", + "snapshotted_at", + "max_loaded_at_time_ago_in_s", + "status", + "criteria", + "adapter_response", + "timing", + "thread_id", + "execution_time" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "max_loaded_at": { + "type": "string", + "format": "date-time" + }, + "snapshotted_at": { + "type": "string", + "format": "date-time" + }, + "max_loaded_at_time_ago_in_s": { + "type": "number" + }, + "status": { + "type": "string", + "enum": [ + "pass", + "warn", + "error", + "runtime error" + ] + }, + "criteria": { + "$ref": "#/definitions/FreshnessThreshold" + }, + "adapter_response": { + "type": "object" + }, + "timing": { + "type": "array", + "items": { + "$ref": "#/definitions/TimingInfo" + } + }, + "thread_id": { + "type": "string" + }, + "execution_time": { + "type": "number" + } + }, + "additionalProperties": false, + "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" + }, + "FreshnessThreshold": { + "type": "object", + "required": [], + "properties": { + "warn_after": { + "oneOf": [ + { + "$ref": "#/definitions/Time" + }, + { + "type": "null" + } + ], + "default": { + "count": null, + "period": null + } + }, + "error_after": { + "oneOf": [ + { + "$ref": "#/definitions/Time" + }, + { + "type": "null" + } + ], + "default": { + "count": null, + "period": null + } + }, + "filter": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "FreshnessThreshold(warn_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , error_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , filter: Union[str, NoneType] = None)" + }, + "Time": { + "type": "object", + "required": [], + "properties": { + "count": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "period": { + "oneOf": [ + { + "type": "string", + "enum": [ + "minute", + "hour", + "day" + ] + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "Time(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.TimePeriod, NoneType] = None)" + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://schemas.getdbt.com/dbt/run-results/v4.json" +} diff --git a/schemas/dbt/sources/v3.json b/schemas/dbt/sources/v3.json new file mode 100644 index 00000000000..dfeaf6372c0 --- /dev/null +++ b/schemas/dbt/sources/v3.json @@ -0,0 +1,281 @@ +{ + "type": "object", + "required": [ + "metadata", + "results", + "elapsed_time" + ], + "properties": { + "metadata": { + "$ref": "#/definitions/FreshnessMetadata" + }, + "results": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/SourceFreshnessRuntimeError" + }, + { + "$ref": "#/definitions/SourceFreshnessOutput" + } + ] + } + }, + "elapsed_time": { + "type": "number" + } + }, + "additionalProperties": false, + "description": "FreshnessExecutionResultArtifact(metadata: dbt.contracts.results.FreshnessMetadata, results: Sequence[Union[dbt.contracts.results.SourceFreshnessRuntimeError, dbt.contracts.results.SourceFreshnessOutput]], elapsed_time: float)", + "definitions": { + "FreshnessMetadata": { + "type": "object", + "required": [], + "properties": { + "dbt_schema_version": { + "type": "string", + "default": "https://schemas.getdbt.com/dbt/sources/v3.json" + }, + "dbt_version": { + "type": "string", + "default": "1.2.0a1" + }, + "generated_at": { + "type": "string", + "format": "date-time", + "default": "2022-04-14T19:46:57.671562Z" + }, + "invocation_id": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "16540468-9312-488f-9721-e3fb46751526" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "default": {} + } + }, + "additionalProperties": false, + "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" + }, + "SourceFreshnessRuntimeError": { + "type": "object", + "required": [ + "unique_id", + "status" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "error": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "status": { + "type": "string", + "enum": [ + "runtime error" + ] + } + }, + "additionalProperties": false, + "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" + }, + "SourceFreshnessOutput": { + "type": "object", + "required": [ + "unique_id", + "max_loaded_at", + "snapshotted_at", + "max_loaded_at_time_ago_in_s", + "status", + "criteria", + "adapter_response", + "timing", + "thread_id", + "execution_time" + ], + "properties": { + "unique_id": { + "type": "string" + }, + "max_loaded_at": { + "type": "string", + "format": "date-time" + }, + "snapshotted_at": { + "type": "string", + "format": "date-time" + }, + "max_loaded_at_time_ago_in_s": { + "type": "number" + }, + "status": { + "type": "string", + "enum": [ + "pass", + "warn", + "error", + "runtime error" + ] + }, + "criteria": { + "$ref": "#/definitions/FreshnessThreshold" + }, + "adapter_response": { + "type": "object" + }, + "timing": { + "type": "array", + "items": { + "$ref": "#/definitions/TimingInfo" + } + }, + "thread_id": { + "type": "string" + }, + "execution_time": { + "type": "number" + } + }, + "additionalProperties": false, + "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" + }, + "FreshnessThreshold": { + "type": "object", + "required": [], + "properties": { + "warn_after": { + "oneOf": [ + { + "$ref": "#/definitions/Time" + }, + { + "type": "null" + } + ], + "default": { + "count": null, + "period": null + } + }, + "error_after": { + "oneOf": [ + { + "$ref": "#/definitions/Time" + }, + { + "type": "null" + } + ], + "default": { + "count": null, + "period": null + } + }, + "filter": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "FreshnessThreshold(warn_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , error_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , filter: Union[str, NoneType] = None)" + }, + "Time": { + "type": "object", + "required": [], + "properties": { + "count": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "period": { + "oneOf": [ + { + "type": "string", + "enum": [ + "minute", + "hour", + "day" + ] + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "Time(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.TimePeriod, NoneType] = None)" + }, + "TimingInfo": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "started_at": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "completed_at": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "description": "TimingInfo(name: str, started_at: Union[datetime.datetime, NoneType] = None, completed_at: Union[datetime.datetime, NoneType] = None)" + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://schemas.getdbt.com/dbt/sources/v3.json" +} diff --git a/tests/functional/docs_generate/expected_catalog.py b/tests/adapter/dbt/tests/adapter/basic/expected_catalog.py similarity index 100% rename from tests/functional/docs_generate/expected_catalog.py rename to tests/adapter/dbt/tests/adapter/basic/expected_catalog.py diff --git a/tests/functional/docs_generate/test_docs_generate.py b/tests/adapter/dbt/tests/adapter/basic/test_docs_generate.py similarity index 76% rename from tests/functional/docs_generate/test_docs_generate.py rename to tests/adapter/dbt/tests/adapter/basic/test_docs_generate.py index 7821c2b6d37..0558409ddae 100644 --- a/tests/functional/docs_generate/test_docs_generate.py +++ b/tests/adapter/dbt/tests/adapter/basic/test_docs_generate.py @@ -5,19 +5,11 @@ from dbt.tests.util import run_dbt, rm_file, get_artifact, check_datetime_between from dbt.tests.fixtures.project import write_project_files -from tests.functional.docs_generate.expected_catalog import ( +from dbt.tests.adapter.basic.expected_catalog import ( base_expected_catalog, no_stats, expected_references_catalog, ) -from tests.functional.docs_generate.expected_manifest import ( - expected_seeded_manifest, - expected_references_manifest, -) -from tests.functional.docs_generate.expected_run_results import ( - expected_run_results, - expected_references_run_results, -) models__schema_yml = """ version: 2 @@ -352,80 +344,6 @@ def verify_metadata(metadata, dbt_schema_version, start_time): assert metadata["env"] == {key: "env_value"} -def verify_manifest(project, expected_manifest, start_time): - manifest_path = os.path.join(project.project_root, "target", "manifest.json") - assert os.path.exists(manifest_path) - - manifest = get_artifact(manifest_path) - - manifest_keys = { - "nodes", - "sources", - "macros", - "parent_map", - "child_map", - "metrics", - "docs", - "metadata", - "docs", - "disabled", - "exposures", - "selectors", - } - - assert set(manifest.keys()) == manifest_keys - - for key in manifest_keys: - if key == "macros": - verify_manifest_macros(manifest, expected_manifest.get("macros")) - elif key == "metadata": - metadata = manifest["metadata"] - verify_metadata( - metadata, "https://schemas.getdbt.com/dbt/manifest/v5.json", start_time - ) - assert ( - "project_id" in metadata - and metadata["project_id"] == "098f6bcd4621d373cade4e832627b4f6" - ) - assert ( - "send_anonymous_usage_stats" in metadata - and metadata["send_anonymous_usage_stats"] is False - ) - assert "adapter_type" in metadata and metadata["adapter_type"] == project.adapter_type - elif key in ["nodes", "sources", "exposures", "metrics", "disabled", "docs"]: - for unique_id, node in expected_manifest[key].items(): - assert unique_id in manifest[key] - assert manifest[key][unique_id] == node - else: # ['docs', 'parent_map', 'child_map', 'selectors'] - assert manifest[key] == expected_manifest[key] - - -def verify_manifest_macros(manifest, expected=None): - assert "macros" in manifest - if expected: - for unique_id, expected_macro in expected.items(): - assert unique_id in manifest["macros"] - actual_macro = manifest["macros"][unique_id] - assert expected_macro == actual_macro - - -def verify_run_results(project, expected_run_results, start_time): - run_results_path = os.path.join(project.project_root, "target", "run_results.json") - run_results = get_artifact(run_results_path) - assert "metadata" in run_results - verify_metadata( - run_results["metadata"], "https://schemas.getdbt.com/dbt/run-results/v4.json", start_time - ) - assert "elapsed_time" in run_results - assert run_results["elapsed_time"] > 0 - assert isinstance(run_results["elapsed_time"], float) - assert "args" in run_results - # sort the results so we can make reasonable assertions - run_results["results"].sort(key=lambda r: r["unique_id"]) - assert run_results["results"] == expected_run_results - set(run_results) == {"elapsed_time", "results", "metadata"} - - def run_and_generate(project, args=None): results = run_dbt(["run"]) assert len(results) == 2 @@ -485,7 +403,7 @@ def project_config_update(self, unique_schema): } -class TestDocsGenerate(BaseGenerateProject): +class BaseDocsGenerate(BaseGenerateProject): @pytest.fixture(scope="class") def models(self): return { @@ -518,16 +436,17 @@ def test_run_and_generate(self, project, expected_catalog): start_time = run_and_generate(project) verify_catalog(project, expected_catalog, start_time) - verify_manifest(project, expected_seeded_manifest(project, quote_model=False), start_time) - verify_run_results(project, expected_run_results(), start_time) - # Check that assets have been copied to the target directory for use in the docs html page assert os.path.exists(os.path.join(".", "target", "assets")) assert os.path.exists(os.path.join(".", "target", "assets", "lorem-ipsum.txt")) assert not os.path.exists(os.path.join(".", "target", "non-existent-assets")) -class TestReferences(BaseGenerateProject): +class TestDocsGenerate(BaseDocsGenerate): + pass + + +class BaseReferences(BaseGenerateProject): @pytest.fixture(scope="class") def models(self): return { @@ -541,5 +460,7 @@ def models(self): def test_references(self, project): start_time = run_and_generate(project) verify_catalog(project, expected_references_catalog(project), start_time) - verify_manifest(project, expected_references_manifest(project), start_time) - verify_run_results(project, expected_references_run_results(), start_time) + + +class TestReferences(BaseReferences): + pass diff --git a/tests/functional/docs_generate/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py similarity index 100% rename from tests/functional/docs_generate/expected_manifest.py rename to tests/functional/artifacts/expected_manifest.py diff --git a/tests/functional/docs_generate/expected_run_results.py b/tests/functional/artifacts/expected_run_results.py similarity index 100% rename from tests/functional/docs_generate/expected_run_results.py rename to tests/functional/artifacts/expected_run_results.py diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py new file mode 100644 index 00000000000..b8927b4144d --- /dev/null +++ b/tests/functional/artifacts/test_artifacts.py @@ -0,0 +1,484 @@ +import pytest +import os +from datetime import datetime +import dbt + +from dbt.tests.util import run_dbt, get_artifact, check_datetime_between +from tests.functional.artifacts.expected_manifest import ( + expected_seeded_manifest, + expected_references_manifest, +) +from tests.functional.artifacts.expected_run_results import ( + expected_run_results, + expected_references_run_results, +) + +models__schema_yml = """ +version: 2 + +models: + - name: model + description: "The test model" + docs: + show: false + columns: + - name: id + description: The user ID number + tests: + - unique + - not_null + - name: first_name + description: The user's first name + - name: email + description: The user's email + - name: ip_address + description: The user's IP address + - name: updated_at + description: The last time this user's email was updated + tests: + - test.nothing + + - name: second_model + description: "The second test model" + docs: + show: false + columns: + - name: id + description: The user ID number + - name: first_name + description: The user's first name + - name: email + description: The user's email + - name: ip_address + description: The user's IP address + - name: updated_at + description: The last time this user's email was updated + + +sources: + - name: my_source + description: "My source" + loader: a_loader + schema: "{{ var('test_schema') }}" + tables: + - name: my_table + description: "My table" + identifier: seed + quoting: + identifier: True + columns: + - name: id + description: "An ID field" + + +exposures: + - name: simple_exposure + type: dashboard + depends_on: + - ref('model') + - source('my_source', 'my_table') + owner: + email: something@example.com + - name: notebook_exposure + type: notebook + depends_on: + - ref('model') + - ref('second_model') + owner: + email: something@example.com + name: Some name + description: > + A description of the complex exposure + maturity: medium + meta: + tool: 'my_tool' + languages: + - python + tags: ['my_department'] + url: http://example.com/notebook/1 +""" + +models__second_model_sql = """ +{{ + config( + materialized='view', + schema='test', + ) +}} + +select * from {{ ref('seed') }} +""" + +models__readme_md = """ +This is a readme.md file with {{ invalid-ish jinja }} in it +""" + +models__model_sql = """ +{{ + config( + materialized='view', + ) +}} + +select * from {{ ref('seed') }} +""" + +seed__schema_yml = """ +version: 2 +seeds: + - name: seed + description: "The test seed" + columns: + - name: id + description: The user ID number + - name: first_name + description: The user's first name + - name: email + description: The user's email + - name: ip_address + description: The user's IP address + - name: updated_at + description: The last time this user's email was updated +""" + +seed__seed_csv = """id,first_name,email,ip_address,updated_at +1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 +""" + +macros__schema_yml = """ +version: 2 +macros: + - name: test_nothing + description: "{{ doc('macro_info') }}" + meta: + some_key: 100 + arguments: + - name: model + type: Relation + description: "{{ doc('macro_arg_info') }}" +""" + +macros__macro_md = """ +{% docs macro_info %} +My custom test that I wrote that does nothing +{% enddocs %} + +{% docs macro_arg_info %} +The model for my custom test +{% enddocs %} +""" + +macros__dummy_test_sql = """ +{% test nothing(model) %} + +-- a silly test to make sure that table-level tests show up in the manifest +-- without a column_name field +select 0 + +{% endtest %} +""" + +snapshot__snapshot_seed_sql = """ +{% snapshot snapshot_seed %} +{{ + config( + unique_key='id', + strategy='check', + check_cols='all', + target_schema=var('alternate_schema') + ) +}} +select * from {{ ref('seed') }} +{% endsnapshot %} +""" + +ref_models__schema_yml = """ +version: 2 + +models: + - name: ephemeral_summary + description: "{{ doc('ephemeral_summary') }}" + columns: &summary_columns + - name: first_name + description: "{{ doc('summary_first_name') }}" + - name: ct + description: "{{ doc('summary_count') }}" + - name: view_summary + description: "{{ doc('view_summary') }}" + columns: *summary_columns + +sources: + - name: my_source + description: "{{ doc('source_info') }}" + loader: a_loader + schema: "{{ var('test_schema') }}" + quoting: + database: False + identifier: False + tables: + - name: my_table + description: "{{ doc('table_info') }}" + identifier: seed + quoting: + identifier: True + columns: + - name: id + description: "{{ doc('column_info') }}" + +exposures: + - name: notebook_exposure + type: notebook + depends_on: + - ref('view_summary') + owner: + email: something@example.com + name: Some name + description: "{{ doc('notebook_info') }}" + maturity: medium + url: http://example.com/notebook/1 + meta: + tool: 'my_tool' + languages: + - python + tags: ['my_department'] + +""" + +ref_models__view_summary_sql = """ +{{ + config( + materialized = "view" + ) +}} + +select first_name, ct from {{ref('ephemeral_summary')}} +order by ct asc + +""" + +ref_models__ephemeral_summary_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select first_name, count(*) as ct from {{ref('ephemeral_copy')}} +group by first_name +order by first_name asc + +""" + +ref_models__ephemeral_copy_sql = """ +{{ + config( + materialized = "ephemeral" + ) +}} + +select * from {{ source("my_source", "my_table") }} + +""" + +ref_models__docs_md = """ +{% docs ephemeral_summary %} +A summmary table of the ephemeral copy of the seed data +{% enddocs %} + +{% docs summary_first_name %} +The first name being summarized +{% enddocs %} + +{% docs summary_count %} +The number of instances of the first name +{% enddocs %} + +{% docs view_summary %} +A view of the summary of the ephemeral copy of the seed data +{% enddocs %} + +{% docs source_info %} +My source +{% enddocs %} + +{% docs table_info %} +My table +{% enddocs %} + +{% docs column_info %} +An ID field +{% enddocs %} + +{% docs notebook_info %} +A description of the complex exposure +{% enddocs %} + +""" + + +def verify_metadata(metadata, dbt_schema_version, start_time): + assert "generated_at" in metadata + check_datetime_between(metadata["generated_at"], start=start_time) + assert "dbt_version" in metadata + assert metadata["dbt_version"] == dbt.version.__version__ + assert "dbt_schema_version" in metadata + assert metadata["dbt_schema_version"] == dbt_schema_version + assert metadata["invocation_id"] == dbt.tracking.active_user.invocation_id + key = "env_key" + if os.name == "nt": + key = key.upper() + assert metadata["env"] == {key: "env_value"} + + +def verify_manifest(project, expected_manifest, start_time): + manifest_path = os.path.join(project.project_root, "target", "manifest.json") + assert os.path.exists(manifest_path) + + manifest = get_artifact(manifest_path) + + manifest_keys = { + "nodes", + "sources", + "macros", + "parent_map", + "child_map", + "metrics", + "docs", + "metadata", + "docs", + "disabled", + "exposures", + "selectors", + } + + assert set(manifest.keys()) == manifest_keys + + for key in manifest_keys: + if key == "macros": + verify_manifest_macros(manifest, expected_manifest.get("macros")) + elif key == "metadata": + metadata = manifest["metadata"] + verify_metadata( + metadata, "https://schemas.getdbt.com/dbt/manifest/v5.json", start_time + ) + assert ( + "project_id" in metadata + and metadata["project_id"] == "098f6bcd4621d373cade4e832627b4f6" + ) + assert ( + "send_anonymous_usage_stats" in metadata + and metadata["send_anonymous_usage_stats"] is False + ) + assert "adapter_type" in metadata and metadata["adapter_type"] == project.adapter_type + elif key in ["nodes", "sources", "exposures", "metrics", "disabled", "docs"]: + for unique_id, node in expected_manifest[key].items(): + assert unique_id in manifest[key] + assert manifest[key][unique_id] == node + else: # ['docs', 'parent_map', 'child_map', 'selectors'] + assert manifest[key] == expected_manifest[key] + + +def verify_manifest_macros(manifest, expected=None): + assert "macros" in manifest + if expected: + for unique_id, expected_macro in expected.items(): + assert unique_id in manifest["macros"] + actual_macro = manifest["macros"][unique_id] + assert expected_macro == actual_macro + + +def verify_run_results(project, expected_run_results, start_time): + run_results_path = os.path.join(project.project_root, "target", "run_results.json") + run_results = get_artifact(run_results_path) + assert "metadata" in run_results + verify_metadata( + run_results["metadata"], "https://schemas.getdbt.com/dbt/run-results/v4.json", start_time + ) + assert "elapsed_time" in run_results + assert run_results["elapsed_time"] > 0 + assert isinstance(run_results["elapsed_time"], float) + assert "args" in run_results + # sort the results so we can make reasonable assertions + run_results["results"].sort(key=lambda r: r["unique_id"]) + assert run_results["results"] == expected_run_results + set(run_results) == {"elapsed_time", "results", "metadata"} + + +class BaseVerifyProject: + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + alternate_schema_name = project.test_schema + "_test" + project.create_test_schema(schema_name=alternate_schema_name) + os.environ["DBT_ENV_CUSTOM_ENV_env_key"] = "env_value" + run_dbt(["seed"]) + yield + del os.environ["DBT_ENV_CUSTOM_ENV_env_key"] + + @pytest.fixture(scope="class") + def seeds(self): + return {"schema.yml": seed__schema_yml, "seed.csv": seed__seed_csv} + + @pytest.fixture(scope="class") + def macros(self): + return { + "schema.yml": macros__schema_yml, + "macro.md": macros__macro_md, + "dummy_test.sql": macros__dummy_test_sql, + } + + @pytest.fixture(scope="class") + def snapshots(self): + return {"snapshot_seed.sql": snapshot__snapshot_seed_sql} + + @pytest.fixture(scope="class") + def project_config_update(self, unique_schema): + alternate_schema = unique_schema + "_test" + return { + "asset-paths": ["assets", "invalid-asset-paths"], + "vars": { + "test_schema": unique_schema, + "alternate_schema": alternate_schema, + }, + "seeds": { + "quote_columns": True, + }, + "quoting": {"identifier": False}, + } + + +class TestVerifyArtifacts(BaseVerifyProject): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models__schema_yml, + "second_model.sql": models__second_model_sql, + "readme.md": models__readme_md, + "model.sql": models__model_sql, + } + + # Test generic "docs generate" command + def test_run_and_generate(self, project): + start_time = datetime.utcnow() + results = run_dbt(["compile"]) + assert len(results) == 7 + verify_manifest(project, expected_seeded_manifest(project, quote_model=False), start_time) + verify_run_results(project, expected_run_results(), start_time) + + +class TestVerifyArtifactsReferences(BaseVerifyProject): + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": ref_models__schema_yml, + "view_summary.sql": ref_models__view_summary_sql, + "ephemeral_summary.sql": ref_models__ephemeral_summary_sql, + "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, + "docs.md": ref_models__docs_md, + } + + def test_references(self, project): + start_time = datetime.utcnow() + results = run_dbt(["compile"]) + assert len(results) == 4 + verify_manifest(project, expected_references_manifest(project), start_time) + verify_run_results(project, expected_references_run_results(), start_time) diff --git a/tests/functional/docs_generate/test_override.py b/tests/functional/artifacts/test_override.py similarity index 100% rename from tests/functional/docs_generate/test_override.py rename to tests/functional/artifacts/test_override.py From de28f8c3303a2c4b313160bb322fb2b8c576aa0a Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 15 Apr 2022 15:33:06 -0400 Subject: [PATCH 6/8] remove asset-paths from test_artifacts.py --- tests/functional/artifacts/test_artifacts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index b8927b4144d..0dad1546534 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -434,7 +434,6 @@ def snapshots(self): def project_config_update(self, unique_schema): alternate_schema = unique_schema + "_test" return { - "asset-paths": ["assets", "invalid-asset-paths"], "vars": { "test_schema": unique_schema, "alternate_schema": alternate_schema, From 67b712b86d3b16fcedf7508f93cee067e34d509d Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 15 Apr 2022 16:51:24 -0400 Subject: [PATCH 7/8] Add validation of manifest and run_results artifacts by current and stored jsonschemas --- schemas/dbt/catalog/v1.json | 4 +- schemas/dbt/manifest/v5.json | 52 ++++++------ schemas/dbt/run-results/v4.json | 8 +- schemas/dbt/sources/v3.json | 4 +- tests/functional/artifacts/test_artifacts.py | 84 ++++++++++++++++---- 5 files changed, 103 insertions(+), 49 deletions(-) diff --git a/schemas/dbt/catalog/v1.json b/schemas/dbt/catalog/v1.json index ab969e349ac..f925a18a7d6 100644 --- a/schemas/dbt/catalog/v1.json +++ b/schemas/dbt/catalog/v1.json @@ -53,7 +53,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2022-04-14T19:46:57.674922Z" + "default": "2022-04-15T20:38:22.701177Z" }, "invocation_id": { "oneOf": [ @@ -64,7 +64,7 @@ "type": "null" } ], - "default": "16540468-9312-488f-9721-e3fb46751526" + "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", diff --git a/schemas/dbt/manifest/v5.json b/schemas/dbt/manifest/v5.json index addffd2f342..6c73ab8876b 100644 --- a/schemas/dbt/manifest/v5.json +++ b/schemas/dbt/manifest/v5.json @@ -244,7 +244,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2022-04-14T19:46:57.676592Z" + "default": "2022-04-15T20:38:22.703053Z" }, "invocation_id": { "oneOf": [ @@ -255,7 +255,7 @@ "type": "null" } ], - "default": "16540468-9312-488f-9721-e3fb46751526" + "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", @@ -501,7 +501,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.680463 + "default": 1650055102.707036 }, "config_call_dict": { "type": "object", @@ -999,7 +999,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.682705 + "default": 1650055102.7093382 }, "config_call_dict": { "type": "object", @@ -1340,7 +1340,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.684142 + "default": 1650055102.710903 }, "config_call_dict": { "type": "object", @@ -1569,7 +1569,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.685675 + "default": 1650055102.712516 }, "config_call_dict": { "type": "object", @@ -1808,7 +1808,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.687135 + "default": 1650055102.714043 }, "config_call_dict": { "type": "object", @@ -2037,7 +2037,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.688512 + "default": 1650055102.715457 }, "config_call_dict": { "type": "object", @@ -2269,7 +2269,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.690442 + "default": 1650055102.717103 }, "config_call_dict": { "type": "object", @@ -2546,7 +2546,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.693683 + "default": 1650055102.719672 }, "config_call_dict": { "type": "object", @@ -2912,7 +2912,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.695231 + "default": 1650055102.72113 }, "config_call_dict": { "type": "object", @@ -3137,7 +3137,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.6965492 + "default": 1650055102.7224698 }, "config_call_dict": { "type": "object", @@ -3330,7 +3330,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.697702 + "default": 1650055102.723626 }, "config_call_dict": { "type": "object", @@ -3524,7 +3524,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.6988769 + "default": 1650055102.724789 }, "config_call_dict": { "type": "object", @@ -3728,7 +3728,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.700115 + "default": 1650055102.725993 }, "config_call_dict": { "type": "object", @@ -3922,7 +3922,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.701325 + "default": 1650055102.727343 }, "config_call_dict": { "type": "object", @@ -4116,7 +4116,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.7026129 + "default": 1650055102.728572 }, "config_call_dict": { "type": "object", @@ -4313,7 +4313,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.7037811 + "default": 1650055102.729745 }, "config_call_dict": { "type": "object", @@ -4528,7 +4528,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.705039 + "default": 1650055102.731012 }, "config_call_dict": { "type": "object", @@ -4706,7 +4706,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.707365 + "default": 1650055102.733336 }, "config_call_dict": { "type": "object", @@ -5063,7 +5063,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.7098281 + "default": 1650055102.735436 } }, "additionalProperties": false, @@ -5178,7 +5178,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2022-04-14T19:46:57.671562Z" + "default": "2022-04-15T20:38:22.697740Z" }, "invocation_id": { "oneOf": [ @@ -5189,7 +5189,7 @@ "type": "null" } ], - "default": "16540468-9312-488f-9721-e3fb46751526" + "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", @@ -5546,7 +5546,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.710664 + "default": 1650055102.736266 } }, "additionalProperties": false, @@ -5770,7 +5770,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.711809 + "default": 1650055102.7375019 } }, "additionalProperties": false, @@ -5951,7 +5951,7 @@ }, "created_at": { "type": "number", - "default": 1649965617.712818 + "default": 1650055102.738508 } }, "additionalProperties": false, diff --git a/schemas/dbt/run-results/v4.json b/schemas/dbt/run-results/v4.json index 654a35c4713..27326fe8b62 100644 --- a/schemas/dbt/run-results/v4.json +++ b/schemas/dbt/run-results/v4.json @@ -42,7 +42,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2022-04-14T19:46:57.673939Z" + "default": "2022-04-15T20:38:22.700175Z" }, "invocation_id": { "oneOf": [ @@ -53,7 +53,7 @@ "type": "null" } ], - "default": "16540468-9312-488f-9721-e3fb46751526" + "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", @@ -200,7 +200,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2022-04-14T19:46:57.671562Z" + "default": "2022-04-15T20:38:22.697740Z" }, "invocation_id": { "oneOf": [ @@ -211,7 +211,7 @@ "type": "null" } ], - "default": "16540468-9312-488f-9721-e3fb46751526" + "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", diff --git a/schemas/dbt/sources/v3.json b/schemas/dbt/sources/v3.json index dfeaf6372c0..95cd76fec9f 100644 --- a/schemas/dbt/sources/v3.json +++ b/schemas/dbt/sources/v3.json @@ -44,7 +44,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2022-04-14T19:46:57.671562Z" + "default": "2022-04-15T20:38:22.697740Z" }, "invocation_id": { "oneOf": [ @@ -55,7 +55,7 @@ "type": "null" } ], - "default": "16540468-9312-488f-9721-e3fb46751526" + "default": "34abf75e-59d3-442f-920c-fa3843d98014" }, "env": { "type": "object", diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index 0dad1546534..f752756893c 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -2,6 +2,7 @@ import os from datetime import datetime import dbt +import jsonschema from dbt.tests.util import run_dbt, get_artifact, check_datetime_between from tests.functional.artifacts.expected_manifest import ( @@ -13,6 +14,9 @@ expected_references_run_results, ) +from dbt.contracts.graph.manifest import WritableManifest +from dbt.contracts.results import RunResultsArtifact + models__schema_yml = """ version: 2 @@ -330,12 +334,21 @@ def verify_metadata(metadata, dbt_schema_version, start_time): assert metadata["env"] == {key: "env_value"} -def verify_manifest(project, expected_manifest, start_time): +def verify_manifest(project, expected_manifest, start_time, manifest_schema_path): manifest_path = os.path.join(project.project_root, "target", "manifest.json") assert os.path.exists(manifest_path) - manifest = get_artifact(manifest_path) + # Verify that manifest jsonschema from WritableManifest works + manifest_schema = WritableManifest.json_schema() + validate(manifest_schema, manifest) + + # Verify that stored manifest jsonschema works. + # If this fails, schemas need to be updated with: + # scripts/collect-artifact-schema.py --path schemas + stored_manifest_schema = get_artifact(manifest_schema_path) + validate(stored_manifest_schema, manifest) + manifest_keys = { "nodes", "sources", @@ -358,9 +371,8 @@ def verify_manifest(project, expected_manifest, start_time): verify_manifest_macros(manifest, expected_manifest.get("macros")) elif key == "metadata": metadata = manifest["metadata"] - verify_metadata( - metadata, "https://schemas.getdbt.com/dbt/manifest/v5.json", start_time - ) + dbt_schema_version = str(WritableManifest.dbt_schema_version) + verify_metadata(metadata, dbt_schema_version, start_time) assert ( "project_id" in metadata and metadata["project_id"] == "098f6bcd4621d373cade4e832627b4f6" @@ -387,13 +399,23 @@ def verify_manifest_macros(manifest, expected=None): assert expected_macro == actual_macro -def verify_run_results(project, expected_run_results, start_time): +def verify_run_results(project, expected_run_results, start_time, run_results_schema_path): run_results_path = os.path.join(project.project_root, "target", "run_results.json") run_results = get_artifact(run_results_path) assert "metadata" in run_results - verify_metadata( - run_results["metadata"], "https://schemas.getdbt.com/dbt/run-results/v4.json", start_time - ) + + # Verify that jsonschema for RunResultsArtifact works + run_results_schema = RunResultsArtifact.json_schema() + validate(run_results_schema, run_results) + + # Verify that stored run_results jsonschema works. + # If this fails, schemas need to be updated with: + # scripts/collect-artifact-schema.py --path schemas + stored_run_results_schema = get_artifact(run_results_schema_path) + validate(stored_run_results_schema, run_results) + + dbt_schema_version = str(RunResultsArtifact.dbt_schema_version) + verify_metadata(run_results["metadata"], dbt_schema_version, start_time) assert "elapsed_time" in run_results assert run_results["elapsed_time"] > 0 assert isinstance(run_results["elapsed_time"], float) @@ -444,6 +466,28 @@ def project_config_update(self, unique_schema): "quoting": {"identifier": False}, } + @pytest.fixture(scope="class") + def manifest_schema_path(self, request): + schema_version_paths = WritableManifest.dbt_schema_version.path.split("/") + manifest_schema_path = os.path.join( + request.config.rootdir, "schemas", *schema_version_paths + ) + return manifest_schema_path + + @pytest.fixture(scope="class") + def run_results_schema_path(self, request): + schema_version_paths = RunResultsArtifact.dbt_schema_version.path.split("/") + run_results_schema_path = os.path.join( + request.config.rootdir, "schemas", *schema_version_paths + ) + return run_results_schema_path + + +def validate(artifact_schema, artifact_dict): + validator = jsonschema.Draft7Validator(artifact_schema) + error = next(iter(validator.iter_errors(artifact_dict)), None) + assert error is None + class TestVerifyArtifacts(BaseVerifyProject): @pytest.fixture(scope="class") @@ -456,12 +500,18 @@ def models(self): } # Test generic "docs generate" command - def test_run_and_generate(self, project): + def test_run_and_generate(self, project, manifest_schema_path, run_results_schema_path): start_time = datetime.utcnow() results = run_dbt(["compile"]) assert len(results) == 7 - verify_manifest(project, expected_seeded_manifest(project, quote_model=False), start_time) - verify_run_results(project, expected_run_results(), start_time) + + verify_manifest( + project, + expected_seeded_manifest(project, quote_model=False), + start_time, + manifest_schema_path, + ) + verify_run_results(project, expected_run_results(), start_time, run_results_schema_path) class TestVerifyArtifactsReferences(BaseVerifyProject): @@ -475,9 +525,13 @@ def models(self): "docs.md": ref_models__docs_md, } - def test_references(self, project): + def test_references(self, project, manifest_schema_path, run_results_schema_path): start_time = datetime.utcnow() results = run_dbt(["compile"]) assert len(results) == 4 - verify_manifest(project, expected_references_manifest(project), start_time) - verify_run_results(project, expected_references_run_results(), start_time) + verify_manifest( + project, expected_references_manifest(project), start_time, manifest_schema_path + ) + verify_run_results( + project, expected_references_run_results(), start_time, run_results_schema_path + ) From a37509a669444e00d6cacf968d1cd24c029a286b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 18 Apr 2022 14:07:45 -0400 Subject: [PATCH 8/8] Updates to support docs generate tests in adapter repos --- core/dbt/tests/fixtures/project.py | 5 + core/dbt/tests/util.py | 50 ++++++++- .../tests/adapter/basic/expected_catalog.py | 105 +++++++++++------- .../tests/adapter/basic/test_docs_generate.py | 73 ++++++++---- .../functional/artifacts/expected_manifest.py | 18 +-- .../artifacts/expected_run_results.py | 8 +- tests/functional/artifacts/test_artifacts.py | 2 +- .../sources/test_source_fresher_state.py | 26 +---- .../sources/test_source_freshness.py | 26 +---- 9 files changed, 174 insertions(+), 139 deletions(-) diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 56756ac0e22..40ddf849cb6 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -120,6 +120,11 @@ def dbt_profile_target(): } +@pytest.fixture(scope="class") +def profile_user(dbt_profile_target): + return dbt_profile_target["user"] + + # This fixture can be overridden in a project. The data provided in this # fixture will be merged into the default project dictionary via a python 'update'. @pytest.fixture(scope="class") diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index b87f2e11b39..2603de4bf88 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -39,7 +39,12 @@ # get_relation_columns # update_rows # generate_update_clause - +# +# Classes for comparing fields in dictionaries +# AnyFloat +# AnyInteger +# AnyString +# AnyStringWith # ============================================================================= @@ -430,3 +435,46 @@ def check_table_does_not_exist(adapter, name): def check_table_does_exist(adapter, name): columns = get_relation_columns(adapter, name) assert len(columns) > 0 + + +# Utility classes for enabling comparison of dictionaries + + +class AnyFloat: + """Any float. Use this in assert calls""" + + def __eq__(self, other): + return isinstance(other, float) + + +class AnyInteger: + """Any Integer. Use this in assert calls""" + + def __eq__(self, other): + return isinstance(other, int) + + +class AnyString: + """Any string. Use this in assert calls""" + + def __eq__(self, other): + return isinstance(other, str) + + +class AnyStringWith: + """AnyStringWith("AUTO")""" + + def __init__(self, contains=None): + self.contains = contains + + def __eq__(self, other): + if not isinstance(other, str): + return False + + if self.contains is None: + return True + + return self.contains in other + + def __repr__(self): + return "AnyStringWith<{!r}>".format(self.contains) diff --git a/tests/adapter/dbt/tests/adapter/basic/expected_catalog.py b/tests/adapter/dbt/tests/adapter/basic/expected_catalog.py index 43937f48c9a..d7d558f4583 100644 --- a/tests/adapter/dbt/tests/adapter/basic/expected_catalog.py +++ b/tests/adapter/dbt/tests/adapter/basic/expected_catalog.py @@ -1,3 +1,6 @@ +from dbt.tests.util import AnyInteger + + def no_stats(): return { "has_stats": { @@ -12,6 +15,7 @@ def no_stats(): def base_expected_catalog( project, + role, id_type, text_type, time_type, @@ -21,7 +25,6 @@ def base_expected_catalog( seed_stats=None, case=None, case_columns=False, - model_database=None, ): if case is None: @@ -34,40 +37,38 @@ def case(x): if seed_stats is None: seed_stats = model_stats - if model_database is None: - model_database = project.database - my_schema_name = project.test_schema - role = "root" - alternate_schema = project.test_schema + "_test" + model_database = project.database + my_schema_name = case(project.test_schema) + alternate_schema = case(project.test_schema + "_test") expected_cols = { col_case("id"): { "name": col_case("id"), - "index": 1, + "index": AnyInteger(), "type": id_type, "comment": None, }, col_case("first_name"): { "name": col_case("first_name"), - "index": 2, + "index": AnyInteger(), "type": text_type, "comment": None, }, col_case("email"): { "name": col_case("email"), - "index": 3, + "index": AnyInteger(), "type": text_type, "comment": None, }, col_case("ip_address"): { "name": col_case("ip_address"), - "index": 4, + "index": AnyInteger(), "type": text_type, "comment": None, }, col_case("updated_at"): { "name": col_case("updated_at"), - "index": 5, + "index": AnyInteger(), "type": time_type, "comment": None, }, @@ -132,55 +133,81 @@ def case(x): } -def expected_references_catalog(project): +def expected_references_catalog( + project, + role, + id_type, + text_type, + time_type, + view_type, + table_type, + model_stats, + bigint_type=None, + seed_stats=None, + case=None, + case_columns=False, + view_summary_stats=None, +): + if case is None: + + def case(x): + return x + + col_case = case if case_columns else lambda x: x + + if seed_stats is None: + seed_stats = model_stats + + if view_summary_stats is None: + view_summary_stats = model_stats + model_database = project.database - my_schema_name = project.test_schema - role = "root" - stats = no_stats() + my_schema_name = case(project.test_schema) + summary_columns = { "first_name": { "name": "first_name", "index": 1, - "type": "text", + "type": text_type, "comment": None, }, "ct": { "name": "ct", "index": 2, - "type": "bigint", + "type": bigint_type, "comment": None, }, } seed_columns = { "id": { - "name": "id", + "name": col_case("id"), "index": 1, - "type": "integer", + "type": id_type, "comment": None, }, "first_name": { - "name": "first_name", + "name": col_case("first_name"), "index": 2, - "type": "text", + "type": text_type, "comment": None, }, "email": { - "name": "email", + "name": col_case("email"), "index": 3, - "type": "text", + "type": text_type, "comment": None, }, "ip_address": { - "name": "ip_address", + "name": col_case("ip_address"), "index": 4, - "type": "text", + "type": text_type, "comment": None, }, "updated_at": { - "name": "updated_at", + "name": col_case("updated_at"), "index": 5, - "type": "timestamp without time zone", + "type": time_type, "comment": None, }, } @@ -191,12 +218,12 @@ def expected_references_catalog(project): "metadata": { "schema": my_schema_name, "database": project.database, - "name": "seed", - "type": "BASE TABLE", + "name": case("seed"), + "type": table_type, "comment": None, "owner": role, }, - "stats": stats, + "stats": seed_stats, "columns": seed_columns, }, "model.test.ephemeral_summary": { @@ -204,12 +231,12 @@ def expected_references_catalog(project): "metadata": { "schema": my_schema_name, "database": model_database, - "name": "ephemeral_summary", - "type": "BASE TABLE", + "name": case("ephemeral_summary"), + "type": table_type, "comment": None, "owner": role, }, - "stats": stats, + "stats": model_stats, "columns": summary_columns, }, "model.test.view_summary": { @@ -217,12 +244,12 @@ def expected_references_catalog(project): "metadata": { "schema": my_schema_name, "database": model_database, - "name": "view_summary", - "type": "VIEW", + "name": case("view_summary"), + "type": view_type, "comment": None, "owner": role, }, - "stats": stats, + "stats": view_summary_stats, "columns": summary_columns, }, }, @@ -232,12 +259,12 @@ def expected_references_catalog(project): "metadata": { "schema": my_schema_name, "database": project.database, - "name": "seed", - "type": "BASE TABLE", + "name": case("seed"), + "type": table_type, "comment": None, "owner": role, }, - "stats": stats, + "stats": seed_stats, "columns": seed_columns, }, }, diff --git a/tests/adapter/dbt/tests/adapter/basic/test_docs_generate.py b/tests/adapter/dbt/tests/adapter/basic/test_docs_generate.py index 0558409ddae..59c4241d148 100644 --- a/tests/adapter/dbt/tests/adapter/basic/test_docs_generate.py +++ b/tests/adapter/dbt/tests/adapter/basic/test_docs_generate.py @@ -205,24 +205,6 @@ description: "{{ doc('view_summary') }}" columns: *summary_columns -sources: - - name: my_source - description: "{{ doc('source_info') }}" - loader: a_loader - schema: "{{ var('test_schema') }}" - quoting: - database: False - identifier: False - tables: - - name: my_table - description: "{{ doc('table_info') }}" - identifier: seed - quoting: - identifier: True - columns: - - name: id - description: "{{ doc('column_info') }}" - exposures: - name: notebook_exposure type: notebook @@ -242,6 +224,27 @@ """ +ref_sources__schema_yml = """ +version: 2 +sources: + - name: my_source + description: "{{ doc('source_info') }}" + loader: a_loader + schema: "{{ var('test_schema') }}" + quoting: + database: False + identifier: False + tables: + - name: my_table + description: "{{ doc('table_info') }}" + identifier: seed + quoting: + identifier: True + columns: + - name: id + description: "{{ doc('column_info') }}" +""" + ref_models__view_summary_sql = """ {{ config( @@ -327,7 +330,13 @@ def verify_catalog(project, expected_catalog, start_time): ) assert not catalog["errors"] for key in "nodes", "sources": - assert catalog[key] == expected_catalog[key] + for unique_id, expected_node in expected_catalog[key].items(): + found_node = catalog[key][unique_id] + for node_key in expected_node: + assert node_key in found_node + assert ( + found_node[node_key] == expected_node[node_key] + ), f"Key '{node_key}' in '{unique_id}' did not match" def verify_metadata(metadata, dbt_schema_version, start_time): @@ -414,9 +423,10 @@ def models(self): } @pytest.fixture(scope="class") - def expected_catalog(self, project): + def expected_catalog(self, project, profile_user): return base_expected_catalog( project, + role=profile_user, id_type="integer", text_type="text", time_type="timestamp without time zone", @@ -446,21 +456,36 @@ class TestDocsGenerate(BaseDocsGenerate): pass -class BaseReferences(BaseGenerateProject): +class BaseDocsGenReferences(BaseGenerateProject): @pytest.fixture(scope="class") def models(self): return { "schema.yml": ref_models__schema_yml, + "sources.yml": ref_sources__schema_yml, "view_summary.sql": ref_models__view_summary_sql, "ephemeral_summary.sql": ref_models__ephemeral_summary_sql, "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, "docs.md": ref_models__docs_md, } - def test_references(self, project): + @pytest.fixture(scope="class") + def expected_catalog(self, project, profile_user): + return expected_references_catalog( + project, + role=profile_user, + id_type="integer", + text_type="text", + time_type="timestamp without time zone", + bigint_type="bigint", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + ) + + def test_references(self, project, expected_catalog): start_time = run_and_generate(project) - verify_catalog(project, expected_references_catalog(project), start_time) + verify_catalog(project, expected_catalog, start_time) -class TestReferences(BaseReferences): +class TestDocsGenReferences(BaseDocsGenReferences): pass diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index ef7a9250331..305e1bbeae5 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -2,6 +2,7 @@ import dbt import os from unittest.mock import ANY +from dbt.tests.util import AnyStringWith # This produces an "expected manifest", with a number of the fields # modified to avoid ephemeral changes. @@ -172,23 +173,6 @@ def __str__(self): return self.__repr__() -class AnyStringWith: - def __init__(self, contains=None): - self.contains = contains - - def __eq__(self, other): - if not isinstance(other, str): - return False - - if self.contains is None: - return True - - return self.contains in other - - def __repr__(self): - return "AnyStringWith<{!r}>".format(self.contains) - - def expected_seeded_manifest(project, model_database=None, quote_model=False): model_sql_path = os.path.join("models", "model.sql") diff --git a/tests/functional/artifacts/expected_run_results.py b/tests/functional/artifacts/expected_run_results.py index 83cf6511124..04ae9c80dbf 100644 --- a/tests/functional/artifacts/expected_run_results.py +++ b/tests/functional/artifacts/expected_run_results.py @@ -1,11 +1,5 @@ from unittest.mock import ANY - - -class AnyFloat: - """Any float. Use this in assert calls to assert that it is a float.""" - - def __eq__(self, other): - return isinstance(other, float) +from dbt.tests.util import AnyFloat def expected_run_results(): diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index f752756893c..0644479486b 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -385,7 +385,7 @@ def verify_manifest(project, expected_manifest, start_time, manifest_schema_path elif key in ["nodes", "sources", "exposures", "metrics", "disabled", "docs"]: for unique_id, node in expected_manifest[key].items(): assert unique_id in manifest[key] - assert manifest[key][unique_id] == node + assert manifest[key][unique_id] == node, f"{unique_id} did not match" else: # ['docs', 'parent_map', 'child_map', 'selectors'] assert manifest[key] == expected_manifest[key] diff --git a/tests/functional/sources/test_source_fresher_state.py b/tests/functional/sources/test_source_fresher_state.py index df3aeffccd7..73519f5c5c6 100644 --- a/tests/functional/sources/test_source_fresher_state.py +++ b/tests/functional/sources/test_source_fresher_state.py @@ -7,6 +7,7 @@ from dbt.exceptions import InternalException +from dbt.tests.util import AnyStringWith, AnyFloat import dbt.version from tests.functional.sources.common_source_setup import BaseSourcesTest @@ -23,31 +24,6 @@ def copy_to_previous_state(): shutil.copyfile("target/run_results.json", "previous_state/run_results.json") -# put these here for now to get tests working -class AnyStringWith: - def __init__(self, contains=None): - self.contains = contains - - def __eq__(self, other): - if not isinstance(other, str): - return False - - if self.contains is None: - return True - - return self.contains in other - - def __repr__(self): - return "AnyStringWith<{!r}>".format(self.contains) - - -class AnyFloat: - """Any float. Use this in assertEqual() calls to assert that it is a float.""" - - def __eq__(self, other): - return isinstance(other, float) - - class SuccessfulSourceFreshnessTest(BaseSourcesTest): @pytest.fixture(scope="class", autouse=True) def setUp(self, project): diff --git a/tests/functional/sources/test_source_freshness.py b/tests/functional/sources/test_source_freshness.py index 4bbcd85706a..3b69aa7a0d6 100644 --- a/tests/functional/sources/test_source_freshness.py +++ b/tests/functional/sources/test_source_freshness.py @@ -11,31 +11,7 @@ filtered_models__schema_yml, override_freshness_models__schema_yml, ) - - -# put these here for now to get tests working -class AnyStringWith: - def __init__(self, contains=None): - self.contains = contains - - def __eq__(self, other): - if not isinstance(other, str): - return False - - if self.contains is None: - return True - - return self.contains in other - - def __repr__(self): - return "AnyStringWith<{!r}>".format(self.contains) - - -class AnyFloat: - """Any float. Use this in assertEqual() calls to assert that it is a float.""" - - def __eq__(self, other): - return isinstance(other, float) +from dbt.tests.util import AnyStringWith, AnyFloat class SuccessfulSourceFreshnessTest(BaseSourcesTest):