diff --git a/.changes/unreleased/Features-20221118-141120.yaml b/.changes/unreleased/Features-20221118-141120.yaml new file mode 100644 index 00000000000..100fb45a8eb --- /dev/null +++ b/.changes/unreleased/Features-20221118-141120.yaml @@ -0,0 +1,8 @@ +kind: Features +body: Data type constraints are now native to SQL table materializations. Enforce + columns are specific data types and not null depending on database functionality. +time: 2022-11-18T14:11:20.868062-08:00 +custom: + Author: sungchun12 + Issue: "6079" + PR: "6271" diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 407c5435786..786ecc8a496 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -446,6 +446,7 @@ class NodeConfig(NodeAndTestConfig): default_factory=Docs, metadata=MergeBehavior.Update.meta(), ) + constraints_enabled: Optional[bool] = False # we validate that node_color has a suitable value to prevent dbt-docs from crashing def __post_init__(self): diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 9516ebaccf6..f4bb533ebea 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -61,6 +61,7 @@ SnapshotConfig, ) + # ===================================================================== # This contains the classes for all of the nodes and node-like objects # in the manifest. In the "nodes" dictionary of the manifest we find @@ -146,6 +147,8 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable description: str = "" meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None + constraints: Optional[List[str]] = None + constraints_check: Optional[str] = None quote: Optional[bool] = None tags: List[str] = field(default_factory=list) _extra: Dict[str, Any] = field(default_factory=dict) @@ -400,6 +403,7 @@ class CompiledNode(ParsedNode): extra_ctes_injected: bool = False extra_ctes: List[InjectedCTE] = field(default_factory=list) _pre_injected_sql: Optional[str] = None + constraints_enabled: bool = False @property def empty(self): diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 3fd561d2394..271fb4f4508 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -93,6 +93,8 @@ class HasDocs(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable): description: str = "" meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None + constraints: Optional[List[str]] = None + constraints_check: Optional[str] = None docs: Docs = field(default_factory=Docs) _extra: Dict[str, Any] = field(default_factory=dict) diff --git a/core/dbt/docs/build/doctrees/environment.pickle b/core/dbt/docs/build/doctrees/environment.pickle index 24041fc88a2..37a21654aec 100644 Binary files a/core/dbt/docs/build/doctrees/environment.pickle and b/core/dbt/docs/build/doctrees/environment.pickle differ diff --git a/core/dbt/docs/build/doctrees/index.doctree b/core/dbt/docs/build/doctrees/index.doctree index a31c50cfc63..b6df7c64b8f 100644 Binary files a/core/dbt/docs/build/doctrees/index.doctree and b/core/dbt/docs/build/doctrees/index.doctree differ diff --git a/core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql b/core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql new file mode 100644 index 00000000000..7eea90a5fd9 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/table/columns_spec_ddl.sql @@ -0,0 +1,51 @@ +{%- macro get_columns_spec_ddl() -%} + {{ adapter.dispatch('get_columns_spec_ddl', 'dbt')() }} +{%- endmacro -%} + +{% macro default__get_columns_spec_ddl() -%} + {{ return(columns_spec_ddl()) }} +{%- endmacro %} + +{% macro columns_spec_ddl() %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set user_provided_columns = model['columns'] -%} + ( + {% for i in user_provided_columns %} + {% set col = user_provided_columns[i] %} + {% set constraints = col['constraints'] %} + {% set constraints_check = col['constraints_check'] %} + {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {% if constraints_check -%} check {{ constraints_check or "" }} {%- endif %} {{ "," if not loop.last }} + {% endfor %} + ) +{% endmacro %} + +{%- macro get_assert_columns_equivalent(sql) -%} + {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }} +{%- endmacro -%} + +{% macro default__get_assert_columns_equivalent(sql) -%} + {{ return(assert_columns_equivalent(sql)) }} +{%- endmacro %} + +{% macro assert_columns_equivalent(sql) %} + {#- loop through user_provided_columns to get column names -#} + {%- set user_provided_columns = model['columns'] -%} + {%- set column_names_config_only = [] -%} + {%- for i in user_provided_columns -%} + {%- set col = user_provided_columns[i] -%} + {%- set col_name = col['name'] -%} + {%- set column_names_config_only = column_names_config_only.append(col_name) -%} + {%- endfor -%} + {%- set sql_file_provided_columns = get_columns_in_query(sql) -%} + + {#- uppercase both schema and sql file columns -#} + {%- set column_names_config_upper= column_names_config_only|map('upper')|join(',') -%} + {%- set column_names_config_formatted = column_names_config_upper.split(',') -%} + {%- set sql_file_provided_columns_upper = sql_file_provided_columns|map('upper')|join(',') -%} + {%- set sql_file_provided_columns_formatted = sql_file_provided_columns_upper.split(',') -%} + + {%- if column_names_config_formatted != sql_file_provided_columns_formatted -%} + {%- do exceptions.raise_compiler_error('Please ensure the name, order, and number of columns in your `yml` file match the columns in your SQL file.\nSchema File Columns: ' ~ column_names_config_formatted ~ '\nSQL File Columns: ' ~ sql_file_provided_columns_formatted ~ ' ' ) %} + {%- endif -%} + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql b/core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql index 7f463edf0ae..04b9bd6fcc9 100644 --- a/core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql +++ b/core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql @@ -25,6 +25,10 @@ create {% if temporary: -%}temporary{%- endif %} table {{ relation.include(database=(not temporary), schema=(not temporary)) }} + {% if config.get('constraints_enabled', False) %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_columns_spec_ddl() }} + {% endif %} as ( {{ sql }} ); diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 1f01aff36f1..4d295fac134 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -18,7 +18,7 @@ from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ManifestNode, BaseNode from dbt.contracts.graph.unparsed import UnparsedNode, Docs -from dbt.exceptions import DbtInternalError, ConfigUpdateError, DictParseError +from dbt.exceptions import DbtInternalError, ConfigUpdateError, DictParseError, ParsingError from dbt import hooks from dbt.node_types import NodeType, ModelLanguage from dbt.parser.search import FileBlock @@ -306,6 +306,19 @@ def update_parsed_node_config( else: parsed_node.docs = Docs(show=docs_show) + # If we have constraints_enabled in the config, copy to node level, for backwards + # compatibility with earlier node-only config. + if config_dict.get("constraints_enabled", False): + parsed_node.constraints_enabled = True + + parser_name = type(self).__name__ + if parser_name == "ModelParser": + original_file_path = parsed_node.original_file_path + error_message = "\n `constraints_enabled=true` can only be configured within `schema.yml` files\n NOT within a model file(ex: .sql, .py) or `dbt_project.yml`." + raise ParsingError( + f"Original File Path: ({original_file_path})\nConstraints must be defined in a `yml` schema configuration file like `schema.yml`.\nOnly the SQL table materialization is supported for constraints. \n`data_type` values must be defined for all columns and NOT be null or blank.{error_message}" + ) + # unrendered_config is used to compare the original database/schema/alias # values and to handle 'same_config' and 'same_contents' calls parsed_node.unrendered_config = config.build_config_dict( diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 482eb5b6e35..05da989709e 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -119,6 +119,8 @@ def add( column: Union[HasDocs, UnparsedColumn], description: str, data_type: Optional[str], + constraints: Optional[List[str]], + constraints_check: Optional[str], meta: Dict[str, Any], ): tags: List[str] = [] @@ -132,6 +134,8 @@ def add( name=column.name, description=description, data_type=data_type, + constraints=constraints, + constraints_check=constraints_check, meta=meta, tags=tags, quote=quote, @@ -144,8 +148,10 @@ def from_target(cls, target: Union[HasColumnDocs, HasColumnTests]) -> "ParserRef for column in target.columns: description = column.description data_type = column.data_type + constraints = column.constraints + constraints_check = column.constraints_check meta = column.meta - refs.add(column, description, data_type, meta) + refs.add(column, description, data_type, constraints, constraints_check, meta) return refs @@ -914,6 +920,75 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: self.patch_node_config(node, patch) node.patch(patch) + self.validate_constraints(node) + + def validate_constraints(self, patched_node): + error_messages = [] + if ( + patched_node.resource_type == "model" + and patched_node.config.constraints_enabled is True + ): + validators = [ + self.constraints_schema_validator(patched_node), + self.constraints_materialization_validator(patched_node), + self.constraints_language_validator(patched_node), + self.constraints_data_type_validator(patched_node), + ] + error_messages = [validator for validator in validators if validator != "None"] + + if error_messages: + original_file_path = patched_node.original_file_path + raise ParsingError( + f"Original File Path: ({original_file_path})\nConstraints must be defined in a `yml` schema configuration file like `schema.yml`.\nOnly the SQL table materialization is supported for constraints. \n`data_type` values must be defined for all columns and NOT be null or blank.{self.convert_errors_to_string(error_messages)}" + ) + + def convert_errors_to_string(self, error_messages: List[str]): + n = len(error_messages) + if not n: + return "" + if n == 1: + return error_messages[0] + error_messages_string = "".join(error_messages[:-1]) + f"{error_messages[-1]}" + return error_messages_string + + def constraints_schema_validator(self, patched_node): + schema_error = False + if patched_node.columns == {}: + schema_error = True + schema_error_msg = "\n Schema Error: `yml` configuration does NOT exist" + schema_error_msg_payload = f"{schema_error_msg if schema_error else None}" + return schema_error_msg_payload + + def constraints_materialization_validator(self, patched_node): + materialization_error = {} + if patched_node.config.materialized != "table": + materialization_error = {"materialization": patched_node.config.materialized} + materialization_error_msg = f"\n Materialization Error: {materialization_error}" + materialization_error_msg_payload = ( + f"{materialization_error_msg if materialization_error else None}" + ) + return materialization_error_msg_payload + + def constraints_language_validator(self, patched_node): + language_error = {} + language = str(patched_node.language) + if language != "sql": + language_error = {"language": language} + language_error_msg = f"\n Language Error: {language_error}" + language_error_msg_payload = f"{language_error_msg if language_error else None}" + return language_error_msg_payload + + def constraints_data_type_validator(self, patched_node): + data_type_errors = set() + for column, column_info in patched_node.columns.items(): + if column_info.data_type is None: + data_type_error = {column} + data_type_errors.update(data_type_error) + data_type_errors_msg = ( + f"\n Columns with `data_type` Blank/Null Errors: {data_type_errors}" + ) + data_type_errors_msg_payload = f"{data_type_errors_msg if data_type_errors else None}" + return data_type_errors_msg_payload class TestablePatchParser(NodePatchParser[UnparsedNodeUpdate]): diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 3aca43ab010..7dbc1093f5f 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -9,7 +9,14 @@ {%- elif unlogged -%} unlogged {%- endif %} table {{ relation }} - as ( + {% if config.get('constraints_enabled', False) %} + {{ get_assert_columns_equivalent(sql) }} + {{ get_columns_spec_ddl() }} ; + insert into {{ relation }} {{ get_column_names() }} + {% else %} + as + {% endif %} + ( {{ sql }} ); {%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/utils/columns_spec_ddl.sql b/plugins/postgres/dbt/include/postgres/macros/utils/columns_spec_ddl.sql new file mode 100644 index 00000000000..72533dbff24 --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/macros/utils/columns_spec_ddl.sql @@ -0,0 +1,23 @@ +{% macro postgres__get_columns_spec_ddl() %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set user_provided_columns = model['columns'] -%} + ( + {% for i in user_provided_columns %} + {% set col = user_provided_columns[i] %} + {% set constraints = col['constraints'] %} + {% set constraints_check = col['constraints_check'] %} + {{ col['name'] }} {{ col['data_type'] }} {% for x in constraints %} {{ x or "" }} {% endfor %} {% if constraints_check -%} check {{ constraints_check or "" }} {%- endif %} {{ "," if not loop.last }} + {% endfor %} + ) +{% endmacro %} + +{% macro get_column_names() %} + {# loop through user_provided_columns to get column names #} + {%- set user_provided_columns = model['columns'] -%} + ( + {% for i in user_provided_columns %} + {% set col = user_provided_columns[i] %} + {{ col['name'] }} {{ "," if not loop.last }} + {% endfor %} + ) +{% endmacro %} diff --git a/schemas/dbt/manifest/v8.json b/schemas/dbt/manifest/v8.json index 4930109c277..bce96ddee14 100644 --- a/schemas/dbt/manifest/v8.json +++ b/schemas/dbt/manifest/v8.json @@ -126,6 +126,12 @@ }, { "$ref": "#/definitions/SourceDefinition" + }, + { + "$ref": "#/definitions/Exposure" + }, + { + "$ref": "#/definitions/Metric" } ] } @@ -173,7 +179,7 @@ } }, "additionalProperties": false, - "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], selectors: Mapping[str, Any], disabled: Optional[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition]]]], parent_map: Optional[Dict[str, List[str]]], child_map: Optional[Dict[str, List[str]]])", + "description": "WritableManifest(metadata: dbt.contracts.graph.manifest.ManifestMetadata, nodes: Mapping[str, Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode]], sources: Mapping[str, dbt.contracts.graph.nodes.SourceDefinition], macros: Mapping[str, dbt.contracts.graph.nodes.Macro], docs: Mapping[str, dbt.contracts.graph.nodes.Documentation], exposures: Mapping[str, dbt.contracts.graph.nodes.Exposure], metrics: Mapping[str, dbt.contracts.graph.nodes.Metric], selectors: Mapping[str, Any], disabled: Optional[Mapping[str, List[Union[dbt.contracts.graph.nodes.AnalysisNode, dbt.contracts.graph.nodes.SingularTestNode, dbt.contracts.graph.nodes.HookNode, dbt.contracts.graph.nodes.ModelNode, dbt.contracts.graph.nodes.RPCNode, dbt.contracts.graph.nodes.SqlNode, dbt.contracts.graph.nodes.GenericTestNode, dbt.contracts.graph.nodes.SnapshotNode, dbt.contracts.graph.nodes.SeedNode, dbt.contracts.graph.nodes.SourceDefinition, dbt.contracts.graph.nodes.Exposure, dbt.contracts.graph.nodes.Metric]]]], parent_map: Optional[Dict[str, List[str]]], child_map: Optional[Dict[str, List[str]]])", "definitions": { "ManifestMetadata": { "type": "object", @@ -190,7 +196,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2023-02-09T23:46:55.265899Z" + "default": "2023-02-14T21:54:19.556509Z" }, "invocation_id": { "oneOf": [ @@ -201,7 +207,7 @@ "type": "null" } ], - "default": "e6a9b266-203d-4fec-93af-fb8f55423a6b" + "default": "94c0abb0-a621-40a5-abe5-b7f2d26c3bd6" }, "env": { "type": "object", @@ -343,6 +349,7 @@ "show": true, "node_color": null }, + "constraints_enabled": false, "post-hook": [], "pre-hook": [] } @@ -406,7 +413,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.2675269 + "default": 1676411659.559374 }, "config_call_dict": { "type": "object", @@ -501,10 +508,14 @@ "$ref": "#/definitions/InjectedCTE" }, "default": [] + }, + "constraints_enabled": { + "type": "boolean", + "default": false } }, "additionalProperties": false, - "description": "AnalysisNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" + "description": "AnalysisNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False)" }, "FileHash": { "type": "object", @@ -673,10 +684,21 @@ "show": true, "node_color": null } + }, + "constraints_enabled": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false } }, "additionalProperties": true, - "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'view', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = )" + "description": "NodeConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'view', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , constraints_enabled: Optional[bool] = False)" }, "Hook": { "type": "object", @@ -754,6 +776,29 @@ } ] }, + "constraints": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "constraints_check": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, "quote": { "oneOf": [ { @@ -953,7 +998,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.2685802 + "default": 1676411659.561325 }, "config_call_dict": { "type": "object", @@ -1048,10 +1093,14 @@ "$ref": "#/definitions/InjectedCTE" }, "default": [] + }, + "constraints_enabled": { + "type": "boolean", + "default": false } }, "additionalProperties": false, - "description": "SingularTestNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" + "description": "SingularTestNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False)" }, "TestConfig": { "type": "object", @@ -1249,6 +1298,7 @@ "show": true, "node_color": null }, + "constraints_enabled": false, "post-hook": [], "pre-hook": [] } @@ -1312,7 +1362,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.269182 + "default": 1676411659.562434 }, "config_call_dict": { "type": "object", @@ -1408,6 +1458,10 @@ }, "default": [] }, + "constraints_enabled": { + "type": "boolean", + "default": false + }, "index": { "oneOf": [ { @@ -1420,7 +1474,7 @@ } }, "additionalProperties": false, - "description": "HookNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, index: Optional[int] = None)" + "description": "HookNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False, index: Optional[int] = None)" }, "ModelNode": { "type": "object", @@ -1506,6 +1560,7 @@ "show": true, "node_color": null }, + "constraints_enabled": false, "post-hook": [], "pre-hook": [] } @@ -1569,7 +1624,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.2698119 + "default": 1676411659.5636091 }, "config_call_dict": { "type": "object", @@ -1664,10 +1719,14 @@ "$ref": "#/definitions/InjectedCTE" }, "default": [] + }, + "constraints_enabled": { + "type": "boolean", + "default": false } }, "additionalProperties": false, - "description": "ModelNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" + "description": "ModelNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False)" }, "RPCNode": { "type": "object", @@ -1753,6 +1812,7 @@ "show": true, "node_color": null }, + "constraints_enabled": false, "post-hook": [], "pre-hook": [] } @@ -1816,7 +1876,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.2704 + "default": 1676411659.5646772 }, "config_call_dict": { "type": "object", @@ -1911,10 +1971,14 @@ "$ref": "#/definitions/InjectedCTE" }, "default": [] + }, + "constraints_enabled": { + "type": "boolean", + "default": false } }, "additionalProperties": false, - "description": "RPCNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" + "description": "RPCNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False)" }, "SqlNode": { "type": "object", @@ -2000,6 +2064,7 @@ "show": true, "node_color": null }, + "constraints_enabled": false, "post-hook": [], "pre-hook": [] } @@ -2063,7 +2128,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.270981 + "default": 1676411659.5657442 }, "config_call_dict": { "type": "object", @@ -2158,10 +2223,14 @@ "$ref": "#/definitions/InjectedCTE" }, "default": [] + }, + "constraints_enabled": { + "type": "boolean", + "default": false } }, "additionalProperties": false, - "description": "SqlNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" + "description": "SqlNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False)" }, "GenericTestNode": { "type": "object", @@ -2306,7 +2375,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.2716632 + "default": 1676411659.5669792 }, "config_call_dict": { "type": "object", @@ -2402,6 +2471,10 @@ }, "default": [] }, + "constraints_enabled": { + "type": "boolean", + "default": false + }, "column_name": { "oneOf": [ { @@ -2424,7 +2497,7 @@ } }, "additionalProperties": false, - "description": "GenericTestNode(test_metadata: dbt.contracts.graph.nodes.TestMetadata, database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, column_name: Optional[str] = None, file_key_name: Optional[str] = None)" + "description": "GenericTestNode(test_metadata: dbt.contracts.graph.nodes.TestMetadata, database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.TestConfig = , _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False, column_name: Optional[str] = None, file_key_name: Optional[str] = None)" }, "TestMetadata": { "type": "object", @@ -2577,7 +2650,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.272834 + "default": 1676411659.569141 }, "config_call_dict": { "type": "object", @@ -2672,10 +2745,14 @@ "$ref": "#/definitions/InjectedCTE" }, "default": [] + }, + "constraints_enabled": { + "type": "boolean", + "default": false } }, "additionalProperties": false, - "description": "SnapshotNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None)" + "description": "SnapshotNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.SnapshotConfig, _event_status: Dict[str, Any] = , tags: List[str] = , description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = , meta: Dict[str, Any] = , docs: dbt.contracts.graph.unparsed.Docs = , patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = , created_at: float = , config_call_dict: Dict[str, Any] = , relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , depends_on: dbt.contracts.graph.nodes.DependsOn = , compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = , _pre_injected_sql: Optional[str] = None, constraints_enabled: bool = False)" }, "SnapshotConfig": { "type": "object", @@ -2822,6 +2899,17 @@ "node_color": null } }, + "constraints_enabled": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false + }, "strategy": { "oneOf": [ { @@ -2880,7 +2968,7 @@ } }, "additionalProperties": true, - "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'snapshot', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Optional[str] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , strategy: Optional[str] = None, target_schema: Optional[str] = None, target_database: Optional[str] = None, updated_at: Optional[str] = None, check_cols: Union[str, List[str], NoneType] = None)" + "description": "SnapshotConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'snapshot', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Optional[str] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , constraints_enabled: Optional[bool] = False, strategy: Optional[str] = None, target_schema: Optional[str] = None, target_database: Optional[str] = None, updated_at: Optional[str] = None, check_cols: Union[str, List[str], NoneType] = None)" }, "SeedNode": { "type": "object", @@ -2966,6 +3054,7 @@ "show": true, "node_color": null }, + "constraints_enabled": false, "quote_columns": null, "post-hook": [], "pre-hook": [] @@ -3030,7 +3119,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.27388 + "default": 1676411659.571127 }, "config_call_dict": { "type": "object", @@ -3221,6 +3310,17 @@ "node_color": null } }, + "constraints_enabled": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false + }, "quote_columns": { "oneOf": [ { @@ -3233,7 +3333,7 @@ } }, "additionalProperties": true, - "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'seed', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , quote_columns: Optional[bool] = None)" + "description": "SeedConfig(_extra: Dict[str, Any] = , enabled: bool = True, alias: Optional[str] = None, schema: Optional[str] = None, database: Optional[str] = None, tags: Union[List[str], str] = , meta: Dict[str, Any] = , materialized: str = 'seed', incremental_strategy: Optional[str] = None, persist_docs: Dict[str, Any] = , post_hook: List[dbt.contracts.graph.model_config.Hook] = , pre_hook: List[dbt.contracts.graph.model_config.Hook] = , quoting: Dict[str, Any] = , column_types: Dict[str, Any] = , full_refresh: Optional[bool] = None, unique_key: Union[str, List[str], NoneType] = None, on_schema_change: Optional[str] = 'ignore', grants: Dict[str, Any] = , packages: List[str] = , docs: dbt.contracts.graph.unparsed.Docs = , constraints_enabled: Optional[bool] = False, quote_columns: Optional[bool] = None)" }, "MacroDependsOn": { "type": "object", @@ -3416,7 +3516,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.2750158 + "default": 1676411659.572991 } }, "additionalProperties": false, @@ -3531,7 +3631,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2023-02-09T23:46:55.263337Z" + "default": "2023-02-14T21:54:19.552782Z" }, "invocation_id": { "oneOf": [ @@ -3542,7 +3642,7 @@ "type": "null" } ], - "default": "e6a9b266-203d-4fec-93af-fb8f55423a6b" + "default": "94c0abb0-a621-40a5-abe5-b7f2d26c3bd6" }, "env": { "type": "object", @@ -3895,7 +3995,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.275425 + "default": 1676411659.573476 }, "supported_languages": { "oneOf": [ @@ -4037,7 +4137,7 @@ ] }, "owner": { - "$ref": "#/definitions/ExposureOwner" + "$ref": "#/definitions/Owner" }, "description": { "type": "string", @@ -4138,20 +4238,25 @@ }, "created_at": { "type": "number", - "default": 1675986415.27617 + "default": 1676411659.574655 } }, "additionalProperties": false, - "description": "Exposure(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.ExposureOwner, description: str = '', label: Optional[str] = None, maturity: Optional[dbt.contracts.graph.unparsed.MaturityType] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.ExposureConfig = , unrendered_config: Dict[str, Any] = , url: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" + "description": "Exposure(name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], type: dbt.contracts.graph.unparsed.ExposureType, owner: dbt.contracts.graph.unparsed.Owner, description: str = '', label: Optional[str] = None, maturity: Optional[dbt.contracts.graph.unparsed.MaturityType] = None, meta: Dict[str, Any] = , tags: List[str] = , config: dbt.contracts.graph.model_config.ExposureConfig = , unrendered_config: Dict[str, Any] = , url: Optional[str] = None, depends_on: dbt.contracts.graph.nodes.DependsOn = , refs: List[List[str]] = , sources: List[List[str]] = , metrics: List[List[str]] = , created_at: float = )" }, - "ExposureOwner": { + "Owner": { "type": "object", - "required": [ - "email" - ], + "required": [], "properties": { "email": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] }, "name": { "oneOf": [ @@ -4164,8 +4269,8 @@ ] } }, - "additionalProperties": false, - "description": "ExposureOwner(email: str, name: Optional[str] = None)" + "additionalProperties": true, + "description": "Owner(_extra: Dict[str, Any] = , email: Optional[str] = None, name: Optional[str] = None)" }, "ExposureConfig": { "type": "object", @@ -4355,7 +4460,7 @@ }, "created_at": { "type": "number", - "default": 1675986415.2768772 + "default": 1676411659.575802 } }, "additionalProperties": false, diff --git a/test/unit/test_contracts_graph_compiled.py b/test/unit/test_contracts_graph_compiled.py index fe1e25d7925..c3f3348924c 100644 --- a/test/unit/test_contracts_graph_compiled.py +++ b/test/unit/test_contracts_graph_compiled.py @@ -75,6 +75,7 @@ def basic_compiled_model(): alias='bar', tags=[], config=NodeConfig(), + constraints_enabled=False, meta={}, compiled=True, extra_ctes=[InjectedCTE('whatever', 'select * from other')], @@ -192,10 +193,12 @@ def basic_compiled_dict(): 'meta': {}, 'grants': {}, 'packages': [], + 'constraints_enabled': False, 'docs': {'show': True}, }, 'docs': {'show': True}, 'columns': {}, + 'constraints_enabled': False, 'meta': {}, 'compiled': True, 'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}], @@ -400,6 +403,7 @@ def basic_compiled_schema_test_node(): alias='bar', tags=[], config=TestConfig(severity='warn'), + constraints_enabled=False, meta={}, compiled=True, extra_ctes=[InjectedCTE('whatever', 'select * from other')], @@ -500,6 +504,7 @@ def basic_compiled_schema_test_dict(): }, 'docs': {'show': True}, 'columns': {}, + 'constraints_enabled': False, 'meta': {}, 'compiled': True, 'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}], diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index 5d97a0567bf..f1b6514bccb 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -76,6 +76,7 @@ def populated_node_config_dict(): 'grants': {}, 'packages': [], 'docs': {'show': True}, + 'constraints_enabled': False, } @@ -158,10 +159,12 @@ def base_parsed_model_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'deferred': False, 'docs': {'show': True}, + 'constraints_enabled': False, 'columns': {}, 'meta': {}, 'checksum': {'name': 'sha256', 'checksum': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'}, @@ -256,9 +259,11 @@ def complex_parsed_model_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'docs': {'show': True}, + 'constraints_enabled': False, 'columns': { 'a': { 'name': 'a', @@ -315,6 +320,10 @@ def complex_parsed_model_object(): ) +{'enabled': True, 'tags': [], 'meta': {}, 'materialized': 'ephemeral', 'persist_docs': {}, 'quoting': {}, 'column_types': {'a': 'text'}, 'on_schema_change': 'ignore', 'grants': {}, 'packages': [], 'docs': {'show': True}, 'constraints_enabled': False, 'post-hook': [{'sql': 'insert into blah(a, b) select "1", 1', 'transaction': True}], 'pre-hook': []} + +{'column_types': {'a': 'text'}, 'enabled': True, 'materialized': 'ephemeral', 'persist_docs': {}, 'post-hook': [{'sql': 'insert into blah(a, b) select "1", 1', 'transaction': True}], 'pre-hook': [], 'quoting': {}, 'tags': [], 'on_schema_change': 'ignore', 'meta': {}, 'grants': {}, 'docs': {'show': True}, 'packages': []} + def test_model_basic(basic_parsed_model_object, base_parsed_model_dict, minimal_parsed_model_dict): node = basic_parsed_model_object node_dict = base_parsed_model_dict @@ -450,6 +459,7 @@ def basic_parsed_seed_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'deferred': False, @@ -540,6 +550,7 @@ def complex_parsed_seed_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'deferred': False, @@ -796,9 +807,11 @@ def base_parsed_hook_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'docs': {'show': True}, + 'constraints_enabled': False, 'columns': {}, 'meta': {}, 'checksum': {'name': 'sha256', 'checksum': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'}, @@ -873,9 +886,11 @@ def complex_parsed_hook_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'docs': {'show': True}, + 'constraints_enabled': False, 'columns': { 'a': { 'name': 'a', @@ -1023,6 +1038,7 @@ def basic_parsed_schema_test_dict(): 'schema': 'dbt_test__audit', }, 'docs': {'show': True}, + 'constraints_enabled': False, 'columns': {}, 'test_metadata': { 'name': 'foo', @@ -1099,6 +1115,7 @@ def complex_parsed_schema_test_dict(): 'schema': 'dbt_test__audit', }, 'docs': {'show': False}, + 'constraints_enabled': False, 'columns': { 'a': { 'name': 'a', @@ -1219,6 +1236,7 @@ def basic_timestamp_snapshot_config_dict(): 'grants': {}, 'packages': [], 'docs': {'show': True}, + 'constraints_enabled': False, } @@ -1255,6 +1273,7 @@ def complex_timestamp_snapshot_config_dict(): 'grants': {}, 'packages': [], 'docs': {'show': True}, + 'constraints_enabled': False, } @@ -1315,6 +1334,7 @@ def basic_check_snapshot_config_dict(): 'grants': {}, 'packages': [], 'docs': {'show': True}, + 'constraints_enabled': False, } @@ -1351,6 +1371,7 @@ def complex_set_snapshot_config_dict(): 'grants': {}, 'packages': [], 'docs': {'show': True}, + 'constraints_enabled': False, } @@ -1460,9 +1481,11 @@ def basic_timestamp_snapshot_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'docs': {'show': True}, + 'constraints_enabled': False, 'columns': {}, 'meta': {}, 'checksum': {'name': 'sha256', 'checksum': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'}, @@ -1600,9 +1623,11 @@ def basic_check_snapshot_dict(): 'meta': {}, 'grants': {}, 'docs': {'show': True}, + 'constraints_enabled': False, 'packages': [], }, 'docs': {'show': True}, + 'constraints_enabled': False, 'columns': {}, 'meta': {}, 'checksum': {'name': 'sha256', 'checksum': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'}, diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 2b34e0ed5b1..cecb28c2759 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -48,7 +48,7 @@ 'depends_on', 'database', 'schema', 'name', 'resource_type', 'package_name', 'path', 'original_file_path', 'raw_code', 'language', 'description', 'columns', 'fqn', 'build_path', 'compiled_path', 'patch_path', 'docs', - 'deferred', 'checksum', 'unrendered_config', 'created_at', 'config_call_dict', 'relation_name', + 'deferred', 'checksum', 'unrendered_config', 'created_at', 'config_call_dict', 'relation_name', 'constraints_enabled' }) REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | { diff --git a/tests/adapter/dbt/tests/adapter/constraints/fixtures.py b/tests/adapter/dbt/tests/adapter/constraints/fixtures.py new file mode 100644 index 00000000000..46b9449ece6 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/constraints/fixtures.py @@ -0,0 +1,119 @@ +my_model_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +my_model_wrong_order_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 1 as color, + 'blue' as id, + cast('2019-01-01' as date) as date_day +""" + +my_model_wrong_name_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 1 as error, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +my_model_with_nulls_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + -- null value for 'id' + cast(null as {{ dbt.type_int() }}) as id, + -- change the color as well (to test rollback) + 'red' as color, + cast('2019-01-01' as date) as date_day +""" + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + constraints_enabled: true + columns: + - name: id + quote: true + data_type: integer + description: hello + constraints: ['not null','primary key'] + constraints_check: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: date + - name: my_model_error + config: + constraints_enabled: true + columns: + - name: id + data_type: integer + description: hello + constraints: ['not null','primary key'] + constraints_check: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: date + - name: my_model_wrong_order + config: + constraints_enabled: true + columns: + - name: id + data_type: integer + description: hello + constraints: ['not null','primary key'] + constraints_check: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: date + - name: my_model_wrong_name + config: + constraints_enabled: true + columns: + - name: id + data_type: integer + description: hello + constraints: ['not null','primary key'] + constraints_check: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: date +""" diff --git a/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py b/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py new file mode 100644 index 00000000000..780c92c5a74 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/constraints/test_constraints.py @@ -0,0 +1,180 @@ +import pytest +import re + +from dbt.tests.util import ( + run_dbt, + get_manifest, + run_dbt_and_capture, + write_file, + read_file, + relation_from_name, +) + +from dbt.tests.adapter.constraints.fixtures import ( + my_model_sql, + my_model_wrong_order_sql, + my_model_wrong_name_sql, + my_model_with_nulls_sql, + model_schema_yml, +) + + +class BaseConstraintsColumnsEqual: + """ + dbt should catch these mismatches during its "preflight" checks. + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + def test__constraints_wrong_column_order(self, project): + results, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_wrong_order"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_order" + my_model_config = manifest.nodes[model_id].config + constraints_enabled_actual_config = my_model_config.constraints_enabled + + assert constraints_enabled_actual_config is True + + expected_compile_error = "Please ensure the name, order, and number of columns in your `yml` file match the columns in your SQL file." + expected_schema_file_columns = "Schema File Columns: ['ID', 'COLOR', 'DATE_DAY']" + expected_sql_file_columns = "SQL File Columns: ['COLOR', 'ID', 'DATE_DAY']" + + assert expected_compile_error in log_output + assert expected_schema_file_columns in log_output + assert expected_sql_file_columns in log_output + + def test__constraints_wrong_column_names(self, project): + results, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_wrong_name"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_name" + my_model_config = manifest.nodes[model_id].config + constraints_enabled_actual_config = my_model_config.constraints_enabled + + assert constraints_enabled_actual_config is True + + expected_compile_error = "Please ensure the name, order, and number of columns in your `yml` file match the columns in your SQL file." + expected_schema_file_columns = "Schema File Columns: ['ID', 'COLOR', 'DATE_DAY']" + expected_sql_file_columns = "SQL File Columns: ['ERROR', 'COLOR', 'DATE_DAY']" + + assert expected_compile_error in log_output + assert expected_schema_file_columns in log_output + assert expected_sql_file_columns in log_output + + +# This is SUPER specific to Postgres, and will need replacing on other adapters +# TODO: make more generic +_expected_sql = """ +create table {0} ( + id integer not null primary key check (id > 0) , + color text , + date_day date +) ; +insert into {0} ( + id , + color , + date_day +) ( + select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +); +""" + + +class BaseConstraintsRuntimeEnforcement: + """ + These constraints pass muster for dbt's preflight checks. Make sure they're + passed into the DDL statement. If they don't match up with the underlying data, + the data platform should raise an error at runtime. + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self, project): + relation = relation_from_name(project.adapter, "my_model") + tmp_relation = relation.incorporate(path={"identifier": relation.identifier + "__dbt_tmp"}) + return _expected_sql.format(tmp_relation) + + @pytest.fixture(scope="class") + def expected_color(self): + return "blue" + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ['null value in column "id"', "violates not-null constraint"] + + def assert_expected_error_messages(self, error_message, expected_error_messages): + assert all(msg in error_message for msg in expected_error_messages) + + def test__constraints_ddl(self, project, expected_sql): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + # TODO: consider refactoring this to introspect logs instead + generated_sql = read_file("target", "run", "test", "models", "my_model.sql") + + generated_sql_check = re.sub(r"\s+", " ", generated_sql).lower().strip() + expected_sql_check = re.sub(r"\s+", " ", expected_sql).lower().strip() + assert ( + expected_sql_check == generated_sql_check + ), f""" +-- GENERATED SQL +{generated_sql} + +-- EXPECTED SQL +{expected_sql} +""" + + def test__constraints_enforcement_rollback( + self, project, expected_color, expected_error_messages + ): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + + # Make a contract-breaking change to the model + write_file(my_model_with_nulls_sql, "models", "my_model.sql") + + failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=False) + assert len(failing_results) == 1 + + # Verify the previous table still exists + relation = relation_from_name(project.adapter, "my_model") + old_model_exists_sql = f"select * from {relation}" + old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") + assert len(old_model_exists) == 1 + assert old_model_exists[0][1] == expected_color + + # Confirm this model was contracted + # TODO: is this step really necessary? + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + constraints_enabled_actual_config = my_model_config.constraints_enabled + assert constraints_enabled_actual_config is True + + # Its result includes the expected error messages + self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) + + +class TestConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + pass + + +class TestConstraintsRuntimeEnforcement(BaseConstraintsRuntimeEnforcement): + pass diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index 0ce826dd378..fd2b51248da 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -34,6 +34,7 @@ def get_rendered_model_config(**updates): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, } result.update(updates) return result @@ -65,6 +66,7 @@ def get_rendered_seed_config(**updates): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, } result.update(updates) return result @@ -102,6 +104,7 @@ def get_rendered_snapshot_config(**updates): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, } result.update(updates) return result @@ -274,6 +277,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "first_name": { "name": "first_name", @@ -282,6 +287,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "email": { "name": "email", @@ -290,6 +297,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "ip_address": { "name": "ip_address", @@ -298,6 +307,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "updated_at": { "name": "updated_at", @@ -306,8 +317,11 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, }, + "constraints_enabled": False, "patch_path": "test://" + model_schema_yml_path, "docs": {"node_color": None, "show": False}, "compiled": True, @@ -355,6 +369,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "first_name": { "name": "first_name", @@ -363,6 +379,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "email": { "name": "email", @@ -371,6 +389,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "ip_address": { "name": "ip_address", @@ -379,6 +399,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "updated_at": { "name": "updated_at", @@ -387,8 +409,11 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, }, + "constraints_enabled": False, "patch_path": "test://" + model_schema_yml_path, "docs": {"node_color": None, "show": False}, "compiled": True, @@ -428,6 +453,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "first_name": { "name": "first_name", @@ -436,6 +463,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "email": { "name": "email", @@ -444,6 +473,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "ip_address": { "name": "ip_address", @@ -452,6 +483,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, "updated_at": { "name": "updated_at", @@ -460,6 +493,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, }, }, "docs": {"node_color": None, "show": True}, @@ -519,6 +554,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "constraints_enabled": False, }, "snapshot.test.snapshot_seed": { "alias": "snapshot_seed", @@ -530,6 +566,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "compiled": True, "compiled_code": ANY, "config": snapshot_config, + "constraints_enabled": False, "database": project.database, "deferred": False, "depends_on": { @@ -575,6 +612,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "column_name": None, "columns": {}, "config": test_config, + "constraints_enabled": False, "sources": [], "depends_on": { "macros": ["macro.test.test_nothing", "macro.dbt.get_where_subquery"], @@ -625,6 +663,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "column_name": "id", "columns": {}, "config": test_config, + "constraints_enabled": False, "sources": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], @@ -678,6 +717,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "quote": None, "tags": [], + "constraints": None, + "constraints_check": None, } }, "config": { @@ -883,6 +924,7 @@ def expected_references_manifest(project): "unique_id": "model.test.ephemeral_copy", "compiled": True, "compiled_code": ANY, + "constraints_enabled": False, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(ephemeral_copy_path), @@ -901,6 +943,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, "ct": { "description": "The number of instances of the first name", @@ -909,9 +953,12 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, }, "config": get_rendered_model_config(materialized="table"), + "constraints_enabled": False, "sources": [], "depends_on": {"macros": [], "nodes": ["model.test.ephemeral_copy"]}, "deferred": False, @@ -956,6 +1003,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, "ct": { "description": "The number of instances of the first name", @@ -964,9 +1013,12 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, }, "config": get_rendered_model_config(), + "constraints_enabled": False, "database": project.database, "depends_on": {"macros": [], "nodes": ["model.test.ephemeral_summary"]}, "deferred": False, @@ -1008,6 +1060,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, "first_name": { "name": "first_name", @@ -1016,6 +1070,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, "email": { "name": "email", @@ -1024,6 +1080,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, "ip_address": { "name": "ip_address", @@ -1032,6 +1090,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, "updated_at": { "name": "updated_at", @@ -1040,6 +1100,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, }, }, "config": get_rendered_seed_config(), @@ -1075,6 +1137,7 @@ def expected_references_manifest(project): "compiled": True, "compiled_code": ANY, "config": get_rendered_snapshot_config(target_schema=alternate_schema), + "constraints_enabled": False, "database": model_database, "deferred": False, "depends_on": {"macros": [], "nodes": ["seed.test.seed"]}, @@ -1116,6 +1179,8 @@ def expected_references_manifest(project): "meta": {}, "quote": None, "tags": [], + "constraints_check": None, + "constraints": None, } }, "config": { diff --git a/tests/functional/configs/test_constraint_configs.py b/tests/functional/configs/test_constraint_configs.py new file mode 100644 index 00000000000..8036f045879 --- /dev/null +++ b/tests/functional/configs/test_constraint_configs.py @@ -0,0 +1,269 @@ +import pytest +from dbt.exceptions import ParsingError +from dbt.tests.util import run_dbt, get_manifest + +my_model_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +my_model_constraints_enabled_sql = """ +{{ + config( + materialized = "table", + constraints_enabled = true + ) +}} + +select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +my_model_constraints_disabled_sql = """ +{{ + config( + materialized = "table", + constraints_enabled = false + ) +}} + +select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +my_view_model_sql = """ +{{ + config( + materialized = "view" + ) +}} + +select + 1 as id, + 'blue' as color, + cast('2019-01-01' as date) as date_day +""" + +my_model_python_error = """ +import holidays, s3fs + + +def model(dbt, _): + dbt.config( + materialized="table", + packages=["holidays", "s3fs"], # how to import python libraries in dbt's context + ) + df = dbt.ref("my_model") + df_describe = df.describe() # basic statistics profiling + return df_describe +""" + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + constraints_enabled: true + columns: + - name: id + quote: true + data_type: integer + description: hello + constraints: ['not null','primary key'] + constraints_check: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: date +""" + +model_schema_errors_yml = """ +version: 2 +models: + - name: my_model + config: + constraints_enabled: true + columns: + - name: id + data_type: integer + description: hello + constraints: ['not null','primary key'] + constraints_check: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + - name: python_model + config: + constraints_enabled: true + columns: + - name: id + data_type: integer + description: hello + constraints: ['not null','primary key'] + constraints_check: (id > 0) + tests: + - unique + - name: color + data_type: text + - name: date_day + data_type: date +""" + +model_schema_blank_yml = """ +version: 2 +models: + - name: my_model + config: + constraints_enabled: true +""" + + +class TestModelLevelConstraintsEnabledConfigs: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + def test__model_constraints_enabled_true(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_columns = manifest.nodes[model_id].columns + my_model_config = manifest.nodes[model_id].config + constraints_enabled_actual_config = my_model_config.constraints_enabled + + assert constraints_enabled_actual_config is True + + expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=['not null', 'primary key'], constraints_check='(id > 0)', quote=True, tags=[], _extra={}), 'color': ColumnInfo(name='color', description='', meta={}, data_type='text', constraints=None, constraints_check=None, quote=None, tags=[], _extra={}), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=None, constraints_check=None, quote=None, tags=[], _extra={})}" + + assert expected_columns == str(my_model_columns) + + +class TestProjectConstraintsEnabledConfigs: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "+constraints_enabled": True, + "subdirectory": { + "+constraints_enabled": False, + }, + } + } + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + } + + def test__project_error(self, project): + with pytest.raises(ParsingError) as err_info: + run_dbt(["parse"], expect_pass=False) + + error_message_expected = "NOT within a model file(ex: .sql, .py) or `dbt_project.yml`." + assert error_message_expected in str(err_info) + + +class TestModelConstraintsEnabledConfigs: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_constraints_enabled_sql, + } + + def test__model_error(self, project): + with pytest.raises(ParsingError) as err_info: + run_dbt(["parse"], expect_pass=False) + + error_message_expected = "NOT within a model file(ex: .sql, .py) or `dbt_project.yml`." + assert error_message_expected in str(err_info) + + +class TestModelLevelConstraintsDisabledConfigs: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_constraints_disabled_sql, + "constraints_schema.yml": model_schema_yml, + } + + def test__model_constraints_enabled_false(self, project): + + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + constraints_enabled_actual_config = my_model_config.constraints_enabled + + assert constraints_enabled_actual_config is False + + +class TestModelLevelConstraintsErrorMessages: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_view_model_sql, + "constraints_schema.yml": model_schema_errors_yml, + } + + def test__config_errors(self, project): + with pytest.raises(ParsingError) as err_info: + run_dbt(["parse"], expect_pass=False) + + expected_materialization_error = "Materialization Error: {'materialization': 'view'}" + expected_empty_data_type_error = "Columns with `data_type` Blank/Null Errors: {'date_day'}" + assert expected_materialization_error in str(err_info) + assert expected_empty_data_type_error in str(err_info) + + +class TestSchemaConstraintsEnabledConfigs: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_blank_yml, + } + + def test__schema_error(self, project): + with pytest.raises(ParsingError) as err_info: + run_dbt(["parse"], expect_pass=False) + + schema_error_expected = "Schema Error: `yml` configuration does NOT exist" + assert schema_error_expected in str(err_info) + + +class TestPythonModelLevelConstraintsErrorMessages: + @pytest.fixture(scope="class") + def models(self): + return { + "python_model.py": my_model_python_error, + "constraints_schema.yml": model_schema_errors_yml, + } + + def test__python_errors(self, project): + with pytest.raises(ParsingError) as err_info: + run_dbt(["parse"], expect_pass=False) + + expected_python_error = "Language Error: {'language': 'python'}" + assert expected_python_error in str(err_info) diff --git a/tests/functional/docs/test_good_docs_blocks.py b/tests/functional/docs/test_good_docs_blocks.py index 9fc9a7f0bb5..97be58d6e18 100644 --- a/tests/functional/docs/test_good_docs_blocks.py +++ b/tests/functional/docs/test_good_docs_blocks.py @@ -87,6 +87,8 @@ def test_valid_doc_ref(self, project): "name": "id", "description": "The user ID number", "data_type": None, + "constraints": None, + "constraints_check": None, "meta": {}, "quote": None, "tags": [], @@ -96,6 +98,8 @@ def test_valid_doc_ref(self, project): "name": "first_name", "description": "The user's first name", "data_type": None, + "constraints": None, + "constraints_check": None, "meta": {}, "quote": None, "tags": [], @@ -105,6 +109,8 @@ def test_valid_doc_ref(self, project): "name": "last_name", "description": "The user's last name", "data_type": None, + "constraints": None, + "constraints_check": None, "meta": {}, "quote": None, "tags": [], @@ -145,6 +151,8 @@ def test_alternative_docs_path(self, project): "name": "id", "description": "The user ID number with alternative text", "data_type": None, + "constraints": None, + "constraints_check": None, "meta": {}, "quote": None, "tags": [], @@ -154,6 +162,8 @@ def test_alternative_docs_path(self, project): "name": "first_name", "description": "The user's first name", "data_type": None, + "constraints": None, + "constraints_check": None, "meta": {}, "quote": None, "tags": [], @@ -163,6 +173,8 @@ def test_alternative_docs_path(self, project): "name": "last_name", "description": "The user's last name in this other file", "data_type": None, + "constraints": None, + "constraints_check": None, "meta": {}, "quote": None, "tags": [], diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index d4ecb111195..d57cb2e9187 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -94,6 +94,7 @@ def expect_snapshot_output(self, project): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, }, "unique_id": "snapshot.test.my_snapshot", "original_file_path": normalize("snapshots/snapshot.sql"), @@ -133,6 +134,7 @@ def expect_analyses_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, }, "unique_id": "analysis.test.a", "original_file_path": normalize("analyses/a.sql"), @@ -173,6 +175,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, }, "original_file_path": normalize("models/ephemeral.sql"), "unique_id": "model.test.ephemeral", @@ -207,6 +210,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": "delete+insert", "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, }, "original_file_path": normalize("models/incremental.sql"), "unique_id": "model.test.incremental", @@ -238,6 +242,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, }, "original_file_path": normalize("models/sub/inner.sql"), "unique_id": "model.test.inner", @@ -269,6 +274,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, }, "original_file_path": normalize("models/outer.sql"), "unique_id": "model.test.outer", @@ -379,6 +385,7 @@ def expect_seed_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, + "constraints_enabled": False, }, "depends_on": {"macros": []}, "unique_id": "seed.test.seed",