Improve error handling on invalid archive configs

Added a special archive-only node that has extra config restrictions add tests for invalid archive config
dbt-labs · Apr 9, 2019 · f1a5bce · f1a5bce
1 parent 41f4bfa
commit f1a5bce
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 1 deletion.
diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py
@@ -441,6 +441,44 @@ def config(self, value):
         self._contents['config'] = value
 
 
+ARCHIVE_CONFIG_CONTRACT = {
+    'properties': {
+        'target_database': {
+            'type': 'string',
+        },
+        'target_schema': {
+            'type': 'string',
+        },
+        'unique_key': {
+            'type': 'string',
+        },
+        'strategy': {
+            'enum': ['timestamp'],
+        },
+    },
+    'required': [
+        'target_database', 'target_schema', 'unique_key', 'strategy',
+    ],
+}
+
+
+PARSED_ARCHIVE_NODE_CONTRACT = deep_merge(
+    PARSED_NODE_CONTRACT,
+    {
+        'properties': {
+            'config': ARCHIVE_CONFIG_CONTRACT,
+            'resource_type': {
+                'enum': [NodeType.Archive],
+            },
+        },
+    }
+)
+
+
+class ParsedArchiveNode(ParsedNode):
+    SCHEMA = PARSED_ARCHIVE_NODE_CONTRACT
+
+
 # The parsed node update is only the 'patch', not the test. The test became a
 # regular parsed node. Note that description and columns must be present, but
 # may be empty.

diff --git a/core/dbt/parser/archives.py b/core/dbt/parser/archives.py
@@ -1,4 +1,5 @@
 from dbt.contracts.graph.unparsed import UnparsedNode
+from dbt.contracts.graph.parsed import ParsedArchiveNode
 from dbt.node_types import NodeType
 from dbt.parser.base import MacrosKnownParser
 from dbt.parser.base_sql import BaseSqlParser, SQLParseResult
@@ -119,7 +120,7 @@ def parse_archives_from_file(self, file_node, tags=None):
 
     @classmethod
     def get_compiled_path(cls, name, relative_path):
-        return os.path.join('archives', relative_path)
+        return relative_path
 
     @classmethod
     def get_fqn(cls, node, package_project_config, extra=[]):
@@ -131,6 +132,16 @@ def get_fqn(cls, node, package_project_config, extra=[]):
 
         return fqn
 
+    @staticmethod
+    def validate_archives(node):
+        if node.resource_type == NodeType.Archive:
+            try:
+                return ParsedArchiveNode(**node.to_shallow_dict())
+            except dbt.exceptions.JSONValidationException as exc:
+                raise dbt.exceptions.CompilationException(str(exc), node)
+        else:
+            return node
+
     def parse_sql_nodes(self, nodes, tags=None):
         if tags is None:
             tags = []
@@ -145,5 +156,10 @@ def parse_sql_nodes(self, nodes, tags=None):
             found = super(ArchiveBlockParser, self).parse_sql_nodes(
                 nodes=archive_nodes, tags=tags
             )
+            # make sure our blocks are going to work when we try to archive
+            # them!
+            found.parsed = {k: self.validate_archives(v) for
+                            k, v in found.parsed.items()}
+
             results.update(found)
         return results
diff --git a/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql b/test/integration/004_simple_archive_test/test-archives-invalid/archive.sql
@@ -0,0 +1,14 @@
+{% archive archive_sad %}
+
+    {# missing target_database #}
+    {{
+        config(
+            target_schema=schema,
+            unique_key='"id" || ' ~ "'-'" ~ ' || "first_name"',
+            strategy='timestamp',
+            updated_at='"updated_at"',
+        )
+    }}
+    select * from {{database}}.{{schema}}.seed
+
+{% endarchive %}
diff --git a/test/integration/004_simple_archive_test/test_simple_archive.py b/test/integration/004_simple_archive_test/test_simple_archive.py
@@ -1,5 +1,6 @@
 from nose.plugins.attrib import attr
 from test.integration.base import DBTIntegrationTest
+import dbt.exceptions
 
 
 class TestSimpleArchive(DBTIntegrationTest):
@@ -363,3 +364,26 @@ def project_config(self):
 
     def run_archive(self):
         return self.run_dbt(['archive', '--vars', '{{"target_database": {}}}'.format(self.alternative_database)])
+
+
+class TestBadArchive(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "simple_archive_004"
+
+    @property
+    def models(self):
+        return "test/integration/004_simple_archive_test/models"
+
+    @property
+    def project_config(self):
+        return {
+            "archive-paths": ['test/integration/004_simple_archive_test/test-archives-invalid'],
+        }
+
+    @attr(type='postgres')
+    def test__postgres__invalid(self):
+        with self.assertRaises(dbt.exceptions.CompilationException) as exc:
+            self.run_dbt(['compile'], expect_pass=False)
+
+        self.assertIn('target_database', str(exc.exception))