Convert docs_generate_tests to new framework

.changes/unreleased/Under the Hood-20220413-183014.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: Convert 029_docs_generate tests to new framework
+time: 2022-04-13T18:30:14.706391-04:00
+custom:
+ Author: gshank
+ Issue: "5035"
+ PR: "5058"

core/dbt/tests/fixtures/project.py

@@ -253,16 +253,17 @@ def write_project_files(project_root, dir_name, file_dict):
 # Write files out from file_dict. Can be nested directories...
 def write_project_files_recursively(path, file_dict):
  if type(file_dict) is not dict:
- raise TestProcessingException(f"Error creating {path}. Did you forget the file extension?")
+ raise TestProcessingException(f"File dict is not a dict: '{file_dict}' for path '{path}'")
+ suffix_list = [".sql", ".csv", ".md", ".txt"]
  for name, value in file_dict.items():
- if name.endswith(".sql") or name.endswith(".csv") or name.endswith(".md"):
- write_file(value, path, name)
- elif name.endswith(".yml") or name.endswith(".yaml"):
+ if name.endswith(".yml") or name.endswith(".yaml"):
  if isinstance(value, str):
  data = value
  else:
  data = yaml.safe_dump(value)
  write_file(data, path, name)
+ elif name.endswith(tuple(suffix_list)):
+ write_file(value, path, name)
  else:
  write_project_files_recursively(path.mkdir(name), value)
 
@@ -356,6 +357,7 @@ def __init__(
  self.test_schema = test_schema
  self.database = database
  self.test_config = test_config
+ self.created_schemas = []
 
  @property
  def adapter(self):
@@ -377,20 +379,21 @@ def run_sql(self, sql, fetch=None):
 
  # Create the unique test schema. Used in test setup, so that we're
  # ready for initial sql prior to a run_dbt command.
- def create_test_schema(self):
+ def create_test_schema(self, schema_name=None):
+ if schema_name is None:
+ schema_name = self.test_schema
  with get_connection(self.adapter):
- relation = self.adapter.Relation.create(
- database=self.database, schema=self.test_schema
- )
+ relation = self.adapter.Relation.create(database=self.database, schema=schema_name)
  self.adapter.create_schema(relation)
+ self.created_schemas.append(schema_name)
 
  # Drop the unique test schema, usually called in test cleanup
  def drop_test_schema(self):
  with get_connection(self.adapter):
- relation = self.adapter.Relation.create(
- database=self.database, schema=self.test_schema
- )
- self.adapter.drop_schema(relation)
+ for schema_name in self.created_schemas:
+ relation = self.adapter.Relation.create(database=self.database, schema=schema_name)
+  self.adapter.drop_schema(relation)
+ self.created_schemas = []
 
  # This return a dictionary of table names to 'view' or 'table' values.
  def get_tables_in_schema(self):

core/dbt/tests/util.py

@@ -3,6 +3,7 @@
 import yaml
 import json
 import warnings
+from datetime import datetime
 from typing import List
 from contextlib import contextmanager
 
@@ -104,9 +105,9 @@ def copy_file(src_path, src, dest_path, dest) -> None:
 
 
 # Used in tests when you want to remove a file from the project directory
-def rm_file(src_path, src) -> None:
+def rm_file(*paths) -> None:
  # remove files from proj_path
- os.remove(os.path.join(src_path, src))
+ os.remove(os.path.join(*paths))
 
 
 # Used in tests to write out the string contents of a file to a
@@ -167,6 +168,16 @@ def check_result_nodes_by_unique_id(results, unique_ids):
  assert set(unique_ids) == set(result_unique_ids)
 
 
+# Check datetime is between start and end/now
+def check_datetime_between(timestr, start, end=None):
+ datefmt = "%Y-%m-%dT%H:%M:%S.%fZ"
+ if end is None:
+ end = datetime.utcnow()
+ parsed = datetime.strptime(timestr, datefmt)
+ assert start <= parsed
+ assert end >= parsed
+
+
 class TestProcessingException(Exception):
  pass
 

schemas/dbt/catalog/v1.json

@@ -0,0 +1,249 @@
+{
+ "type": "object",
+ "required": [
+ "metadata",
+ "nodes",
+ "sources"
+ ],
+ "properties": {
+ "metadata": {
+ "$ref": "#/definitions/CatalogMetadata"
+ },
+ "nodes": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/CatalogTable"
+ }
+ },
+ "sources": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/CatalogTable"
+ }
+ },
+ "errors": {
+ "oneOf": [
+ {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "description": "CatalogArtifact(metadata: dbt.contracts.results.CatalogMetadata, nodes: Dict[str, dbt.contracts.results.CatalogTable], sources: Dict[str, dbt.contracts.results.CatalogTable], errors: Union[List[str], NoneType] = None, _compile_results: Union[Any, NoneType] = None)",
+ "definitions": {
+ "CatalogMetadata": {
+ "type": "object",
+ "required": [],
+ "properties": {
+ "dbt_schema_version": {
+ "type": "string",
+ "default": "https://schemas.getdbt.com/dbt/catalog/v1.json"
+ },
+ "dbt_version": {
+ "type": "string",
+ "default": "1.2.0a1"
+ },
+ "generated_at": {
+ "type": "string",
+ "format": "date-time",
+ "default": "2022-04-14T19:46:57.674922Z"
+ },
+ "invocation_id": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": "16540468-9312-488f-9721-e3fb46751526"
+ },
+ "env": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ },
+ "default": {}
+ }
+ },
+ "additionalProperties": false,
+ "description": "CatalogMetadata(dbt_schema_version: str = <factory>, dbt_version: str = '1.2.0a1', generated_at: datetime.datetime = <factory>, invocation_id: Union[str, NoneType] = <factory>, env: Dict[str, str] = <factory>)"
+ },
+ "CatalogTable": {
+ "type": "object",
+ "required": [
+ "metadata",
+ "columns",
+ "stats"
+ ],
+ "properties": {
+ "metadata": {
+ "$ref": "#/definitions/TableMetadata"
+ },
+ "columns": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/ColumnMetadata"
+ }
+ },
+ "stats": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/StatsItem"
+ }
+ },
+ "unique_id": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "description": "CatalogTable(metadata: dbt.contracts.results.TableMetadata, columns: Dict[str, dbt.contracts.results.ColumnMetadata], stats: Dict[str, dbt.contracts.results.StatsItem], unique_id: Union[str, NoneType] = None)"
+ },
+ "TableMetadata": {
+ "type": "object",
+ "required": [
+ "type",
+ "schema",
+ "name"
+ ],
+ "properties": {
+ "type": {
+ "type": "string"
+ },
+ "schema": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "database": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "comment": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "owner": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "description": "TableMetadata(type: str, schema: str, name: str, database: Union[str, NoneType] = None, comment: Union[str, NoneType] = None, owner: Union[str, NoneType] = None)"
+ },
+ "ColumnMetadata": {
+ "type": "object",
+ "required": [
+ "type",
+ "index",
+ "name"
+ ],
+ "properties": {
+ "type": {
+ "type": "string"
+ },
+ "index": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "comment": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "description": "ColumnMetadata(type: str, index: int, name: str, comment: Union[str, NoneType] = None)"
+ },
+ "StatsItem": {
+ "type": "object",
+ "required": [
+ "id",
+ "label",
+ "include"
+ ],
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "label": {
+ "type": "string"
+ },
+ "value": {
+ "oneOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "string"
+ },
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "include": {
+ "type": "boolean"
+ },
+ "description": {
+ "oneOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ },
+ "additionalProperties": false,
+ "description": "StatsItem(id: str, label: str, value: Union[bool, str, float, NoneType], include: bool, description: Union[str, NoneType] = None)"
+ }
+ },
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://schemas.getdbt.com/dbt/catalog/v1.json"
+}