Merge pull request #2036 from fishtown-analytics/feature/toyaml-fromyaml

add toyaml/fromyaml (#1911)

core/dbt/context/base.py

@@ -14,7 +14,7 @@
 
 from dbt.node_types import NodeType
 
-
+import yaml
 # These modules are added to the context. Consider alternative
 # approaches which will extend well to potentially many modules
 import pytz
@@ -139,6 +139,21 @@ def tojson(value, default=None, sort_keys=False):
         return default
 
 
+def fromyaml(value, default=None):
+    try:
+        return yaml.safe_load(value)
+    except (AttributeError, ValueError, yaml.YAMLError):
+        return default
+
+
+# safe_dump defaults to sort_keys=True, but act like json.dumps (the opposite)
+def toyaml(value, default=None, sort_keys=False):
+    try:
+        return yaml.safe_dump(data=value, sort_keys=sort_keys)
+    except (ValueError, yaml.YAMLError):
+        return default
+
+
 def log(msg, info=False):
     if info:
         logger.info(msg)
@@ -164,6 +179,8 @@ def to_dict(self) -> Dict[str, Any]:
             'return': _return,
             'fromjson': fromjson,
             'tojson': tojson,
+            'fromyaml': fromyaml,
+            'toyaml': toyaml,
             'log': log,
         }
         if os.environ.get('DBT_MACRO_DEBUGGING'):

core/dbt/contracts/project.py

@@ -107,6 +107,7 @@ class RegistryPackageMetadata(
     'execute',
     'flags',
     'fromjson',
+    'fromyaml',
     'graph',
     'invocation_id',
     'load_agate_table',
@@ -128,6 +129,7 @@ class RegistryPackageMetadata(
     'target',
     'this',
     'tojson',
+    'toyaml',
     'try_or_compiler_error',
     'var',
     'write',

test/integration/013_context_var_tests/test_context_members.py

@@ -0,0 +1,19 @@
+from test.integration.base import DBTIntegrationTest, use_profile
+
+
+class TestContextVars(DBTIntegrationTest):
+    @property
+    def schema(self):
+        return "context_members_013"
+
+    @property
+    def models(self):
+        return "context-member-models"
+
+    @property
+    def project_config(self):
+        return {'test-paths': ['tests']}
+
+    @use_profile('postgres')
+    def test_json_data_tests_postgres(self):
+        self.assertEqual(len(self.run_dbt(['test'])), 2)

test/integration/013_context_var_tests/tests/from_yaml.sql

@@ -0,0 +1,14 @@
+{% set simplest = (fromyaml('a: 1') == {'a': 1}) %}
+{% set nested_data %}
+a:
+  b:
+   - c: 1
+     d: 2
+   - c: 3
+     d: 4
+{% endset %}
+{% set nested = (fromyaml(nested_data) == {'a': {'b': [{'c': 1, 'd': 2}, {'c': 3, 'd': 4}]}}) %}
+
+(select 'simplest' as name {% if simplest %}limit 0{% endif %})
+union all
+(select 'nested' as name {% if nested %}limit 0{% endif %})

test/integration/013_context_var_tests/tests/to_yaml.sql

@@ -0,0 +1,17 @@
+
+{% set simplest = (toyaml({'a': 1}) == 'a: 1\n') %}
+{% set default_sort = (toyaml({'b': 2, 'a': 1}) == 'b: 2\na: 1\n') %}
+{% set unsorted = (toyaml({'b': 2, 'a': 1}, sort_keys=False) == 'b: 2\na: 1\n') %}
+{% set sorted = (toyaml({'b': 2, 'a': 1}, sort_keys=True) == 'a: 1\nb: 2\n') %}
+{% set default_results = (toyaml({'a': adapter}, 'failed') == 'failed') %}
+
+(select 'simplest' as name {% if simplest %}limit 0{% endif %})
+union all
+(select 'default_sort' as name {% if default_sort %}limit 0{% endif %})
+union all
+(select 'unsorted' as name {% if unsorted %}limit 0{% endif %})
+union all
+(select 'sorted' as name {% if sorted %}limit 0{% endif %})
+union all
+(select 'default_results' as name {% if default_results %}limit 0{% endif %})
+