Updates DocumentParser to look for docs in .sql files

Since the .sql files have macros that aren't available to the docs
parser (e.g., `ref`), we can instead get the AST for the docs macros and
directly read out the contents without rendering them.

This assumes that when people write `docs` blocks, they don't contain to
any other macros or other Jinja tomfoolery. It's hard for me to tell if
that's a reasonable assumption or not.

Relevant to dbt-labs#979

dbt/clients/jinja.py

@@ -173,23 +173,25 @@ def __call__(self, *args, **kwargs):
     return ParserMacroCapture
 
 
-def get_template(string, ctx, node=None, capture_macros=False):
-    try:
-        args = {
-            'extensions': []
-        }
+def get_env(node=None, capture_macros=False):
+    args = {
+        'extensions': []
+    }
 
-        if capture_macros:
-            args['undefined'] = create_macro_capture_env(node)
+    if capture_macros:
+        args['undefined'] = create_macro_capture_env(node)
 
-        args['extensions'].append(MaterializationExtension)
-        args['extensions'].append(OperationExtension)
-        args['extensions'].append(DocumentationExtension)
+    args['extensions'].append(MaterializationExtension)
+    args['extensions'].append(OperationExtension)
+    args['extensions'].append(DocumentationExtension)
 
-        env = MacroFuzzEnvironment(**args)
+    return MacroFuzzEnvironment(**args)
 
-        return env.from_string(dbt.compat.to_string(string), globals=ctx)
 
+def get_template(string, ctx, node=None, capture_macros=False):
+    try:
+        env = get_env(node, capture_macros)
+        return env.from_string(dbt.compat.to_string(string), globals=ctx)
     except (jinja2.exceptions.TemplateSyntaxError,
             jinja2.exceptions.UndefinedError) as e:
         e.translated = False

dbt/parser/docs.py

@@ -14,12 +14,14 @@ def load_file(cls, package_name, root_dir, relative_dirs):
         """Load and parse documentation in a list of projects. Returns a list
         of ParsedNodes.
         """
-        extension = "[!.#~]*.md"
 
-        file_matches = dbt.clients.system.find_matching(
-            root_dir,
-            relative_dirs,
-            extension)
+        extensions = ["[!.#~]*.md", "[!.#~]*.sql"]
+
+        file_matches = []
+        for extension in extensions:
+            file_matches.extend(
+                dbt.clients.system.find_matching(
+                    root_dir, relative_dirs, extension))
 
         for file_match in file_matches:
             file_contents = dbt.clients.system.load_file_contents(
@@ -44,19 +46,11 @@ def load_file(cls, package_name, root_dir, relative_dirs):
 
     @classmethod
     def parse(cls, all_projects, root_project_config, docfile):
-        try:
-            template = dbt.clients.jinja.get_template(docfile.file_contents,
-                                                      {})
-        except dbt.exceptions.CompilationException as e:
-            e.node = docfile
-            raise
-
-        schema = getattr(root_project_config.credentials, 'schema', 'public')
-
-        for key, item in template.module.__dict__.items():
-            if type(item) != jinja2.runtime.Macro:
-                continue
+        env = dbt.clients.jinja.get_env()
+        ast = env.parse(docfile.file_contents)
 
+        for macro in ast.find_all(jinja2.nodes.Macro):
+            key = macro.name
             if not key.startswith(dbt.utils.DOCS_PREFIX):
                 continue
 
@@ -68,12 +62,19 @@ def parse(cls, all_projects, root_project_config, docfile):
             fqn = cls.get_fqn(docfile.path,
                               all_projects[docfile.package_name])
 
+            # HACK: Assume that docs macros do not call other macros and only
+            # have a single relevant Output node. Alternatively we could render
+            # this particular block.
+            if macro.body and macro.body[0].nodes:
+                contents = macro.body[0].nodes[0].data
+            else:
+                contents = ''
             merged = dbt.utils.deep_merge(
                 docfile.serialize(),
                 {
                     'name': name,
                     'unique_id': unique_id,
-                    'block_contents': item().strip(),
+                    'block_contents': contents,
                 }
             )
             yield ParsedDocumentation(**merged)