Abstract manifest generation from tasks

.changes/unreleased/Under the Hood-20230110-115725.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Abstract manifest generation
+time: 2023-01-10T11:57:25.193965-06:00
+custom:
+  Author: stu-k
+  Issue: "6357"

core/dbt/cli/main.py

@@ -5,11 +5,11 @@
 
 import click
 from dbt.cli import requires, params as p
-from dbt.config import RuntimeConfig
 from dbt.config.project import Project
 from dbt.config.profile import Profile
 from dbt.contracts.graph.manifest import Manifest
 from dbt.task.clean import CleanTask
+from dbt.parser.manifest import write_manifest
 from dbt.task.compile import CompileTask
 from dbt.task.deps import DepsTask
 from dbt.task.run import RunTask
@@ -156,7 +156,6 @@ def docs(ctx, **kwargs):
 @p.exclude
 @p.profile
 @p.profiles_dir
-@p.project_dir
 @p.select
 @p.selector
 @p.state
@@ -208,11 +207,15 @@ def docs_serve(ctx, **kwargs):
 @p.vars
 @p.version_check
 @requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def compile(ctx, **kwargs):
     """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the
     target/ directory."""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = CompileTask(ctx.obj["flags"], config)
+    task = CompileTask(ctx.obj["flags"], ctx.obj["runtime_config"])
+    _set_manifest_on_task(task, ctx)
 
     results = task.run()
     success = task.interpret_results(results)
@@ -250,7 +253,6 @@ def debug(ctx, **kwargs):
 def deps(ctx, **kwargs):
     """Pull the most recent version of the dependencies listed in packages.yml"""
     task = DepsTask(ctx.obj["flags"], ctx.obj["project"])
-
     results = task.run()
     success = task.interpret_results(results)
     return results, success
@@ -309,9 +311,13 @@ def list(ctx, **kwargs):
 @p.version_check
 @p.write_manifest
 @requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest(write_perf_info=True)
 def parse(ctx, **kwargs):
     """Parses the project and provides information on performance"""
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {ctx.obj['flags']}")
+    _write_manifest(ctx)
     return None, True
 
 
@@ -336,10 +342,12 @@ def parse(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def run(ctx, **kwargs):
     """Compile SQL and execute against the current target database."""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = RunTask(ctx.obj["flags"], config)
+    task = RunTask(ctx.obj["flags"], ctx.obj["runtime_config"])
+    _set_manifest_on_task(task, ctx)
 
     results = task.run()
     success = task.interpret_results(results)
@@ -455,16 +463,30 @@ def freshness(ctx, **kwargs):
 @p.vars
 @p.version_check
 @requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def test(ctx, **kwargs):
     """Runs tests on data in deployed models. Run this after `dbt run`"""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = TestTask(ctx.obj["flags"], config)
+    task = TestTask(ctx.obj["flags"], ctx.obj["runtime_config"])
+    _set_manifest_on_task(task, ctx)
 
     results = task.run()
     success = task.interpret_results(results)
     return results, success
 
 
+def _set_manifest_on_task(task, ctx):
+    _write_manifest(ctx)
+    task.set_manifest(ctx.obj["manifest"])
+
+
+def _write_manifest(ctx):
+    if ctx.obj["flags"].write_json:
+        write_manifest(ctx.obj["manifest"], ctx.obj["runtime_config"].target_path)
+
+
 # Support running as a module
 if __name__ == "__main__":
     cli_runner()

core/dbt/cli/requires.py

@@ -1,8 +1,10 @@
-from dbt.adapters.factory import adapter_management
+from dbt.adapters.factory import adapter_management, register_adapter
 from dbt.cli.flags import Flags
+from dbt.config import RuntimeConfig
 from dbt.config.runtime import load_project, load_profile
 from dbt.events.functions import setup_event_logger
 from dbt.exceptions import DbtProjectError
+from dbt.parser.manifest import ManifestLoader
 from dbt.profiler import profiler
 from dbt.tracking import initialize_from_flags, track_run
 
@@ -85,3 +87,64 @@ def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
 
     return update_wrapper(wrapper, func)
+
+
+def runtime_config(func):
+    """A decorator used by click command functions for generating a runtime
+    config given a profile and project.
+    """
+
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        assert isinstance(ctx, Context)
+
+        req_strs = ["profile", "project"]
+        reqs = [ctx.obj.get(req_str) for req_str in req_strs]
+
+        if None in reqs:
+            raise DbtProjectError("profile and project required for runtime_config")
+
+        ctx.obj["runtime_config"] = RuntimeConfig.from_parts(
+            ctx.obj["project"],
+            ctx.obj["profile"],
+            ctx.obj["flags"],
+        )
+
+        return func(*args, **kwargs)
+
+    return update_wrapper(wrapper, func)
+
+
+def manifest(*args0, write_perf_info=False):
+    """A decorator used by click command functions for generating a manifest given a profile, project,
+    and runtime config. This also registers the adaper from the runtime config.
+    """
+
+    def outer_wrapper(func):
+        def wrapper(*args, **kwargs):
+            ctx = args[0]
+            assert isinstance(ctx, Context)
+
+            req_strs = ["profile", "project", "runtime_config"]
+            reqs = [ctx.obj.get(dep) for dep in req_strs]
+
+            if None in reqs:
+                raise DbtProjectError("profile, project, and runtime_config required for manifest")
+
+            runtime_config = ctx.obj["runtime_config"]
+            register_adapter(runtime_config)
+            manifest = ManifestLoader.get_full_manifest(
+                runtime_config, write_perf_info=write_perf_info
+            )
+
+            ctx.obj["manifest"] = manifest
+
+            return func(*args, **kwargs)
+
+        return update_wrapper(wrapper, func)
+
+    # if there are no args, the decorator was used without params @decorator
+    # otherwise, the decorator was called with params @decorator(arg)
+    if len(args0) == 0:
+        return outer_wrapper
+    return outer_wrapper(args0[0])

core/dbt/docs/build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: e27d6c1c419f2f0af393858cdf674109
+config: 0d25ef12a43286020bcd8b805064f01c
 tags: 645f666f9bcd5a90fca523b33c5a78b7

core/dbt/docs/build/html/_static/_sphinx_javascript_frameworks_compat.js

@@ -0,0 +1,134 @@
+/*
+ * _sphinx_javascript_frameworks_compat.js
+ * ~~~~~~~~~~
+ *
+ * Compatability shim for jQuery and underscores.js.
+ *
+ * WILL BE REMOVED IN Sphinx 6.0
+ * xref RemovedInSphinx60Warning
+ *
+ */
+
+/**
+ * select a different prefix for underscore
+ */
+$u = _.noConflict();
+
+
+/**
+ * small helper function to urldecode strings
+ *
+ * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#Decoding_query_parameters_from_a_URL
+ */
+jQuery.urldecode = function(x) {
+    if (!x) {
+        return x
+    }
+    return decodeURIComponent(x.replace(/\+/g, ' '));
+};
+
+/**
+ * small helper function to urlencode strings
+ */
+jQuery.urlencode = encodeURIComponent;
+
+/**
+ * This function returns the parsed url parameters of the
+ * current request. Multiple values per key are supported,
+ * it will always return arrays of strings for the value parts.
+ */
+jQuery.getQueryParameters = function(s) {
+    if (typeof s === 'undefined')
+        s = document.location.search;
+    var parts = s.substr(s.indexOf('?') + 1).split('&');
+    var result = {};
+    for (var i = 0; i < parts.length; i++) {
+        var tmp = parts[i].split('=', 2);
+        var key = jQuery.urldecode(tmp[0]);
+        var value = jQuery.urldecode(tmp[1]);
+        if (key in result)
+            result[key].push(value);
+        else
+            result[key] = [value];
+    }
+    return result;
+};
+
+/**
+ * highlight a given string on a jquery object by wrapping it in
+ * span elements with the given class name.
+ */
+jQuery.fn.highlightText = function(text, className) {
+    function highlight(node, addItems) {
+        if (node.nodeType === 3) {
+            var val = node.nodeValue;
+            var pos = val.toLowerCase().indexOf(text);
+            if (pos >= 0 &&
+                !jQuery(node.parentNode).hasClass(className) &&
+                !jQuery(node.parentNode).hasClass("nohighlight")) {
+                var span;
+                var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg");
+                if (isInSVG) {
+                    span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
+                } else {
+                    span = document.createElement("span");
+                    span.className = className;
+                }
+                span.appendChild(document.createTextNode(val.substr(pos, text.length)));
+                node.parentNode.insertBefore(span, node.parentNode.insertBefore(
+                    document.createTextNode(val.substr(pos + text.length)),
+                    node.nextSibling));
+                node.nodeValue = val.substr(0, pos);
+                if (isInSVG) {
+                    var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
+                    var bbox = node.parentElement.getBBox();
+                    rect.x.baseVal.value = bbox.x;
+                    rect.y.baseVal.value = bbox.y;
+                    rect.width.baseVal.value = bbox.width;
+                    rect.height.baseVal.value = bbox.height;
+                    rect.setAttribute('class', className);
+                    addItems.push({
+                        "parent": node.parentNode,
+                        "target": rect});
+                }
+            }
+        }
+        else if (!jQuery(node).is("button, select, textarea")) {
+            jQuery.each(node.childNodes, function() {
+                highlight(this, addItems);
+            });
+        }
+    }
+    var addItems = [];
+    var result = this.each(function() {
+        highlight(this, addItems);
+    });
+    for (var i = 0; i < addItems.length; ++i) {
+        jQuery(addItems[i].parent).before(addItems[i].target);
+    }
+    return result;
+};
+
+/*
+ * backward compatibility for jQuery.browser
+ * This will be supported until firefox bug is fixed.
+ */
+if (!jQuery.browser) {
+    jQuery.uaMatch = function(ua) {
+        ua = ua.toLowerCase();
+
+        var match = /(chrome)[ \/]([\w.]+)/.exec(ua) ||
+            /(webkit)[ \/]([\w.]+)/.exec(ua) ||
+            /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) ||
+            /(msie) ([\w.]+)/.exec(ua) ||
+            ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) ||
+            [];
+
+        return {
+            browser: match[ 1 ] || "",
+            version: match[ 2 ] || "0"
+        };
+    };
+    jQuery.browser = {};
+    jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true;
+}

core/dbt/docs/build/html/_static/basic.css

@@ -4,7 +4,7 @@
  *
  * Sphinx stylesheet -- basic theme.
  *
- * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */
@@ -324,15 +324,13 @@ aside.sidebar {
 p.sidebar-title {
     font-weight: bold;
 }
-
 nav.contents,
 aside.topic,
 div.admonition, div.topic, blockquote {
     clear: left;
 }
 
 /* -- topics ---------------------------------------------------------------- */
-
 nav.contents,
 aside.topic,
 div.topic {
@@ -608,7 +606,6 @@ ol.simple p,
 ul.simple p {
     margin-bottom: 0;
 }
-
 aside.footnote > span,
 div.citation > span {
     float: left;

core/dbt/docs/build/html/_static/doctools.js

@@ -4,7 +4,7 @@
  *
  * Base JavaScript utilities for all Sphinx HTML documentation.
  *
- * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */