Abstract manifest generation from tasks (#6565)

.changes/unreleased/Under the Hood-20230110-115725.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Abstract manifest generation
+time: 2023-01-10T11:57:25.193965-06:00
+custom:
+  Author: stu-k
+  Issue: "6357"

core/dbt/cli/main.py

@@ -5,7 +5,6 @@
 
 import click
 from dbt.cli import requires, params as p
-from dbt.config import RuntimeConfig
 from dbt.config.project import Project
 from dbt.config.profile import Profile
 from dbt.contracts.graph.manifest import Manifest
@@ -120,10 +119,15 @@ def cli(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def build(ctx, **kwargs):
     """Run all Seeds, Models, Snapshots, and tests in DAG order"""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = BuildTask(ctx.obj["flags"], config)
+    task = BuildTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -166,7 +170,6 @@ def docs(ctx, **kwargs):
 @p.models
 @p.profile
 @p.profiles_dir
-@p.project_dir
 @p.select
 @p.selector
 @p.state
@@ -178,10 +181,11 @@ def docs(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def docs_generate(ctx, **kwargs):
     """Generate the documentation website for your project"""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = GenerateTask(ctx.obj["flags"], config)
+    task = GenerateTask(ctx.obj["flags"], ctx.obj["runtime_config"])
 
     results = task.run()
     success = task.interpret_results(results)
@@ -225,11 +229,18 @@ def docs_serve(ctx, **kwargs):
 @p.vars
 @p.version_check
 @requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def compile(ctx, **kwargs):
     """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the
     target/ directory."""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = CompileTask(ctx.obj["flags"], config)
+    task = CompileTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -267,7 +278,6 @@ def debug(ctx, **kwargs):
 def deps(ctx, **kwargs):
     """Pull the most recent version of the dependencies listed in packages.yml"""
     task = DepsTask(ctx.obj["flags"], ctx.obj["project"])
-
     results = task.run()
     success = task.interpret_results(results)
     return results, success
@@ -311,10 +321,15 @@ def init(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def list(ctx, **kwargs):
     """List the resources in your project"""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = ListTask(ctx.obj["flags"], config)
+    task = ListTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -341,9 +356,13 @@ def list(ctx, **kwargs):
 @p.version_check
 @p.write_manifest
 @requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest(write_perf_info=True)
 def parse(ctx, **kwargs):
     """Parses the project and provides information on performance"""
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {ctx.obj['flags']}")
+    # manifest generation and writing happens in @requires.manifest
     return None, True
 
 
@@ -369,10 +388,15 @@ def parse(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def run(ctx, **kwargs):
     """Compile SQL and execute against the current target database."""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = RunTask(ctx.obj["flags"], config)
+    task = RunTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -392,10 +416,15 @@ def run(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def run_operation(ctx, **kwargs):
     """Run the named macro with any supplied arguments."""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = RunOperationTask(ctx.obj["flags"], config)
+    task = RunOperationTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -423,10 +452,15 @@ def run_operation(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def seed(ctx, **kwargs):
     """Load data from csv files into your data warehouse."""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = SeedTask(ctx.obj["flags"], config)
+    task = SeedTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -451,10 +485,15 @@ def seed(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def snapshot(ctx, **kwargs):
     """Execute snapshots defined in your project"""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = SnapshotTask(ctx.obj["flags"], config)
+    task = SnapshotTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -486,10 +525,15 @@ def source(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def freshness(ctx, **kwargs):
     """check the current freshness of the project's sources"""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = FreshnessTask(ctx.obj["flags"], config)
+    task = FreshnessTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)
@@ -525,10 +569,15 @@ def freshness(ctx, **kwargs):
 @requires.preflight
 @requires.profile
 @requires.project
+@requires.runtime_config
+@requires.manifest
 def test(ctx, **kwargs):
     """Runs tests on data in deployed models. Run this after `dbt run`"""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
-    task = TestTask(ctx.obj["flags"], config)
+    task = TestTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
 
     results = task.run()
     success = task.interpret_results(results)

core/dbt/cli/requires.py

@@ -1,8 +1,10 @@
-from dbt.adapters.factory import adapter_management
+from dbt.adapters.factory import adapter_management, register_adapter
 from dbt.cli.flags import Flags
+from dbt.config import RuntimeConfig
 from dbt.config.runtime import load_project, load_profile
 from dbt.events.functions import setup_event_logger
 from dbt.exceptions import DbtProjectError
+from dbt.parser.manifest import ManifestLoader, write_manifest
 from dbt.profiler import profiler
 from dbt.tracking import initialize_from_flags, track_run
 
@@ -85,3 +87,70 @@ def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
 
     return update_wrapper(wrapper, func)
+
+
+def runtime_config(func):
+    """A decorator used by click command functions for generating a runtime
+    config given a profile and project.
+    """
+
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        assert isinstance(ctx, Context)
+
+        req_strs = ["profile", "project"]
+        reqs = [ctx.obj.get(req_str) for req_str in req_strs]
+
+        if None in reqs:
+            raise DbtProjectError("profile and project required for runtime_config")
+
+        ctx.obj["runtime_config"] = RuntimeConfig.from_parts(
+            ctx.obj["project"],
+            ctx.obj["profile"],
+            ctx.obj["flags"],
+        )
+
+        return func(*args, **kwargs)
+
+    return update_wrapper(wrapper, func)
+
+
+def manifest(*args0, write_perf_info=False):
+    """A decorator used by click command functions for generating a manifest
+    given a profile, project, and runtime config. This also registers the adaper
+    from the runtime config and writes the manifest to disc.
+    """
+
+    def outer_wrapper(func):
+        def wrapper(*args, **kwargs):
+            ctx = args[0]
+            assert isinstance(ctx, Context)
+
+            req_strs = ["profile", "project", "runtime_config"]
+            reqs = [ctx.obj.get(dep) for dep in req_strs]
+
+            if None in reqs:
+                raise DbtProjectError("profile, project, and runtime_config required for manifest")
+
+            runtime_config = ctx.obj["runtime_config"]
+            register_adapter(runtime_config)
+
+            # a manifest has already been set on the context, so don't overwrite it
+            if ctx.obj.get("manifest") is None:
+                manifest = ManifestLoader.get_full_manifest(
+                    runtime_config, write_perf_info=write_perf_info
+                )
+
+                ctx.obj["manifest"] = manifest
+                if ctx.obj["flags"].write_json:
+                    write_manifest(manifest, ctx.obj["runtime_config"].target_path)
+
+            return func(*args, **kwargs)
+
+        return update_wrapper(wrapper, func)
+
+    # if there are no args, the decorator was used without params @decorator
+    # otherwise, the decorator was called with params @decorator(arg)
+    if len(args0) == 0:
+        return outer_wrapper
+    return outer_wrapper(args0[0])

core/dbt/docs/build/html/_sources/index.rst.txt

@@ -6,6 +6,7 @@ How to invoke dbt commands in python runtime
 Right now the best way to invoke a command from python runtime is to use the `dbtRunner` we exposed
 
 .. code-block:: python
+
     from dbt.cli.main import dbtRunner
     cli_args = ['run', '--project-dir', 'jaffle_shop']
 

core/dbt/docs/build/html/index.html

@@ -35,6 +35,15 @@ <h1>dbt-core’s API documentation<a class="headerlink" href="#dbt-core-s-api-do
 <section id="how-to-invoke-dbt-commands-in-python-runtime">
 <h2>How to invoke dbt commands in python runtime<a class="headerlink" href="#how-to-invoke-dbt-commands-in-python-runtime" title="Permalink to this heading">¶</a></h2>
 <p>Right now the best way to invoke a command from python runtime is to use the <cite>dbtRunner</cite> we exposed</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">dbt.cli.main</span> <span class="kn">import</span> <span class="n">dbtRunner</span>
+<span class="n">cli_args</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;run&#39;</span><span class="p">,</span> <span class="s1">&#39;--project-dir&#39;</span><span class="p">,</span> <span class="s1">&#39;jaffle_shop&#39;</span><span class="p">]</span>
+
+<span class="c1"># initialize the dbt runner</span>
+<span class="n">dbt</span> <span class="o">=</span> <span class="n">dbtRunner</span><span class="p">()</span>
+<span class="c1"># run the command</span>
+<span class="n">res</span><span class="p">,</span> <span class="n">success</span> <span class="o">=</span> <span class="n">dbt</span><span class="o">.</span><span class="n">invoke</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
+</pre></div>
+</div>
 <p>You can also pass in pre constructed object into dbtRunner, and we will use those objects instead of loading up from the disk.</p>
 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># preload profile and project</span>
 <span class="n">profile</span> <span class="o">=</span> <span class="n">load_profile</span><span class="p">(</span><span class="n">project_dir</span><span class="p">,</span> <span class="p">{},</span> <span class="s1">&#39;testing-postgres&#39;</span><span class="p">)</span>
@@ -92,6 +101,11 @@ <h4>project_dir<a class="headerlink" href="#build|project_dir" title="Permalink
 <p>Type: path</p>
 <p>Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.</p>
 </section>
+<section id="build|resource_types">
+<h4>resource_types<a class="headerlink" href="#build|resource_types" title="Permalink to this heading">¶</a></h4>
+<p>Type: unknown</p>
+<p>TODO: No current help text</p>
+</section>
 <section id="build|select">
 <h4>select<a class="headerlink" href="#build|select" title="Permalink to this heading">¶</a></h4>
 <p>Type: unknown</p>
@@ -313,6 +327,10 @@ <h4>vars<a class="headerlink" href="#deps|vars" title="Permalink to this heading
 </section>
 <h3>Command: docs<a class="headerlink" href="#dbt-section" title="Permalink to this heading">¶</a></h3>
 <h3>Command: init<a class="headerlink" href="#dbt-section" title="Permalink to this heading">¶</a></h3>
+<section id="init|project_name">
+<h4>project_name<a class="headerlink" href="#init|project_name" title="Permalink to this heading">¶</a></h4>
+<p>Type: string</p>
+</section>
 <section id="init|profile">
 <h4>profile<a class="headerlink" href="#init|profile" title="Permalink to this heading">¶</a></h4>
 <p>Type: string</p>
@@ -618,6 +636,10 @@ <h4>version_check<a class="headerlink" href="#run|version_check" title="Permalin
 <p>Ensure dbt’s version matches the one specified in the dbt_project.yml file (‘require-dbt-version’)</p>
 </section>
 <h3>Command: run_operation<a class="headerlink" href="#dbt-section" title="Permalink to this heading">¶</a></h3>
+<section id="run-operation|macro">
+<h4>macro<a class="headerlink" href="#run-operation|macro" title="Permalink to this heading">¶</a></h4>
+<p>Type: string</p>
+</section>
 <section id="run-operation|args">
 <h4>args<a class="headerlink" href="#run-operation|args" title="Permalink to this heading">¶</a></h4>
 <p>Type: YAML</p>