Abstract manifest generation from tasks

.changes/unreleased/Under the Hood-20230110-115725.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Abstract manifest generation
+time: 2023-01-10T11:57:25.193965-06:00
+custom:
+  Author: stu-k
+  Issue: "6357"

core/dbt/cli/main.py

@@ -2,6 +2,7 @@
 from copy import copy
 from pprint import pformat as pf  # This is temporary for RAT-ing
 from typing import List, Tuple, Optional
+import os
 
 import click
 from dbt.cli import requires, params as p
@@ -10,6 +11,7 @@
 from dbt.config.profile import Profile
 from dbt.contracts.graph.manifest import Manifest
 from dbt.task.clean import CleanTask
+from dbt.parser.manifest import ManifestLoader, write_manifest
 from dbt.task.deps import DepsTask
 from dbt.task.run import RunTask
 
@@ -242,7 +244,7 @@ def debug(ctx, **kwargs):
 def deps(ctx, **kwargs):
     """Pull the most recent version of the dependencies listed in packages.yml"""
     task = DepsTask(ctx.obj["flags"], ctx.obj["project"])
-
+    _set_manifest_on_task(task, ctx)
     results = task.run()
     success = task.interpret_results(results)
     return results, success
@@ -301,9 +303,18 @@ def list(ctx, **kwargs):
 @p.version_check
 @p.write_manifest
 @requires.preflight
+@requires.profile
+@requires.project
 def parse(ctx, **kwargs):
     """Parses the project and provides information on performance"""
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {ctx.obj['flags']}")
+    flags = ctx.obj["flags"]
+    config = RuntimeConfig.from_parts(
+        ctx.obj["project"],
+        ctx.obj["profile"],
+        flags,
+    )
+    manifest = ManifestLoader.get_full_manifest(config, write_perf_info=True)
+    _write_manifest(manifest, flags)
     return None, True
 
 
@@ -330,8 +341,13 @@ def parse(ctx, **kwargs):
 @requires.project
 def run(ctx, **kwargs):
     """Compile SQL and execute against the current target database."""
-    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
+    config = RuntimeConfig.from_parts(
+        ctx.obj["project"],
+        ctx.obj["profile"],
+        ctx.obj["flags"],
+    )
     task = RunTask(ctx.obj["flags"], config)
+    _set_manifest_on_task(task, ctx)
 
     results = task.run()
     success = task.interpret_results(results)
@@ -453,6 +469,30 @@ def test(ctx, **kwargs):
     return None, True
 
 
+def _get_config_from_ctx(ctx):
+    return RuntimeConfig.from_parts(
+        ctx.obj["project"],
+        ctx.obj["profile"],
+        ctx.obj["flags"],
+    )
+
+
+def _set_manifest_on_task(task, ctx, write_perf_info=False):
+    config = _get_config_from_ctx(ctx)
+    manifest = ManifestLoader.get_full_manifest(config, write_perf_info=write_perf_info)
+    flags = ctx.obj["flags"]
+    _write_manifest(manifest, flags)
+    task.set_manifest(manifest)
+
+
+def _write_manifest(manifest, flags):
+    write_manifest(
+        manifest,
+        write_files=os.getenv("DBT_WRITE_FILES"),
+        write_json=flags.write_json,
+    )
+
+
 # Support running as a module
 if __name__ == "__main__":
     cli_runner()

core/dbt/docs/build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 1ee31fc16e025fb98598189ba2cb5fcb
+config: e27d6c1c419f2f0af393858cdf674109
 tags: 645f666f9bcd5a90fca523b33c5a78b7

core/dbt/docs/build/html/_sources/index.rst.txt

@@ -1,4 +1,36 @@
 dbt-core's API documentation
 ============================
+How to invoke dbt commands in python runtime
+--------------------------------------------
+
+Right now the best way to invoke a command from python runtime is to use the `dbtRunner` we exposed
+
+.. code-block:: python
+    from dbt.cli.main import dbtRunner
+    cli_args = ['run', '--project-dir', 'jaffle_shop']
+
+    # initialize the dbt runner
+    dbt = dbtRunner()
+    # run the command
+    res, success = dbt.invoke(args)
+
+You can also pass in pre constructed object into dbtRunner, and we will use those objects instead of loading up from the disk.
+
+.. code-block:: python
+
+    # preload profile and project
+    profile = load_profile(project_dir, {}, 'testing-postgres')
+    project = load_project(project_dir, False, profile, {})
+
+    # initialize the runner with pre-loaded profile and project
+    dbt = dbtRunner(profile=profile, project=project)
+    # run the command, this will use the pre-loaded profile and project instead of loading
+    res, success = dbt.invoke(cli_args)
+
+
+For the full example code, you can refer to `core/dbt/cli/example.py`
+
+API documentation
+-----------------
 
 .. dbt_click:: dbt.cli.main:cli

core/dbt/docs/build/html/_static/basic.css

@@ -4,7 +4,7 @@
  *
  * Sphinx stylesheet -- basic theme.
  *
- * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */
@@ -324,13 +324,15 @@ aside.sidebar {
 p.sidebar-title {
     font-weight: bold;
 }
+
 nav.contents,
 aside.topic,
 div.admonition, div.topic, blockquote {
     clear: left;
 }
 
 /* -- topics ---------------------------------------------------------------- */
+
 nav.contents,
 aside.topic,
 div.topic {
@@ -606,6 +608,7 @@ ol.simple p,
 ul.simple p {
     margin-bottom: 0;
 }
+
 aside.footnote > span,
 div.citation > span {
     float: left;

core/dbt/docs/build/html/_static/doctools.js

@@ -4,7 +4,7 @@
  *
  * Base JavaScript utilities for all Sphinx HTML documentation.
  *
- * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */