From 9a827a95204350c5dfe456e2362d9147f5044736 Mon Sep 17 00:00:00 2001 From: pquadri Date: Fri, 17 Jun 2022 10:28:59 +0200 Subject: [PATCH 1/6] fix: add quotes to relation --- .../macros/materializations/snapshots/strategies.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 43779b05bf6..f5317e1e7df 100644 --- a/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -126,7 +126,7 @@ {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %} {% endif %} - {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%} + {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | map(adapter.quote) | list -%} {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#} {%- set ns.column_added = false -%} From a8ba96dbfaa460635cdb3821e4319949181e3265 Mon Sep 17 00:00:00 2001 From: pquadri Date: Fri, 17 Jun 2022 11:34:22 +0200 Subject: [PATCH 2/6] fix: add quotes correctly --- .../materializations/snapshots/strategies.sql | 4 +- core/dbt/main.py | 92 ++++++++++++------- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index f5317e1e7df..63fa1b52808 100644 --- a/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -126,14 +126,14 @@ {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %} {% endif %} - {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | map(adapter.quote) | list -%} + {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%} {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#} {%- set ns.column_added = false -%} {%- set intersection = [] -%} {%- for col in query_columns -%} {%- if col in existing_cols -%} - {%- do intersection.append(col) -%} + {%- do intersection.append(adapter.quote(col)) -%} {%- else -%} {% set ns.column_added = true %} {%- endif -%} diff --git a/core/dbt/main.py b/core/dbt/main.py index af30854fade..89483ee4c3c 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -1,6 +1,3 @@ -from typing import List -from dbt.logger import log_cache_events, log_manager - import argparse import os.path import sys @@ -8,17 +5,8 @@ import warnings from contextlib import contextmanager from pathlib import Path +from typing import List -import dbt.version -from dbt.events.functions import fire_event, setup_event_logger -from dbt.events.types import ( - MainEncounteredError, - MainKeyboardInterrupt, - MainReportVersion, - MainReportArgs, - MainTrackingUserState, - MainStackTrace, -) import dbt.flags as flags import dbt.task.build as build_task import dbt.task.clean as clean_task @@ -36,14 +24,27 @@ import dbt.task.serve as serve_task import dbt.task.snapshot as snapshot_task import dbt.task.test as test_task -from dbt.profiler import profiler -from dbt.adapters.factory import reset_adapters, cleanup_connections - import dbt.tracking - -from dbt.utils import ExitCodes, args_to_dict +import dbt.version +from dbt.adapters.factory import cleanup_connections, reset_adapters from dbt.config.profile import DEFAULT_PROFILES_DIR, read_user_config -from dbt.exceptions import InternalException, NotImplementedException, FailedToConnectException +from dbt.events.functions import fire_event, setup_event_logger +from dbt.events.types import ( + MainEncounteredError, + MainKeyboardInterrupt, + MainReportArgs, + MainReportVersion, + MainStackTrace, + MainTrackingUserState, +) +from dbt.exceptions import ( + FailedToConnectException, + InternalException, + NotImplementedException, +) +from dbt.logger import log_cache_events, log_manager +from dbt.profiler import profiler +from dbt.utils import ExitCodes, args_to_dict class DBTVersion(argparse.Action): @@ -60,7 +61,11 @@ def __init__( help="show program's version number and exit", ): super().__init__( - option_strings=option_strings, dest=dest, default=default, nargs=0, help=help + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help, ) def __call__(self, parser, namespace, values, option_string=None): @@ -121,6 +126,7 @@ def add_optional_argument_inverse( def main(args=None): # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. # This _only_ works for regular cli invocations. + print("ciao") warnings.filterwarnings("ignore", category=DeprecationWarning, module="logbook") if args is None: args = sys.argv[1:] @@ -199,12 +205,18 @@ def track_run(task): dbt.tracking.track_invocation_start(config=task.config, args=task.args) try: yield - dbt.tracking.track_invocation_end(config=task.config, args=task.args, result_type="ok") + dbt.tracking.track_invocation_end( + config=task.config, args=task.args, result_type="ok" + ) except (NotImplementedException, FailedToConnectException) as e: fire_event(MainEncounteredError(e=str(e))) - dbt.tracking.track_invocation_end(config=task.config, args=task.args, result_type="error") + dbt.tracking.track_invocation_end( + config=task.config, args=task.args, result_type="error" + ) except Exception: - dbt.tracking.track_invocation_end(config=task.config, args=task.args, result_type="error") + dbt.tracking.track_invocation_end( + config=task.config, args=task.args, result_type="error" + ) raise finally: dbt.tracking.flush() @@ -389,9 +401,9 @@ def _build_build_subparser(subparsers, base_subparser): """, ) - resource_values: List[str] = [str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES] + [ - "all" - ] + resource_values: List[str] = [ + str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES + ] + ["all"] sub.add_argument( "--resource-type", choices=resource_values, @@ -477,7 +489,9 @@ def _build_snapshot_subparser(subparsers, base_subparser): Overrides settings in profiles.yml. """, ) - sub.set_defaults(cls=snapshot_task.SnapshotTask, which="snapshot", rpc_method="snapshot") + sub.set_defaults( + cls=snapshot_task.SnapshotTask, which="snapshot", rpc_method="snapshot" + ) return sub @@ -527,7 +541,9 @@ def _build_compile_subparser(subparsers, base_subparser): Compiled SQL files are written to the target/ directory. """, ) - sub.set_defaults(cls=compile_task.CompileTask, which="compile", rpc_method="compile") + sub.set_defaults( + cls=compile_task.CompileTask, which="compile", rpc_method="compile" + ) sub.add_argument("--parse-only", action="store_true") return sub @@ -792,7 +808,9 @@ def _build_list_subparser(subparsers, base_subparser): aliases=["ls"], ) sub.set_defaults(cls=list_task.ListTask, which="list", rpc_method=None) - resource_values: List[str] = [str(s) for s in list_task.ListTask.ALL_RESOURCE_VALUES] + [ + resource_values: List[str] = [ + str(s) for s in list_task.ListTask.ALL_RESOURCE_VALUES + ] + [ "default", "all", ] @@ -803,7 +821,9 @@ def _build_list_subparser(subparsers, base_subparser): default=[], dest="resource_types", ) - sub.add_argument("--output", choices=["json", "name", "path", "selector"], default="selector") + sub.add_argument( + "--output", choices=["json", "name", "path", "selector"], default="selector" + ) sub.add_argument("--output-keys") sub.add_argument( @@ -870,7 +890,9 @@ def _build_run_operation_subparser(subparsers, base_subparser): """, ) sub.set_defaults( - cls=run_operation_task.RunOperationTask, which="run-operation", rpc_method="run-operation" + cls=run_operation_task.RunOperationTask, + which="run-operation", + rpc_method="run-operation", ) return sub @@ -1148,7 +1170,9 @@ def parse_args(args, cls=DBTArgumentParser): ) # --select, --exclude # list_sub sets up its own arguments. - _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub, snapshot_sub, seed_sub) + _add_selection_arguments( + run_sub, compile_sub, generate_sub, test_sub, snapshot_sub, seed_sub + ) # --defer _add_defer_argument(run_sub, test_sub, build_sub, snapshot_sub) # --full-refresh @@ -1201,3 +1225,7 @@ def parse_args(args, cls=DBTArgumentParser): p.exit(1) return parsed + + +if __name__ == "__main__": + main() From b16252ed13d6756a16100fe9ed3fa9ac77f06783 Mon Sep 17 00:00:00 2001 From: pquadri Date: Fri, 17 Jun 2022 11:39:54 +0200 Subject: [PATCH 3/6] chore: reset main.py --- core/dbt/main.py | 1276 ++++++++++++++++------------------------------ 1 file changed, 428 insertions(+), 848 deletions(-) diff --git a/core/dbt/main.py b/core/dbt/main.py index 89483ee4c3c..3031f2a04c8 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -1,156 +1,112 @@ +from dbt.logger import initialize_logger, GLOBAL_LOGGER as logger, \ + logger_initialized, log_cache_events + import argparse import os.path import sys import traceback -import warnings from contextlib import contextmanager -from pathlib import Path -from typing import List +import dbt.version import dbt.flags as flags -import dbt.task.build as build_task -import dbt.task.clean as clean_task +import dbt.task.run as run_task import dbt.task.compile as compile_task import dbt.task.debug as debug_task +import dbt.task.clean as clean_task import dbt.task.deps as deps_task -import dbt.task.freshness as freshness_task -import dbt.task.generate as generate_task import dbt.task.init as init_task -import dbt.task.list as list_task -import dbt.task.parse as parse_task -import dbt.task.run as run_task -import dbt.task.run_operation as run_operation_task import dbt.task.seed as seed_task -import dbt.task.serve as serve_task -import dbt.task.snapshot as snapshot_task import dbt.task.test as test_task +import dbt.task.snapshot as snapshot_task +import dbt.task.generate as generate_task +import dbt.task.serve as serve_task +import dbt.task.freshness as freshness_task +import dbt.task.run_operation as run_operation_task +from dbt.task.list import ListTask +from dbt.task.migrate import MigrationTask +from dbt.task.rpc_server import RPCServerTask +from dbt.adapters.factory import reset_adapters + import dbt.tracking -import dbt.version -from dbt.adapters.factory import cleanup_connections, reset_adapters -from dbt.config.profile import DEFAULT_PROFILES_DIR, read_user_config -from dbt.events.functions import fire_event, setup_event_logger -from dbt.events.types import ( - MainEncounteredError, - MainKeyboardInterrupt, - MainReportArgs, - MainReportVersion, - MainStackTrace, - MainTrackingUserState, -) -from dbt.exceptions import ( - FailedToConnectException, - InternalException, - NotImplementedException, -) -from dbt.logger import log_cache_events, log_manager -from dbt.profiler import profiler -from dbt.utils import ExitCodes, args_to_dict +import dbt.ui.printer +import dbt.compat +import dbt.deprecations +import dbt.profiler + +from dbt.utils import ExitCodes +from dbt.config import UserConfig, PROFILES_DIR +from dbt.exceptions import RuntimeException + + +PROFILES_HELP_MESSAGE = """ +For more information on configuring profiles, please consult the dbt docs: + +https://docs.getdbt.com/docs/configure-your-profile +""" class DBTVersion(argparse.Action): - """This is very similar to the built-in argparse._Version action, + """This is very very similar to the builtin argparse._Version action, except it just calls dbt.version.get_version_information(). """ - - def __init__( - self, - option_strings, - version=None, - dest=argparse.SUPPRESS, - default=argparse.SUPPRESS, - help="show program's version number and exit", - ): - super().__init__( + def __init__(self, + option_strings, + version=None, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + help="show program's version number and exit"): + super(DBTVersion, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, - help=help, - ) + help=help) def __call__(self, parser, namespace, values, option_string=None): - formatter = argparse.RawTextHelpFormatter(prog=parser.prog) + formatter = parser._get_formatter() formatter.add_text(dbt.version.get_version_information()) parser.exit(message=formatter.format_help()) class DBTArgumentParser(argparse.ArgumentParser): def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.register("action", "dbtversion", DBTVersion) - - def add_optional_argument_inverse( - self, - name, - *, - enable_help=None, - disable_help=None, - dest=None, - no_name=None, - default=None, - ): - mutex_group = self.add_mutually_exclusive_group() - if not name.startswith("--"): - raise InternalException( - 'cannot handle optional argument without "--" prefix: ' f'got "{name}"' - ) - if dest is None: - dest_name = name[2:].replace("-", "_") - else: - dest_name = dest + super(DBTArgumentParser, self).__init__(*args, **kwargs) + self.register('action', 'dbtversion', DBTVersion) - if no_name is None: - no_name = f"--no-{name[2:]}" - mutex_group.add_argument( - name, - action="store_const", - const=True, - dest=dest_name, - default=default, - help=enable_help, - ) +def main(args=None): + if args is None: + args = sys.argv[1:] - mutex_group.add_argument( - f"--no-{name[2:]}", - action="store_const", - const=False, - dest=dest_name, - default=default, - help=disable_help, - ) + try: + results, succeeded = handle_and_check(args) + if succeeded: + exit_code = ExitCodes.Success + else: + exit_code = ExitCodes.ModelError - return mutex_group + except KeyboardInterrupt: + logger.info("ctrl-c") + exit_code = ExitCodes.UnhandledError + # This can be thrown by eg. argparse + except SystemExit as e: + exit_code = e.code -def main(args=None): - # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. - # This _only_ works for regular cli invocations. - print("ciao") - warnings.filterwarnings("ignore", category=DeprecationWarning, module="logbook") - if args is None: - args = sys.argv[1:] - with log_manager.applicationbound(): - try: - results, succeeded = handle_and_check(args) - if succeeded: - exit_code = ExitCodes.Success.value - else: - exit_code = ExitCodes.ModelError.value - - except KeyboardInterrupt: - # if the logger isn't configured yet, it will use the default logger - fire_event(MainKeyboardInterrupt()) - exit_code = ExitCodes.UnhandledError.value - - # This can be thrown by eg. argparse - except SystemExit as e: - exit_code = e.code - - except BaseException as e: - fire_event(MainEncounteredError(e=str(e))) - fire_event(MainStackTrace(stack_trace=traceback.format_exc())) - exit_code = ExitCodes.UnhandledError.value + except BaseException as e: + logger.warn("Encountered an error:") + logger.warn(str(e)) + + if logger_initialized(): + logger.debug(traceback.format_exc()) + elif not isinstance(e, RuntimeException): + # if it did not come from dbt proper and the logger is not + # initialized (so there's no safe path to log to), log the stack + # trace at error level. + logger.error(traceback.format_exc()) + exit_code = ExitCodes.UnhandledError + + _python2_compatibility_message() sys.exit(exit_code) @@ -161,43 +117,41 @@ def handle(args): return res -@contextmanager -def adapter_management(): - reset_adapters() - try: - yield - finally: - cleanup_connections() +def initialize_config_values(parsed): + """Given the parsed args, initialize the dbt tracking code. + It would be nice to re-use this profile later on instead of parsing it + twice, but dbt's intialization is not structured in a way that makes that + easy. + """ + try: + cfg = UserConfig.from_directory(parsed.profiles_dir) + except RuntimeException: + cfg = UserConfig.from_dict(None) -def handle_and_check(args): - with log_manager.applicationbound(): - parsed = parse_args(args) + cfg.set_values(parsed.profiles_dir) - # Set flags from args, user config, and env vars - user_config = read_user_config(flags.PROFILES_DIR) # This is read again later - flags.set_from_args(parsed, user_config) - dbt.tracking.initialize_from_flags() - # Set log_format from flags - parsed.cls.set_log_format() - # we've parsed the args and set the flags - we can now decide if we're debug or not - if flags.DEBUG: - log_manager.set_debug() +def handle_and_check(args): + parsed = parse_args(args) + profiler_enabled = False - profiler_enabled = False + if parsed.record_timing_info: + profiler_enabled = True - if parsed.record_timing_info: - profiler_enabled = True + with dbt.profiler.profiler( + enable=profiler_enabled, + outfile=parsed.record_timing_info + ): - with profiler(enable=profiler_enabled, outfile=parsed.record_timing_info): + initialize_config_values(parsed) - with adapter_management(): + reset_adapters() - task, res = run_from_args(parsed) - success = task.interpret_results(res) + task, res = run_from_args(parsed) + success = task.interpret_results(res) - return res, success + return res, success @contextmanager @@ -208,8 +162,9 @@ def track_run(task): dbt.tracking.track_invocation_end( config=task.config, args=task.args, result_type="ok" ) - except (NotImplementedException, FailedToConnectException) as e: - fire_event(MainEncounteredError(e=str(e))) + except (dbt.exceptions.NotImplementedException, + dbt.exceptions.FailedToConnectException) as e: + logger.error('ERROR: {}'.format(e)) dbt.tracking.track_invocation_end( config=task.config, args=task.args, result_type="error" ) @@ -222,32 +177,44 @@ def track_run(task): dbt.tracking.flush() +_PYTHON_27_WARNING = ''' +Python 2.7 will reach the end of its life on January 1st, 2020. +Please upgrade your Python as Python 2.7 won't be maintained after that date. +A future version of dbt will drop support for Python 2.7. +'''.strip() + + +def _python2_compatibility_message(): + if dbt.compat.WHICH_PYTHON != 2: + return + + logger.critical( + dbt.ui.printer.red('DEPRECATION: ') + _PYTHON_27_WARNING + ) + + def run_from_args(parsed): - log_cache_events(getattr(parsed, "log_cache_events", False)) + log_cache_events(getattr(parsed, 'log_cache_events', False)) + flags.set_from_args(parsed) + + parsed.cls.pre_init_hook() + logger.info("Running with dbt{}".format(dbt.version.installed)) # this will convert DbtConfigErrors into RuntimeExceptions - # task could be any one of the task objects task = parsed.cls.from_args(args=parsed) + logger.debug("running dbt with arguments %s", parsed) - # Set up logging log_path = None if task.config is not None: - log_path = getattr(task.config, "log_path", None) - log_manager.set_path(log_path) - # if 'list' task: set stdout to WARN instead of INFO - level_override = parsed.cls.pre_init_hook(parsed) - setup_event_logger(log_path or "logs", level_override) - - fire_event(MainReportVersion(v=str(dbt.version.installed))) - fire_event(MainReportArgs(args=args_to_dict(parsed))) - - if dbt.tracking.active_user is not None: # mypy appeasement, always true - fire_event(MainTrackingUserState(user_state=dbt.tracking.active_user.state())) + log_path = getattr(task.config, 'log_path', None) + initialize_logger(parsed.debug, log_path) + logger.debug("Tracking: {}".format(dbt.tracking.active_user.state())) results = None with track_run(task): results = task.run() + return task, results @@ -255,774 +222,519 @@ def _build_base_subparser(): base_subparser = argparse.ArgumentParser(add_help=False) base_subparser.add_argument( - "--project-dir", + '--project-dir', default=None, type=str, help=""" Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents. - """, + """ ) base_subparser.add_argument( - "--profiles-dir", - default=None, - dest="sub_profiles_dir", # Main cli arg precedes subcommand + '--profiles-dir', + default=PROFILES_DIR, type=str, help=""" Which directory to look in for the profiles.yml file. Default = {} - """.format( - DEFAULT_PROFILES_DIR - ), + """.format(PROFILES_DIR) ) base_subparser.add_argument( - "--profile", + '--profile', required=False, type=str, help=""" Which profile to load. Overrides setting in dbt_project.yml. - """, + """ ) base_subparser.add_argument( - "-t", - "--target", + '--target', default=None, type=str, - help=""" - Which target to load for the given profile - """, + help='Which target to load for the given profile' ) base_subparser.add_argument( - "--vars", + '--vars', type=str, - default="{}", + default='{}', help=""" - Supply variables to the project. This argument overrides variables - defined in your dbt_project.yml file. This argument should be a YAML - string, eg. '{my_variable: my_value}' - """, + Supply variables to the project. This argument overrides + variables defined in your dbt_project.yml file. This argument + should be a YAML string, eg. '{my_variable: my_value}'""" ) # if set, log all cache events. This is extremely verbose! base_subparser.add_argument( - "--log-cache-events", - action="store_true", + '--log-cache-events', + action='store_true', help=argparse.SUPPRESS, ) - base_subparser.set_defaults(defer=None, state=None) + base_subparser.add_argument( + '--bypass-cache', + action='store_false', + dest='use_cache', + help='If set, bypass the adapter-level cache of database state', + ) return base_subparser def _build_docs_subparser(subparsers, base_subparser): docs_sub = subparsers.add_parser( - "docs", - help=""" - Generate or serve the documentation website for your project. - """, - ) + 'docs', + parents=[base_subparser], + help="Generate or serve the documentation " + "website for your project.") return docs_sub def _build_source_subparser(subparsers, base_subparser): source_sub = subparsers.add_parser( - "source", - help=""" - Manage your project's sources - """, - ) + 'source', + parents=[base_subparser], + help="Manage your project's sources") return source_sub def _build_init_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "init", - parents=[base_subparser], - help=""" - Initialize a new DBT project. - """, - ) - sub.add_argument( - "project_name", - nargs="?", - help=""" - Name of the new DBT project. - """, - ) - sub.add_argument( - "-s", - "--skip-profile-setup", - dest="skip_profile_setup", - action="store_true", - help=""" - Skip interative profile setup. - """, - ) - sub.set_defaults(cls=init_task.InitTask, which="init", rpc_method=None) - return sub - - -def _build_build_subparser(subparsers, base_subparser): - sub = subparsers.add_parser( - "build", + 'init', parents=[base_subparser], - help=""" - Run all Seeds, Models, Snapshots, and tests in DAG order - """, - ) - sub.set_defaults(cls=build_task.BuildTask, which="build", rpc_method="build") - sub.add_argument( - "-x", - "--fail-fast", - dest="sub_fail_fast", - action="store_true", - help=""" - Stop execution upon a first failure. - """, - ) - sub.add_argument( - "--store-failures", - action="store_true", - help=""" - Store test results (failing rows) in the database - """, - ) - sub.add_argument( - "--indirect-selection", - choices=["eager", "cautious"], - default="eager", - dest="indirect_selection", - help=""" - Select all tests that are adjacent to selected resources, - even if they those resources have been explicitly selected. - """, - ) - - resource_values: List[str] = [ - str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES - ] + ["all"] - sub.add_argument( - "--resource-type", - choices=resource_values, - action="append", - default=[], - dest="resource_types", - ) - # explicity don't support --models - sub.add_argument( - "-s", - "--select", - dest="select", - nargs="+", - help=""" - Specify the nodes to include. - """, - ) - _add_common_selector_arguments(sub) + help="Initialize a new DBT project.") + sub.add_argument('project_name', type=str, help='Name of the new project') + sub.set_defaults(cls=init_task.InitTask, which='init') return sub def _build_clean_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "clean", + 'clean', parents=[base_subparser], - help=""" - Delete all folders in the clean-targets list - (usually the dbt_packages and target directories.) - """, - ) - sub.set_defaults(cls=clean_task.CleanTask, which="clean", rpc_method=None) + help="Delete all folders in the clean-targets list" + "\n(usually the dbt_modules and target directories.)") + sub.set_defaults(cls=clean_task.CleanTask, which='clean') return sub def _build_debug_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "debug", + 'debug', parents=[base_subparser], - help=""" - Show some helpful information about dbt for debugging. - - Not to be confused with the --debug option which increases verbosity. - """, - ) + help="Show some helpful information about dbt for debugging." + "\nNot to be confused with the --debug option which increases " + "verbosity.") sub.add_argument( - "--config-dir", - action="store_true", + '--config-dir', + action='store_true', help=""" If specified, DBT will show path information for this project - """, + """ ) - _add_version_check(sub) - sub.set_defaults(cls=debug_task.DebugTask, which="debug", rpc_method=None) + sub.set_defaults(cls=debug_task.DebugTask, which='debug') return sub def _build_deps_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "deps", + 'deps', parents=[base_subparser], - help=""" - Pull the most recent version of the dependencies listed in packages.yml - """, - ) - sub.set_defaults(cls=deps_task.DepsTask, which="deps", rpc_method="deps") + help="Pull the most recent version of the dependencies " + "listed in packages.yml") + sub.set_defaults(cls=deps_task.DepsTask, which='deps') return sub -def _build_snapshot_subparser(subparsers, base_subparser): +def _build_snapshot_subparser(subparsers, base_subparser, which='snapshot'): + if which == 'archive': + helpmsg = ( + 'DEPRECATED: This command is deprecated and will\n' + 'be removed in a future release. Use dbt snapshot instead.' + ) + else: + helpmsg = 'Execute snapshots defined in your project' + sub = subparsers.add_parser( - "snapshot", + which, parents=[base_subparser], - help=""" - Execute snapshots defined in your project - """, - ) + help=helpmsg) sub.add_argument( - "--threads", + '--threads', type=int, required=False, help=""" - Specify number of threads to use while snapshotting tables. - Overrides settings in profiles.yml. - """, - ) - sub.set_defaults( - cls=snapshot_task.SnapshotTask, which="snapshot", rpc_method="snapshot" + Specify number of threads to use while snapshotting tables. Overrides + settings in profiles.yml. + """ ) + sub.set_defaults(cls=snapshot_task.SnapshotTask, which=which) return sub -def _add_defer_argument(*subparsers): - for sub in subparsers: - sub.add_optional_argument_inverse( - "--defer", - enable_help=""" - If set, defer to the state variable for resolving unselected nodes. - """, - disable_help=""" - If set, do not defer to the state variable for resolving unselected - nodes. - """, - default=flags.DEFER_MODE, - ) - - def _build_run_subparser(subparsers, base_subparser): run_sub = subparsers.add_parser( - "run", + 'run', parents=[base_subparser], - help=""" - Compile SQL and execute against the current target database. - """, - ) - run_sub.add_argument( - "-x", - "--fail-fast", - dest="sub_fail_fast", - action="store_true", - help=""" - Stop execution upon a first failure. - """, - ) - - run_sub.set_defaults(cls=run_task.RunTask, which="run", rpc_method="run") + help="Compile SQL and execute against the current " + "target database.") + run_sub.set_defaults(cls=run_task.RunTask, which='run') return run_sub def _build_compile_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "compile", + 'compile', parents=[base_subparser], - help=""" - Generates executable SQL from source, model, test, and analysis files. - Compiled SQL files are written to the target/ directory. - """, - ) - sub.set_defaults( - cls=compile_task.CompileTask, which="compile", rpc_method="compile" - ) - sub.add_argument("--parse-only", action="store_true") - return sub - - -def _build_parse_subparser(subparsers, base_subparser): - sub = subparsers.add_parser( - "parse", - parents=[base_subparser], - help=""" - Parses the project and provides information on performance - """, - ) - sub.set_defaults(cls=parse_task.ParseTask, which="parse", rpc_method="parse") - sub.add_argument("--write-manifest", action="store_true") - sub.add_argument("--compile", action="store_true") + help="Generates executable SQL from source model, test, and" + "analysis files. \nCompiled SQL files are written to the target/" + "directory.") + sub.set_defaults(cls=compile_task.CompileTask, which='compile') return sub def _build_docs_generate_subparser(subparsers, base_subparser): # it might look like docs_sub is the correct parents entry, but that # will cause weird errors about 'conflicting option strings'. - generate_sub = subparsers.add_parser("generate", parents=[base_subparser]) - generate_sub.set_defaults( - cls=generate_task.GenerateTask, which="generate", rpc_method="docs.generate" - ) + generate_sub = subparsers.add_parser('generate', parents=[base_subparser]) + generate_sub.set_defaults(cls=generate_task.GenerateTask, + which='generate') generate_sub.add_argument( - "--no-compile", - action="store_false", - dest="compile", - help=""" - Do not run "dbt compile" as part of docs generation - """, + '--no-compile', + action='store_false', + dest='compile', + help='Do not run "dbt compile" as part of docs generation' ) return generate_sub -def _add_common_selector_arguments(sub): - sub.add_argument( - "--exclude", - required=False, - nargs="+", - help=""" - Specify the models to exclude. - """, - ) - sub.add_argument( - "--selector", - dest="selector_name", - metavar="SELECTOR_NAME", - help=""" - The selector name to use, as defined in selectors.yml - """, - ) - sub.add_argument( - "--state", - help=""" - If set, use the given directory as the source for json files to - compare with this project. - """, - type=Path, - default=flags.ARTIFACT_STATE_PATH, - ) - - -def _add_selection_arguments(*subparsers): +def _add_selection_arguments(*subparsers, **kwargs): + models_name = kwargs.get('models_name', 'models') for sub in subparsers: sub.add_argument( - "-m", - "--models", - dest="select", - nargs="+", + '-{}'.format(models_name[0]), + '--{}'.format(models_name), + dest='models', + required=False, + nargs='+', help=""" - Specify the nodes to include. - """, + Specify the models to include. + """ ) sub.add_argument( - "-s", - "--select", - dest="select", - nargs="+", + '--exclude', + required=False, + nargs='+', help=""" - Specify the nodes to include. - """, + Specify the models to exclude. + """ ) - _add_common_selector_arguments(sub) def _add_table_mutability_arguments(*subparsers): for sub in subparsers: sub.add_argument( - "--full-refresh", - action="store_true", + '--full-refresh', + action='store_true', help=""" - If specified, dbt will drop incremental models and + If specified, DBT will drop incremental models and fully-recalculate the incremental table from the model definition. - """, - ) - - -def _add_version_check(sub): - sub.add_argument( - "--no-version-check", - dest="sub_version_check", # main cli arg precedes subcommands - action="store_false", - default=None, - help=""" - If set, skip ensuring dbt's version matches the one specified in - the dbt_project.yml file ('require-dbt-version') - """, - ) + """) def _add_common_arguments(*subparsers): for sub in subparsers: sub.add_argument( - "--threads", + '--threads', type=int, required=False, help=""" Specify number of threads to use while executing models. Overrides settings in profiles.yml. - """, + """ ) - _add_version_check(sub) + sub.add_argument( + '--no-version-check', + dest='version_check', + action='store_false', + help=""" + If set, skip ensuring dbt's version matches the one specified in + the dbt_project.yml file ('require-dbt-version') + """) def _build_seed_subparser(subparsers, base_subparser): seed_sub = subparsers.add_parser( - "seed", + 'seed', parents=[base_subparser], - help=""" - Load data from csv files into your data warehouse. - """, - ) + help="Load data from csv files into your data warehouse.") seed_sub.add_argument( - "--full-refresh", - action="store_true", - help=""" - Drop existing seed tables and recreate them - """, + '--full-refresh', + action='store_true', + help='Drop existing seed tables and recreate them' ) seed_sub.add_argument( - "--show", - action="store_true", - help=""" - Show a sample of the loaded data in the terminal - """, + '--show', + action='store_true', + help='Show a sample of the loaded data in the terminal' ) - seed_sub.set_defaults(cls=seed_task.SeedTask, which="seed", rpc_method="seed") + seed_sub.set_defaults(cls=seed_task.SeedTask, which='seed') return seed_sub def _build_docs_serve_subparser(subparsers, base_subparser): - serve_sub = subparsers.add_parser("serve", parents=[base_subparser]) + serve_sub = subparsers.add_parser('serve', parents=[base_subparser]) serve_sub.add_argument( - "--port", + '--port', default=8080, type=int, - help=""" - Specify the port number for the docs server. - """, - ) - serve_sub.add_argument( - "--no-browser", - dest="open_browser", - action="store_false", + help='Specify the port number for the docs server.' ) - serve_sub.set_defaults(cls=serve_task.ServeTask, which="serve", rpc_method=None) + serve_sub.set_defaults(cls=serve_task.ServeTask, which='serve') return serve_sub def _build_test_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "test", + 'test', parents=[base_subparser], - help=""" - Runs tests on data in deployed models. Run this after `dbt run` - """, - ) + help="Runs tests on data in deployed models." + "Run this after `dbt run`") sub.add_argument( - "-x", - "--fail-fast", - dest="sub_fail_fast", - action="store_true", - help=""" - Stop execution upon a first test failure. - """, + '--data', + action='store_true', + help='Run data tests defined in "tests" directory.' ) sub.add_argument( - "--store-failures", - action="store_true", - help=""" - Store test results (failing rows) in the database - """, - ) - sub.add_argument( - "--indirect-selection", - choices=["eager", "cautious"], - default="eager", - dest="indirect_selection", - help=""" - Select all tests that are adjacent to selected resources, - even if they those resources have been explicitly selected. - """, + '--schema', + action='store_true', + help='Run constraint validations from schema.yml files' ) - sub.set_defaults(cls=test_task.TestTask, which="test", rpc_method="test") + sub.set_defaults(cls=test_task.TestTask, which='test') return sub -def _build_source_freshness_subparser(subparsers, base_subparser): +def _build_source_snapshot_freshness_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "freshness", + 'snapshot-freshness', parents=[base_subparser], + help="Snapshots the current freshness of the project's sources", + ) + sub.add_argument( + '-s', + '--select', + required=False, + nargs='+', help=""" - Snapshots the current freshness of the project's sources + Specify the sources to snapshot freshness """, - aliases=["snapshot-freshness"], + dest='selected' ) sub.add_argument( - "-o", - "--output", + '-o', + '--output', required=False, help=""" Specify the output path for the json report. By default, outputs to target/sources.json - """, + """ ) sub.add_argument( - "--threads", + '--threads', type=int, required=False, help=""" Specify number of threads to use. Overrides settings in profiles.yml - """, - ) - sub.set_defaults( - cls=freshness_task.FreshnessTask, - which="source-freshness", - rpc_method="source-freshness", - ) - sub.add_argument( - "-s", - "--select", - dest="select", - nargs="+", - help=""" - Specify the nodes to include. - """, + """ ) - _add_common_selector_arguments(sub) + sub.set_defaults(cls=freshness_task.FreshnessTask, + which='snapshot-freshness') return sub -def _build_list_subparser(subparsers, base_subparser): +def _build_rpc_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "list", + 'rpc', parents=[base_subparser], - help=""" - List the resources in your project - """, - aliases=["ls"], + help='Start a json-rpc server', ) - sub.set_defaults(cls=list_task.ListTask, which="list", rpc_method=None) - resource_values: List[str] = [ - str(s) for s in list_task.ListTask.ALL_RESOURCE_VALUES - ] + [ - "default", - "all", - ] sub.add_argument( - "--resource-type", - choices=resource_values, - action="append", - default=[], - dest="resource_types", + '--host', + default='0.0.0.0', + help='Specify the host to listen on for the rpc server.' ) sub.add_argument( - "--output", choices=["json", "name", "path", "selector"], default="selector" + '--port', + default=8580, + type=int, + help='Specify the port number for the rpc server.' ) - sub.add_argument("--output-keys") + sub.set_defaults(cls=RPCServerTask, which='rpc') + # the rpc task does a 'compile', so we need these attributes to exist, but + # we don't want users to be allowed to set them. + sub.set_defaults(models=None, exclude=None) + return sub + +def _build_list_subparser(subparsers, base_subparser): + sub = subparsers.add_parser( + 'list', + parents=[base_subparser], + help='List the resources in your project' + ) + sub.set_defaults(cls=ListTask, which='list') + resource_values = list(ListTask.ALL_RESOURCE_VALUES) + ['default', 'all'] + sub.add_argument('--resource-type', + choices=resource_values, + action='append', + default=[], + dest='resource_types') + sub.add_argument('--output', + choices=['json', 'name', 'path', 'selector'], + default='selector') sub.add_argument( - "-m", - "--models", - dest="models", - nargs="+", - help=""" - Specify the models to select and set the resource-type to 'model'. - Mutually exclusive with '--select' (or '-s') and '--resource-type' - """, - metavar="SELECTOR", + '-s', + '--select', required=False, + nargs='+', + metavar='SELECTOR', + help="Specify the nodes to select.", ) sub.add_argument( - "-s", - "--select", - dest="select", - nargs="+", - help=""" - Specify the nodes to include. - """, - metavar="SELECTOR", + '-m', + '--models', required=False, + nargs='+', + metavar='SELECTOR', + help="Specify the models to select and set the resource-type to " + "'model'. Mutually exclusive with '--select' (or '-s') and " + "'--resource-type'", ) sub.add_argument( - "--indirect-selection", - choices=["eager", "cautious"], - default="eager", - dest="indirect_selection", - help=""" - Select all tests that are adjacent to selected resources, - even if they those resources have been explicitly selected. - """, + '--exclude', + required=False, + nargs='+', + metavar='SELECTOR', + help="Specify the models to exclude." ) - _add_common_selector_arguments(sub) - + # in python 3.x you can use the 'aliases' kwarg, but in python 2.7 you get + # to do this + subparsers._name_parser_map['ls'] = sub return sub def _build_run_operation_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - "run-operation", + 'run-operation', parents=[base_subparser], help=""" - Run the named macro with any supplied arguments. - """, + (beta) Run the named macro with any supplied arguments. This + subcommand is unstable and subject to change in a future release + of dbt. Please use it with caution""" ) sub.add_argument( - "macro", + 'macro', help=""" - Specify the macro to invoke. dbt will call this macro with the supplied - arguments and then exit - """, + Specify the macro to invoke. dbt will call this macro with the + supplied arguments and then exit""" ) sub.add_argument( - "--args", + '--args', type=str, - default="{}", + default='{}', help=""" - Supply arguments to the macro. This dictionary will be mapped to the - keyword arguments defined in the selected macro. This argument should - be a YAML string, eg. '{my_variable: my_value}' - """, - ) - sub.set_defaults( - cls=run_operation_task.RunOperationTask, - which="run-operation", - rpc_method="run-operation", + Supply arguments to the macro. This dictionary will be mapped + to the keyword arguments defined in the selected macro. This + argument should be a YAML string, eg. '{my_variable: my_value}'""" ) + sub.set_defaults(cls=run_operation_task.RunOperationTask, + which='run-operation') return sub -def parse_args(args, cls=DBTArgumentParser): - p = cls( - prog="dbt", - description=""" - An ELT tool for managing your SQL transformations and data models. - For more documentation on these commands, visit: docs.getdbt.com - """, - epilog=""" - Specify one of these sub-commands and you can find more help from - there. - """, +def _build_snapshot_migrate_subparser(subparsers, base_subparser): + sub = subparsers.add_parser( + 'snapshot-migrate', + parents=[base_subparser], + help='Run the snapshot migration script' ) - - p.add_argument( - "--version", - action="dbtversion", - help=""" - Show version information - """, + sub.add_argument( + '--from-archive', + action='store_true', + help=('This flag is required for the 0.14.0 archive to snapshot ' + 'migration') ) - - p.add_argument( - "-r", - "--record-timing-info", - default=None, - type=str, - help=""" - When this option is passed, dbt will output low-level timing stats to - the specified file. Example: `--record-timing-info output.profile` - """, + sub.add_argument( + '--apply-files', + action='store_true', + dest='write_files', + help='If set, write .sql files to disk instead of logging them' ) - - p.add_argument( - "-d", - "--debug", - action="store_true", - default=None, - help=""" - Display debug logging during dbt execution. Useful for debugging and - making bug reports. - """, + sub.add_argument( + '--apply-database', + action='store_true', + dest='migrate_database', + help='If set, perform just the database migration' ) + sub.add_argument( + '--apply', + action='store_true', + help='If set, implies --apply-database --apply-files' + ) + sub.set_defaults(cls=MigrationTask, which='migration') + + +def parse_args(args): + p = DBTArgumentParser( + prog='dbt', + formatter_class=argparse.RawTextHelpFormatter, + description="An ELT tool for managing your SQL " + "transformations and data models." + "\nFor more documentation on these commands, visit: " + "docs.getdbt.com", + epilog="Specify one of these sub-commands and you can " + "find more help from there.") p.add_argument( - "--log-format", - choices=["text", "json", "default"], - default=None, - help="""Specify the log format, overriding the command's default.""", - ) + '--version', + action='dbtversion', + help="Show version information") p.add_argument( - "--no-write-json", - action="store_false", + '-r', + '--record-timing-info', default=None, - dest="write_json", - help=""" - If set, skip writing the manifest and run_results.json files to disk - """, - ) - colors_flag = p.add_mutually_exclusive_group() - colors_flag.add_argument( - "--use-colors", - action="store_const", - const=True, - default=None, - dest="use_colors", - help=""" - Colorize the output DBT prints to the terminal. Output is colorized by - default and may also be set in a profile or at the command line. - Mutually exclusive with --no-use-colors - """, - ) - colors_flag.add_argument( - "--no-use-colors", - action="store_const", - const=False, - dest="use_colors", + type=str, help=""" - Do not colorize the output DBT prints to the terminal. Output is - colorized by default and may also be set in a profile or at the - command line. - Mutually exclusive with --use-colors - """, + When this option is passed, dbt will output low-level timing + stats to the specified file. Example: + `--record-timing-info output.profile` + """ ) p.add_argument( - "--printer-width", - dest="printer_width", - help=""" - Sets the width of terminal output - """, - ) + '-d', + '--debug', + action='store_true', + help='''Display debug logging during dbt execution. Useful for + debugging and making bug reports.''') p.add_argument( - "--warn-error", - action="store_true", - default=None, - help=""" - If dbt would normally warn, instead raise an exception. Examples - include --models that selects nothing, deprecations, configurations - with no associated models, invalid test configurations, and missing - sources/refs in tests. - """, - ) + '-S', + '--strict', + action='store_true', + help='''Run schema validations at runtime. This will surface + bugs in dbt, but may incur a performance penalty.''') p.add_argument( - "--no-version-check", - dest="version_check", - action="store_false", - default=None, - help=""" - If set, skip ensuring dbt's version matches the one specified in - the dbt_project.yml file ('require-dbt-version') - """, - ) - - p.add_optional_argument_inverse( - "--partial-parse", - enable_help=""" - Allow for partial parsing by looking for and writing to a pickle file - in the target directory. This overrides the user configuration file. - """, - disable_help=""" - Disallow partial parsing. This overrides the user configuration file. - """, - ) + '--warn-error', + action='store_true', + help='''If dbt would normally warn, instead raise an exception. + Examples include --models that selects nothing, deprecations, + configurations with no associated models, invalid test configurations, + and missing sources/refs in tests''') # if set, run dbt in single-threaded mode: thread count is ignored, and # calls go through `map` instead of the thread pool. This is useful for @@ -1030,114 +742,18 @@ def parse_args(args, cls=DBTArgumentParser): # a thread, as the profiler ignores child threads. Users should really # never use this. p.add_argument( - "--single-threaded", - action="store_true", + '--single-threaded', + action='store_true', help=argparse.SUPPRESS, ) - # if set, will use the latest features from the static parser instead of - # the stable static parser. - p.add_argument( - "--use-experimental-parser", - action="store_true", - default=None, - help=""" - Enables experimental parsing features. - """, - ) - - # if set, will disable the use of the stable static parser and instead - # always rely on jinja rendering. + # if set, extract all models and blocks with the jinja block extractor, and + # verify that we don't fail anywhere the actual jinja parser passes. The + # reverse (passing files that ends up failing jinja) is fine. p.add_argument( - "--no-static-parser", - default=None, - dest="static_parser", - action="store_false", - help=""" - Disables the static parser. - """, - ) - - p.add_argument( - "--profiles-dir", - default=None, - dest="profiles_dir", - type=str, - help=""" - Which directory to look in for the profiles.yml file. Default = {} - """.format( - DEFAULT_PROFILES_DIR - ), - ) - - p.add_argument( - "--no-anonymous-usage-stats", - action="store_false", - default=None, - dest="send_anonymous_usage_stats", - help=""" - Do not send anonymous usage stat to dbt Labs - """, - ) - - p.add_argument( - "-x", - "--fail-fast", - dest="fail_fast", - action="store_true", - default=None, - help=""" - Stop execution upon a first failure. - """, - ) - - p.add_argument( - "--event-buffer-size", - dest="event_buffer_size", - help=""" - Sets the max number of events to buffer in EVENT_HISTORY - """, - ) - - p.add_argument( - "-q", - "--quiet", - action="store_true", - default=None, - help=""" - Suppress all non-error logging to stdout. Does not affect - {{ print() }} macro calls. - """, - ) - - p.add_argument( - "--no-print", - action="store_true", - default=None, - help=""" - Suppress all {{ print() }} macro calls. - """, - ) - - schema_cache_flag = p.add_mutually_exclusive_group() - schema_cache_flag.add_argument( - "--cache-selected-only", - action="store_const", - const=True, - default=None, - dest="cache_selected_only", - help=""" - Pre cache database objects relevant to selected resource only. - """, - ) - schema_cache_flag.add_argument( - "--no-cache-selected-only", - action="store_const", - const=False, - dest="cache_selected_only", - help=""" - Pre cache all database objects related to the project. - """, + '--test-new-parser', + action='store_true', + help=argparse.SUPPRESS ) subs = p.add_subparsers(title="Available sub-commands") @@ -1155,31 +771,28 @@ def parse_args(args, cls=DBTArgumentParser): _build_debug_subparser(subs, base_subparser) _build_deps_subparser(subs, base_subparser) _build_list_subparser(subs, base_subparser) + _build_snapshot_migrate_subparser(subs, base_subparser) - build_sub = _build_build_subparser(subs, base_subparser) snapshot_sub = _build_snapshot_subparser(subs, base_subparser) + archive_sub = _build_snapshot_subparser(subs, base_subparser, 'archive') + rpc_sub = _build_rpc_subparser(subs, base_subparser) run_sub = _build_run_subparser(subs, base_subparser) compile_sub = _build_compile_subparser(subs, base_subparser) - parse_sub = _build_parse_subparser(subs, base_subparser) generate_sub = _build_docs_generate_subparser(docs_subs, base_subparser) test_sub = _build_test_subparser(subs, base_subparser) - seed_sub = _build_seed_subparser(subs, base_subparser) # --threads, --no-version-check - _add_common_arguments( - run_sub, compile_sub, generate_sub, test_sub, seed_sub, parse_sub, build_sub - ) - # --select, --exclude - # list_sub sets up its own arguments. - _add_selection_arguments( - run_sub, compile_sub, generate_sub, test_sub, snapshot_sub, seed_sub - ) - # --defer - _add_defer_argument(run_sub, test_sub, build_sub, snapshot_sub) + _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub, + rpc_sub) + # --models, --exclude + _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub, + archive_sub) + _add_selection_arguments(snapshot_sub, models_name='select') # --full-refresh - _add_table_mutability_arguments(run_sub, compile_sub, build_sub) + _add_table_mutability_arguments(run_sub, compile_sub) + _build_seed_subparser(subs, base_subparser) _build_docs_serve_subparser(docs_subs, base_subparser) - _build_source_freshness_subparser(source_subs, base_subparser) + _build_source_snapshot_freshness_subparser(source_subs, base_subparser) _build_run_operation_subparser(subs, base_subparser) if len(args) == 0: @@ -1187,45 +800,12 @@ def parse_args(args, cls=DBTArgumentParser): sys.exit(1) parsed = p.parse_args(args) + parsed.profiles_dir = os.path.expanduser(parsed.profiles_dir) - # profiles_dir is set before subcommands and after, so normalize - if hasattr(parsed, "sub_profiles_dir"): - if parsed.sub_profiles_dir is not None: - parsed.profiles_dir = parsed.sub_profiles_dir - delattr(parsed, "sub_profiles_dir") - if hasattr(parsed, "profiles_dir"): - if parsed.profiles_dir is None: - parsed.profiles_dir = flags.PROFILES_DIR - else: - parsed.profiles_dir = os.path.abspath(parsed.profiles_dir) - # needs to be set before the other flags, because it's needed to - # read the profile that contains them - flags.PROFILES_DIR = parsed.profiles_dir - - # version_check is set before subcommands and after, so normalize - if hasattr(parsed, "sub_version_check"): - if parsed.sub_version_check is False: - parsed.version_check = False - delattr(parsed, "sub_version_check") - - # fail_fast is set before subcommands and after, so normalize - if hasattr(parsed, "sub_fail_fast"): - if parsed.sub_fail_fast is True: - parsed.fail_fast = True - delattr(parsed, "sub_fail_fast") - - if getattr(parsed, "project_dir", None) is not None: - expanded_user = os.path.expanduser(parsed.project_dir) - parsed.project_dir = os.path.abspath(expanded_user) - - if not hasattr(parsed, "which"): + if not hasattr(parsed, 'which'): # the user did not provide a valid subcommand. trigger the help message # and exit with a error p.print_help() p.exit(1) return parsed - - -if __name__ == "__main__": - main() From 8b6ef6aef8230cc2de7c821f2a2fe92a86814914 Mon Sep 17 00:00:00 2001 From: pquadri Date: Fri, 17 Jun 2022 11:41:27 +0200 Subject: [PATCH 4/6] chore: reset main.py from master --- core/dbt/main.py | 1272 ++++++++++++++++++++++++++++++---------------- 1 file changed, 832 insertions(+), 440 deletions(-) diff --git a/core/dbt/main.py b/core/dbt/main.py index 3031f2a04c8..6111d0a5f7b 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -1,112 +1,150 @@ -from dbt.logger import initialize_logger, GLOBAL_LOGGER as logger, \ - logger_initialized, log_cache_events +from typing import List +from dbt.logger import log_cache_events, log_manager import argparse import os.path import sys import traceback +import warnings from contextlib import contextmanager +from pathlib import Path import dbt.version +from dbt.events.functions import fire_event, setup_event_logger +from dbt.events.types import ( + MainEncounteredError, + MainKeyboardInterrupt, + MainReportVersion, + MainReportArgs, + MainTrackingUserState, + MainStackTrace, +) import dbt.flags as flags -import dbt.task.run as run_task +import dbt.task.build as build_task +import dbt.task.clean as clean_task import dbt.task.compile as compile_task import dbt.task.debug as debug_task -import dbt.task.clean as clean_task import dbt.task.deps as deps_task +import dbt.task.freshness as freshness_task +import dbt.task.generate as generate_task import dbt.task.init as init_task +import dbt.task.list as list_task +import dbt.task.parse as parse_task +import dbt.task.run as run_task +import dbt.task.run_operation as run_operation_task import dbt.task.seed as seed_task -import dbt.task.test as test_task -import dbt.task.snapshot as snapshot_task -import dbt.task.generate as generate_task import dbt.task.serve as serve_task -import dbt.task.freshness as freshness_task -import dbt.task.run_operation as run_operation_task -from dbt.task.list import ListTask -from dbt.task.migrate import MigrationTask -from dbt.task.rpc_server import RPCServerTask -from dbt.adapters.factory import reset_adapters +import dbt.task.snapshot as snapshot_task +import dbt.task.test as test_task +from dbt.profiler import profiler +from dbt.adapters.factory import reset_adapters, cleanup_connections import dbt.tracking -import dbt.ui.printer -import dbt.compat -import dbt.deprecations -import dbt.profiler - -from dbt.utils import ExitCodes -from dbt.config import UserConfig, PROFILES_DIR -from dbt.exceptions import RuntimeException - - -PROFILES_HELP_MESSAGE = """ -For more information on configuring profiles, please consult the dbt docs: -https://docs.getdbt.com/docs/configure-your-profile -""" +from dbt.utils import ExitCodes, args_to_dict +from dbt.config.profile import DEFAULT_PROFILES_DIR, read_user_config +from dbt.exceptions import InternalException, NotImplementedException, FailedToConnectException class DBTVersion(argparse.Action): - """This is very very similar to the builtin argparse._Version action, + """This is very similar to the built-in argparse._Version action, except it just calls dbt.version.get_version_information(). """ - def __init__(self, - option_strings, - version=None, - dest=argparse.SUPPRESS, - default=argparse.SUPPRESS, - help="show program's version number and exit"): - super(DBTVersion, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) + + def __init__( + self, + option_strings, + version=None, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + help="show program's version number and exit", + ): + super().__init__( + option_strings=option_strings, dest=dest, default=default, nargs=0, help=help + ) def __call__(self, parser, namespace, values, option_string=None): - formatter = parser._get_formatter() + formatter = argparse.RawTextHelpFormatter(prog=parser.prog) formatter.add_text(dbt.version.get_version_information()) parser.exit(message=formatter.format_help()) class DBTArgumentParser(argparse.ArgumentParser): def __init__(self, *args, **kwargs): - super(DBTArgumentParser, self).__init__(*args, **kwargs) - self.register('action', 'dbtversion', DBTVersion) - - -def main(args=None): - if args is None: - args = sys.argv[1:] - - try: - results, succeeded = handle_and_check(args) - if succeeded: - exit_code = ExitCodes.Success + super().__init__(*args, **kwargs) + self.register("action", "dbtversion", DBTVersion) + + def add_optional_argument_inverse( + self, + name, + *, + enable_help=None, + disable_help=None, + dest=None, + no_name=None, + default=None, + ): + mutex_group = self.add_mutually_exclusive_group() + if not name.startswith("--"): + raise InternalException( + 'cannot handle optional argument without "--" prefix: ' f'got "{name}"' + ) + if dest is None: + dest_name = name[2:].replace("-", "_") else: - exit_code = ExitCodes.ModelError + dest_name = dest + + if no_name is None: + no_name = f"--no-{name[2:]}" - except KeyboardInterrupt: - logger.info("ctrl-c") - exit_code = ExitCodes.UnhandledError + mutex_group.add_argument( + name, + action="store_const", + const=True, + dest=dest_name, + default=default, + help=enable_help, + ) - # This can be thrown by eg. argparse - except SystemExit as e: - exit_code = e.code + mutex_group.add_argument( + f"--no-{name[2:]}", + action="store_const", + const=False, + dest=dest_name, + default=default, + help=disable_help, + ) - except BaseException as e: - logger.warn("Encountered an error:") - logger.warn(str(e)) + return mutex_group - if logger_initialized(): - logger.debug(traceback.format_exc()) - elif not isinstance(e, RuntimeException): - # if it did not come from dbt proper and the logger is not - # initialized (so there's no safe path to log to), log the stack - # trace at error level. - logger.error(traceback.format_exc()) - exit_code = ExitCodes.UnhandledError - _python2_compatibility_message() +def main(args=None): + # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. + # This _only_ works for regular cli invocations. + warnings.filterwarnings("ignore", category=DeprecationWarning, module="logbook") + if args is None: + args = sys.argv[1:] + with log_manager.applicationbound(): + try: + results, succeeded = handle_and_check(args) + if succeeded: + exit_code = ExitCodes.Success.value + else: + exit_code = ExitCodes.ModelError.value + + except KeyboardInterrupt: + # if the logger isn't configured yet, it will use the default logger + fire_event(MainKeyboardInterrupt()) + exit_code = ExitCodes.UnhandledError.value + + # This can be thrown by eg. argparse + except SystemExit as e: + exit_code = e.code + + except BaseException as e: + fire_event(MainEncounteredError(e=str(e))) + fire_event(MainStackTrace(stack_trace=traceback.format_exc())) + exit_code = ExitCodes.UnhandledError.value sys.exit(exit_code) @@ -117,41 +155,43 @@ def handle(args): return res -def initialize_config_values(parsed): - """Given the parsed args, initialize the dbt tracking code. - - It would be nice to re-use this profile later on instead of parsing it - twice, but dbt's intialization is not structured in a way that makes that - easy. - """ +@contextmanager +def adapter_management(): + reset_adapters() try: - cfg = UserConfig.from_directory(parsed.profiles_dir) - except RuntimeException: - cfg = UserConfig.from_dict(None) - - cfg.set_values(parsed.profiles_dir) + yield + finally: + cleanup_connections() def handle_and_check(args): - parsed = parse_args(args) - profiler_enabled = False + with log_manager.applicationbound(): + parsed = parse_args(args) - if parsed.record_timing_info: - profiler_enabled = True + # Set flags from args, user config, and env vars + user_config = read_user_config(flags.PROFILES_DIR) # This is read again later + flags.set_from_args(parsed, user_config) + dbt.tracking.initialize_from_flags() + # Set log_format from flags + parsed.cls.set_log_format() - with dbt.profiler.profiler( - enable=profiler_enabled, - outfile=parsed.record_timing_info - ): + # we've parsed the args and set the flags - we can now decide if we're debug or not + if flags.DEBUG: + log_manager.set_debug() + + profiler_enabled = False + + if parsed.record_timing_info: + profiler_enabled = True - initialize_config_values(parsed) + with profiler(enable=profiler_enabled, outfile=parsed.record_timing_info): - reset_adapters() + with adapter_management(): - task, res = run_from_args(parsed) - success = task.interpret_results(res) + task, res = run_from_args(parsed) + success = task.interpret_results(res) - return res, success + return res, success @contextmanager @@ -159,62 +199,43 @@ def track_run(task): dbt.tracking.track_invocation_start(config=task.config, args=task.args) try: yield - dbt.tracking.track_invocation_end( - config=task.config, args=task.args, result_type="ok" - ) - except (dbt.exceptions.NotImplementedException, - dbt.exceptions.FailedToConnectException) as e: - logger.error('ERROR: {}'.format(e)) - dbt.tracking.track_invocation_end( - config=task.config, args=task.args, result_type="error" - ) + dbt.tracking.track_invocation_end(config=task.config, args=task.args, result_type="ok") + except (NotImplementedException, FailedToConnectException) as e: + fire_event(MainEncounteredError(e=str(e))) + dbt.tracking.track_invocation_end(config=task.config, args=task.args, result_type="error") except Exception: - dbt.tracking.track_invocation_end( - config=task.config, args=task.args, result_type="error" - ) + dbt.tracking.track_invocation_end(config=task.config, args=task.args, result_type="error") raise finally: dbt.tracking.flush() -_PYTHON_27_WARNING = ''' -Python 2.7 will reach the end of its life on January 1st, 2020. -Please upgrade your Python as Python 2.7 won't be maintained after that date. -A future version of dbt will drop support for Python 2.7. -'''.strip() - - -def _python2_compatibility_message(): - if dbt.compat.WHICH_PYTHON != 2: - return - - logger.critical( - dbt.ui.printer.red('DEPRECATION: ') + _PYTHON_27_WARNING - ) - - def run_from_args(parsed): - log_cache_events(getattr(parsed, 'log_cache_events', False)) - flags.set_from_args(parsed) - - parsed.cls.pre_init_hook() - logger.info("Running with dbt{}".format(dbt.version.installed)) + log_cache_events(getattr(parsed, "log_cache_events", False)) # this will convert DbtConfigErrors into RuntimeExceptions + # task could be any one of the task objects task = parsed.cls.from_args(args=parsed) - logger.debug("running dbt with arguments %s", parsed) + # Set up logging log_path = None if task.config is not None: - log_path = getattr(task.config, 'log_path', None) - initialize_logger(parsed.debug, log_path) - logger.debug("Tracking: {}".format(dbt.tracking.active_user.state())) + log_path = getattr(task.config, "log_path", None) + log_manager.set_path(log_path) + # if 'list' task: set stdout to WARN instead of INFO + level_override = parsed.cls.pre_init_hook(parsed) + setup_event_logger(log_path or "logs", level_override) + + fire_event(MainReportVersion(v=str(dbt.version.installed))) + fire_event(MainReportArgs(args=args_to_dict(parsed))) + + if dbt.tracking.active_user is not None: # mypy appeasement, always true + fire_event(MainTrackingUserState(user_state=dbt.tracking.active_user.state())) results = None with track_run(task): results = task.run() - return task, results @@ -222,519 +243,764 @@ def _build_base_subparser(): base_subparser = argparse.ArgumentParser(add_help=False) base_subparser.add_argument( - '--project-dir', + "--project-dir", default=None, type=str, help=""" Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents. - """ + """, ) base_subparser.add_argument( - '--profiles-dir', - default=PROFILES_DIR, + "--profiles-dir", + default=None, + dest="sub_profiles_dir", # Main cli arg precedes subcommand type=str, help=""" Which directory to look in for the profiles.yml file. Default = {} - """.format(PROFILES_DIR) + """.format( + DEFAULT_PROFILES_DIR + ), ) base_subparser.add_argument( - '--profile', + "--profile", required=False, type=str, help=""" Which profile to load. Overrides setting in dbt_project.yml. - """ + """, ) base_subparser.add_argument( - '--target', + "-t", + "--target", default=None, type=str, - help='Which target to load for the given profile' + help=""" + Which target to load for the given profile + """, ) base_subparser.add_argument( - '--vars', + "--vars", type=str, - default='{}', + default="{}", help=""" - Supply variables to the project. This argument overrides - variables defined in your dbt_project.yml file. This argument - should be a YAML string, eg. '{my_variable: my_value}'""" + Supply variables to the project. This argument overrides variables + defined in your dbt_project.yml file. This argument should be a YAML + string, eg. '{my_variable: my_value}' + """, ) # if set, log all cache events. This is extremely verbose! base_subparser.add_argument( - '--log-cache-events', - action='store_true', + "--log-cache-events", + action="store_true", help=argparse.SUPPRESS, ) - base_subparser.add_argument( - '--bypass-cache', - action='store_false', - dest='use_cache', - help='If set, bypass the adapter-level cache of database state', - ) + base_subparser.set_defaults(defer=None, state=None) return base_subparser def _build_docs_subparser(subparsers, base_subparser): docs_sub = subparsers.add_parser( - 'docs', - parents=[base_subparser], - help="Generate or serve the documentation " - "website for your project.") + "docs", + help=""" + Generate or serve the documentation website for your project. + """, + ) return docs_sub def _build_source_subparser(subparsers, base_subparser): source_sub = subparsers.add_parser( - 'source', - parents=[base_subparser], - help="Manage your project's sources") + "source", + help=""" + Manage your project's sources + """, + ) return source_sub def _build_init_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'init', + "init", + parents=[base_subparser], + help=""" + Initialize a new DBT project. + """, + ) + sub.add_argument( + "project_name", + nargs="?", + help=""" + Name of the new DBT project. + """, + ) + sub.add_argument( + "-s", + "--skip-profile-setup", + dest="skip_profile_setup", + action="store_true", + help=""" + Skip interative profile setup. + """, + ) + sub.set_defaults(cls=init_task.InitTask, which="init", rpc_method=None) + return sub + + +def _build_build_subparser(subparsers, base_subparser): + sub = subparsers.add_parser( + "build", parents=[base_subparser], - help="Initialize a new DBT project.") - sub.add_argument('project_name', type=str, help='Name of the new project') - sub.set_defaults(cls=init_task.InitTask, which='init') + help=""" + Run all Seeds, Models, Snapshots, and tests in DAG order + """, + ) + sub.set_defaults(cls=build_task.BuildTask, which="build", rpc_method="build") + sub.add_argument( + "-x", + "--fail-fast", + dest="sub_fail_fast", + action="store_true", + help=""" + Stop execution upon a first failure. + """, + ) + sub.add_argument( + "--store-failures", + action="store_true", + help=""" + Store test results (failing rows) in the database + """, + ) + sub.add_argument( + "--indirect-selection", + choices=["eager", "cautious"], + default="eager", + dest="indirect_selection", + help=""" + Select all tests that are adjacent to selected resources, + even if they those resources have been explicitly selected. + """, + ) + + resource_values: List[str] = [str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES] + [ + "all" + ] + sub.add_argument( + "--resource-type", + choices=resource_values, + action="append", + default=[], + dest="resource_types", + ) + # explicity don't support --models + sub.add_argument( + "-s", + "--select", + dest="select", + nargs="+", + help=""" + Specify the nodes to include. + """, + ) + _add_common_selector_arguments(sub) return sub def _build_clean_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'clean', + "clean", parents=[base_subparser], - help="Delete all folders in the clean-targets list" - "\n(usually the dbt_modules and target directories.)") - sub.set_defaults(cls=clean_task.CleanTask, which='clean') + help=""" + Delete all folders in the clean-targets list + (usually the dbt_packages and target directories.) + """, + ) + sub.set_defaults(cls=clean_task.CleanTask, which="clean", rpc_method=None) return sub def _build_debug_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'debug', + "debug", parents=[base_subparser], - help="Show some helpful information about dbt for debugging." - "\nNot to be confused with the --debug option which increases " - "verbosity.") + help=""" + Show some helpful information about dbt for debugging. + + Not to be confused with the --debug option which increases verbosity. + """, + ) sub.add_argument( - '--config-dir', - action='store_true', + "--config-dir", + action="store_true", help=""" If specified, DBT will show path information for this project - """ + """, ) - sub.set_defaults(cls=debug_task.DebugTask, which='debug') + _add_version_check(sub) + sub.set_defaults(cls=debug_task.DebugTask, which="debug", rpc_method=None) return sub def _build_deps_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'deps', + "deps", parents=[base_subparser], - help="Pull the most recent version of the dependencies " - "listed in packages.yml") - sub.set_defaults(cls=deps_task.DepsTask, which='deps') + help=""" + Pull the most recent version of the dependencies listed in packages.yml + """, + ) + sub.set_defaults(cls=deps_task.DepsTask, which="deps", rpc_method="deps") return sub -def _build_snapshot_subparser(subparsers, base_subparser, which='snapshot'): - if which == 'archive': - helpmsg = ( - 'DEPRECATED: This command is deprecated and will\n' - 'be removed in a future release. Use dbt snapshot instead.' - ) - else: - helpmsg = 'Execute snapshots defined in your project' - +def _build_snapshot_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - which, + "snapshot", parents=[base_subparser], - help=helpmsg) + help=""" + Execute snapshots defined in your project + """, + ) sub.add_argument( - '--threads', + "--threads", type=int, required=False, help=""" - Specify number of threads to use while snapshotting tables. Overrides - settings in profiles.yml. - """ + Specify number of threads to use while snapshotting tables. + Overrides settings in profiles.yml. + """, ) - sub.set_defaults(cls=snapshot_task.SnapshotTask, which=which) + sub.set_defaults(cls=snapshot_task.SnapshotTask, which="snapshot", rpc_method="snapshot") return sub +def _add_defer_argument(*subparsers): + for sub in subparsers: + sub.add_optional_argument_inverse( + "--defer", + enable_help=""" + If set, defer to the state variable for resolving unselected nodes. + """, + disable_help=""" + If set, do not defer to the state variable for resolving unselected + nodes. + """, + default=flags.DEFER_MODE, + ) + + def _build_run_subparser(subparsers, base_subparser): run_sub = subparsers.add_parser( - 'run', + "run", parents=[base_subparser], - help="Compile SQL and execute against the current " - "target database.") - run_sub.set_defaults(cls=run_task.RunTask, which='run') + help=""" + Compile SQL and execute against the current target database. + """, + ) + run_sub.add_argument( + "-x", + "--fail-fast", + dest="sub_fail_fast", + action="store_true", + help=""" + Stop execution upon a first failure. + """, + ) + + run_sub.set_defaults(cls=run_task.RunTask, which="run", rpc_method="run") return run_sub def _build_compile_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'compile', + "compile", parents=[base_subparser], - help="Generates executable SQL from source model, test, and" - "analysis files. \nCompiled SQL files are written to the target/" - "directory.") - sub.set_defaults(cls=compile_task.CompileTask, which='compile') + help=""" + Generates executable SQL from source, model, test, and analysis files. + Compiled SQL files are written to the target/ directory. + """, + ) + sub.set_defaults(cls=compile_task.CompileTask, which="compile", rpc_method="compile") + sub.add_argument("--parse-only", action="store_true") + return sub + + +def _build_parse_subparser(subparsers, base_subparser): + sub = subparsers.add_parser( + "parse", + parents=[base_subparser], + help=""" + Parses the project and provides information on performance + """, + ) + sub.set_defaults(cls=parse_task.ParseTask, which="parse", rpc_method="parse") + sub.add_argument("--write-manifest", action="store_true") + sub.add_argument("--compile", action="store_true") return sub def _build_docs_generate_subparser(subparsers, base_subparser): # it might look like docs_sub is the correct parents entry, but that # will cause weird errors about 'conflicting option strings'. - generate_sub = subparsers.add_parser('generate', parents=[base_subparser]) - generate_sub.set_defaults(cls=generate_task.GenerateTask, - which='generate') + generate_sub = subparsers.add_parser("generate", parents=[base_subparser]) + generate_sub.set_defaults( + cls=generate_task.GenerateTask, which="generate", rpc_method="docs.generate" + ) generate_sub.add_argument( - '--no-compile', - action='store_false', - dest='compile', - help='Do not run "dbt compile" as part of docs generation' + "--no-compile", + action="store_false", + dest="compile", + help=""" + Do not run "dbt compile" as part of docs generation + """, ) return generate_sub -def _add_selection_arguments(*subparsers, **kwargs): - models_name = kwargs.get('models_name', 'models') +def _add_common_selector_arguments(sub): + sub.add_argument( + "--exclude", + required=False, + nargs="+", + help=""" + Specify the models to exclude. + """, + ) + sub.add_argument( + "--selector", + dest="selector_name", + metavar="SELECTOR_NAME", + help=""" + The selector name to use, as defined in selectors.yml + """, + ) + sub.add_argument( + "--state", + help=""" + If set, use the given directory as the source for json files to + compare with this project. + """, + type=Path, + default=flags.ARTIFACT_STATE_PATH, + ) + + +def _add_selection_arguments(*subparsers): for sub in subparsers: sub.add_argument( - '-{}'.format(models_name[0]), - '--{}'.format(models_name), - dest='models', - required=False, - nargs='+', + "-m", + "--models", + dest="select", + nargs="+", help=""" - Specify the models to include. - """ + Specify the nodes to include. + """, ) sub.add_argument( - '--exclude', - required=False, - nargs='+', + "-s", + "--select", + dest="select", + nargs="+", help=""" - Specify the models to exclude. - """ + Specify the nodes to include. + """, ) + _add_common_selector_arguments(sub) def _add_table_mutability_arguments(*subparsers): for sub in subparsers: sub.add_argument( - '--full-refresh', - action='store_true', + "--full-refresh", + action="store_true", help=""" - If specified, DBT will drop incremental models and + If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition. - """) + """, + ) + + +def _add_version_check(sub): + sub.add_argument( + "--no-version-check", + dest="sub_version_check", # main cli arg precedes subcommands + action="store_false", + default=None, + help=""" + If set, skip ensuring dbt's version matches the one specified in + the dbt_project.yml file ('require-dbt-version') + """, + ) def _add_common_arguments(*subparsers): for sub in subparsers: sub.add_argument( - '--threads', + "--threads", type=int, required=False, help=""" Specify number of threads to use while executing models. Overrides settings in profiles.yml. - """ + """, ) - sub.add_argument( - '--no-version-check', - dest='version_check', - action='store_false', - help=""" - If set, skip ensuring dbt's version matches the one specified in - the dbt_project.yml file ('require-dbt-version') - """) + _add_version_check(sub) def _build_seed_subparser(subparsers, base_subparser): seed_sub = subparsers.add_parser( - 'seed', + "seed", parents=[base_subparser], - help="Load data from csv files into your data warehouse.") + help=""" + Load data from csv files into your data warehouse. + """, + ) seed_sub.add_argument( - '--full-refresh', - action='store_true', - help='Drop existing seed tables and recreate them' + "--full-refresh", + action="store_true", + help=""" + Drop existing seed tables and recreate them + """, ) seed_sub.add_argument( - '--show', - action='store_true', - help='Show a sample of the loaded data in the terminal' + "--show", + action="store_true", + help=""" + Show a sample of the loaded data in the terminal + """, ) - seed_sub.set_defaults(cls=seed_task.SeedTask, which='seed') + seed_sub.set_defaults(cls=seed_task.SeedTask, which="seed", rpc_method="seed") return seed_sub def _build_docs_serve_subparser(subparsers, base_subparser): - serve_sub = subparsers.add_parser('serve', parents=[base_subparser]) + serve_sub = subparsers.add_parser("serve", parents=[base_subparser]) serve_sub.add_argument( - '--port', + "--port", default=8080, type=int, - help='Specify the port number for the docs server.' + help=""" + Specify the port number for the docs server. + """, + ) + serve_sub.add_argument( + "--no-browser", + dest="open_browser", + action="store_false", ) - serve_sub.set_defaults(cls=serve_task.ServeTask, which='serve') + serve_sub.set_defaults(cls=serve_task.ServeTask, which="serve", rpc_method=None) return serve_sub def _build_test_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'test', + "test", parents=[base_subparser], - help="Runs tests on data in deployed models." - "Run this after `dbt run`") + help=""" + Runs tests on data in deployed models. Run this after `dbt run` + """, + ) sub.add_argument( - '--data', - action='store_true', - help='Run data tests defined in "tests" directory.' + "-x", + "--fail-fast", + dest="sub_fail_fast", + action="store_true", + help=""" + Stop execution upon a first test failure. + """, ) sub.add_argument( - '--schema', - action='store_true', - help='Run constraint validations from schema.yml files' + "--store-failures", + action="store_true", + help=""" + Store test results (failing rows) in the database + """, + ) + sub.add_argument( + "--indirect-selection", + choices=["eager", "cautious"], + default="eager", + dest="indirect_selection", + help=""" + Select all tests that are adjacent to selected resources, + even if they those resources have been explicitly selected. + """, ) - sub.set_defaults(cls=test_task.TestTask, which='test') + sub.set_defaults(cls=test_task.TestTask, which="test", rpc_method="test") return sub -def _build_source_snapshot_freshness_subparser(subparsers, base_subparser): +def _build_source_freshness_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'snapshot-freshness', + "freshness", parents=[base_subparser], - help="Snapshots the current freshness of the project's sources", - ) - sub.add_argument( - '-s', - '--select', - required=False, - nargs='+', help=""" - Specify the sources to snapshot freshness + Snapshots the current freshness of the project's sources """, - dest='selected' + aliases=["snapshot-freshness"], ) sub.add_argument( - '-o', - '--output', + "-o", + "--output", required=False, help=""" Specify the output path for the json report. By default, outputs to target/sources.json - """ + """, ) sub.add_argument( - '--threads', + "--threads", type=int, required=False, help=""" Specify number of threads to use. Overrides settings in profiles.yml - """ + """, ) - sub.set_defaults(cls=freshness_task.FreshnessTask, - which='snapshot-freshness') - return sub - - -def _build_rpc_subparser(subparsers, base_subparser): - sub = subparsers.add_parser( - 'rpc', - parents=[base_subparser], - help='Start a json-rpc server', + sub.set_defaults( + cls=freshness_task.FreshnessTask, + which="source-freshness", + rpc_method="source-freshness", ) sub.add_argument( - '--host', - default='0.0.0.0', - help='Specify the host to listen on for the rpc server.' - ) - sub.add_argument( - '--port', - default=8580, - type=int, - help='Specify the port number for the rpc server.' + "-s", + "--select", + dest="select", + nargs="+", + help=""" + Specify the nodes to include. + """, ) - sub.set_defaults(cls=RPCServerTask, which='rpc') - # the rpc task does a 'compile', so we need these attributes to exist, but - # we don't want users to be allowed to set them. - sub.set_defaults(models=None, exclude=None) + _add_common_selector_arguments(sub) return sub def _build_list_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'list', + "list", parents=[base_subparser], - help='List the resources in your project' - ) - sub.set_defaults(cls=ListTask, which='list') - resource_values = list(ListTask.ALL_RESOURCE_VALUES) + ['default', 'all'] - sub.add_argument('--resource-type', - choices=resource_values, - action='append', - default=[], - dest='resource_types') - sub.add_argument('--output', - choices=['json', 'name', 'path', 'selector'], - default='selector') + help=""" + List the resources in your project + """, + aliases=["ls"], + ) + sub.set_defaults(cls=list_task.ListTask, which="list", rpc_method=None) + resource_values: List[str] = [str(s) for s in list_task.ListTask.ALL_RESOURCE_VALUES] + [ + "default", + "all", + ] sub.add_argument( - '-s', - '--select', - required=False, - nargs='+', - metavar='SELECTOR', - help="Specify the nodes to select.", + "--resource-type", + choices=resource_values, + action="append", + default=[], + dest="resource_types", ) + sub.add_argument("--output", choices=["json", "name", "path", "selector"], default="selector") + sub.add_argument("--output-keys") + sub.add_argument( - '-m', - '--models', + "-m", + "--models", + dest="models", + nargs="+", + help=""" + Specify the models to select and set the resource-type to 'model'. + Mutually exclusive with '--select' (or '-s') and '--resource-type' + """, + metavar="SELECTOR", required=False, - nargs='+', - metavar='SELECTOR', - help="Specify the models to select and set the resource-type to " - "'model'. Mutually exclusive with '--select' (or '-s') and " - "'--resource-type'", ) sub.add_argument( - '--exclude', + "-s", + "--select", + dest="select", + nargs="+", + help=""" + Specify the nodes to include. + """, + metavar="SELECTOR", required=False, - nargs='+', - metavar='SELECTOR', - help="Specify the models to exclude." ) - # in python 3.x you can use the 'aliases' kwarg, but in python 2.7 you get - # to do this - subparsers._name_parser_map['ls'] = sub + sub.add_argument( + "--indirect-selection", + choices=["eager", "cautious"], + default="eager", + dest="indirect_selection", + help=""" + Select all tests that are adjacent to selected resources, + even if they those resources have been explicitly selected. + """, + ) + _add_common_selector_arguments(sub) + return sub def _build_run_operation_subparser(subparsers, base_subparser): sub = subparsers.add_parser( - 'run-operation', + "run-operation", parents=[base_subparser], help=""" - (beta) Run the named macro with any supplied arguments. This - subcommand is unstable and subject to change in a future release - of dbt. Please use it with caution""" + Run the named macro with any supplied arguments. + """, ) sub.add_argument( - 'macro', + "macro", help=""" - Specify the macro to invoke. dbt will call this macro with the - supplied arguments and then exit""" + Specify the macro to invoke. dbt will call this macro with the supplied + arguments and then exit + """, ) sub.add_argument( - '--args', + "--args", type=str, - default='{}', + default="{}", help=""" - Supply arguments to the macro. This dictionary will be mapped - to the keyword arguments defined in the selected macro. This - argument should be a YAML string, eg. '{my_variable: my_value}'""" + Supply arguments to the macro. This dictionary will be mapped to the + keyword arguments defined in the selected macro. This argument should + be a YAML string, eg. '{my_variable: my_value}' + """, + ) + sub.set_defaults( + cls=run_operation_task.RunOperationTask, which="run-operation", rpc_method="run-operation" ) - sub.set_defaults(cls=run_operation_task.RunOperationTask, - which='run-operation') return sub -def _build_snapshot_migrate_subparser(subparsers, base_subparser): - sub = subparsers.add_parser( - 'snapshot-migrate', - parents=[base_subparser], - help='Run the snapshot migration script' - ) - sub.add_argument( - '--from-archive', - action='store_true', - help=('This flag is required for the 0.14.0 archive to snapshot ' - 'migration') - ) - sub.add_argument( - '--apply-files', - action='store_true', - dest='write_files', - help='If set, write .sql files to disk instead of logging them' - ) - sub.add_argument( - '--apply-database', - action='store_true', - dest='migrate_database', - help='If set, perform just the database migration' +def parse_args(args, cls=DBTArgumentParser): + p = cls( + prog="dbt", + description=""" + An ELT tool for managing your SQL transformations and data models. + For more documentation on these commands, visit: docs.getdbt.com + """, + epilog=""" + Specify one of these sub-commands and you can find more help from + there. + """, ) - sub.add_argument( - '--apply', - action='store_true', - help='If set, implies --apply-database --apply-files' + + p.add_argument( + "--version", + action="dbtversion", + help=""" + Show version information + """, ) - sub.set_defaults(cls=MigrationTask, which='migration') + p.add_argument( + "-r", + "--record-timing-info", + default=None, + type=str, + help=""" + When this option is passed, dbt will output low-level timing stats to + the specified file. Example: `--record-timing-info output.profile` + """, + ) -def parse_args(args): - p = DBTArgumentParser( - prog='dbt', - formatter_class=argparse.RawTextHelpFormatter, - description="An ELT tool for managing your SQL " - "transformations and data models." - "\nFor more documentation on these commands, visit: " - "docs.getdbt.com", - epilog="Specify one of these sub-commands and you can " - "find more help from there.") + p.add_argument( + "-d", + "--debug", + action="store_true", + default=None, + help=""" + Display debug logging during dbt execution. Useful for debugging and + making bug reports. + """, + ) p.add_argument( - '--version', - action='dbtversion', - help="Show version information") + "--log-format", + choices=["text", "json", "default"], + default=None, + help="""Specify the log format, overriding the command's default.""", + ) p.add_argument( - '-r', - '--record-timing-info', + "--no-write-json", + action="store_false", default=None, - type=str, + dest="write_json", + help=""" + If set, skip writing the manifest and run_results.json files to disk + """, + ) + colors_flag = p.add_mutually_exclusive_group() + colors_flag.add_argument( + "--use-colors", + action="store_const", + const=True, + default=None, + dest="use_colors", + help=""" + Colorize the output DBT prints to the terminal. Output is colorized by + default and may also be set in a profile or at the command line. + Mutually exclusive with --no-use-colors + """, + ) + colors_flag.add_argument( + "--no-use-colors", + action="store_const", + const=False, + dest="use_colors", help=""" - When this option is passed, dbt will output low-level timing - stats to the specified file. Example: - `--record-timing-info output.profile` - """ + Do not colorize the output DBT prints to the terminal. Output is + colorized by default and may also be set in a profile or at the + command line. + Mutually exclusive with --use-colors + """, ) p.add_argument( - '-d', - '--debug', - action='store_true', - help='''Display debug logging during dbt execution. Useful for - debugging and making bug reports.''') + "--printer-width", + dest="printer_width", + help=""" + Sets the width of terminal output + """, + ) p.add_argument( - '-S', - '--strict', - action='store_true', - help='''Run schema validations at runtime. This will surface - bugs in dbt, but may incur a performance penalty.''') + "--warn-error", + action="store_true", + default=None, + help=""" + If dbt would normally warn, instead raise an exception. Examples + include --models that selects nothing, deprecations, configurations + with no associated models, invalid test configurations, and missing + sources/refs in tests. + """, + ) p.add_argument( - '--warn-error', - action='store_true', - help='''If dbt would normally warn, instead raise an exception. - Examples include --models that selects nothing, deprecations, - configurations with no associated models, invalid test configurations, - and missing sources/refs in tests''') + "--no-version-check", + dest="version_check", + action="store_false", + default=None, + help=""" + If set, skip ensuring dbt's version matches the one specified in + the dbt_project.yml file ('require-dbt-version') + """, + ) + + p.add_optional_argument_inverse( + "--partial-parse", + enable_help=""" + Allow for partial parsing by looking for and writing to a pickle file + in the target directory. This overrides the user configuration file. + """, + disable_help=""" + Disallow partial parsing. This overrides the user configuration file. + """, + ) # if set, run dbt in single-threaded mode: thread count is ignored, and # calls go through `map` instead of the thread pool. This is useful for @@ -742,18 +1008,114 @@ def parse_args(args): # a thread, as the profiler ignores child threads. Users should really # never use this. p.add_argument( - '--single-threaded', - action='store_true', + "--single-threaded", + action="store_true", help=argparse.SUPPRESS, ) - # if set, extract all models and blocks with the jinja block extractor, and - # verify that we don't fail anywhere the actual jinja parser passes. The - # reverse (passing files that ends up failing jinja) is fine. + # if set, will use the latest features from the static parser instead of + # the stable static parser. + p.add_argument( + "--use-experimental-parser", + action="store_true", + default=None, + help=""" + Enables experimental parsing features. + """, + ) + + # if set, will disable the use of the stable static parser and instead + # always rely on jinja rendering. + p.add_argument( + "--no-static-parser", + default=None, + dest="static_parser", + action="store_false", + help=""" + Disables the static parser. + """, + ) + p.add_argument( - '--test-new-parser', - action='store_true', - help=argparse.SUPPRESS + "--profiles-dir", + default=None, + dest="profiles_dir", + type=str, + help=""" + Which directory to look in for the profiles.yml file. Default = {} + """.format( + DEFAULT_PROFILES_DIR + ), + ) + + p.add_argument( + "--no-anonymous-usage-stats", + action="store_false", + default=None, + dest="send_anonymous_usage_stats", + help=""" + Do not send anonymous usage stat to dbt Labs + """, + ) + + p.add_argument( + "-x", + "--fail-fast", + dest="fail_fast", + action="store_true", + default=None, + help=""" + Stop execution upon a first failure. + """, + ) + + p.add_argument( + "--event-buffer-size", + dest="event_buffer_size", + help=""" + Sets the max number of events to buffer in EVENT_HISTORY + """, + ) + + p.add_argument( + "-q", + "--quiet", + action="store_true", + default=None, + help=""" + Suppress all non-error logging to stdout. Does not affect + {{ print() }} macro calls. + """, + ) + + p.add_argument( + "--no-print", + action="store_true", + default=None, + help=""" + Suppress all {{ print() }} macro calls. + """, + ) + + schema_cache_flag = p.add_mutually_exclusive_group() + schema_cache_flag.add_argument( + "--cache-selected-only", + action="store_const", + const=True, + default=None, + dest="cache_selected_only", + help=""" + Pre cache database objects relevant to selected resource only. + """, + ) + schema_cache_flag.add_argument( + "--no-cache-selected-only", + action="store_const", + const=False, + dest="cache_selected_only", + help=""" + Pre cache all database objects related to the project. + """, ) subs = p.add_subparsers(title="Available sub-commands") @@ -771,28 +1133,29 @@ def parse_args(args): _build_debug_subparser(subs, base_subparser) _build_deps_subparser(subs, base_subparser) _build_list_subparser(subs, base_subparser) - _build_snapshot_migrate_subparser(subs, base_subparser) + build_sub = _build_build_subparser(subs, base_subparser) snapshot_sub = _build_snapshot_subparser(subs, base_subparser) - archive_sub = _build_snapshot_subparser(subs, base_subparser, 'archive') - rpc_sub = _build_rpc_subparser(subs, base_subparser) run_sub = _build_run_subparser(subs, base_subparser) compile_sub = _build_compile_subparser(subs, base_subparser) + parse_sub = _build_parse_subparser(subs, base_subparser) generate_sub = _build_docs_generate_subparser(docs_subs, base_subparser) test_sub = _build_test_subparser(subs, base_subparser) + seed_sub = _build_seed_subparser(subs, base_subparser) # --threads, --no-version-check - _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub, - rpc_sub) - # --models, --exclude - _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub, - archive_sub) - _add_selection_arguments(snapshot_sub, models_name='select') + _add_common_arguments( + run_sub, compile_sub, generate_sub, test_sub, seed_sub, parse_sub, build_sub + ) + # --select, --exclude + # list_sub sets up its own arguments. + _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub, snapshot_sub, seed_sub) + # --defer + _add_defer_argument(run_sub, test_sub, build_sub, snapshot_sub) # --full-refresh - _add_table_mutability_arguments(run_sub, compile_sub) + _add_table_mutability_arguments(run_sub, compile_sub, build_sub) - _build_seed_subparser(subs, base_subparser) _build_docs_serve_subparser(docs_subs, base_subparser) - _build_source_snapshot_freshness_subparser(source_subs, base_subparser) + _build_source_freshness_subparser(source_subs, base_subparser) _build_run_operation_subparser(subs, base_subparser) if len(args) == 0: @@ -800,12 +1163,41 @@ def parse_args(args): sys.exit(1) parsed = p.parse_args(args) - parsed.profiles_dir = os.path.expanduser(parsed.profiles_dir) - if not hasattr(parsed, 'which'): + # profiles_dir is set before subcommands and after, so normalize + if hasattr(parsed, "sub_profiles_dir"): + if parsed.sub_profiles_dir is not None: + parsed.profiles_dir = parsed.sub_profiles_dir + delattr(parsed, "sub_profiles_dir") + if hasattr(parsed, "profiles_dir"): + if parsed.profiles_dir is None: + parsed.profiles_dir = flags.PROFILES_DIR + else: + parsed.profiles_dir = os.path.abspath(parsed.profiles_dir) + # needs to be set before the other flags, because it's needed to + # read the profile that contains them + flags.PROFILES_DIR = parsed.profiles_dir + + # version_check is set before subcommands and after, so normalize + if hasattr(parsed, "sub_version_check"): + if parsed.sub_version_check is False: + parsed.version_check = False + delattr(parsed, "sub_version_check") + + # fail_fast is set before subcommands and after, so normalize + if hasattr(parsed, "sub_fail_fast"): + if parsed.sub_fail_fast is True: + parsed.fail_fast = True + delattr(parsed, "sub_fail_fast") + + if getattr(parsed, "project_dir", None) is not None: + expanded_user = os.path.expanduser(parsed.project_dir) + parsed.project_dir = os.path.abspath(expanded_user) + + if not hasattr(parsed, "which"): # the user did not provide a valid subcommand. trigger the help message # and exit with a error p.print_help() p.exit(1) - return parsed + return parsed \ No newline at end of file From 2a16de5ee1568b3fe3c61ace7172917e9cb8a332 Mon Sep 17 00:00:00 2001 From: pquadri Date: Fri, 17 Jun 2022 11:43:35 +0200 Subject: [PATCH 5/6] chore: newline --- core/dbt/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/main.py b/core/dbt/main.py index 6111d0a5f7b..af30854fade 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -1200,4 +1200,4 @@ def parse_args(args, cls=DBTArgumentParser): p.print_help() p.exit(1) - return parsed \ No newline at end of file + return parsed From abeddc56511bb72bbe44e8493c0df74e8674cd3e Mon Sep 17 00:00:00 2001 From: pquadri Date: Fri, 17 Jun 2022 18:44:29 +0200 Subject: [PATCH 6/6] chore: changie --- .changes/unreleased/Fixes-20220617-114443.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Fixes-20220617-114443.yaml diff --git a/.changes/unreleased/Fixes-20220617-114443.yaml b/.changes/unreleased/Fixes-20220617-114443.yaml new file mode 100644 index 00000000000..4e3854d496f --- /dev/null +++ b/.changes/unreleased/Fixes-20220617-114443.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Properly use quotes for Snowflake snapshots when checking all columns +time: 2022-06-17T11:44:43.978834+02:00 +custom: + Author: pquadri + Issue: "2975" + PR: "5389"