diff --git a/dlt/cli/_dlt.py b/dlt/cli/_dlt.py index 4b7f217e24..ac7f5c1b5b 100644 --- a/dlt/cli/_dlt.py +++ b/dlt/cli/_dlt.py @@ -1,11 +1,13 @@ from typing import Any, Sequence, Type, cast, List, Dict import argparse +import click from dlt.version import __version__ from dlt.common.runners import Venv from dlt.cli import SupportsCliCommand import dlt.cli.echo as fmt +from dlt.cli.exceptions import CliCommandException from dlt.cli.command_wrappers import ( deploy_command_wrapper, @@ -15,6 +17,7 @@ ACTION_EXECUTED = False +DEFAULT_DOCS_URL = "https://dlthub.com/docs/intro" def print_help(parser: argparse.ArgumentParser) -> None: @@ -153,12 +156,35 @@ def main() -> int: " the current virtual environment instead." ) - if args.command in installed_commands: - return installed_commands[args.command].execute(args) + if cmd := installed_commands.get(args.command): + try: + cmd.execute(args) + except Exception as ex: + docs_url = cmd.docs_url if hasattr(cmd, "docs_url") else DEFAULT_DOCS_URL + error_code = -1 + raiseable_exception = ex + + # overwrite some values if this is a CliCommandException + if isinstance(ex, CliCommandException): + error_code = ex.error_code + docs_url = ex.docs_url or docs_url + raiseable_exception = ex.raiseable_exception + + # print exception if available + if raiseable_exception: + click.secho(str(ex), err=True, fg="red") + + fmt.note("Please refer to our docs at '%s' for further assistance." % docs_url) + if debug.is_debug_enabled() and raiseable_exception: + raise raiseable_exception + + return error_code else: print_help(parser) return -1 + return 0 + def _main() -> None: """Script entry point""" diff --git a/dlt/cli/command_wrappers.py b/dlt/cli/command_wrappers.py index 6b98bac0e1..0e6491688e 100644 --- a/dlt/cli/command_wrappers.py +++ b/dlt/cli/command_wrappers.py @@ -11,6 +11,7 @@ import dlt.cli.echo as fmt from dlt.cli import utils from dlt.pipeline.exceptions import CannotRestorePipelineException +from dlt.cli.exceptions import CliCommandException from dlt.cli.init_command import ( init_command, @@ -29,17 +30,11 @@ from dlt.cli import deploy_command from dlt.cli.deploy_command import ( PipelineWasNotRun, - DLT_DEPLOY_DOCS_URL, ) except ModuleNotFoundError: pass - -def on_exception(ex: Exception, info: str) -> None: - click.secho(str(ex), err=True, fg="red") - fmt.note("Please refer to %s for further assistance" % fmt.bold(info)) - if debug.is_debug_enabled(): - raise ex +DLT_DEPLOY_DOCS_URL = "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline" @utils.track_command("init", False, "source_name", "destination_type") @@ -49,48 +44,34 @@ def init_command_wrapper( repo_location: str, branch: str, omit_core_sources: bool = False, -) -> int: - try: - init_command( - source_name, - destination_type, - repo_location, - branch, - omit_core_sources, - ) - except Exception as ex: - on_exception(ex, DLT_INIT_DOCS_URL) - return -1 - return 0 +) -> None: + init_command( + source_name, + destination_type, + repo_location, + branch, + omit_core_sources, + ) @utils.track_command("list_sources", False) -def list_sources_command_wrapper(repo_location: str, branch: str) -> int: - try: - list_sources_command(repo_location, branch) - except Exception as ex: - on_exception(ex, DLT_INIT_DOCS_URL) - return -1 - return 0 +def list_sources_command_wrapper(repo_location: str, branch: str) -> None: + list_sources_command(repo_location, branch) @utils.track_command("pipeline", True, "operation") def pipeline_command_wrapper( operation: str, pipeline_name: str, pipelines_dir: str, verbosity: int, **command_kwargs: Any -) -> int: +) -> None: try: pipeline_command(operation, pipeline_name, pipelines_dir, verbosity, **command_kwargs) - return 0 except CannotRestorePipelineException as ex: click.secho(str(ex), err=True, fg="red") click.secho( "Try command %s to restore the pipeline state from destination" % fmt.bold(f"dlt pipeline {pipeline_name} sync") ) - return -1 - except Exception as ex: - on_exception(ex, DLT_PIPELINE_COMMAND_DOCS_URL) - return -2 + raise CliCommandException(error_code=-2) @utils.track_command("deploy", False, "deployment_method") @@ -100,13 +81,12 @@ def deploy_command_wrapper( repo_location: str, branch: Optional[str] = None, **kwargs: Any, -) -> int: +) -> None: try: utils.ensure_git_command("deploy") except Exception as ex: click.secho(str(ex), err=True, fg="red") - return -1 - + raise CliCommandException(error_code=-2) from git import InvalidGitRepositoryError, NoSuchPathError try: @@ -121,8 +101,7 @@ def deploy_command_wrapper( fmt.note( "You must run the pipeline locally successfully at least once in order to deploy it." ) - on_exception(ex, DLT_DEPLOY_DOCS_URL) - return -2 + raise CliCommandException(error_code=-3, raiseable_exception=ex) except InvalidGitRepositoryError: click.secho( "No git repository found for pipeline script %s." % fmt.bold(pipeline_script_path), @@ -140,18 +119,14 @@ def deploy_command_wrapper( ) ) fmt.note("Please refer to %s for further assistance" % fmt.bold(DLT_DEPLOY_DOCS_URL)) - return -3 + raise CliCommandException(error_code=-4) except NoSuchPathError as path_ex: click.secho("The pipeline script does not exist\n%s" % str(path_ex), err=True, fg="red") - return -4 - except Exception as ex: - on_exception(ex, DLT_DEPLOY_DOCS_URL) - return -5 - return 0 + raise CliCommandException(error_code=-5) @utils.track_command("schema", False, "operation") -def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) -> int: +def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) -> None: with open(file_path, "rb") as f: if os.path.splitext(file_path)[1][1:] == "json": schema_dict: DictStrAny = json.load(f) @@ -163,24 +138,16 @@ def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) else: schema_str = s.to_pretty_yaml(remove_defaults=remove_defaults) fmt.echo(schema_str) - return 0 @utils.track_command("telemetry", False) -def telemetry_status_command_wrapper() -> int: - try: - telemetry_status_command() - except Exception as ex: - on_exception(ex, DLT_TELEMETRY_DOCS_URL) - return -1 - return 0 +def telemetry_status_command_wrapper() -> None: + telemetry_status_command() @utils.track_command("telemetry_switch", False, "enabled") -def telemetry_change_status_command_wrapper(enabled: bool) -> int: +def telemetry_change_status_command_wrapper(enabled: bool) -> None: try: change_telemetry_status_command(enabled) except Exception as ex: - on_exception(ex, DLT_TELEMETRY_DOCS_URL) - return -1 - return 0 + raise CliCommandException(docs_url=DLT_TELEMETRY_DOCS_URL, raiseable_exception=ex) diff --git a/dlt/cli/deploy_command.py b/dlt/cli/deploy_command.py index 88c132f5e2..a397abcd4f 100644 --- a/dlt/cli/deploy_command.py +++ b/dlt/cli/deploy_command.py @@ -26,7 +26,6 @@ from dlt.common.destination.reference import Destination REQUIREMENTS_GITHUB_ACTION = "requirements_github_action.txt" -DLT_DEPLOY_DOCS_URL = "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline" DLT_AIRFLOW_GCP_DOCS_URL = ( "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer" ) diff --git a/dlt/cli/deploy_command_helpers.py b/dlt/cli/deploy_command_helpers.py index 38e95ce5d0..89f13ae4e1 100644 --- a/dlt/cli/deploy_command_helpers.py +++ b/dlt/cli/deploy_command_helpers.py @@ -35,7 +35,7 @@ from dlt.cli import utils from dlt.cli import echo as fmt -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException GITHUB_URL = "https://github.com/" @@ -99,14 +99,14 @@ def _get_origin(self) -> str: try: origin = get_origin(self.repo) if "github.com" not in origin: - raise CliCommandException( + raise CliCommandInnerException( "deploy", f"Your current repository origin is not set to github but to {origin}.\nYou" " must change it to be able to run the pipelines with github actions:" " https://docs.github.com/en/get-started/getting-started-with-git/managing-remote-repositories", ) except ValueError: - raise CliCommandException( + raise CliCommandInnerException( "deploy", "Your current repository has no origin set. Please set it up to be able to run the" " pipelines with github actions:" @@ -293,7 +293,7 @@ def get_state_and_trace(pipeline: Pipeline) -> Tuple[TPipelineState, PipelineTra def get_visitors(pipeline_script: str, pipeline_script_path: str) -> PipelineScriptVisitor: visitor = utils.parse_init_script("deploy", pipeline_script, pipeline_script_path) if n.RUN not in visitor.known_calls: - raise CliCommandException( + raise CliCommandInnerException( "deploy", f"The pipeline script {pipeline_script_path} does not seem to run the pipeline.", ) @@ -323,13 +323,13 @@ def parse_pipeline_info(visitor: PipelineScriptVisitor) -> List[Tuple[str, Optio " abort to set it to False?", default=True, ): - raise CliCommandException("deploy", "Please set the dev_mode to False") + raise CliCommandInnerException("deploy", "Please set the dev_mode to False") p_d_node = call_args.arguments.get("pipelines_dir") if p_d_node: pipelines_dir = evaluate_node_literal(p_d_node) if pipelines_dir is None: - raise CliCommandException( + raise CliCommandInnerException( "deploy", "The value of 'pipelines_dir' argument in call to `dlt_pipeline` cannot be" f" determined from {unparse(p_d_node).strip()}. Pipeline working dir will" @@ -340,7 +340,7 @@ def parse_pipeline_info(visitor: PipelineScriptVisitor) -> List[Tuple[str, Optio if p_n_node: pipeline_name = evaluate_node_literal(p_n_node) if pipeline_name is None: - raise CliCommandException( + raise CliCommandInnerException( "deploy", "The value of 'pipeline_name' argument in call to `dlt_pipeline` cannot be" f" determined from {unparse(p_d_node).strip()}. Pipeline working dir will" @@ -439,9 +439,9 @@ def ask_files_overwrite(files: Sequence[str]) -> None: if existing: fmt.echo("Following files will be overwritten: %s" % fmt.bold(str(existing))) if not fmt.confirm("Do you want to continue?", default=False): - raise CliCommandException("init", "Aborted") + raise CliCommandInnerException("init", "Aborted") -class PipelineWasNotRun(CliCommandException): +class PipelineWasNotRun(CliCommandInnerException): def __init__(self, msg: str) -> None: super().__init__("deploy", msg, None) diff --git a/dlt/cli/exceptions.py b/dlt/cli/exceptions.py index d69066207e..a12f7e7243 100644 --- a/dlt/cli/exceptions.py +++ b/dlt/cli/exceptions.py @@ -1,13 +1,27 @@ from dlt.common.exceptions import DltException -class CliCommandException(DltException): +class CliCommandInnerException(DltException): def __init__(self, cmd: str, msg: str, inner_exc: Exception = None) -> None: self.cmd = cmd self.inner_exc = inner_exc super().__init__(msg) +class CliCommandException(DltException): + """ + Exception that can be thrown inside a cli command and can change the + error code or docs url presented to the user. Will always be caught. + """ + + def __init__( + self, error_code: int = -1, docs_url: str = None, raiseable_exception: Exception = None + ) -> None: + self.error_code = error_code + self.docs_url = docs_url + self.raiseable_exception = raiseable_exception + + class VerifiedSourceRepoError(DltException): def __init__(self, msg: str, source_name: str) -> None: self.source_name = source_name diff --git a/dlt/cli/init_command.py b/dlt/cli/init_command.py index 0d3b5fe99e..51cebad384 100644 --- a/dlt/cli/init_command.py +++ b/dlt/cli/init_command.py @@ -36,7 +36,7 @@ TVerifiedSourceFileEntry, TVerifiedSourceFileIndex, ) -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException DLT_INIT_DOCS_URL = "https://dlthub.com/docs/reference/command-line-interface#dlt-init" @@ -428,14 +428,14 @@ def init_command( source_configuration.src_pipeline_script, ) if visitor.is_destination_imported: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {source_configuration.src_pipeline_script} imports a destination" " from dlt.destinations. You should specify destinations by name when calling" " dlt.pipeline or dlt.run in init scripts.", ) if n.PIPELINE not in visitor.known_calls: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {source_configuration.src_pipeline_script} does not seem to" " initialize a pipeline with dlt.pipeline. Please initialize pipeline explicitly in" @@ -498,7 +498,7 @@ def init_command( (known_sections.SOURCES, source_name), ) if len(checked_sources) == 0: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {source_configuration.src_pipeline_script} is not creating or" " importing any sources or resources. Exiting...", @@ -552,7 +552,7 @@ def init_command( ) if not fmt.confirm("Do you want to proceed?", default=True): - raise CliCommandException("init", "Aborted") + raise CliCommandInnerException("init", "Aborted") dependency_system = _get_dependency_system(dest_storage) _welcome_message( diff --git a/dlt/cli/pipeline_command.py b/dlt/cli/pipeline_command.py index 6aa479a398..d879281808 100644 --- a/dlt/cli/pipeline_command.py +++ b/dlt/cli/pipeline_command.py @@ -1,7 +1,7 @@ import yaml from typing import Any, Optional, Sequence, Tuple import dlt -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.common.json import json from dlt.common.pipeline import resource_state, get_dlt_pipelines_dir, TSourceState @@ -21,7 +21,9 @@ from dlt.cli import echo as fmt -DLT_PIPELINE_COMMAND_DOCS_URL = "https://dlthub.com/docs/reference/command-line-interface" +DLT_PIPELINE_COMMAND_DOCS_URL = ( + "https://dlthub.com/docs/reference/command-line-interface#dlt-pipeline" +) def pipeline_command( @@ -294,7 +296,7 @@ def _display_pending_packages() -> Tuple[Sequence[str], Sequence[str]]: if not packages: packages = sorted(p.list_completed_load_packages()) if not packages: - raise CliCommandException( + raise CliCommandInnerException( "pipeline", "There are no load packages for that pipeline" ) load_id = packages[-1] diff --git a/dlt/cli/plugins.py b/dlt/cli/plugins.py index 2041d6b369..cc2d4594b9 100644 --- a/dlt/cli/plugins.py +++ b/dlt/cli/plugins.py @@ -9,6 +9,7 @@ from dlt.cli.init_command import ( DEFAULT_VERIFIED_SOURCES_REPO, ) +from dlt.cli.exceptions import CliCommandException from dlt.cli.command_wrappers import ( init_command_wrapper, list_sources_command_wrapper, @@ -17,7 +18,12 @@ telemetry_status_command_wrapper, deploy_command_wrapper, ) -from dlt.cli.pipeline_command import DLT_PIPELINE_COMMAND_DOCS_URL +from dlt.cli.command_wrappers import ( + DLT_PIPELINE_COMMAND_DOCS_URL, + DLT_INIT_DOCS_URL, + DLT_TELEMETRY_DOCS_URL, + DLT_DEPLOY_DOCS_URL, +) try: from dlt.cli.deploy_command import ( @@ -42,6 +48,7 @@ class InitCommand(SupportsCliCommand): "Creates a pipeline project in the current folder by adding existing verified source or" " creating a new one from template." ) + docs_url = DLT_INIT_DOCS_URL def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser @@ -87,15 +94,15 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None: ), ) - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: if args.list_sources: - return list_sources_command_wrapper(args.location, args.branch) + list_sources_command_wrapper(args.location, args.branch) else: if not args.source or not args.destination: self.parser.print_usage() - return -1 + raise CliCommandException() else: - return init_command_wrapper( + init_command_wrapper( args.source, args.destination, args.location, @@ -107,6 +114,7 @@ def execute(self, args: argparse.Namespace) -> int: class PipelineCommand(SupportsCliCommand): command = "pipeline" help_string = "Operations on pipelines that were ran locally" + docs_url = DLT_PIPELINE_COMMAND_DOCS_URL def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None: self.parser = pipe_cmd @@ -241,23 +249,24 @@ def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None: help="Load id of completed or normalized package. Defaults to the most recent package.", ) - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: if args.list_pipelines: - return pipeline_command_wrapper("list", "-", args.pipelines_dir, args.verbosity) + pipeline_command_wrapper("list", "-", args.pipelines_dir, args.verbosity) else: command_kwargs = dict(args._get_kwargs()) if not command_kwargs.get("pipeline_name"): self.parser.print_usage() - return -1 + raise CliCommandException(error_code=-1) command_kwargs["operation"] = args.operation or "info" del command_kwargs["command"] del command_kwargs["list_pipelines"] - return pipeline_command_wrapper(**command_kwargs) + pipeline_command_wrapper(**command_kwargs) class SchemaCommand(SupportsCliCommand): command = "schema" help_string = "Shows, converts and upgrades schemas" + docs_url = "https://dlthub.com/docs/reference/command-line-interface#dlt-schema" def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser @@ -279,26 +288,26 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None: default=True, ) - def execute(self, args: argparse.Namespace) -> int: - return schema_command_wrapper(args.file, args.format, args.remove_defaults) + def execute(self, args: argparse.Namespace) -> None: + schema_command_wrapper(args.file, args.format, args.remove_defaults) class TelemetryCommand(SupportsCliCommand): command = "telemetry" help_string = "Shows telemetry status" + docs_url = DLT_TELEMETRY_DOCS_URL def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser - def execute(self, args: argparse.Namespace) -> int: - return telemetry_status_command_wrapper() + def execute(self, args: argparse.Namespace) -> None: + telemetry_status_command_wrapper() -# TODO: ensure the command reacts the correct way if dependencies are not installed -# thsi has changed a bit in this impl class DeployCommand(SupportsCliCommand): command = "deploy" help_string = "Creates a deployment package for a selected pipeline script" + docs_url = DLT_DEPLOY_DOCS_URL def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser @@ -368,7 +377,7 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None: help="Format of the secrets", ) - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: # exit if deploy command is not available if not deploy_command_available: fmt.warning( @@ -379,14 +388,14 @@ def execute(self, args: argparse.Namespace) -> int: "We ask you to install those dependencies separately to keep our core library small" " and make it work everywhere." ) - return -1 + raise CliCommandException() deploy_args = vars(args) if deploy_args.get("deployment_method") is None: self.parser.print_help() - return -1 + raise CliCommandException() else: - return deploy_command_wrapper( + deploy_command_wrapper( pipeline_script_path=deploy_args.pop("pipeline_script_path"), deployment_method=deploy_args.pop("deployment_method"), repo_location=deploy_args.pop("location"), diff --git a/dlt/cli/reference.py b/dlt/cli/reference.py index fd4fbb35f7..dd4bf69fe6 100644 --- a/dlt/cli/reference.py +++ b/dlt/cli/reference.py @@ -1,4 +1,4 @@ -from typing import Protocol +from typing import Protocol, Optional import argparse @@ -7,12 +7,16 @@ class SupportsCliCommand(Protocol): """Protocol for defining one dlt cli command""" command: str + """name of the command""" help_string: str + """the help string for argparse""" + docs_url: Optional[str] + """the default docs url to be printed in case of an exception""" def configure_parser(self, parser: argparse.ArgumentParser) -> None: """Configures the parser for the given argument""" ... - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: """Executes the command with the given arguments""" ... diff --git a/dlt/cli/source_detection.py b/dlt/cli/source_detection.py index 787f28881d..f4e9b3e050 100644 --- a/dlt/cli/source_detection.py +++ b/dlt/cli/source_detection.py @@ -10,7 +10,7 @@ from dlt.sources import SourceReference from dlt.cli.config_toml_writer import WritableConfigValue -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.reflection.script_visitor import PipelineScriptVisitor @@ -28,7 +28,7 @@ def find_call_arguments_to_replace( dn_node: ast.AST = args.arguments.get(t_arg_name) if dn_node is not None: if not isinstance(dn_node, ast.Constant) or not isinstance(dn_node.value, str): - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {init_script_name} must pass the {t_arg_name} as" f" string to '{arg_name}' function in line {dn_node.lineno}", @@ -40,7 +40,7 @@ def find_call_arguments_to_replace( # there was at least one replacement for t_arg_name, _ in replace_nodes: if t_arg_name not in replaced_args: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {init_script_name} is not explicitly passing the" f" '{t_arg_name}' argument to 'pipeline' or 'run' function. In init script the" diff --git a/dlt/cli/utils.py b/dlt/cli/utils.py index 9635348253..e0032ee7d3 100644 --- a/dlt/cli/utils.py +++ b/dlt/cli/utils.py @@ -11,7 +11,7 @@ from dlt.reflection.script_visitor import PipelineScriptVisitor -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException REQUIREMENTS_TXT = "requirements.txt" @@ -32,7 +32,7 @@ def parse_init_script( visitor = PipelineScriptVisitor(script_source) visitor.visit_passes(tree) if len(visitor.mod_aliases) == 0: - raise CliCommandException( + raise CliCommandInnerException( command, f"The pipeline script {init_script_name} does not import dlt and does not seem to run" " any pipelines", @@ -47,7 +47,7 @@ def ensure_git_command(command: str) -> None: except ImportError as imp_ex: if "Bad git executable" not in str(imp_ex): raise - raise CliCommandException( + raise CliCommandInnerException( command, "'git' command is not available. Install and setup git with the following the guide %s" % "https://docs.github.com/en/get-started/quickstart/set-up-git", diff --git a/docs/website/docs/reference/command-line-interface.md b/docs/website/docs/reference/command-line-interface.md index e29b43bcba..825d33d548 100644 --- a/docs/website/docs/reference/command-line-interface.md +++ b/docs/website/docs/reference/command-line-interface.md @@ -245,6 +245,25 @@ pending packages first. The command above removes such packages. Note that **pip were created. Using `dlt pipeline ... sync` is recommended if your destination supports state sync. +## `dlt schema` + +Will load, validate and print out a dlt schema. + +```sh +dlt schema path/to/my_schema_file.yaml +``` + +## `dlt telemetry` + +Shows the current status of dlt telemetry. + +```sh +dlt telemetry +``` + +Lern more about telemetry on the [telemetry reference page](./telemetry) + + ## Show stack traces If the command fails and you want to see the full stack trace, add `--debug` just after the `dlt` executable. ```sh diff --git a/tests/cli/common/test_cli_invoke.py b/tests/cli/common/test_cli_invoke.py index eef1af03ad..5631511f46 100644 --- a/tests/cli/common/test_cli_invoke.py +++ b/tests/cli/common/test_cli_invoke.py @@ -38,7 +38,7 @@ def test_invoke_basic(script_runner: ScriptRunner) -> None: def test_invoke_list_pipelines(script_runner: ScriptRunner) -> None: result = script_runner.run(["dlt", "pipeline", "--list-pipelines"]) # directory does not exist (we point to TEST_STORAGE) - assert result.returncode == -2 + assert result.returncode == -1 # create empty os.makedirs(get_dlt_pipelines_dir()) @@ -50,7 +50,7 @@ def test_invoke_list_pipelines(script_runner: ScriptRunner) -> None: def test_invoke_pipeline(script_runner: ScriptRunner) -> None: # info on non existing pipeline result = script_runner.run(["dlt", "pipeline", "debug_pipeline", "info"]) - assert result.returncode == -1 + assert result.returncode == -2 assert "the pipeline was not found in" in result.stderr # copy dummy pipeline @@ -75,7 +75,7 @@ def test_invoke_pipeline(script_runner: ScriptRunner) -> None: result = script_runner.run( ["dlt", "pipeline", "dummy_pipeline", "load-package", "NON EXISTENT"] ) - assert result.returncode == -2 + assert result.returncode == -1 try: # use debug flag to raise an exception result = script_runner.run( @@ -118,10 +118,10 @@ def test_invoke_deploy_project(script_runner: ScriptRunner) -> None: result = script_runner.run( ["dlt", "deploy", "debug_pipeline.py", "github-action", "--schedule", "@daily"] ) - assert result.returncode == -4 + assert result.returncode == -5 assert "The pipeline script does not exist" in result.stderr result = script_runner.run(["dlt", "deploy", "debug_pipeline.py", "airflow-composer"]) - assert result.returncode == -4 + assert result.returncode == -5 assert "The pipeline script does not exist" in result.stderr # now init result = script_runner.run(["dlt", "init", "chess", "dummy"]) diff --git a/tests/cli/common/test_telemetry_command.py b/tests/cli/common/test_telemetry_command.py index b0a3ff502c..24c91a7a9d 100644 --- a/tests/cli/common/test_telemetry_command.py +++ b/tests/cli/common/test_telemetry_command.py @@ -158,9 +158,12 @@ def test_instrumentation_wrappers() -> None: SENT_ITEMS.clear() with io.StringIO() as buf, contextlib.redirect_stderr(buf): - init_command_wrapper("instrumented_source", "", None, None) - output = buf.getvalue() - assert "is not one of the standard dlt destinations" in output + try: + init_command_wrapper("instrumented_source", "", None, None) + except Exception: + pass + # output = buf.getvalue() + # assert "is not one of the standard dlt destinations" in output msg = SENT_ITEMS[0] assert msg["event"] == "command_init" assert msg["properties"]["source_name"] == "instrumented_source" @@ -179,12 +182,15 @@ def test_instrumentation_wrappers() -> None: # assert msg["properties"]["operation"] == "list" SENT_ITEMS.clear() - deploy_command_wrapper( - "list.py", - DeploymentMethods.github_actions.value, - COMMAND_DEPLOY_REPO_LOCATION, - schedule="* * * * *", - ) + try: + deploy_command_wrapper( + "list.py", + DeploymentMethods.github_actions.value, + COMMAND_DEPLOY_REPO_LOCATION, + schedule="* * * * *", + ) + except Exception: + pass msg = SENT_ITEMS[0] assert msg["event"] == "command_deploy" assert msg["properties"]["deployment_method"] == DeploymentMethods.github_actions.value diff --git a/tests/cli/test_deploy_command.py b/tests/cli/test_deploy_command.py index 5d9163679a..db9ec2ac11 100644 --- a/tests/cli/test_deploy_command.py +++ b/tests/cli/test_deploy_command.py @@ -15,9 +15,10 @@ from dlt.common.utils import set_working_dir from dlt.cli import deploy_command, _dlt, echo -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.pipeline.exceptions import CannotRestorePipelineException from dlt.cli.deploy_command_helpers import get_schedule_description +from dlt.cli.exceptions import CliCommandException from tests.utils import TEST_STORAGE_ROOT, reset_providers, test_storage @@ -47,13 +48,14 @@ def test_deploy_command_no_repo( ) # test wrapper - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -3 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -4 @pytest.mark.parametrize("deployment_method,deployment_args", DEPLOY_PARAMS) @@ -72,7 +74,7 @@ def test_deploy_command( # we have a repo without git origin with Repo.init(".") as repo: # test no origin - with pytest.raises(CliCommandException) as py_ex: + with pytest.raises(CliCommandInnerException) as py_ex: deploy_command.deploy_command( "debug_pipeline.py", deployment_method, @@ -80,13 +82,13 @@ def test_deploy_command( **deployment_args ) assert "Your current repository has no origin set" in py_ex.value.args[0] - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -5 + with pytest.raises(CliCommandInnerException): + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) # we have a repo that was never run Remote.create(repo, "origin", "git@github.com:rudolfix/dlt-cmd-test-2.git") @@ -97,18 +99,19 @@ def test_deploy_command( deploy_command.COMMAND_DEPLOY_REPO_LOCATION, **deployment_args ) - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -2 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -3 # run the script with wrong credentials (it is postgres there) venv = Venv.restore_current() # mod environ so wrong password is passed to override secrets.toml - pg_credentials = os.environ.pop("DESTINATION__POSTGRES__CREDENTIALS") + pg_credentials = os.environ.pop("DESTINATION__POSTGRES__CREDENTIALS", "") # os.environ["DESTINATION__POSTGRES__CREDENTIALS__PASSWORD"] = "password" with pytest.raises(CalledProcessError): venv.run_script("debug_pipeline.py") @@ -121,13 +124,14 @@ def test_deploy_command( **deployment_args ) assert "The last pipeline run ended with error" in py_ex2.value.args[0] - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -2 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -3 os.environ["DESTINATION__POSTGRES__CREDENTIALS"] = pg_credentials # also delete secrets so credentials are not mixed up on CI @@ -172,10 +176,11 @@ def test_deploy_command( **deployment_args ) with echo.always_choose(False, always_choose_value=True): - rc = _dlt.deploy_command_wrapper( - "no_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -4 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "no_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -5 diff --git a/tests/cli/test_init_command.py b/tests/cli/test_init_command.py index 35c68ecfb4..b0af2447e9 100644 --- a/tests/cli/test_init_command.py +++ b/tests/cli/test_init_command.py @@ -37,7 +37,7 @@ _list_template_sources, _list_verified_sources, ) -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.cli.requirements import SourceRequirements from dlt.reflection.script_visitor import PipelineScriptVisitor from dlt.reflection import names as n diff --git a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py index 4377196320..f9ade3c011 100644 --- a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py +++ b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py @@ -9,6 +9,7 @@ from dlt.common.runtime.run_context import RunContext, DOT_DLT from tests.utils import TEST_STORAGE_ROOT +from dlt.cli.exceptions import CliCommandException class RunContextTest(RunContext): @@ -32,28 +33,42 @@ def plug_run_context_impl() -> SupportsRunContext: return RunContextTest() +class ExampleException(Exception): + pass + + class ExampleCommand(SupportsCliCommand): command: str = "example" help_string: str = "Example command" + docs_url: str = "DEFAULT_DOCS_URL" def configure_parser(self, parser: argparse.ArgumentParser) -> None: parser.add_argument("--name", type=str, help="Name to print") + parser.add_argument("--result", type=str, help="How to result") - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: print(f"Example command executed with name: {args.name}") - return 33 + + # pass without return + if args.result == "pass": + pass + if args.result == "known_error": + raise CliCommandException(error_code=-33, docs_url="MODIFIED_DOCS_URL") + if args.result == "unknown_error": + raise ExampleException("No one knows what is going on") class InitCommand(SupportsCliCommand): command: str = "init" help_string: str = "Init command" + docs_url: str = "INIT_DOCS_URL" def configure_parser(self, parser: argparse.ArgumentParser) -> None: pass - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: print("Plugin overwrote init command") - return 55 + raise CliCommandException(error_code=-55) @plugins.hookimpl(specname="plug_cli") diff --git a/tests/plugins/test_plugin_discovery.py b/tests/plugins/test_plugin_discovery.py index 6bb85d04f5..6962e89bf7 100644 --- a/tests/plugins/test_plugin_discovery.py +++ b/tests/plugins/test_plugin_discovery.py @@ -57,10 +57,29 @@ def test_example_plugin() -> None: def test_cli_hook(script_runner: ScriptRunner) -> None: # new command result = script_runner.run(["dlt", "example", "--name", "John"]) - assert result.returncode == 33 + assert result.returncode == 0 assert "Example command executed with name: John" in result.stdout + # raise + result = script_runner.run(["dlt", "example", "--name", "John", "--result", "known_error"]) + assert result.returncode == -33 + assert "MODIFIED_DOCS_URL" in result.stdout + + result = script_runner.run(["dlt", "example", "--name", "John", "--result", "unknown_error"]) + assert result.returncode == -1 + assert "DEFAULT_DOCS_URL" in result.stdout + assert "No one knows what is going on" in result.stderr + assert "Traceback" not in result.stderr # stack trace is not there + + # raise with trace + result = script_runner.run( + ["dlt", "--debug", "example", "--name", "John", "--result", "unknown_error"] + ) + assert "No one knows what is going on" in result.stderr + assert "Traceback" in result.stderr # stacktrace is there + # overwritten pipeline command result = script_runner.run(["dlt", "init"]) - assert result.returncode == 55 + assert result.returncode == -55 assert "Plugin overwrote init command" in result.stdout + assert "INIT_DOCS_URL" in result.stdout