From 339324591163b24ca95858c50be6c5994c89fbfe Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Fri, 23 Jul 2021 14:29:36 +0100 Subject: [PATCH 01/17] Initial --- core/dbt/main.py | 3 +- core/dbt/task/init.py | 149 ++++++++++++++---- core/setup.py | 1 + .../dbt/include/bigquery/target_options.yml | 14 ++ .../dbt/include/postgres/target_options.yml | 8 + .../dbt/include/redshift/target_options.yml | 8 + .../dbt/adapters/snowflake/connections.py | 2 +- .../dbt/include/snowflake/target_options.yml | 15 ++ 8 files changed, 168 insertions(+), 32 deletions(-) create mode 100644 plugins/bigquery/dbt/include/bigquery/target_options.yml create mode 100644 plugins/postgres/dbt/include/postgres/target_options.yml create mode 100644 plugins/redshift/dbt/include/redshift/target_options.yml create mode 100644 plugins/snowflake/dbt/include/snowflake/target_options.yml diff --git a/core/dbt/main.py b/core/dbt/main.py index cd0957a1b2f..25f0d890ed0 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -256,7 +256,6 @@ def run_from_args(parsed): with track_run(task): results = task.run() - return task, results @@ -361,7 +360,7 @@ def _build_init_subparser(subparsers, base_subparser): ''' ) sub.add_argument( - 'project_name', + '--project_name', type=str, help=''' Name of the new project diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index fadee6a760a..0d3bc926405 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -1,5 +1,9 @@ +import copy import os import shutil +import yaml + +import click import dbt.config import dbt.clients.system @@ -10,7 +14,7 @@ from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory -from dbt.task.base import BaseTask +from dbt.task.base import BaseTask, move_to_nearest_project_dir DOCS_URL = 'https://docs.getdbt.com/docs/configure-your-profile' SLACK_URL = 'https://community.getdbt.com/' @@ -54,19 +58,18 @@ def create_profiles_dir(self, profiles_dir): return True return False - def create_profiles_file(self, profiles_file, sample_adapter): + def create_sample_profiles_file(self, profiles_file, adapter): # Line below raises an exception if the specified adapter is not found - load_plugin(sample_adapter) - adapter_path = get_include_paths(sample_adapter)[0] + load_plugin(adapter) + adapter_path = get_include_paths(adapter)[0] sample_profiles_path = adapter_path / 'sample_profiles.yml' if not sample_profiles_path.exists(): - logger.debug(f"No sample profile found for {sample_adapter}, skipping") + logger.debug(f"No sample profile found for {adapter}, skipping") return False if not os.path.exists(profiles_file): - msg = "With sample profiles.yml for {}" - logger.info(msg.format(sample_adapter)) + logger.info(f"With sample profiles.yml for {adapter}") shutil.copyfile(sample_profiles_path, profiles_file) return True @@ -83,29 +86,117 @@ def get_addendum(self, project_name, profiles_path): slack_url=SLACK_URL ) - def run(self): - project_dir = self.args.project_name - sample_adapter = self.args.adapter - if not sample_adapter: - try: - # pick first one available, often postgres - sample_adapter = next(_get_adapter_plugin_names()) - except StopIteration: - logger.debug("No adapters installed, skipping") + def generate_target_from_input(self, target_options, target={}): + target_options_local = copy.deepcopy(target_options) + # value = click.prompt('Please enter a valid integer', type=int) + for key, value in target_options_local.items(): + if not key.startswith("_"): + if isinstance(value, str) and (value[0] + value[-1] == "[]"): + hide_input = key == "password" + target[key] = click.prompt( + f"{key} ({value[1:-1]})", hide_input=hide_input + ) + else: + target[key] = target_options_local[key] + if key.startswith("_choose"): + choice_type = key[8:] + option_list = list(value.keys()) + options_msg = "\n".join([ + f"[{n+1}] {v}" for n, v in enumerate(option_list) + ]) + click.echo(options_msg) + numeric_choice = click.prompt( + f"desired {choice_type} option (enter a number)", type=int + ) + choice = option_list[numeric_choice - 1] + target = self.generate_target_from_input( + target_options_local[key][choice], target + ) + return target + + def get_profile_name_from_current_project(self): + with open("dbt_project.yml") as f: + dbt_project = yaml.load(f) + return dbt_project["profile"] + + def write_profile(self, profiles_file, profile, profile_name=None): + if not profile_name: + profile_name = self.get_profile_name_from_current_project() + if os.path.exists(profiles_file): + with open(profiles_file, "r+") as f: + profiles = yaml.load(f) or {} + profiles[profile_name] = profile + f.seek(0) + yaml.dump(profiles, f) + else: + profiles = {profile_name: profile} + with open(profiles_file, "w") as f: + yaml.dump(profiles, f) + + def configure_profile_from_scratch(self, selected_adapter): + # Line below raises an exception if the specified adapter is not found + load_plugin(selected_adapter) + adapter_path = get_include_paths(selected_adapter)[0] + target_options_path = adapter_path / 'target_options.yml' + profiles_file = os.path.join(dbt.config.PROFILES_DIR, 'profiles.yml') + + if not target_options_path.exists(): + logger.info(f"No options found for {selected_adapter}, using " + + "sample profiles instead. Make sure to update it at" + + "{profiles_file}.") + self.create_sample_profiles_file(profiles_file, selected_adapter) + else: + logger.info(f"Using {selected_adapter} profile options.") + with open(target_options_path) as f: + target_options = yaml.load(f) + target = self.generate_target_from_input(target_options) + profile = { + "outputs": { + "dev": target + }, + "target": "dev" + } + self.write_profile(profiles_file, profile) + + def configure_profile_using_defaults(self, selected_adapter): + raise(NotImplementedError()) + def run(self): + selected_adapter = self.args.adapter profiles_dir = dbt.config.PROFILES_DIR - profiles_file = os.path.join(profiles_dir, 'profiles.yml') - self.create_profiles_dir(profiles_dir) - if sample_adapter: - self.create_profiles_file(profiles_file, sample_adapter) - - if os.path.exists(project_dir): - raise RuntimeError("directory {} already exists!".format( - project_dir - )) - - self.copy_starter_repo(project_dir) - addendum = self.get_addendum(project_dir, profiles_dir) - logger.info(addendum) + # Determine whether we're initializing a new project or configuring a + # profile for an existing one + if self.args.project_name: + project_dir = self.args.project_name + if os.path.exists(project_dir): + raise RuntimeError("directory {} already exists!".format( + project_dir + )) + + self.copy_starter_repo(project_dir) + + addendum = self.get_addendum(project_dir, profiles_dir) + logger.info(addendum) + if not selected_adapter: + try: + # pick first one available, often postgres + selected_adapter = next(_get_adapter_plugin_names()) + except StopIteration: + logger.debug("No adapters installed, skipping") + self.configure_profile_from_scratch( + selected_adapter + ) + else: + logger.info("Setting up your profile.") + move_to_nearest_project_dir(self.args) + if os.path.exists("target_defaults.yml"): + self.configure_profile_using_defaults() + else: + if not selected_adapter: + raise RuntimeError("No adapter specified.") + logger.info("Configuring from scratch.") + self.configure_profile_from_scratch( + selected_adapter + ) diff --git a/core/setup.py b/core/setup.py index ad2fed3884c..d01558347f3 100644 --- a/core/setup.py +++ b/core/setup.py @@ -52,6 +52,7 @@ def read(fname): 'Jinja2==2.11.3', 'PyYAML>=3.11', 'agate>=1.6,<1.6.2', + 'click>=8,<9', 'colorama>=0.3.9,<0.4.5', 'dataclasses>=0.6,<0.9;python_version<"3.7"', 'hologram==0.0.14', diff --git a/plugins/bigquery/dbt/include/bigquery/target_options.yml b/plugins/bigquery/dbt/include/bigquery/target_options.yml new file mode 100644 index 00000000000..4218c5531d3 --- /dev/null +++ b/plugins/bigquery/dbt/include/bigquery/target_options.yml @@ -0,0 +1,14 @@ +type: bigquery +_choose_method: + _oauth: + method: oauth + _service_account: + method: service-account + keyfile: [/path/to/bigquery/keyfile.json] +project: [GCP project id] +dataset: [the name of your dbt dataset] +threads: [1 or more] +timeout_seconds: 300 +location: [one of US or EU] +priority: interactive +retries: 1 diff --git a/plugins/postgres/dbt/include/postgres/target_options.yml b/plugins/postgres/dbt/include/postgres/target_options.yml new file mode 100644 index 00000000000..d4d3700072b --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/target_options.yml @@ -0,0 +1,8 @@ +type: postgres +threads: [1 or more] +host: [host] +port: [port] +user: [dev_username] +pass: [dev_password] +dbname: [dbname] +schema: [dev_schema] \ No newline at end of file diff --git a/plugins/redshift/dbt/include/redshift/target_options.yml b/plugins/redshift/dbt/include/redshift/target_options.yml new file mode 100644 index 00000000000..ddbc380e329 --- /dev/null +++ b/plugins/redshift/dbt/include/redshift/target_options.yml @@ -0,0 +1,8 @@ +type: redshift +threads: [1 or more] +host: [host] +port: [port] +user: [dev_username] +pass: [dev_password] +dbname: [dbname] +schema: [dev_schema] \ No newline at end of file diff --git a/plugins/snowflake/dbt/adapters/snowflake/connections.py b/plugins/snowflake/dbt/adapters/snowflake/connections.py index 81aab29ac41..d8b41e4272d 100644 --- a/plugins/snowflake/dbt/adapters/snowflake/connections.py +++ b/plugins/snowflake/dbt/adapters/snowflake/connections.py @@ -31,9 +31,9 @@ class SnowflakeCredentials(Credentials): account: str user: str + password: str warehouse: Optional[str] = None role: Optional[str] = None - password: Optional[str] = None authenticator: Optional[str] = None private_key_path: Optional[str] = None private_key_passphrase: Optional[str] = None diff --git a/plugins/snowflake/dbt/include/snowflake/target_options.yml b/plugins/snowflake/dbt/include/snowflake/target_options.yml new file mode 100644 index 00000000000..8a2b53c1105 --- /dev/null +++ b/plugins/snowflake/dbt/include/snowflake/target_options.yml @@ -0,0 +1,15 @@ +type: snowflake +account: '[account id + region (if applicable)]' +user: '[username]' +_choose_authentication_type: + password: + password: '[password]' + keypair: + private_key_path: '[path/to/private.key]' + private_key_passphrase: '[passphrase for the private key, if key is encrypted]' +role: '[user role]' +database: '[database name]' +warehouse: '[warehouse name]' +schema: '[dbt schema]' +threads: '[1 or more]' +client_session_keep_alive: False \ No newline at end of file From e1a8463b1cfdf5092a4587bf567639ad506543a1 Mon Sep 17 00:00:00 2001 From: NiallRees Date: Sun, 15 Aug 2021 22:04:21 +0100 Subject: [PATCH 02/17] Further dev --- core/dbt/clients/yaml_helper.py | 2 +- core/dbt/main.py | 14 - core/dbt/task/init.py | 257 ++++++++++++------ .../upgrade_dbt_schema_tests_v1_to_v2.py | 2 +- core/setup.py | 2 +- .../dbt/include/bigquery/target_options.yml | 26 +- .../dbt/include/postgres/target_options.yml | 16 +- .../dbt/include/redshift/target_options.yml | 21 +- .../dbt/include/snowflake/target_options.yml | 2 +- .../test_local_dependency.py | 2 +- .../test_graph_selection.py | 4 +- .../test_intersection_syntax.py | 2 +- .../test_tag_selection.py | 2 +- .../test_cli_invocation.py | 2 +- .../test_bigquery_adapter_functions.py | 2 +- .../test_bigquery_adapter_specific.py | 4 +- .../test_bigquery_copy_failing_models.py | 2 +- .../test_bigquery_copy_models.py | 2 +- .../test_bigquery_date_partitioning.py | 2 +- .../integration/028_cli_vars/test_cli_vars.py | 2 +- test/integration/040_init_test/test_init.py | 2 +- .../042_sources_test/test_sources.py | 4 +- .../test_run_operations.py | 2 +- .../049_dbt_debug_test/test_debug.py | 2 +- .../test_adapter_methods.py | 2 +- .../test_alter_column_types.py | 2 +- test/integration/069_build_test/test_build.py | 4 +- test/integration/base.py | 2 +- test/rpc/conftest.py | 2 +- test/rpc/test_build.py | 4 +- test/rpc/test_test.py | 4 +- test/rpc/util.py | 2 +- test/unit/test_config.py | 2 +- test/unit/test_graph_selector_parsing.py | 2 +- test/unit/test_jinja.py | 2 +- test/unit/test_main.py | 2 +- test/unit/test_manifest_selectors.py | 4 +- test/unit/test_parser.py | 2 +- test/unit/test_selector_errors.py | 2 +- 39 files changed, 244 insertions(+), 172 deletions(-) diff --git a/core/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py index df349c1bb38..1c34ed41cc2 100644 --- a/core/dbt/clients/yaml_helper.py +++ b/core/dbt/clients/yaml_helper.py @@ -1,6 +1,6 @@ import dbt.exceptions from typing import Any, Dict, Optional -import yaml +import oyaml as yaml import yaml.scanner # the C version is faster, but it doesn't always exist diff --git a/core/dbt/main.py b/core/dbt/main.py index 2f207641ef5..437b7f3840b 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -359,20 +359,6 @@ def _build_init_subparser(subparsers, base_subparser): Initialize a new DBT project. ''' ) - sub.add_argument( - '--project_name', - type=str, - help=''' - Name of the new project - ''', - ) - sub.add_argument( - '--adapter', - type=str, - help=''' - Write sample profiles.yml for which adapter - ''', - ) sub.set_defaults(cls=init_task.InitTask, which='init', rpc_method=None) return sub diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 0d3bc926405..edb754e6a99 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -1,9 +1,11 @@ import copy import os import shutil -import yaml +from typing import Tuple +import oyaml as yaml import click +from jinja2 import Template import dbt.config import dbt.clients.system @@ -23,11 +25,7 @@ IGNORE_FILES = ["__init__.py", "__pycache__"] ON_COMPLETE_MESSAGE = """ -Your new dbt project "{project_name}" was created! If this is your first time -using dbt, you'll need to set up your profiles.yml file -- this file will tell dbt how -to connect to your database. You can find this file by running: - - {open_cmd} {profiles_path} +Your new dbt project "{project_name}" was created! For more information on how to configure the profiles.yml file, please consult the dbt documentation here: @@ -50,7 +48,8 @@ def copy_starter_repo(self, project_name): shutil.copytree(starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES)) - def create_profiles_dir(self, profiles_dir): + def create_profiles_dir(self, profiles_dir: str) -> bool: + """Create the user's profiles directory if it doesn't already exist.""" if not os.path.exists(profiles_dir): msg = "Creating dbt configuration folder at {}" logger.info(msg.format(profiles_dir)) @@ -58,24 +57,27 @@ def create_profiles_dir(self, profiles_dir): return True return False - def create_sample_profiles_file(self, profiles_file, adapter): + def create_profile_from_sample(self, adapter: str): + """Create a profile entry using the adapter's sample_profiles.yml""" # Line below raises an exception if the specified adapter is not found load_plugin(adapter) adapter_path = get_include_paths(adapter)[0] - sample_profiles_path = adapter_path / 'sample_profiles.yml' + sample_profiles_path = adapter_path / "sample_profiles.yml" if not sample_profiles_path.exists(): - logger.debug(f"No sample profile found for {adapter}, skipping") - return False - - if not os.path.exists(profiles_file): - logger.info(f"With sample profiles.yml for {adapter}") - shutil.copyfile(sample_profiles_path, profiles_file) - return True - - return False + logger.debug(f"No sample profile found for {adapter}.") + else: + with open(sample_profiles_path, "r") as f: + # Ignore the name given in the sample_profiles.yml + profile = list(yaml.load(f).values())[0] + profiles_filepath, profile_name = self.write_profile(profile) + logger.info( + f"Profile {profile_name} written to {profiles_filepath} " + "using sample configuration. Once updated " + "you'll be able to start developing with dbt." + ) - def get_addendum(self, project_name, profiles_path): + def get_addendum(self, project_name: str, profiles_path: str) -> str: open_cmd = dbt.clients.system.open_dir_cmd() return ON_COMPLETE_MESSAGE.format( @@ -86,117 +88,196 @@ def get_addendum(self, project_name, profiles_path): slack_url=SLACK_URL ) - def generate_target_from_input(self, target_options, target={}): + def generate_target_from_input( + self, + target_options: dict, + target: dict = {} + ) -> dict: + """Generate a target configuration from target_options and user input. + """ target_options_local = copy.deepcopy(target_options) - # value = click.prompt('Please enter a valid integer', type=int) for key, value in target_options_local.items(): - if not key.startswith("_"): - if isinstance(value, str) and (value[0] + value[-1] == "[]"): - hide_input = key == "password" - target[key] = click.prompt( - f"{key} ({value[1:-1]})", hide_input=hide_input - ) - else: - target[key] = target_options_local[key] if key.startswith("_choose"): - choice_type = key[8:] + choice_type = key[8:].replace("_", " ") option_list = list(value.keys()) options_msg = "\n".join([ f"[{n+1}] {v}" for n, v in enumerate(option_list) ]) click.echo(options_msg) numeric_choice = click.prompt( - f"desired {choice_type} option (enter a number)", type=int + f"Desired {choice_type} option (enter a number)", type=int ) choice = option_list[numeric_choice - 1] + # Complete the chosen option's values in a recursive call target = self.generate_target_from_input( target_options_local[key][choice], target ) + else: + if key.startswith("_fixed"): + # _fixed prefixed keys are not presented to the user + target[key[7:]] = value + elif isinstance(value, str) and (value[0] + value[-1] == "[]"): + # A square bracketed value is used as a hint + hide_input = key == "password" + target[key] = click.prompt( + f"{key} ({value[1:-1]})", hide_input=hide_input + ) + elif isinstance(value, list): + # A list can be used to provide both a hint and a default + target[key] = click.prompt( + f"{key} ({value[0]})", default=value[1] + ) + else: + # All other values are used as defaults + target[key] = click.prompt( + key, default=target_options_local[key] + ) return target - def get_profile_name_from_current_project(self): + def get_profile_name_from_current_project(self) -> str: + """Reads dbt_project.yml in the current directory to retrieve the + profile name. + """ with open("dbt_project.yml") as f: dbt_project = yaml.load(f) return dbt_project["profile"] - def write_profile(self, profiles_file, profile, profile_name=None): - if not profile_name: - profile_name = self.get_profile_name_from_current_project() + def write_profile( + self, profile: dict, profile_name: str = None + ) -> Tuple[str, str]: + """Given a profile, write it to the current project's profiles.yml. + This will overwrite any profile with a matching name.""" + profiles_file = os.path.join(dbt.config.PROFILES_DIR, "profiles.yml") + profile_name = ( + profile_name or self.get_profile_name_from_current_project() + ) if os.path.exists(profiles_file): with open(profiles_file, "r+") as f: profiles = yaml.load(f) or {} profiles[profile_name] = profile f.seek(0) yaml.dump(profiles, f) + f.truncate() else: profiles = {profile_name: profile} with open(profiles_file, "w") as f: yaml.dump(profiles, f) + return profiles_file, profile_name + + def create_profile_from_target_options(self, target_options: dict): + """Create and write a profile using the supplied target_options.""" + target = self.generate_target_from_input(target_options) + profile = { + "outputs": { + "dev": target + }, + "target": "dev" + } + profiles_filepath, profile_name = self.write_profile(profile) + logger.info( + f"Profile {profile_name} written to {profiles_filepath} using " + "your supplied values." + ) - def configure_profile_from_scratch(self, selected_adapter): + def create_profile_from_scratch(self, adapter: str): + """Create a profile without defaults using target_options.yml if available, or + sample_profiles.yml as a fallback.""" # Line below raises an exception if the specified adapter is not found - load_plugin(selected_adapter) - adapter_path = get_include_paths(selected_adapter)[0] + load_plugin(adapter) + adapter_path = get_include_paths(adapter)[0] target_options_path = adapter_path / 'target_options.yml' - profiles_file = os.path.join(dbt.config.PROFILES_DIR, 'profiles.yml') - if not target_options_path.exists(): - logger.info(f"No options found for {selected_adapter}, using " + - "sample profiles instead. Make sure to update it at" + - "{profiles_file}.") - self.create_sample_profiles_file(profiles_file, selected_adapter) - else: - logger.info(f"Using {selected_adapter} profile options.") + if target_options_path.exists(): with open(target_options_path) as f: target_options = yaml.load(f) - target = self.generate_target_from_input(target_options) - profile = { - "outputs": { - "dev": target - }, - "target": "dev" - } - self.write_profile(profiles_file, profile) - - def configure_profile_using_defaults(self, selected_adapter): - raise(NotImplementedError()) + self.create_profile_from_target_options(target_options) + else: + # For adapters without a target_options.yml defined, fallback on + # sample_profiles.yml + self.create_profile_from_sample(adapter) + + def check_if_can_write_profile(self, profile_name: str = None) -> bool: + profiles_file = os.path.join(dbt.config.PROFILES_DIR, "profiles.yml") + if not os.path.exists(profiles_file): + return True + profile_name = ( + profile_name or self.get_profile_name_from_current_project() + ) + with open(profiles_file, "r") as f: + profiles = yaml.load(f) or {} + if profile_name in profiles.keys(): + response = click.confirm( + f"The profile {profile_name} already exists in " + f"{profiles_file}. Continue and overwrite it?" + ) + return response + else: + return True + + def create_profile_using_profile_template(self): + """Create a profile using profile_template.yml""" + with open("profile_template.yml") as f: + profile_template = yaml.load(f) + profile_name = list(profile_template["profile"].keys())[0] + self.check_if_can_write_profile(profile_name) + render_vars = {} + for template_variable in profile_template["vars"]: + render_vars[template_variable] = click.prompt(template_variable) + profile = profile_template["profile"][profile_name] + profile_str = yaml.dump(profile) + profile_str = Template(profile_str).render(vars=render_vars) + profile = yaml.load(profile_str) + profiles_filepath, _ = self.write_profile(profile, profile_name) + logger.info( + f"Profile {profile_name} written to {profiles_filepath} using " + "profile_template.yml and your supplied values." + ) + + def ask_for_adapter_choice(self) -> str: + """Ask the user which adapter (database) they'd like to use.""" + click.echo("Which database would you like to use?") + available_adapters = list(_get_adapter_plugin_names()) + click.echo("\n".join([ + f"[{n+1}] {v}" for n, v in enumerate(available_adapters) + ])) + numeric_choice = click.prompt("Enter a number", type=int) + return available_adapters[numeric_choice - 1] def run(self): - selected_adapter = self.args.adapter profiles_dir = dbt.config.PROFILES_DIR self.create_profiles_dir(profiles_dir) - # Determine whether we're initializing a new project or configuring a - # profile for an existing one - if self.args.project_name: - project_dir = self.args.project_name - if os.path.exists(project_dir): - raise RuntimeError("directory {} already exists!".format( - project_dir - )) - - self.copy_starter_repo(project_dir) + try: + move_to_nearest_project_dir(self.args) + in_project = True + except dbt.exceptions.RuntimeException: + in_project = False - addendum = self.get_addendum(project_dir, profiles_dir) - logger.info(addendum) - if not selected_adapter: - try: - # pick first one available, often postgres - selected_adapter = next(_get_adapter_plugin_names()) - except StopIteration: - logger.debug("No adapters installed, skipping") - self.configure_profile_from_scratch( - selected_adapter - ) - else: + if in_project: logger.info("Setting up your profile.") - move_to_nearest_project_dir(self.args) - if os.path.exists("target_defaults.yml"): - self.configure_profile_using_defaults() + if os.path.exists("profile_template.yml"): + self.create_profile_using_profile_template() else: - if not selected_adapter: - raise RuntimeError("No adapter specified.") - logger.info("Configuring from scratch.") - self.configure_profile_from_scratch( - selected_adapter + if not self.check_if_can_write_profile(): + return + adapter = self.ask_for_adapter_choice() + self.create_profile_from_scratch( + adapter ) + else: + project_dir = click.prompt("What is the desired project name?") + if os.path.exists(project_dir): + logger.info( + f"Existing project found at directory {project_dir}" + ) + return + + self.copy_starter_repo(project_dir) + os.chdir(project_dir) + if not self.check_if_can_write_profile(): + return + adapter = self.ask_for_adapter_choice() + self.create_profile_from_scratch( + adapter + ) + logger.info(self.get_addendum(project_dir, profiles_dir)) diff --git a/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py b/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py index 29a2eed1e7c..c803b5b7e7f 100644 --- a/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py +++ b/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py @@ -5,7 +5,7 @@ import os import re import sys -import yaml +import oyaml as yaml LOGGER = logging.getLogger('upgrade_dbt_schema') LOGFILE = 'upgrade_dbt_schema_tests_v1_to_v2.txt' diff --git a/core/setup.py b/core/setup.py index f6b381b1844..6e2b3c5d7d3 100644 --- a/core/setup.py +++ b/core/setup.py @@ -50,7 +50,7 @@ def read(fname): ], install_requires=[ 'Jinja2==2.11.3', - 'PyYAML>=3.11', + 'oyaml>=1.0', 'agate>=1.6,<1.6.2', 'click>=8,<9', 'colorama>=0.3.9,<0.4.5', diff --git a/plugins/bigquery/dbt/include/bigquery/target_options.yml b/plugins/bigquery/dbt/include/bigquery/target_options.yml index 4218c5531d3..cfd5113b621 100644 --- a/plugins/bigquery/dbt/include/bigquery/target_options.yml +++ b/plugins/bigquery/dbt/include/bigquery/target_options.yml @@ -1,14 +1,14 @@ -type: bigquery -_choose_method: - _oauth: - method: oauth - _service_account: - method: service-account - keyfile: [/path/to/bigquery/keyfile.json] -project: [GCP project id] -dataset: [the name of your dbt dataset] -threads: [1 or more] +_fixed_type: bigquery +_choose_authentication_method: + oauth: + _fixed_method: oauth + service_account: + _fixed_method: service-account + keyfile: '[/path/to/bigquery/keyfile.json]' +project: '[GCP project id]' +dataset: '[the name of your dbt dataset]' +threads: '[1 or more]' timeout_seconds: 300 -location: [one of US or EU] -priority: interactive -retries: 1 +location: '[one of US or EU]' +_fixed_priority: interactive +_fixed_retries: 1 diff --git a/plugins/postgres/dbt/include/postgres/target_options.yml b/plugins/postgres/dbt/include/postgres/target_options.yml index d4d3700072b..96698d7ccbf 100644 --- a/plugins/postgres/dbt/include/postgres/target_options.yml +++ b/plugins/postgres/dbt/include/postgres/target_options.yml @@ -1,8 +1,8 @@ -type: postgres -threads: [1 or more] -host: [host] -port: [port] -user: [dev_username] -pass: [dev_password] -dbname: [dbname] -schema: [dev_schema] \ No newline at end of file +_fixed_type: postgres +threads: '[1 or more]' +host: '[host]' +port: '[port]' +user: '[dev_username]' +pass: '[dev_password]' +dbname: '[dbname]' +schema: '[dev_schema]' \ No newline at end of file diff --git a/plugins/redshift/dbt/include/redshift/target_options.yml b/plugins/redshift/dbt/include/redshift/target_options.yml index ddbc380e329..20fc722b66f 100644 --- a/plugins/redshift/dbt/include/redshift/target_options.yml +++ b/plugins/redshift/dbt/include/redshift/target_options.yml @@ -1,8 +1,13 @@ -type: redshift -threads: [1 or more] -host: [host] -port: [port] -user: [dev_username] -pass: [dev_password] -dbname: [dbname] -schema: [dev_schema] \ No newline at end of file +_fixed_type: redshift +_choose_authentication_method: + password: + password: '[password]' + iam: + method: iam +host: '[hostname.region.redshift.amazonaws.com]' +user: '[username]' +port: 5439 +dbname: '[database name]' +schema: '[dbt schema]' +threads: '[1 or more]' +keepalives_idle: 0 # default 0, indicating the system default \ No newline at end of file diff --git a/plugins/snowflake/dbt/include/snowflake/target_options.yml b/plugins/snowflake/dbt/include/snowflake/target_options.yml index 8a2b53c1105..d7cfbdf34cb 100644 --- a/plugins/snowflake/dbt/include/snowflake/target_options.yml +++ b/plugins/snowflake/dbt/include/snowflake/target_options.yml @@ -1,4 +1,4 @@ -type: snowflake +_fixed_type: snowflake account: '[account id + region (if applicable)]' user: '[username]' _choose_authentication_type: diff --git a/test/integration/006_simple_dependency_test/test_local_dependency.py b/test/integration/006_simple_dependency_test/test_local_dependency.py index 3691b19c5d7..f054f47c12f 100644 --- a/test/integration/006_simple_dependency_test/test_local_dependency.py +++ b/test/integration/006_simple_dependency_test/test_local_dependency.py @@ -2,7 +2,7 @@ import os import json import shutil -import yaml +import oyaml as yaml from unittest import mock import dbt.semver diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index e7069ae1a38..b42ad4a3021 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml import json import os @@ -408,7 +408,7 @@ def test__postgres__exposure_parents(self): results = self.run_dbt(['ls', '--select', '1+exposure:user_exposure']) assert len(results) == 5 - assert sorted(results) == ['exposure:test.user_exposure', 'test.schema_test.unique_users_id', + assert sorted(results) == ['exposure:test.user_exposure', 'test.schema_test.unique_users_id', 'test.schema_test.unique_users_rollup_gender', 'test.users', 'test.users_rollup'] results = self.run_dbt(['run', '-m', '+exposure:user_exposure']) diff --git a/test/integration/007_graph_selection_tests/test_intersection_syntax.py b/test/integration/007_graph_selection_tests/test_intersection_syntax.py index d725d03c39f..65f0ba3cfae 100644 --- a/test/integration/007_graph_selection_tests/test_intersection_syntax.py +++ b/test/integration/007_graph_selection_tests/test_intersection_syntax.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml class TestGraphSelection(DBTIntegrationTest): diff --git a/test/integration/007_graph_selection_tests/test_tag_selection.py b/test/integration/007_graph_selection_tests/test_tag_selection.py index 3d3b0c3baf7..c5ea7434c9c 100644 --- a/test/integration/007_graph_selection_tests/test_tag_selection.py +++ b/test/integration/007_graph_selection_tests/test_tag_selection.py @@ -1,6 +1,6 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml class TestGraphSelection(DBTIntegrationTest): diff --git a/test/integration/015_cli_invocation_tests/test_cli_invocation.py b/test/integration/015_cli_invocation_tests/test_cli_invocation.py index ef898ae84fe..9c881d4da80 100644 --- a/test/integration/015_cli_invocation_tests/test_cli_invocation.py +++ b/test/integration/015_cli_invocation_tests/test_cli_invocation.py @@ -4,7 +4,7 @@ import shutil import pytest import tempfile -import yaml +import oyaml as yaml from typing import Dict diff --git a/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py b/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py index 480dca07d93..9df84d806f5 100644 --- a/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py +++ b/test/integration/022_bigquery_test/test_bigquery_adapter_functions.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile -import yaml +import oyaml as yaml class TestBigqueryAdapterFunctions(DBTIntegrationTest): diff --git a/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py b/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py index 244d7ce56a0..76124fb24e9 100644 --- a/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py +++ b/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py @@ -3,7 +3,7 @@ from test.integration.base import DBTIntegrationTest, use_profile import textwrap -import yaml +import oyaml as yaml class TestBigqueryAdapterSpecific(DBTIntegrationTest): @@ -28,7 +28,7 @@ def project_config(self): test: materialized: table expiring_table: - hours_to_expiration: 4 + hours_to_expiration: 4 ''')) @use_profile('bigquery') diff --git a/test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py b/test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py index da5316da8cc..de4e6e00b6e 100644 --- a/test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py +++ b/test/integration/022_bigquery_test/test_bigquery_copy_failing_models.py @@ -1,6 +1,6 @@ from test.integration.base import DBTIntegrationTest, use_profile import textwrap -import yaml +import oyaml as yaml class TestBigqueryCopyTableFails(DBTIntegrationTest): diff --git a/test/integration/022_bigquery_test/test_bigquery_copy_models.py b/test/integration/022_bigquery_test/test_bigquery_copy_models.py index 00028357faf..90e0449c276 100644 --- a/test/integration/022_bigquery_test/test_bigquery_copy_models.py +++ b/test/integration/022_bigquery_test/test_bigquery_copy_models.py @@ -1,6 +1,6 @@ from test.integration.base import DBTIntegrationTest, use_profile import textwrap -import yaml +import oyaml as yaml class TestBigqueryCopyTable(DBTIntegrationTest): diff --git a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py b/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py index c787d8d3f6f..94a31a5e279 100644 --- a/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py +++ b/test/integration/022_bigquery_test/test_bigquery_date_partitioning.py @@ -1,6 +1,6 @@ from test.integration.base import DBTIntegrationTest, use_profile import textwrap -import yaml +import oyaml as yaml class TestBigqueryDatePartitioning(DBTIntegrationTest): diff --git a/test/integration/028_cli_vars/test_cli_vars.py b/test/integration/028_cli_vars/test_cli_vars.py index d0873b6e107..bcf6c018a84 100644 --- a/test/integration/028_cli_vars/test_cli_vars.py +++ b/test/integration/028_cli_vars/test_cli_vars.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml import json diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index c7b5a85363d..bb2761df81c 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -2,7 +2,7 @@ from test.integration.base import DBTIntegrationTest, use_profile import os import shutil -import yaml +import oyaml as yaml class TestInit(DBTIntegrationTest): diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index c9633e86e41..e948b7c9ab7 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -2,7 +2,7 @@ import os from datetime import datetime, timedelta -import yaml +import oyaml as yaml from dbt.exceptions import CompilationException import dbt.tracking @@ -371,7 +371,7 @@ def test_postgres_source_freshness_selection_select(self): @use_profile('postgres') def test_postgres_source_freshness_selection_exclude(self): - """Tests node selection using the --select argument. It 'excludes' the + """Tests node selection using the --select argument. It 'excludes' the only source in the project so it should return no results.""" self._set_updated_at_to(timedelta(hours=-2)) self.freshness_start_time = datetime.utcnow() diff --git a/test/integration/044_run_operations_test/test_run_operations.py b/test/integration/044_run_operations_test/test_run_operations.py index 98715191e3e..10b1ec8ecd0 100644 --- a/test/integration/044_run_operations_test/test_run_operations.py +++ b/test/integration/044_run_operations_test/test_run_operations.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml class TestOperations(DBTIntegrationTest): diff --git a/test/integration/049_dbt_debug_test/test_debug.py b/test/integration/049_dbt_debug_test/test_debug.py index 74fabfcbc7b..137935042fe 100644 --- a/test/integration/049_dbt_debug_test/test_debug.py +++ b/test/integration/049_dbt_debug_test/test_debug.py @@ -1,7 +1,7 @@ from test.integration.base import DBTIntegrationTest, use_profile import os import re -import yaml +import oyaml as yaml import pytest diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index 1fcaf5b1a49..4fa0b9feff7 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml class TestBaseCaching(DBTIntegrationTest): diff --git a/test/integration/056_column_type_tests/test_alter_column_types.py b/test/integration/056_column_type_tests/test_alter_column_types.py index 51970244b84..7e3048a4ddd 100644 --- a/test/integration/056_column_type_tests/test_alter_column_types.py +++ b/test/integration/056_column_type_tests/test_alter_column_types.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml class TestAlterColumnTypes(DBTIntegrationTest): diff --git a/test/integration/069_build_test/test_build.py b/test/integration/069_build_test/test_build.py index 4b2c24024ce..999108c2aae 100644 --- a/test/integration/069_build_test/test_build.py +++ b/test/integration/069_build_test/test_build.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import yaml +import oyaml as yaml class TestBuild(DBTIntegrationTest): @@ -44,7 +44,7 @@ def schema(self): @property def models(self): return "models-failing" - + @use_profile("postgres") def test__postgres_build_happy_path(self): results = self.build(expect_pass=False) diff --git a/test/integration/base.py b/test/integration/base.py index 463948de4b8..dedeef420e3 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -13,7 +13,7 @@ from functools import wraps import pytest -import yaml +import oyaml as yaml from unittest.mock import patch import dbt.main as dbt diff --git a/test/rpc/conftest.py b/test/rpc/conftest.py index 561c009f92e..10db7d49206 100644 --- a/test/rpc/conftest.py +++ b/test/rpc/conftest.py @@ -5,7 +5,7 @@ import time from typing import Dict, Any, Set -import yaml +import oyaml as yaml def pytest_addoption(parser): diff --git a/test/rpc/test_build.py b/test/rpc/test_build.py index 24183b947f2..8bbd4583313 100644 --- a/test/rpc/test_build.py +++ b/test/rpc/test_build.py @@ -1,6 +1,6 @@ import os import pytest -import yaml +import oyaml as yaml from .util import ( assert_has_threads, get_querier, @@ -128,7 +128,7 @@ def test_rpc_build_state( querier.build(state='./state', models=['state:modified']), ) assert len(results['results']) == 0 - + # a better test of defer would require multiple targets results = querier.async_wait_for_result( querier.build(state='./state', models=['state:modified'], defer=True) diff --git a/test/rpc/test_test.py b/test/rpc/test_test.py index 9f938c3c966..08aba9f4bbd 100644 --- a/test/rpc/test_test.py +++ b/test/rpc/test_test.py @@ -1,6 +1,6 @@ import os import pytest -import yaml +import oyaml as yaml from .util import ( assert_has_threads, get_querier, @@ -105,7 +105,7 @@ def test_rpc_test_state( querier.test(state='./state', models=['state:modified']), ) assert len(results['results']) == 0 - + # a better test of defer would require multiple targets results = querier.async_wait_for_result( querier.run(state='./state', models=['state:modified'], defer=True) diff --git a/test/rpc/util.py b/test/rpc/util.py index f6beb832af5..1bacc397116 100644 --- a/test/rpc/util.py +++ b/test/rpc/util.py @@ -10,7 +10,7 @@ from typing import Dict, Any, Optional, Union, List import requests -import yaml +import oyaml as yaml import dbt.flags from dbt.adapters.factory import get_adapter, register_adapter diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 544bf473ce1..3a59484c6dc 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -8,7 +8,7 @@ import pytest from unittest import mock -import yaml +import oyaml as yaml import dbt.config import dbt.exceptions diff --git a/test/unit/test_graph_selector_parsing.py b/test/unit/test_graph_selector_parsing.py index 759930dd9dc..7a241d1ea1d 100644 --- a/test/unit/test_graph_selector_parsing.py +++ b/test/unit/test_graph_selector_parsing.py @@ -7,7 +7,7 @@ ) from dbt.graph.selector_methods import MethodName import textwrap -import yaml +import oyaml as yaml from dbt.contracts.selection import SelectorFile diff --git a/test/unit/test_jinja.py b/test/unit/test_jinja.py index 19a25ee3f56..67503c4ab52 100644 --- a/test/unit/test_jinja.py +++ b/test/unit/test_jinja.py @@ -1,7 +1,7 @@ from contextlib import contextmanager import pytest import unittest -import yaml +import oyaml as yaml from dbt.clients.jinja import get_rendered from dbt.clients.jinja import get_template diff --git a/test/unit/test_main.py b/test/unit/test_main.py index 09f65aa1d7b..6827b28ba49 100644 --- a/test/unit/test_main.py +++ b/test/unit/test_main.py @@ -4,7 +4,7 @@ import unittest from unittest import mock -import yaml +import oyaml as yaml from dbt import main diff --git a/test/unit/test_manifest_selectors.py b/test/unit/test_manifest_selectors.py index d7e7c3d1fe8..cf3f31319df 100644 --- a/test/unit/test_manifest_selectors.py +++ b/test/unit/test_manifest_selectors.py @@ -1,6 +1,6 @@ import dbt.exceptions import textwrap -import yaml +import oyaml as yaml import unittest from dbt.config.selectors import SelectorDict @@ -71,7 +71,7 @@ def test_single_string_definition(self): definition = sel_dict['nightly_selector']['definition'] self.assertEqual(expected, definition) - + def test_single_key_value_definition(self): dct = get_selector_dict('''\ selectors: diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index fc60b39a111..ca6f6c81802 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -3,7 +3,7 @@ from unittest import mock import os -import yaml +import oyaml as yaml import dbt.flags import dbt.parser diff --git a/test/unit/test_selector_errors.py b/test/unit/test_selector_errors.py index a5019c1211a..47ea553a084 100644 --- a/test/unit/test_selector_errors.py +++ b/test/unit/test_selector_errors.py @@ -1,6 +1,6 @@ import dbt.exceptions import textwrap -import yaml +import oyaml as yaml import unittest from dbt.config.selectors import ( selector_config_from_data From 2a29dbd1b764996b37899838e4a333300e56a022 Mon Sep 17 00:00:00 2001 From: NiallRees Date: Sun, 15 Aug 2021 22:30:11 +0100 Subject: [PATCH 03/17] Make mypy happy --- core/dbt/clients/yaml_helper.py | 5 ++--- core/dbt/task/init.py | 6 +++--- mypy.ini | 3 +++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py index 1c34ed41cc2..9ed41d7c6ea 100644 --- a/core/dbt/clients/yaml_helper.py +++ b/core/dbt/clients/yaml_helper.py @@ -1,17 +1,16 @@ import dbt.exceptions from typing import Any, Dict, Optional import oyaml as yaml -import yaml.scanner # the C version is faster, but it doesn't always exist try: - from yaml import ( + from oyaml import ( CLoader as Loader, CSafeLoader as SafeLoader, CDumper as Dumper ) except ImportError: - from yaml import ( # type: ignore # noqa: F401 + from oyaml import ( # type: ignore # noqa: F401 Loader, SafeLoader, Dumper ) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index edb754e6a99..5bc17ced116 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -105,7 +105,7 @@ def generate_target_from_input( ]) click.echo(options_msg) numeric_choice = click.prompt( - f"Desired {choice_type} option (enter a number)", type=int + f"Desired {choice_type} option (enter a number)", type=click.INT ) choice = option_list[numeric_choice - 1] # Complete the chosen option's values in a recursive call @@ -185,7 +185,7 @@ def create_profile_from_scratch(self, adapter: str): # Line below raises an exception if the specified adapter is not found load_plugin(adapter) adapter_path = get_include_paths(adapter)[0] - target_options_path = adapter_path / 'target_options.yml' + target_options_path = adapter_path / "target_options.yml" if target_options_path.exists(): with open(target_options_path) as f: @@ -240,7 +240,7 @@ def ask_for_adapter_choice(self) -> str: click.echo("\n".join([ f"[{n+1}] {v}" for n, v in enumerate(available_adapters) ])) - numeric_choice = click.prompt("Enter a number", type=int) + numeric_choice = click.prompt("Enter a number", type=click.INT) return available_adapters[numeric_choice - 1] def run(self): diff --git a/mypy.ini b/mypy.ini index 51fada1b1dc..e27ef73d8f0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,6 @@ [mypy] mypy_path = ./third-party-stubs namespace_packages = True + +[mypy-oyaml.*] +ignore_missing_imports = True \ No newline at end of file From 164e983061e5336fd636598ebcbfea5333048d17 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Sun, 5 Sep 2021 17:08:31 +0100 Subject: [PATCH 04/17] Further dev --- .../include/starter_project/dbt_project.yml | 8 +- core/dbt/task/init.py | 138 +++++++++++------- .../dbt/include/bigquery/target_options.yml | 23 ++- .../dbt/include/postgres/target_options.yml | 25 +++- .../dbt/include/redshift/target_options.yml | 28 ++-- .../dbt/include/snowflake/target_options.yml | 35 +++-- 6 files changed, 169 insertions(+), 88 deletions(-) diff --git a/core/dbt/include/starter_project/dbt_project.yml b/core/dbt/include/starter_project/dbt_project.yml index e5c84238977..54cc3cfad17 100644 --- a/core/dbt/include/starter_project/dbt_project.yml +++ b/core/dbt/include/starter_project/dbt_project.yml @@ -2,12 +2,12 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'my_new_project' +name: '{project_name}' version: '1.0.0' config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: 'default' +profile: '{profile_name}' # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be @@ -30,9 +30,9 @@ clean-targets: # directories to be removed by `dbt clean` # In this example config, we tell dbt to build all models in the example/ directory # as tables. These settings can be overridden in the individual model files -# using the `{{ config(...) }}` macro. +# using the `{{{{ config(...) }}}}` macro. models: - my_new_project: + {project_name}: # Config indicated by + and applies to all files under models/example/ example: +materialized: view diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 5bc17ced116..4dfa63ee052 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -1,7 +1,8 @@ import copy import os +import re import shutil -from typing import Tuple +from typing import Optional import oyaml as yaml import click @@ -41,6 +42,15 @@ Happy modeling! """ +# https://click.palletsprojects.com/en/8.0.x/api/?highlight=float#types +click_type_mapping = { + "string": click.STRING, + "int": click.INT, + "float": click.FLOAT, + "bool": click.BOOL, + None: None +} + class InitTask(BaseTask): def copy_starter_repo(self, project_name): @@ -57,7 +67,7 @@ def create_profiles_dir(self, profiles_dir: str) -> bool: return True return False - def create_profile_from_sample(self, adapter: str): + def create_profile_from_sample(self, adapter: str, profile_name: str): """Create a profile entry using the adapter's sample_profiles.yml""" # Line below raises an exception if the specified adapter is not found load_plugin(adapter) @@ -68,13 +78,24 @@ def create_profile_from_sample(self, adapter: str): logger.debug(f"No sample profile found for {adapter}.") else: with open(sample_profiles_path, "r") as f: - # Ignore the name given in the sample_profiles.yml - profile = list(yaml.load(f).values())[0] - profiles_filepath, profile_name = self.write_profile(profile) + sample_profile = f.read() + sample_profile_name = list(yaml.load(sample_profile).keys())[0] + sample_profile = re.sub( + f"^{sample_profile_name}:", + f"{profile_name}:", + sample_profile + ) + profiles_filepath = os.path.join(dbt.config.PROFILES_DIR, "profiles.yml") + if os.path.exists(profiles_filepath): + with open(profiles_filepath, "a") as f: + f.write("\n" + sample_profile) + else: + with open(profiles_filepath, "w") as f: + f.write(sample_profile) logger.info( f"Profile {profile_name} written to {profiles_filepath} " - "using sample configuration. Once updated " - "you'll be able to start developing with dbt." + "using sample configuration. Once updated, you'll be able to " + "start developing with dbt." ) def get_addendum(self, project_name: str, profiles_path: str) -> str: @@ -116,21 +137,17 @@ def generate_target_from_input( if key.startswith("_fixed"): # _fixed prefixed keys are not presented to the user target[key[7:]] = value - elif isinstance(value, str) and (value[0] + value[-1] == "[]"): - # A square bracketed value is used as a hint - hide_input = key == "password" - target[key] = click.prompt( - f"{key} ({value[1:-1]})", hide_input=hide_input - ) - elif isinstance(value, list): - # A list can be used to provide both a hint and a default - target[key] = click.prompt( - f"{key} ({value[0]})", default=value[1] - ) else: - # All other values are used as defaults + hide_input = value.get("hide_input", False) + default = value.get("default", None) + hint = value.get("hint", None) + type = click_type_mapping[value.get("type", None)] + text = key + (f" ({hint})" if hint else "") target[key] = click.prompt( - key, default=target_options_local[key] + text, + default=default, + hide_input=hide_input, + type=type ) return target @@ -143,16 +160,13 @@ def get_profile_name_from_current_project(self) -> str: return dbt_project["profile"] def write_profile( - self, profile: dict, profile_name: str = None - ) -> Tuple[str, str]: + self, profile: dict, profile_name: str + ) -> str: """Given a profile, write it to the current project's profiles.yml. This will overwrite any profile with a matching name.""" - profiles_file = os.path.join(dbt.config.PROFILES_DIR, "profiles.yml") - profile_name = ( - profile_name or self.get_profile_name_from_current_project() - ) - if os.path.exists(profiles_file): - with open(profiles_file, "r+") as f: + profiles_filepath = os.path.join(dbt.config.PROFILES_DIR, "profiles.yml") + if os.path.exists(profiles_filepath): + with open(profiles_filepath, "r+") as f: profiles = yaml.load(f) or {} profiles[profile_name] = profile f.seek(0) @@ -160,11 +174,11 @@ def write_profile( f.truncate() else: profiles = {profile_name: profile} - with open(profiles_file, "w") as f: + with open(profiles_filepath, "w") as f: yaml.dump(profiles, f) - return profiles_file, profile_name + return profiles_filepath - def create_profile_from_target_options(self, target_options: dict): + def create_profile_from_target_options(self, target_options: dict, profile_name: str): """Create and write a profile using the supplied target_options.""" target = self.generate_target_from_input(target_options) profile = { @@ -173,13 +187,13 @@ def create_profile_from_target_options(self, target_options: dict): }, "target": "dev" } - profiles_filepath, profile_name = self.write_profile(profile) + profiles_filepath = self.write_profile(profile, profile_name) logger.info( f"Profile {profile_name} written to {profiles_filepath} using " - "your supplied values." + "your supplied values. Run 'dbt debug' to validate the connection." ) - def create_profile_from_scratch(self, adapter: str): + def create_profile_from_scratch(self, adapter: str, profile_name: str): """Create a profile without defaults using target_options.yml if available, or sample_profiles.yml as a fallback.""" # Line below raises an exception if the specified adapter is not found @@ -190,13 +204,16 @@ def create_profile_from_scratch(self, adapter: str): if target_options_path.exists(): with open(target_options_path) as f: target_options = yaml.load(f) - self.create_profile_from_target_options(target_options) + self.create_profile_from_target_options(target_options, profile_name) else: # For adapters without a target_options.yml defined, fallback on # sample_profiles.yml - self.create_profile_from_sample(adapter) + self.create_profile_from_sample(adapter, profile_name) - def check_if_can_write_profile(self, profile_name: str = None) -> bool: + def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool: + """Using either a provided profile name or that specified in dbt_project.yml, + check if the profile already exists in profiles.yml, and if so ask the + user whether to proceed and overwrite it.""" profiles_file = os.path.join(dbt.config.PROFILES_DIR, "profiles.yml") if not os.path.exists(profiles_file): return True @@ -221,16 +238,17 @@ def create_profile_using_profile_template(self): profile_name = list(profile_template["profile"].keys())[0] self.check_if_can_write_profile(profile_name) render_vars = {} - for template_variable in profile_template["vars"]: + for template_variable in profile_template["prompts"]: render_vars[template_variable] = click.prompt(template_variable) profile = profile_template["profile"][profile_name] profile_str = yaml.dump(profile) profile_str = Template(profile_str).render(vars=render_vars) profile = yaml.load(profile_str) - profiles_filepath, _ = self.write_profile(profile, profile_name) + profiles_filepath = self.write_profile(profile, profile_name) logger.info( f"Profile {profile_name} written to {profiles_filepath} using " - "profile_template.yml and your supplied values." + "profile_template.yml and your supplied values. Run 'dbt debug' " + "to validate the connection." ) def ask_for_adapter_choice(self) -> str: @@ -240,10 +258,14 @@ def ask_for_adapter_choice(self) -> str: click.echo("\n".join([ f"[{n+1}] {v}" for n, v in enumerate(available_adapters) ])) + click.echo( + "(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)" + ) numeric_choice = click.prompt("Enter a number", type=click.INT) return available_adapters[numeric_choice - 1] def run(self): + """Entry point for the init task.""" profiles_dir = dbt.config.PROFILES_DIR self.create_profiles_dir(profiles_dir) @@ -254,30 +276,44 @@ def run(self): in_project = False if in_project: + # When dbt init is run inside an existing project, + # just setup the user's profile. logger.info("Setting up your profile.") + default_profile_name = self.get_profile_name_from_current_project() if os.path.exists("profile_template.yml"): - self.create_profile_using_profile_template() + self.create_profile_using_profile_template(default_profile_name) else: - if not self.check_if_can_write_profile(): + if not self.check_if_can_write_profile(profile_name=default_profile_name): return adapter = self.ask_for_adapter_choice() self.create_profile_from_scratch( - adapter + adapter, profile_name=default_profile_name ) else: - project_dir = click.prompt("What is the desired project name?") - if os.path.exists(project_dir): + # When dbt init is run outside of an existing project, + # create a new project and set up their profile. + project_name = click.prompt("What is the desired project name?") + if os.path.exists(project_name): logger.info( - f"Existing project found at directory {project_dir}" + f"Existing project found at directory {project_name}" ) return - self.copy_starter_repo(project_dir) - os.chdir(project_dir) - if not self.check_if_can_write_profile(): + self.copy_starter_repo(project_name) + os.chdir(project_name) + with open("dbt_project.yml", "r+") as f: + content = f"{f.read()}".format( + project_name=project_name, + profile_name=project_name + ) + f.seek(0) + f.write(content) + f.truncate() + + if not self.check_if_can_write_profile(profile_name=project_name): return adapter = self.ask_for_adapter_choice() self.create_profile_from_scratch( - adapter + adapter, profile_name=project_name ) - logger.info(self.get_addendum(project_dir, profiles_dir)) + logger.info(self.get_addendum(project_name, profiles_dir)) diff --git a/plugins/bigquery/dbt/include/bigquery/target_options.yml b/plugins/bigquery/dbt/include/bigquery/target_options.yml index cfd5113b621..b76400f2022 100644 --- a/plugins/bigquery/dbt/include/bigquery/target_options.yml +++ b/plugins/bigquery/dbt/include/bigquery/target_options.yml @@ -4,11 +4,22 @@ _choose_authentication_method: _fixed_method: oauth service_account: _fixed_method: service-account - keyfile: '[/path/to/bigquery/keyfile.json]' -project: '[GCP project id]' -dataset: '[the name of your dbt dataset]' -threads: '[1 or more]' -timeout_seconds: 300 -location: '[one of US or EU]' + keyfile: + hint: '/path/to/bigquery/keyfile.json' +project: + hint: 'GCP project id' +dataset: + hint: 'the name of your dbt dataset' +threads: + hint: '1 or more' + type: 'int' +timeout_seconds: + default: 300 + type: 'int' +_choose_location: + US: + _fixed_location: US + EU: + _fixed_location: EU _fixed_priority: interactive _fixed_retries: 1 diff --git a/plugins/postgres/dbt/include/postgres/target_options.yml b/plugins/postgres/dbt/include/postgres/target_options.yml index 96698d7ccbf..d353304e7af 100644 --- a/plugins/postgres/dbt/include/postgres/target_options.yml +++ b/plugins/postgres/dbt/include/postgres/target_options.yml @@ -1,8 +1,19 @@ _fixed_type: postgres -threads: '[1 or more]' -host: '[host]' -port: '[port]' -user: '[dev_username]' -pass: '[dev_password]' -dbname: '[dbname]' -schema: '[dev_schema]' \ No newline at end of file +threads: + hint: '1 or more' + type: 'int' + default: 1 +host: + hint: 'hostname for the instance' +port: + default: 5432 + type: 'int' +user: + hint: 'dev username' +pass: + hint: 'dev password' + hide_input: true +dbname: + hint: 'default database that dbt will build objects in' +schema: + hint: 'default schema that dbt will build objects in' \ No newline at end of file diff --git a/plugins/redshift/dbt/include/redshift/target_options.yml b/plugins/redshift/dbt/include/redshift/target_options.yml index 20fc722b66f..16182775e02 100644 --- a/plugins/redshift/dbt/include/redshift/target_options.yml +++ b/plugins/redshift/dbt/include/redshift/target_options.yml @@ -1,13 +1,23 @@ _fixed_type: redshift +host: + hint: 'hostname.region.redshift.amazonaws.com' +port: + default: 5439 + type: 'int' +user: + hint: 'dev username' _choose_authentication_method: password: - password: '[password]' + password: + hint: 'dev password' + hide_input: true iam: - method: iam -host: '[hostname.region.redshift.amazonaws.com]' -user: '[username]' -port: 5439 -dbname: '[database name]' -schema: '[dbt schema]' -threads: '[1 or more]' -keepalives_idle: 0 # default 0, indicating the system default \ No newline at end of file + _fixed_method: iam +dbname: + hint: 'default database that dbt will build objects in' +schema: + hint: 'default schema that dbt will build objects in' +threads: + hint: '1 or more' + type: 'int' + default: 1 \ No newline at end of file diff --git a/plugins/snowflake/dbt/include/snowflake/target_options.yml b/plugins/snowflake/dbt/include/snowflake/target_options.yml index d7cfbdf34cb..9a37b1fd06e 100644 --- a/plugins/snowflake/dbt/include/snowflake/target_options.yml +++ b/plugins/snowflake/dbt/include/snowflake/target_options.yml @@ -1,15 +1,28 @@ _fixed_type: snowflake -account: '[account id + region (if applicable)]' -user: '[username]' +account: + hint: 'https://.snowflakecomputing.com' +user: + hint: 'dev username' _choose_authentication_type: password: - password: '[password]' + password: + hint: 'dev password' + hide_input: true keypair: - private_key_path: '[path/to/private.key]' - private_key_passphrase: '[passphrase for the private key, if key is encrypted]' -role: '[user role]' -database: '[database name]' -warehouse: '[warehouse name]' -schema: '[dbt schema]' -threads: '[1 or more]' -client_session_keep_alive: False \ No newline at end of file + private_key_path: + hint: 'path/to/private.key' + private_key_passphrase: + hint: 'passphrase for the private key, if key is encrypted' + hide_input: true +role: + hint: 'dev role' +warehouse: + hint: 'warehouse name' +database: + hint: 'default database that dbt will build objects in' +schema: + hint: 'default schema that dbt will build objects in' +threads: + hint: '1 or more' + type: 'int' + default: 1 \ No newline at end of file From 83c0888cdf685cfc61ac01216fedf98283537dc4 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Sun, 5 Sep 2021 17:13:55 +0100 Subject: [PATCH 05/17] Existing tests passing --- plugins/snowflake/dbt/adapters/snowflake/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/snowflake/dbt/adapters/snowflake/connections.py b/plugins/snowflake/dbt/adapters/snowflake/connections.py index fcb20c97819..8c4f8cdf949 100644 --- a/plugins/snowflake/dbt/adapters/snowflake/connections.py +++ b/plugins/snowflake/dbt/adapters/snowflake/connections.py @@ -31,7 +31,7 @@ class SnowflakeCredentials(Credentials): account: str user: str - password: str + password: Optional[str] = None warehouse: Optional[str] = None role: Optional[str] = None authenticator: Optional[str] = None From 6184fb3b8b73cfa0d7c0f515fecbe6f0c0db62d0 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Tue, 7 Sep 2021 11:56:39 +0100 Subject: [PATCH 06/17] Functioning integration test --- core/dbt/task/init.py | 34 ++++++------- test/integration/040_init_test/test_init.py | 53 ++++++++++++++------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 4dfa63ee052..bbed18e45c0 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -121,13 +121,10 @@ def generate_target_from_input( if key.startswith("_choose"): choice_type = key[8:].replace("_", " ") option_list = list(value.keys()) - options_msg = "\n".join([ + prompt_msg = "\n".join([ f"[{n+1}] {v}" for n, v in enumerate(option_list) - ]) - click.echo(options_msg) - numeric_choice = click.prompt( - f"Desired {choice_type} option (enter a number)", type=click.INT - ) + ]) + f"\nDesired {choice_type} option (enter a number)" + numeric_choice = click.prompt(prompt_msg, type=click.INT) choice = option_list[numeric_choice - 1] # Complete the chosen option's values in a recursive call target = self.generate_target_from_input( @@ -253,15 +250,14 @@ def create_profile_using_profile_template(self): def ask_for_adapter_choice(self) -> str: """Ask the user which adapter (database) they'd like to use.""" - click.echo("Which database would you like to use?") available_adapters = list(_get_adapter_plugin_names()) - click.echo("\n".join([ - f"[{n+1}] {v}" for n, v in enumerate(available_adapters) - ])) - click.echo( - "(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)" + prompt_msg = ( + "Which database would you like to use?\n" + + "\n".join([f"[{n+1}] {v}" for n, v in enumerate(available_adapters)]) + + "\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)" + + "\n\nEnter a number" ) - numeric_choice = click.prompt("Enter a number", type=click.INT) + numeric_choice = click.prompt(prompt_msg, type=click.INT) return available_adapters[numeric_choice - 1] def run(self): @@ -279,23 +275,23 @@ def run(self): # When dbt init is run inside an existing project, # just setup the user's profile. logger.info("Setting up your profile.") - default_profile_name = self.get_profile_name_from_current_project() + profile_name = self.get_profile_name_from_current_project() if os.path.exists("profile_template.yml"): - self.create_profile_using_profile_template(default_profile_name) + self.create_profile_using_profile_template(profile_name) else: - if not self.check_if_can_write_profile(profile_name=default_profile_name): + if not self.check_if_can_write_profile(profile_name=profile_name): return adapter = self.ask_for_adapter_choice() self.create_profile_from_scratch( - adapter, profile_name=default_profile_name + adapter, profile_name=profile_name ) else: # When dbt init is run outside of an existing project, - # create a new project and set up their profile. + # create a new project and set up the user's profile. project_name = click.prompt("What is the desired project name?") if os.path.exists(project_name): logger.info( - f"Existing project found at directory {project_name}" + f"A project called {project_name} already exists here." ) return diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index bb2761df81c..433d904b4e9 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -1,8 +1,11 @@ - -from test.integration.base import DBTIntegrationTest, use_profile import os import shutil -import oyaml as yaml +from unittest import mock +from unittest.mock import Mock, call + +import click + +from test.integration.base import DBTIntegrationTest, use_profile class TestInit(DBTIntegrationTest): @@ -26,18 +29,32 @@ def models(self): return "models" @use_profile('postgres') - def test_postgres_init_task(self): - project_name = self.get_project_name() - self.run_dbt(['init', project_name, '--adapter', 'postgres']) - - assert os.path.exists(project_name) - project_file = os.path.join(project_name, 'dbt_project.yml') - assert os.path.exists(project_file) - with open(project_file) as fp: - project_data = yaml.safe_load(fp.read()) - - assert 'config-version' in project_data - assert project_data['config-version'] == 2 - - git_dir = os.path.join(project_name, '.git') - assert not os.path.exists(git_dir) + @mock.patch('click.confirm') + @mock.patch('click.prompt') + def test_postgres_init_task_in_project(self, mock_prompt, mock_confirm): + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.confirm.side_effect = ["y"] + manager.prompt.side_effect = [ + 1, + 4, + "localhost", + 5432, + "test_user", + "test_password", + "test_db", + "test_schema", + ] + self.run_dbt(['init', '--profiles-dir', 'dbt-profile'], profiles_dir=False) + manager.assert_has_calls([ + call.confirm('The profile test already exists in /Users/niall.woodward/.dbt/profiles.yml. Continue and overwrite it?'), + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) From 182c82dcbbc6e04c3c311f0e3ae9fba88d0ea436 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Mon, 4 Oct 2021 10:52:37 +0100 Subject: [PATCH 07/17] Passing integration test --- test/integration/040_init_test/test_init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 433d904b4e9..85a2e97bf50 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -46,9 +46,9 @@ def test_postgres_init_task_in_project(self, mock_prompt, mock_confirm): "test_db", "test_schema", ] - self.run_dbt(['init', '--profiles-dir', 'dbt-profile'], profiles_dir=False) + self.run_dbt(['init']) manager.assert_has_calls([ - call.confirm('The profile test already exists in /Users/niall.woodward/.dbt/profiles.yml. Continue and overwrite it?'), + call.confirm(f'The profile test already exists in {self.test_root_dir}/profiles.yml. Continue and overwrite it?'), call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), From c105ab75ad310fa1731b1d099ec709e5fb846159 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Mon, 4 Oct 2021 13:22:58 +0100 Subject: [PATCH 08/17] Integration tests --- core/dbt/task/init.py | 34 +-- test/integration/040_init_test/test_init.py | 226 +++++++++++++++++++- 2 files changed, 246 insertions(+), 14 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 5a0b989e016..a69e7b68673 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -1,5 +1,6 @@ import copy import os +from pathlib import Path import re import shutil from typing import Optional @@ -61,7 +62,8 @@ def copy_starter_repo(self, project_name): def create_profiles_dir(self, profiles_dir: str) -> bool: """Create the user's profiles directory if it doesn't already exist.""" - if not os.path.exists(profiles_dir): + profiles_path = Path(profiles_dir) + if profiles_path.exists(): msg = "Creating dbt configuration folder at {}" logger.info(msg.format(profiles_dir)) dbt.clients.system.make_directory(profiles_dir) @@ -69,7 +71,9 @@ def create_profiles_dir(self, profiles_dir: str) -> bool: return False def create_profile_from_sample(self, adapter: str, profile_name: str): - """Create a profile entry using the adapter's sample_profiles.yml""" + """Create a profile entry using the adapter's sample_profiles.yml + + Renames the profile in sample_profiles.yml to match that of the project.""" # Line below raises an exception if the specified adapter is not found load_plugin(adapter) adapter_path = get_include_paths(adapter)[0] @@ -81,13 +85,15 @@ def create_profile_from_sample(self, adapter: str, profile_name: str): with open(sample_profiles_path, "r") as f: sample_profile = f.read() sample_profile_name = list(yaml.load(sample_profile).keys())[0] + # Use a regex to replace the name of the sample_profile with + # that of the project without losing any comments from the sample sample_profile = re.sub( f"^{sample_profile_name}:", f"{profile_name}:", sample_profile ) - profiles_filepath = os.path.join(flags.PROFILES_DIR, "profiles.yml") - if os.path.exists(profiles_filepath): + profiles_filepath = Path(flags.PROFILES_DIR) / Path("profiles.yml") + if profiles_filepath.exists(): with open(profiles_filepath, "a") as f: f.write("\n" + sample_profile) else: @@ -159,11 +165,11 @@ def get_profile_name_from_current_project(self) -> str: def write_profile( self, profile: dict, profile_name: str - ) -> str: + ) -> Path: """Given a profile, write it to the current project's profiles.yml. This will overwrite any profile with a matching name.""" - profiles_filepath = os.path.join(flags.PROFILES_DIR, "profiles.yml") - if os.path.exists(profiles_filepath): + profiles_filepath = Path(flags.PROFILES_DIR) / Path("profiles.yml") + if profiles_filepath.exists(): with open(profiles_filepath, "r+") as f: profiles = yaml.load(f) or {} profiles[profile_name] = profile @@ -212,8 +218,8 @@ def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool """Using either a provided profile name or that specified in dbt_project.yml, check if the profile already exists in profiles.yml, and if so ask the user whether to proceed and overwrite it.""" - profiles_file = os.path.join(flags.PROFILES_DIR, "profiles.yml") - if not os.path.exists(profiles_file): + profiles_file = Path(flags.PROFILES_DIR) / Path("profiles.yml") + if not profiles_file.exists(): return True profile_name = ( profile_name or self.get_profile_name_from_current_project() @@ -240,7 +246,7 @@ def create_profile_using_profile_template(self): render_vars[template_variable] = click.prompt(template_variable) profile = profile_template["profile"][profile_name] profile_str = yaml.dump(profile) - profile_str = Template(profile_str).render(vars=render_vars) + profile_str = Template(profile_str).render(render_vars) profile = yaml.load(profile_str) profiles_filepath = self.write_profile(profile, profile_name) logger.info( @@ -277,8 +283,9 @@ def run(self): # just setup the user's profile. logger.info("Setting up your profile.") profile_name = self.get_profile_name_from_current_project() - if os.path.exists("profile_template.yml"): - self.create_profile_using_profile_template(profile_name) + profile_template_path = Path("profile_template.yml") + if profile_template_path.exists(): + self.create_profile_using_profile_template() else: if not self.check_if_can_write_profile(profile_name=profile_name): return @@ -290,7 +297,8 @@ def run(self): # When dbt init is run outside of an existing project, # create a new project and set up the user's profile. project_name = click.prompt("What is the desired project name?") - if os.path.exists(project_name): + project_path = Path(project_name) + if project_path.exists(): logger.info( f"A project called {project_name} already exists here." ) diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 85a2e97bf50..1a436d33ab6 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -2,6 +2,7 @@ import shutil from unittest import mock from unittest.mock import Mock, call +from pathlib import Path import click @@ -31,7 +32,7 @@ def models(self): @use_profile('postgres') @mock.patch('click.confirm') @mock.patch('click.prompt') - def test_postgres_init_task_in_project(self, mock_prompt, mock_confirm): + def test_postgres_init_task_in_project_with_existing_profiles_yml(self, mock_prompt, mock_confirm): manager = Mock() manager.attach_mock(mock_prompt, 'prompt') manager.attach_mock(mock_confirm, 'confirm') @@ -46,7 +47,9 @@ def test_postgres_init_task_in_project(self, mock_prompt, mock_confirm): "test_db", "test_schema", ] + self.run_dbt(['init']) + manager.assert_has_calls([ call.confirm(f'The profile test already exists in {self.test_root_dir}/profiles.yml. Continue and overwrite it?'), call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), @@ -58,3 +61,224 @@ def test_postgres_init_task_in_project(self, mock_prompt, mock_confirm): call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + assert f.read() == """config: + send_anonymous_usage_stats: false +test: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: test_user + pass: test_password + dbname: test_db + schema: test_schema + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + @mock.patch.object(Path, 'exists', autospec=True) + def test_postgres_init_task_in_project_without_existing_profiles_yml(self, exists, mock_prompt, mock_confirm): + + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return { + 'profiles.yml': False + }.get(path.name, os.path.exists(path)) + + exists.side_effect = exists_side_effect + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.prompt.side_effect = [ + 1, + 4, + "localhost", + 5432, + "test_user", + "test_password", + "test_db", + "test_schema", + ] + + self.run_dbt(['init']) + + manager.assert_has_calls([ + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + assert f.read() == """test: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: test_user + pass: test_password + dbname: test_db + schema: test_schema + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + @mock.patch.object(Path, 'exists', autospec=True) + def test_postgres_init_task_in_project_without_existing_profiles_yml_or_target_options(self, exists, mock_prompt, mock_confirm): + + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return { + 'profiles.yml': False, + 'target_options.yml': False, + }.get(path.name, os.path.exists(path)) + + exists.side_effect = exists_side_effect + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.prompt.side_effect = [ + 1, + ] + self.run_dbt(['init']) + manager.assert_has_calls([ + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + ]) + with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + assert f.read() == """test: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: test_user + pass: test_password + dbname: test_db + schema: test_schema + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + @mock.patch.object(Path, 'exists', autospec=True) + def test_postgres_init_task_in_project_without_existing_profiles_yml_or_target_options(self, exists, mock_prompt, mock_confirm): + + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return { + 'profiles.yml': False, + 'target_options.yml': False, + }.get(path.name, os.path.exists(path)) + + exists.side_effect = exists_side_effect + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.prompt.side_effect = [ + 1, + ] + self.run_dbt(['init']) + manager.assert_has_calls([ + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + assert f.read() == """test: + outputs: + + dev: + type: postgres + threads: [1 or more] + host: [host] + port: [port] + user: [dev_username] + pass: [dev_password] + dbname: [dbname] + schema: [dev_schema] + + prod: + type: postgres + threads: [1 or more] + host: [host] + port: [port] + user: [prod_username] + pass: [prod_password] + dbname: [dbname] + schema: [prod_schema] + + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + @mock.patch.object(Path, 'exists', autospec=True) + def test_postgres_init_task_in_project_with_profile_template_without_existing_profiles_yml(self, exists, mock_prompt, mock_confirm): + + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return { + 'profiles.yml': False, + }.get(path.name, os.path.exists(path)) + exists.side_effect = exists_side_effect + + with open("profile_template.yml", 'w') as f: + f.write("""prompts: + - pg_username + - pg_password +profile: + my_profile: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: "{{ pg_username }}" + pass: "{{ pg_password }}" + dbname: my_db + schema: my_schema + target: dev""") + + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.prompt.side_effect = [ + "test_username", + "test_password" + ] + self.run_dbt(['init']) + manager.assert_has_calls([ + call.prompt('pg_username'), + call.prompt('pg_password') + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + assert f.read() == """my_profile: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: test_username + pass: test_password + dbname: my_db + schema: my_schema + target: dev +""" From 9bd1bed89a5d43dd672973d3c613ece47b8a7614 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Mon, 4 Oct 2021 14:03:41 +0100 Subject: [PATCH 09/17] Add changelog entry --- CHANGELOG.md | 2 + test/integration/040_init_test/test_init.py | 72 +++++---------------- 2 files changed, 19 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a38ab631df..c54a9d9cb4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Turns on the static parser by default and adds the flag `--no-static-parser` to disable it. ([#3377](https://github.com/dbt-labs/dbt/issues/3377), [#3939](https://github.com/dbt-labs/dbt/pull/3939)) - Generic test FQNs have changed to include the relative path, resource, and column (if applicable) where they are defined. This makes it easier to configure them from the `tests` block in `dbt_project.yml` ([#3259](https://github.com/dbt-labs/dbt/pull/3259), [#3880](https://github.com/dbt-labs/dbt/pull/3880) - Turn on partial parsing by default ([#3867](https://github.com/dbt-labs/dbt/issues/3867), [#3989](https://github.com/dbt-labs/dbt/issues/3989)) +- `dbt init` is now interactive, generating profiles.yml when run inside existing project ([#3625](https://github.com/dbt-labs/dbt/pull/3625)) ### Fixes - Add generic tests defined on sources to the manifest once, not twice ([#3347](https://github.com/dbt-labs/dbt/issues/3347), [#3880](https://github.com/dbt-labs/dbt/pull/3880)) @@ -29,6 +30,7 @@ Contributors: - [@dave-connors-3](https://github.com/dave-connors-3) ([#3920](https://github.com/dbt-labs/dbt/issues/3920)) - [@kadero](https://github.com/kadero) ([#3952](https://github.com/dbt-labs/dbt/issues/3952)) +- [@NiallRees](https://github.com/NiallRees) ([#3625](https://github.com/dbt-labs/dbt/pull/3625)) ## dbt 0.21.0 (Release TBD) diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 1a436d33ab6..431c0142e62 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -40,12 +40,12 @@ def test_postgres_init_task_in_project_with_existing_profiles_yml(self, mock_pro manager.prompt.side_effect = [ 1, 4, - "localhost", + 'localhost', 5432, - "test_user", - "test_password", - "test_db", - "test_schema", + 'test_user', + 'test_password', + 'test_db', + 'test_schema', ] self.run_dbt(['init']) @@ -62,7 +62,7 @@ def test_postgres_init_task_in_project_with_existing_profiles_yml(self, mock_pro call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) ]) - with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: assert f.read() == """config: send_anonymous_usage_stats: false test: @@ -97,12 +97,12 @@ def exists_side_effect(path): manager.prompt.side_effect = [ 1, 4, - "localhost", + 'localhost', 5432, - "test_user", - "test_password", - "test_db", - "test_schema", + 'test_user', + 'test_password', + 'test_db', + 'test_schema', ] self.run_dbt(['init']) @@ -118,7 +118,7 @@ def exists_side_effect(path): call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) ]) - with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: assert f.read() == """test: outputs: dev: @@ -157,47 +157,8 @@ def exists_side_effect(path): manager.assert_has_calls([ call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), ]) - with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: - assert f.read() == """test: - outputs: - dev: - type: postgres - threads: 4 - host: localhost - port: 5432 - user: test_user - pass: test_password - dbname: test_db - schema: test_schema - target: dev -""" - - @use_profile('postgres') - @mock.patch('click.confirm') - @mock.patch('click.prompt') - @mock.patch.object(Path, 'exists', autospec=True) - def test_postgres_init_task_in_project_without_existing_profiles_yml_or_target_options(self, exists, mock_prompt, mock_confirm): - def exists_side_effect(path): - # Override responses on specific files, default to 'real world' if not overriden - return { - 'profiles.yml': False, - 'target_options.yml': False, - }.get(path.name, os.path.exists(path)) - - exists.side_effect = exists_side_effect - manager = Mock() - manager.attach_mock(mock_prompt, 'prompt') - manager.attach_mock(mock_confirm, 'confirm') - manager.prompt.side_effect = [ - 1, - ] - self.run_dbt(['init']) - manager.assert_has_calls([ - call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), - ]) - - with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: assert f.read() == """test: outputs: @@ -259,8 +220,8 @@ def exists_side_effect(path): manager.attach_mock(mock_prompt, 'prompt') manager.attach_mock(mock_confirm, 'confirm') manager.prompt.side_effect = [ - "test_username", - "test_password" + 'test_username', + 'test_password' ] self.run_dbt(['init']) manager.assert_has_calls([ @@ -268,7 +229,7 @@ def exists_side_effect(path): call.prompt('pg_password') ]) - with open(os.path.join(self.test_root_dir, 'profiles.yml'), "r") as f: + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: assert f.read() == """my_profile: outputs: dev: @@ -282,3 +243,4 @@ def exists_side_effect(path): schema: my_schema target: dev """ + From 622840e18f19a4d0dbf61cde6263069e72827cf9 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Tue, 5 Oct 2021 10:41:48 +0100 Subject: [PATCH 10/17] Add integration test for init outside of project --- test/integration/040_init_test/test_init.py | 114 ++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 431c0142e62..1099a7053ac 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -244,3 +244,117 @@ def exists_side_effect(path): target: dev """ + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + def test_postgres_init_task_outside_of_project(self, mock_prompt, mock_confirm): + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + + # Start by removing the dbt_project.yml so that we're not in an existing project + os.remove('dbt_project.yml') + + project_name = self.get_project_name() + manager.prompt.side_effect = [ + project_name, + 1, + 4, + 'localhost', + 5432, + 'test_username', + 'test_password', + 'test_db', + 'test_schema', + ] + self.run_dbt(['init']) + manager.assert_has_calls([ + call.prompt('What is the desired project name?'), + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == f"""config: + send_anonymous_usage_stats: false +test: + outputs: + default2: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: root + pass: password + dbname: dbt + schema: {self.unique_schema()} + noaccess: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: noaccess + pass: password + dbname: dbt + schema: {self.unique_schema()} + target: default2 +{project_name}: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: test_username + pass: test_password + dbname: test_db + schema: test_schema + target: dev +""" + + with open(os.path.join(self.test_root_dir, project_name, 'dbt_project.yml'), 'r') as f: + assert f.read() == f""" +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: '{project_name}' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: '{project_name}' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{{{ config(...) }}}}` macro. +models: + {project_name}: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: view +""" From 5ceb484be914dd5fc1f133a00a050153e87d0cc6 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Wed, 6 Oct 2021 19:15:23 +0100 Subject: [PATCH 11/17] Fall back to target_options.yml when invalid profile_template.yml is provided --- core/dbt/task/init.py | 19 ++++--- test/integration/040_init_test/test_init.py | 56 +++++++++++++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index a69e7b68673..e608e193649 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -285,14 +285,19 @@ def run(self): profile_name = self.get_profile_name_from_current_project() profile_template_path = Path("profile_template.yml") if profile_template_path.exists(): - self.create_profile_using_profile_template() - else: - if not self.check_if_can_write_profile(profile_name=profile_name): + try: + # This relies on a valid profile_template.yml from the user, + # so use a try: except to fall back to the default on failure + self.create_profile_using_profile_template() return - adapter = self.ask_for_adapter_choice() - self.create_profile_from_scratch( - adapter, profile_name=profile_name - ) + except Exception: + logger.info("Invalid profile_template.yml in project.") + if not self.check_if_can_write_profile(profile_name=profile_name): + return + adapter = self.ask_for_adapter_choice() + self.create_profile_from_scratch( + adapter, profile_name=profile_name + ) else: # When dbt init is run outside of an existing project, # create a new project and set up the user's profile. diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 1099a7053ac..33447c62be4 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -244,6 +244,62 @@ def exists_side_effect(path): target: dev """ + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + def test_postgres_init_task_in_project_with_invalid_profile_template(self, mock_prompt, mock_confirm): + """Test that when an invalid profile_template.yml is provided, + init command falls back to the target_options.yml""" + + with open("profile_template.yml", 'w') as f: + f.write("""invalid template""") + + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.confirm.side_effect = ["y"] + manager.prompt.side_effect = [ + 1, + 4, + 'localhost', + 5432, + 'test_username', + 'test_password', + 'test_db', + 'test_schema', + ] + + self.run_dbt(['init']) + + manager.assert_has_calls([ + call.confirm(f'The profile test already exists in {self.test_root_dir}/profiles.yml. Continue and overwrite it?'), + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == """config: + send_anonymous_usage_stats: false +test: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: test_username + pass: test_password + dbname: test_db + schema: test_schema + target: dev +""" + @use_profile('postgres') @mock.patch('click.confirm') @mock.patch('click.prompt') From 6dcae8c365e8f2ef38e7d1a44c9af9ed38d4e419 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Wed, 6 Oct 2021 21:03:45 +0100 Subject: [PATCH 12/17] Use built-in yaml with exception of in init --- core/dbt/clients/yaml_helper.py | 6 +++--- core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py | 2 +- .../006_simple_dependency_test/test_local_dependency.py | 2 +- .../007_graph_selection_tests/test_graph_selection.py | 2 +- .../007_graph_selection_tests/test_intersection_syntax.py | 2 +- .../007_graph_selection_tests/test_tag_selection.py | 2 +- .../015_cli_invocation_tests/test_cli_invocation.py | 2 +- test/integration/028_cli_vars/test_cli_vars.py | 2 +- test/integration/042_sources_test/test_sources.py | 2 +- .../044_run_operations_test/test_run_operations.py | 2 +- test/integration/049_dbt_debug_test/test_debug.py | 2 +- .../054_adapter_methods_test/test_adapter_methods.py | 2 +- .../056_column_type_tests/test_alter_column_types.py | 2 +- test/integration/069_build_test/test_build.py | 2 +- test/integration/base.py | 2 +- test/rpc/conftest.py | 2 +- test/rpc/test_build.py | 2 +- test/rpc/test_test.py | 2 +- test/rpc/util.py | 2 +- test/unit/test_config.py | 2 +- test/unit/test_graph_selector_parsing.py | 2 +- test/unit/test_jinja.py | 2 +- test/unit/test_manifest_selectors.py | 2 +- test/unit/test_parser.py | 2 +- test/unit/test_selector_errors.py | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/core/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py index 9ed41d7c6ea..34ce235767b 100644 --- a/core/dbt/clients/yaml_helper.py +++ b/core/dbt/clients/yaml_helper.py @@ -1,16 +1,16 @@ import dbt.exceptions from typing import Any, Dict, Optional -import oyaml as yaml +import yaml # the C version is faster, but it doesn't always exist try: - from oyaml import ( + from yaml import ( CLoader as Loader, CSafeLoader as SafeLoader, CDumper as Dumper ) except ImportError: - from oyaml import ( # type: ignore # noqa: F401 + from yaml import ( # type: ignore # noqa: F401 Loader, SafeLoader, Dumper ) diff --git a/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py b/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py index c803b5b7e7f..29a2eed1e7c 100644 --- a/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py +++ b/core/scripts/upgrade_dbt_schema_tests_v1_to_v2.py @@ -5,7 +5,7 @@ import os import re import sys -import oyaml as yaml +import yaml LOGGER = logging.getLogger('upgrade_dbt_schema') LOGFILE = 'upgrade_dbt_schema_tests_v1_to_v2.txt' diff --git a/test/integration/006_simple_dependency_test/test_local_dependency.py b/test/integration/006_simple_dependency_test/test_local_dependency.py index d637c15e00d..0f8233fd868 100644 --- a/test/integration/006_simple_dependency_test/test_local_dependency.py +++ b/test/integration/006_simple_dependency_test/test_local_dependency.py @@ -2,7 +2,7 @@ import os import json import shutil -import oyaml as yaml +import yaml from unittest import mock import dbt.semver diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index 68598e10b0e..dba2950c26c 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml import json import os diff --git a/test/integration/007_graph_selection_tests/test_intersection_syntax.py b/test/integration/007_graph_selection_tests/test_intersection_syntax.py index 65f0ba3cfae..d725d03c39f 100644 --- a/test/integration/007_graph_selection_tests/test_intersection_syntax.py +++ b/test/integration/007_graph_selection_tests/test_intersection_syntax.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml class TestGraphSelection(DBTIntegrationTest): diff --git a/test/integration/007_graph_selection_tests/test_tag_selection.py b/test/integration/007_graph_selection_tests/test_tag_selection.py index c5ea7434c9c..3d3b0c3baf7 100644 --- a/test/integration/007_graph_selection_tests/test_tag_selection.py +++ b/test/integration/007_graph_selection_tests/test_tag_selection.py @@ -1,6 +1,6 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml class TestGraphSelection(DBTIntegrationTest): diff --git a/test/integration/015_cli_invocation_tests/test_cli_invocation.py b/test/integration/015_cli_invocation_tests/test_cli_invocation.py index 9c881d4da80..ef898ae84fe 100644 --- a/test/integration/015_cli_invocation_tests/test_cli_invocation.py +++ b/test/integration/015_cli_invocation_tests/test_cli_invocation.py @@ -4,7 +4,7 @@ import shutil import pytest import tempfile -import oyaml as yaml +import yaml from typing import Dict diff --git a/test/integration/028_cli_vars/test_cli_vars.py b/test/integration/028_cli_vars/test_cli_vars.py index bcf6c018a84..d0873b6e107 100644 --- a/test/integration/028_cli_vars/test_cli_vars.py +++ b/test/integration/028_cli_vars/test_cli_vars.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml import json diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index 2eaf623b74d..e2c95d8175a 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -2,7 +2,7 @@ import os from datetime import datetime, timedelta -import oyaml as yaml +import yaml from dbt.exceptions import CompilationException import dbt.tracking diff --git a/test/integration/044_run_operations_test/test_run_operations.py b/test/integration/044_run_operations_test/test_run_operations.py index 10b1ec8ecd0..98715191e3e 100644 --- a/test/integration/044_run_operations_test/test_run_operations.py +++ b/test/integration/044_run_operations_test/test_run_operations.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml class TestOperations(DBTIntegrationTest): diff --git a/test/integration/049_dbt_debug_test/test_debug.py b/test/integration/049_dbt_debug_test/test_debug.py index 471cf7e9cba..8a5fbd774f3 100644 --- a/test/integration/049_dbt_debug_test/test_debug.py +++ b/test/integration/049_dbt_debug_test/test_debug.py @@ -1,7 +1,7 @@ from test.integration.base import DBTIntegrationTest, use_profile import os import re -import oyaml as yaml +import yaml import pytest diff --git a/test/integration/054_adapter_methods_test/test_adapter_methods.py b/test/integration/054_adapter_methods_test/test_adapter_methods.py index 7fb071af4ef..9093a7b76c3 100644 --- a/test/integration/054_adapter_methods_test/test_adapter_methods.py +++ b/test/integration/054_adapter_methods_test/test_adapter_methods.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml class TestBaseCaching(DBTIntegrationTest): diff --git a/test/integration/056_column_type_tests/test_alter_column_types.py b/test/integration/056_column_type_tests/test_alter_column_types.py index e2507a5b384..e06e1f5697c 100644 --- a/test/integration/056_column_type_tests/test_alter_column_types.py +++ b/test/integration/056_column_type_tests/test_alter_column_types.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml class TestAlterColumnTypes(DBTIntegrationTest): diff --git a/test/integration/069_build_test/test_build.py b/test/integration/069_build_test/test_build.py index 48a34bbd2a2..2b7b9e69936 100644 --- a/test/integration/069_build_test/test_build.py +++ b/test/integration/069_build_test/test_build.py @@ -1,5 +1,5 @@ from test.integration.base import DBTIntegrationTest, use_profile -import oyaml as yaml +import yaml class TestBuildBase(DBTIntegrationTest): diff --git a/test/integration/base.py b/test/integration/base.py index d8b4d00b830..9899aed2d64 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -13,7 +13,7 @@ from functools import wraps import pytest -import oyaml as yaml +import yaml from unittest.mock import patch import dbt.main as dbt diff --git a/test/rpc/conftest.py b/test/rpc/conftest.py index 7c1bfdfba8b..3b3a3dc262c 100644 --- a/test/rpc/conftest.py +++ b/test/rpc/conftest.py @@ -5,7 +5,7 @@ import time from typing import Dict, Any, Set -import oyaml as yaml +import yaml from dbt import flags diff --git a/test/rpc/test_build.py b/test/rpc/test_build.py index 2525302fdce..751e3db537c 100644 --- a/test/rpc/test_build.py +++ b/test/rpc/test_build.py @@ -1,6 +1,6 @@ import os import pytest -import oyaml as yaml +import yaml from .util import ( assert_has_threads, get_querier, diff --git a/test/rpc/test_test.py b/test/rpc/test_test.py index 08aba9f4bbd..fe863006002 100644 --- a/test/rpc/test_test.py +++ b/test/rpc/test_test.py @@ -1,6 +1,6 @@ import os import pytest -import oyaml as yaml +import yaml from .util import ( assert_has_threads, get_querier, diff --git a/test/rpc/util.py b/test/rpc/util.py index 320a97afd91..2c667eeb02a 100644 --- a/test/rpc/util.py +++ b/test/rpc/util.py @@ -10,7 +10,7 @@ from typing import Dict, Any, Optional, Union, List import requests -import oyaml as yaml +import yaml import dbt.flags from dbt.adapters.factory import get_adapter, register_adapter diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 4d69dca3f23..1005b4e7e83 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -8,7 +8,7 @@ import pytest from unittest import mock -import oyaml as yaml +import yaml import dbt.config import dbt.exceptions diff --git a/test/unit/test_graph_selector_parsing.py b/test/unit/test_graph_selector_parsing.py index 15b5c2935b5..435e04c3709 100644 --- a/test/unit/test_graph_selector_parsing.py +++ b/test/unit/test_graph_selector_parsing.py @@ -7,7 +7,7 @@ ) from dbt.graph.selector_methods import MethodName import textwrap -import oyaml as yaml +import yaml from dbt.contracts.selection import SelectorFile diff --git a/test/unit/test_jinja.py b/test/unit/test_jinja.py index 67503c4ab52..19a25ee3f56 100644 --- a/test/unit/test_jinja.py +++ b/test/unit/test_jinja.py @@ -1,7 +1,7 @@ from contextlib import contextmanager import pytest import unittest -import oyaml as yaml +import yaml from dbt.clients.jinja import get_rendered from dbt.clients.jinja import get_template diff --git a/test/unit/test_manifest_selectors.py b/test/unit/test_manifest_selectors.py index cf3f31319df..eea2ade8edf 100644 --- a/test/unit/test_manifest_selectors.py +++ b/test/unit/test_manifest_selectors.py @@ -1,6 +1,6 @@ import dbt.exceptions import textwrap -import oyaml as yaml +import yaml import unittest from dbt.config.selectors import SelectorDict diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 9376cd8dcc6..9f1672ff86c 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -3,7 +3,7 @@ from unittest import mock import os -import oyaml as yaml +import yaml import dbt.flags import dbt.parser diff --git a/test/unit/test_selector_errors.py b/test/unit/test_selector_errors.py index 07957785a34..8aa75aae488 100644 --- a/test/unit/test_selector_errors.py +++ b/test/unit/test_selector_errors.py @@ -1,6 +1,6 @@ import dbt.exceptions import textwrap -import oyaml as yaml +import yaml import unittest from dbt.config.selectors import ( selector_config_from_data From 52f9efbd24c1b0e490e4b0efd1c977f90b0008b1 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Wed, 6 Oct 2021 22:39:59 +0100 Subject: [PATCH 13/17] Remove oyaml and fix tests --- core/dbt/task/init.py | 2 +- core/setup.py | 1 - mypy.ini | 5 +- test/integration/040_init_test/test_init.py | 84 ++++++++++----------- 4 files changed, 44 insertions(+), 48 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index e608e193649..c28e01b6141 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -5,7 +5,7 @@ import shutil from typing import Optional -import oyaml as yaml +import yaml import click from jinja2 import Template diff --git a/core/setup.py b/core/setup.py index 6ea52bdb942..0011d42e937 100644 --- a/core/setup.py +++ b/core/setup.py @@ -50,7 +50,6 @@ def read(fname): ], install_requires=[ 'Jinja2==2.11.3', - 'oyaml>=1.0', 'agate>=1.6,<1.6.2', 'click>=8,<9', 'colorama>=0.3.9,<0.4.5', diff --git a/mypy.ini b/mypy.ini index e27ef73d8f0..60eb6ccafad 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,3 @@ [mypy] mypy_path = ./third-party-stubs -namespace_packages = True - -[mypy-oyaml.*] -ignore_missing_imports = True \ No newline at end of file +namespace_packages = True \ No newline at end of file diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 33447c62be4..07d41e03b19 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -68,14 +68,14 @@ def test_postgres_init_task_in_project_with_existing_profiles_yml(self, mock_pro test: outputs: dev: - type: postgres - threads: 4 + dbname: test_db host: localhost - port: 5432 - user: test_user pass: test_password - dbname: test_db + port: 5432 schema: test_schema + threads: 4 + type: postgres + user: test_user target: dev """ @@ -122,14 +122,14 @@ def exists_side_effect(path): assert f.read() == """test: outputs: dev: - type: postgres - threads: 4 + dbname: test_db host: localhost - port: 5432 - user: test_user pass: test_password - dbname: test_db + port: 5432 schema: test_schema + threads: 4 + type: postgres + user: test_user target: dev """ @@ -233,14 +233,14 @@ def exists_side_effect(path): assert f.read() == """my_profile: outputs: dev: - type: postgres - threads: 4 + dbname: my_db host: localhost - port: 5432 - user: test_username pass: test_password - dbname: my_db + port: 5432 schema: my_schema + threads: 4 + type: postgres + user: test_username target: dev """ @@ -289,14 +289,14 @@ def test_postgres_init_task_in_project_with_invalid_profile_template(self, mock_ test: outputs: dev: - type: postgres - threads: 4 + dbname: test_db host: localhost - port: 5432 - user: test_username pass: test_password - dbname: test_db + port: 5432 schema: test_schema + threads: 4 + type: postgres + user: test_username target: dev """ @@ -339,39 +339,39 @@ def test_postgres_init_task_outside_of_project(self, mock_prompt, mock_confirm): with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: assert f.read() == f"""config: send_anonymous_usage_stats: false +{project_name}: + outputs: + dev: + dbname: test_db + host: localhost + pass: test_password + port: 5432 + schema: test_schema + threads: 4 + type: postgres + user: test_username + target: dev test: outputs: default2: - type: postgres - threads: 4 + dbname: dbt host: localhost - port: 5432 - user: root pass: password - dbname: dbt + port: 5432 schema: {self.unique_schema()} - noaccess: - type: postgres threads: 4 + type: postgres + user: root + noaccess: + dbname: dbt host: localhost - port: 5432 - user: noaccess pass: password - dbname: dbt + port: 5432 schema: {self.unique_schema()} - target: default2 -{project_name}: - outputs: - dev: - type: postgres threads: 4 - host: localhost - port: 5432 - user: test_username - pass: test_password - dbname: test_db - schema: test_schema - target: dev + type: postgres + user: noaccess + target: default2 """ with open(os.path.join(self.test_root_dir, project_name, 'dbt_project.yml'), 'r') as f: From bd509775e3ea65517b3c50926f308f02c9bea861 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Wed, 6 Oct 2021 22:55:02 +0100 Subject: [PATCH 14/17] Update dbt_project.yml in test comparison --- test/integration/040_init_test/test_init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index 07d41e03b19..bd83df13881 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -390,7 +390,7 @@ def test_postgres_init_task_outside_of_project(self, mock_prompt, mock_confirm): # The `source-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! source-paths: ["models"] -analysis-paths: ["analysis"] +analysis-paths: ["analyses"] test-paths: ["tests"] data-paths: ["data"] macro-paths: ["macros"] @@ -399,7 +399,7 @@ def test_postgres_init_task_outside_of_project(self, mock_prompt, mock_confirm): target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` - "target" - - "dbt_modules" + - "dbt_packages" # Configuring models From 4e490ce836e96116a901f66db4424a5d8c84a035 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Wed, 6 Oct 2021 23:00:49 +0100 Subject: [PATCH 15/17] Create the profiles directory if it doesn't exist --- core/dbt/task/init.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index c28e01b6141..12c30ea90cc 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -168,6 +168,8 @@ def write_profile( ) -> Path: """Given a profile, write it to the current project's profiles.yml. This will overwrite any profile with a matching name.""" + # Create the profile directory if it doesn't exist + os.makedirs(flags.PROFILES_DIR, exist_ok=True) profiles_filepath = Path(flags.PROFILES_DIR) / Path("profiles.yml") if profiles_filepath.exists(): with open(profiles_filepath, "r+") as f: From f9ac9123f56662aa0d4a1a7d40462e9e55378da6 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Wed, 20 Oct 2021 17:03:06 +0100 Subject: [PATCH 16/17] Use safe_load --- core/dbt/task/init.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 12c30ea90cc..d00006413ac 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -84,7 +84,7 @@ def create_profile_from_sample(self, adapter: str, profile_name: str): else: with open(sample_profiles_path, "r") as f: sample_profile = f.read() - sample_profile_name = list(yaml.load(sample_profile).keys())[0] + sample_profile_name = list(yaml.safe_load(sample_profile).keys())[0] # Use a regex to replace the name of the sample_profile with # that of the project without losing any comments from the sample sample_profile = re.sub( @@ -160,7 +160,7 @@ def get_profile_name_from_current_project(self) -> str: profile name. """ with open("dbt_project.yml") as f: - dbt_project = yaml.load(f) + dbt_project = yaml.safe_load(f) return dbt_project["profile"] def write_profile( @@ -173,7 +173,7 @@ def write_profile( profiles_filepath = Path(flags.PROFILES_DIR) / Path("profiles.yml") if profiles_filepath.exists(): with open(profiles_filepath, "r+") as f: - profiles = yaml.load(f) or {} + profiles = yaml.safe_load(f) or {} profiles[profile_name] = profile f.seek(0) yaml.dump(profiles, f) @@ -209,7 +209,7 @@ def create_profile_from_scratch(self, adapter: str, profile_name: str): if target_options_path.exists(): with open(target_options_path) as f: - target_options = yaml.load(f) + target_options = yaml.safe_load(f) self.create_profile_from_target_options(target_options, profile_name) else: # For adapters without a target_options.yml defined, fallback on @@ -227,7 +227,7 @@ def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool profile_name or self.get_profile_name_from_current_project() ) with open(profiles_file, "r") as f: - profiles = yaml.load(f) or {} + profiles = yaml.safe_load(f) or {} if profile_name in profiles.keys(): response = click.confirm( f"The profile {profile_name} already exists in " @@ -240,7 +240,7 @@ def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool def create_profile_using_profile_template(self): """Create a profile using profile_template.yml""" with open("profile_template.yml") as f: - profile_template = yaml.load(f) + profile_template = yaml.safe_load(f) profile_name = list(profile_template["profile"].keys())[0] self.check_if_can_write_profile(profile_name) render_vars = {} @@ -249,7 +249,7 @@ def create_profile_using_profile_template(self): profile = profile_template["profile"][profile_name] profile_str = yaml.dump(profile) profile_str = Template(profile_str).render(render_vars) - profile = yaml.load(profile_str) + profile = yaml.safe_load(profile_str) profiles_filepath = self.write_profile(profile, profile_name) logger.info( f"Profile {profile_name} written to {profiles_filepath} using " From 4f213ba4d549041ede0966e48153137c8b551bc8 Mon Sep 17 00:00:00 2001 From: Niall Woodward Date: Wed, 20 Oct 2021 17:27:30 +0100 Subject: [PATCH 17/17] Update integration test --- test/integration/040_init_test/test_init.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index bd83df13881..3fc612869f1 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -387,12 +387,12 @@ def test_postgres_init_task_outside_of_project(self, mock_prompt, mock_confirm): profile: '{project_name}' # These configurations specify where dbt should look for different types of files. -# The `source-paths` config, for example, states that models in this project can be +# The `model-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! -source-paths: ["models"] +model-paths: ["models"] analysis-paths: ["analyses"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["seeds"] macro-paths: ["macros"] snapshot-paths: ["snapshots"]