diff --git a/CHANGELOG.md b/CHANGELOG.md index dd21b3598f7..4c61b7aabc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,12 @@ - Enable `on-run-start` and `on-run-end` hooks for `dbt test`. Add `flags.WHICH` to execution context, representing current task ([#3463](https://github.com/dbt-labs/dbt-core/issues/3463), [#4004](https://github.com/dbt-labs/dbt-core/pull/4004)) ### Features - +- Normalize global CLI arguments/flags ([#2990](https://github.com/dbt-labs/dbt/issues/2990), [#3839](https://github.com/dbt-labs/dbt/pull/3839)) +- Turns on the static parser by default and adds the flag `--no-static-parser` to disable it. ([#3377](https://github.com/dbt-labs/dbt/issues/3377), [#3939](https://github.com/dbt-labs/dbt/pull/3939)) +- Generic test FQNs have changed to include the relative path, resource, and column (if applicable) where they are defined. This makes it easier to configure them from the `tests` block in `dbt_project.yml` ([#3259](https://github.com/dbt-labs/dbt/pull/3259), [#3880](https://github.com/dbt-labs/dbt/pull/3880) +- Turn on partial parsing by default ([#3867](https://github.com/dbt-labs/dbt/issues/3867), [#3989](https://github.com/dbt-labs/dbt/issues/3989)) - Add `result:` selectors to automatically rerun failed tests and erroneous models. This makes it easier to rerun failed dbt jobs with a simple selector flag instead of restarting from the beginning or manually running the dbt models in scope. ([#3859](https://github.com/dbt-labs/dbt/issues/3891), [#4017](https://github.com/dbt-labs/dbt/pull/4017)) +- `dbt init` is now interactive, generating profiles.yml when run inside existing project ([#3625](https://github.com/dbt-labs/dbt/pull/3625)) ### Under the hood - Fix intermittent errors in partial parsing tests ([#4060](https://github.com/dbt-labs/dbt-core/issues/4060), [#4068](https://github.com/dbt-labs/dbt-core/pull/4068)) @@ -16,6 +20,7 @@ Contributors: - [@sungchun12](https://github.com/sungchun12) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) - [@matt-winkler](https://github.com/matt-winkler) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) +- [@NiallRees](https://github.com/NiallRees) ([#3625](https://github.com/dbt-labs/dbt/pull/3625)) ## dbt-core 1.0.0b1 (October 11, 2021) diff --git a/core/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py index df349c1bb38..34ce235767b 100644 --- a/core/dbt/clients/yaml_helper.py +++ b/core/dbt/clients/yaml_helper.py @@ -1,7 +1,6 @@ import dbt.exceptions from typing import Any, Dict, Optional import yaml -import yaml.scanner # the C version is faster, but it doesn't always exist try: diff --git a/core/dbt/include/starter_project/dbt_project.yml b/core/dbt/include/starter_project/dbt_project.yml index 47b85cf9108..b016ce308f7 100644 --- a/core/dbt/include/starter_project/dbt_project.yml +++ b/core/dbt/include/starter_project/dbt_project.yml @@ -2,12 +2,12 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'my_new_project' +name: '{project_name}' version: '1.0.0' config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: 'default' +profile: '{profile_name}' # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be @@ -30,9 +30,9 @@ clean-targets: # directories to be removed by `dbt clean` # In this example config, we tell dbt to build all models in the example/ directory # as tables. These settings can be overridden in the individual model files -# using the `{{ config(...) }}` macro. +# using the `{{{{ config(...) }}}}` macro. models: - my_new_project: + {project_name}: # Config indicated by + and applies to all files under models/example/ example: +materialized: view diff --git a/core/dbt/main.py b/core/dbt/main.py index 8e59ecb72aa..ba7058f88bf 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -248,7 +248,6 @@ def run_from_args(parsed): with track_run(task): results = task.run() - return task, results @@ -344,20 +343,6 @@ def _build_init_subparser(subparsers, base_subparser): Initialize a new DBT project. ''' ) - sub.add_argument( - 'project_name', - type=str, - help=''' - Name of the new project - ''', - ) - sub.add_argument( - '--adapter', - type=str, - help=''' - Write sample profiles.yml for which adapter - ''', - ) sub.set_defaults(cls=init_task.InitTask, which='init', rpc_method=None) return sub diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 1e141d0c9fe..d00006413ac 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -1,5 +1,13 @@ +import copy import os +from pathlib import Path +import re import shutil +from typing import Optional + +import yaml +import click +from jinja2 import Template import dbt.config import dbt.clients.system @@ -11,7 +19,7 @@ from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory -from dbt.task.base import BaseTask +from dbt.task.base import BaseTask, move_to_nearest_project_dir DOCS_URL = 'https://docs.getdbt.com/docs/configure-your-profile' SLACK_URL = 'https://community.getdbt.com/' @@ -20,11 +28,7 @@ IGNORE_FILES = ["__init__.py", "__pycache__"] ON_COMPLETE_MESSAGE = """ -Your new dbt project "{project_name}" was created! If this is your first time -using dbt, you'll need to set up your profiles.yml file -- this file will tell dbt how -to connect to your database. You can find this file by running: - - {open_cmd} {profiles_path} +Your new dbt project "{project_name}" was created! For more information on how to configure the profiles.yml file, please consult the dbt documentation here: @@ -40,6 +44,15 @@ Happy modeling! """ +# https://click.palletsprojects.com/en/8.0.x/api/?highlight=float#types +click_type_mapping = { + "string": click.STRING, + "int": click.INT, + "float": click.FLOAT, + "bool": click.BOOL, + None: None +} + class InitTask(BaseTask): def copy_starter_repo(self, project_name): @@ -47,33 +60,52 @@ def copy_starter_repo(self, project_name): shutil.copytree(starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES)) - def create_profiles_dir(self, profiles_dir): - if not os.path.exists(profiles_dir): + def create_profiles_dir(self, profiles_dir: str) -> bool: + """Create the user's profiles directory if it doesn't already exist.""" + profiles_path = Path(profiles_dir) + if profiles_path.exists(): msg = "Creating dbt configuration folder at {}" logger.info(msg.format(profiles_dir)) dbt.clients.system.make_directory(profiles_dir) return True return False - def create_profiles_file(self, profiles_file, sample_adapter): + def create_profile_from_sample(self, adapter: str, profile_name: str): + """Create a profile entry using the adapter's sample_profiles.yml + + Renames the profile in sample_profiles.yml to match that of the project.""" # Line below raises an exception if the specified adapter is not found - load_plugin(sample_adapter) - adapter_path = get_include_paths(sample_adapter)[0] - sample_profiles_path = adapter_path / 'sample_profiles.yml' + load_plugin(adapter) + adapter_path = get_include_paths(adapter)[0] + sample_profiles_path = adapter_path / "sample_profiles.yml" if not sample_profiles_path.exists(): - logger.debug(f"No sample profile found for {sample_adapter}, skipping") - return False - - if not os.path.exists(profiles_file): - msg = "With sample profiles.yml for {}" - logger.info(msg.format(sample_adapter)) - shutil.copyfile(sample_profiles_path, profiles_file) - return True - - return False + logger.debug(f"No sample profile found for {adapter}.") + else: + with open(sample_profiles_path, "r") as f: + sample_profile = f.read() + sample_profile_name = list(yaml.safe_load(sample_profile).keys())[0] + # Use a regex to replace the name of the sample_profile with + # that of the project without losing any comments from the sample + sample_profile = re.sub( + f"^{sample_profile_name}:", + f"{profile_name}:", + sample_profile + ) + profiles_filepath = Path(flags.PROFILES_DIR) / Path("profiles.yml") + if profiles_filepath.exists(): + with open(profiles_filepath, "a") as f: + f.write("\n" + sample_profile) + else: + with open(profiles_filepath, "w") as f: + f.write(sample_profile) + logger.info( + f"Profile {profile_name} written to {profiles_filepath} " + "using sample configuration. Once updated, you'll be able to " + "start developing with dbt." + ) - def get_addendum(self, project_name, profiles_path): + def get_addendum(self, project_name: str, profiles_path: str) -> str: open_cmd = dbt.clients.system.open_dir_cmd() return ON_COMPLETE_MESSAGE.format( @@ -84,29 +116,216 @@ def get_addendum(self, project_name, profiles_path): slack_url=SLACK_URL ) - def run(self): - project_dir = self.args.project_name - sample_adapter = self.args.adapter - if not sample_adapter: - try: - # pick first one available, often postgres - sample_adapter = next(_get_adapter_plugin_names()) - except StopIteration: - logger.debug("No adapters installed, skipping") + def generate_target_from_input( + self, + target_options: dict, + target: dict = {} + ) -> dict: + """Generate a target configuration from target_options and user input. + """ + target_options_local = copy.deepcopy(target_options) + for key, value in target_options_local.items(): + if key.startswith("_choose"): + choice_type = key[8:].replace("_", " ") + option_list = list(value.keys()) + prompt_msg = "\n".join([ + f"[{n+1}] {v}" for n, v in enumerate(option_list) + ]) + f"\nDesired {choice_type} option (enter a number)" + numeric_choice = click.prompt(prompt_msg, type=click.INT) + choice = option_list[numeric_choice - 1] + # Complete the chosen option's values in a recursive call + target = self.generate_target_from_input( + target_options_local[key][choice], target + ) + else: + if key.startswith("_fixed"): + # _fixed prefixed keys are not presented to the user + target[key[7:]] = value + else: + hide_input = value.get("hide_input", False) + default = value.get("default", None) + hint = value.get("hint", None) + type = click_type_mapping[value.get("type", None)] + text = key + (f" ({hint})" if hint else "") + target[key] = click.prompt( + text, + default=default, + hide_input=hide_input, + type=type + ) + return target - profiles_dir = flags.PROFILES_DIR - profiles_file = os.path.join(profiles_dir, 'profiles.yml') + def get_profile_name_from_current_project(self) -> str: + """Reads dbt_project.yml in the current directory to retrieve the + profile name. + """ + with open("dbt_project.yml") as f: + dbt_project = yaml.safe_load(f) + return dbt_project["profile"] + + def write_profile( + self, profile: dict, profile_name: str + ) -> Path: + """Given a profile, write it to the current project's profiles.yml. + This will overwrite any profile with a matching name.""" + # Create the profile directory if it doesn't exist + os.makedirs(flags.PROFILES_DIR, exist_ok=True) + profiles_filepath = Path(flags.PROFILES_DIR) / Path("profiles.yml") + if profiles_filepath.exists(): + with open(profiles_filepath, "r+") as f: + profiles = yaml.safe_load(f) or {} + profiles[profile_name] = profile + f.seek(0) + yaml.dump(profiles, f) + f.truncate() + else: + profiles = {profile_name: profile} + with open(profiles_filepath, "w") as f: + yaml.dump(profiles, f) + return profiles_filepath + def create_profile_from_target_options(self, target_options: dict, profile_name: str): + """Create and write a profile using the supplied target_options.""" + target = self.generate_target_from_input(target_options) + profile = { + "outputs": { + "dev": target + }, + "target": "dev" + } + profiles_filepath = self.write_profile(profile, profile_name) + logger.info( + f"Profile {profile_name} written to {profiles_filepath} using " + "your supplied values. Run 'dbt debug' to validate the connection." + ) + + def create_profile_from_scratch(self, adapter: str, profile_name: str): + """Create a profile without defaults using target_options.yml if available, or + sample_profiles.yml as a fallback.""" + # Line below raises an exception if the specified adapter is not found + load_plugin(adapter) + adapter_path = get_include_paths(adapter)[0] + target_options_path = adapter_path / "target_options.yml" + + if target_options_path.exists(): + with open(target_options_path) as f: + target_options = yaml.safe_load(f) + self.create_profile_from_target_options(target_options, profile_name) + else: + # For adapters without a target_options.yml defined, fallback on + # sample_profiles.yml + self.create_profile_from_sample(adapter, profile_name) + + def check_if_can_write_profile(self, profile_name: Optional[str] = None) -> bool: + """Using either a provided profile name or that specified in dbt_project.yml, + check if the profile already exists in profiles.yml, and if so ask the + user whether to proceed and overwrite it.""" + profiles_file = Path(flags.PROFILES_DIR) / Path("profiles.yml") + if not profiles_file.exists(): + return True + profile_name = ( + profile_name or self.get_profile_name_from_current_project() + ) + with open(profiles_file, "r") as f: + profiles = yaml.safe_load(f) or {} + if profile_name in profiles.keys(): + response = click.confirm( + f"The profile {profile_name} already exists in " + f"{profiles_file}. Continue and overwrite it?" + ) + return response + else: + return True + + def create_profile_using_profile_template(self): + """Create a profile using profile_template.yml""" + with open("profile_template.yml") as f: + profile_template = yaml.safe_load(f) + profile_name = list(profile_template["profile"].keys())[0] + self.check_if_can_write_profile(profile_name) + render_vars = {} + for template_variable in profile_template["prompts"]: + render_vars[template_variable] = click.prompt(template_variable) + profile = profile_template["profile"][profile_name] + profile_str = yaml.dump(profile) + profile_str = Template(profile_str).render(render_vars) + profile = yaml.safe_load(profile_str) + profiles_filepath = self.write_profile(profile, profile_name) + logger.info( + f"Profile {profile_name} written to {profiles_filepath} using " + "profile_template.yml and your supplied values. Run 'dbt debug' " + "to validate the connection." + ) + + def ask_for_adapter_choice(self) -> str: + """Ask the user which adapter (database) they'd like to use.""" + available_adapters = list(_get_adapter_plugin_names()) + prompt_msg = ( + "Which database would you like to use?\n" + + "\n".join([f"[{n+1}] {v}" for n, v in enumerate(available_adapters)]) + + "\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)" + + "\n\nEnter a number" + ) + numeric_choice = click.prompt(prompt_msg, type=click.INT) + return available_adapters[numeric_choice - 1] + + def run(self): + """Entry point for the init task.""" + profiles_dir = flags.PROFILES_DIR self.create_profiles_dir(profiles_dir) - if sample_adapter: - self.create_profiles_file(profiles_file, sample_adapter) - if os.path.exists(project_dir): - raise RuntimeError("directory {} already exists!".format( - project_dir - )) + try: + move_to_nearest_project_dir(self.args) + in_project = True + except dbt.exceptions.RuntimeException: + in_project = False + + if in_project: + # When dbt init is run inside an existing project, + # just setup the user's profile. + logger.info("Setting up your profile.") + profile_name = self.get_profile_name_from_current_project() + profile_template_path = Path("profile_template.yml") + if profile_template_path.exists(): + try: + # This relies on a valid profile_template.yml from the user, + # so use a try: except to fall back to the default on failure + self.create_profile_using_profile_template() + return + except Exception: + logger.info("Invalid profile_template.yml in project.") + if not self.check_if_can_write_profile(profile_name=profile_name): + return + adapter = self.ask_for_adapter_choice() + self.create_profile_from_scratch( + adapter, profile_name=profile_name + ) + else: + # When dbt init is run outside of an existing project, + # create a new project and set up the user's profile. + project_name = click.prompt("What is the desired project name?") + project_path = Path(project_name) + if project_path.exists(): + logger.info( + f"A project called {project_name} already exists here." + ) + return - self.copy_starter_repo(project_dir) + self.copy_starter_repo(project_name) + os.chdir(project_name) + with open("dbt_project.yml", "r+") as f: + content = f"{f.read()}".format( + project_name=project_name, + profile_name=project_name + ) + f.seek(0) + f.write(content) + f.truncate() - addendum = self.get_addendum(project_dir, profiles_dir) - logger.info(addendum) + if not self.check_if_can_write_profile(profile_name=project_name): + return + adapter = self.ask_for_adapter_choice() + self.create_profile_from_scratch( + adapter, profile_name=project_name + ) + logger.info(self.get_addendum(project_name, profiles_dir)) diff --git a/core/setup.py b/core/setup.py index 7114dacaccd..5214645112e 100644 --- a/core/setup.py +++ b/core/setup.py @@ -50,8 +50,8 @@ def read(fname): ], install_requires=[ 'Jinja2==2.11.3', - 'PyYAML>=3.11', 'agate>=1.6,<1.6.2', + 'click>=8,<9', 'colorama>=0.3.9,<0.4.5', 'dataclasses>=0.6,<0.9;python_version<"3.7"', 'hologram==0.0.14', diff --git a/mypy.ini b/mypy.ini index 51fada1b1dc..60eb6ccafad 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,3 @@ [mypy] mypy_path = ./third-party-stubs -namespace_packages = True +namespace_packages = True \ No newline at end of file diff --git a/plugins/postgres/dbt/include/postgres/target_options.yml b/plugins/postgres/dbt/include/postgres/target_options.yml new file mode 100644 index 00000000000..d353304e7af --- /dev/null +++ b/plugins/postgres/dbt/include/postgres/target_options.yml @@ -0,0 +1,19 @@ +_fixed_type: postgres +threads: + hint: '1 or more' + type: 'int' + default: 1 +host: + hint: 'hostname for the instance' +port: + default: 5432 + type: 'int' +user: + hint: 'dev username' +pass: + hint: 'dev password' + hide_input: true +dbname: + hint: 'default database that dbt will build objects in' +schema: + hint: 'default schema that dbt will build objects in' \ No newline at end of file diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index 8122f609da7..dba2950c26c 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -326,7 +326,7 @@ def test__postgres__exposure_parents(self): results = self.run_dbt(['ls', '--select', '1+exposure:user_exposure']) assert len(results) == 5 - assert sorted(results) == ['exposure:test.user_exposure', 'test.unique_users_id', + assert sorted(results) == ['exposure:test.user_exposure', 'test.unique_users_id', 'test.unique_users_rollup_gender', 'test.users', 'test.users_rollup'] results = self.run_dbt(['run', '-m', '+exposure:user_exposure']) diff --git a/test/integration/040_init_test/test_init.py b/test/integration/040_init_test/test_init.py index c7b5a85363d..3fc612869f1 100644 --- a/test/integration/040_init_test/test_init.py +++ b/test/integration/040_init_test/test_init.py @@ -1,8 +1,12 @@ - -from test.integration.base import DBTIntegrationTest, use_profile import os import shutil -import yaml +from unittest import mock +from unittest.mock import Mock, call +from pathlib import Path + +import click + +from test.integration.base import DBTIntegrationTest, use_profile class TestInit(DBTIntegrationTest): @@ -26,18 +30,387 @@ def models(self): return "models" @use_profile('postgres') - def test_postgres_init_task(self): + @mock.patch('click.confirm') + @mock.patch('click.prompt') + def test_postgres_init_task_in_project_with_existing_profiles_yml(self, mock_prompt, mock_confirm): + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.confirm.side_effect = ["y"] + manager.prompt.side_effect = [ + 1, + 4, + 'localhost', + 5432, + 'test_user', + 'test_password', + 'test_db', + 'test_schema', + ] + + self.run_dbt(['init']) + + manager.assert_has_calls([ + call.confirm(f'The profile test already exists in {self.test_root_dir}/profiles.yml. Continue and overwrite it?'), + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == """config: + send_anonymous_usage_stats: false +test: + outputs: + dev: + dbname: test_db + host: localhost + pass: test_password + port: 5432 + schema: test_schema + threads: 4 + type: postgres + user: test_user + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + @mock.patch.object(Path, 'exists', autospec=True) + def test_postgres_init_task_in_project_without_existing_profiles_yml(self, exists, mock_prompt, mock_confirm): + + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return { + 'profiles.yml': False + }.get(path.name, os.path.exists(path)) + + exists.side_effect = exists_side_effect + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.prompt.side_effect = [ + 1, + 4, + 'localhost', + 5432, + 'test_user', + 'test_password', + 'test_db', + 'test_schema', + ] + + self.run_dbt(['init']) + + manager.assert_has_calls([ + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == """test: + outputs: + dev: + dbname: test_db + host: localhost + pass: test_password + port: 5432 + schema: test_schema + threads: 4 + type: postgres + user: test_user + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + @mock.patch.object(Path, 'exists', autospec=True) + def test_postgres_init_task_in_project_without_existing_profiles_yml_or_target_options(self, exists, mock_prompt, mock_confirm): + + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return { + 'profiles.yml': False, + 'target_options.yml': False, + }.get(path.name, os.path.exists(path)) + + exists.side_effect = exists_side_effect + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.prompt.side_effect = [ + 1, + ] + self.run_dbt(['init']) + manager.assert_has_calls([ + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == """test: + outputs: + + dev: + type: postgres + threads: [1 or more] + host: [host] + port: [port] + user: [dev_username] + pass: [dev_password] + dbname: [dbname] + schema: [dev_schema] + + prod: + type: postgres + threads: [1 or more] + host: [host] + port: [port] + user: [prod_username] + pass: [prod_password] + dbname: [dbname] + schema: [prod_schema] + + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + @mock.patch.object(Path, 'exists', autospec=True) + def test_postgres_init_task_in_project_with_profile_template_without_existing_profiles_yml(self, exists, mock_prompt, mock_confirm): + + def exists_side_effect(path): + # Override responses on specific files, default to 'real world' if not overriden + return { + 'profiles.yml': False, + }.get(path.name, os.path.exists(path)) + exists.side_effect = exists_side_effect + + with open("profile_template.yml", 'w') as f: + f.write("""prompts: + - pg_username + - pg_password +profile: + my_profile: + outputs: + dev: + type: postgres + threads: 4 + host: localhost + port: 5432 + user: "{{ pg_username }}" + pass: "{{ pg_password }}" + dbname: my_db + schema: my_schema + target: dev""") + + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.prompt.side_effect = [ + 'test_username', + 'test_password' + ] + self.run_dbt(['init']) + manager.assert_has_calls([ + call.prompt('pg_username'), + call.prompt('pg_password') + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == """my_profile: + outputs: + dev: + dbname: my_db + host: localhost + pass: test_password + port: 5432 + schema: my_schema + threads: 4 + type: postgres + user: test_username + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + def test_postgres_init_task_in_project_with_invalid_profile_template(self, mock_prompt, mock_confirm): + """Test that when an invalid profile_template.yml is provided, + init command falls back to the target_options.yml""" + + with open("profile_template.yml", 'w') as f: + f.write("""invalid template""") + + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + manager.confirm.side_effect = ["y"] + manager.prompt.side_effect = [ + 1, + 4, + 'localhost', + 5432, + 'test_username', + 'test_password', + 'test_db', + 'test_schema', + ] + + self.run_dbt(['init']) + + manager.assert_has_calls([ + call.confirm(f'The profile test already exists in {self.test_root_dir}/profiles.yml. Continue and overwrite it?'), + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == """config: + send_anonymous_usage_stats: false +test: + outputs: + dev: + dbname: test_db + host: localhost + pass: test_password + port: 5432 + schema: test_schema + threads: 4 + type: postgres + user: test_username + target: dev +""" + + @use_profile('postgres') + @mock.patch('click.confirm') + @mock.patch('click.prompt') + def test_postgres_init_task_outside_of_project(self, mock_prompt, mock_confirm): + manager = Mock() + manager.attach_mock(mock_prompt, 'prompt') + manager.attach_mock(mock_confirm, 'confirm') + + # Start by removing the dbt_project.yml so that we're not in an existing project + os.remove('dbt_project.yml') + project_name = self.get_project_name() - self.run_dbt(['init', project_name, '--adapter', 'postgres']) + manager.prompt.side_effect = [ + project_name, + 1, + 4, + 'localhost', + 5432, + 'test_username', + 'test_password', + 'test_db', + 'test_schema', + ] + self.run_dbt(['init']) + manager.assert_has_calls([ + call.prompt('What is the desired project name?'), + call.prompt("Which database would you like to use?\n[1] postgres\n\n(Don't see the one you want? https://docs.getdbt.com/docs/available-adapters)\n\nEnter a number", type=click.INT), + call.prompt('threads (1 or more)', default=1, hide_input=False, type=click.INT), + call.prompt('host (hostname for the instance)', default=None, hide_input=False, type=None), + call.prompt('port', default=5432, hide_input=False, type=click.INT), + call.prompt('user (dev username)', default=None, hide_input=False, type=None), + call.prompt('pass (dev password)', default=None, hide_input=True, type=None), + call.prompt('dbname (default database that dbt will build objects in)', default=None, hide_input=False, type=None), + call.prompt('schema (default schema that dbt will build objects in)', default=None, hide_input=False, type=None) + ]) + + with open(os.path.join(self.test_root_dir, 'profiles.yml'), 'r') as f: + assert f.read() == f"""config: + send_anonymous_usage_stats: false +{project_name}: + outputs: + dev: + dbname: test_db + host: localhost + pass: test_password + port: 5432 + schema: test_schema + threads: 4 + type: postgres + user: test_username + target: dev +test: + outputs: + default2: + dbname: dbt + host: localhost + pass: password + port: 5432 + schema: {self.unique_schema()} + threads: 4 + type: postgres + user: root + noaccess: + dbname: dbt + host: localhost + pass: password + port: 5432 + schema: {self.unique_schema()} + threads: 4 + type: postgres + user: noaccess + target: default2 +""" + + with open(os.path.join(self.test_root_dir, project_name, 'dbt_project.yml'), 'r') as f: + assert f.read() == f""" +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: '{project_name}' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: '{project_name}' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" - assert os.path.exists(project_name) - project_file = os.path.join(project_name, 'dbt_project.yml') - assert os.path.exists(project_file) - with open(project_file) as fp: - project_data = yaml.safe_load(fp.read()) - assert 'config-version' in project_data - assert project_data['config-version'] == 2 +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models - git_dir = os.path.join(project_name, '.git') - assert not os.path.exists(git_dir) +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{{{ config(...) }}}}` macro. +models: + {project_name}: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: view +""" diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index be70d06819e..412f27f5dec 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -372,7 +372,7 @@ def test_postgres_source_freshness_selection_select(self): @use_profile('postgres') def test_postgres_source_freshness_selection_exclude(self): - """Tests node selection using the --select argument. It 'excludes' the + """Tests node selection using the --select argument. It 'excludes' the only source in the project so it should return no results.""" self._set_updated_at_to(timedelta(hours=-2)) self.freshness_start_time = datetime.utcnow() diff --git a/test/unit/test_manifest_selectors.py b/test/unit/test_manifest_selectors.py index d7e7c3d1fe8..eea2ade8edf 100644 --- a/test/unit/test_manifest_selectors.py +++ b/test/unit/test_manifest_selectors.py @@ -71,7 +71,7 @@ def test_single_string_definition(self): definition = sel_dict['nightly_selector']['definition'] self.assertEqual(expected, definition) - + def test_single_key_value_definition(self): dct = get_selector_dict('''\ selectors: