diff --git a/.changes/unreleased/Features-20231218-195854.yaml b/.changes/unreleased/Features-20231218-195854.yaml new file mode 100644 index 00000000000..2a78826aff0 --- /dev/null +++ b/.changes/unreleased/Features-20231218-195854.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Move flags from UserConfig in profiles.yml to flags in dbt_project.yml +time: 2023-12-18T19:58:54.075811-05:00 +custom: + Author: gshank + Issue: "9183" diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 5b1769346ba..f5e7ca18104 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from importlib import import_module from multiprocessing import get_context +from pathlib import Path from pprint import pformat as pf from typing import Any, Callable, Dict, List, Optional, Set, Union @@ -11,8 +12,8 @@ from dbt.cli.exceptions import DbtUsageException from dbt.cli.resolvers import default_log_path, default_project_dir from dbt.cli.types import Command as CliCommand -from dbt.config.profile import read_user_config -from dbt.contracts.project import UserConfig +from dbt.config.project import read_project_flags +from dbt.contracts.project import ProjectFlags from dbt.exceptions import DbtInternalError from dbt.deprecations import renamed_env_var from dbt.helper_types import WarnErrorOptions @@ -25,7 +26,7 @@ "INDIRECT_SELECTION": "eager", "TARGET_PATH": None, "WARN_ERROR": None, - # Cli args without user_config or env var option. + # Cli args without project_flags or env var option. "FULL_REFRESH": False, "STRICT_MODE": False, "STORE_FAILURES": False, @@ -77,7 +78,7 @@ class Flags: """Primary configuration artifact for running dbt""" def __init__( - self, ctx: Optional[Context] = None, user_config: Optional[UserConfig] = None + self, ctx: Optional[Context] = None, project_flags: Optional[ProjectFlags] = None ) -> None: # Set the default flags. for key, value in FLAGS_DEFAULTS.items(): @@ -200,23 +201,29 @@ def _assign_params( invoked_subcommand_ctx, params_assigned_from_default, deprecated_env_vars ) - if not user_config: + if not project_flags: + project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir())) profiles_dir = getattr(self, "PROFILES_DIR", None) - user_config = read_user_config(profiles_dir) if profiles_dir else None + if profiles_dir and project_dir: + project_flags = read_project_flags(project_dir, profiles_dir) + else: + project_flags = None # Add entire invocation command to flags object.__setattr__(self, "INVOCATION_COMMAND", "dbt " + " ".join(sys.argv[1:])) # Overwrite default assignments with user config if available. - if user_config: + if project_flags: param_assigned_from_default_copy = params_assigned_from_default.copy() for param_assigned_from_default in params_assigned_from_default: - user_config_param_value = getattr(user_config, param_assigned_from_default, None) - if user_config_param_value is not None: + project_flags_param_value = getattr( + project_flags, param_assigned_from_default, None + ) + if project_flags_param_value is not None: object.__setattr__( self, param_assigned_from_default.upper(), - convert_config(param_assigned_from_default, user_config_param_value), + convert_config(param_assigned_from_default, project_flags_param_value), ) param_assigned_from_default_copy.remove(param_assigned_from_default) params_assigned_from_default = param_assigned_from_default_copy @@ -234,9 +241,11 @@ def _assign_params( # Starting in v1.5, if `log-path` is set in `dbt_project.yml`, it will raise a deprecation warning, # with the possibility of removing it in a future release. if getattr(self, "LOG_PATH", None) is None: - project_dir = getattr(self, "PROJECT_DIR", default_project_dir()) + project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir())) version_check = getattr(self, "VERSION_CHECK", True) - object.__setattr__(self, "LOG_PATH", default_log_path(project_dir, version_check)) + object.__setattr__( + self, "LOG_PATH", default_log_path(Path(project_dir), version_check) + ) # Support console DO NOT TRACK initiative. if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"): diff --git a/core/dbt/config/__init__.py b/core/dbt/config/__init__.py index 1fa43bed3a5..767901c84eb 100644 --- a/core/dbt/config/__init__.py +++ b/core/dbt/config/__init__.py @@ -1,4 +1,4 @@ # all these are just exports, they need "noqa" so flake8 will not complain. -from .profile import Profile, read_user_config # noqa +from .profile import Profile # noqa from .project import Project, IsFQNResource, PartialProject # noqa from .runtime import RuntimeConfig # noqa diff --git a/core/dbt/config/profile.py b/core/dbt/config/profile.py index acd06e6a8a7..740c0bfe5e9 100644 --- a/core/dbt/config/profile.py +++ b/core/dbt/config/profile.py @@ -8,7 +8,7 @@ from dbt.clients.system import load_file_contents from dbt.clients.yaml_helper import load_yaml_text from dbt.contracts.connection import Credentials, HasCredentials -from dbt.contracts.project import ProfileConfig, UserConfig +from dbt.contracts.project import ProfileConfig from dbt.exceptions import ( CompilationError, DbtProfileError, @@ -19,7 +19,6 @@ ) from dbt.events.types import MissingProfileTarget from dbt.events.functions import fire_event -from dbt.utils import coerce_dict_str from .renderer import ProfileRenderer @@ -51,19 +50,6 @@ def read_profile(profiles_dir: str) -> Dict[str, Any]: return {} -def read_user_config(directory: str) -> UserConfig: - try: - profile = read_profile(directory) - if profile: - user_config = coerce_dict_str(profile.get("config", {})) - if user_config is not None: - UserConfig.validate(user_config) - return UserConfig.from_dict(user_config) - except (DbtRuntimeError, ValidationError): - pass - return UserConfig() - - # The Profile class is included in RuntimeConfig, so any attribute # additions must also be set where the RuntimeConfig class is created # `init=False` is a workaround for https://bugs.python.org/issue45081 @@ -71,7 +57,6 @@ def read_user_config(directory: str) -> UserConfig: class Profile(HasCredentials): profile_name: str target_name: str - user_config: UserConfig threads: int credentials: Credentials profile_env_vars: Dict[str, Any] @@ -80,7 +65,6 @@ def __init__( self, profile_name: str, target_name: str, - user_config: UserConfig, threads: int, credentials: Credentials, ) -> None: @@ -89,7 +73,6 @@ def __init__( """ self.profile_name = profile_name self.target_name = target_name - self.user_config = user_config self.threads = threads self.credentials = credentials self.profile_env_vars = {} # never available on init @@ -106,12 +89,10 @@ def to_profile_info(self, serialize_credentials: bool = False) -> Dict[str, Any] result = { "profile_name": self.profile_name, "target_name": self.target_name, - "user_config": self.user_config, "threads": self.threads, "credentials": self.credentials, } if serialize_credentials: - result["user_config"] = self.user_config.to_dict(omit_none=True) result["credentials"] = self.credentials.to_dict(omit_none=True) return result @@ -124,7 +105,6 @@ def to_target_dict(self) -> Dict[str, Any]: "name": self.target_name, "target_name": self.target_name, "profile_name": self.profile_name, - "config": self.user_config.to_dict(omit_none=True), } ) return target @@ -246,7 +226,6 @@ def from_credentials( threads: int, profile_name: str, target_name: str, - user_config: Optional[Dict[str, Any]] = None, ) -> "Profile": """Create a profile from an existing set of Credentials and the remaining information. @@ -255,20 +234,13 @@ def from_credentials( :param threads: The number of threads to use for connections. :param profile_name: The profile name used for this profile. :param target_name: The target name used for this profile. - :param user_config: The user-level config block from the - raw profiles, if specified. :raises DbtProfileError: If the profile is invalid. :returns: The new Profile object. """ - if user_config is None: - user_config = {} - UserConfig.validate(user_config) - user_config_obj: UserConfig = UserConfig.from_dict(user_config) profile = cls( profile_name=profile_name, target_name=target_name, - user_config=user_config_obj, threads=threads, credentials=credentials, ) @@ -316,7 +288,6 @@ def from_raw_profile_info( raw_profile: Dict[str, Any], profile_name: str, renderer: ProfileRenderer, - user_config: Optional[Dict[str, Any]] = None, target_override: Optional[str] = None, threads_override: Optional[int] = None, ) -> "Profile": @@ -328,8 +299,6 @@ def from_raw_profile_info( disk as yaml and its values rendered with jinja. :param profile_name: The profile name used. :param renderer: The config renderer. - :param user_config: The global config for the user, if it - was present. :param target_override: The target to use, if provided on the command line. :param threads_override: The thread count to use, if @@ -338,9 +307,6 @@ def from_raw_profile_info( target could not be found :returns: The new Profile object. """ - # user_config is not rendered. - if user_config is None: - user_config = raw_profile.get("config") # TODO: should it be, and the values coerced to bool? target_name, profile_data = cls.render_profile( raw_profile, profile_name, target_override, renderer @@ -361,7 +327,6 @@ def from_raw_profile_info( profile_name=profile_name, target_name=target_name, threads=threads, - user_config=user_config, ) @classmethod @@ -396,13 +361,11 @@ def from_raw_profiles( if not raw_profile: msg = f"Profile {profile_name} in profiles.yml is empty" raise DbtProfileError(INVALID_PROFILE_MESSAGE.format(error_string=msg)) - user_config = raw_profiles.get("config") return cls.from_raw_profile_info( raw_profile=raw_profile, profile_name=profile_name, renderer=renderer, - user_config=user_config, target_override=target_override, threads_override=threads_override, ) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 696e2668762..3d7426c6f8f 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -20,6 +20,7 @@ DEPENDENCIES_FILE_NAME, PACKAGES_FILE_NAME, PACKAGE_LOCK_HASH_KEY, + DBT_PROJECT_FILE_NAME, ) from dbt.clients.system import path_exists, load_file_contents from dbt.clients.yaml_helper import load_yaml_text @@ -35,12 +36,13 @@ from dbt.helper_types import NoValue from dbt.semver import VersionSpecifier, versions_compatible from dbt.version import get_installed_version -from dbt.utils import MultiDict, md5 +from dbt.utils import MultiDict, md5, coerce_dict_str from dbt.node_types import NodeType from dbt.config.selectors import SelectorDict from dbt.contracts.project import ( Project as ProjectContract, SemverString, + ProjectFlags, ) from dbt.contracts.project import PackageConfig, ProjectPackageMetadata from dbt.dataclass_schema import ValidationError @@ -81,8 +83,8 @@ """ MISSING_DBT_PROJECT_ERROR = """\ -No dbt_project.yml found at expected path {path} -Verify that each entry within packages.yml (and their transitive dependencies) contains a file named dbt_project.yml +No {DBT_PROJECT_FILE_NAME} found at expected path {path} +Verify that each entry within packages.yml (and their transitive dependencies) contains a file named {DBT_PROJECT_FILE_NAME} """ @@ -199,16 +201,20 @@ def value_or(value: Optional[T], default: T) -> T: def load_raw_project(project_root: str) -> Dict[str, Any]: project_root = os.path.normpath(project_root) - project_yaml_filepath = os.path.join(project_root, "dbt_project.yml") + project_yaml_filepath = os.path.join(project_root, DBT_PROJECT_FILE_NAME) # get the project.yml contents if not path_exists(project_yaml_filepath): - raise DbtProjectError(MISSING_DBT_PROJECT_ERROR.format(path=project_yaml_filepath)) + raise DbtProjectError( + MISSING_DBT_PROJECT_ERROR.format( + path=project_yaml_filepath, DBT_PROJECT_FILE_NAME=DBT_PROJECT_FILE_NAME + ) + ) project_dict = _load_yaml(project_yaml_filepath) if not isinstance(project_dict, dict): - raise DbtProjectError("dbt_project.yml does not parse to a dictionary") + raise DbtProjectError(f"{DBT_PROJECT_FILE_NAME} does not parse to a dictionary") return project_dict @@ -323,21 +329,21 @@ def get_rendered( selectors_dict=rendered_selectors, ) - # Called by Project.from_project_root (not PartialProject.from_project_root!) + # Called by Project.from_project_root which first calls PartialProject.from_project_root def render(self, renderer: DbtProjectYamlRenderer) -> "Project": try: rendered = self.get_rendered(renderer) return self.create_project(rendered) except DbtProjectError as exc: if exc.path is None: - exc.path = os.path.join(self.project_root, "dbt_project.yml") + exc.path = os.path.join(self.project_root, DBT_PROJECT_FILE_NAME) raise def render_package_metadata(self, renderer: PackageRenderer) -> ProjectPackageMetadata: packages_data = renderer.render_data(self.packages_dict) packages_config = package_config_from_data(packages_data, self.packages_dict) if not self.project_name: - raise DbtProjectError("Package dbt_project.yml must have a name!") + raise DbtProjectError(f"Package defined in {DBT_PROJECT_FILE_NAME} must have a name!") return ProjectPackageMetadata(self.project_name, packages_config.packages) def check_config_path( @@ -348,7 +354,7 @@ def check_config_path( msg = ( "{deprecated_path} and {expected_path} cannot both be defined. The " "`{deprecated_path}` config has been deprecated in favor of `{expected_path}`. " - "Please update your `dbt_project.yml` configuration to reflect this " + f"Please update your `{DBT_PROJECT_FILE_NAME}` configuration to reflect this " "change." ) raise DbtProjectError( @@ -420,11 +426,11 @@ def create_project(self, rendered: RenderComponents) -> "Project": docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths) asset_paths: List[str] = value_or(cfg.asset_paths, []) - flags = get_flags() + global_flags = get_flags() - flag_target_path = str(flags.TARGET_PATH) if flags.TARGET_PATH else None + flag_target_path = str(global_flags.TARGET_PATH) if global_flags.TARGET_PATH else None target_path: str = flag_or(flag_target_path, cfg.target_path, "target") - log_path: str = str(flags.LOG_PATH) + log_path: str = str(global_flags.LOG_PATH) clean_targets: List[str] = value_or(cfg.clean_targets, [target_path]) packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages") @@ -569,6 +575,11 @@ def from_project_root( ) = package_and_project_data_from_root(project_root) selectors_dict = selector_data_from_root(project_root) + if "flags" in project_dict: + # We don't want to include "flags" in the Project, + # it goes in ProjectFlags + project_dict.pop("flags") + return cls.from_dicts( project_root=project_root, project_dict=project_dict, @@ -709,7 +720,6 @@ def to_project_config(self, with_packages=False): "exposures": self.exposures, "vars": self.vars.to_dict(), "require-dbt-version": [v.to_version_string() for v in self.dbt_version], - "config-version": self.config_version, "restrict-access": self.restrict_access, "dbt-cloud": self.dbt_cloud, } @@ -773,3 +783,52 @@ def get_macro_search_order(self, macro_namespace: str): def project_target_path(self): # If target_path is absolute, project_root will not be included return os.path.join(self.project_root, self.target_path) + + +def read_project_flags(project_dir: str, profiles_dir: str) -> ProjectFlags: + try: + project_flags: Dict[str, Any] = {} + # Read project_flags from dbt_project.yml first + # Flags are instantiated before the project, so we don't + # want to throw an error for non-existence of dbt_project.yml here + # because it breaks things. + project_root = os.path.normpath(project_dir) + project_yaml_filepath = os.path.join(project_root, DBT_PROJECT_FILE_NAME) + if path_exists(project_yaml_filepath): + try: + project_dict = load_raw_project(project_root) + if "flags" in project_dict: + project_flags = project_dict.pop("flags") + except Exception: + # This is probably a yaml load error.The error will be reported + # later, when the project loads. + pass + + from dbt.config.profile import read_profile + + profile = read_profile(profiles_dir) + profile_project_flags: Optional[Dict[str, Any]] = {} + if profile: + profile_project_flags = coerce_dict_str(profile.get("config", {})) + + if project_flags and profile_project_flags: + raise DbtProjectError( + f"Do not specify both 'config' in profiles.yml and 'flags' in {DBT_PROJECT_FILE_NAME}. " + "Using 'config' in profiles.yml is deprecated." + ) + + if profile_project_flags: + # This can't use WARN_ERROR or WARN_ERROR_OPTIONS because they're in + # the config that we're loading. Uses special "warn" method. + deprecations.warn("project-flags-moved") + project_flags = profile_project_flags + + if project_flags is not None: + ProjectFlags.validate(project_flags) + return ProjectFlags.from_dict(project_flags) + except (DbtProjectError) as exc: + # We don't want to eat the DbtProjectError for UserConfig to ProjectFlags + raise exc + except (DbtRuntimeError, ValidationError): + pass + return ProjectFlags() diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index ab92be2f128..913fed53cd8 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -20,7 +20,7 @@ from dbt.config.project import load_raw_project from dbt.contracts.connection import AdapterRequiredConfig, Credentials, HasCredentials from dbt.contracts.graph.manifest import ManifestMetadata -from dbt.contracts.project import Configuration, UserConfig +from dbt.contracts.project import Configuration from dbt.contracts.relation import ComponentName from dbt.dataclass_schema import ValidationError from dbt.events.functions import warn_or_error @@ -178,7 +178,6 @@ def from_parts( profile_env_vars=profile.profile_env_vars, profile_name=profile.profile_name, target_name=profile.target_name, - user_config=profile.user_config, threads=profile.threads, credentials=profile.credentials, args=args, @@ -432,7 +431,6 @@ def _connection_keys(self): class UnsetProfile(Profile): def __init__(self): self.credentials = UnsetCredentials() - self.user_config = UserConfig() # This will be read in _get_rendered_profile self.profile_name = "" self.target_name = "" self.threads = -1 diff --git a/core/dbt/contracts/connection.py b/core/dbt/contracts/connection.py index 692f40f71b7..b4e4b086612 100644 --- a/core/dbt/contracts/connection.py +++ b/core/dbt/contracts/connection.py @@ -178,17 +178,9 @@ def __post_serialize__(self, dct): return dct -class UserConfigContract(Protocol): - send_anonymous_usage_stats: bool - use_colors: Optional[bool] = None - partial_parse: Optional[bool] = None - printer_width: Optional[int] = None - - class HasCredentials(Protocol): credentials: Credentials profile_name: str - user_config: UserConfigContract target_name: str threads: int diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index a19cba4263e..1442c5bd6ed 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -1,5 +1,5 @@ from dbt.contracts.util import Replaceable, Mergeable, list_str, Identifier -from dbt.contracts.connection import QueryComment, UserConfigContract +from dbt.contracts.connection import QueryComment from dbt.helper_types import NoValue from dbt.dataclass_schema import ( dbtClassMixin, @@ -283,7 +283,7 @@ def validate(cls, data): @dataclass -class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract): +class ProjectFlags(ExtensibleDbtClassMixin, Replaceable): cache_selected_only: Optional[bool] = None debug: Optional[bool] = None fail_fast: Optional[bool] = None @@ -310,7 +310,6 @@ class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract): class ProfileConfig(dbtClassMixin, Replaceable): profile_name: str target_name: str - user_config: UserConfig threads: int # TODO: make this a dynamic union of some kind? credentials: Optional[Dict[str, Any]] diff --git a/core/dbt/flags.py b/core/dbt/flags.py index 891d510f2e1..241189f556a 100644 --- a/core/dbt/flags.py +++ b/core/dbt/flags.py @@ -39,23 +39,24 @@ def get_flags(): return GLOBAL_FLAGS -def set_from_args(args: Namespace, user_config): +def set_from_args(args: Namespace, project_flags): global GLOBAL_FLAGS from dbt.cli.main import cli from dbt.cli.flags import Flags, convert_config - # we set attributes of args after initialize the flags, but user_config + # we set attributes of args after initialize the flags, but project_flags # is being read in the Flags constructor, so we need to read it here and pass in - # to make sure we use the correct user_config - if (hasattr(args, "PROFILES_DIR") or hasattr(args, "profiles_dir")) and not user_config: - from dbt.config.profile import read_user_config + # to make sure we use the correct project_flags + profiles_dir = getattr(args, "PROFILES_DIR", None) or getattr(args, "profiles_dir", None) + project_dir = getattr(args, "PROJECT_DIR", None) or getattr(args, "project_dir", None) + if profiles_dir and project_dir: + from dbt.config.project import read_project_flags - profiles_dir = getattr(args, "PROFILES_DIR", None) or getattr(args, "profiles_dir") - user_config = read_user_config(profiles_dir) + project_flags = read_project_flags(project_dir, profiles_dir) # make a dummy context to get the flags, totally arbitrary ctx = cli.make_context("run", ["run"]) - flags = Flags(ctx, user_config) + flags = Flags(ctx, project_flags) for arg_name, args_param_value in vars(args).items(): args_param_value = convert_config(arg_name, args_param_value) object.__setattr__(flags, arg_name.upper(), args_param_value) diff --git a/core/dbt/include/starter_project/dbt_project.yml b/core/dbt/include/starter_project/dbt_project.yml index 630001eed2f..c7e1fcdb0ef 100644 --- a/core/dbt/include/starter_project/dbt_project.yml +++ b/core/dbt/include/starter_project/dbt_project.yml @@ -4,7 +4,6 @@ # name or the intended use of these models name: '{project_name}' version: '1.0.0' -config-version: 2 # This setting configures which "profile" dbt uses for this project. profile: '{profile_name}' diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 1b7ef899bd0..8ea01050367 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -142,7 +142,6 @@ def profiles_config_update(): @pytest.fixture(scope="class") def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update): profile = { - "config": {"send_anonymous_usage_stats": False}, "test": { "outputs": { "default": {}, @@ -181,6 +180,7 @@ def dbt_project_yml(project_root, project_config_update): project_config = { "name": "test", "profile": "test", + "flags": {"send_anonymous_usage_stats": False}, } if project_config_update: if isinstance(project_config_update, dict): diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index 88022c93f0f..7febe4acdf6 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -471,7 +471,6 @@ def process(self, record): def initialize_from_flags(send_anonymous_usage_stats, profiles_dir): - # Setting these used to be in UserConfig, but had to be moved here global active_user if send_anonymous_usage_stats: active_user = User(profiles_dir) diff --git a/core/dbt/utils.py b/core/dbt/utils.py index c50df0d1f52..31697ad6146 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -631,7 +631,7 @@ def _connection_exception_retry(fn, max_attempts: int, attempt: int = 0): def args_to_dict(args): var_args = vars(args).copy() # update the args with the flags, which could also come from environment - # variables or user_config + # variables or project_flags flag_dict = flags.get_flag_dict() var_args.update(flag_dict) dict_args = {} diff --git a/tests/functional/basic/test_mixed_case_db.py b/tests/functional/basic/test_mixed_case_db.py index 19b2077cede..13519cc4bb4 100644 --- a/tests/functional/basic/test_mixed_case_db.py +++ b/tests/functional/basic/test_mixed_case_db.py @@ -16,7 +16,6 @@ def models(): def dbt_profile_data(unique_schema): return { - "config": {"send_anonymous_usage_stats": False}, "test": { "outputs": { "default": { diff --git a/tests/functional/basic/test_project.py b/tests/functional/basic/test_project.py index 080c5d591d0..6602c5e300f 100644 --- a/tests/functional/basic/test_project.py +++ b/tests/functional/basic/test_project.py @@ -77,11 +77,16 @@ def test_dbt_cloud(self, project): conf = yaml.safe_load( Path(os.path.join(project.project_root, "dbt_project.yml")).read_text() ) - assert conf == {"name": "test", "profile": "test"} + assert conf == { + "name": "test", + "profile": "test", + "flags": {"send_anonymous_usage_stats": False}, + } config = { "name": "test", "profile": "test", + "flags": {"send_anonymous_usage_stats": False}, "dbt-cloud": { "account_id": "123", "application": "test", diff --git a/tests/functional/configs/test_disabled_configs.py b/tests/functional/configs/test_disabled_configs.py index ee56a39a867..0d7cff755d8 100644 --- a/tests/functional/configs/test_disabled_configs.py +++ b/tests/functional/configs/test_disabled_configs.py @@ -9,7 +9,6 @@ class TestDisabledConfigs(BaseConfigProject): @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { - "config": {"send_anonymous_usage_stats": False}, "test": { "outputs": { "default": { diff --git a/tests/functional/dependencies/test_local_dependency.py b/tests/functional/dependencies/test_local_dependency.py index 5305659b95b..c537f0068a5 100644 --- a/tests/functional/dependencies/test_local_dependency.py +++ b/tests/functional/dependencies/test_local_dependency.py @@ -243,6 +243,10 @@ class TestSimpleDependencyNoVersionCheckConfig(BaseDependencyTest): @pytest.fixture(scope="class") def project_config_update(self): return { + "flags": { + "send_anonymous_usage_stats": False, + "version_check": False, + }, "models": { "schema": "dbt_test", }, @@ -251,15 +255,6 @@ def project_config_update(self): }, } - @pytest.fixture(scope="class") - def profiles_config_update(self): - return { - "config": { - "send_anonymous_usage_stats": False, - "version_check": False, - } - } - @pytest.fixture(scope="class") def macros(self): return {"macro.sql": macros__macro_override_schema_sql} diff --git a/tests/functional/deprecations/test_deprecations.py b/tests/functional/deprecations/test_deprecations.py index 6c2678433b0..68430ff4b8b 100644 --- a/tests/functional/deprecations/test_deprecations.py +++ b/tests/functional/deprecations/test_deprecations.py @@ -2,7 +2,8 @@ from dbt import deprecations import dbt.exceptions -from dbt.tests.util import run_dbt +from dbt.tests.util import run_dbt, write_file +import yaml models__already_exists_sql = """ @@ -157,3 +158,31 @@ def test_exposure_name_fail(self, project): exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "Starting in v1.3, the 'name' of an exposure should contain only letters, numbers, and underscores." assert expected_msg in exc_str + + +class TestPrjectFlagsMovedDeprecation: + @pytest.fixture(scope="class") + def profiles_config_update(self): + return { + "config": {"send_anonymous_usage_stats": False}, + } + + @pytest.fixture(scope="class") + def dbt_project_yml(self, project_root, project_config_update): + project_config = { + "name": "test", + "profile": "test", + } + write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml") + return project_config + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": "select 1 as fun"} + + def test_profile_config_deprecation(self, project): + deprecations.reset_deprecations() + assert deprecations.active_deprecations == set() + run_dbt(["parse"]) + expected = {"project-flags-moved"} + assert expected == deprecations.active_deprecations diff --git a/tests/functional/fail_fast/test_fail_fast_run.py b/tests/functional/fail_fast/test_fail_fast_run.py index ea956a2d540..457d620cd8d 100644 --- a/tests/functional/fail_fast/test_fail_fast_run.py +++ b/tests/functional/fail_fast/test_fail_fast_run.py @@ -44,15 +44,15 @@ def test_fail_fast_run( class TestFailFastFromConfig(FailFastBase): @pytest.fixture(scope="class") - def profiles_config_update(self): + def project_config_update(self): return { - "config": { + "flags": { "send_anonymous_usage_stats": False, "fail_fast": True, } } - def test_fail_fast_run_user_config( + def test_fail_fast_run_project_flags( self, project, models, # noqa: F811 diff --git a/tests/functional/init/test_init.py b/tests/functional/init/test_init.py index 9ac821d7c26..6aee523320c 100644 --- a/tests/functional/init/test_init.py +++ b/tests/functional/init/test_init.py @@ -70,9 +70,7 @@ def test_init_task_in_project_with_existing_profiles_yml( with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() - == """config: - send_anonymous_usage_stats: false -test: + == """test: outputs: dev: dbname: test_db @@ -391,9 +389,7 @@ def test_init_task_in_project_with_invalid_profile_template( with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() - == """config: - send_anonymous_usage_stats: false -test: + == """test: outputs: dev: dbname: test_db @@ -430,7 +426,6 @@ class TestInitOutsideOfProject(TestInitOutsideOfProjectBase): @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { - "config": {"send_anonymous_usage_stats": False}, "test": { "outputs": { "default2": { @@ -513,9 +508,7 @@ def test_init_task_outside_of_project( with open(os.path.join(project.profiles_dir, "profiles.yml"), "r") as f: assert ( f.read() - == f"""config: - send_anonymous_usage_stats: false -{project_name}: + == f"""{project_name}: outputs: dev: dbname: test_db @@ -560,7 +553,6 @@ def test_init_task_outside_of_project( # name or the intended use of these models name: '{project_name}' version: '1.0.0' -config-version: 2 # This setting configures which "profile" dbt uses for this project. profile: '{project_name}' @@ -679,7 +671,6 @@ def test_init_provided_project_name_and_skip_profile_setup( # name or the intended use of these models name: '{project_name}' version: '1.0.0' -config-version: 2 # This setting configures which "profile" dbt uses for this project. profile: '{project_name}' @@ -766,7 +757,6 @@ def test_init_task_outside_of_project_with_specified_profile( # name or the intended use of these models name: '{project_name}' version: '1.0.0' -config-version: 2 # This setting configures which "profile" dbt uses for this project. profile: 'test' diff --git a/tests/functional/metrics/test_metric_deferral.py b/tests/functional/metrics/test_metric_deferral.py index 620c8dba25f..8803bf249da 100644 --- a/tests/functional/metrics/test_metric_deferral.py +++ b/tests/functional/metrics/test_metric_deferral.py @@ -23,7 +23,6 @@ def setup(self, project): @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { - "config": {"send_anonymous_usage_stats": False}, "test": { "outputs": { "default": { diff --git a/tests/functional/run_operations/test_run_operations.py b/tests/functional/run_operations/test_run_operations.py index aa6d908b8ce..c713c8d1939 100644 --- a/tests/functional/run_operations/test_run_operations.py +++ b/tests/functional/run_operations/test_run_operations.py @@ -28,7 +28,6 @@ def macros(self): @pytest.fixture(scope="class") def dbt_profile_data(self, unique_schema): return { - "config": {"send_anonymous_usage_stats": False}, "test": { "outputs": { "default": { diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index 83c0e251deb..da53c203239 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -9,7 +9,7 @@ from dbt.cli.flags import Flags from dbt.cli.main import cli from dbt.cli.types import Command -from dbt.contracts.project import UserConfig +from dbt.contracts.project import ProjectFlags from dbt.exceptions import DbtInternalError from dbt.helper_types import WarnErrorOptions from dbt.tests.util import rm_file, write_file @@ -27,8 +27,8 @@ def run_context(self) -> click.Context: return self.make_dbt_context("run", ["run"]) @pytest.fixture - def user_config(self) -> UserConfig: - return UserConfig() + def project_flags(self) -> ProjectFlags: + return ProjectFlags() def test_which(self, run_context): flags = Flags(run_context) @@ -110,35 +110,35 @@ def test_anonymous_usage_state( flags = Flags(run_context) assert flags.SEND_ANONYMOUS_USAGE_STATS == expected_anonymous_usage_stats - def test_empty_user_config_uses_default(self, run_context, user_config): - flags = Flags(run_context, user_config) + def test_empty_project_flags_uses_default(self, run_context, project_flags): + flags = Flags(run_context, project_flags) assert flags.USE_COLORS == run_context.params["use_colors"] - def test_none_user_config_uses_default(self, run_context): + def test_none_project_flags_uses_default(self, run_context): flags = Flags(run_context, None) assert flags.USE_COLORS == run_context.params["use_colors"] - def test_prefer_user_config_to_default(self, run_context, user_config): - user_config.use_colors = False + def test_prefer_project_flags_to_default(self, run_context, project_flags): + project_flags.use_colors = False # ensure default value is not the same as user config - assert run_context.params["use_colors"] is not user_config.use_colors + assert run_context.params["use_colors"] is not project_flags.use_colors - flags = Flags(run_context, user_config) - assert flags.USE_COLORS == user_config.use_colors + flags = Flags(run_context, project_flags) + assert flags.USE_COLORS == project_flags.use_colors - def test_prefer_param_value_to_user_config(self): - user_config = UserConfig(use_colors=False) + def test_prefer_param_value_to_project_flags(self): + project_flags = ProjectFlags(use_colors=False) context = self.make_dbt_context("run", ["--use-colors", "True", "run"]) - flags = Flags(context, user_config) + flags = Flags(context, project_flags) assert flags.USE_COLORS - def test_prefer_env_to_user_config(self, monkeypatch, user_config): - user_config.use_colors = False + def test_prefer_env_to_project_flags(self, monkeypatch, project_flags): + project_flags.use_colors = False monkeypatch.setenv("DBT_USE_COLORS", "True") context = self.make_dbt_context("run", ["run"]) - flags = Flags(context, user_config) + flags = Flags(context, project_flags) assert flags.USE_COLORS def test_mutually_exclusive_options_passed_separately(self): @@ -163,14 +163,14 @@ def test_mutually_exclusive_options_from_cli(self): Flags(context) @pytest.mark.parametrize("warn_error", [True, False]) - def test_mutually_exclusive_options_from_user_config(self, warn_error, user_config): - user_config.warn_error = warn_error + def test_mutually_exclusive_options_from_project_flags(self, warn_error, project_flags): + project_flags.warn_error = warn_error context = self.make_dbt_context( "run", ["--warn-error-options", '{"include": "all"}', "run"] ) with pytest.raises(DbtUsageException): - Flags(context, user_config) + Flags(context, project_flags) @pytest.mark.parametrize("warn_error", ["True", "False"]) def test_mutually_exclusive_options_from_envvar(self, warn_error, monkeypatch): @@ -182,14 +182,16 @@ def test_mutually_exclusive_options_from_envvar(self, warn_error, monkeypatch): Flags(context) @pytest.mark.parametrize("warn_error", [True, False]) - def test_mutually_exclusive_options_from_cli_and_user_config(self, warn_error, user_config): - user_config.warn_error = warn_error + def test_mutually_exclusive_options_from_cli_and_project_flags( + self, warn_error, project_flags + ): + project_flags.warn_error = warn_error context = self.make_dbt_context( "run", ["--warn-error-options", '{"include": "all"}', "run"] ) with pytest.raises(DbtUsageException): - Flags(context, user_config) + Flags(context, project_flags) @pytest.mark.parametrize("warn_error", ["True", "False"]) def test_mutually_exclusive_options_from_cli_and_envvar(self, warn_error, monkeypatch): @@ -202,15 +204,15 @@ def test_mutually_exclusive_options_from_cli_and_envvar(self, warn_error, monkey Flags(context) @pytest.mark.parametrize("warn_error", ["True", "False"]) - def test_mutually_exclusive_options_from_user_config_and_envvar( - self, user_config, warn_error, monkeypatch + def test_mutually_exclusive_options_from_project_flags_and_envvar( + self, project_flags, warn_error, monkeypatch ): - user_config.warn_error = warn_error + project_flags.warn_error = warn_error monkeypatch.setenv("DBT_WARN_ERROR_OPTIONS", '{"include": "all"}') context = self.make_dbt_context("run", ["run"]) with pytest.raises(DbtUsageException): - Flags(context, user_config) + Flags(context, project_flags) @pytest.mark.parametrize( "cli_colors,cli_colors_file,flag_colors,flag_colors_file", @@ -319,10 +321,10 @@ def test_log_format_interaction( assert flags.LOG_FORMAT_FILE == flag_log_format_file def test_log_settings_from_config(self): - """Test that values set in UserConfig for log settings will set flags as expected""" + """Test that values set in ProjectFlags for log settings will set flags as expected""" context = self.make_dbt_context("run", ["run"]) - config = UserConfig(log_format="json", log_level="warn", use_colors=False) + config = ProjectFlags(log_format="json", log_level="warn", use_colors=False) flags = Flags(context, config) @@ -334,11 +336,11 @@ def test_log_settings_from_config(self): assert flags.USE_COLORS_FILE is False def test_log_file_settings_from_config(self): - """Test that values set in UserConfig for log *file* settings will set flags as expected, leaving the console + """Test that values set in ProjectFlags for log *file* settings will set flags as expected, leaving the console logging flags with their default values""" context = self.make_dbt_context("run", ["run"]) - config = UserConfig(log_format_file="json", log_level_file="warn", use_colors_file=False) + config = ProjectFlags(log_format_file="json", log_level_file="warn", use_colors_file=False) flags = Flags(context, config) @@ -409,3 +411,38 @@ def test_from_dict_0_value(self): args_dict = {"log_file_max_bytes": 0} flags = Flags.from_dict(Command.RUN, args_dict) assert flags.LOG_FILE_MAX_BYTES == 0 + + +def test_project_flag_defaults(): + flags = ProjectFlags() + # From # 9183: Let's add a unit test that ensures that: + # every attribute of ProjectFlags that has a corresponding click option + # in params.py should be set to None by default (except for anon user + # tracking). Going forward, flags can have non-None defaults if they + # do not have a corresponding CLI option/env var. These will be used + # to control backwards incompatible interface or behaviour changes. + + # List of all flags except send_anonymous_usage_stats + project_flags = [ + "cache_selected_only", + "debug", + "fail_fast", + "indirect_selection", + "log_format", + "log_format_file", + "log_level", + "log_level_file", + "partial_parse", + "populate_cache", + "printer_width", + "static_parser", + "use_colors", + "use_colors_file", + "use_experimental_parser", + "version_check", + "warn_error", + "warn_error_options", + "write_json", + ] + for flag in project_flags: + assert getattr(flags, flag) is None diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 38516fdea0a..e8728091c5d 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -119,12 +119,17 @@ class BaseConfigTest(unittest.TestCase): """ def setUp(self): + # Write project + self.project_dir = normalize(tempfile.mkdtemp()) self.default_project_data = { "version": "0.0.1", "name": "my_test_project", "profile": "default", - "config-version": 2, } + self.write_project(self.default_project_data) + + # Write profile + self.profiles_dir = normalize(tempfile.mkdtemp()) self.default_profile_data = { "default": { "outputs": { @@ -176,6 +181,8 @@ def setUp(self): }, "empty_profile_data": {}, } + self.write_profile(self.default_profile_data) + self.args = Namespace( profiles_dir=self.profiles_dir, cli_vars={}, @@ -203,13 +210,6 @@ def assertRaisesOrReturns(self, exc): else: return self.assertRaises(exc) - -class BaseFileTest(BaseConfigTest): - def setUp(self): - self.project_dir = normalize(tempfile.mkdtemp()) - self.profiles_dir = normalize(tempfile.mkdtemp()) - super().setUp() - def tearDown(self): try: shutil.rmtree(self.project_dir) @@ -248,11 +248,6 @@ def write_empty_profile(self): class TestProfile(BaseConfigTest): - def setUp(self): - self.profiles_dir = "/invalid-path" - self.project_dir = "/invalid-project-path" - super().setUp() - def from_raw_profiles(self): renderer = empty_profile_renderer() return dbt.config.Profile.from_raw_profiles(self.default_profile_data, "default", renderer) @@ -262,8 +257,6 @@ def test_from_raw_profiles(self): self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "postgres") self.assertEqual(profile.threads, 7) - self.assertTrue(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) self.assertTrue(isinstance(profile.credentials, PostgresCredentials)) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "postgres-db-hostname") @@ -273,29 +266,6 @@ def test_from_raw_profiles(self): self.assertEqual(profile.credentials.schema, "postgres-schema") self.assertEqual(profile.credentials.database, "postgres-db-name") - def test_config_override(self): - self.default_profile_data["config"] = { - "send_anonymous_usage_stats": False, - "use_colors": False, - } - profile = self.from_raw_profiles() - self.assertEqual(profile.profile_name, "default") - self.assertEqual(profile.target_name, "postgres") - self.assertFalse(profile.user_config.send_anonymous_usage_stats) - self.assertFalse(profile.user_config.use_colors) - - def test_partial_config_override(self): - self.default_profile_data["config"] = { - "send_anonymous_usage_stats": False, - "printer_width": 60, - } - profile = self.from_raw_profiles() - self.assertEqual(profile.profile_name, "default") - self.assertEqual(profile.target_name, "postgres") - self.assertFalse(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) - self.assertEqual(profile.user_config.printer_width, 60) - def test_missing_type(self): del self.default_profile_data["default"]["outputs"]["postgres"]["type"] with self.assertRaises(dbt.exceptions.DbtProfileError) as exc: @@ -337,7 +307,7 @@ def test_extra_path(self): } ) with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: - project_from_config_norender(self.default_project_data) + project_from_config_norender(self.default_project_data, project_root=self.project_dir) self.assertIn("source-paths and model-paths", str(exc.exception)) self.assertIn("cannot both be defined.", str(exc.exception)) @@ -403,11 +373,7 @@ def test_invalid_env_vars(self): self.assertIn("Could not convert value 'hello' into type 'number'", str(exc.exception)) -class TestProfileFile(BaseFileTest): - def setUp(self): - super().setUp() - self.write_profile(self.default_profile_data) - +class TestProfileFile(BaseConfigTest): def from_raw_profile_info(self, raw_profile=None, profile_name="default", **kwargs): if raw_profile is None: raw_profile = self.default_profile_data["default"] @@ -438,8 +404,6 @@ def test_profile_simple(self): self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "postgres") self.assertEqual(profile.threads, 7) - self.assertTrue(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) self.assertTrue(isinstance(profile.credentials, PostgresCredentials)) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "postgres-db-hostname") @@ -464,8 +428,6 @@ def test_profile_override(self): self.assertEqual(profile.profile_name, "other") self.assertEqual(profile.target_name, "other-postgres") self.assertEqual(profile.threads, 3) - self.assertTrue(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) self.assertTrue(isinstance(profile.credentials, PostgresCredentials)) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "other-postgres-db-hostname") @@ -485,8 +447,6 @@ def test_env_vars(self): self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "with-vars") self.assertEqual(profile.threads, 1) - self.assertTrue(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "env-postgres-host") self.assertEqual(profile.credentials.port, 6543) @@ -505,8 +465,6 @@ def test_env_vars_env_target(self): self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "with-vars") self.assertEqual(profile.threads, 1) - self.assertTrue(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "env-postgres-host") self.assertEqual(profile.credentials.port, 6543) @@ -537,8 +495,6 @@ def test_cli_and_env_vars(self): self.assertEqual(profile.profile_name, "default") self.assertEqual(profile.target_name, "cli-and-env-vars") self.assertEqual(profile.threads, 1) - self.assertTrue(profile.user_config.send_anonymous_usage_stats) - self.assertIsNone(profile.user_config.use_colors) self.assertEqual(profile.credentials.type, "postgres") self.assertEqual(profile.credentials.host, "cli-postgres-host") self.assertEqual(profile.credentials.port, 6543) @@ -567,18 +523,19 @@ def test_profile_with_empty_profile_data(self): def project_from_config_norender( - cfg, packages=None, path="/invalid-root-path", verify_version=False + cfg, packages=None, project_root="/invalid-root-path", verify_version=False ): if packages is None: packages = {} partial = dbt.config.project.PartialProject.from_dicts( - path, + project_root, project_dict=cfg, packages_dict=packages, selectors_dict={}, verify_version=verify_version, ) - # no rendering + # no rendering ... Why? + partial.project_dict["project-root"] = project_root rendered = dbt.config.project.RenderComponents( project_dict=partial.project_dict, packages_dict=partial.packages_dict, @@ -590,14 +547,14 @@ def project_from_config_norender( def project_from_config_rendered( cfg, packages=None, - path="/invalid-root-path", + project_root="/invalid-root-path", verify_version=False, packages_specified_path=PACKAGES_FILE_NAME, ): if packages is None: packages = {} partial = dbt.config.project.PartialProject.from_dicts( - path, + project_root, project_dict=cfg, packages_dict=packages, selectors_dict={}, @@ -608,18 +565,14 @@ def project_from_config_rendered( class TestProject(BaseConfigTest): - def setUp(self): - self.profiles_dir = "/invalid-profiles-path" - self.project_dir = "/invalid-root-path" - super().setUp() - self.default_project_data["project-root"] = self.project_dir - def test_defaults(self): - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project.project_name, "my_test_project") self.assertEqual(project.version, "0.0.1") self.assertEqual(project.profile_name, "default") - self.assertEqual(project.project_root, "/invalid-root-path") + self.assertEqual(project.project_root, self.project_dir) self.assertEqual(project.model_paths, ["models"]) self.assertEqual(project.macro_paths, ["macros"]) self.assertEqual(project.seed_paths, ["seeds"]) @@ -645,30 +598,38 @@ def test_defaults(self): str(project) def test_eq(self): - project = project_from_config_norender(self.default_project_data) - other = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) + other = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project, other) def test_neq(self): - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertNotEqual(project, object()) def test_implicit_overrides(self): self.default_project_data.update( { "model-paths": ["other-models"], - "target-path": "other-target", } ) - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual( set(project.docs_paths), set(["other-models", "seeds", "snapshots", "analyses", "macros"]), ) - self.assertEqual(project.clean_targets, ["other-target"]) def test_hashed_name(self): - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project.hashed_name(), "754cd47eac1d6f50a5f7cd399ec43da4") def test_all_overrides(self): @@ -682,7 +643,6 @@ def test_all_overrides(self): "analysis-paths": ["other-analyses"], "docs-paths": ["docs"], "asset-paths": ["other-assets"], - "target-path": "other-target", "clean-targets": ["another-target"], "packages-install-path": "other-dbt_packages", "quoting": {"identifier": False}, @@ -731,11 +691,12 @@ def test_all_overrides(self): {"git": "git@example.com:dbt-labs/dbt-utils.git", "revision": "test-rev"}, ], } - project = project_from_config_norender(self.default_project_data, packages=packages) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir, packages=packages + ) self.assertEqual(project.project_name, "my_test_project") self.assertEqual(project.version, "0.0.1") self.assertEqual(project.profile_name, "default") - self.assertEqual(project.project_root, "/invalid-root-path") self.assertEqual(project.model_paths, ["other-models"]) self.assertEqual(project.macro_paths, ["other-macros"]) self.assertEqual(project.seed_paths, ["other-seeds"]) @@ -743,7 +704,6 @@ def test_all_overrides(self): self.assertEqual(project.analysis_paths, ["other-analyses"]) self.assertEqual(project.docs_paths, ["docs"]) self.assertEqual(project.asset_paths, ["other-assets"]) - self.assertEqual(project.target_path, "other-target") self.assertEqual(project.clean_targets, ["another-target"]) self.assertEqual(project.packages_install_path, "other-dbt_packages") self.assertEqual(project.quoting, {"identifier": False}) @@ -822,11 +782,12 @@ def test_string_run_hooks(self): def test_invalid_project_name(self): self.default_project_data["name"] = "invalid-project-name" with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: - project_from_config_norender(self.default_project_data) + project_from_config_norender(self.default_project_data, project_root=self.project_dir) self.assertIn("invalid-project-name", str(exc.exception)) def test_no_project(self): + os.remove(os.path.join(self.project_dir, "dbt_project.yml")) renderer = empty_project_renderer() with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: dbt.config.Project.from_project_root(self.project_dir, renderer) @@ -836,12 +797,12 @@ def test_no_project(self): def test_invalid_version(self): self.default_project_data["require-dbt-version"] = "hello!" with self.assertRaises(dbt.exceptions.DbtProjectError): - project_from_config_norender(self.default_project_data) + project_from_config_norender(self.default_project_data, project_root=self.project_dir) def test_unsupported_version(self): self.default_project_data["require-dbt-version"] = ">99999.0.0" # allowed, because the RuntimeConfig checks, not the Project itself - project_from_config_norender(self.default_project_data) + project_from_config_norender(self.default_project_data, project_root=self.project_dir) def test_none_values(self): self.default_project_data.update( @@ -891,7 +852,9 @@ def test_query_comment_disabled(self): "query-comment": None, } ) - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project.query_comment.comment, "") self.assertEqual(project.query_comment.append, False) @@ -900,12 +863,16 @@ def test_query_comment_disabled(self): "query-comment": "", } ) - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project.query_comment.comment, "") self.assertEqual(project.query_comment.append, False) def test_default_query_comment(self): - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project.query_comment, QueryComment()) def test_default_query_comment_append(self): @@ -914,7 +881,9 @@ def test_default_query_comment_append(self): "query-comment": {"append": True}, } ) - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project.query_comment.comment, DEFAULT_QUERY_COMMENT) self.assertEqual(project.query_comment.append, True) @@ -924,7 +893,9 @@ def test_custom_query_comment_append(self): "query-comment": {"comment": "run by user test", "append": True}, } ) - project = project_from_config_norender(self.default_project_data) + project = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project.query_comment.comment, "run by user test") self.assertEqual(project.query_comment.append, True) @@ -946,17 +917,13 @@ def test_packages_from_dependencies(self): assert git_package.git == "{{ env_var('some_package') }}" -class TestProjectFile(BaseFileTest): - def setUp(self): - super().setUp() - self.write_project(self.default_project_data) - # and after the fact, add the project root - self.default_project_data["project-root"] = self.project_dir - +class TestProjectFile(BaseConfigTest): def test_from_project_root(self): renderer = empty_project_renderer() project = dbt.config.Project.from_project_root(self.project_dir, renderer) - from_config = project_from_config_norender(self.default_project_data) + from_config = project_from_config_norender( + self.default_project_data, project_root=self.project_dir + ) self.assertEqual(project, from_config) self.assertEqual(project.version, "0.0.1") self.assertEqual(project.project_name, "my_test_project") @@ -973,12 +940,7 @@ def run(self): pass -class TestConfiguredTask(BaseFileTest): - def setUp(self): - super().setUp() - self.write_project(self.default_project_data) - self.write_profile(self.default_profile_data) - +class TestConfiguredTask(BaseConfigTest): def tearDown(self): super().tearDown() # These tests will change the directory to the project path, @@ -997,15 +959,13 @@ def test_configured_task_dir_change_with_bad_path(self): InheritsFromConfiguredTask.from_args(self.args) -class TestVariableProjectFile(BaseFileTest): +class TestVariableProjectFile(BaseConfigTest): def setUp(self): super().setUp() self.default_project_data["version"] = "{{ var('cli_version') }}" self.default_project_data["name"] = "blah" self.default_project_data["profile"] = "{{ env_var('env_value_profile') }}" self.write_project(self.default_project_data) - # and after the fact, add the project root - self.default_project_data["project-root"] = self.project_dir def test_cli_and_env_vars(self): renderer = dbt.config.renderer.DbtProjectYamlRenderer(None, {"cli_version": "0.1.2"}) @@ -1022,15 +982,11 @@ def test_cli_and_env_vars(self): class TestRuntimeConfig(BaseConfigTest): - def setUp(self): - self.profiles_dir = "/invalid-profiles-path" - self.project_dir = "/invalid-root-path" - super().setUp() - self.default_project_data["project-root"] = self.project_dir - def get_project(self): return project_from_config_norender( - self.default_project_data, verify_version=self.args.version_check + self.default_project_data, + project_root=self.project_dir, + verify_version=self.args.version_check, ) def get_profile(self): @@ -1079,14 +1035,6 @@ def test_str(self): # to make sure nothing terrible happens str(config) - def test_validate_fails(self): - project = self.get_project() - profile = self.get_profile() - # invalid - must be boolean - profile.user_config.use_colors = 100 - with self.assertRaises(dbt.exceptions.DbtProjectError): - dbt.config.RuntimeConfig.from_parts(project, profile, {}) - def test_supported_version(self): self.default_project_data["require-dbt-version"] = ">0.0.0" conf = self.from_parts() @@ -1210,7 +1158,9 @@ def setUp(self): } def get_project(self): - return project_from_config_norender(self.default_project_data, verify_version=True) + return project_from_config_norender( + self.default_project_data, project_root=self.project_dir, verify_version=True + ) def get_profile(self): renderer = empty_profile_renderer() @@ -1242,14 +1192,7 @@ def test__warn_for_unused_resource_config_paths(self): assert expected_msg in msg -class TestRuntimeConfigFiles(BaseFileTest): - def setUp(self): - super().setUp() - self.write_profile(self.default_profile_data) - self.write_project(self.default_project_data) - # and after the fact, add the project root - self.default_project_data["project-root"] = self.project_dir - +class TestRuntimeConfigFiles(BaseConfigTest): def test_from_args(self): with temp_cd(self.project_dir): config = dbt.config.RuntimeConfig.from_args(self.args) @@ -1279,7 +1222,7 @@ def test_from_args(self): self.assertEqual(config.project_name, "my_test_project") -class TestVariableRuntimeConfigFiles(BaseFileTest): +class TestVariableRuntimeConfigFiles(BaseConfigTest): def setUp(self): super().setUp() self.default_project_data.update( @@ -1311,9 +1254,6 @@ def setUp(self): } ) self.write_project(self.default_project_data) - self.write_profile(self.default_profile_data) - # and after the fact, add the project root - self.default_project_data["project-root"] = self.project_dir def test_cli_and_env_vars(self): self.args.target = "cli-and-env-vars" @@ -1387,3 +1327,30 @@ def test_lookups(self): for node, key, expected_value in expected: value = vars_provider.vars_for(node, "postgres").get(key) assert value == expected_value + + +class TestMultipleProjectFlags(BaseConfigTest): + def setUp(self): + super().setUp() + + self.default_project_data.update( + { + "flags": { + "send_anonymous_usage_data": False, + } + } + ) + self.write_project(self.default_project_data) + + self.default_profile_data.update( + { + "config": { + "send_anonymous_usage_data": False, + } + } + ) + self.write_profile(self.default_profile_data) + + def test_setting_multiple_flags(self): + with pytest.raises(dbt.exceptions.DbtProjectError): + set_from_args(self.args, None) diff --git a/tests/unit/test_flags.py b/tests/unit/test_flags.py deleted file mode 100644 index 69d8913b675..00000000000 --- a/tests/unit/test_flags.py +++ /dev/null @@ -1,340 +0,0 @@ -import os -from unittest import TestCase -from argparse import Namespace -import pytest - -from dbt import flags -from dbt.contracts.project import UserConfig -from dbt.graph.selector_spec import IndirectSelection -from dbt.helper_types import WarnErrorOptions - -# Skip due to interface for flag updated -pytestmark = pytest.mark.skip - - -class TestFlags(TestCase): - def setUp(self): - self.args = Namespace() - self.user_config = UserConfig() - - def test__flags(self): - - # use_experimental_parser - self.user_config.use_experimental_parser = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.USE_EXPERIMENTAL_PARSER, True) - os.environ["DBT_USE_EXPERIMENTAL_PARSER"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.USE_EXPERIMENTAL_PARSER, False) - setattr(self.args, "use_experimental_parser", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.USE_EXPERIMENTAL_PARSER, True) - # cleanup - os.environ.pop("DBT_USE_EXPERIMENTAL_PARSER") - delattr(self.args, "use_experimental_parser") - flags.USE_EXPERIMENTAL_PARSER = False - self.user_config.use_experimental_parser = None - - # static_parser - self.user_config.static_parser = False - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.STATIC_PARSER, False) - os.environ["DBT_STATIC_PARSER"] = "true" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.STATIC_PARSER, True) - setattr(self.args, "static_parser", False) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.STATIC_PARSER, False) - # cleanup - os.environ.pop("DBT_STATIC_PARSER") - delattr(self.args, "static_parser") - flags.STATIC_PARSER = True - self.user_config.static_parser = None - - # warn_error - self.user_config.warn_error = False - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WARN_ERROR, False) - os.environ["DBT_WARN_ERROR"] = "true" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WARN_ERROR, True) - setattr(self.args, "warn_error", False) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WARN_ERROR, False) - # cleanup - os.environ.pop("DBT_WARN_ERROR") - delattr(self.args, "warn_error") - flags.WARN_ERROR = False - self.user_config.warn_error = None - - # warn_error_options - self.user_config.warn_error_options = '{"include": "all"}' - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WARN_ERROR_OPTIONS, WarnErrorOptions(include="all")) - os.environ["DBT_WARN_ERROR_OPTIONS"] = '{"include": []}' - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WARN_ERROR_OPTIONS, WarnErrorOptions(include=[])) - setattr(self.args, "warn_error_options", '{"include": "all"}') - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WARN_ERROR_OPTIONS, WarnErrorOptions(include="all")) - # cleanup - os.environ.pop("DBT_WARN_ERROR_OPTIONS") - delattr(self.args, "warn_error_options") - self.user_config.warn_error_options = None - - # write_json - self.user_config.write_json = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WRITE_JSON, True) - os.environ["DBT_WRITE_JSON"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WRITE_JSON, False) - setattr(self.args, "write_json", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.WRITE_JSON, True) - # cleanup - os.environ.pop("DBT_WRITE_JSON") - delattr(self.args, "write_json") - - # partial_parse - self.user_config.partial_parse = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.PARTIAL_PARSE, True) - os.environ["DBT_PARTIAL_PARSE"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.PARTIAL_PARSE, False) - setattr(self.args, "partial_parse", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.PARTIAL_PARSE, True) - # cleanup - os.environ.pop("DBT_PARTIAL_PARSE") - delattr(self.args, "partial_parse") - self.user_config.partial_parse = False - - # use_colors - self.user_config.use_colors = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.USE_COLORS, True) - os.environ["DBT_USE_COLORS"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.USE_COLORS, False) - setattr(self.args, "use_colors", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.USE_COLORS, True) - # cleanup - os.environ.pop("DBT_USE_COLORS") - delattr(self.args, "use_colors") - - # debug - self.user_config.debug = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.DEBUG, True) - os.environ["DBT_DEBUG"] = "True" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.DEBUG, True) - os.environ["DBT_DEBUG"] = "False" - setattr(self.args, "debug", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.DEBUG, True) - # cleanup - os.environ.pop("DBT_DEBUG") - delattr(self.args, "debug") - self.user_config.debug = None - - # log_format -- text, json, default - self.user_config.log_format = "text" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.LOG_FORMAT, "text") - os.environ["DBT_LOG_FORMAT"] = "json" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.LOG_FORMAT, "json") - setattr(self.args, "log_format", "text") - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.LOG_FORMAT, "text") - # cleanup - os.environ.pop("DBT_LOG_FORMAT") - delattr(self.args, "log_format") - self.user_config.log_format = None - - # version_check - self.user_config.version_check = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.VERSION_CHECK, True) - os.environ["DBT_VERSION_CHECK"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.VERSION_CHECK, False) - setattr(self.args, "version_check", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.VERSION_CHECK, True) - # cleanup - os.environ.pop("DBT_VERSION_CHECK") - delattr(self.args, "version_check") - - # fail_fast - self.user_config.fail_fast = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.FAIL_FAST, True) - os.environ["DBT_FAIL_FAST"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.FAIL_FAST, False) - setattr(self.args, "fail_fast", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.FAIL_FAST, True) - # cleanup - os.environ.pop("DBT_FAIL_FAST") - delattr(self.args, "fail_fast") - self.user_config.fail_fast = False - - # send_anonymous_usage_stats - self.user_config.send_anonymous_usage_stats = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.SEND_ANONYMOUS_USAGE_STATS, True) - os.environ["DBT_SEND_ANONYMOUS_USAGE_STATS"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.SEND_ANONYMOUS_USAGE_STATS, False) - setattr(self.args, "send_anonymous_usage_stats", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.SEND_ANONYMOUS_USAGE_STATS, True) - os.environ["DO_NOT_TRACK"] = "1" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.SEND_ANONYMOUS_USAGE_STATS, False) - # cleanup - os.environ.pop("DBT_SEND_ANONYMOUS_USAGE_STATS") - os.environ.pop("DO_NOT_TRACK") - delattr(self.args, "send_anonymous_usage_stats") - - # printer_width - self.user_config.printer_width = 100 - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.PRINTER_WIDTH, 100) - os.environ["DBT_PRINTER_WIDTH"] = "80" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.PRINTER_WIDTH, 80) - setattr(self.args, "printer_width", "120") - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.PRINTER_WIDTH, 120) - # cleanup - os.environ.pop("DBT_PRINTER_WIDTH") - delattr(self.args, "printer_width") - self.user_config.printer_width = None - - # indirect_selection - self.user_config.indirect_selection = "eager" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Eager) - self.user_config.indirect_selection = "cautious" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Cautious) - self.user_config.indirect_selection = "buildable" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Buildable) - self.user_config.indirect_selection = None - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Eager) - os.environ["DBT_INDIRECT_SELECTION"] = "cautious" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Cautious) - setattr(self.args, "indirect_selection", "cautious") - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.INDIRECT_SELECTION, IndirectSelection.Cautious) - # cleanup - os.environ.pop("DBT_INDIRECT_SELECTION") - delattr(self.args, "indirect_selection") - self.user_config.indirect_selection = None - - # quiet - self.user_config.quiet = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.QUIET, True) - # cleanup - self.user_config.quiet = None - - # no_print - self.user_config.no_print = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.NO_PRINT, True) - # cleanup - self.user_config.no_print = None - - # cache_selected_only - self.user_config.cache_selected_only = True - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.CACHE_SELECTED_ONLY, True) - os.environ["DBT_CACHE_SELECTED_ONLY"] = "false" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.CACHE_SELECTED_ONLY, False) - setattr(self.args, "cache_selected_only", True) - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.CACHE_SELECTED_ONLY, True) - # cleanup - os.environ.pop("DBT_CACHE_SELECTED_ONLY") - delattr(self.args, "cache_selected_only") - self.user_config.cache_selected_only = False - - # target_path/log_path - flags.set_from_args(self.args, self.user_config) - self.assertIsNone(flags.LOG_PATH) - os.environ["DBT_LOG_PATH"] = "a/b/c" - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.LOG_PATH, "a/b/c") - setattr(self.args, "log_path", "d/e/f") - flags.set_from_args(self.args, self.user_config) - self.assertEqual(flags.LOG_PATH, "d/e/f") - # cleanup - os.environ.pop("DBT_LOG_PATH") - delattr(self.args, "log_path") - - def test__flags_are_mutually_exclusive(self): - # options from user config - self.user_config.warn_error = False - self.user_config.warn_error_options = '{"include":"all"}' - with pytest.raises(ValueError): - flags.set_from_args(self.args, self.user_config) - # cleanup - self.user_config.warn_error = None - self.user_config.warn_error_options = None - - # options from args - setattr(self.args, "warn_error", False) - setattr(self.args, "warn_error_options", '{"include":"all"}') - with pytest.raises(ValueError): - flags.set_from_args(self.args, self.user_config) - # cleanup - delattr(self.args, "warn_error") - delattr(self.args, "warn_error_options") - - # options from environment - os.environ["DBT_WARN_ERROR"] = "false" - os.environ["DBT_WARN_ERROR_OPTIONS"] = '{"include": []}' - with pytest.raises(ValueError): - flags.set_from_args(self.args, self.user_config) - # cleanup - os.environ.pop("DBT_WARN_ERROR") - os.environ.pop("DBT_WARN_ERROR_OPTIONS") - - # options from user config + args - self.user_config.warn_error = False - setattr(self.args, "warn_error_options", '{"include":"all"}') - with pytest.raises(ValueError): - flags.set_from_args(self.args, self.user_config) - # cleanup - self.user_config.warn_error = None - delattr(self.args, "warn_error_options") - - # options from user config + environ - self.user_config.warn_error = False - os.environ["DBT_WARN_ERROR_OPTIONS"] = '{"include": []}' - with pytest.raises(ValueError): - flags.set_from_args(self.args, self.user_config) - # cleanup - self.user_config.warn_error = None - os.environ.pop("DBT_WARN_ERROR_OPTIONS") - - # options from args + environ - setattr(self.args, "warn_error", False) - os.environ["DBT_WARN_ERROR_OPTIONS"] = '{"include": []}' - with pytest.raises(ValueError): - flags.set_from_args(self.args, self.user_config) - # cleanup - delattr(self.args, "warn_error") - os.environ.pop("DBT_WARN_ERROR_OPTIONS") diff --git a/tests/unit/test_graph_selection.py b/tests/unit/test_graph_selection.py index 572c8fed10d..5700cba6606 100644 --- a/tests/unit/test_graph_selection.py +++ b/tests/unit/test_graph_selection.py @@ -13,9 +13,9 @@ from dbt import flags from argparse import Namespace -from dbt.contracts.project import UserConfig +from dbt.contracts.project import ProjectFlags -flags.set_from_args(Namespace(), UserConfig()) +flags.set_from_args(Namespace(), ProjectFlags()) def _get_graph():