diff --git a/dbt/deprecations.py b/dbt/deprecations.py index e7c43fb6fe0..00d656b7be0 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -14,9 +14,16 @@ def show(self, *args, **kwargs): class DBTRepositoriesDeprecation(DBTDeprecation): name = "repositories" - description = """dbt_project.yml configuration option 'repositories' is - deprecated. Please use 'packages' instead. The 'repositories' option will - be removed in a later version of DBT.""" + description = """The dbt_project.yml configuration option 'repositories' is + deprecated. Please place dependencies in the `packages.yml` file instead. + The 'repositories' option will be removed in a future version of dbt. + + For more information, see: https://docs.getdbt.com/docs/package-management + + # Example packages.yml contents: + +{recommendation} + """ class SeedDropExistingDeprecation(DBTDeprecation): diff --git a/dbt/project.py b/dbt/project.py index 2fb7b0bb156..802f05226de 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -11,6 +11,7 @@ import dbt.clients.jinja import dbt.compat import dbt.context.common +import dbt.clients.system from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -25,7 +26,7 @@ 'target': 'default', 'models': {}, 'profile': None, - 'repositories': [], + 'packages': [], 'modules-path': 'dbt_modules' } @@ -244,16 +245,37 @@ def read_profiles(profiles_dir=None): return profiles -def read_project(filename, profiles_dir=None, validate=True, +def read_packages(project_dir): + + package_filepath = dbt.clients.system.resolve_path_from_base( + 'packages.yml', project_dir) + + if dbt.clients.system.path_exists(package_filepath): + package_file_contents = dbt.clients.system.load_file_contents( + package_filepath) + package_cfg = dbt.clients.yaml_helper.load_yaml_text( + package_file_contents) + else: + package_cfg = {} + + return package_cfg.get('packages', []) + + +def read_project(project_filepath, profiles_dir=None, validate=True, profile_to_load=None, args=None): if profiles_dir is None: profiles_dir = default_profiles_dir - project_file_contents = dbt.clients.system.load_file_contents(filename) + project_dir = os.path.dirname(os.path.abspath(project_filepath)) + project_file_contents = dbt.clients.system.load_file_contents( + project_filepath) project_cfg = dbt.clients.yaml_helper.load_yaml_text(project_file_contents) - project_cfg['project-root'] = os.path.dirname( - os.path.abspath(filename)) + package_cfg = read_packages(project_dir) + + project_cfg['project-root'] = project_dir + project_cfg['packages'] = package_cfg + profiles = read_profiles(profiles_dir) proj = Project(project_cfg, profiles, diff --git a/dbt/task/deps.py b/dbt/task/deps.py index 8a6d36e5db8..f722a6e3565 100644 --- a/dbt/task/deps.py +++ b/dbt/task/deps.py @@ -3,12 +3,13 @@ import hashlib import tempfile import six +import yaml +import dbt.project import dbt.deprecations import dbt.clients.git import dbt.clients.system import dbt.clients.registry as registry -from dbt.clients.yaml_helper import load_yaml_text from dbt.compat import basestring from dbt.logger import GLOBAL_LOGGER as logger @@ -205,8 +206,7 @@ def _checkout(self, project): def _fetch_metadata(self, project): path = self._checkout(project) - with open(os.path.join(path, 'dbt_project.yml')) as f: - return load_yaml_text(f.read()) + return dbt.utils.load_project_with_profile(project, path) def install(self, project): dest_path = self.get_installation_path(project) @@ -237,8 +237,7 @@ def _fetch_metadata(self, project): self.local, project['project-root']) - with open(os.path.join(project_file_path, 'dbt_project.yml')) as f: - return load_yaml_text(f.read()) + return dbt.utils.load_project_with_profile(project, project_file_path) def install(self, project): src_path = dbt.clients.system.resolve_path_from_base( @@ -351,8 +350,12 @@ def _read_packages(project_yaml): packages = project_yaml.get('packages', []) repos = project_yaml.get('repositories', []) if repos: - dbt.deprecations.warn('repositories') - packages += [_convert_repo(r) for r in repos] + bad_packages = [_convert_repo(r) for r in repos] + packages += bad_packages + + fixed_packages = {"packages": bad_packages} + recommendation = yaml.dump(fixed_packages, default_flow_style=False) + dbt.deprecations.warn('repositories', recommendation=recommendation) return packages @@ -374,7 +377,7 @@ def run(self): packages = _read_packages(self.project) if not packages: - logger.info('Warning: No packages found in dbt_project.yml') + logger.info('Warning: No packages were found in packages.yml') return pending_deps = PackageListing.create(packages) diff --git a/dbt/utils.py b/dbt/utils.py index 78705be89a1..24c75538a50 100644 --- a/dbt/utils.py +++ b/dbt/utils.py @@ -198,6 +198,15 @@ def get_materialization_macro(flat_graph, materialization_name, return macro +def load_project_with_profile(source_project, project_dir): + project_filepath = os.path.join(project_dir, 'dbt_project.yml') + return dbt.project.read_project( + project_filepath, + source_project.profiles_dir, + profile_to_load=source_project.profile_to_load, + args=source_project.args) + + def dependency_projects(project): import dbt.project module_paths = [ @@ -218,11 +227,7 @@ def dependency_projects(project): continue try: - yield dbt.project.read_project( - os.path.join(full_obj, 'dbt_project.yml'), - project.profiles_dir, - profile_to_load=project.profile_to_load, - args=project.args) + yield load_project_with_profile(project, full_obj) except dbt.project.DbtProjectError as e: logger.info( "Error reading dependency project at {}".format( diff --git a/sample.dbt_project.yml b/sample.dbt_project.yml index 158a64b411d..d581b70b826 100644 --- a/sample.dbt_project.yml +++ b/sample.dbt_project.yml @@ -141,7 +141,6 @@ models: dist: "user_id" # This is the configuration for the "quickbooks" open source package - # which is included in the `repositories` section below. # These configs override those defined within the quickbooks package quickbooks: base: @@ -170,20 +169,7 @@ on-run-end: # Package configurations # -# repositories: Optional. Contains a list of packages (git repositories) that -# should be downloaded and included in this project. When a package is -# included, its models are added to the dbt model graph and executed with -# `dbt run`. These configs can be specified above (as with quickbooks). -# -# Each repository should be a git-cloneable url. Branches and Tags can be -# specified by adding @branch-or-version to the end of the url, ex: -# -# - https://github.com/fishtown-analytics/quickbooks -# - https://github.com/fishtown-analytics/quickbooks@v0.1.0 -# - https://github.com/fishtown-analytics/quickbooks@master - -repositories: - - https://github.com/fishtown-analytics/quickbooks +# More info here: https://docs.getdbt.com/docs/package-management diff --git a/sample.packages.yml b/sample.packages.yml new file mode 100644 index 00000000000..69288c9ff2d --- /dev/null +++ b/sample.packages.yml @@ -0,0 +1,10 @@ + + +packages: + - local: /opt/dbt/snowplow + + - git: https://github.com/fishtown-analytics/redshift.git + revision: master + + - git: https://github.com/fishtown-analytics/dbt-utils.git + revision: 0.1.0