pull packages from packages.yml file

dbt/deprecations.py

@@ -14,9 +14,16 @@ def show(self, *args, **kwargs):
 
 class DBTRepositoriesDeprecation(DBTDeprecation):
     name = "repositories"
-    description = """dbt_project.yml configuration option 'repositories' is
-    deprecated. Please use 'packages' instead. The 'repositories' option will
-    be removed in a later version of DBT."""
+    description = """The dbt_project.yml configuration option 'repositories' is
+  deprecated. Please place dependencies in the `packages.yml` file instead.
+  The 'repositories' option will be removed in a future version of dbt.
+
+  For more information, see: https://docs.getdbt.com/docs/package-management
+
+  # Example packages.yml contents:
+
+{recommendation}
+  """
 
 
 class SeedDropExistingDeprecation(DBTDeprecation):

dbt/project.py

@@ -11,6 +11,7 @@
 import dbt.clients.jinja
 import dbt.compat
 import dbt.context.common
+import dbt.clients.system
 
 from dbt.logger import GLOBAL_LOGGER as logger  # noqa
 
@@ -25,7 +26,7 @@
     'target': 'default',
     'models': {},
     'profile': None,
-    'repositories': [],
+    'packages': [],
     'modules-path': 'dbt_modules'
 }
 
@@ -244,16 +245,37 @@ def read_profiles(profiles_dir=None):
     return profiles
 
 
-def read_project(filename, profiles_dir=None, validate=True,
+def read_packages(project_dir):
+
+    package_filepath = dbt.clients.system.resolve_path_from_base(
+            'packages.yml', project_dir)
+
+    if dbt.clients.system.path_exists(package_filepath):
+        package_file_contents = dbt.clients.system.load_file_contents(
+                package_filepath)
+        package_cfg = dbt.clients.yaml_helper.load_yaml_text(
+                package_file_contents)
+    else:
+        package_cfg = {}
+
+    return package_cfg.get('packages', [])
+
+
+def read_project(project_filepath, profiles_dir=None, validate=True,
                  profile_to_load=None, args=None):
     if profiles_dir is None:
         profiles_dir = default_profiles_dir
 
-    project_file_contents = dbt.clients.system.load_file_contents(filename)
+    project_dir = os.path.dirname(os.path.abspath(project_filepath))
+    project_file_contents = dbt.clients.system.load_file_contents(
+            project_filepath)
 
     project_cfg = dbt.clients.yaml_helper.load_yaml_text(project_file_contents)
-    project_cfg['project-root'] = os.path.dirname(
-        os.path.abspath(filename))
+    package_cfg = read_packages(project_dir)
+
+    project_cfg['project-root'] = project_dir
+    project_cfg['packages'] = package_cfg
+
     profiles = read_profiles(profiles_dir)
     proj = Project(project_cfg,
                    profiles,

dbt/task/deps.py

@@ -3,12 +3,13 @@
 import hashlib
 import tempfile
 import six
+import yaml
 
+import dbt.project
 import dbt.deprecations
 import dbt.clients.git
 import dbt.clients.system
 import dbt.clients.registry as registry
-from dbt.clients.yaml_helper import load_yaml_text
 
 from dbt.compat import basestring
 from dbt.logger import GLOBAL_LOGGER as logger
@@ -205,8 +206,7 @@ def _checkout(self, project):
 
     def _fetch_metadata(self, project):
         path = self._checkout(project)
-        with open(os.path.join(path, 'dbt_project.yml')) as f:
-            return load_yaml_text(f.read())
+        return dbt.utils.load_project_with_profile(project, path)
 
     def install(self, project):
         dest_path = self.get_installation_path(project)
@@ -237,8 +237,7 @@ def _fetch_metadata(self, project):
             self.local,
             project['project-root'])
 
-        with open(os.path.join(project_file_path, 'dbt_project.yml')) as f:
-            return load_yaml_text(f.read())
+        return dbt.utils.load_project_with_profile(project, project_file_path)
 
     def install(self, project):
         src_path = dbt.clients.system.resolve_path_from_base(
@@ -351,8 +350,12 @@ def _read_packages(project_yaml):
     packages = project_yaml.get('packages', [])
     repos = project_yaml.get('repositories', [])
     if repos:
-        dbt.deprecations.warn('repositories')
-        packages += [_convert_repo(r) for r in repos]
+        bad_packages = [_convert_repo(r) for r in repos]
+        packages += bad_packages
+
+        fixed_packages = {"packages": bad_packages}
+        recommendation = yaml.dump(fixed_packages, default_flow_style=False)
+        dbt.deprecations.warn('repositories', recommendation=recommendation)
     return packages
 
 
@@ -374,7 +377,7 @@ def run(self):
 
         packages = _read_packages(self.project)
         if not packages:
-            logger.info('Warning: No packages found in dbt_project.yml')
+            logger.info('Warning: No packages were found in packages.yml')
             return
 
         pending_deps = PackageListing.create(packages)

dbt/utils.py

@@ -198,6 +198,15 @@ def get_materialization_macro(flat_graph, materialization_name,
     return macro
 
 
+def load_project_with_profile(source_project, project_dir):
+    project_filepath = os.path.join(project_dir, 'dbt_project.yml')
+    return dbt.project.read_project(
+        project_filepath,
+        source_project.profiles_dir,
+        profile_to_load=source_project.profile_to_load,
+        args=source_project.args)
+
+
 def dependency_projects(project):
     import dbt.project
     module_paths = [
@@ -218,11 +227,7 @@ def dependency_projects(project):
                 continue
 
             try:
-                yield dbt.project.read_project(
-                    os.path.join(full_obj, 'dbt_project.yml'),
-                    project.profiles_dir,
-                    profile_to_load=project.profile_to_load,
-                    args=project.args)
+                yield load_project_with_profile(project, full_obj)
             except dbt.project.DbtProjectError as e:
                 logger.info(
                     "Error reading dependency project at {}".format(

sample.dbt_project.yml

@@ -141,7 +141,6 @@ models:
                 dist: "user_id"
 
     # This is the configuration for the "quickbooks" open source package
-    # which is included in the `repositories` section below.
     # These configs override those defined within the quickbooks package
     quickbooks:
         base:
@@ -170,20 +169,7 @@ on-run-end:
 # Package configurations
 #
 
-# repositories: Optional. Contains a list of packages (git repositories) that
-# should be downloaded and included in this project. When a package is
-# included, its models are added to the dbt model graph and executed with
-# `dbt run`. These configs can be specified above (as with quickbooks).
-#
-# Each repository should be a git-cloneable url. Branches and Tags can be
-# specified by adding @branch-or-version to the end of the url, ex:
-#
-# - https://github.com/fishtown-analytics/quickbooks
-# - https://github.com/fishtown-analytics/quickbooks@v0.1.0
-# - https://github.com/fishtown-analytics/quickbooks@master
-
-repositories:
-  - https://github.com/fishtown-analytics/quickbooks
+# More info here: https://docs.getdbt.com/docs/package-management
 
 
 

sample.packages.yml

@@ -0,0 +1,10 @@
+
+
+packages:
+    - local: /opt/dbt/snowplow
+
+    - git: https://github.com/fishtown-analytics/redshift.git
+      revision: master
+
+    - git: https://github.com/fishtown-analytics/dbt-utils.git
+      revision: 0.1.0