Merge pull request #1 from fishtown-analytics/dev/stephen-girard

Merge upstream

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.12.2a1
+current_version = 0.13.0a1
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)
@@ -20,7 +20,15 @@ values =
 [bumpversion:part:num]
 first_value = 1
 
-[bumpversion:file:setup.py]
+[bumpversion:file:core/setup.py]
 
-[bumpversion:file:dbt/version.py]
+[bumpversion:file:core/dbt/version.py]
+
+[bumpversion:file:plugins/postgres/setup.py]
+
+[bumpversion:file:plugins/redshift/setup.py]
+
+[bumpversion:file:plugins/snowflake/setup.py]
+
+[bumpversion:file:plugins/bigquery/setup.py]
 

.coveragerc

@@ -1,3 +1,4 @@
 [report]
 include =
-  dbt/*
+  core/dbt/*
+  plugins/adapters/dbt/*

CHANGELOG.md → core/CHANGELOG.md

@@ -1,3 +1,12 @@
+## dbt dev/stephen-girard (0.13.0? - To be released)
+
+## Overview
+
+This release makes dbt and its adapters into a core-and-plugin architecture.
+
+### Breaking Changes
+- '{{this}}' is no longer respected in hooks [#1176](https://github.com/fishtown-analytics/dbt/pull/1176), implementing [#878](https://github.com/fishtown-analytics/dbt/issues/878)
+
 ## dbt 0.12.1 - (November 15, 2018)
 
 ### Overview

CONTRIBUTING.md → core/CONTRIBUTING.md

@@ -50,6 +50,7 @@ Set up a fresh virtualenv with pyenv-virtualenv for dbt:
 pyenv virtualenv 3.6.5 dbt36
 cd ~/git/dbt
 pyenv local dbt36
+pyenv activate
 ```
 
 This makes a new virtualenv based on python 3.6.5 named `dbt36`, and tells pyenv that when you're in the `dbt` directory it should automatically use that virtualenv.
@@ -131,7 +132,11 @@ Sometimes, you're going to have to pretend to be an end user to reproduce bugs a
 
 ### installation
 
-First, from the `dbt` directory, install dbt in 'editable' mode. There are a couple ways to do it, but I'm in the habit of `pip install -e .`, which tells pip to install the package in the current directory in "editable" mode. What's cool about this mode is any changes you make to the current dbt directory will be reflected immediately in your next `dbt` run.
+First make sure that you setup your `virtualenv` as described in section _Setting up your environment_.
+
+From the `dbt` directory, install dbt in 'editable' mode. There are a couple ways to do it, but I'm in the habit of `pip install -e .`, which tells pip to install the package in the current directory in "editable" mode. Since you are running `pip install` in virtualenv, this will not change your normal dbt installation, if you have one.
+
+What's cool about this mode is any changes you make to the current dbt directory will be reflected immediately in your next `dbt` run.
 
 ### Profile
 
@@ -204,7 +209,7 @@ Found 2 models, 0 tests, 0 archives, 0 analyses, 59 macros, 1 operations, 1 seed
 So what does that mean? Well:
 
 - `2 models` refers to the contents of the `models` directory
-- `59 macros` are the builtin global macros defind by dbt itself
+- `59 macros` are the builtin global macros defined by dbt itself
 - `1 operations` is the catalog generation operation that runs by default
 - `1 seed files` refers to the seed data in `data/moby_dick.csv`
 

core/README.md

@@ -0,0 +1,58 @@
+<p align="center">
+  <img src="/etc/dbt-horizontal.png" alt="dbt logo"/>
+</p>
+<p align="center">
+  <a href="https://codeclimate.com/github/fishtown-analytics/dbt">
+    <img src="https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg" alt="Code Climate"/>
+  </a>
+  <a href="https://circleci.com/gh/fishtown-analytics/dbt/tree/master">
+    <img src="https://circleci.com/gh/fishtown-analytics/dbt/tree/master.svg?style=svg"  alt="CircleCI" />
+  </a>
+  <a href="https://ci.appveyor.com/project/DrewBanin/dbt/branch/development">
+    <img src="https://ci.appveyor.com/api/projects/status/v01rwd3q91jnwp9m/branch/development?svg=true" alt="AppVeyor" />
+  </a>
+  <a href="https://slack.getdbt.com">
+    <img src="https://slack.getdbt.com/badge.svg" alt="Slack" />
+  </a>
+</p>
+
+**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
+
+dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
+
+![dbt architecture](/etc/dbt-arch.png?raw=true)
+
+dbt can be used to [aggregate pageviews into sessions](https://github.com/fishtown-analytics/snowplow), calculate [ad spend ROI](https://github.com/fishtown-analytics/facebook-ads), or report on [email campaign performance](https://github.com/fishtown-analytics/mailchimp).
+
+## Understanding dbt
+
+Analysts using dbt can transform their data by simply writing select statements, while dbt handles turning these statements into tables and views in a data warehouse.
+
+These select statements, or "models", form a dbt project. Models frequently build on top of one another – dbt makes it easy to [manage relationships](https://docs.getdbt.com/reference#ref) between models, and [visualize these relationships](https://docs.getdbt.com/docs/documentation), as well as assure the quality of your transformations through [testing](https://docs.getdbt.com/docs/testing).
+
+![dbt dag](/etc/dbt-dag.png?raw=true)
+
+## Getting started
+
+-   [Install dbt](https://docs.getdbt.com/docs/installation)
+-   Read the [documentation](https://docs.getdbt.com/).
+-   Productionize your dbt project with [Sinter](https://www.sinterdata.com)
+
+## Find out more
+
+-   Check out the [Introduction to dbt](https://dbt.readme.io/docs/introduction).
+-   Read the [dbt Viewpoint](https://dbt.readme.io/docs/viewpoint).
+
+## Join thousands of analysts in the dbt community
+
+-   Join the [chat](http://slack.getdbt.com/) on Slack.
+-   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
+
+## Reporting bugs and contributing code
+
+-   Want to report a bug or request a feature? Let us know on [Slack](http://slack.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt/issues/new).
+-   Want to help us build dbt? Check out the [Contributing Getting Started Guide](/CONTRIBUTING.md)
+
+## Code of Conduct
+
+Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct](https://www.pypa.io/en/latest/code-of-conduct/).

core/dbt/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

core/dbt/adapters/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

dbt/adapters/base/__init__.py → core/dbt/adapters/base/__init__.py

@@ -2,3 +2,4 @@
 from dbt.adapters.base.relation import BaseRelation
 from dbt.adapters.base.connections import BaseConnectionManager, Credentials
 from dbt.adapters.base.impl import BaseAdapter
+from dbt.adapters.base.plugin import AdapterPlugin

dbt/adapters/base/impl.py → core/dbt/adapters/base/impl.py

@@ -21,7 +21,7 @@
 from dbt.adapters.base import BaseRelation
 from dbt.adapters.cache import RelationsCache
 
-GET_CATALOG_OPERATION_NAME = 'get_catalog_data'
+GET_CATALOG_MACRO_NAME = 'get_catalog'
 
 
 def _expect_row_value(key, row):
@@ -655,25 +655,29 @@ def convert_agate_type(cls, agate_table, col_idx):
     ###
     # Operations involving the manifest
     ###
-    def run_operation(self, manifest, operation_name):
-        """Look the operation identified by operation_name up in the manifest
-        and run it.
+    def execute_macro(self, manifest, macro_name, project=None):
+        """Look macro_name up in the manifest and execute its results.
 
         Return an an AttrDict with three attributes: 'table', 'data', and
             'status'. 'table' is an agate.Table.
         """
-        operation = manifest.find_operation_by_name(operation_name, 'dbt')
+        macro = manifest.find_macro_by_name(macro_name, project)
+        if macro is None:
+            raise dbt.exceptions.RuntimeException(
+                'Could not find macro with name {} in project {}'
+                .format(macro_name, project)
+            )
 
         # This causes a reference cycle, as dbt.context.runtime.generate()
         # ends up calling get_adapter, so the import has to be here.
         import dbt.context.runtime
         context = dbt.context.runtime.generate(
-            operation,
+            macro,
             self.config,
             manifest,
         )
 
-        result = operation.generator(context)()
+        result = macro.generator(context)()
         return result
 
     @classmethod
@@ -684,13 +688,13 @@ def _catalog_filter_table(cls, table, manifest):
         return table.where(_catalog_filter_schemas(manifest))
 
     def get_catalog(self, manifest):
-        """Get the catalog for this manifest by running the get catalog
-        operation. Returns an agate.Table of catalog information.
+        """Get the catalog for this manifest by running the get catalog macro.
+        Returns an agate.Table of catalog information.
         """
         try:
-            table = self.run_operation(manifest, GET_CATALOG_OPERATION_NAME)
+            table = self.execute_macro(manifest, GET_CATALOG_MACRO_NAME)
         finally:
-            self.release_connection(GET_CATALOG_OPERATION_NAME)
+            self.release_connection(GET_CATALOG_MACRO_NAME)
 
         results = self._catalog_filter_table(table, manifest)
         return results

core/dbt/adapters/base/plugin.py

@@ -0,0 +1,26 @@
+import os
+
+from dbt.config.project import Project
+
+
+class AdapterPlugin(object):
+    """Defines the basic requirements for a dbt adapter plugin.
+
+    :param type adapter: An adapter class, derived from BaseAdapter
+    :param type credentials: A credentials object, derived from Credentials
+    :param str project_name: The name of this adapter plugin's associated dbt
+        project.
+    :param str include_path: The path to this adapter plugin's root
+    :param Optional[List[str]] dependencies: A list of adapter names that this\
+        adapter depends upon.
+    """
+    def __init__(self, adapter, credentials, include_path, dependencies=None):
+        self.adapter = adapter
+        self.credentials = credentials
+        self.include_path = include_path
+        project_path = os.path.join(self.include_path, adapter.type())
+        project = Project.from_project_root(project_path, {})
+        self.project_name = project.project_name
+        if dependencies is None:
+            dependencies = []
+        self.dependencies = dependencies

dbt/adapters/factory.py → core/dbt/adapters/factory.py

@@ -2,10 +2,10 @@
 
 import dbt.exceptions
 from importlib import import_module
+from dbt.include.global_project import PACKAGES
 
 import threading
 
-
 ADAPTER_TYPES = {}
 
 _ADAPTERS = {}
@@ -28,25 +28,31 @@ def get_relation_class_by_name(adapter_name):
     return adapter.Relation
 
 
-def load_adapter(adapter_name):
-    """Load an adapter package with the class of adapter_name, put it in the
-    ADAPTER_TYPES dict, and return its associated Credentials
-    """
+def load_plugin(adapter_name):
     try:
         mod = import_module('.'+adapter_name, 'dbt.adapters')
     except ImportError:
         raise dbt.exceptions.RuntimeException(
             "Could not find adapter type {}!".format(adapter_name)
         )
-    if mod.Adapter.type() != adapter_name:
+    plugin = mod.Plugin
+
+    if plugin.adapter.type() != adapter_name:
         raise dbt.exceptions.RuntimeException(
             'Expected to find adapter with type named {}, got adapter with '
             'type {}'
-            .format(adapter_name, mod.Adapter.type())
+            .format(adapter_name, plugin.adapter.type())
         )
 
-    ADAPTER_TYPES[adapter_name] = mod.Adapter
-    return mod.Credentials
+    with _ADAPTER_LOCK:
+        ADAPTER_TYPES[adapter_name] = plugin.adapter
+
+    PACKAGES[plugin.project_name] = plugin.include_path
+
+    for dep in plugin.dependencies:
+        load_plugin(dep)
+
+    return plugin.credentials
 
 
 def get_adapter(config):

dbt/clients/jinja.py → core/dbt/clients/jinja.py

@@ -92,10 +92,7 @@ def call(*args, **kwargs):
             template = template_cache.get_node_template(node)
             module = template.make_module(context, False, context)
 
-            if node['resource_type'] == NodeType.Operation:
-                macro = module.__dict__[dbt.utils.get_dbt_operation_name(name)]
-            else:
-                macro = module.__dict__[dbt.utils.get_dbt_macro_name(name)]
+            macro = module.__dict__[dbt.utils.get_dbt_macro_name(name)]
             module.__dict__.update(context)
 
             try:
@@ -148,28 +145,6 @@ def parse(self, parser):
         return node
 
 
-class OperationExtension(jinja2.ext.Extension):
-    tags = ['operation']
-
-    def parse(self, parser):
-        node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
-        operation_name = \
-            parser.parse_assign_target(name_only=True).name
-
-        node.args = []
-        node.defaults = []
-
-        while parser.stream.skip_if('comma'):
-            target = parser.parse_assign_target(name_only=True)
-
-        node.name = dbt.utils.get_operation_macro_name(operation_name)
-
-        node.body = parser.parse_statements(('name:endoperation',),
-                                            drop_needle=True)
-
-        return node
-
-
 class DocumentationExtension(jinja2.ext.Extension):
     tags = ['docs']
 
@@ -253,7 +228,6 @@ def get_environment(node=None, capture_macros=False):
         args['undefined'] = create_macro_capture_env(node)
 
     args['extensions'].append(MaterializationExtension)
-    args['extensions'].append(OperationExtension)
     args['extensions'].append(DocumentationExtension)
 
     return MacroFuzzEnvironment(**args)