Merge pull request #1241 from fishtown-analytics/0.12.latest

Merge 0.12.latest into master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.12.1
+current_version = 0.12.2
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)

.circleci/config.yml

@@ -1,111 +1,83 @@
 version: 2
 jobs:
   unit:
-    docker: &py36
-      - image: python:3.6
+    docker: &test_and_postgres
+      - image: fishtownjacob/test-container
       - image: postgres
+        name: database
         environment: &pgenv
           POSTGRES_USER: "root"
           POSTGRES_PASSWORD: "password"
           POSTGRES_DB: "dbt"
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run: &setupdb
           name: Setup postgres
           command: bash test/setup_db.sh
           environment:
-            PGHOST: 127.0.0.1
+            PGHOST: database
             PGUSER: root
             PGPASSWORD: password
             PGDATABASE: postgres
       - run: tox -e pep8,unit-py27,unit-py36
   integration-postgres-py36:
-    docker: *py36
+    docker: *test_and_postgres
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run: *setupdb
       - run:
           name: Run tests
           command: tox -e integration-postgres-py36
   integration-snowflake-py36:
-    docker: *py36
+    docker: &test_only
+      - image: fishtownjacob/test-container
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-snowflake-py36
           no_output_timeout: 1h
   integration-redshift-py36:
-    docker: *py36
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-redshift-py36
   integration-bigquery-py36:
-    docker: *py36
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-bigquery-py36
   integration-postgres-py27:
-    docker: &py27
-      - image: python:2.7
-      - image: postgres
-        environment: *pgenv
+    docker: *test_and_postgres
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run: *setupdb
       - run:
           name: Run tests
           command: tox -e integration-postgres-py27
   integration-snowflake-py27:
-    docker: *py27
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-snowflake-py27
           no_output_timeout: 1h
   integration-redshift-py27:
-    docker: *py27
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-redshift-py27
   integration-bigquery-py27:
-    docker: *py27
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-bigquery-py27

.dockerignore

@@ -0,0 +1 @@
+*

CHANGELOG.md

@@ -1,3 +1,37 @@
+## dbt 0.12.2 - Grace Kelly (January 8, 2019)
+
+### Overview
+
+This release reduces the runtime of dbt projects by improving dbt's approach to model running. Additionally, a number of workflow improvements have been added. 
+
+### Deprecations
+- Deprecate `sql_where` ([#744](https://github.com/fishtown-analytics/dbt/issues/744)) ([docs](https://docs.getdbt.com/v0.12/docs/configuring-incremental-models))
+
+### Features
+- More intelligently order and execute nodes in the graph. This _significantly_ speeds up the runtime of most dbt projects ([#813](https://github.com/fishtown-analytics/dbt/issues/813))
+- Add `-m` flag as an alias for `--models` ([#1160](https://github.com/fishtown-analytics/dbt/issues/1160))
+- Add `post_hook` and `pre_hook` as aliases for `post-hook` and `pre-hook`, respectively ([#1124](https://github.com/fishtown-analytics/dbt/issues/1124)) ([docs](https://docs.getdbt.com/v0.12/docs/using-hooks))
+- Better handling of git errors in `dbt deps` + full support for Windows ([#994](https://github.com/fishtown-analytics/dbt/issues/994), [#778](https://github.com/fishtown-analytics/dbt/issues/778), [#895](https://github.com/fishtown-analytics/dbt/issues/895))
+- Add support for specifying a `location` in BigQuery datasets ([#969](https://github.com/fishtown-analytics/dbt/issues/969)) ([docs](https://docs.getdbt.com/v0.12/docs/supported-databases#section-dataset-locations))
+- Add support for Jinja expressions using the `{% do ... %}` block ([#1113](https://github.com/fishtown-analytics/dbt/issues/1113))
+- The `dbt debug` command is actually useful now ([#1061](https://github.com/fishtown-analytics/dbt/issues/1061))
+- The `config` function can now be called multiple times in a model ([#558](https://github.com/fishtown-analytics/dbt/issues/558))
+- Source the latest version of dbt from PyPi instead of GitHub ([#1122](https://github.com/fishtown-analytics/dbt/issues/1122))
+- Add a peformance profiling mechnanism to dbt ([#1001](https://github.com/fishtown-analytics/dbt/issues/1001))
+- Add caching for dbt's macros-only manifest to speedup parsing ([#1098](https://github.com/fishtown-analytics/dbt/issues/1098))
+
+### Fixes
+- Fix for custom schemas used alongside the `generate_schema_name` macro ([#801](https://github.com/fishtown-analytics/dbt/issues/801))
+- Fix for silent failure of tests that reference nonexistent models ([#968](https://github.com/fishtown-analytics/dbt/issues/968))
+- Fix for `generate_schema_name` macros that return whitespace-padded schema names ([#1074](https://github.com/fishtown-analytics/dbt/issues/1074))
+- Fix for incorrect relation type for backup tables on Snowflake ([#1103](https://github.com/fishtown-analytics/dbt/issues/1103))
+- Fix for incorrectly cased values in the relation cache ([#1140](https://github.com/fishtown-analytics/dbt/issues/1140))
+- Fix for JSON decoding error on Python2 installed with Anaconda ([#1155](https://github.com/fishtown-analytics/dbt/issues/1155))
+- Fix for unhandled exceptions that occur in anonymous event tracking ([#1180](https://github.com/fishtown-analytics/dbt/issues/1180))
+- Fix for analysis files that contain `raw` tags ([#1152](https://github.com/fishtown-analytics/dbt/issues/1152))
+- Fix for packages which reference the [hubsite](hub.getdbt.com) ([#1095](https://github.com/fishtown-analytics/dbt/issues/1095))
+
+
 ## dbt 0.12.1 - (November 15, 2018)
 
 ### Overview

Dockerfile

@@ -1,14 +1,16 @@
 FROM python:3.6
 
-RUN apt-get update
-
-RUN apt-get install -y python-pip netcat
-RUN apt-get install -y python-dev python3-dev
+# do these on one line so changes trigger apt-get update
+RUN apt-get update && \
+    apt-get install -y python-pip netcat python-dev python3-dev postgresql
 
 RUN pip install pip --upgrade
 RUN pip install virtualenv
 RUN pip install virtualenvwrapper
 RUN pip install tox
 
+RUN useradd -mU dbt_test_user
+USER dbt_test_user
+
 WORKDIR /usr/src/app
 RUN cd /usr/src/app

README.md

@@ -1,49 +1,58 @@
-# dbt
+<p align="center">
+  <img src="/etc/dbt-horizontal.png" alt="dbt logo"/>
+</p>
+<p align="center">
+  <a href="https://codeclimate.com/github/fishtown-analytics/dbt">
+    <img src="https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg" alt="Code Climate"/>
+  </a>
+  <a href="https://circleci.com/gh/fishtown-analytics/dbt/tree/master">
+    <img src="https://circleci.com/gh/fishtown-analytics/dbt/tree/master.svg?style=svg"  alt="CircleCI" />
+  </a>
+  <a href="https://ci.appveyor.com/project/DrewBanin/dbt/branch/development">
+    <img src="https://ci.appveyor.com/api/projects/status/v01rwd3q91jnwp9m/branch/development?svg=true" alt="AppVeyor" />
+  </a>
+  <a href="https://slack.getdbt.com">
+    <img src="https://slack.getdbt.com/badge.svg" alt="Slack" />
+  </a>
+</p>
 
-dbt (data build tool) helps analysts write reliable, modular code using a workflow that closely mirrors software development.
+**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
 
-A dbt project primarily consists of "models". These models are SQL `select` statements that filter, aggregate, and otherwise transform data to facilitate analytics. Analysts use dbt to [aggregate pageviews into sessions](https://github.com/fishtown-analytics/snowplow), calculate [ad spend ROI](https://github.com/fishtown-analytics/facebook-ads), or report on [email campaign performance](https://github.com/fishtown-analytics/mailchimp).
+dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
 
-These models frequently build on top of one another. Fortunately, dbt makes it easy to [manage relationships](https://docs.getdbt.com/reference#ref) between models, [test](https://docs.getdbt.com/docs/testing) your assumptions, and [visualize](https://graph.sinterdata.com/) your projects.
+![dbt architecture](/etc/dbt-arch.png?raw=true)
 
-Still reading? Check out the [docs](https://docs.getdbt.com/docs/overview) for more information.
+dbt can be used to [aggregate pageviews into sessions](https://github.com/fishtown-analytics/snowplow), calculate [ad spend ROI](https://github.com/fishtown-analytics/facebook-ads), or report on [email campaign performance](https://github.com/fishtown-analytics/mailchimp).
 
-![dbt dag](/etc/dag.png?raw=true)
+## Understanding dbt
 
----
-### Getting Started
+Analysts using dbt can transform their data by simply writing select statements, while dbt handles turning these statements into tables and views in a data warehouse.
 
-- [What is dbt]?
-- Read the [dbt viewpoint]
-- [Installation]
-- Join the [chat][slack-url] on Slack for live questions and support.
+These select statements, or "models", form a dbt project. Models frequently build on top of one another – dbt makes it easy to [manage relationships](https://docs.getdbt.com/reference#ref) between models, and [visualize these relationships](https://docs.getdbt.com/docs/documentation), as well as assure the quality of your transformations through [testing](https://docs.getdbt.com/docs/testing).
 
----
-### The dbt ecosystem
-- Visualize your dbt graph [here](https://graph.sinterdata.com/)
-- Run your dbt projects on a schedule [here](http://sinterdata.com/)
+![dbt dag](/etc/dbt-dag.png?raw=true)
 
----
+## Getting started
 
-[![Code Climate](https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg)](https://codeclimate.com/github/fishtown-analytics/dbt) [![Slack](https://slack.getdbt.com/badge.svg)](https://slack.getdbt.com)
+-   [Install dbt](https://docs.getdbt.com/docs/installation)
+-   Read the [documentation](https://docs.getdbt.com/).
+-   Productionize your dbt project with [Sinter](https://www.sinterdata.com)
 
-### Testing
+## Find out more
 
-| service | development | master |
-| --- | --- | --- |
-| CircleCI| [![CircleCI](https://circleci.com/gh/fishtown-analytics/dbt/tree/development.svg?style=svg)](https://circleci.com/gh/fishtown-analytics/dbt/tree/development) | [![CircleCI](https://circleci.com/gh/fishtown-analytics/dbt/tree/master.svg?style=svg)](https://circleci.com/gh/fishtown-analytics/dbt/tree/master) |
-| AppVeyor | [![AppVeyor](https://ci.appveyor.com/api/projects/status/v01rwd3q91jnwp9m/branch/development?svg=true)](https://ci.appveyor.com/project/DrewBanin/dbt/branch/development) | [![AppVeyor](https://ci.appveyor.com/api/projects/status/v01rwd3q91jnwp9m/branch/master?svg=true)](https://ci.appveyor.com/project/DrewBanin/dbt/branch/master) |
+-   Check out the [Introduction to dbt](https://dbt.readme.io/docs/introduction).
+-   Read the [dbt Viewpoint](https://dbt.readme.io/docs/viewpoint).
 
-[Coverage](https://circleci.com/api/v1/project/fishtown-analytics/dbt/latest/artifacts/0/$CIRCLE_ARTIFACTS/htmlcov/index.html?branch=development)
+## Join thousands of analysts in the dbt community
 
-## Code of Conduct
+-   Join the [chat](http://slack.getdbt.com/) on Slack.
+-   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
 
-Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct].
+## Reporting bugs and contributing code
 
+-   Want to report a bug or request a feature? Let us know on [Slack](http://slack.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt/issues/new).
+-   Want to help us build dbt? Check out the [Contributing Getting Started Guide](/CONTRIBUTING.md)
 
+## Code of Conduct
 
-[PyPA Code of Conduct]: https://www.pypa.io/en/latest/code-of-conduct/
-[slack-url]: https://slack.getdbt.com/
-[Installation]: https://docs.getdbt.com/docs/installation
-[What is dbt]: https://docs.getdbt.com/docs/overview
-[dbt viewpoint]: https://docs.getdbt.com/docs/viewpoint
+Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct](https://www.pypa.io/en/latest/code-of-conduct/).

dbt/adapters/bigquery/impl.py

@@ -142,8 +142,9 @@ def get_bigquery_credentials(cls, profile_credentials):
     def get_bigquery_client(cls, profile_credentials):
         project_name = profile_credentials.project
         creds = cls.get_bigquery_credentials(profile_credentials)
-
-        return google.cloud.bigquery.Client(project_name, creds)
+        location = getattr(profile_credentials, 'location', None)
+        return google.cloud.bigquery.Client(project_name, creds,
+                                            location=location)
 
     @classmethod
     def open_connection(cls, connection):
@@ -447,13 +448,6 @@ def create_schema(self, schema, model_name=None):
             with self.exception_handler('create dataset', model_name):
                 client.create_dataset(dataset)
 
-    def drop_tables_in_schema(self, dataset):
-        conn = self.get_connection()
-        client = conn.handle
-
-        for table in client.list_tables(dataset):
-            client.delete_table(table.reference)
-
     def drop_schema(self, schema, model_name=None):
         logger.debug('Dropping schema "%s".', schema)
 
@@ -465,8 +459,7 @@ def drop_schema(self, schema, model_name=None):
 
         dataset = self.get_dataset(schema, model_name)
         with self.exception_handler('drop dataset', model_name):
-            self.drop_tables_in_schema(dataset)
-            client.delete_dataset(dataset)
+            client.delete_dataset(dataset, delete_contents=True)
 
     def get_existing_schemas(self, model_name=None):
         conn = self.get_connection(model_name)