Merge pull request #1110 from fishtown-analytics/fix/dbt-deps-issues

Fix dbt deps issues (#778 #994 #895)

.circleci/config.yml

@@ -1,113 +1,83 @@
 version: 2
 jobs:
   unit:
-    docker: &py36_postgres
-      - image: python:3.6
+    docker: &test_and_postgres
+      - image: fishtownjacob/test-container
       - image: postgres
+        name: database
         environment: &pgenv
           POSTGRES_USER: "root"
           POSTGRES_PASSWORD: "password"
           POSTGRES_DB: "dbt"
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run: &setupdb
           name: Setup postgres
           command: bash test/setup_db.sh
           environment:
-            PGHOST: 127.0.0.1
+            PGHOST: database
             PGUSER: root
             PGPASSWORD: password
             PGDATABASE: postgres
       - run: tox -e pep8,unit-py27,unit-py36
   integration-postgres-py36:
-    docker: *py36_postgres
+    docker: *test_and_postgres
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run: *setupdb
       - run:
           name: Run tests
           command: tox -e integration-postgres-py36
   integration-snowflake-py36:
-    docker: &py36
-      - image: python:3.6
+    docker: &test_only
+      - image: fishtownjacob/test-container
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-snowflake-py36
           no_output_timeout: 1h
   integration-redshift-py36:
-    docker: *py36
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-redshift-py36
   integration-bigquery-py36:
-    docker: *py36
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python3-dev
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-bigquery-py36
   integration-postgres-py27:
-    docker: &py27_postgres
-      - image: python:2.7
-      - image: postgres
-        environment: *pgenv
+    docker: *test_and_postgres
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run: *setupdb
       - run:
           name: Run tests
           command: tox -e integration-postgres-py27
   integration-snowflake-py27:
-    docker: &py27
-      - image: python:2.7
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-snowflake-py27
           no_output_timeout: 1h
   integration-redshift-py27:
-    docker: *py27
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev postgresql
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-redshift-py27
   integration-bigquery-py27:
-    docker: *py27
+    docker: *test_only
     steps:
       - checkout
-      - run: apt-get update && apt-get install -y python-dev
-      - run: echo 127.0.0.1 database | tee -a /etc/hosts
-      - run: pip install virtualenvwrapper tox
       - run:
           name: Run tests
           command: tox -e integration-bigquery-py27

.dockerignore

@@ -0,0 +1 @@
+*

Dockerfile

@@ -1,14 +1,16 @@
 FROM python:3.6
 
-RUN apt-get update
-
-RUN apt-get install -y python-pip netcat
-RUN apt-get install -y python-dev python3-dev
+# do these on one line so changes trigger apt-get update
+RUN apt-get update && \
+    apt-get install -y python-pip netcat python-dev python3-dev postgresql
 
 RUN pip install pip --upgrade
 RUN pip install virtualenv
 RUN pip install virtualenvwrapper
 RUN pip install tox
 
+RUN useradd -mU dbt_test_user
+USER dbt_test_user
+
 WORKDIR /usr/src/app
 RUN cd /usr/src/app

dbt/clients/git.py

@@ -26,10 +26,7 @@ def list_tags(cwd):
     return tags
 
 
-def checkout(cwd, repo, branch=None):
-    if branch is None:
-        branch = 'master'
-
+def _checkout(cwd, repo, branch):
     logger.debug('  Checking out branch {}.'.format(branch))
 
     run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', branch])
@@ -44,12 +41,17 @@ def checkout(cwd, repo, branch=None):
         spec = 'origin/{}'.format(branch)
 
     out, err = run_cmd(cwd, ['git', 'reset', '--hard', spec])
-    stderr = err.decode('utf-8').strip()
+    return out, err
 
-    if stderr.startswith('fatal:'):
-        dbt.exceptions.bad_package_spec(repo, branch, stderr)
-    else:
-        return out, err
+
+def checkout(cwd, repo, branch=None):
+    if branch is None:
+        branch = 'master'
+    try:
+        return _checkout(cwd, repo, branch)
+    except dbt.exceptions.CommandResultError as exc:
+        stderr = exc.stderr.decode('utf-8').strip()
+    dbt.exceptions.bad_package_spec(repo, branch, stderr)
 
 
 def get_current_sha(cwd):
@@ -64,9 +66,16 @@ def remove_remote(cwd):
 
 def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
                        branch=None):
-    _, err = clone(repo, cwd, dirname=dirname, remove_git_dir=remove_git_dir)
-    exists = re.match("fatal: destination path '(.+)' already exists",
-                      err.decode('utf-8'))
+    exists = None
+    try:
+        _, err = clone(repo, cwd, dirname=dirname,
+                       remove_git_dir=remove_git_dir)
+    except dbt.exceptions.CommandResultError as exc:
+        err = exc.stderr.decode('utf-8')
+        exists = re.match("fatal: destination path '(.+)' already exists", err)
+        if not exists:  # something else is wrong, raise it
+            raise
+
     directory = None
     start_sha = None
     if exists: