package management

converter.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+import json
+import yaml
+import sys
+import argparse
+from datetime import datetime, timezone
+import dbt.clients.registry as registry
+
+
+def yaml_type(fname):
+ with open(fname) as f:
+ return yaml.load(f)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--project", type=yaml_type, default="dbt_project.yml")
+ parser.add_argument("--namespace", required=True)
+ return parser.parse_args()
+
+
+def get_full_name(args):
+ return "{}/{}".format(args.namespace, args.project["name"])
+
+
+def init_project_in_packages(args, packages):
+ full_name = get_full_name(args)
+ if full_name not in packages:
+ packages[full_name] = {
+ "name": args.project["name"],
+ "namespace": args.namespace,
+ "latest": args.project["version"],
+ "assets": {},
+ "versions": {},
+ }
+ return packages[full_name]
+
+
+def add_version_to_package(args, project_json):
+ project_json["versions"][args.project["version"]] = {
+ "id": "{}/{}".format(get_full_name(args), args.project["version"]),
+ "name": args.project["name"],
+ "version": args.project["version"],
+ "description": "",
+ "published_at": datetime.now(timezone.utc).astimezone().isoformat(),
+ "packages": args.project.get("packages") or [],
+ "works_with": [],
+ "_source": {
+ "type": "github",
+ "url": "",
+ "readme": "",
+ },
+ "downloads": {
+ "tarball": "",
+ "format": "tgz",
+ "sha1": "",
+ },
+ }
+
+
+def main():
+ args = parse_args()
+ packages = registry.packages()
+ project_json = init_project_in_packages(args, packages)
+ if args.project["version"] in project_json["versions"]:
+ raise Exception("Version {} already in packages JSON"
+ .format(args.project["version"]),
+ file=sys.stderr)
+ add_version_to_package(args, project_json)
+ print(json.dumps(packages, indent=2))
+
+if __name__ == "__main__":
+ main()

dbt/adapters/bigquery.py

@@ -372,6 +372,12 @@ def get_columns_in_table(cls, profile, schema_name, table_name,
  raise dbt.exceptions.NotImplementedException(
  '`get_columns_in_table` is not implemented for this adapter!')
 
+ @classmethod
+ def get_columns_in_table(cls, profile, schema_name, table_name,
+ model_name=None):
+ raise dbt.exceptions.NotImplementedException(
+ '`get_columns_in_table` is not implemented for this adapter!')
+
  @classmethod
  def check_schema_exists(cls, profile, schema, model_name=None):
  conn = cls.get_connection(profile, model_name)

dbt/clients/git.py

@@ -1,3 +1,4 @@
+import re
 import os.path
 
 from dbt.clients.system import run_cmd, rmdir
@@ -29,7 +30,7 @@ def checkout(cwd, repo, branch=None):
  if branch is None:
  branch = 'master'
 
- logger.info(' Checking out branch {}.'.format(branch))
+ logger.debug(' Checking out branch {}.'.format(branch))
 
  run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', branch])
  run_cmd(cwd, ['git', 'fetch', '--tags', '--depth', '1', 'origin', branch])
@@ -59,3 +60,32 @@ def get_current_sha(cwd):
 
 def remove_remote(cwd):
  return run_cmd(cwd, ['git', 'remote', 'rm', 'origin'])
+
+
+def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
+ branch=None):
+ _, err = clone(repo, cwd, dirname=dirname, remove_git_dir=remove_git_dir)
+ exists = re.match("fatal: destination path '(.+)' already exists",
+ err.decode('utf-8'))
+ directory = None
+ start_sha = None
+ if exists:
+ directory = exists.group(1)
+ logger.debug('Updating existing dependency %s.', directory)
+ else:
+ matches = re.match("Cloning into '(.+)'", err.decode('utf-8'))
+ directory = matches.group(1)
+ logger.debug('Pulling new dependency %s.', directory)
+ full_path = os.path.join(cwd, directory)
+ start_sha = get_current_sha(full_path)
+ checkout(full_path, repo, branch)
+ end_sha = get_current_sha(full_path)
+ if exists:
+ if start_sha == end_sha:
+ logger.debug(' Already at %s, nothing to do.', start_sha[:7])
+ else:
+ logger.debug(' Updated checkout from %s to %s.',
+ start_sha[:7], end_sha[:7])
+ else:
+ logger.debug(' Checked out at %s.', end_sha[:7])
+ return directory

dbt/clients/registry.py

@@ -0,0 +1,61 @@
+from functools import wraps
+import six
+import requests
+from dbt.exceptions import RegistryException
+from dbt.utils import memoized
+import os
+
+if os.getenv('DBT_PACKAGE_HUB_URL'):
+ DEFAULT_REGISTRY_BASE_URL = os.getenv('DBT_PACKAGE_HUB_URL')
+else:
+ DEFAULT_REGISTRY_BASE_URL = 'https://hub.getdbt.com/'
+
+
+def _get_url(url, registry_base_url=None):
+ if registry_base_url is None:
+ registry_base_url = DEFAULT_REGISTRY_BASE_URL
+
+ return '{}{}'.format(registry_base_url, url)
+
+
+def _wrap_exceptions(fn):
+ @wraps(fn)
+ def wrapper(*args, **kwargs):
+ try:
+ return fn(*args, **kwargs)
+ except requests.exceptions.ConnectionError as e:
+ six.raise_from(
+ RegistryException('Unable to connect to registry hub'), e)
+ return wrapper
+
+
+@_wrap_exceptions
+def _get(path, registry_base_url=None):
+ url = _get_url(path, registry_base_url)
+ resp = requests.get(url)
+ resp.raise_for_status()
+ return resp.json()
+
+
+def index(registry_base_url=None):
+ return _get('api/v1/index.json', registry_base_url)
+
+
+index_cached = memoized(index)
+
+
+def packages(registry_base_url=None):
+ return _get('api/v1/packages.json', registry_base_url)
+
+
+def package(name, registry_base_url=None):
+ return _get('api/v1/{}.json'.format(name), registry_base_url)
+
+
+def package_version(name, version, registry_base_url=None):
+ return _get('api/v1/{}/{}.json'.format(name, version), registry_base_url)
+
+
+def get_available_versions(name):
+ response = package(name)
+ return list(response['versions'])

dbt/clients/system.py

@@ -5,8 +5,12 @@
 import shutil
 import subprocess
 import sys
+import tarfile
+import requests
+import stat
 
 import dbt.compat
+import dbt.exceptions
 
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -92,19 +96,61 @@ def make_file(path, contents='', overwrite=False):
  return False
 
 
+def make_symlink(source, link_path):
+ """
+ Create a symlink at `link_path` referring to `source`.
+ """
+ if not supports_symlinks():
+ dbt.exceptions.system_error('create a symbolic link')
+
+ return os.symlink(source, link_path)
+
+
+def supports_symlinks():
+ return getattr(os, "symlink", None) is not None
+
+
 def write_file(path, contents=''):
  make_directory(os.path.dirname(path))
  dbt.compat.write_file(path, contents)
 
  return True
 
 
+def _windows_rmdir_readonly(func, path, exc):
+ exception_val = exc[1]
+ if exception_val.errno == errno.EACCES:
+ os.chmod(path, stat.S_IWUSR)
+ func(path)
+ else:
+ raise
+
+
 def rmdir(path):
  """
- Make a file at `path` assuming that the directory it resides in already
- exists. The file is saved with contents `contents`
+ Recursively deletes a directory. Includes an error handler to retry with
+ different permissions on Windows. Otherwise, removing directories (eg.
+ cloned via git) can cause rmtree to throw a PermissionError exception
  """
- return shutil.rmtree(path)
+ logger.debug("DEBUG** Window rmdir sys.platform: {}".format(sys.platform))
+ if sys.platform == 'win32':
+ onerror = _windows_rmdir_readonly
+ else:
+ onerror = None
+
+ return shutil.rmtree(path, onerror=onerror)
+
+
+def remove_file(path):
+ return os.remove(path)
+
+
+def path_exists(path):
+ return os.path.lexists(path)
+
+
+def path_is_symlink(path):
+ return os.path.islink(path)
 
 
 def open_dir_cmd():
@@ -133,3 +179,27 @@ def run_cmd(cwd, cmd):
  logger.debug('STDERR: "{}"'.format(err))
 
  return out, err
+
+
+def download(url, path):
+ response = requests.get(url)
+ with open(path, 'wb') as handle:
+ for block in response.iter_content(1024*64):
+ handle.write(block)
+
+
+def rename(from_path, to_path, force=False):
+ if os.path.exists(to_path) and force:
+ rmdir(to_path)
+ os.rename(from_path, to_path)
+
+
+def untar_package(tar_path, dest_dir, rename_to=None):
+ tar_dir_name = None
+ with tarfile.open(tar_path, 'r') as tarball:
+ tarball.extractall(dest_dir)
+ tar_dir_name = os.path.commonprefix(tarball.getnames())
+ if rename_to:
+ downloaded_path = os.path.join(dest_dir, tar_dir_name)
+ desired_path = os.path.join(dest_dir, rename_to)
+ dbt.clients.system.rename(downloaded_path, desired_path, force=True)

dbt/context/common.py

@@ -1,5 +1,6 @@
 import json
 import os
+import pytz
 import voluptuous
 
 from dbt.adapters.factory import get_adapter

dbt/deprecations.py

@@ -11,21 +11,19 @@ def show(self, *args, **kwargs):
  logger.info("* Deprecation Warning: {}\n".format(desc))
  active_deprecations.add(self.name)
 
-# Leaving this as an example. Make sure to add new ones to deprecations_list
-# - Connor
-#
-# class DBTRunTargetDeprecation(DBTDeprecation):
-# name = 'run-target'
-# description = """profiles.yml configuration option 'run-target' is
-# deprecated. Please use 'target' instead. The 'run-target' option will be
-# removed (in favor of 'target') in DBT version 0.7.0"""
+
+class DBTRepositoriesDeprecation(DBTDeprecation):
+ name = "repositories"
+ description = """dbt_project.yml configuration option 'repositories' is
+ deprecated. Please use 'packages' instead. The 'repositories' option will
+ be removed in a later version of DBT."""
 
 
 class SeedDropExistingDeprecation(DBTDeprecation):
  name = 'drop-existing'
- description = """The --drop-existing argument has been deprecated. Please
- use --full-refresh instead. The --drop-existing option will be removed in a
- future version of dbt."""
+ description = """The --drop-existing argument to `dbt seed` has been
+ deprecated. Please use --full-refresh instead. The --drop-existing option
+ will be removed in a future version of dbt."""
 
 
 def warn(name, *args, **kwargs):
@@ -44,6 +42,7 @@ def warn(name, *args, **kwargs):
 active_deprecations = set()
 
 deprecations_list = [
+ DBTRepositoriesDeprecation(),
  SeedDropExistingDeprecation()
 ]