Merge pull request #165 from analyst-collective/feature/specify-dep-b…

…ranch

Feature/specify dep branch

dbt/compilation.py

@@ -82,6 +82,15 @@ def do_config(*args, **kwargs):
  return ""
  return do_config
 
+ def model_can_reference(self, src_model, other_model):
+ """returns True if the src_model can reference the other_model. Models can access
+ other models in their package and dependency models, but a dependency model cannot
+ access models "up" the dependency chain"""
+
+ # hack for now b/c we don't support recursive dependencies
+ return other_model.own_project['name'] == src_model.own_project['name'] \
+ or src_model.own_project['name'] == src_model.project['name']
+
  def __ref(self, linker, ctx, model, all_models):
  schema = ctx['env']['schema']
 
@@ -96,15 +105,24 @@ def do_ref(*args):
  other_model_package, other_model_name = args
  other_model_name = self.create_template.model_name(other_model_name)
  other_model = find_model_by_name(all_models, other_model_name, package_namespace=other_model_package)
+ else:
+ compiler_error(model, "ref() takes at most two arguments ({} given)".format(len(args)))
 
  other_model_fqn = tuple(other_model.fqn[:-1] + [other_model_name])
+ src_fqn = ".".join(source_model)
+ ref_fqn = ".".join(other_model_fqn)
+
+ if not self.model_can_reference(model, other_model):
+ compiler_error(model, "Model '{}' exists but cannot be referenced from dependency model '{}'".format(ref_fqn, src_fqn))
+
  if not other_model.is_enabled:
- src_fqn = ".".join(source_model)
- ref_fqn = ".".join(other_model_fqn)
  raise RuntimeError("Model '{}' depends on model '{}' which is disabled in the project config".format(src_fqn, ref_fqn))
 
  # this creates a trivial cycle -- should this be a compiler error?
- if source_model != other_model_fqn:
+ # we can still interpolate the name w/o making a self-cycle
+ if source_model == other_model_fqn:
+ pass
+ else:
  linker.dependency(source_model, other_model_fqn)
 
  if other_model.is_ephemeral:
@@ -117,7 +135,9 @@ def wrapped_do_ref(*args):
  try:
  return do_ref(*args)
  except RuntimeError as e:
- print("Compiler error in {}".format(model.filepath))
+ root = os.path.relpath(model.root_dir, model.project['project-root'])
+ filepath = os.path.join(root, model.rel_filepath)
+ print("Compiler error in {}".format(filepath))
  print("Enabled models:")
  for m in all_models:
  print(" - {}".format(".".join(m.fqn)))

dbt/task/deps.py

@@ -16,7 +16,16 @@ def __init__(self, args, project):
  self.args = args
  self.project = project
 
- def __pull_repo(self, repo):
+ def __checkout_branch(self, branch, full_path):
+ print(" checking out branch {}".format(branch))
+ proc = subprocess.Popen(
+ ['git', 'checkout', branch],
+ cwd=full_path,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+
+ def __pull_repo(self, repo, branch=None):
  proc = subprocess.Popen(
  ['git', 'clone', repo],
  cwd=self.project['modules-path'],
@@ -43,28 +52,51 @@ def __pull_repo(self, repo):
  stdout=subprocess.PIPE,
  stderr=subprocess.PIPE)
  out, err = proc.communicate()
+ if branch is not None:
+ self.__checkout_branch(branch, full_path)
  else:
  matches = re.match("Cloning into '(.+)'", err.decode('utf-8'))
  folder = matches.group(1)
+ full_path = os.path.join(self.project['modules-path'], folder)
  print("pulled new dependency {}".format(folder))
+ if branch is not None:
+ self.__checkout_branch(branch, full_path)
 
  return folder
 
- def __pull_deps_recursive(self, repos, processed_repos = set()):
- for repo in repos:
+ def __split_at_branch(self, repo):
+ last_slash = repo.rfind('/') + 1
+ end = repo[last_slash:]
+
+ res = re.search('^([^@]*)(\.git)?@?(.*)$', end)
+
+ if res is None:
+ raise RuntimeError("Invalid dep specified: '{}' -- not a repo we can clone".format(repo))
+
+ repo, _, branch = res.groups()
+ repo = repo.replace('.git', '')
+ if branch == '':
+ branch = None
+
+ return repo, branch
+
+ def __pull_deps_recursive(self, repos, processed_repos = set(), i=0):
+ for repo_string in repos:
+ repo, branch = self.__split_at_branch(repo_string)
  repo_folder = folder_from_git_remote(repo)
+
  try:
  if repo_folder in processed_repos:
  print("skipping already processed dependency {}".format(repo_folder))
  else:
- dep_folder = self.__pull_repo(repo)
+ dep_folder = self.__pull_repo(repo, branch)
  dep_project = project.read_project(
  os.path.join(self.project['modules-path'],
  dep_folder,
  'dbt_project.yml')
  )
  processed_repos.add(dep_folder)
- self.__pull_deps_recursive(dep_project['repositories'], processed_repos)
+ self.__pull_deps_recursive(dep_project['repositories'], processed_repos, i+1)
  except IOError as e:
  if e.errno == errno.ENOENT:
  print("'{}' is not a valid dbt project - dbt_project.yml not found".format(repo))