Add a "dbt deps" api call

Refactor deps
 - split it up
 - move most of it into dbt.deps
Add the idea of a remote method that does not require a manifest
 - it does still require config/args
 - set up the various reloading logic to not care about reloading these methods
Add dbt deps api call
 - also available as if it were from the cli
Add tests

core/dbt/contracts/rpc.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from numbers import Real
 from typing import Optional, Union, List, Any, Dict
 
@@ -55,30 +55,43 @@ class RPCCliParameters(RPCParameters):
     cli: str
 
 
+@dataclass
+class RPCNoParameters(RPCParameters):
+    pass
+
+
 # Outputs
 
+@dataclass
+class RemoteResult(JsonSchemaMixin):
+    logs: List[LogMessage]
+
 
 @dataclass
-class RemoteCatalogResults(CatalogResults):
-    logs: List[LogMessage] = field(default_factory=list)
+class RemoteEmptyResult(RemoteResult):
+    pass
 
 
 @dataclass
-class RemoteCompileResult(JsonSchemaMixin):
+class RemoteCatalogResults(CatalogResults, RemoteResult):
+    pass
+
+
+@dataclass
+class RemoteCompileResult(RemoteResult):
     raw_sql: str
     compiled_sql: str
     node: CompileResultNode
     timing: List[TimingInfo]
-    logs: List[LogMessage]
 
     @property
     def error(self):
         return None
 
 
 @dataclass
-class RemoteExecutionResult(ExecutionResult):
-    logs: List[LogMessage]
+class RemoteExecutionResult(ExecutionResult, RemoteResult):
+    pass
 
 
 @dataclass
@@ -90,3 +103,11 @@ class ResultTable(JsonSchemaMixin):
 @dataclass
 class RemoteRunResult(RemoteCompileResult):
     table: ResultTable
+
+
+RPCResult = Union[
+    RemoteCompileResult,
+    RemoteExecutionResult,
+    RemoteCatalogResults,
+    RemoteEmptyResult,
+]

core/dbt/deps/base.py

@@ -0,0 +1,112 @@
+import abc
+import os
+import tempfile
+from contextlib import contextmanager
+from typing import List, Optional, Generic, TypeVar
+
+from dbt.clients import system
+from dbt.contracts.project import ProjectPackageMetadata
+from dbt.logger import GLOBAL_LOGGER as logger
+
+DOWNLOADS_PATH = None
+
+
+def get_downloads_path():
+    return DOWNLOADS_PATH
+
+
+@contextmanager
+def downloads_directory():
+    global DOWNLOADS_PATH
+    remove_downloads = False
+    # the user might have set an environment variable. Set it to that, and do
+    # not remove it when finished.
+    if DOWNLOADS_PATH is None:
+        DOWNLOADS_PATH = os.getenv('DBT_DOWNLOADS_DIR')
+        remove_downloads = False
+    # if we are making a per-run temp directory, remove it at the end of
+    # successful runs
+    if DOWNLOADS_PATH is None:
+        DOWNLOADS_PATH = tempfile.mkdtemp(prefix='dbt-downloads-')
+        remove_downloads = True
+
+    system.make_directory(DOWNLOADS_PATH)
+    logger.debug("Set downloads directory='{}'".format(DOWNLOADS_PATH))
+
+    yield DOWNLOADS_PATH
+
+    if remove_downloads:
+        system.rmtree(DOWNLOADS_PATH)
+        DOWNLOADS_PATH = None
+
+
+class BasePackage(metaclass=abc.ABCMeta):
+    @abc.abstractproperty
+    def name(self) -> str:
+        raise NotImplementedError
+
+    def all_names(self) -> List[str]:
+        return [self.name]
+
+    @abc.abstractmethod
+    def source_type(self) -> str:
+        raise NotImplementedError
+
+
+class PinnedPackage(BasePackage):
+    def __init__(self) -> None:
+        self._cached_metadata: Optional[ProjectPackageMetadata] = None
+
+    def __str__(self) -> str:
+        version = self.get_version()
+        if not version:
+            return self.name
+
+        return '{}@{}'.format(self.name, version)
+
+    @abc.abstractmethod
+    def get_version(self) -> Optional[str]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def _fetch_metadata(self, project):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def install(self, project):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def nice_version_name(self):
+        raise NotImplementedError
+
+    def fetch_metadata(self, project):
+        if not self._cached_metadata:
+            self._cached_metadata = self._fetch_metadata(project)
+        return self._cached_metadata
+
+    def get_project_name(self, project):
+        metadata = self.fetch_metadata(project)
+        return metadata.name
+
+    def get_installation_path(self, project):
+        dest_dirname = self.get_project_name(project)
+        return os.path.join(project.modules_path, dest_dirname)
+
+
+SomePinned = TypeVar('SomePinned', bound=PinnedPackage)
+SomeUnpinned = TypeVar('SomeUnpinned', bound='UnpinnedPackage')
+
+
+class UnpinnedPackage(Generic[SomePinned], BasePackage):
+    @abc.abstractclassmethod
+    def from_contract(cls, contract):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def incorporate(self: SomeUnpinned, other: SomeUnpinned) -> SomeUnpinned:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def resolved(self) -> SomePinned:
+        raise NotImplementedError

core/dbt/deps/git.py

@@ -0,0 +1,144 @@
+import os
+import hashlib
+from typing import List
+
+from dbt.clients import git, system
+from dbt.contracts.project import (
+    ProjectPackageMetadata,
+    GitPackage,
+)
+from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
+from dbt.exceptions import (
+    ExecutableError, warn_or_error, raise_dependency_error
+)
+from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.ui import printer
+
+PIN_PACKAGE_URL = 'https://docs.getdbt.com/docs/package-management#section-specifying-package-versions' # noqa
+
+
+def md5sum(s: str):
+    return hashlib.md5(s.encode('latin-1')).hexdigest()
+
+
+class GitPackageMixin:
+    def __init__(self, git: str) -> None:
+        super().__init__()
+        self.git = git
+
+    @property
+    def name(self):
+        return self.git
+
+    def source_type(self) -> str:
+        return 'git'
+
+
+class GitPinnedPackage(GitPackageMixin, PinnedPackage):
+    def __init__(
+        self, git: str, revision: str, warn_unpinned: bool = True
+    ) -> None:
+        super().__init__(git)
+        self.revision = revision
+        self.warn_unpinned = warn_unpinned
+        self._checkout_name = md5sum(self.git)
+
+    def get_version(self):
+        return self.revision
+
+    def nice_version_name(self):
+        return 'revision {}'.format(self.revision)
+
+    def _checkout(self):
+        """Performs a shallow clone of the repository into the downloads
+        directory. This function can be called repeatedly. If the project has
+        already been checked out at this version, it will be a no-op. Returns
+        the path to the checked out directory."""
+        try:
+            dir_ = git.clone_and_checkout(
+                self.git, get_downloads_path(), branch=self.revision,
+                dirname=self._checkout_name
+            )
+        except ExecutableError as exc:
+            if exc.cmd and exc.cmd[0] == 'git':
+                logger.error(
+                    'Make sure git is installed on your machine. More '
+                    'information: '
+                    'https://docs.getdbt.com/docs/package-management'
+                )
+            raise
+        return os.path.join(get_downloads_path(), dir_)
+
+    def _fetch_metadata(self, project) -> ProjectPackageMetadata:
+        path = self._checkout()
+        if self.revision == 'master' and self.warn_unpinned:
+            warn_or_error(
+                'The git package "{}" is not pinned.\n\tThis can introduce '
+                'breaking changes into your project without warning!\n\nSee {}'
+                .format(self.git, PIN_PACKAGE_URL),
+                log_fmt=printer.yellow('WARNING: {}')
+            )
+        loaded = project.from_project_root(path, {})
+        return ProjectPackageMetadata.from_project(loaded)
+
+    def install(self, project):
+        dest_path = self.get_installation_path(project)
+        if os.path.exists(dest_path):
+            if system.path_is_symlink(dest_path):
+                system.remove_file(dest_path)
+            else:
+                system.rmdir(dest_path)
+
+        system.move(self._checkout(), dest_path)
+
+
+class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
+    def __init__(
+        self, git: str, revisions: List[str], warn_unpinned: bool = True
+    ) -> None:
+        super().__init__(git)
+        self.revisions = revisions
+        self.warn_unpinned = warn_unpinned
+
+    @classmethod
+    def from_contract(
+        cls, contract: GitPackage
+    ) -> 'GitUnpinnedPackage':
+        revisions = [contract.revision] if contract.revision else []
+
+        # we want to map None -> True
+        warn_unpinned = contract.warn_unpinned is not False
+        return cls(git=contract.git, revisions=revisions,
+                   warn_unpinned=warn_unpinned)
+
+    def all_names(self) -> List[str]:
+        if self.git.endswith('.git'):
+            other = self.git[:-4]
+        else:
+            other = self.git + '.git'
+        return [self.git, other]
+
+    def incorporate(
+        self, other: 'GitUnpinnedPackage'
+    ) -> 'GitUnpinnedPackage':
+        warn_unpinned = self.warn_unpinned and other.warn_unpinned
+
+        return GitUnpinnedPackage(
+            git=self.git,
+            revisions=self.revisions + other.revisions,
+            warn_unpinned=warn_unpinned,
+        )
+
+    def resolved(self) -> GitPinnedPackage:
+        requested = set(self.revisions)
+        if len(requested) == 0:
+            requested = {'master'}
+        elif len(requested) > 1:
+            raise_dependency_error(
+                'git dependencies should contain exactly one version. '
+                '{} contains: {}'.format(self.git, requested))
+
+        return GitPinnedPackage(
+            git=self.git, revision=requested.pop(),
+            warn_unpinned=self.warn_unpinned
+        )