move system client to common

.changes/unreleased/Under the Hood-20231214-164107.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: move system.py to common as dbt-bigquery relies on it to call gcloud
+time: 2023-12-14T16:41:07.539814-08:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "9293"

core/dbt/clients/git.py

@@ -1,7 +1,7 @@
 import re
 import os.path
 
-from dbt.clients.system import run_cmd, rmdir
+from dbt.common.clients.system import run_cmd, rmdir
 from dbt.common.events.functions import fire_event
 from dbt.common.events.types import (
     GitSparseCheckoutSubdirectory,

core/dbt/clients/registry.py

@@ -12,7 +12,8 @@
     RegistryResponseMissingNestedKeys,
     RegistryResponseExtraNestedKeys,
 )
-from dbt.utils import memoized, _connection_exception_retry as connection_exception_retry
+from dbt.utils import memoized
+from dbt.common.utils.connection import connection_exception_retry
 from dbt import deprecations
 from dbt.common import semver
 import os

core/dbt/clients/system.py → core/dbt/common/clients/system.py

@@ -1,3 +1,4 @@
+import dbt.common.exceptions.base
 import errno
 import fnmatch
 import functools
@@ -24,7 +25,7 @@
     SystemReportReturnCode,
 )
 from dbt.common.exceptions import DbtInternalError
-from dbt.utils import _connection_exception_retry as connection_exception_retry
+from dbt.common.utils.connection import connection_exception_retry
 from pathspec import PathSpec  # type: ignore
 
 if sys.platform == "win32":
@@ -197,7 +198,7 @@
 
 
 def write_json(path: str, data: Dict[str, Any]) -> bool:
-    return write_file(path, json.dumps(data, cls=dbt.utils.JSONEncoder))
+    return write_file(path, json.dumps(data, cls=dbt.common.utils.encoding.JSONEncoder))
 
 
 def _windows_rmdir_readonly(func: Callable[[str], Any], path: str, exc: Tuple[Any, OSError, Any]):
@@ -385,7 +386,7 @@
 
 
 def _handle_windows_error(exc: OSError, cwd: str, cmd: List[str]) -> NoReturn:
-    cls: Type[dbt.common.exceptions.DbtBaseException] = dbt.exceptions.CommandError
+    cls: Type[dbt.common.exceptions.DbtBaseException] = dbt.common.exceptions.base.CommandError
     if exc.errno == errno.ENOENT:
         message = (
             "Could not find command, ensure it is in the user's PATH "
@@ -411,7 +412,7 @@
 def _interpret_oserror(exc: OSError, cwd: str, cmd: List[str]) -> NoReturn:
     """Interpret an OSError exception and raise the appropriate dbt exception."""
     if len(cmd) == 0:
-        raise dbt.exceptions.CommandError(cwd, cmd)
+        raise dbt.common.exceptions.base.CommandError(cwd, cmd)
 
     # all of these functions raise unconditionally
     if os.name == "nt":
@@ -428,7 +429,7 @@
 def run_cmd(cwd: str, cmd: List[str], env: Optional[Dict[str, Any]] = None) -> Tuple[bytes, bytes]:
     fire_event(SystemExecutingCmd(cmd=cmd))
     if len(cmd) == 0:
-        raise dbt.exceptions.CommandError(cwd, cmd)
+        raise dbt.common.exceptions.base.CommandError(cwd, cmd)
 
     # the env argument replaces the environment entirely, which has exciting
     # consequences on Windows! Do an update instead.
@@ -502,7 +503,7 @@
     if rename_to:
         downloaded_path = os.path.join(dest_dir, tar_dir_name)
         desired_path = os.path.join(dest_dir, rename_to)
-        dbt.clients.system.rename(downloaded_path, desired_path, force=True)
+        dbt.common.clients.system.rename(downloaded_path, desired_path, force=True)
 
 
 def chmod_and_retry(func, path, exc_info):

core/dbt/common/exceptions/base.py

@@ -25,7 +25,7 @@
 
     def data(self):
         # if overriding, make sure the result is json-serializable.
         return {
             "type": self.__class__.__name__,
             "message": str(self),
         }
@@ -46,16 +46,16 @@
         first = True
 
         if len(stack) > 1:
             lines.append("")
 
             for item in stack:
                 msg = "called by"
 
                 if first:
                     msg = "in"
                     first = False
 
                 lines.append(f"> {msg}")
 
         return lines
 
@@ -63,7 +63,7 @@
         if hasattr(self.msg, "split"):
             split_msg = self.msg.split("\n")
         else:
             split_msg = str(self.msg).split("\n")
 
         lines = ["{}".format(self.type + " Error")] + split_msg
 
@@ -84,7 +84,7 @@
     def add_node(self, node=None):
         if node is not None and node is not self.node:
             if self.node is not None:
                 self.stack.append(self.node)
             self.node = node
 
     @property
@@ -129,7 +129,7 @@
         jsonschema.ValidationError), return the relevant parts as a string
         """
         if not isinstance(exc, ValidationError):
             return str(exc)
         path = "[%s]" % "][".join(map(repr, exc.relative_path))
         return f"at path {path}: {exc.message}"
 
@@ -142,7 +142,7 @@
         if hasattr(self.msg, "split"):
             split_msg = self.msg.split("\n")
         else:
             split_msg = str(self.msg).split("\n")
 
         lines = ["{}{}".format(self.type + " Error", node_string)] + split_msg
 
@@ -151,18 +151,18 @@
         return lines[0] + "\n" + "\n".join(["  " + line for line in lines[1:]])
 
     def data(self):
         result = DbtBaseException.data(self)
         if self.node is None:
             return result
 
         result.update(
             {
                 "raw_code": self.node.raw_code,
                 # the node isn't always compiled, but if it is, include that!
                 "compiled_code": getattr(self.node, "compiled_code", None),
             }
         )
         return result
 
 
 class CompilationError(DbtRuntimeError):
@@ -175,7 +175,7 @@
 
     def _fix_dupe_msg(self, path_1: str, path_2: str, name: str, type_name: str) -> str:
         if path_1 == path_2:
             return (
                 f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n"
             )
         else:
@@ -252,10 +252,24 @@
 
 class UnexpectedNullError(DbtDatabaseError):
     def __init__(self, field_name: str, source):
         self.field_name = field_name
         self.source = source
         msg = (
             f"Expected a non-null value when querying field '{self.field_name}' of table "
             f" {self.source} but received value 'null' instead"
         )
         super().__init__(msg)
+
+
+class CommandError(DbtRuntimeError):
+    def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command") -> None:
+        cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd)
+        super().__init__(msg)
+        self.cwd = cwd
+        self.cmd = cmd_scrubbed
+        self.args = (cwd, cmd_scrubbed, msg)
+
+    def __str__(self):
+        if len(self.cmd) == 0:
+            return f"{self.msg}: No arguments given"
+        return f'{self.msg}: "{self.cmd[0]}"'

core/dbt/common/semver.py

@@ -4,7 +4,6 @@
 
 import dbt.common.exceptions.base
 from dbt.common.exceptions import VersionsNotCompatibleError
-import dbt.utils
 
 from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
 from typing import Optional

core/dbt/common/utils/connection.py

@@ -0,0 +1,33 @@
+import time
+
+from dbt.common.events.types import RecordRetryException, RetryExternalCall
+from dbt.exceptions import ConnectionError
+from tarfile import ReadError
+
+import requests
+
+
+def connection_exception_retry(fn, max_attempts: int, attempt: int = 0):
+    """Attempts to run a function that makes an external call, if the call fails
+    on a Requests exception or decompression issue (ReadError), it will be tried
+    up to 5 more times.  All exceptions that Requests explicitly raises inherit from
+    requests.exceptions.RequestException.  See https://github.com/dbt-labs/dbt-core/issues/4579
+    for context on this decompression issues specifically.
+    """
+    try:
+        return fn()
+    except (
+        requests.exceptions.RequestException,
+        ReadError,
+        EOFError,
+    ) as exc:
+        if attempt <= max_attempts - 1:
+            # This import needs to be inline to avoid circular dependency
+            from dbt.common.events.functions import fire_event
+
+            fire_event(RecordRetryException(exc=str(exc)))
+            fire_event(RetryExternalCall(attempt=attempt, max=max_attempts))
+            time.sleep(1)
+            return connection_exception_retry(fn, max_attempts, attempt + 1)
+        else:
+            raise ConnectionError("External connection exception occurred: " + str(exc))

core/dbt/compilation.py

@@ -11,7 +11,7 @@
 from dbt.flags import get_flags
 from dbt.adapters.factory import get_adapter
 from dbt.clients import jinja
-from dbt.clients.system import make_directory
+from dbt.common.clients.system import make_directory
 from dbt.context.providers import generate_runtime_model_context
 from dbt.contracts.graph.manifest import Manifest, UniqueID
 from dbt.contracts.graph.nodes import (

core/dbt/config/profile.py

@@ -5,7 +5,7 @@
 from dbt.common.dataclass_schema import ValidationError
 
 from dbt.flags import get_flags
-from dbt.clients.system import load_file_contents
+from dbt.common.clients.system import load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.adapters.contracts.connection import Credentials, HasCredentials
 from dbt.contracts.project import ProfileConfig, UserConfig

core/dbt/config/project.py

@@ -21,7 +21,7 @@
     PACKAGES_FILE_NAME,
     PACKAGE_LOCK_HASH_KEY,
 )
-from dbt.clients.system import path_exists, load_file_contents
+from dbt.common.clients.system import path_exists, load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.adapters.contracts.connection import QueryComment
 from dbt.exceptions import (

core/dbt/config/selectors.py

@@ -6,7 +6,7 @@
 
 from .renderer import BaseRenderer
 
-from dbt.clients.system import (
+from dbt.common.clients.system import (
     load_file_contents,
     path_exists,
     resolve_path_from_base,

core/dbt/contracts/graph/nodes.py

@@ -14,7 +14,7 @@
 )
 from dbt.common.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
 
-from dbt.clients.system import write_file
+from dbt.common.clients.system import write_file
 from dbt.contracts.files import FileHash
 from dbt.contracts.graph.saved_queries import Export, QueryParams
 from dbt.contracts.graph.semantic_models import (
@@ -1742,16 +1742,16 @@
         return self.depends_on.nodes
 
     def same_metrics(self, old: "SavedQuery") -> bool:
         return self.query_params.metrics == old.query_params.metrics
 
     def same_group_by(self, old: "SavedQuery") -> bool:
         return self.query_params.group_by == old.query_params.group_by
 
     def same_description(self, old: "SavedQuery") -> bool:
         return self.description == old.description
 
     def same_where(self, old: "SavedQuery") -> bool:
         return self.query_params.where == old.query_params.where
 
     def same_label(self, old: "SavedQuery") -> bool:
         return self.label == old.label
@@ -1763,15 +1763,15 @@
         return self.group == old.group
 
     def same_exports(self, old: "SavedQuery") -> bool:
         if len(self.exports) != len(old.exports):
             return False
 
         # exports should be in the same order, so we zip them for easy iteration
         for (old_export, new_export) in zip(old.exports, self.exports):
             if not new_export.same_contents(old_export):
                 return False
 
         return True
 
     def same_contents(self, old: Optional["SavedQuery"]) -> bool:
         # existing when it didn't before is a change!

core/dbt/contracts/graph/semantic_manifest.py

@@ -13,7 +13,7 @@
     SemanticManifestValidator,
 )
 
-from dbt.clients.system import write_file
+from dbt.common.clients.system import write_file
 from dbt.common.events.base_types import EventLevel
 from dbt.common.events.functions import fire_event
 from dbt.common.events.types import SemanticValidationFailure

core/dbt/contracts/results.py

@@ -37,7 +37,7 @@
     Tuple,
 )
 
-from dbt.clients.system import write_json
+from dbt.common.clients.system import write_json
 
 
 @dataclass

core/dbt/contracts/util.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import List, Tuple, ClassVar, Type, TypeVar, Dict, Any, Optional
 
-from dbt.clients.system import write_json, read_json
+from dbt.common.clients.system import write_json, read_json
 from dbt.exceptions import (
     DbtInternalError,
     DbtRuntimeError,
@@ -208,7 +208,7 @@
 def get_artifact_schema_version(dct: dict) -> int:
     schema_version = dct.get("metadata", {}).get("dbt_schema_version", None)
     if not schema_version:
         raise ValueError("Artifact is missing schema version")
 
     # schema_version is in this format: https://schemas.getdbt.com/dbt/manifest/v10.json
     # What the code below is doing:

core/dbt/deps/README.md

@@ -16,7 +16,7 @@ Defines the base classes of `PinnedPackage` and `UnpinnedPackage`.
 
 `downloads_directory` sets the directory packages will be downloaded to.
 
-`_install` has retry logic if the download or untarring process hit exceptions (see `dbt.utils._connection_exception_retry`).
+`_install` has retry logic if the download or untarring process hit exceptions (see `dbt.common.utils.connection_exception_retry`).
 
 ## `git.py`
 

core/dbt/deps/base.py

@@ -6,11 +6,11 @@
 from pathlib import Path
 from typing import List, Optional, Generic, TypeVar, Dict
 
-from dbt.clients import system
+from dbt.common.clients import system
 from dbt.contracts.project import ProjectPackageMetadata
 from dbt.common.events.functions import fire_event
 from dbt.common.events.types import DepsSetDownloadDirectory
-from dbt.utils import _connection_exception_retry as connection_exception_retry
+from dbt.common.utils.connection import connection_exception_retry
 
 DOWNLOADS_PATH = None
 
@@ -86,7 +86,7 @@
 
     @abc.abstractmethod
     def to_dict(self) -> Dict[str, str]:
         raise NotImplementedError
 
     def fetch_metadata(self, project, renderer):
         if not self._cached_metadata:
@@ -126,7 +126,7 @@
         download appears successful but the file did not make it through as expected
         (generally due to a github incident).  Either way we want to retry downloading
         and untarring to see if we can get a success.  Call this within
-        `_connection_exception_retry`
+        `connection_exception_retry`
         """
 
         system.download(download_url, tar_path)

core/dbt/deps/git.py

@@ -1,7 +1,8 @@
 import os
 from typing import List, Optional, Dict
 
-from dbt.clients import git, system
+from dbt.clients import git
+from dbt.common.clients import system
 from dbt.config.project import PartialProject, Project
 from dbt.config.renderer import PackageRenderer
 from dbt.contracts.project import (
@@ -58,7 +59,7 @@
     def to_dict(self) -> Dict[str, str]:
         git_scrubbed = scrub_secrets(self.git_unrendered, env_secrets())
         if self.git_unrendered != git_scrubbed:
             warn_or_error(DepsScrubbedPackageName(package_name=git_scrubbed))
         ret = {
             "git": git_scrubbed,
             "revision": self.revision,

core/dbt/deps/local.py

@@ -1,7 +1,7 @@
 import shutil
 from typing import Dict
 
-from dbt.clients import system
+from dbt.common.clients import system
 from dbt.deps.base import PinnedPackage, UnpinnedPackage
 from dbt.contracts.project import (
     ProjectPackageMetadata,

core/dbt/deps/tarball.py

@@ -3,14 +3,14 @@
 from pathlib import Path
 from typing import Dict
 
-from dbt.clients import system
+from dbt.common.clients import system
 from dbt.config.project import PartialProject
 from dbt.contracts.project import TarballPackage
 from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
 from dbt.exceptions import DependencyError, scrub_secrets, env_secrets
 from dbt.common.events.functions import warn_or_error
 from dbt.events.types import DepsScrubbedPackageName
-from dbt.utils import _connection_exception_retry as connection_exception_retry
+from dbt.common.utils.connection import connection_exception_retry
 
 
 class TarballPackageMixin:
@@ -42,7 +42,7 @@
     def to_dict(self) -> Dict[str, str]:
         tarball_scrubbed = scrub_secrets(self.tarball_unrendered, env_secrets())
         if self.tarball_unrendered != tarball_scrubbed:
             warn_or_error(DepsScrubbedPackageName(package_name=tarball_scrubbed))
         return {
             "tarball": tarball_scrubbed,
             "name": self.package,
@@ -82,10 +82,10 @@
         connection_exception_retry(download_untar_fn, 5)
         dest_path = self.get_installation_path(project, renderer)
         if os.path.exists(dest_path):
             if system.path_is_symlink(dest_path):
                 system.remove_file(dest_path)
             else:
                 system.rmdir(dest_path)
         system.move(self.untarred_path, dest_path)
 
 
@@ -110,7 +110,7 @@
         )
 
     def incorporate(self, other: "TarballUnpinnedPackage") -> "TarballUnpinnedPackage":
         return TarballUnpinnedPackage(
             tarball=self.tarball, tarball_unrendered=self.tarball_unrendered, package=self.package
         )
 

core/dbt/exceptions.py

@@ -12,6 +12,7 @@
     env_secrets,
     scrub_secrets,
     DbtValidationError,
+    CommandError,
 )
 from dbt.node_types import NodeType, AccessType
 
@@ -135,20 +136,6 @@
     pass
 
 
-class CommandError(DbtRuntimeError):
-    def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command") -> None:
-        cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd)
-        super().__init__(msg)
-        self.cwd = cwd
-        self.cmd = cmd_scrubbed
-        self.args = (cwd, cmd_scrubbed, msg)
-
-    def __str__(self):
-        if len(self.cmd) == 0:
-            return f"{self.msg}: No arguments given"
-        return f'{self.msg}: "{self.cmd[0]}"'
-
-
 class ExecutableError(CommandError):
     def __init__(self, cwd: str, cmd: List[str], msg: str) -> None:
         super().__init__(cwd, cmd, msg)
@@ -631,7 +618,7 @@
     def get_message(self) -> str:
         target_package_string = ""
         if self.target_doc_package is not None:
             target_package_string = f"in package '{self.target_doc_package}' "
         msg = f"Documentation for '{self.node.unique_id}' depends on doc '{self.target_doc_name}' {target_package_string} which was not found"
         return msg
 

core/dbt/logger.py

@@ -327,9 +327,9 @@ def process(self, record):
 
 
 def make_log_dir_if_missing(log_dir):
-    import dbt.clients.system
+    import dbt.common.clients.system
 
-    dbt.clients.system.make_directory(log_dir)
+    dbt.common.clients.system.make_directory(log_dir)
 
 
 class DebugWarnings(logbook.compat.redirected_warnings):

core/dbt/parser/manifest.py

@@ -65,7 +65,7 @@
 from dbt.node_types import NodeType, AccessType
 from dbt.clients.jinja import get_rendered, MacroStack
 from dbt.clients.jinja_static import statically_extract_macro_calls
-from dbt.clients.system import (
+from dbt.common.clients.system import (
     make_directory,
     path_exists,
     read_json,

core/dbt/parser/read_files.py

@@ -2,7 +2,7 @@
 import pathspec  # type: ignore
 import pathlib
 from dataclasses import dataclass, field
-from dbt.clients.system import load_file_contents
+from dbt.common.clients.system import load_file_contents
 from dbt.contracts.files import (
     FilePath,
     ParseFileType,

core/dbt/parser/search.py

@@ -4,7 +4,7 @@
 from pathspec import PathSpec  # type: ignore
 
 from dbt.common.clients.jinja import extract_toplevel_blocks, BlockTag
-from dbt.clients.system import find_matching
+from dbt.common.clients.system import find_matching
 from dbt.config import Project
 from dbt.contracts.files import FilePath, AnySourceFile
 from dbt.exceptions import ParsingError, DbtInternalError

core/dbt/task/debug.py

@@ -15,7 +15,7 @@
     DebugCmdOut,
     DebugCmdResult,
 )
-import dbt.clients.system
+import dbt.common.clients.system
 import dbt.exceptions
 import dbt.common.exceptions
 from dbt.adapters.factory import get_adapter, register_adapter
@@ -110,7 +110,8 @@ def run(self) -> bool:
         if self.args.config_dir:
             fire_event(
                 OpenCommand(
-                    open_cmd=dbt.clients.system.open_dir_cmd(), profiles_dir=str(self.profiles_dir)
+                    open_cmd=dbt.common.clients.system.open_dir_cmd(),
+                    profiles_dir=str(self.profiles_dir),
                 )
             )
             return DebugRunStatus.SUCCESS.value
@@ -202,7 +203,9 @@ def _load_profile(self) -> SubtaskStatus:
                 ),
             )
 
-        raw_profile_data = load_yaml_text(dbt.clients.system.load_file_contents(self.profile_path))
+        raw_profile_data = load_yaml_text(
+            dbt.common.clients.system.load_file_contents(self.profile_path)
+        )
         if isinstance(raw_profile_data, dict):
             self.raw_profile_data = raw_profile_data
 
@@ -395,7 +398,7 @@ def _target_found(self) -> str:
 
     def test_git(self) -> SubtaskStatus:
         try:
-            dbt.clients.system.run_cmd(os.getcwd(), ["git", "--help"])
+            dbt.common.clients.system.run_cmd(os.getcwd(), ["git", "--help"])
         except dbt.exceptions.ExecutableError as exc:
             return SubtaskStatus(
                 log_msg=red("ERROR"),