move system client to common (#9294)

* moving types_pb2.py to common/events

* move system.py to common

* add changie update README

* remove dbt.utils from semver.py

* remove aliasing connection_exception_retry

.changes/unreleased/Under the Hood-20231214-164107.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: move system.py to common as dbt-bigquery relies on it to call gcloud
+time: 2023-12-14T16:41:07.539814-08:00
+custom:
+  Author: colin-rogers-dbt
+  Issue: "9293"

core/dbt/clients/git.py

@@ -1,7 +1,7 @@
 import re
 import os.path
 
-from dbt.clients.system import run_cmd, rmdir
+from dbt.common.clients.system import run_cmd, rmdir
 from dbt.common.events.functions import fire_event
 from dbt.common.events.types import (
     GitSparseCheckoutSubdirectory,

core/dbt/clients/registry.py

@@ -12,7 +12,8 @@
     RegistryResponseMissingNestedKeys,
     RegistryResponseExtraNestedKeys,
 )
-from dbt.utils import memoized, _connection_exception_retry as connection_exception_retry
+from dbt.utils import memoized
+from dbt.common.utils.connection import connection_exception_retry
 from dbt import deprecations
 from dbt.common import semver
 import os

core/dbt/clients/system.py → core/dbt/common/clients/system.py

@@ -1,3 +1,4 @@
+import dbt.common.exceptions.base
 import errno
 import fnmatch
 import functools
@@ -24,7 +25,7 @@
     SystemReportReturnCode,
 )
 from dbt.common.exceptions import DbtInternalError
-from dbt.utils import _connection_exception_retry as connection_exception_retry
+from dbt.common.utils.connection import connection_exception_retry
 from pathspec import PathSpec  # type: ignore
 
 if sys.platform == "win32":
@@ -197,7 +198,7 @@ def read_json(path: str) -> Dict[str, Any]:
 
 
 def write_json(path: str, data: Dict[str, Any]) -> bool:
-    return write_file(path, json.dumps(data, cls=dbt.utils.JSONEncoder))
+    return write_file(path, json.dumps(data, cls=dbt.common.utils.encoding.JSONEncoder))
 
 
 def _windows_rmdir_readonly(func: Callable[[str], Any], path: str, exc: Tuple[Any, OSError, Any]):
@@ -385,7 +386,7 @@ def _handle_posix_error(exc: OSError, cwd: str, cmd: List[str]) -> NoReturn:
 
 
 def _handle_windows_error(exc: OSError, cwd: str, cmd: List[str]) -> NoReturn:
-    cls: Type[dbt.common.exceptions.DbtBaseException] = dbt.exceptions.CommandError
+    cls: Type[dbt.common.exceptions.DbtBaseException] = dbt.common.exceptions.base.CommandError
     if exc.errno == errno.ENOENT:
         message = (
             "Could not find command, ensure it is in the user's PATH "
@@ -411,7 +412,7 @@ def _handle_windows_error(exc: OSError, cwd: str, cmd: List[str]) -> NoReturn:
 def _interpret_oserror(exc: OSError, cwd: str, cmd: List[str]) -> NoReturn:
     """Interpret an OSError exception and raise the appropriate dbt exception."""
     if len(cmd) == 0:
-        raise dbt.exceptions.CommandError(cwd, cmd)
+        raise dbt.common.exceptions.base.CommandError(cwd, cmd)
 
     # all of these functions raise unconditionally
     if os.name == "nt":
@@ -428,7 +429,7 @@ def _interpret_oserror(exc: OSError, cwd: str, cmd: List[str]) -> NoReturn:
 def run_cmd(cwd: str, cmd: List[str], env: Optional[Dict[str, Any]] = None) -> Tuple[bytes, bytes]:
     fire_event(SystemExecutingCmd(cmd=cmd))
     if len(cmd) == 0:
-        raise dbt.exceptions.CommandError(cwd, cmd)
+        raise dbt.common.exceptions.base.CommandError(cwd, cmd)
 
     # the env argument replaces the environment entirely, which has exciting
     # consequences on Windows! Do an update instead.
@@ -502,7 +503,7 @@ def untar_package(tar_path: str, dest_dir: str, rename_to: Optional[str] = None)
     if rename_to:
         downloaded_path = os.path.join(dest_dir, tar_dir_name)
         desired_path = os.path.join(dest_dir, rename_to)
-        dbt.clients.system.rename(downloaded_path, desired_path, force=True)
+        dbt.common.clients.system.rename(downloaded_path, desired_path, force=True)
 
 
 def chmod_and_retry(func, path, exc_info):

core/dbt/common/exceptions/base.py

@@ -259,3 +259,17 @@ def __init__(self, field_name: str, source):
             f" {self.source} but received value 'null' instead"
         )
         super().__init__(msg)
+
+
+class CommandError(DbtRuntimeError):
+    def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command") -> None:
+        cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd)
+        super().__init__(msg)
+        self.cwd = cwd
+        self.cmd = cmd_scrubbed
+        self.args = (cwd, cmd_scrubbed, msg)
+
+    def __str__(self):
+        if len(self.cmd) == 0:
+            return f"{self.msg}: No arguments given"
+        return f'{self.msg}: "{self.cmd[0]}"'

core/dbt/common/semver.py

@@ -4,7 +4,6 @@
 
 import dbt.common.exceptions.base
 from dbt.common.exceptions import VersionsNotCompatibleError
-import dbt.utils
 
 from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
 from typing import Optional

core/dbt/common/utils/connection.py

@@ -0,0 +1,33 @@
+import time
+
+from dbt.common.events.types import RecordRetryException, RetryExternalCall
+from dbt.exceptions import ConnectionError
+from tarfile import ReadError
+
+import requests
+
+
+def connection_exception_retry(fn, max_attempts: int, attempt: int = 0):
+    """Attempts to run a function that makes an external call, if the call fails
+    on a Requests exception or decompression issue (ReadError), it will be tried
+    up to 5 more times.  All exceptions that Requests explicitly raises inherit from
+    requests.exceptions.RequestException.  See https://github.com/dbt-labs/dbt-core/issues/4579
+    for context on this decompression issues specifically.
+    """
+    try:
+        return fn()
+    except (
+        requests.exceptions.RequestException,
+        ReadError,
+        EOFError,
+    ) as exc:
+        if attempt <= max_attempts - 1:
+            # This import needs to be inline to avoid circular dependency
+            from dbt.common.events.functions import fire_event
+
+            fire_event(RecordRetryException(exc=str(exc)))
+            fire_event(RetryExternalCall(attempt=attempt, max=max_attempts))
+            time.sleep(1)
+            return connection_exception_retry(fn, max_attempts, attempt + 1)
+        else:
+            raise ConnectionError("External connection exception occurred: " + str(exc))

core/dbt/compilation.py

@@ -11,7 +11,7 @@
 from dbt.flags import get_flags
 from dbt.adapters.factory import get_adapter
 from dbt.clients import jinja
-from dbt.clients.system import make_directory
+from dbt.common.clients.system import make_directory
 from dbt.context.providers import generate_runtime_model_context
 from dbt.contracts.graph.manifest import Manifest, UniqueID
 from dbt.contracts.graph.nodes import (

core/dbt/config/profile.py

@@ -5,7 +5,7 @@
 from dbt.common.dataclass_schema import ValidationError
 
 from dbt.flags import get_flags
-from dbt.clients.system import load_file_contents
+from dbt.common.clients.system import load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.adapters.contracts.connection import Credentials, HasCredentials
 from dbt.contracts.project import ProfileConfig, UserConfig

core/dbt/config/project.py

@@ -21,7 +21,7 @@
     PACKAGES_FILE_NAME,
     PACKAGE_LOCK_HASH_KEY,
 )
-from dbt.clients.system import path_exists, load_file_contents
+from dbt.common.clients.system import path_exists, load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.adapters.contracts.connection import QueryComment
 from dbt.exceptions import (

core/dbt/config/selectors.py

@@ -6,7 +6,7 @@
 
 from .renderer import BaseRenderer
 
-from dbt.clients.system import (
+from dbt.common.clients.system import (
     load_file_contents,
     path_exists,
     resolve_path_from_base,

core/dbt/contracts/graph/nodes.py

@@ -14,7 +14,7 @@
 )
 from dbt.common.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
 
-from dbt.clients.system import write_file
+from dbt.common.clients.system import write_file
 from dbt.contracts.files import FileHash
 from dbt.contracts.graph.saved_queries import Export, QueryParams
 from dbt.contracts.graph.semantic_models import (

core/dbt/contracts/graph/semantic_manifest.py

@@ -13,7 +13,7 @@
     SemanticManifestValidator,
 )
 
-from dbt.clients.system import write_file
+from dbt.common.clients.system import write_file
 from dbt.common.events.base_types import EventLevel
 from dbt.common.events.functions import fire_event
 from dbt.common.events.types import SemanticValidationFailure

core/dbt/contracts/results.py

@@ -37,7 +37,7 @@
     Tuple,
 )
 
-from dbt.clients.system import write_json
+from dbt.common.clients.system import write_json
 
 
 @dataclass

core/dbt/contracts/util.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import List, Tuple, ClassVar, Type, TypeVar, Dict, Any, Optional
 
-from dbt.clients.system import write_json, read_json
+from dbt.common.clients.system import write_json, read_json
 from dbt.exceptions import (
     DbtInternalError,
     DbtRuntimeError,

core/dbt/deps/README.md

@@ -16,7 +16,7 @@ Defines the base classes of `PinnedPackage` and `UnpinnedPackage`.
 
 `downloads_directory` sets the directory packages will be downloaded to.
 
-`_install` has retry logic if the download or untarring process hit exceptions (see `dbt.utils._connection_exception_retry`).
+`_install` has retry logic if the download or untarring process hit exceptions (see `dbt.common.utils.connection_exception_retry`).
 
 ## `git.py`
 

core/dbt/deps/base.py

@@ -6,11 +6,11 @@
 from pathlib import Path
 from typing import List, Optional, Generic, TypeVar, Dict
 
-from dbt.clients import system
+from dbt.common.clients import system
 from dbt.contracts.project import ProjectPackageMetadata
 from dbt.common.events.functions import fire_event
 from dbt.common.events.types import DepsSetDownloadDirectory
-from dbt.utils import _connection_exception_retry as connection_exception_retry
+from dbt.common.utils.connection import connection_exception_retry
 
 DOWNLOADS_PATH = None
 
@@ -126,7 +126,7 @@ def download_and_untar(self, download_url, tar_path, deps_path, package_name):
         download appears successful but the file did not make it through as expected
         (generally due to a github incident).  Either way we want to retry downloading
         and untarring to see if we can get a success.  Call this within
-        `_connection_exception_retry`
+        `connection_exception_retry`
         """
 
         system.download(download_url, tar_path)

core/dbt/deps/git.py

@@ -1,7 +1,8 @@
 import os
 from typing import List, Optional, Dict
 
-from dbt.clients import git, system
+from dbt.clients import git
+from dbt.common.clients import system
 from dbt.config.project import PartialProject, Project
 from dbt.config.renderer import PackageRenderer
 from dbt.contracts.project import (

core/dbt/deps/local.py

@@ -1,7 +1,7 @@
 import shutil
 from typing import Dict
 
-from dbt.clients import system
+from dbt.common.clients import system
 from dbt.deps.base import PinnedPackage, UnpinnedPackage
 from dbt.contracts.project import (
     ProjectPackageMetadata,

core/dbt/deps/tarball.py

@@ -3,14 +3,14 @@
 from pathlib import Path
 from typing import Dict
 
-from dbt.clients import system
+from dbt.common.clients import system
 from dbt.config.project import PartialProject
 from dbt.contracts.project import TarballPackage
 from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
 from dbt.exceptions import DependencyError, scrub_secrets, env_secrets
 from dbt.common.events.functions import warn_or_error
 from dbt.events.types import DepsScrubbedPackageName
-from dbt.utils import _connection_exception_retry as connection_exception_retry
+from dbt.common.utils.connection import connection_exception_retry
 
 
 class TarballPackageMixin:

core/dbt/exceptions.py

@@ -12,6 +12,7 @@
     env_secrets,
     scrub_secrets,
     DbtValidationError,
+    CommandError,
 )
 from dbt.node_types import NodeType, AccessType
 
@@ -135,20 +136,6 @@ class DbtProfileError(DbtConfigError):
     pass
 
 
-class CommandError(DbtRuntimeError):
-    def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command") -> None:
-        cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd)
-        super().__init__(msg)
-        self.cwd = cwd
-        self.cmd = cmd_scrubbed
-        self.args = (cwd, cmd_scrubbed, msg)
-
-    def __str__(self):
-        if len(self.cmd) == 0:
-            return f"{self.msg}: No arguments given"
-        return f'{self.msg}: "{self.cmd[0]}"'
-
-
 class ExecutableError(CommandError):
     def __init__(self, cwd: str, cmd: List[str], msg: str) -> None:
         super().__init__(cwd, cmd, msg)

core/dbt/logger.py

@@ -327,9 +327,9 @@ def process(self, record):
 
 
 def make_log_dir_if_missing(log_dir):
-    import dbt.clients.system
+    import dbt.common.clients.system
 
-    dbt.clients.system.make_directory(log_dir)
+    dbt.common.clients.system.make_directory(log_dir)
 
 
 class DebugWarnings(logbook.compat.redirected_warnings):

core/dbt/parser/manifest.py

@@ -65,7 +65,7 @@
 from dbt.node_types import NodeType, AccessType
 from dbt.clients.jinja import get_rendered, MacroStack
 from dbt.clients.jinja_static import statically_extract_macro_calls
-from dbt.clients.system import (
+from dbt.common.clients.system import (
     make_directory,
     path_exists,
     read_json,

core/dbt/parser/read_files.py

@@ -2,7 +2,7 @@
 import pathspec  # type: ignore
 import pathlib
 from dataclasses import dataclass, field
-from dbt.clients.system import load_file_contents
+from dbt.common.clients.system import load_file_contents
 from dbt.contracts.files import (
     FilePath,
     ParseFileType,

core/dbt/parser/search.py

@@ -4,7 +4,7 @@
 from pathspec import PathSpec  # type: ignore
 
 from dbt.common.clients.jinja import extract_toplevel_blocks, BlockTag
-from dbt.clients.system import find_matching
+from dbt.common.clients.system import find_matching
 from dbt.config import Project
 from dbt.contracts.files import FilePath, AnySourceFile
 from dbt.exceptions import ParsingError, DbtInternalError

core/dbt/task/debug.py

@@ -15,7 +15,7 @@
     DebugCmdOut,
     DebugCmdResult,
 )
-import dbt.clients.system
+import dbt.common.clients.system
 import dbt.exceptions
 import dbt.common.exceptions
 from dbt.adapters.factory import get_adapter, register_adapter
@@ -110,7 +110,8 @@ def run(self) -> bool:
         if self.args.config_dir:
             fire_event(
                 OpenCommand(
-                    open_cmd=dbt.clients.system.open_dir_cmd(), profiles_dir=str(self.profiles_dir)
+                    open_cmd=dbt.common.clients.system.open_dir_cmd(),
+                    profiles_dir=str(self.profiles_dir),
                 )
             )
             return DebugRunStatus.SUCCESS.value
@@ -202,7 +203,9 @@ def _load_profile(self) -> SubtaskStatus:
                 ),
             )
 
-        raw_profile_data = load_yaml_text(dbt.clients.system.load_file_contents(self.profile_path))
+        raw_profile_data = load_yaml_text(
+            dbt.common.clients.system.load_file_contents(self.profile_path)
+        )
         if isinstance(raw_profile_data, dict):
             self.raw_profile_data = raw_profile_data
 
@@ -395,7 +398,7 @@ def _target_found(self) -> str:
 
     def test_git(self) -> SubtaskStatus:
         try:
-            dbt.clients.system.run_cmd(os.getcwd(), ["git", "--help"])
+            dbt.common.clients.system.run_cmd(os.getcwd(), ["git", "--help"])
         except dbt.exceptions.ExecutableError as exc:
             return SubtaskStatus(
                 log_msg=red("ERROR"),