Move exclude_keys() to dlt.common.utils module

dlt/common/utils.py

@@ -504,6 +504,19 @@ def without_none(d: Mapping[TKey, Optional[TValue]]) -> Mapping[TKey, TValue]:
  return {k: v for k, v in d.items() if v is not None}
 
 
+def exclude_keys(d: Mapping[str, Any], keys: Iterable[str]) -> Dict[str, Any]:
+ """Removes specified keys from a dictionary and returns a new dictionary.
+
+ Args:
+ d (Mapping[str, Any]): The dictionary to remove keys from.
+ keys (Iterable[str]): The keys to remove.
+
+ Returns:
+ Dict[str, Any]: A new dictionary with the specified keys removed.
+ """
+ return {k: v for k, v in d.items() if k not in keys}
+
+
 def get_full_class_name(obj: Any) -> str:
  cls = obj.__class__
  module = cls.__module__

dlt/sources/rest_api/__init__.py

@@ -9,6 +9,7 @@
 from dlt.common import jsonpath
 from dlt.common.schema.schema import Schema
 from dlt.common.schema.typing import TSchemaContract
+from dlt.common.utils import exclude_keys
 
 from dlt.extract import Incremental, DltResource, DltSource, decorators
 
@@ -44,7 +45,7 @@
  setup_incremental_object,
  create_response_hooks,
 )
-from .utils import check_connection, exclude_keys # noqa: F401
+from .utils import check_connection # noqa: F401
 
 PARAM_TYPES: List[ParamBindType] = ["incremental", "resolve"]
 MIN_SECRET_MASKING_LENGTH = 3

dlt/sources/rest_api/config_setup.py

@@ -19,7 +19,7 @@
 from dlt.common import logger
 from dlt.common.configuration import resolve_configuration
 from dlt.common.schema.utils import merge_columns
-from dlt.common.utils import update_dict_nested
+from dlt.common.utils import update_dict_nested, exclude_keys
 from dlt.common import jsonpath
 
 from dlt.extract.incremental import Incremental
@@ -65,7 +65,6 @@
  Endpoint,
  EndpointResource,
 )
-from .utils import exclude_keys
 
 
 PAGINATOR_MAP: Dict[str, Type[BasePaginator]] = {

dlt/sources/rest_api/utils.py

@@ -1,4 +1,4 @@
-from typing import Tuple, Dict, Any, Mapping, Iterable
+from typing import Tuple
 
 from dlt.common import logger
 from dlt.extract.source import DltSource
@@ -10,19 +10,6 @@ def join_url(base_url: str, path: str) -> str:
  return base_url + path.lstrip("/")
 
 
-def exclude_keys(d: Mapping[str, Any], keys: Iterable[str]) -> Dict[str, Any]:
- """Removes specified keys from a dictionary and returns a new dictionary.
-
- Args:
- d (Mapping[str, Any]): The dictionary to remove keys from.
- keys (Iterable[str]): The keys to remove.
-
- Returns:
- Dict[str, Any]: A new dictionary with the specified keys removed.
- """
- return {k: v for k, v in d.items() if k not in keys}
-
-
 def check_connection(
  source: DltSource,
  *resource_names: str,