change ksexception -> ddsexception, first option (#107)

Co-authored-by: Tim Hunter <tjhunter@cs.stanford.edu>

Author: tjhunter

dds/_api.py

@@ -14,10 +14,11 @@
 from .fun_args import get_arg_ctx
 from .introspect import introspect, _accepted_packages
 from ._lru_store import LRUCacheStore, default_cache_size
+
 from .store import LocalFileStore, Store
 from .structures import (
     DDSPath,
-    KSException,
+    DDSException,
     EvalContext,
     PyHash,
     ProcessingStage,
@@ -27,6 +28,7 @@
     FunctionInteractionsUtils,
     FunctionIndirectInteractionUtils,
 )
+from . import _options
 
 _Out = TypeVar("_Out")
 _In = TypeVar("_In")
@@ -64,7 +66,7 @@ def load(path: Union[str, DDSPath, pathlib.Path]) -> Any:
     path_ = DDSPathUtils.create(path)
     key = _store.fetch_paths([path_]).get(path_)
     if key is None:
-        raise KSException(f"The store {_store} did not return path {path_}")
+        raise DDSException(f"The store {_store} did not return path {path_}")
     else:
         return _store.fetch_blob(key)
 
@@ -85,7 +87,7 @@ def set_store(
     global _store
     if isinstance(store, Store):
         if cache_objects is not None:
-            raise KSException(
+            raise DDSException(
                 f"Cannot provide a caching option and a store object of type 'Store' at the same time"
             )
         # Directly setting the store
@@ -99,12 +101,12 @@ def set_store(
         _store = LocalFileStore(internal_dir, data_dir)
     elif store == "dbfs":
         if data_dir is None:
-            raise KSException("Missing data_dir argument")
+            raise DDSException("Missing data_dir argument")
         if internal_dir is None:
-            raise KSException("Missing internal_dir argument")
+            raise DDSException("Missing internal_dir argument")
         dbutils = dbutils or _fetch_ipython_vars().get("dbutils")
         if dbutils is None:
-            raise KSException(
+            raise DDSException(
                 "Missing dbutils objects from input or from arguments."
                 " You must be using a databricks notebook to use the DBFS store"
             )
@@ -117,13 +119,13 @@ def set_store(
             DBFSURI.parse(internal_dir), DBFSURI.parse(data_dir), dbutils, commit_type_
         )
     else:
-        raise KSException(f"Unknown store {store}")
+        raise DDSException(f"Unknown store {store}")
 
     if cache_objects is not None:
         num_objects: Optional[int] = None
 
         if not isinstance(cache_objects, (int, bool)):
-            raise KSException(
+            raise DDSException(
                 f"cached_object should be int or bool, received type {type(cache_objects)}"
             )
         if isinstance(cache_objects, bool) and cache_objects:
@@ -148,16 +150,16 @@ def check(s: Union[str, ProcessingStage], cur: ProcessingStage) -> ProcessingSta
         if isinstance(s, str):
             s = s.upper()
             if s not in dir(ProcessingStage):
-                raise KSException(
+                raise DDSException(
                     f"{s} is not a valid stage name. Valid names are {dir(ProcessingStage)}"
                 )
             x = ProcessingStage[s]
         elif isinstance(s, ProcessingStage):
             x = s
         else:
-            raise KSException(f"Not a valid type: {s} {type(s)}")
+            raise DDSException(f"Not a valid type: {s} {type(s)}")
         if x != cur:
-            raise KSException(
+            raise DDSException(
                 f"Wrong order for the stage name, expected {cur} but got {x}"
             )
         return cur
@@ -182,15 +184,15 @@ def _eval(
 
     stages = _parse_stages(dds_stages)
 
-    extra_debug = dds_extra_debug or False
+    extra_debug = dds_extra_debug or _options._dds_extra_debug
 
     if not _eval_ctx:
         # Not in an evaluation context, create one and introspect
         return _eval_new_ctx(fun, path, args, kwargs, export_graph, extra_debug, stages)
     else:
         if not path:
-            raise KSException(
-                "Already in eval() context. Nested eval contexts are not supported"
+            raise DDSException(
+                "Already in dds.eval() context. Nested eval contexts are not supported"
             )
         key = None if path is None else _eval_ctx.requested_paths[path]
         t = _time()
@@ -302,7 +304,7 @@ def _eval_new_ctx(
 
         _logger.debug(f"Interaction tree:")
         FunctionInteractionsUtils.pprint_tree(
-            inters, present_blobs, printer=lambda s: _logger.debug(s)
+            inters, present_blobs, printer=_logger.debug
         )
         if export_graph is not None:
             # Attempt to run the export module:

dds/_introspect_indirect.py

@@ -33,7 +33,7 @@
 from .structures import (
     FunctionArgContext,
     DDSPath,
-    KSException,
+    DDSException,
     CanonicalPath,
     LocalDepPath,
     FunctionIndirectInteractions,
@@ -59,7 +59,7 @@ def _introspect(
         return _introspect_fun(obj, gctx, call_stack)
     if isinstance(obj, type):
         return _introspect_class(obj, gctx, call_stack)
-    raise KSException(
+    raise DDSException(
         f"Expected function or class, got object of type {type(obj)} instead: {obj}"
     )
 
@@ -74,7 +74,7 @@ def _introspect_class(
 
     fun_module = inspect.getmodule(c)
     if fun_module is None:
-        raise KSException(f"Could not find module: class:{c} module: {fun_module}")
+        raise DDSException(f"Could not find module: class:{c} module: {fun_module}")
     # _logger.debug(f"_introspect: {f}: fun_path={fun_path} fun_module={fun_module}")
     fiis_ = gctx.cached_indirect_interactions.get(fun_path)
     if fiis_ is not None:
@@ -103,7 +103,7 @@ def _introspect_fun(
 
     fun_module = inspect.getmodule(f)
     if fun_module is None:
-        raise KSException(f"Could not find module: f; {f} module: {fun_module}")
+        raise DDSException(f"Could not find module: f; {f} module: {fun_module}")
     # _logger.debug(f"_introspect: {f}: fun_path={fun_path} fun_module={fun_module}")
     ast_f: Union[ast.Lambda, ast.FunctionDef]
     if is_lambda(f):
@@ -160,7 +160,7 @@ def inspect_fun(
         elif isinstance(node, ast.Lambda):
             body = [node.body]
         else:
-            raise KSException(f"unknown ast node {type(node)}")
+            raise DDSException(f"unknown ast node {type(node)}")
         dummy_arg_ctx = FunctionArgContext(OrderedDict(), None)
         local_vars = set(
             InspectFunction.get_local_vars(body, dummy_arg_ctx) + arg_names
@@ -255,7 +255,9 @@ def inspect_call(
             # - parse the arguments
             # - introspect the callee
             if len(node.args) < 2:
-                raise KSException(f"Wrong number of args: expected 2+, got {node.args}")
+                raise DDSException(
+                    f"Wrong number of args: expected 2+, got {node.args}"
+                )
             store_path = InspectFunction._retrieve_store_path(node.args[0], mod, gctx)
             called_path_ast = node.args[1]
             if isinstance(called_path_ast, ast.Name):
@@ -276,7 +278,7 @@ def inspect_call(
             assert isinstance(called_z, AuthorizedObject)
             called_fun, call_fun_path = called_z.object_val, called_z.resolved_path
             if call_fun_path in call_stack:
-                raise KSException(
+                raise DDSException(
                     f"Detected circular function calls or (co-)recursive calls."
                     f"This is currently not supported. Change your code to split the "
                     f"recursive section into a separate function. "
@@ -291,7 +293,7 @@ def inspect_call(
         if caller_fun_path == CPU.from_list(["dds", "load"]):
             # Evaluation call: get the argument and returns the function interaction for this call.
             if len(node.args) != 1:
-                raise KSException(f"Wrong number of args: expected 1, got {node.args}")
+                raise DDSException(f"Wrong number of args: expected 1, got {node.args}")
             store_path = InspectFunction._retrieve_store_path(node.args[0], mod, gctx)
             _logger.debug(f"inspect_call:eval: store_path: {store_path}")
             return store_path
@@ -300,7 +302,7 @@ def inspect_call(
             raise NotImplementedError("eval")
 
         if caller_fun_path in call_stack:
-            raise KSException(
+            raise DDSException(
                 f"Detected circular function calls or (co-)recursive calls."
                 f"This is currently not supported. Change your code to split the "
                 f"recursive section into a separate function. "

dds/_lambda_funs.py

@@ -11,7 +11,7 @@
 import inspect
 import logging
 from ._print_ast import pformat
-from .structures import KSException
+from .structures import DDSException
 
 _logger = logging.getLogger(__name__)
 
@@ -79,9 +79,9 @@ def inspect_lambda_condition(fun: Callable[..., Any]) -> ast.Lambda:
     _logger.debug(f"_parse_lambda: call_node: {pformat(call_node)}")
 
     if call_node is None:
-        raise KSException(f"Could not find call node {pformat(call_node)}")
+        raise DDSException(f"Could not find call node {pformat(call_node)}")
 
     for node in ast.walk(call_node):
         if isinstance(node, ast.Lambda):
             return node
-    raise KSException(f"Could not parse lambda {pformat(call_node)}")
+    raise DDSException(f"Could not parse lambda {pformat(call_node)}")

dds/_options.py

@@ -0,0 +1,7 @@
+"""
+All the options for the package.
+
+TODO: provide eventually a configuration interface.
+"""
+
+_dds_extra_debug = True

dds/_retrieve_objects.py

@@ -22,7 +22,7 @@
     AuthorizedObject,
 )
 from .structures import (
-    KSException,
+    DDSException,
     CanonicalPath,
     LocalDepPath,
 )
@@ -38,7 +38,7 @@ def _mod_path(m: ModuleType) -> CanonicalPath:
 def function_path(f: Union[type, FunctionType]) -> CanonicalPath:
     mod = inspect.getmodule(f)
     if mod is None:
-        raise KSException(f"Function {f} has no module")
+        raise DDSException(f"Function {f} has no module")
     return CanonicalPath(_mod_path(mod)._path.joinpath(f.__name__))
 
 
@@ -209,15 +209,15 @@ def retrieve_object_global(
 
         mod = importlib.import_module(mod_name)
         if mod is None:
-            raise KSException(
+            raise DDSException(
                 f"Cannot process path {path}: module {mod_name} cannot be loaded"
             )
         sub_path = CanonicalPathUtils.tail(path)
         dep_path = LocalDepPath(sub_path._path)
         # _logger.debug(f"Calling retrieve_object on {dep_path}, {mod}")
         z = cls.retrieve_object(dep_path, mod, gctx)
         if z is None or isinstance(z, ExternalObject):
-            raise KSException(
+            raise DDSException(
                 f"Cannot load path {path}: this object cannot be retrieved, however "
                 f"the module '{mod_name}' exists. The typical cause of the issue is "
                 f"that the module {mod_name} has not been whitelisted for use by DDS. Use the "

dds/codec.py

@@ -2,7 +2,7 @@
 from typing import Optional, Dict, List, Union
 
 from .structures import (
-    KSException,
+    DDSException,
     CodecProtocol,
     ProtocolRef,
     SupportedType,
@@ -58,7 +58,7 @@ def get_codec(
         # First the reference
         if ref:
             if ref not in self._protocols:
-                raise KSException(f"Requested protocol {ref}, which is not registered")
+                raise DDSException(f"Requested protocol {ref}, which is not registered")
             return self._protocols[ref]
         # Then the object type
         if obj_type is not None:
@@ -68,11 +68,11 @@ def get_codec(
                 SupportedTypeUtils.from_type(object)
             )
             if cp is None:
-                raise KSException(
+                raise DDSException(
                     f"Requested protocol for type {obj_type}, which is not registered"
                 )
             return cp
-        raise KSException(f"No protocol found")
+        raise DDSException(f"No protocol found")
 
 
 def _build_default_registry() -> CodecRegistry:

dds/codecs/databricks.py

@@ -15,7 +15,7 @@
 
 from ..codec import CodecRegistry
 from ..store import Store, current_timestamp
-from ..structures import CodecProtocol, ProtocolRef, FileCodecProtocol, KSException
+from ..structures import CodecProtocol, ProtocolRef, FileCodecProtocol, DDSException
 from ..structures import PyHash, DDSPath, GenericLocation, SupportedType as ST
 from ..structures_utils import SupportedTypeUtils as STU
 
@@ -186,7 +186,7 @@ def fetch_blob(self, key: PyHash) -> Optional[Any]:
                 self._dbutils.fs.cp(str(p), str(lp2))
                 return codec.deserialize_from(lp)
         else:
-            raise KSException(f"{type(codec)} codec")
+            raise DDSException(f"{type(codec)} codec")
 
     def store_blob(self, key: PyHash, blob: Any, codec: Optional[ProtocolRef]) -> None:
         protocol = self._registry.get_codec(STU.from_type(type(blob)), codec)
@@ -205,7 +205,7 @@ def store_blob(self, key: PyHash, blob: Any, codec: Optional[ProtocolRef]) -> No
                 _logger.debug(f"Temporary copy from DBFS: {lp2} -> {p}")
                 self._dbutils.fs.cp(lp2, str(p))
         else:
-            raise KSException(f"{type(protocol)} codec")
+            raise DDSException(f"{type(protocol)} codec")
         meta_p = self._blob_meta_path(key)
         try:
             meta = json.dumps(