diff --git a/docs/source/conf.py b/docs/source/conf.py index bfe8da4760..940f9c44b0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# + import importlib import os import sys @@ -166,9 +166,8 @@ "cupy", # Avoid loading GPU libraries during the documentation build "merlin", "morpheus.cli.commands", # Dont document the CLI in Sphinx - "morpheus.utils.nvt.mutate.annotate", "nvtabular", - "pandas", # Avoid documenting pandas for the purposes of the dfencoder.dataframe + "pandas", "tensorrt", "torch", "tqdm", diff --git a/docs/source/sphinxext/github_link.py b/docs/source/sphinxext/github_link.py index 48e8ef8de2..e047b3af42 100644 --- a/docs/source/sphinxext/github_link.py +++ b/docs/source/sphinxext/github_link.py @@ -100,7 +100,11 @@ def _linkcode_resolve(domain, info, package, url_fmt, revision): # Unwrap the object to get the correct source # file in case that is wrapped by a decorator - obj = inspect.unwrap(obj) + # Note: objects mocked by autodoc_mock_imports will raise an exception when we try to unwrap them + try: + obj = inspect.unwrap(obj) + except: # noqa: E722 + return fn: str = None lineno: str = None diff --git a/morpheus/utils/nvt/__init__.py b/morpheus/utils/nvt/__init__.py index b07b75f6ee..081b2ae826 100644 --- a/morpheus/utils/nvt/__init__.py +++ b/morpheus/utils/nvt/__init__.py @@ -11,8 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from .extensions import register_morpheus_extensions -from .mutate import MutateOp - -__all__ = ["MutateOp", "register_morpheus_extensions"] diff --git a/morpheus/utils/nvt/decorators.py b/morpheus/utils/nvt/decorators.py index 5a196bd24d..217d391feb 100644 --- a/morpheus/utils/nvt/decorators.py +++ b/morpheus/utils/nvt/decorators.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import inspect +import os import typing import pandas as pd @@ -98,3 +100,24 @@ def wrapper(*args, **kwargs) -> typing.Union[pd.DataFrame, cudf.DataFrame]: return wrapper return decorator + + +# Avoid using the annotate decorator in sphinx builds, instead define a simple pass-through decorator +if os.environ.get("MORPHEUS_IN_SPHINX_BUILD") is None: + from merlin.core.dispatch import annotate # pylint: disable=unused-import +else: + + def annotate(*args, **kwargs): # pylint: disable=unused-argument + """ + `merlin.core.dispatch.annotate` + """ + + def decorator(func): + + @functools.wraps(func) + def wrappper(*args, **kwargs): + return func(*args, **kwargs) + + return wrappper + + return decorator diff --git a/morpheus/utils/nvt/mutate.py b/morpheus/utils/nvt/mutate.py index b100d2aab7..92c62bc7e1 100644 --- a/morpheus/utils/nvt/mutate.py +++ b/morpheus/utils/nvt/mutate.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools -import os import typing from inspect import getsourcelines @@ -24,18 +22,7 @@ from nvtabular.ops.operator import ColumnSelector from nvtabular.ops.operator import Operator -# Avoid using the annotate decorator in sphinx builds, instead define a simple pass-through decorator -if os.environ.get("MORPHEUS_IN_SPHINX_BUILD") is None: - from merlin.core.dispatch import annotate # pylint: disable=ungrouped-imports -else: - - def annotate(func, *args, **kwargs): # pylint: disable=unused-argument - - @functools.wraps(func) - def decorator(func): - return func - - return decorator +from morpheus.utils.nvt.decorators import annotate class MutateOp(Operator): @@ -108,7 +95,7 @@ def label(self): try: # otherwise get the lambda source code from the inspect module if possible - source = getsourcelines(self.f)[0][0] + source = getsourcelines(self.f)[0][0] # pylint: disable=no-member lambdas = [op.strip() for op in source.split(">>") if "lambda " in op] if len(lambdas) == 1 and lambdas[0].count("lambda") == 1: return lambdas[0] diff --git a/morpheus/utils/nvt/schema_converters.py b/morpheus/utils/nvt/schema_converters.py index 64b64bd284..1889fcd70c 100644 --- a/morpheus/utils/nvt/schema_converters.py +++ b/morpheus/utils/nvt/schema_converters.py @@ -42,8 +42,8 @@ from morpheus.utils.column_info import StringCatColumn from morpheus.utils.column_info import StringJoinColumn from morpheus.utils.column_info import create_increment_col -from morpheus.utils.nvt import MutateOp from morpheus.utils.nvt.decorators import sync_df_as_pandas +from morpheus.utils.nvt.mutate import MutateOp from morpheus.utils.nvt.transforms import json_flatten diff --git a/morpheus/utils/schema_transforms.py b/morpheus/utils/schema_transforms.py index 3daa3c5903..37fa539fb1 100644 --- a/morpheus/utils/schema_transforms.py +++ b/morpheus/utils/schema_transforms.py @@ -22,20 +22,20 @@ import cudf from morpheus.utils.column_info import DataFrameInputSchema -from morpheus.utils.nvt import register_morpheus_extensions -from morpheus.utils.nvt.patches import patch_numpy_dtype_registry +from morpheus.utils.nvt import patches +from morpheus.utils.nvt.extensions import morpheus_ext from morpheus.utils.nvt.schema_converters import create_and_attach_nvt_workflow if os.environ.get("MORPHEUS_IN_SPHINX_BUILD") is None: # Apply patches to NVT # TODO(Devin): Can be removed, once numpy mappings are updated in Merlin # ======================================================================== - patch_numpy_dtype_registry() + patches.patch_numpy_dtype_registry() # ======================================================================== # Add morpheus conversion mappings # ======================================================================== - register_morpheus_extensions() + morpheus_ext.register_morpheus_extensions() # ========================================================================= logger = logging.getLogger(__name__) diff --git a/tests/utils/nvt/integration/test_mutate_op.py b/tests/utils/nvt/integration/test_mutate_op.py index 73f41857d4..c690dfc0a7 100644 --- a/tests/utils/nvt/integration/test_mutate_op.py +++ b/tests/utils/nvt/integration/test_mutate_op.py @@ -20,7 +20,7 @@ import cudf -from morpheus.utils.nvt import MutateOp +from morpheus.utils.nvt.mutate import MutateOp from morpheus.utils.nvt.transforms import json_flatten diff --git a/tests/utils/nvt/test_mutate_op.py b/tests/utils/nvt/test_mutate_op.py index 562c3a6491..034e4f9049 100644 --- a/tests/utils/nvt/test_mutate_op.py +++ b/tests/utils/nvt/test_mutate_op.py @@ -20,7 +20,7 @@ from merlin.schema import Schema from nvtabular.ops.operator import ColumnSelector -from morpheus.utils.nvt import MutateOp +from morpheus.utils.nvt.mutate import MutateOp @pytest.fixture(name="df")