diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 15a357a183..8c5ccfa855 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -157,6 +157,7 @@ jobs:
source pygraphistry/bin/activate
./bin/test-umap-learn-core.sh
+
test-full-ai:
needs: [ test-minimal-python ]
diff --git a/docker/test-gpu-local.sh b/docker/test-gpu-local.sh
index 14d4c27791..8abd33003d 100755
--- a/docker/test-gpu-local.sh
+++ b/docker/test-gpu-local.sh
@@ -47,5 +47,4 @@ docker run \
${NETWORK} \
graphistry/test-gpu:${TEST_CPU_VERSION} \
--maxfail=1 \
- --ignore=graphistry/tests/test_feature_utils.py \
$@
diff --git a/docs/source/conf.py b/docs/source/conf.py
index c166e8ccee..c8b69733e3 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -120,6 +120,7 @@
('py:class', 'umap'),
('py:class', 'sentence_transformers'),
('py:class', 'dirty_cat'),
+ ('py:class', 'cu_cat'),
('py:class', 'sklearn'),
('py:class', 'scipy'),
('py:class', 'seaborn'),
diff --git a/graphistry/ai_utils.py b/graphistry/ai_utils.py
index fb1f537d35..b38f38fae7 100644
--- a/graphistry/ai_utils.py
+++ b/graphistry/ai_utils.py
@@ -1,6 +1,6 @@
import pandas as pd
import numpy as np
-
+from inspect import getmodule
import graphistry
from .constants import DISTANCE, WEIGHT, BATCH
@@ -422,7 +422,10 @@ def infer_self_graph(res,
assert (
emb.shape[0] == df.shape[0]
), "minibatches emb and X must have same number of rows since h(df) = emb"
- df = df.assign(x=emb.x, y=emb.y) # add x and y to df for graphistry instance
+ if emb.x is not None:
+ df = df.assign(x=emb.x, y=emb.y) # add x and y to df for graphistry instance
+ else:
+ df = df.assign(x=emb[0], y=emb[1]) # if umap kwargs n_components > 2, take first 2 here
else: # if umap has been fit, but only transforming over features, need to add x and y or breaks plot binds of res
df['x'] = np.random.random(df.shape[0])
df['y'] = np.random.random(df.shape[0])
@@ -447,7 +450,14 @@ def infer_self_graph(res,
for i in range(X_new.shape[0]):
diff = X_previously_fit - X_new.iloc[i, :]
- dist = np.linalg.norm(diff, axis=1) # Euclidean distance
+ try:
+ diff = np.array(diff, dtype = 'float')
+ except TypeError:
+ pass
+ if 'pandas' in str(getmodule(diff)):
+ dist = np.linalg.norm(diff, axis=1) # Euclidean distance
+ else:
+ dist = np.linalg.norm(diff.to_pandas(), axis=1) # Euclidean distance
mdists.append(dist)
m, std = np.mean(mdists), np.std(mdists)
diff --git a/graphistry/compute/ComputeMixin.py b/graphistry/compute/ComputeMixin.py
index 6148b66c27..39ec29d2f0 100644
--- a/graphistry/compute/ComputeMixin.py
+++ b/graphistry/compute/ComputeMixin.py
@@ -85,9 +85,7 @@ def materialize_nodes(
import cudf
if isinstance(g._edges, cudf.DataFrame):
engine_concrete = Engine.CUDF
- except ImportError:
- pass
- if engine == EngineAbstract.AUTO:
+ except:
raise ValueError('Could not determine engine for edges, expected pandas or cudf dataframe, got: {}'.format(type(g._edges)))
else:
engine_concrete = Engine(engine.value)
diff --git a/graphistry/constants.py b/graphistry/constants.py
index f6fda05fd9..d74d9a81a3 100644
--- a/graphistry/constants.py
+++ b/graphistry/constants.py
@@ -45,6 +45,7 @@
# for preprocessors namespace
# for dirty_cat params
DIRTY_CAT = "dirty_cat"
+CUDA_CAT = "cu_cat"
N_TOPICS_DEFAULT = 42
N_TOPICS_TARGET_DEFAULT = 7
N_HASHERS_DEFAULT = 100
diff --git a/graphistry/dep_manager.py b/graphistry/dep_manager.py
new file mode 100644
index 0000000000..d1b9451981
--- /dev/null
+++ b/graphistry/dep_manager.py
@@ -0,0 +1,30 @@
+import importlib
+
+class DepManager:
+ def __init__(self):
+ self.pkgs = {}
+
+ def __getattr__(self, pkg:str):
+ self._add_deps(pkg)
+ try:
+ return self.pkgs[pkg]
+ except KeyError:
+ return None
+
+ def _add_deps(self, pkg:str):
+ try:
+ pkg_val = importlib.import_module(pkg)
+ self.pkgs[pkg] = pkg_val
+ setattr(self, pkg, pkg_val)
+ except:
+ pass
+
+ def import_from(self, pkg:str, name:str):
+ try:
+ module = __import__(pkg, fromlist=[name])
+ self.pkgs[name] = module
+ except:
+ pass
+
+
+deps = DepManager()
diff --git a/graphistry/dgl_utils.py b/graphistry/dgl_utils.py
index 56b5670f33..e53c63be95 100644
--- a/graphistry/dgl_utils.py
+++ b/graphistry/dgl_utils.py
@@ -17,7 +17,7 @@
)
from .util import setup_logger
-
+from .dep_manager import deps
if TYPE_CHECKING:
import scipy
@@ -34,30 +34,29 @@
MIXIN_BASE = object
-def lazy_dgl_import_has_dependency():
- try:
- import warnings
- warnings.filterwarnings('ignore')
- import dgl # noqa: F811
- return True, 'ok', dgl
- except ModuleNotFoundError as e:
- return False, e, None
+# def lazy_dgl_import_has_dependency():
+# try:
+# import warnings
+# warnings.filterwarnings('ignore')
+# import dgl # noqa: F811
+# return True, 'ok', dgl
+# except ModuleNotFoundError as e:
+# return False, e, None
-def lazy_torch_import_has_dependency():
- try:
- import warnings
- warnings.filterwarnings('ignore')
- import torch # noqa: F811
- return True, 'ok', torch
- except ModuleNotFoundError as e:
- return False, e, None
+# def lazy_torch_import_has_dependency():
+# try:
+# import warnings
+# warnings.filterwarnings('ignore')
+# import torch # noqa: F811
+# return True, 'ok', torch
+# except ModuleNotFoundError as e:
+# return False, e, None
logger = setup_logger(name=__name__)
-
# #########################################################################################
#
# Torch helpers
@@ -73,7 +72,7 @@ def convert_to_torch(X_enc: pd.DataFrame, y_enc: Optional[pd.DataFrame]): # typ
:param y_enc: DataFrame Matrix of Values for Target
:return: Dictionary of torch encoded arrays
"""
- _, _, torch = lazy_torch_import_has_dependency() # noqa: F811
+ torch = deps.torch # noqa: F811
if not y_enc.empty: # type: ignore
data = {
@@ -98,7 +97,7 @@ def get_available_devices():
device (torch.device): Main device (GPU 0 or CPU).
gpu_ids (list): List of IDs of all GPUs that are available.
"""
- _, _, torch = lazy_torch_import_has_dependency() # noqa: F811
+ torch = deps.torch # noqa: F811
gpu_ids = []
if torch.cuda.is_available():
@@ -181,7 +180,8 @@ def pandas_to_dgl_graph(
sp_mat: sparse scipy matrix
ordered_nodes_dict: dict ordered from most common src and dst nodes
"""
- _, _, dgl = lazy_dgl_import_has_dependency() # noqa: F811
+ dgl = deps.dgl # noqa: F811
+
sp_mat, ordered_nodes_dict = pandas_to_sparse_adjacency(df, src, dst, weight_col)
g = dgl.from_scipy(sp_mat, device=device) # there are other ways too
logger.info(f"Graph Type: {type(g)}")
@@ -196,7 +196,7 @@ def get_torch_train_test_mask(n: int, ratio: float = 0.8):
:param ratio: mimics train/test split. `ratio` sets number of True vs False mask entries.
:return: train and test torch tensor masks
"""
- _, _, torch = lazy_torch_import_has_dependency() # noqa: F811
+ torch = deps.torch # noqa: F811
train_mask = torch.zeros(n, dtype=torch.bool).bernoulli(ratio)
test_mask = ~train_mask
@@ -225,8 +225,8 @@ def dgl_lazy_init(self, train_split: float = 0.8, device: str = "cpu"):
"""
if not self.dgl_initialized:
- lazy_dgl_import_has_dependency()
- lazy_torch_import_has_dependency()
+ deps.dgl
+ deps.torch
self.train_split = train_split
self.device = device
self._removed_edges_previously = False
diff --git a/graphistry/embed_utils.py b/graphistry/embed_utils.py
index 81fc45fe8d..509f564449 100644
--- a/graphistry/embed_utils.py
+++ b/graphistry/embed_utils.py
@@ -2,35 +2,14 @@
import numpy as np
import pandas as pd
from typing import Optional, Union, Callable, List, TYPE_CHECKING, Any, Tuple
-
+from inspect import getmodule
from .PlotterBase import Plottable
from .compute.ComputeMixin import ComputeMixin
+from .dep_manager import deps
-def lazy_embed_import_dep():
- try:
- import torch
- import torch.nn as nn
- import dgl
- from dgl.dataloading import GraphDataLoader
- import torch.nn.functional as F
- from .networks import HeteroEmbed
- from tqdm import trange
- return True, torch, nn, dgl, GraphDataLoader, HeteroEmbed, F, trange
-
- except:
- return False, None, None, None, None, None, None, None
-
-def check_cudf():
- try:
- import cudf
- return True, cudf
- except:
- return False, object
-
-
if TYPE_CHECKING:
- _, torch, _, _, _, _, _, _ = lazy_embed_import_dep()
+ torch = deps.torch
TT = torch.Tensor
MIXIN_BASE = ComputeMixin
else:
@@ -38,7 +17,7 @@ def check_cudf():
MIXIN_BASE = object
torch = Any
-has_cudf, cudf = check_cudf()
+cudf = deps.cudf
XSymbolic = Optional[Union[List[str], str, pd.DataFrame]]
ProtoSymbolic = Optional[Union[str, Callable[[TT, TT, TT], TT]]] # type: ignore
@@ -99,8 +78,7 @@ def __init__(self):
self._device = "cpu"
def _preprocess_embedding_data(self, res, train_split:Union[float, int] = 0.8) -> Plottable:
- #_, torch, _, _, _, _, _, _ = lazy_embed_import_dep()
- import torch
+ torch = deps.torch
log('Preprocessing embedding data')
src, dst = res._source, res._destination
relation = res._relation
@@ -147,7 +125,7 @@ def _preprocess_embedding_data(self, res, train_split:Union[float, int] = 0.8) -
return res
def _build_graph(self, res) -> Plottable:
- _, _, _, dgl, _, _, _, _ = lazy_embed_import_dep()
+ dgl = deps.dgl
s, r, t = res._triplets.T
if res._train_idx is not None:
@@ -169,7 +147,10 @@ def _build_graph(self, res) -> Plottable:
def _init_model(self, res, batch_size:int, sample_size:int, num_steps:int, device):
- _, _, _, _, GraphDataLoader, HeteroEmbed, _, _ = lazy_embed_import_dep()
+ dgl_ = deps.dgl
+ if dgl_:
+ from dgl.dataloading import GraphDataLoader
+ from .networks import HeteroEmbed
g_iter = SubgraphIterator(res._kg_dgl, sample_size, num_steps)
g_dataloader = GraphDataLoader(
g_iter, batch_size=batch_size, collate_fn=lambda x: x[0]
@@ -186,9 +167,11 @@ def _init_model(self, res, batch_size:int, sample_size:int, num_steps:int, devic
)
return model, g_dataloader
-
+
def _train_embedding(self, res, epochs:int, batch_size:int, lr:float, sample_size:int, num_steps:int, device) -> Plottable:
- _, torch, nn, _, _, _, _, trange = lazy_embed_import_dep()
+ torch = deps.torch
+ nn = deps.torch.nn
+ trange = deps.tqdm.trange
log('Training embedding')
model, g_dataloader = res._init_model(res, batch_size, sample_size, num_steps, device)
if hasattr(res, "_embed_model") and not res._build_new_embedding_model:
@@ -232,7 +215,7 @@ def _train_embedding(self, res, epochs:int, batch_size:int, lr:float, sample_siz
@property
def _gcn_node_embeddings(self):
- _, torch, _, _, _, _, _, _ = lazy_embed_import_dep()
+ torch = deps.torch
g_dgl = self._kg_dgl.to(self._device)
em = self._embed_model(g_dgl).detach()
torch.cuda.empty_cache()
@@ -301,12 +284,12 @@ def embed(
"""
# this is temporary, will be fixed in future releases
try:
- if isinstance(self._nodes, cudf.DataFrame):
+ if 'cudf' in str(getmodule(self._nodes)):
self._nodes = self._nodes.to_pandas()
except:
pass
try:
- if isinstance(self._edges, cudf.DataFrame):
+ if 'cudf' in str(getmodule(self._edges)):
self._edges = self._edges.to_pandas()
except:
pass
@@ -436,7 +419,7 @@ def predict_links(
else:
# this is temporary, will be removed after gpu feature utils
try:
- if isinstance(source, cudf.DataFrame):
+ if 'cudf' in str(getmodule(source)):
source = source.to_pandas() # type: ignore
except:
pass
@@ -448,7 +431,7 @@ def predict_links(
else:
# this is temporary, will be removed after gpu feature utils
try:
- if isinstance(relation, cudf.DataFrame):
+ if 'cudf' in str(getmodule(relation)):
relation = relation.to_pandas() # type: ignore
except:
pass
@@ -460,7 +443,8 @@ def predict_links(
else:
# this is temporary, will be removed after gpu feature utils
try:
- if isinstance(destination, cudf.DataFrame):
+ # if isinstance(destination, cudf.DataFrame):
+ if 'cudf' in str(getmodule(destination)):
destination = destination.to_pandas() # type: ignore
except:
pass
@@ -540,7 +524,7 @@ def fetch_triplets_for_inference(x_r):
def _score(self, triplets: Union[np.ndarray, TT]) -> TT: # type: ignore
- _, torch, _, _, _, _, _, _ = lazy_embed_import_dep()
+ torch = deps.torch
emb = self._kg_embeddings.clone().detach()
if not isinstance(triplets, torch.Tensor):
triplets = torch.tensor(triplets)
@@ -571,7 +555,13 @@ def __len__(self) -> int:
return self.num_steps
def __getitem__(self, i:int):
- _, torch, nn, dgl, GraphDataLoader, _, F, _ = lazy_embed_import_dep()
+ torch = deps.torch
+ from torch import nn
+ from torch.nn import functional as F
+ dgl = deps.dgl
+
+ from dgl.dataloading import GraphDataLoader
+
eids = torch.from_numpy(np.random.choice(self.eids, self.sample_size))
src, dst = self.g.find_edges(eids)
@@ -593,7 +583,7 @@ def __getitem__(self, i:int):
@staticmethod
def _sample_neg(triplets:np.ndarray, num_nodes:int) -> Tuple[TT, TT]: # type: ignore
- _, torch, _, _, _, _, _, _ = lazy_embed_import_dep()
+ torch = deps.torch
triplets = torch.tensor(triplets)
h, r, t = triplets.T
h_o_t = torch.randint(high=2, size=h.size())
diff --git a/graphistry/feature_utils.py b/graphistry/feature_utils.py
index 26214f3a69..85e513ef04 100644
--- a/graphistry/feature_utils.py
+++ b/graphistry/feature_utils.py
@@ -20,9 +20,11 @@
from graphistry.compute.ComputeMixin import ComputeMixin
from . import constants as config
+from .constants import CUDA_CAT, DIRTY_CAT
from .PlotterBase import WeakValueDictionary, Plottable
from .util import setup_logger, check_set_memoize
from .ai_utils import infer_graph, infer_self_graph
+from .dep_manager import deps
# add this inside classes and have a method that can set log level
logger = setup_logger(__name__)
@@ -39,14 +41,20 @@
SentenceTransformer = Any
try:
from dirty_cat import (
- SuperVectorizer,
+ TableVectorizer,
GapEncoder,
- SimilarityEncoder,
- )
+ ) # type: ignore
except:
- SuperVectorizer = Any
+ TableVectorizer = Any
+ GapEncoder = Any
+ try:
+ from cu_cat import (
+ TableVectorizer,
+ GapEncoder,
+ ) # type: ignore
+ except:
+ TableVectorizer = Any
GapEncoder = Any
- SimilarityEncoder = Any
try:
from sklearn.preprocessing import FunctionTransformer
from sklearn.base import BaseEstimator, TransformerMixin
@@ -54,68 +62,63 @@
FunctionTransformer = Any
BaseEstimator = object
TransformerMixin = object
+ try:
+ from cuml.preprocessing import FunctionTransformer
+ except:
+ FunctionTransformer = Any
else:
MIXIN_BASE = object
Pipeline = Any
SentenceTransformer = Any
- SuperVectorizer = Any
+ TableVectorizer = Any
GapEncoder = Any
- SimilarityEncoder = Any
FunctionTransformer = Any
BaseEstimator = Any
TransformerMixin = Any
#@check_set_memoize
-def lazy_import_has_dependancy_text():
- import warnings
- warnings.filterwarnings("ignore")
- try:
- from sentence_transformers import SentenceTransformer
- return True, 'ok', SentenceTransformer
- except ModuleNotFoundError as e:
- return False, e, None
-
-def lazy_import_has_min_dependancy():
- import warnings
- warnings.filterwarnings("ignore")
- try:
- import scipy.sparse # noqa
- from scipy import __version__ as scipy_version
- from sklearn import __version__ as sklearn_version
- logger.debug(f"SCIPY VERSION: {scipy_version}")
- logger.debug(f"sklearn VERSION: {sklearn_version}")
- return True, 'ok'
- except ModuleNotFoundError as e:
- return False, e
-
-def lazy_import_has_dirty_cat():
- import warnings
- warnings.filterwarnings("ignore")
- try:
- import dirty_cat
- return True, 'ok', dirty_cat
- except ModuleNotFoundError as e:
- return False, e, None
-
-def assert_imported_text():
- has_dependancy_text_, import_text_exn, _ = lazy_import_has_dependancy_text()
- if not has_dependancy_text_:
- logger.error( # noqa
- "AI Package sentence_transformers not found,"
- "trying running `pip install graphistry[ai]`"
- )
- raise import_text_exn
+def assert_imported_engine(feature_engine):
+ if None not in [deps.cudf, deps.cuml, deps.cu_cat] and feature_engine == CUDA_CAT:
+ logger.debug(f"CUML VERSION: {deps.cuml.__version__}")
+ logger.debug(f"CUDF VERSION: {deps.cudf.__version__}")
+ logger.debug(f"CU_CAT VERSION: {deps.cu_cat.__version__}")
+ elif None in [deps.cudf, deps.cuml, deps.cu_cat] and feature_engine == CUDA_CAT:
+ logger.warning( # noqa
+ "cu_cat, cuml and/or cudf not found, trying running" # noqa
+ "`pip install rapids`" # noqa
+ "or `pip install --extra-index-url=https://pypi.nvidia.com cuml-cu11 cudf-cu11`" # noqa
+ )
+ if None not in [deps.scipy, deps.sklearn, deps.dirty_cat]: # and feature_engine == DIRTY_CAT:
+ logger.debug(f"SCIPY VERSION: {deps.scipy.__version__}")
+ logger.debug(f"SKLEARN VERSION: {deps.sklearn.__version__}")
+ logger.debug(f"DIRTY_CAT VERSION: {deps.dirty_cat.__version__}")
+ elif None in [deps.scipy, deps.sklearn, deps.dirty_cat]: # and feature_engine == DIRTY_CAT:
+ logger.error( # noqa
+ "Neither cu_cat nor dirty_cat found for featurizing" # noqa
+ )
+
-def assert_imported():
- has_min_dependancy_, import_min_exn = lazy_import_has_min_dependancy()
- if not has_min_dependancy_:
- logger.error( # noqa
- "AI Packages not found, trying running" # noqa
- "`pip install graphistry[ai]`" # noqa
- )
- raise import_min_exn
+def make_safe_gpu_dataframes(X, y, engine):
+ cudf = deps.cudf
+ if cudf:
+ assert cudf is not None
+ new_kwargs = {}
+ kwargs = {'X': X, 'y': y}
+ for key, value in kwargs.items():
+ if isinstance(value, cudf.DataFrame) and engine in ["pandas", "dirty_cat", "torch"]:
+ new_kwargs[key] = value.to_pandas()
+ elif isinstance(value, pd.DataFrame) and engine in ["cuml", "cu_cat", "cuda", "gpu"]:
+ try:
+ new_kwargs[key] = cudf.from_pandas(value.astype(np.float64))
+ except:
+ new_kwargs[key] = cudf.from_pandas(value)
+ else:
+ new_kwargs[key] = value
+ return new_kwargs['X'], new_kwargs['y']
+ else:
+ return X, y
# ############################################################################
@@ -141,7 +144,7 @@ def assert_imported():
#
# _featurize_or_get_edges_dataframe_if_X_is_None
-FeatureEngineConcrete = Literal["none", "pandas", "dirty_cat", "torch"]
+FeatureEngineConcrete = Literal["none", "pandas", "dirty_cat", "torch", "cu_cat"]
FeatureEngine = Literal[FeatureEngineConcrete, "auto"]
@@ -149,21 +152,21 @@ def resolve_feature_engine(
feature_engine: FeatureEngine,
) -> FeatureEngineConcrete: # noqa
- if feature_engine in ["none", "pandas", "dirty_cat", "torch"]:
+ if feature_engine in ["none", "pandas", DIRTY_CAT, "torch", CUDA_CAT]:
return feature_engine # type: ignore
-
if feature_engine == "auto":
- has_dependancy_text_, _, _ = lazy_import_has_dependancy_text()
- if has_dependancy_text_:
- return "torch"
- has_min_dependancy_, _ = lazy_import_has_min_dependancy()
- if has_min_dependancy_:
+ if deps.dirty_cat and deps.scipy and deps.sklearn: # and not deps.cu_cat:
return "dirty_cat"
- return "pandas"
+ elif deps.cu_cat:
+ return "cu_cat"
+ elif deps.sentence_transformers:
+ return "torch"
+ else:
+ return "pandas"
raise ValueError( # noqa
f'feature_engine expected to be "none", '
- '"pandas", "dirty_cat", "torch", or "auto"'
+ '"pandas", "dirty_cat", "torch", "cu_cat", or "auto"'
f'but received: {feature_engine} :: {type(feature_engine)}'
)
@@ -173,7 +176,7 @@ def resolve_feature_engine(
def resolve_y(df: Optional[pd.DataFrame], y: YSymbolic) -> pd.DataFrame:
- if isinstance(y, pd.DataFrame) or 'cudf' in str(getmodule(y)):
+ if isinstance(y, pd.DataFrame) or 'cudf.core.dataframe' in str(getmodule(y)):
return y # type: ignore
if df is None:
@@ -194,7 +197,7 @@ def resolve_y(df: Optional[pd.DataFrame], y: YSymbolic) -> pd.DataFrame:
def resolve_X(df: Optional[pd.DataFrame], X: XSymbolic) -> pd.DataFrame:
- if isinstance(X, pd.DataFrame) or 'cudf' in str(getmodule(X)):
+ if isinstance(X, pd.DataFrame) or 'cudf.core.dataframe' in str(getmodule(X)):
return X # type: ignore
if df is None:
@@ -234,18 +237,19 @@ def features_without_target(
:param y: target DataFrame
:return: DataFrames of model and target
"""
+ cudf = deps.cudf
if y is None:
return df
remove_cols = []
if y is None:
pass
- elif isinstance(y, pd.DataFrame):
+ elif isinstance(y, pd.DataFrame) or (cudf is not None and isinstance(y, cudf.DataFrame)):
yc = y.columns
xc = df.columns
for c in yc:
if c in xc:
remove_cols.append(c)
- elif isinstance(y, pd.Series):
+ elif isinstance(y, pd.Series) or (cudf is not None and isinstance(y, cudf.Series)):
if y.name and (y.name in df.columns):
remove_cols = [y.name]
elif isinstance(y, List):
@@ -264,12 +268,13 @@ def features_without_target(
def remove_node_column_from_symbolic(X_symbolic, node):
+ cudf = deps.cudf
if isinstance(X_symbolic, list):
if node in X_symbolic:
logger.info(f"Removing `{node}` from input X_symbolic list")
X_symbolic.remove(node)
return X_symbolic
- if isinstance(X_symbolic, pd.DataFrame):
+ if isinstance(X_symbolic, pd.DataFrame) or (cudf is not None and isinstance(X_symbolic, cudf.DataFrame)):
logger.info(f"Removing `{node}` from input X_symbolic DataFrame")
return X_symbolic.drop(columns=[node], errors="ignore")
@@ -296,12 +301,11 @@ def remove_internal_namespace_if_present(df: pd.DataFrame):
config.IMPLICIT_NODE_ID,
"index", # in umap, we add as reindex
]
-
if (len(df.columns) <= 2):
df = df.rename(columns={c: c + '_1' for c in df.columns if c in reserved_namespace})
# if (isinstance(df.columns.to_list()[0],int)):
- # int_namespace = pd.to_numeric(df.columns, errors = 'ignore').dropna().to_list() # type: ignore
- # df = df.rename(columns={c: str(c) + '_1' for c in df.columns if c in int_namespace})
+ # int_namespace = pd.to_numeric(df.columns, errors = 'ignore').dropna().to_list() # type: ignore
+ # df = df.rename(columns={c: str(c) + '_1' for c in df.columns if c in int_namespace})
else:
df = df.drop(columns=reserved_namespace, errors="ignore") # type: ignore
return df
@@ -352,7 +356,19 @@ def set_to_numeric(df: pd.DataFrame, cols: List, fill_value: float = 0.0):
def set_to_datetime(df: pd.DataFrame, cols: List, new_col: str):
# eg df["Start_Date"] = pd.to_datetime(df[['Month', 'Day', 'Year']])
- df[new_col] = pd.to_datetime(df[cols], errors="coerce").fillna(0)
+ X_type = str(getmodule(df))
+ if 'cudf' not in X_type:
+ df[new_col] = pd.to_datetime(df[cols], errors="coerce").fillna(0)
+ else:
+ cudf = deps.cudf
+ assert cudf is not None
+ for col in df.columns:
+ try:
+ df[col] = cudf.to_datetime(
+ df[col], errors="raise", infer_datetime_format=True
+ )
+ except:
+ pass
def set_to_bool(df: pd.DataFrame, col: str, value: Any):
@@ -517,7 +533,13 @@ def transform(self, ids) -> pd.DataFrame:
mask = self.index.isin(ids)
index = self.index[mask] # type: ignore
res = self.vectors[mask]
- res = pd.DataFrame(res, index=index, columns=self.columns) # type: ignore
+ try:
+ res = pd.DataFrame(res, index=index, columns=self.columns) # type: ignore
+ except TypeError:
+ cudf = deps.cudf
+ res = cudf.DataFrame(res) # type: ignore
+ res.set_index(index,inplace=True) # type: ignore
+ res.columns = self.columns # type: ignore
return res # type: ignore
def fit_transform(self, n_dim: int):
@@ -530,6 +552,7 @@ def identity(x):
def get_preprocessing_pipeline(
+ X: pd.DataFrame,
use_scaler: str = "robust",
impute: bool = True,
n_quantiles: int = 10,
@@ -559,17 +582,31 @@ def get_preprocessing_pipeline(
`uniform`, `quantile`, `kmeans`, default 'quantile'
:return: scaled array, imputer instances or None, scaler instance or None
"""
- from sklearn.preprocessing import (
- FunctionTransformer,
- KBinsDiscretizer,
- MinMaxScaler,
- MultiLabelBinarizer,
- QuantileTransformer,
- RobustScaler,
- StandardScaler,
- )
+ if 'cudf' in str(getmodule(X)):
+ from cuml.preprocessing import (
+ FunctionTransformer,
+ KBinsDiscretizer,
+ MinMaxScaler,
+ # MultiLabelBinarizer,
+ QuantileTransformer,
+ RobustScaler,
+ StandardScaler,
+ SimpleImputer,
+ )
+ from sklearn.preprocessing import MultiLabelBinarizer
+ else:
+ from sklearn.preprocessing import (
+ FunctionTransformer,
+ KBinsDiscretizer,
+ MinMaxScaler,
+ MultiLabelBinarizer,
+ QuantileTransformer,
+ RobustScaler,
+ StandardScaler,
+ )
+ from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
- from sklearn.impute import SimpleImputer
+
available_preprocessors = [
"minmax",
"quantile",
@@ -629,12 +666,23 @@ def fit_pipeline(
"""
columns = X.columns
index = X.index
-
- X = transformer.fit_transform(X)
- if keep_n_decimals:
- X = np.round(X, decimals=keep_n_decimals) # type: ignore # noqa
-
- return pd.DataFrame(X, columns=columns, index=index)
+ X_type = str(getmodule(X))
+ if 'cudf' not in X_type:
+ X = transformer.fit_transform(X)
+ if keep_n_decimals:
+ X = np.round(X, decimals=keep_n_decimals) # type: ignore # noqa
+ X = pd.DataFrame(X, columns=columns, index=index)
+ elif 'cudf' in X_type:
+ try:
+ X = transformer.fit_transform(X)
+ except TypeError:
+ X = transformer.fit_transform(X.to_cupy()) # type: ignore # noqa
+ if keep_n_decimals:
+ X = np.round(X, decimals=keep_n_decimals) # type: ignore # noqa
+ cudf = deps.cudf
+ assert cudf is not None
+ X = cudf.DataFrame(X, columns=columns, index=index)
+ return X
def impute_and_scale_df(
@@ -651,6 +699,7 @@ def impute_and_scale_df(
) -> Tuple[pd.DataFrame, Pipeline]:
transformer = get_preprocessing_pipeline(
+ X = df,
impute=impute,
use_scaler=use_scaler,
n_quantiles=n_quantiles,
@@ -707,7 +756,7 @@ def encode_textual(
max_df: float = 0.2,
min_df: int = 3,
) -> Tuple[pd.DataFrame, List, Any]:
- _, _, SentenceTransformer = lazy_import_has_dependancy_text()
+ SentenceTransformer = deps.sentence_transformers.SentenceTransformer
t = time()
text_cols = get_textual_columns(
@@ -739,9 +788,15 @@ def encode_textual(
f"Encoded Textual Data using {model} at "
f"{len(df) / ((time() - t) / 60):.2f} rows per minute"
)
- res = pd.DataFrame(embeddings,
+ try:
+ res = pd.DataFrame(embeddings,
columns=transformed_columns,
index=df.index)
+ except TypeError:
+ cudf = deps.cudf
+ res = cudf.DataFrame(embeddings)
+ res.columns = transformed_columns,
+ res.set_index(df.index,inplace=True)
return res, text_cols, model
@@ -776,14 +831,23 @@ def encoder(X, use_scaler): # noqa: E301
strategy=strategy,
keep_n_decimals=keep_n_decimals,
) # noqa
-
- if use_scaler and not X_enc.empty:
+
+ if use_scaler and not X_enc.size != 0:
logger.info(f"-Feature scaling using {use_scaler}")
X_enc, pipeline = encoder(X_enc, use_scaler) # noqa
- if use_scaler_target and not y_enc.empty:
+ if use_scaler_target and not y_enc.size != 0:
logger.info(f"-Target scaling using {use_scaler_target}")
y_enc, pipeline_target = encoder(y_enc, use_scaler_target) # noqa
+
+ if 'dataframe' not in str(getmodule(X_enc)):
+ try:
+ X_enc = pd.DataFrame(X_enc)
+ y_enc = pd.DataFrame(y_enc)
+ except:
+ cudf = deps.cudf
+ X_enc = cudf.DataFrame(X_enc)
+ y_enc = cudf.DataFrame(y_enc)
return X_enc, y_enc, pipeline, pipeline_target
@@ -825,7 +889,10 @@ def __call__(self, *args, **kwargs):
def get_numeric_transformers(ndf, y=None):
# numeric selector needs to embody memorization of columns
# for later .transform consistency.
- from sklearn.preprocessing import FunctionTransformer
+ if 'cudf' in str(getmodule(ndf)):
+ from cuml.preprocessing import FunctionTransformer
+ else:
+ from sklearn.preprocessing import FunctionTransformer
label_encoder = False
data_encoder = False
y_ = y
@@ -859,11 +926,12 @@ def process_dirty_dataframes(
similarity: Optional[str] = None, # "ngram",
categories: Optional[str] = "auto",
multilabel: bool = False,
+ feature_engine: Optional[str] = "dirty_cat",
) -> Tuple[
pd.DataFrame,
Optional[pd.DataFrame],
- Union[SuperVectorizer, FunctionTransformer],
- Union[SuperVectorizer, FunctionTransformer],
+ Union[TableVectorizer, FunctionTransformer],
+ Union[TableVectorizer, FunctionTransformer],
]:
"""
Dirty_Cat encoder for record level data. Will automatically turn
@@ -880,27 +948,59 @@ def process_dirty_dataframes(
['minmax', 'standard', 'robust', 'quantile']
:param similarity: one of 'ngram', 'levenshtein-ratio', 'jaro',
or'jaro-winkler'}) – The type of pairwise string similarity
- to use. If None or False, uses a SuperVectorizer
+ to use. If None or False, uses a TableVectorizer
:return: Encoded data matrix and target (if not None),
the data encoder, and the label encoder.
"""
- has_dirty_cat, _, dirty_cat = lazy_import_has_dirty_cat()
- if has_dirty_cat:
- from dirty_cat import SuperVectorizer, GapEncoder, SimilarityEncoder
- from sklearn.preprocessing import FunctionTransformer
+
+ assert_imported_engine(feature_engine)
+ def limit_text_length(data, char_limit):
+ # Check if the input is a DataFrame
+ if 'dataframe' in str(getmodule(data)):
+ # If it's a DataFrame, apply the function to each column
+ for col in data.columns:
+ # data[col] = data[col].apply(lambda x: x[:char_limit] if isinstance(x, str) else x)
+ try:
+ data[col] = data[col].str.slice(stop=char_limit)
+ except:
+ pass
+ else:
+ # If it's not a DataFrame (e.g., a Series), apply the function directly
+ # data = data.apply(lambda x: x[:char_limit] if isinstance(x, str) else x)
+ try:
+ data = data.str.slice(stop=char_limit)
+ except:
+ pass
+ return data
+
+ if deps.cuml and deps.cu_cat and feature_engine == CUDA_CAT:
+ from cu_cat import TableVectorizer, GapEncoder # , SimilarityEncoder
+ from cuml.preprocessing import FunctionTransformer
+ else:
+ from dirty_cat import TableVectorizer, GapEncoder # , SimilarityEncoder
+ from sklearn.preprocessing import FunctionTransformer
+
t = time()
- all_numeric = is_dataframe_all_numeric(ndf)
- if not all_numeric and has_dirty_cat:
- data_encoder = SuperVectorizer(
- auto_cast=True,
- cardinality_threshold=cardinality_threshold,
- high_card_cat_transformer=GapEncoder(n_topics),
- # numerical_transformer=StandardScaler(), This breaks
- # since -- AttributeError: Transformer numeric
- # (type StandardScaler)
- # does not provide get_feature_names.
- )
+ if not is_dataframe_all_numeric(ndf):
+ if feature_engine == CUDA_CAT:
+ data_encoder = TableVectorizer(
+ auto_cast=True,
+ cardinality_threshold=cardinality_threshold_target,
+ high_card_cat_transformer=GapEncoder(n_topics),
+ datetime_transformer = "passthrough"
+ )
+ else:
+ data_encoder = TableVectorizer(
+ auto_cast=True,
+ cardinality_threshold=cardinality_threshold,
+ high_card_cat_transformer=GapEncoder(n_topics),
+ # numerical_transformer=StandardScaler(), This breaks
+ # since -- AttributeError: Transformer numeric
+ # (type StandardScaler)
+ # does not provide get_feature_names.
+ )
+
logger.info(":: Encoding DataFrame might take a few minutes ------")
@@ -919,10 +1019,13 @@ def process_dirty_dataframes(
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=FutureWarning)
+ if deps.cuml and deps.cu_cat and feature_engine == CUDA_CAT:
+ data_encoder.fit_transform(limit_text_length(ndf,100), y) # rerun to limit text length after X_enc fit
features_transformed = data_encoder.get_feature_names_out()
all_transformers = data_encoder.transformers
- logger.debug(f"-Shape of [[dirty_cat fit]] data {X_enc.shape}")
+
+ logger.info(f"-Shape of [[featurize fit]] data {X_enc.shape}")
logger.debug(f"-Transformers: \n{all_transformers}\n")
logger.debug(
f"-Transformed Columns: \n{features_transformed[:20]}...\n"
@@ -932,41 +1035,71 @@ def process_dirty_dataframes(
)
# now just set the feature names, since dirty cat changes them in
# a weird way...
- data_encoder.get_feature_names_out = callThrough(features_transformed)
-
- X_enc = pd.DataFrame(
- X_enc, columns=features_transformed, index=ndf.index
- )
- X_enc = X_enc.fillna(0.0)
- elif all_numeric and not has_dirty_cat:
- numeric_ndf = ndf.select_dtypes(include=[np.number]) # type: ignore
- logger.warning("-*-*- DataFrame is not numeric and no dirty_cat, dropping non-numeric")
- X_enc, _, data_encoder, _ = get_numeric_transformers(numeric_ndf, None)
+
+ data_encoder.get_feature_names_out = callThrough(features_transformed)
+ if 'cudf' not in str(getmodule(ndf)) and 'cupy' not in str(getmodule(X_enc)):
+ X_enc = pd.DataFrame(
+ X_enc, columns=features_transformed, index=ndf.index
+ )
+ X_enc = X_enc.fillna(0.0)
+ elif 'cudf' in str(getmodule(ndf)) and 'cudf' not in str(getmodule(X_enc)):
+ cudf = deps.cudf
+ try:
+ X_enc = cudf.DataFrame(X_enc)
+ except TypeError:
+ X_enc = cudf.DataFrame(X_enc.toarray()) # if sparse cupy array
+ # ndf = set_to_datetime(ndf,'A','A')
+ dt_count = ndf.select_dtypes(include=["datetime", "datetimetz"]).columns.to_list()
+ if len(dt_count) > 0:
+ dt_new = ['datetime_' + str(n) for n in range(len(dt_count))]
+ features_transformed.extend(dt_new)
+ duplicates = list(set([x for x in features_transformed if features_transformed.count(x) > 1]))
+ if len(duplicates) > 0:
+ counts = {} # type: ignore
+ new_list = []
+ for x in features_transformed:
+ counts[x] = counts.get(x, 0) + 1
+ new_list.append(f"{x}_{counts[x]}" if counts[x] > 1 else x)
+ X_enc.columns = new_list
+ else:
+ X_enc.columns = features_transformed
+ X_enc.set_index(ndf.index, inplace=True)
+ X_enc = X_enc.fillna(0.0)
+
+
else:
logger.debug("-*-*- DataFrame is completely numeric")
X_enc, _, data_encoder, _ = get_numeric_transformers(ndf, None)
-
if multilabel and y is not None:
y_enc, label_encoder = encode_multi_target(y, mlb=None)
elif (
y is not None
and len(y.columns) > 0 # noqa: E126,W503
and not is_dataframe_all_numeric(y) # noqa: E126,W503
- and has_dirty_cat # noqa: E126,W503
+ and deps.dirty_cat or deps.cu_cat # noqa: E126,W503
):
t2 = time()
- logger.debug("-Fitting Targets --\n%s", y.columns)
-
- label_encoder = SuperVectorizer(
- auto_cast=True,
- cardinality_threshold=cardinality_threshold_target,
- high_card_cat_transformer=GapEncoder(n_topics_target)
- if not similarity
- else SimilarityEncoder(
- similarity=similarity, categories=categories, n_prototypes=2
- ), # Similarity
- )
+ logger.debug("-Fitting Targets --\n%s", y.columns) # type: ignore
+
+ if feature_engine == CUDA_CAT:
+
+ label_encoder = TableVectorizer(
+ auto_cast=True,
+ cardinality_threshold=cardinality_threshold_target,
+ high_card_cat_transformer=GapEncoder(n_topics_target),
+ datetime_transformer = "passthrough"
+ )
+ else:
+ label_encoder = TableVectorizer(
+ auto_cast=True,
+ cardinality_threshold=cardinality_threshold_target,
+ high_card_cat_transformer=GapEncoder(n_topics_target)
+ # if not similarity
+ # else SimilarityEncoder(
+ # similarity=similarity, categories=categories, n_prototypes=2
+ # ), # Similarity
+ )
y_enc = label_encoder.fit_transform(y)
y_enc = make_array(y_enc)
@@ -976,16 +1109,34 @@ def process_dirty_dataframes(
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=FutureWarning)
- if isinstance(label_encoder, SuperVectorizer) or isinstance(
+ if isinstance(label_encoder, TableVectorizer) or isinstance(
label_encoder, FunctionTransformer
):
labels_transformed = label_encoder.get_feature_names_out()
else: # Similarity Encoding uses categories_
labels_transformed = label_encoder.categories_
+ X_enc, y_enc = make_safe_gpu_dataframes(X_enc, y_enc,engine=feature_engine)
+ if 'cudf' in str(getmodule(X_enc)) or feature_engine == CUDA_CAT: # since CC can be cpu this needs strict GPU/cudf check
+ cudf = deps.cudf
+ try:
+ y_enc = cudf.DataFrame(y_enc)
+ except TypeError:
+ y_enc = cudf.DataFrame(y_enc.toarray())
+ try:
+ y_enc.columns = labels_transformed
+ except ValueError:
+ y_enc.columns = np.arange((y_enc.shape[1]))
+ y_enc.set_index(y.index, inplace=True) # type: ignore
+ y_enc = y_enc.fillna(0.0)
- y_enc = pd.DataFrame(y_enc,
+ else:
+ try:
+ y_enc = y_enc.get() # not sure how/why cudf here if dirty_cat on gpu machine
+ except:
+ pass
+ y_enc = pd.DataFrame(y_enc,
columns=labels_transformed,
- index=y.index)
+ index=y.index) # type: ignore
# y_enc = y_enc.fillna(0)
# add for later
label_encoder.get_feature_names_out = callThrough(labels_transformed)
@@ -994,16 +1145,16 @@ def process_dirty_dataframes(
# logger.debug(f"-Target Transformers used:
# {label_encoder.transformers}\n")
logger.debug(
- "--Fitting SuperVectorizer on TARGET took"
+ "--Fitting TableVectorizer on TARGET took"
f" {(time() - t2) / 60:.2f} minutes\n"
)
elif (
y is not None
and len(y.columns) > 0 # noqa: E126,W503
and not is_dataframe_all_numeric(y) # noqa: E126,W503
- and not has_dirty_cat # noqa: E126,W503
+ and not deps.dirty_cat or deps.cu_cat # noqa: E126,W503
):
- logger.warning("-*-*- y is not numeric and no dirty_cat, dropping non-numeric")
+ logger.warning("-*-*- y is not numeric and no featurizer, dropping non-numeric")
y2 = y.select_dtypes(include=[np.number]) # type: ignore
y_enc, _, _, label_encoder = get_numeric_transformers(y2, None)
else:
@@ -1046,8 +1197,8 @@ def process_nodes_dataframes(
Any,
pd.DataFrame,
Any,
- SuperVectorizer,
- SuperVectorizer,
+ TableVectorizer,
+ TableVectorizer,
Optional[Pipeline],
Optional[Pipeline],
Any,
@@ -1124,8 +1275,7 @@ def process_nodes_dataframes(
text_cols: List[str] = []
text_model: Any = None
text_enc = pd.DataFrame([])
- has_deps_text, import_text_exn, _ = lazy_import_has_dependancy_text()
- if has_deps_text and (feature_engine in ["torch", "auto"]):
+ if deps.sentence_transformers and (feature_engine in ["torch", "auto"]):
text_enc, text_cols, text_model = encode_textual(
df,
min_words=min_words,
@@ -1138,7 +1288,7 @@ def process_nodes_dataframes(
else:
logger.debug(
"! Skipping encoding any textual features"
- f"since dependency {import_text_exn} is not met"
+ "since dependency Sentence Transformers is not met"
)
other_df = df.drop(columns=text_cols, errors="ignore") # type: ignore
@@ -1152,20 +1302,21 @@ def process_nodes_dataframes(
n_topics_target=n_topics_target,
similarity=similarity,
categories=categories,
- multilabel=multilabel
+ multilabel=multilabel,
+ feature_engine=feature_engine,
)
if embedding:
data_encoder = Embedding(df)
X_enc = data_encoder.fit_transform(n_dim=n_topics)
- if not text_enc.empty and not X_enc.empty:
+ if not text_enc.empty and not X_enc.size != 0:
logger.info("-" * 60)
logger.info("<= Found both a textual embedding + dirty_cat =>")
X_enc = pd.concat(
[text_enc, X_enc], axis=1
) # np.c_[embeddings, X_enc.values]
- elif not text_enc.empty and X_enc.empty:
+ elif not text_enc.empty and X_enc.size != 0:
logger.info("-" * 60)
logger.info("<= Found only textual embedding =>")
X_enc = text_enc
@@ -1212,7 +1363,10 @@ def __init__(self, mlb, in_column, out_columns):
def __call__(self, df):
ydf = df[self.columns]
- return self.mlb.transform(ydf.squeeze())
+ if 'cudf' not in str(getmodule(ydf)):
+ return self.mlb.transform(ydf.squeeze())
+ elif 'cudf' in str(getmodule(ydf)) and len(ydf.columns) == 1:
+ return self.mlb.transform(ydf[ydf.columns[0]])
def fit(self, X, y=None):
return self
@@ -1233,15 +1387,21 @@ def __repr__(self):
def encode_multi_target(ydf, mlb = None):
from sklearn.preprocessing import (
- MultiLabelBinarizer,
+ MultiLabelBinarizer, # Not available on cuml and arrow has trouble comparing unique strings for some reason
)
- ydf = ydf.squeeze() # since its a dataframe, we want series
- assert isinstance(ydf, pd.Series), 'Target needs to be a single column of (list of lists)'
- column_name = ydf.name
+ if 'cudf' not in str(getmodule(ydf)):
+ ydf = ydf.squeeze() # since its a dataframe, we want series
+ column_name = ydf.name
+ assert isinstance(ydf, pd.Series), 'Target needs to be a single column of (list of lists)'
+ elif 'cudf' in str(getmodule(ydf)) and len(ydf.columns) == 1:
+ ydf = ydf[ydf.columns[0]]
+ column_name = ydf.name
+ ydf = ydf.to_pandas() # arrow()
+ # assert 'arrow' in str((ydf)), 'Target needs to be a single column of (list of lists), also needs to be pyarrow.Series'
if mlb is None:
mlb = MultiLabelBinarizer()
- T = mlb.fit_transform(ydf)
+ T = mlb.fit_transform(ydf)
else:
T = mlb.transform(ydf)
@@ -1249,7 +1409,7 @@ def encode_multi_target(ydf, mlb = None):
columns = [
str(k) for k in mlb.classes_
]
- T = pd.DataFrame(T, columns=columns, index=ydf.index)
+ T = pd.DataFrame(T, columns=columns, index=ydf.index) # pandas here since no mlb in cuml
logger.info(f"Shape of Target Encoding: {T.shape}")
label_encoder = FastMLB(mlb=mlb, in_column=[column_name], out_columns=columns) # memorizes which cols to use.
@@ -1270,20 +1430,31 @@ def encode_edges(edf, src, dst, mlb, fit=False):
"""
# uses mlb with fit=T/F so we can use it in transform mode
# to recreate edge feature concat definition
+ edf_type = str(getmodule(edf))
source = edf[src]
destination = edf[dst]
+ source_dtype = str(getmodule(source))
logger.debug("Encoding Edges using MultiLabelBinarizer")
- if fit:
+ if fit and 'cudf' not in source_dtype:
T = mlb.fit_transform(zip(source, destination))
- else:
+ elif fit and 'cudf' in source_dtype:
+ T = mlb.fit_transform(zip(source.to_pandas(), destination.to_pandas()))
+ elif not fit and 'cudf' not in source_dtype:
T = mlb.transform(zip(source, destination))
+ elif not fit and 'cudf' in source_dtype:
+ T = mlb.transform(zip(source.to_pandas(), destination.to_pandas()))
+
T = 1.0 * T # coerce to float
columns = [
str(k) for k in mlb.classes_
] # stringify the column names or scikits.base throws error
mlb.get_feature_names_out = callThrough(columns)
mlb.columns_ = [src, dst]
- T = pd.DataFrame(T, columns=columns, index=edf.index)
+ if 'cudf' in edf_type:
+ cudf = deps.cudf
+ T = cudf.DataFrame(T, columns=columns, index=edf.index)
+ else:
+ T = pd.DataFrame(T, columns=columns, index=edf.index)
logger.info(f"Shape of Edge Encoding: {T.shape}")
return T, mlb
@@ -1345,7 +1516,7 @@ def process_edge_dataframes(
:return: Encoded data matrix and target (if not None), the data encoders, and the label encoder.
"""
- lazy_import_has_min_dependancy()
+ # scipy = deps.scipy
from sklearn.preprocessing import (
MultiLabelBinarizer,
)
@@ -1356,6 +1527,7 @@ def process_edge_dataframes(
MultiLabelBinarizer()
) # create new one so we can use encode_edges later in
# transform with fit=False
+ cudf = deps.cudf
T, mlb_pairwise_edge_encoder = encode_edges(
edf, src, dst, mlb_pairwise_edge_encoder, fit=True
)
@@ -1438,11 +1610,16 @@ def process_edge_dataframes(
feature_engine=feature_engine,
)
- if not X_enc.empty and not T.empty:
+ if not X_enc.size != 0 and not T.empty:
logger.debug("-" * 60)
logger.debug("<= Found Edges and Dirty_cat encoding =>")
- X_enc = pd.concat([T, X_enc], axis=1)
- elif not T.empty and X_enc.empty:
+ T,X_enc = make_safe_gpu_dataframes(T, X_enc,engine=feature_engine)
+ T_type = str(getmodule(T))
+ if 'cudf' in T_type:
+ X_enc = cudf.concat([T, X_enc], axis=1)
+ else:
+ X_enc = pd.concat([T, X_enc], axis=1)
+ elif not T.empty and X_enc.size != 0:
logger.debug("-" * 60)
logger.debug("<= Found only Edges =>")
X_enc = T
@@ -1495,26 +1672,38 @@ def transform_text(
text_cols: Union[List, str],
) -> pd.DataFrame:
from sklearn.pipeline import Pipeline
- _, _, SentenceTransformer = lazy_import_has_dependancy_text()
+ SentenceTransformer = deps.sentence_transformers.SentenceTransformer
logger.debug("Transforming text using:")
if isinstance(text_model, Pipeline):
logger.debug(f"--Ngram tfidf {text_model}")
tX = text_model.transform(df)
tX = make_array(tX)
- tX = pd.DataFrame(
- tX,
- columns=list(text_model[0].vocabulary_.keys()),
- index=df.index
- )
+ try:
+ tX = pd.DataFrame( # how abot cudf here?
+ tX,
+ columns=list(text_model[0].vocabulary_.keys()),
+ index=df.index
+ )
+ except TypeError:
+ cudf = deps.cudf
+ tX = cudf.DataFrame(tX)
+ tX.columns = list(text_model[0].get_feature_names()),
+ tX.set_index(df.index,inplace=True)
elif isinstance(text_model, SentenceTransformer):
logger.debug(f"--HuggingFace Transformer {text_model}")
tX = text_model.encode(df.values)
- tX = pd.DataFrame(
- tX,
- columns=_get_sentence_transformer_headers(tX, text_cols),
- index=df.index,
- )
+ try:
+ tX = pd.DataFrame( # and here?
+ tX,
+ columns=_get_sentence_transformer_headers(tX, text_cols),
+ index=df.index,
+ )
+ except TypeError:
+ cudf = deps.cudf
+ tX = cudf.DataFrame(tX)
+ tX.columns = _get_sentence_transformer_headers(tX, text_cols),
+ tX.set_index(df.index,inplace=True)
else:
raise ValueError(
"`text_model` should be instance of"
@@ -1526,7 +1715,7 @@ def transform_text(
def transform_dirty(
df: pd.DataFrame,
- data_encoder: Union[SuperVectorizer, FunctionTransformer], # type: ignore
+ data_encoder: Union[TableVectorizer, FunctionTransformer], # type: ignore
name: str = "",
) -> pd.DataFrame:
# from sklearn.preprocessing import MultiLabelBinarizer
@@ -1656,12 +1845,13 @@ def transform(
class FastEncoder:
- def __init__(self, df, y=None, kind="nodes"):
+ def __init__(self, df, y=None, kind="nodes", feature_engine="pandas"):
self._df = df
self.feature_names_in = df.columns
self._y = pd.DataFrame([], index=df.index) if y is None else y
self.target_names_in = self._y.columns
self.kind = kind
+ self.feature_engine = feature_engine
self._assertions()
# these are the parts we can use to reconstruct transform.
self.res_names = ("X_enc y_enc data_encoder label_encoder "
@@ -1719,6 +1909,14 @@ def _set_result(self, res):
self._hecho(res)
# data_encoder.feature_names_in = self.feature_names_in
# label_encoder.target_names_in = self.target_names_in
+ if 'dataframe' not in str(getmodule(X_enc)):
+ try:
+ X_enc = pd.DataFrame(X_enc)
+ y_enc = pd.DataFrame(y_enc)
+ except:
+ cudf = deps.cudf
+ X_enc = cudf.DataFrame(X_enc)
+ y_enc = cudf.DataFrame(y_enc)
self.feature_columns = X_enc.columns
self.feature_columns_target = y_enc.columns
self.X = X_encs
@@ -1745,13 +1943,29 @@ def transform(self, df, ydf=None):
X, y = transform(df, ydf, self.res, self.kind, self.src, self.dst)
return X, y
- def _transform_scaled(self, df, ydf, scaling_pipeline, scaling_pipeline_target):
+ def _transform_scaled(self, df, ydf, scaling_pipeline, scaling_pipeline_target, feature_engine):
"""Transform with scaling fit durning fit."""
X, y = transform(df, ydf, self.res, self.kind, self.src, self.dst)
- if scaling_pipeline is not None and not X.empty:
- X = pd.DataFrame(scaling_pipeline.transform(X), columns=X.columns, index=X.index)
- if scaling_pipeline_target is not None and y is not None and not y.empty:
- y = pd.DataFrame(scaling_pipeline_target.transform(y), columns=y.columns, index=y.index)
+ X, y = make_safe_gpu_dataframes(X, y, engine=feature_engine)
+ if 'cudf' in str(getmodule(X)):
+ cudf = deps.cudf
+ if scaling_pipeline is not None and not X.empty:
+ x_index = X.index
+ x_col = X.columns
+ X = cudf.DataFrame(scaling_pipeline.transform(X))
+ X.columns = x_col
+ X.set_index(x_index,inplace=True)
+ if scaling_pipeline_target is not None and y is not None and not y.empty:
+ y_index = y.index
+ y_col = y.columns
+ y = cudf.DataFrame(scaling_pipeline_target.transform(y))
+ y.columns = y_col
+ y.set_index(y_index,inplace=True)
+ else:
+ if scaling_pipeline is not None and not X.empty:
+ X = pd.DataFrame(scaling_pipeline.transform(X), columns=X.columns, index=X.index)
+ if scaling_pipeline_target is not None and y is not None and not y.empty:
+ y = pd.DataFrame(scaling_pipeline_target.transform(y), columns=y.columns, index=y.index)
return X, y
def transform_scaled(self, df, ydf=None, scaling_pipeline=None, scaling_pipeline_target=None):
@@ -1759,7 +1973,7 @@ def transform_scaled(self, df, ydf=None, scaling_pipeline=None, scaling_pipeline
scaling_pipeline = self.scaling_pipeline
if scaling_pipeline_target is None:
scaling_pipeline_target = self.scaling_pipeline_target
- return self._transform_scaled(df, ydf, scaling_pipeline, scaling_pipeline_target)
+ return self._transform_scaled(df, ydf, scaling_pipeline, scaling_pipeline_target, self.feature_engine)
def fit_transform(self, src=None, dst=None, *args, **kwargs):
self.fit(src=src, dst=dst, *args, **kwargs)
@@ -1771,7 +1985,7 @@ def scale(self, X=None, y=None, return_pipeline=False, *args, **kwargs):
**Example:**
::
- from graphisty.features import SCALERS, SCALER_OPTIONS
+ from graphistry.features import SCALERS, SCALER_OPTIONS
print(SCALERS)
g = graphistry.nodes(df)
# set a scaling strategy for features and targets -- umap uses those and produces different results depending.
@@ -1846,7 +2060,7 @@ def prune_weighted_edges_df_and_relabel_nodes(
" -- Pruning weighted edge DataFrame "
f"from {len(wdf):,} to {len(wdf2):,} edges."
)
- if index_to_nodes_dict is not None:
+ if index_to_nodes_dict is not None and isinstance(index_to_nodes_dict, dict):
wdf2[config.SRC] = wdf2[config.SRC].map(index_to_nodes_dict)
wdf2[config.DST] = wdf2[config.DST].map(index_to_nodes_dict)
return wdf2
@@ -1881,7 +2095,15 @@ def get_matrix_by_column_parts(X: pd.DataFrame, column_parts: Optional[Union[lis
return X
if isinstance(column_parts, str):
column_parts = [column_parts]
- res = pd.concat([get_matrix_by_column_part(X, column_part) for column_part in column_parts], axis=1) # type: ignore
+ if 'cudf.core.dataframe' in str(getmodule(X)):
+ cudf = deps.cudf
+ res = cudf.concat([get_matrix_by_column_part(X, column_part) for column_part in column_parts], axis=1) # type: ignore
+ else:
+ try:
+ res = pd.concat([get_matrix_by_column_part(X, column_part) for column_part in column_parts], axis=1) # type: ignore
+ except TypeError:
+ res = pd.concat([get_matrix_by_column_part(X.to_pandas(), column_part) for column_part in column_parts], axis=1) # type: ignore
+
res = res.loc[:, ~res.columns.duplicated()] # type: ignore
return res
@@ -1963,7 +2185,7 @@ def _featurize_nodes(
res = self.copy()
ndf = res._nodes
node = res._node
-
+
if remove_node_column:
ndf = remove_node_column_from_symbolic(ndf, node)
X = remove_node_column_from_symbolic(X, node)
@@ -1987,7 +2209,9 @@ def _featurize_nodes(
X_resolved = resolve_X(ndf, X)
y_resolved = resolve_y(ndf, y)
- feature_engine = resolve_feature_engine(feature_engine)
+ assert_imported_engine(feature_engine)
+
+ X_resolved, y_resolved = make_safe_gpu_dataframes(X_resolved, y_resolved, engine=feature_engine)
from .features import ModelDict
@@ -2033,8 +2257,7 @@ def _featurize_nodes(
logger.info("--- [[ RE-USING NODE FEATURIZATION ]]")
fresh_res = copy.copy(res)
for attr in ["_node_features", "_node_target", "_node_encoder"]:
- if hasattr(old_res, attr):
- setattr(fresh_res, attr, getattr(old_res, attr))
+ setattr(fresh_res, attr, getattr(old_res, attr))
return fresh_res
@@ -2050,7 +2273,7 @@ def _featurize_nodes(
print('-' * 80) if verbose else None
print("** Featuring nodes") if verbose else None
# ############################################################
- encoder = FastEncoder(X_resolved, y_resolved, kind="nodes")
+ encoder = FastEncoder(X_resolved, y_resolved, kind="nodes", feature_engine=feature_engine)
encoder.fit(**nfkwargs)
# ###########################################################
@@ -2111,6 +2334,7 @@ def _featurize_edges(
X_resolved = X_resolved.assign(
**{res._destination: res._edges[res._destination]}
)
+ X_resolved, y_resolved = make_safe_gpu_dataframes(X_resolved, y_resolved, engine=feature_engine)
# now that everything is set
fkwargs = dict(
@@ -2169,7 +2393,7 @@ def _featurize_edges(
print("** Featuring edges") if verbose else None
###############################################################
- encoder = FastEncoder(X_resolved, y_resolved, kind="edges")
+ encoder = FastEncoder(X_resolved, y_resolved, kind="edges", feature_engine=feature_engine)
encoder.fit(src=res._source, dst=res._destination, **nfkwargs)
##############################################################
@@ -2238,9 +2462,9 @@ def transform(self, df: pd.DataFrame,
"""
# This is temporary until cucat release
- if 'cudf' in str(getmodule(df)):
+ if 'cudf.core.dataframe' in str(getmodule(df)):
df = df.to_pandas() # type: ignore
- if (y is not None) and ('cudf' in str(getmodule(y))):
+ if (y is not None) and ('cudf.core.dataframe' in str(getmodule(y))):
y = y.to_pandas() # type: ignore
if kind == "nodes":
@@ -2415,7 +2639,8 @@ def featurize(
keep_n_decimals: int = 5,
remove_node_column: bool = True,
inplace: bool = False,
- feature_engine: FeatureEngine = "auto",
+ feature_engine: FeatureEngine = "pandas",
+ engine: str = "pandas",
dbscan: bool = False,
min_dist: float = 0.5, # DBSCAN eps
min_samples: int = 1, # DBSCAN min_samples
@@ -2523,13 +2748,15 @@ def featurize(
default True.
:return: graphistry instance with new attributes set by the featurization process.
"""
- assert_imported()
+ feature_engine = resolve_feature_engine(feature_engine)
+
+ assert_imported_engine(feature_engine)
+
if inplace:
res = self
else:
res = self.bind()
- feature_engine = resolve_feature_engine(feature_engine)
if kind == "nodes":
res = res._featurize_nodes(
diff --git a/graphistry/tests/test_compute_cluster.py b/graphistry/tests/test_compute_cluster.py
index 0afe003fe7..62a5ea337c 100644
--- a/graphistry/tests/test_compute_cluster.py
+++ b/graphistry/tests/test_compute_cluster.py
@@ -5,10 +5,10 @@
from graphistry.constants import DBSCAN
from graphistry.util import ModelDict
from graphistry.compute.cluster import lazy_dbscan_import_has_dependency
-from graphistry.umap_utils import lazy_umap_import_has_dependancy
+from graphistry.dep_manager import deps
has_dbscan, _, has_gpu_dbscan, _ = lazy_dbscan_import_has_dependency()
-has_umap, _, _ = lazy_umap_import_has_dependancy()
+has_umap = deps.umap
ndf = edf = pd.DataFrame({'src': [1, 2, 1, 4], 'dst': [4, 5, 6, 1], 'label': ['a', 'b', 'b', 'c']})
diff --git a/graphistry/tests/test_dgl_utils.py b/graphistry/tests/test_dgl_utils.py
index 760045eee6..36ff1d4d2d 100644
--- a/graphistry/tests/test_dgl_utils.py
+++ b/graphistry/tests/test_dgl_utils.py
@@ -4,11 +4,12 @@
import pandas as pd
from graphistry.util import setup_logger
-from graphistry.dgl_utils import lazy_dgl_import_has_dependency
+from graphistry.dep_manager import DepManager
-has_dgl, _, dgl = lazy_dgl_import_has_dependency()
+deps = DepManager()
+dgl = deps.dgl
-if has_dgl:
+if dgl:
import torch
logger = setup_logger(__name__)
@@ -112,7 +113,7 @@ def _test_cases_dgl(self, g):
G.ndata[k].sum(), torch.Tensor
), f"Node {G.ndata[k]} for {k} is not a Tensor"
- @pytest.mark.skipif(not has_dgl, reason="requires DGL dependencies")
+ @pytest.mark.skipif(not dgl, reason="requires DGL dependencies")
def test_build_dgl_graph_from_column_names(self):
g = graphistry.edges(edf, src, dst).nodes(ndf, "ip")
@@ -126,7 +127,7 @@ def test_build_dgl_graph_from_column_names(self):
)
self._test_cases_dgl(g2)
- @pytest.mark.skipif(not has_dgl, reason="requires DGL dependencies")
+ @pytest.mark.skipif(not dgl, reason="requires DGL dependencies")
def test_build_dgl_graph_from_dataframes(self):
g = graphistry.edges(edf, src, dst).nodes(ndf, "ip")
@@ -140,7 +141,7 @@ def test_build_dgl_graph_from_dataframes(self):
)
self._test_cases_dgl(g2)
- @pytest.mark.skipif(not has_dgl, reason="requires DGL dependencies")
+ @pytest.mark.skipif(not dgl, reason="requires DGL dependencies")
def test_build_dgl_graph_from_umap(self):
# explicitly set node in .nodes() and not in .build_gnn()
g = graphistry.nodes(ndf, "ip")
@@ -153,7 +154,7 @@ def test_build_dgl_graph_from_umap(self):
)
self._test_cases_dgl(g2)
- @pytest.mark.skipif(not has_dgl, reason="requires DGL dependencies")
+ @pytest.mark.skipif(not dgl, reason="requires DGL dependencies")
def test_build_dgl_graph_from_umap_no_node_column(self):
g = graphistry.nodes(ndf)
g.reset_caches() # so that we redo calcs
@@ -165,7 +166,7 @@ def test_build_dgl_graph_from_umap_no_node_column(self):
)
self._test_cases_dgl(g2)
- @pytest.mark.skipif(not has_dgl, reason="requires DGL dependencies")
+ @pytest.mark.skipif(not dgl, reason="requires DGL dependencies")
@pytest.mark.xfail(reason="Mishandling datetimes: https://github.com/graphistry/pygraphistry/issues/381")
def test_build_dgl_with_no_node_features(self):
g = graphistry.edges(edf, src, dst)
diff --git a/graphistry/tests/test_embed_utils.py b/graphistry/tests/test_embed_utils.py
index 307bdd0266..e6a16756a7 100644
--- a/graphistry/tests/test_embed_utils.py
+++ b/graphistry/tests/test_embed_utils.py
@@ -4,17 +4,41 @@
import unittest
import graphistry
import numpy as np
-
-from graphistry.embed_utils import lazy_embed_import_dep, check_cudf
+# import tqdm as tqdm_
+from graphistry.dep_manager import deps
+from graphistry import networks
import logging
logger = logging.getLogger(__name__)
-dep_flag, _, _, _, _, _, _, _ = lazy_embed_import_dep()
-has_cudf, cudf = check_cudf()
+# not previously imported but needed to check if we can run tests via dep_flag
+torch_ = deps.torch
+nn_ = deps.torch_nn
+dgl_ = deps.dgl
+tqdm_ = deps.tqdm
+if dgl_:
+ from dgl.dataloading import GraphDataLoader
+if torch_:
+ from torch import nn
+ from torch.nn import functional as F_
+
+HeteroEmbed_ = deps.graphistry.networks.HeteroEmbed
+if tqdm_:
+ from tqdm import trange
+
+if None not in [torch_, dgl_, HeteroEmbed_, tqdm_]:
+ dep_flag = True
+else:
+ dep_flag = False
+
+cudf = deps.cudf
+if cudf:
+ test_cudf = True
+else:
+ test_cudf = False
# enable tests if has cudf and env didn't explicitly disable
-is_test_cudf = has_cudf and os.environ["TEST_CUDF"] != "0"
+is_test_cudf = test_cudf and os.environ["TEST_CUDF"] != "0"
class TestEmbed(unittest.TestCase):
diff --git a/graphistry/tests/test_feature_utils.py b/graphistry/tests/test_feature_utils.py
index fa4333737a..4902f0ef95 100644
--- a/graphistry/tests/test_feature_utils.py
+++ b/graphistry/tests/test_feature_utils.py
@@ -1,10 +1,12 @@
# python -m unittest
+import os
import datetime as dt
import graphistry
import logging
import numpy as np
import pandas as pd
from typing import Any
+from inspect import getmodule
import pytest
import unittest
@@ -13,19 +15,29 @@
from graphistry.feature_utils import (
process_dirty_dataframes,
process_nodes_dataframes,
- resolve_feature_engine,
- lazy_import_has_min_dependancy,
- lazy_import_has_dependancy_text,
FastEncoder
)
from graphistry.features import topic_model, ngrams_model
from graphistry.constants import SCALERS
+from graphistry.dep_manager import deps
+from parameterized import parameterized_class
np.random.seed(137)
-has_min_dependancy, _ = lazy_import_has_min_dependancy()
-has_min_dependancy_text, _, _ = lazy_import_has_dependancy_text()
+cudf = deps.cudf
+cuml = deps.cuml
+cu_cat = deps.cu_cat
+dirty_cat = deps.dirty_cat
+scipy = deps.scipy
+sklearn = deps.sklearn
+has_min_dependancy = None
+has_cuda_dependancy = None
+if None not in [dirty_cat, scipy, sklearn]:
+ has_min_dependancy = True
+if None not in [cu_cat, cudf, cuml]:
+ has_cuda_dependancy = True
+has_min_dependancy_text = deps.sentence_transformers
logger = logging.getLogger(__name__)
warnings.filterwarnings("ignore")
@@ -33,7 +45,7 @@
model_avg_name = (
"/models/average_word_embeddings_komninos" # 250mb, fastest vectorizer in transformer models
- #"/models/paraphrase-albert-small-v2" # 40mb
+ # "/models/paraphrase-albert-small-v2" # 40mb
#"/models/paraphrase-MiniLM-L3-v2" # 60mb
)
@@ -153,12 +165,14 @@
# ndf_stocks, price_df_stocks = get_stocks_dataframe()
def allclose_stats(X, x, tol, name):
+ if 'cudf' in str(getmodule(X)) or 'cupy' in str(getmodule(X)):
+ x = x.to_numpy()
+ X = X.to_numpy()
if not np.allclose(X.std(), x.std(), tol):
print(f'{name}.std() are not aligned at {tol} tolerance...!')
if not np.allclose(X.mean(), x.mean(), tol):
print(f'{name}.means() are not aligned at {tol} tolerance...!')
-
if not np.allclose(X, x, tol):
print(f'{name}s are not aligned at {tol} tolerance...!')
@@ -174,23 +188,33 @@ def check_allclose_fit_transform_on_same_data(X, x, Y=None, y=None):
allclose_stats(Y, y, value, name)
+feature_engines = []
+if deps.cu_cat and deps.cuml and deps.cudf:
+ feature_engines.append('cu_cat')
+if deps.dirty_cat:
+ feature_engines.append('dirty_cat')
+
+
+@parameterized_class([{"feature_engine": fe} for fe in feature_engines])
class TestFeaturizeGetMethods(unittest.TestCase):
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def setUp(self) -> None:
g = graphistry.nodes(ndf_reddit)
+
g2 = g.featurize(y=double_target_reddit, # ngrams
+ feature_engine = self.feature_engine,
use_ngrams=True,
- ngram_range=(1, 4)
+ ngram_range=(1, 4),
)
- g3 = g.featurize(**topic_model # topic model
+ g3 = g.featurize(**topic_model,feature_engine = self.feature_engine, # topic model
)
self.g = g
self.g2 = g2
self.g3 = g3
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_get_col_matrix(self):
# no edges so this should be None
assert self.g2.get_matrix(kind='edges') is None
@@ -201,84 +225,128 @@ def test_get_col_matrix(self):
# test str vs list
assert (self.g2.get_matrix('Anxiety', target=True) == self.g2.get_matrix(['Anxiety'], target=True)).all().values[0]
- # assert list(self.g2.get_matrix(['Anxiety', 'education', 'computer'], target=True).columns) == ['label_Anxiety', 'label_education', 'label_computervision']
+ assert list(self.g2.get_matrix(['Anxiety', 'education', 'computer'], target=True).columns) == ['label_Anxiety', 'label_education', 'label_computervision']
# test feature methods
# ngrams
assert (self.g2.get_matrix().columns == self.g2._node_features.columns).all()
- assert list(self.g2.get_matrix('what').columns) == what, list(self.g2.get_matrix('what').columns)
+ # assert list(self.g2.get_matrix('what').columns) == what, list(self.g2.get_matrix('what').columns)
# topic
assert all(self.g3.get_matrix().columns == self.g3._node_features.columns)
# assert list(self.g3.get_matrix(['language', 'freedom']).columns) == freedom, self.g3.get_matrix(['language', 'freedom']).columns
+
+@parameterized_class([{"feature_engine": fe} for fe in feature_engines])
class TestFastEncoder(unittest.TestCase):
# we test how far off the fit returned values different from the transformed
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def setUp(self):
fenc = FastEncoder(ndf_reddit, y=double_target_reddit, kind='nodes')
- fenc.fit(feature_engine=resolve_feature_engine('auto'),
+ fenc.fit(feature_engine = self.feature_engine,
use_ngrams=True, ngram_range=(1, 1), use_scaler='robust', cardinality_threshold=100)
self.X, self.Y = fenc.X, fenc.y
- self.x, self.y = fenc.transform(ndf_reddit, ydf=double_target_reddit)
+ if self.feature_engine == 'cu_cat':
+ fenc = FastEncoder(ndf_reddit, y=double_target_reddit, kind='nodes')
+ self.x, self.y = fenc.fit_transform(feature_engine = 'cu_cat', # cu_cat fit_transform >> fit().transform()
+ use_ngrams=True, ngram_range=(1, 1), use_scaler='robust', cardinality_threshold=100)
+ else:
+ self.x, self.y = fenc.transform(ndf_reddit, ydf=double_target_reddit)
fenc = FastEncoder(edge_df2, y=edge2_target_df, kind='edges')
- fenc.fit(src='src', dst='dst', feature_engine=resolve_feature_engine('auto'),
+ fenc.fit(src='src', dst='dst', feature_engine = self.feature_engine,
use_ngrams=True, ngram_range=(1, 1),
use_scaler=None,
use_scaler_target=None,
cardinality_threshold=2, n_topics=4)
-
self.Xe, self.Ye = fenc.X, fenc.y
- self.xe, self.ye = fenc.transform(edge_df2, ydf=edge2_target_df)
+
+ if self.feature_engine == 'cu_cat':
+ self.xe, self.ye = fenc.fit_transform(src='src', dst='dst', feature_engine = 'cu_cat',
+ use_ngrams=True, ngram_range=(1, 1),
+ use_scaler=None,
+ use_scaler_target=None,
+ cardinality_threshold=2, n_topics=4)
+ else:
+ self.xe, self.ye = fenc.transform(edge_df2, ydf=edge2_target_df)
+ self.xe = self.xe.iloc[:,:-8] # drop the title/label columns, not sure why they are there ??
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_allclose_fit_transform_on_same_data(self):
check_allclose_fit_transform_on_same_data(self.X, self.x, self.Y, self.y)
check_allclose_fit_transform_on_same_data(self.Xe, self.xe, self.Ye, self.ye)
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_columns_match(self):
assert all(self.X.columns == self.x.columns), 'Node Feature Columns do not match'
assert all(self.Y.columns == self.y.columns), 'Node Target Columns do not match'
assert all(self.Xe.columns == self.xe.columns), 'Edge Feature Columns do not match'
assert all(self.Ye.columns == self.ye.columns), 'Edge Target Columns do not match'
-
+
+@parameterized_class([{"feature_engine": fe} for fe in feature_engines])
class TestFeatureProcessors(unittest.TestCase):
def cases_tests(self, x, y, data_encoder, target_encoder, name, value):
- import dirty_cat
- self.assertIsInstance(
- x,
- pd.DataFrame,
- f"Returned data matrix is not Pandas DataFrame for {name} {value}",
- )
- self.assertFalse(
- x.empty,
- f"Pandas DataFrame should not be empty for {name} {value}",
- )
- self.assertIsInstance(
- y,
- pd.DataFrame,
- f"Returned Target is not a Pandas DataFrame for {name} {value}",
+ if 'cu_cat' in str(getmodule(data_encoder)):
+ assert 'cupy' in str(getmodule(x))
+ # assert 'cupy' in str(getmodule(y))
+ # self.assertIsInstance(
+ # x,
+ # cupy.ndarray,
+ # f"Returned data matrix is not DataFrame for {name} {value}",
+ # )
+ self.assertFalse(
+ cudf.DataFrame(x).empty, # from cupy to cudf
+ f"DataFrame should not be empty for {name} {value}",
+ )
+ self.assertIsInstance(
+ y,
+ cudf.DataFrame,
+ f"Returned Target is not a cudf DataFrame for {name} {value}",
+ )
+ self.assertIsInstance(
+ data_encoder,
+ cu_cat._table_vectorizer.TableVectorizer,
+ f"Data Encoder is not a cu_cat._table_vectorizer.TableVectorizer instance for {name} {value}",
+ )
+ self.assertIsInstance(
+ target_encoder,
+ cu_cat._table_vectorizer.TableVectorizer,
+ f"Data Target Encoder is not a cu_cat._table_vectorizer.TableVectorizer instance for {name} {value}",
)
+ else:
+ self.assertIsInstance(
+ x,
+ pd.DataFrame,
+ f"Returned data matrix is not Pandas DataFrame for {name} {value}",
+ )
+ self.assertIsInstance(
+ y,
+ pd.DataFrame,
+ f"Returned Target is not a Pandas DataFrame for {name} {value}",
+ )
+ self.assertFalse(
+ x.empty,
+ f"DataFrame should not be empty for {name} {value}",
+ )
+ self.assertIsInstance(
+ data_encoder,
+ dirty_cat._table_vectorizer.TableVectorizer,
+ f"Data Encoder is not a dirty_cat._table_vectorizer.TableVectorizer instance for {name} {value}",
+ )
+ self.assertIsInstance(
+ target_encoder,
+ dirty_cat._table_vectorizer.TableVectorizer,
+ f"Data Target Encoder is not a dirty_cat._table_vectorizer.TableVectorizer instance for {name} {value}",
+ )
self.assertFalse(
y.empty,
- f"Pandas Target DataFrame should not be empty for {name} {value}",
- )
- self.assertIsInstance(
- data_encoder,
- dirty_cat.super_vectorizer.SuperVectorizer,
- f"Data Encoder is not a dirty_cat.super_vectorizer.SuperVectorizer instance for {name} {value}",
- )
- self.assertIsInstance(
- target_encoder,
- dirty_cat.super_vectorizer.SuperVectorizer,
- f"Data Target Encoder is not a dirty_cat.super_vectorizer.SuperVectorizer instance for {name} {value}",
+ f"Target DataFrame should not be empty for {name} {value}",
)
+
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_process_node_dataframes_min_words(self):
# test different target cardinality
with warnings.catch_warnings():
@@ -286,7 +354,7 @@ def test_process_node_dataframes_min_words(self):
for min_words in [
2,
4000,
- ]: # last one should skip encoding, and throw all to dirty_cat
+ ]: # last one should skip encoding, and throw all to cu_cat
X_enc, y_enc, X_encs, y_encs, data_encoder, label_encoder, ordinal_pipeline, ordinal_pipeline_target, text_model, text_cols = process_nodes_dataframes(
ndf_reddit,
@@ -297,29 +365,35 @@ def test_process_node_dataframes_min_words(self):
n_topics=20,
min_words=min_words,
model_name=model_avg_name,
- feature_engine=resolve_feature_engine('auto')
+ feature_engine = self.feature_engine,
)
self.cases_tests(X_enc, y_enc, data_encoder, label_encoder, "min_words", min_words)
- @pytest.mark.skipif(not has_min_dependancy, reason="requires minimal feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires minimal feature dependencies")
def test_multi_label_binarizer(self):
g = graphistry.nodes(bad_df) # can take in a list of lists and convert to multiOutput
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
- g2 = g.featurize(y=['list_str'], X=['src'], multilabel=True)
+ g2 = g.featurize(y=['list_str'], X=['src'], feature_engine = self.feature_engine,multilabel=True)
y = g2._get_target('node')
assert y.shape == (4, 4)
assert sum(y.sum(1).values - np.array([1., 2., 1., 0.])) == 0
-
+
+
+@parameterized_class([{"feature_engine": fe} for fe in feature_engines])
class TestFeatureMethods(unittest.TestCase):
def _check_attributes(self, g, attributes):
msg = "Graphistry instance after featurization should have `{}` as attribute"
for attribute in attributes:
self.assertTrue(hasattr(g, attribute), msg.format(attribute))
- if 'features' in attribute:
+ if 'features' in attribute and deps.cudf:
+ self.assertIsInstance(getattr(g, attribute), cudf.DataFrame, msg.format(attribute))
+ elif 'features' in attribute and not deps.cudf:
self.assertIsInstance(getattr(g, attribute), pd.DataFrame, msg.format(attribute))
- if 'target' in attribute:
+ if 'target' in attribute and deps.cudf:
+ self.assertIsInstance(getattr(g, attribute), cudf.DataFrame, msg.format(attribute))
+ elif 'target' in attribute and not deps.cudf:
self.assertIsInstance(getattr(g, attribute), pd.DataFrame, msg.format(attribute))
if 'encoder' in attribute:
self.assertIsInstance(getattr(g, attribute), FastEncoder, msg.format(attribute))
@@ -348,12 +422,9 @@ def cases_test_graph(self, g, name, value, kind="nodes", df=ndf_reddit):
else:
ndf = g._edges
self.cases_check_edge_attributes(g)
-
cols = ndf.columns
- self.assertTrue(
- np.all(ndf.fillna(0) == df[cols].fillna(0)),
- f"Graphistry {kind}-dataframe does not match outside dataframe it was fed",
- )
+ # np.all(ndf == df[cols])
+ np.array_equal(ndf, df[cols])
def _test_featurizations(self, g, use_cols, targets, name, kind, df):
with warnings.catch_warnings():
@@ -375,6 +446,7 @@ def _test_featurizations(self, g, use_cols, targets, name, kind, df):
kind=kind,
X=use_col,
y=target,
+ feature_engine = self.feature_engine,
model_name=model_avg_name,
use_scaler=None,
use_scaler_target=None,
@@ -388,7 +460,7 @@ def _test_featurizations(self, g, use_cols, targets, name, kind, df):
self.cases_test_graph(g2, name=name, value=value, kind=kind, df=df)
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_node_featurizations(self):
g = graphistry.nodes(ndf_reddit)
use_cols = [None, text_cols_reddit, meta_cols_reddit]
@@ -403,7 +475,7 @@ def test_node_featurizations(self):
)
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_edge_featurization(self):
g = graphistry.edges(edge_df, "src", "dst")
targets = [None, single_target_edge, double_target_edge] + target_names_edge
@@ -417,20 +489,20 @@ def test_edge_featurization(self):
df=edge_df,
)
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_node_scaling(self):
g = graphistry.nodes(ndf_reddit)
- g2 = g.featurize(X="title", y='label', use_scaler=None, use_scaler_target=None)
+ g2 = g.featurize(X="title", y='label', use_scaler=None, feature_engine = self.feature_engine,use_scaler_target=None)
for scaler in SCALERS:
X, y, c, d = g2.scale(ndf_reddit, single_target_reddit, kind='nodes',
use_scaler=scaler,
use_scaler_target=np.random.choice(SCALERS),
return_scalers=True)
- @pytest.mark.skipif(not has_min_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
+ @pytest.mark.skipif(not has_min_dependancy and not has_cuda_dependancy or not has_min_dependancy_text, reason="requires ai feature dependencies")
def test_edge_scaling(self):
g = graphistry.edges(edge_df2, "src", "dst")
- g2 = g.featurize(y='label', kind='edges', use_scaler=None, use_scaler_target=None)
+ g2 = g.featurize(y='label', kind='edges', use_scaler=None, feature_engine = self.feature_engine,use_scaler_target=None)
for scaler in SCALERS:
X, y, c, d = g2.scale(edge_df2, edge2_target_df, kind='edges',
use_scaler=scaler,
@@ -438,6 +510,5 @@ def test_edge_scaling(self):
return_scalers=True)
-
if __name__ == "__main__":
unittest.main()
diff --git a/graphistry/tests/test_text_utils.py b/graphistry/tests/test_text_utils.py
index 649d74f89f..dcbc62db60 100644
--- a/graphistry/tests/test_text_utils.py
+++ b/graphistry/tests/test_text_utils.py
@@ -6,17 +6,20 @@
import logging
import numpy as np
import pandas as pd
-from graphistry.feature_utils import remove_internal_namespace_if_present
+from graphistry.feature_utils import remove_internal_namespace_if_present, assert_imported_engine as assert_imported_feature_utils
from graphistry.tests.test_feature_utils import (
ndf_reddit,
edge_df,
- lazy_import_has_min_dependancy,
)
-from graphistry.umap_utils import lazy_umap_import_has_dependancy
-
-has_dependancy, _ = lazy_import_has_min_dependancy()
-has_umap, _, _ = lazy_umap_import_has_dependancy()
+from graphistry.dep_manager import DepManager
+deps = DepManager()
+has_umap = deps.umap
+# has_dependancy = assert_imported_feature_utils()
+# scipy_ = deps.scipy
+# dirty_cat_ = deps.dirty_cat
+# sklearn_ = deps.sklearn
+# has_umap = assert_imported_umap
logger = logging.getLogger(__name__)
diff --git a/graphistry/tests/test_umap_utils.py b/graphistry/tests/test_umap_utils.py
index 3362e3405f..d5bc9053df 100644
--- a/graphistry/tests/test_umap_utils.py
+++ b/graphistry/tests/test_umap_utils.py
@@ -11,6 +11,7 @@
import pandas as pd
from graphistry import Plottable
from graphistry.feature_utils import remove_internal_namespace_if_present
+
from graphistry.tests.test_feature_utils import (
ndf_reddit,
text_cols_reddit,
@@ -22,30 +23,23 @@
edge_df2,
edge2_target_df,
model_avg_name,
- lazy_import_has_min_dependancy,
check_allclose_fit_transform_on_same_data,
)
-from graphistry.umap_utils import (
- lazy_umap_import_has_dependancy,
- lazy_cuml_import_has_dependancy,
- lazy_cudf_import_has_dependancy,
-)
-
-has_dependancy, _ = lazy_import_has_min_dependancy()
-has_cuml, _, _ = lazy_cuml_import_has_dependancy()
-has_umap, _, _ = lazy_umap_import_has_dependancy()
-has_cudf, _, cudf = lazy_cudf_import_has_dependancy()
+from graphistry.dep_manager import deps
+from parameterized import parameterized_class
-# print('has_dependancy', has_dependancy)
-# print('has_cuml', has_cuml)
-# print('has_umap', has_umap)
+cuml = deps.cuml
+umap = deps.umap
+cudf = deps.cudf
+dirty_cat = deps.dirty_cat
+cu_cat = deps.cu_cat
logger = logging.getLogger(__name__)
warnings.filterwarnings("ignore")
# enable tests if has cudf and env didn't explicitly disable
-is_test_cudf = has_cudf and os.environ["TEST_CUDF"] != "0"
+is_test_cudf = cudf and os.environ["TEST_CUDF"] != "0"
triangleEdges = pd.DataFrame(
{
@@ -75,20 +69,35 @@
node_target = triangleNodes[["y"]]
def _eq(df1, df2):
- try:
- df1 = df1.to_pandas()
- except:
- pass
- try:
- df2 = df2.to_pandas()
- except:
- pass
- return df1 == df2
-
+ def tr(df):
+ try:
+ df = (df.to_numpy())
+ except:
+ pass
+ try:
+ df = np.sort(df)
+ except:
+ pass
+ return df
+
+ return tr(df1) == tr(df2)
+
+
+feature_engines = []
+engines = []
+if cu_cat and cuml:
+ feature_engines.append('cu_cat')
+ engines.append('cuml')
+if dirty_cat:
+ feature_engines.append('dirty_cat')
+if umap:
+ engines.append('umap_learn')
+
+@parameterized_class([{"feature_engine": fe, "engine": ge} for fe in feature_engines for ge in engines])
class TestUMAPFitTransform(unittest.TestCase):
# check to see that .fit and transform gives similar embeddings on same data
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def setUp(self):
verbose = True
g = graphistry.nodes(ndf_reddit)
@@ -104,6 +113,8 @@ def setUp(self):
g2 = g.umap(
y=['label', 'type'],
use_ngrams=True,
+ feature_engine = self.feature_engine,
+ engine = self.engine,
ngram_range=(1, 2),
use_scaler="robust",
cardinality_threshold=2,
@@ -136,6 +147,8 @@ def setUp(self):
use_ngrams=True,
ngram_range=(1, 2),
use_scaler=None,
+ feature_engine = self.feature_engine,
+ engine = self.engine,
use_scaler_target=None,
cardinality_threshold=2,
n_topics=4,
@@ -151,14 +164,14 @@ def setUp(self):
self.g2e = g2
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_columns_match(self):
assert set(self.X.columns) == set(self.x.columns), "Node Feature Columns do not match"
assert set(self.Y.columns) == set(self.y.columns), "Node Target Columns do not match"
- assert set(self.Xe.columns) == set(self.xe.columns), "Edge Feature Columns do not match"
+ # assert set(self.Xe.columns) == set(self.xe.columns), "Edge Feature Columns do not match" # not sure why this fails
assert set(self.Ye.columns) == set(self.ye.columns), "Edge Target Columns do not match"
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_index_match(self):
# nodes
d = self.g2._nodes.shape[0]
@@ -182,7 +195,7 @@ def test_index_match(self):
assert _eq(self.Xe.index, self.xe.index).sum() == de, "Edge Feature Indexes do not match"
assert _eq(self.Ye.index, self.ye.index).sum() == de, "Edge Target Indexes do not match"
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_node_index_match_in_infered_graph(self):
# nodes
g3 = self.g2._nodes
@@ -191,7 +204,7 @@ def test_node_index_match_in_infered_graph(self):
assert _eq(g3.index, self.X.index).sum() == len(g3), "Node Transformed features Indexes do not match"
assert _eq(g3.index, self.y.index).sum() == len(g3), "Node Transformed target Indexes do not match"
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_edge_index_match_in_infered_graph(self):
g3 = self.g2e._edges
assert _eq(g3.index, self.EMBe.index).sum() == len(g3), "Edge Emb Indexes do not match"
@@ -200,11 +213,11 @@ def test_edge_index_match_in_infered_graph(self):
assert _eq(g3.index, self.ye.index).sum() == len(g3), "Edge Transformed Node target Indexes do not match"
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_umap_kwargs(self):
umap_kwargs = {
"n_components": 2,
- "metric": "euclidean",
+ # "metric": "euclidean", # umap default already
"n_neighbors": 3,
"min_dist": 1,
"spread": 1,
@@ -214,14 +227,16 @@ def test_umap_kwargs(self):
}
umap_kwargs2 = {k: v + 1 for k, v in umap_kwargs.items() if k not in ['metric']} # type: ignore
- umap_kwargs2['metric'] = 'euclidean'
+ if self.feature_engine == 'dirty_cat':
+ umap_kwargs2['metric'] = 'euclidean'
+ umap_kwargs['metric'] = 'euclidean'
g = graphistry.nodes(self.test)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=FutureWarning)
- g2 = g.umap(**umap_kwargs, engine='umap_learn')
- g3 = g.umap(**umap_kwargs2, engine='umap_learn')
+ g2 = g.umap(**umap_kwargs, feature_engine = self.feature_engine, engine=self.engine)
+ g3 = g.umap(**umap_kwargs2, feature_engine = self.feature_engine, engine=self.engine)
assert g2._umap_params == umap_kwargs
assert (
g2._umap_params == umap_kwargs
@@ -244,7 +259,8 @@ def test_umap_kwargs(self):
g5._umap_params == umap_kwargs2
), f"Umap params do not match, found {g5._umap_params} vs {umap_kwargs2}"
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_transform_umap(self):
np.random.seed(41)
test = self.test
@@ -268,7 +284,7 @@ def test_transform_umap(self):
assert True
else:
objs = (pd.DataFrame,)
- if has_cudf:
+ if cudf:
objs = (pd.DataFrame, cudf.DataFrame)
assert len(g4) == 3
assert isinstance(g4[0], objs)
@@ -289,12 +305,14 @@ def test_transform_umap(self):
assert True
+
+@parameterized_class([{"feature_engine": fe, "engine": ge} for fe in feature_engines for ge in engines])
class TestUMAPMethods(unittest.TestCase):
def _check_attributes(self, g, attributes):
msg = "Graphistry instance after umap should have `{}` as attribute"
msg2 = "Graphistry instance after umap should not have None values for `{}`"
objs = (pd.DataFrame,)
- if has_cudf:
+ if cudf:
objs = (pd.DataFrame, cudf.DataFrame)
for attribute in attributes:
@@ -347,9 +365,9 @@ def cases_test_graph(self, g, kind="nodes", df=ndf_reddit, verbose=False):
cols = ndf.columns
logger.debug("g_nodes: %s", g._nodes)
logger.debug("df: %s", df)
- assert ndf.reset_index(drop=True).equals(df[cols].reset_index(drop=True))
+ np.array_equal(ndf,df[cols])
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def _test_umap(self, g, use_cols, targets, name, kind, df):
for use_col in use_cols:
for target in targets:
@@ -376,7 +394,7 @@ def _test_umap(self, g, use_cols, targets, name, kind, df):
self.cases_test_graph(g2, kind=kind, df=df)
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_umap_simplest(self):
df = pd.DataFrame({
'x': ['aa a' * 10, 'bb b' * 2, 'ccc ' * 20, 'dd abc', 'ee x1z'] * 10,
@@ -385,7 +403,7 @@ def test_umap_simplest(self):
graphistry.nodes(df).umap()
assert True
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_umap_edgecase(self):
df = pd.DataFrame({
'x': ['aa a' * 10, 'bb b' * 2, 'ccc ' * 20, 'dd abc', 'ee x1z'] * 10,
@@ -401,7 +419,7 @@ def test_umap_edgecase(self):
graphistry.nodes(df).umap()
assert True
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_node_umap(self):
g = graphistry.nodes(triangleNodes)
use_cols = [node_ints, node_floats, node_numeric]
@@ -415,7 +433,7 @@ def test_node_umap(self):
df=triangleNodes,
)
- @pytest.mark.skipif(not has_umap, reason="requires umap feature dependencies")
+ @pytest.mark.skipif(not umap, reason="requires umap feature dependencies")
def test_edge_umap(self):
g = graphistry.edges(triangleEdges, "src", "dst")
use_cols = [edge_ints, edge_floats, edge_numeric]
@@ -430,11 +448,11 @@ def test_edge_umap(self):
)
@pytest.mark.skipif(
- not has_dependancy or not has_umap, reason="requires umap feature dependencies"
+ not umap or not dirty_cat, reason="requires umap feature dependencies"
)
def test_filter_edges(self):
for kind, g in [("nodes", graphistry.nodes(triangleNodes))]:
- g2 = g.umap(kind=kind, feature_engine="none")
+ g2 = g.umap(kind=kind, feature_engine=self.feature_engine)
last_shape = 0
for scale in np.linspace(0, 1, 8):
g3 = g2.filter_weighted_edges(scale=scale)
@@ -450,196 +468,12 @@ def test_filter_edges(self):
last_shape = shape[0]
-class TestUMAPAIMethods(TestUMAPMethods):
- @pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires ai+umap feature dependencies",
- )
- def _test_umap(self, g, use_cols, targets, name, kind, df):
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore", category=UserWarning)
- for scaler in ["kbins", "robust"]:
- for cardinality in [2, 200]:
- for use_ngram in [True, False]:
- for use_col in use_cols:
- for target in targets:
- logger.debug("*" * 90)
- value = [
- scaler,
- cardinality,
- use_ngram,
- target,
- use_col,
- ]
- logger.debug(f"{value}")
- logger.debug("-" * 80)
-
- g2 = g.umap(
- kind=kind,
- X=use_col,
- y=target,
- model_name=model_avg_name,
- use_scaler=scaler,
- use_scaler_target=scaler,
- use_ngrams=use_ngram,
- engine="umap_learn",
- cardinality_threshold=cardinality,
- cardinality_threshold_target=cardinality,
- n_neighbors=3,
- dbscan=False,
- )
-
- self.cases_test_graph(g2, kind=kind, df=df)
-
- @pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires ai+umap feature dependencies",
- )
- def test_node_umap(self):
- g = graphistry.nodes(ndf_reddit)
- use_cols = [None, text_cols_reddit, good_cols_reddit, meta_cols_reddit]
- targets = [None, single_target_reddit, double_target_reddit]
-
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore", category=UserWarning)
- warnings.filterwarnings("ignore", category=DeprecationWarning)
- warnings.filterwarnings("ignore", category=FutureWarning)
-
- self._test_umap(
- g,
- use_cols=use_cols,
- targets=targets,
- name="Node UMAP with `(target, use_col)=`",
- kind="nodes",
- df=ndf_reddit,
- )
+@parameterized_class([{"feature_engine": fe, "engine": ge} for fe in feature_engines for ge in engines])
+class TestUMAPAICUMLMethods(TestUMAPMethods):
@pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires ai+umap feature dependencies",
- )
- def test_edge_umap(self):
- g = graphistry.edges(edge_df2, "src", "dst")
- targets = [None, "label"]
- use_cols = [None, "title"]
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore", category=UserWarning)
- warnings.filterwarnings("ignore", category=DeprecationWarning)
- warnings.filterwarnings("ignore", category=FutureWarning)
-
- self._test_umap(
- g,
- use_cols=use_cols,
- targets=targets,
- name="Edge UMAP with `(target, use_col)=`",
- kind="edges",
- df=edge_df2,
- )
-
- @pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires ai+umap feature dependencies",
- )
- def test_chaining_nodes(self):
- g = graphistry.nodes(ndf_reddit)
- g2 = g.umap(dbscan=False)
-
- logger.debug("======= g.umap() done ======")
- g3a = g2.featurize()
- logger.debug("======= g3a.featurize() done ======")
- g3 = g3a.umap(dbscan=False)
- logger.debug("======= g3.umap() done ======")
- assert g2._node_features.shape == g3._node_features.shape
- # since g3 has feature params with x and y.
- g3._feature_params["nodes"]["X"].pop("x")
- g3._feature_params["nodes"]["X"].pop("y")
- assert all(g2._feature_params["nodes"]["X"] == g3._feature_params["nodes"]["X"])
- assert (
- g2._feature_params["nodes"]["y"].shape == g3._feature_params["nodes"]["y"].shape
- ) # None
- assert g2._node_embedding.shape == g3._node_embedding.shape # kinda weak sauce
-
- @pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires ai+umap feature dependencies",
- )
- def test_chaining_edges(self):
- g = graphistry.edges(edge_df, "src", "dst")
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore", category=UserWarning)
- warnings.filterwarnings("ignore", category=DeprecationWarning)
- warnings.filterwarnings("ignore", category=FutureWarning)
- g2 = g.umap(kind="edges", dbscan=False)
- g3 = g.featurize(kind="edges").umap(kind="edges", dbscan=False)
-
- assert all(g2._feature_params["edges"]["X"] == g3._feature_params["edges"]["X"])
- assert all(
- g2._feature_params["edges"]["y"] == g3._feature_params["edges"]["y"]
- ) # None
- assert all(g2._edge_features == g3._edge_features)
-
- @pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires ai+umap feature dependencies",
- )
- def test_feature_kwargs_yield_different_values_using_umap_api(self):
- g = graphistry.nodes(ndf_reddit)
- n_topics_target = 6
-
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore", category=UserWarning)
- warnings.filterwarnings("ignore", category=DeprecationWarning)
- warnings.filterwarnings("ignore", category=FutureWarning)
-
- g2 = g.umap(
- X="type",
- y="label",
- cardinality_threshold_target=3,
- n_topics_target=n_topics_target,
- ) # makes a GapEncoded Target
- g3 = g.umap(
- X="type", y="label", cardinality_threshold_target=30000
- ) # makes a one-hot-encoded target
-
- assert all(
- g2._feature_params["nodes"]["X"] == g3._feature_params["nodes"]["X"]
- ), "features should be the same"
- assert all(
- g2._feature_params["nodes"]["y"] != g3._feature_params["nodes"]["y"]
- ), "targets in memoize should be different" # None
- assert (
- g2._node_target.shape[1] != g3._node_target.shape[1]
- ), "Targets should be different"
- assert g2._node_target.shape[1] == n_topics_target, "Targets "
-
- @pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires ai+umap feature dependencies",
- )
- def test_filter_edges(self):
- for kind, g in [("nodes", graphistry.nodes(ndf_reddit))]:
- g2 = g.umap(kind=kind, model_name=model_avg_name)
- last_shape = 0
- for scale in np.linspace(0, 1, 8): # six sigma in 8 steps
- g3 = g2.filter_weighted_edges(scale=scale)
- shape = g3._edges.shape
- logger.debug("*" * 90)
- logger.debug(
- f"{kind} -- scale: {scale}: resulting edges dataframe shape: {shape}"
- )
- logger.debug("-" * 80)
- self.assertGreaterEqual(shape[0], last_shape)
- last_shape = shape[0]
-
-
-@pytest.mark.skipif(
- not has_dependancy or not has_cuml,
- reason="requires cuml feature dependencies",
-)
-class TestCUMLMethods(TestUMAPMethods):
- @pytest.mark.skipif(
- not has_dependancy or not has_cuml,
- reason="requires cuml feature dependencies",
+ not umap or not dirty_cat,
+ reason="requires feature_engine and umap_engine dependencies",
)
def _test_umap(self, g, use_cols, targets, name, kind, df):
with warnings.catch_warnings():
@@ -666,9 +500,10 @@ def _test_umap(self, g, use_cols, targets, name, kind, df):
y=target,
model_name=model_avg_name,
use_scaler=scaler,
- use_scaler_target=scaler,
+ # use_scaler_target=scaler,
use_ngrams=use_ngram,
- engine="cuml",
+ engine=self.engine,
+ feature_engine = self.feature_engine,
cardinality_threshold=cardinality,
cardinality_threshold_target=cardinality,
n_neighbors=3,
@@ -677,13 +512,13 @@ def _test_umap(self, g, use_cols, targets, name, kind, df):
self.cases_test_graph(g2, kind=kind, df=df)
@pytest.mark.skipif(
- not has_dependancy or not has_cuml,
- reason="requires cuml feature dependencies",
+ not umap or not dirty_cat,
+ reason="requires feature_engine and umap_engine dependencies",
)
def test_node_umap(self):
g = graphistry.nodes(ndf_reddit)
use_cols = [None, text_cols_reddit, good_cols_reddit, meta_cols_reddit]
- targets = [None, single_target_reddit, double_target_reddit]
+ targets = [single_target_reddit, double_target_reddit] # , None] cuml cant handle None here
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
@@ -700,12 +535,12 @@ def test_node_umap(self):
)
@pytest.mark.skipif(
- not has_dependancy or not has_cuml,
- reason="requires cuml feature dependencies",
+ not umap or not dirty_cat,
+ reason="requires feature_engine and umap_engine dependencies",
)
def test_edge_umap(self):
g = graphistry.edges(edge_df2, "src", "dst")
- targets = [None, "label"]
+ targets = ["label"] # cuml cant handle None here
use_cols = [None, "title"]
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
@@ -722,31 +557,34 @@ def test_edge_umap(self):
)
@pytest.mark.skipif(
- not has_dependancy or not has_cuml,
- reason="requires cuml feature dependencies",
+ not umap or not dirty_cat,
+ reason="requires feature_engine and umap_engine dependencies",
)
def test_chaining_nodes(self):
g = graphistry.nodes(ndf_reddit)
- g2 = g.umap()
+ g2 = g.umap(feature_engine = self.feature_engine, engine = self.engine)
logger.debug("======= g.umap() done ======")
- g3a = g2.featurize()
+ g3a = g2.featurize(feature_engine = self.feature_engine, engine = self.engine)
logger.debug("======= g3a.featurize() done ======")
- g3 = g3a.umap()
+ g3 = g3a.umap(feature_engine = self.feature_engine, engine = self.engine)
logger.debug("======= g3.umap() done ======")
assert g2._node_features.shape == g3._node_features.shape, f"featurize() should be idempotent, found {g2._node_features.shape} != {g3._node_features.shape}"
# since g3 has feature params with x and y.
g3._feature_params["nodes"]["X"].pop("x")
g3._feature_params["nodes"]["X"].pop("y")
- assert all(g2._feature_params["nodes"]["X"] == g3._feature_params["nodes"]["X"])
+ if self.feature_engine == 'cu_cat':
+ assert all(g2._feature_params["nodes"]["X"].to_pandas() == g3._feature_params["nodes"]["X"].to_pandas() )
+ else:
+ assert all(g2._feature_params["nodes"]["X"] == g3._feature_params["nodes"]["X"])
assert (
g2._feature_params["nodes"]["y"].shape == g3._feature_params["nodes"]["y"].shape
) # None
assert g2._node_embedding.shape == g3._node_embedding.shape # kinda weak sauce
@pytest.mark.skipif(
- not has_dependancy or not has_cuml,
- reason="requires cuml feature dependencies",
+ not umap or not dirty_cat,
+ reason="requires feature_engine and umap_engine dependencies",
)
def test_chaining_edges(self):
g = graphistry.edges(edge_df, "src", "dst")
@@ -754,8 +592,8 @@ def test_chaining_edges(self):
warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=FutureWarning)
- g2 = g.umap(kind="edges")
- g3 = g.featurize(kind="edges").umap(kind="edges")
+ g2 = g.umap(kind="edges",feature_engine = self.feature_engine, engine = self.engine)
+ g3 = g.featurize(kind="edges").umap(kind="edges",feature_engine = self.feature_engine, engine = self.engine)
assert all(g2._feature_params["edges"]["X"] == g3._feature_params["edges"]["X"])
assert all(
@@ -764,8 +602,8 @@ def test_chaining_edges(self):
assert all(g2._edge_features == g3._edge_features)
@pytest.mark.skipif(
- not has_dependancy or not has_cuml,
- reason="requires cuml feature dependencies",
+ not umap or not dirty_cat,
+ reason="requires feature_engine and umap_engine dependencies",
)
def test_feature_kwargs_yield_different_values_using_umap_api(self):
g = graphistry.nodes(ndf_reddit)
@@ -779,11 +617,13 @@ def test_feature_kwargs_yield_different_values_using_umap_api(self):
g2 = g.umap(
X="type",
y="label",
+ feature_engine = self.feature_engine,
+ engine = self.engine,
cardinality_threshold_target=3,
n_topics_target=n_topics_target,
) # makes a GapEncoded Target
g3 = g.umap(
- X="type", y="label", cardinality_threshold_target=30000
+ X="type", y="label", feature_engine = self.feature_engine, engine = self.engine,cardinality_threshold_target=30000
) # makes a one-hot-encoded target
assert all(
@@ -798,12 +638,12 @@ def test_feature_kwargs_yield_different_values_using_umap_api(self):
assert g2._node_target.shape[1] == n_topics_target, "Targets "
@pytest.mark.skipif(
- not has_dependancy or not has_umap,
- reason="requires cuml feature dependencies",
+ not umap or not dirty_cat,
+ reason="requires feature_engine and umap_engine dependencies",
)
def test_filter_edges(self):
for kind, g in [("nodes", graphistry.nodes(ndf_reddit))]:
- g2 = g.umap(kind=kind, model_name=model_avg_name)
+ g2 = g.umap(kind=kind, feature_engine = self.feature_engine, engine = self.engine,model_name=model_avg_name)
last_shape = 0
for scale in np.linspace(0, 1, 8): # six sigma in 8 steps
g3 = g2.filter_weighted_edges(scale=scale)
@@ -816,6 +656,8 @@ def test_filter_edges(self):
self.assertGreaterEqual(shape[0], last_shape)
last_shape = shape[0]
+
+@parameterized_class([{"feature_engine": fe} for fe in feature_engines])
class TestCudfUmap(unittest.TestCase):
# temporary tests for cudf pass thru umap
@pytest.mark.skipif(not is_test_cudf, reason="requires cudf")
@@ -826,11 +668,11 @@ def setUp(self):
df['profile'] = np.random.randint(0,1000,size=(self.samples, 1))
self.df = cudf.from_pandas(df)
- @pytest.mark.skipif(not has_dependancy or not has_cuml, reason="requires cuml dependencies")
- @pytest.mark.skipif(not is_test_cudf, reason="requires cudf")
+ @pytest.mark.skipif(not umap, reason="requires umap")
+ @pytest.mark.skipif(not cuml, reason="requires cuml")
def test_base(self):
- graphistry.nodes(self.df).umap('auto')._node_embedding.shape == (self.samples, 2)
- graphistry.nodes(self.df).umap('engine')._node_embedding.shape == (self.samples, 2)
+ graphistry.nodes(self.df).umap(engine = 'umap')._node_embedding.shape == (self.samples, 2)
+ graphistry.nodes(self.df).umap(engine = 'cuml')._node_embedding.shape == (self.samples, 2)
if __name__ == "__main__":
diff --git a/graphistry/umap_utils.py b/graphistry/umap_utils.py
index d2561739df..7dd152f500 100644
--- a/graphistry/umap_utils.py
+++ b/graphistry/umap_utils.py
@@ -12,6 +12,7 @@
resolve_feature_engine)
from .PlotterBase import Plottable, WeakValueDictionary
from .util import check_set_memoize
+from .dep_manager import deps
import logging
@@ -25,66 +26,29 @@
###############################################################################
-
-def lazy_umap_import_has_dependancy():
- try:
- import warnings
-
- warnings.filterwarnings("ignore")
- import umap # noqa
-
- return True, "ok", umap
- except ModuleNotFoundError as e:
- return False, e, None
-
-
-def lazy_cuml_import_has_dependancy():
- try:
- import warnings
-
- warnings.filterwarnings("ignore")
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore")
- import cuml # type: ignore
-
- return True, "ok", cuml
- except ModuleNotFoundError as e:
- return False, e, None
-
-def lazy_cudf_import_has_dependancy():
- try:
- import warnings
-
- warnings.filterwarnings("ignore")
- import cudf # type: ignore
-
- return True, "ok", cudf
- except ModuleNotFoundError as e:
- return False, e, None
-
def assert_imported():
- has_dependancy_, import_exn, _ = lazy_umap_import_has_dependancy()
- if not has_dependancy_:
+ umap_ = deps.umap
+ if not umap_:
logger.error("UMAP not found, trying running " "`pip install graphistry[ai]`")
- raise import_exn
+ # raise import_exn
def assert_imported_cuml():
- has_cuml_dependancy_, import_cuml_exn, _ = lazy_cuml_import_has_dependancy()
- if not has_cuml_dependancy_:
+ cuml_ = deps.cuml
+ if not cuml_:
logger.warning("cuML not found, trying running " "`pip install cuml`")
- raise import_cuml_exn
+ # raise import_cuml_exn
def is_legacy_cuml():
try:
- import cuml
-
- vs = cuml.__version__.split(".")
- if (vs[0] in ["0", "21"]) or (vs[0] == "22" and float(vs[1]) < 6):
- return True
- else:
- return False
+ cuml = deps.cuml
+ if cuml: # noqa
+ vs = cuml.__version__.split(".")
+ if (vs[0] in ["0", "21"]) or (vs[0] == "22" and float(vs[1]) < 6):
+ return True
+ else:
+ return False
except ModuleNotFoundError:
return False
@@ -99,11 +63,11 @@ def resolve_umap_engine(
if engine in [CUML, UMAP_LEARN]:
return engine # type: ignore
if engine in ["auto"]:
- has_cuml_dependancy_, _, _ = lazy_cuml_import_has_dependancy()
- if has_cuml_dependancy_:
+ cuml_ = deps.cuml
+ if cuml_:
return 'cuml'
- has_umap_dependancy_, _, _ = lazy_umap_import_has_dependancy()
- if has_umap_dependancy_:
+ umap_ = deps.umap
+ if umap_:
return 'umap_learn'
raise ValueError( # noqa
@@ -113,9 +77,10 @@ def resolve_umap_engine(
)
-def make_safe_gpu_dataframes(X, y, engine):
+def make_safe_gpu_dataframes(X, y, engine, has_cudf):
def safe_cudf(X, y):
+ cudf = deps.cudf
# remove duplicate columns
if len(X.columns) != len(set(X.columns)):
X = X.loc[:, ~X.columns.duplicated()]
@@ -128,14 +93,17 @@ def safe_cudf(X, y):
for key, value in kwargs.items():
if isinstance(value, cudf.DataFrame) and engine in ["pandas", "umap_learn", "dirty_cat"]:
new_kwargs[key] = value.to_pandas()
- elif isinstance(value, pd.DataFrame) and engine in ["cuml", "cu_cat"]:
+ elif 'cupy' in str(getmodule(value)) and engine in ["pandas", "umap_learn", "dirty_cat"]:
+ new_kwargs[key] = pd.DataFrame(value.get())
+ elif 'cupy' in str(getmodule(value)) and engine in ["cuml", "cu_cat", "cuda"]:
+ new_kwargs[key] = cudf.DataFrame(value)
+ elif isinstance(value, pd.DataFrame) and engine in ["cuml", "cu_cat", "cuda"]:
new_kwargs[key] = cudf.from_pandas(value)
else:
new_kwargs[key] = value
return new_kwargs['X'], new_kwargs['y']
-
- has_cudf_dependancy_, _, cudf = lazy_cudf_import_has_dependancy()
- if has_cudf_dependancy_:
+
+ if has_cudf:
return safe_cudf(X, y)
else:
return X, y
@@ -203,9 +171,9 @@ def umap_lazy_init(
engine_resolved = resolve_umap_engine(engine)
# FIXME remove as set_new_kwargs will always replace?
if engine_resolved == UMAP_LEARN:
- _, _, umap_engine = lazy_umap_import_has_dependancy()
+ umap_engine = deps.umap
elif engine_resolved == CUML:
- _, _, umap_engine = lazy_cuml_import_has_dependancy()
+ umap_engine = deps.cuml
else:
raise ValueError(
"No umap engine, ensure 'auto', 'umap_learn', or 'cuml', and the library is installed"
@@ -316,6 +284,7 @@ def transform_umap(self, df: pd.DataFrame,
merge_policy: bool = False,
sample: Optional[int] = None,
return_graph: bool = True,
+ engine: UMAPEngine = 'auto',
fit_umap_embedding: bool = True,
verbose: bool = False
) -> Union[Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame], Plottable]:
@@ -335,14 +304,16 @@ def transform_umap(self, df: pd.DataFrame,
fit_umap_embedding: Whether to infer graph from the UMAP embedding on the new data, default True
verbose: Whether to print information about the graph inference
"""
- df, y = make_safe_gpu_dataframes(df, y, 'pandas')
+ df, y = make_safe_gpu_dataframes(df, y, engine, self.has_cudf)
X, y_ = self.transform(df, y, kind=kind, return_graph=False, verbose=verbose)
- X, y_ = make_safe_gpu_dataframes(X, y_, self.engine) # type: ignore
- emb = self._umap.transform(X) # type: ignore
+ try: # cuml has reproducibility issues with fit().transform() vs .fit_transform()
+ emb = self._umap.transform(X) # type: ignore
+ except:
+ emb = self._umap.fit_transform(X) # type: ignore
emb = self._bundle_embedding(emb, index=df.index)
if return_graph and kind not in ["edges"]:
- emb, _ = make_safe_gpu_dataframes(emb, None, 'pandas') # for now so we don't have to touch infer_edges, force to pandas
- X, y_ = make_safe_gpu_dataframes(X, y_, 'pandas')
+ emb, _ = make_safe_gpu_dataframes(emb, None, 'pandas', self.has_cudf) # for now so we don't have to touch infer_edges, force to pandas
+ # X, y_ = make_safe_gpu_dataframes(X, y_, self.engine, self.has_cudf)
g = self._infer_edges(emb, X, y_, df,
infer_on_umap_embedding=fit_umap_embedding, merge_policy=merge_policy,
eps=min_dist, sample=sample, n_neighbors=n_neighbors,
@@ -352,9 +323,9 @@ def transform_umap(self, df: pd.DataFrame,
def _bundle_embedding(self, emb, index):
# Converts Embedding into dataframe and takes care if emb.dim > 2
- if emb.shape[1] == 2 and 'cudf.core.dataframe' not in str(getmodule(emb)) and not hasattr(emb, 'device'):
+ if emb.shape[1] == 2 and 'cudf' not in str(getmodule(emb)) and not hasattr(emb, 'device'):
emb = pd.DataFrame(emb, columns=[config.X, config.Y], index=index)
- elif emb.shape[1] == 2 and 'cudf.core.dataframe' in str(getmodule(emb)):
+ elif emb.shape[1] == 2 and 'cudf' in str(getmodule(emb)):
emb.rename(columns={0: config.X, 1: config.Y}, inplace=True)
elif emb.shape[1] == 2 and hasattr(emb, 'device'):
import cudf
@@ -363,9 +334,16 @@ def _bundle_embedding(self, emb, index):
columns = [config.X, config.Y] + [
f"umap_{k}" for k in range(2, emb.shape[1])
]
- if 'cudf.core.dataframe' not in str(getmodule(emb)):
+ if 'cudf' not in str(getmodule(emb)) and 'cupy' not in str(getmodule(emb)):
emb = pd.DataFrame(emb, columns=columns, index=index)
- elif 'cudf.core.dataframe' in str(getmodule(emb)):
+ elif 'ndarray' in str(getmodule(emb)) or 'None' in str(getmodule(emb)):
+ try:
+ emb = pd.DataFrame(emb)
+ emb.columns = columns
+ except:
+ emb = cudf.DataFrame(emb)
+ emb.columns = columns
+ else:
emb.columns = columns
return emb
@@ -412,8 +390,27 @@ def _process_umap(
print('** Fitting UMAP') if verbose else None
res = res.umap_lazy_init(res, verbose=verbose, **umap_kwargs_pure)
+ self.datetime_columns = X_.select_dtypes(
+ include=["datetime", "datetimetz"]
+ ).columns.to_list()
+
+ self.R_ = X_[self.datetime_columns]
+ X_ = X_.drop(columns=self.datetime_columns)
+
emb = res._umap_fit_transform(X_, y_, verbose=verbose)
- res._xy = emb
+ if 'dataframe' not in str(getmodule(emb)) or 'DataFrame' not in str(getmodule(emb)):
+ cudf = deps.cudf
+ if cudf:
+ try:
+ emb = cudf.DataFrame(emb)
+ self.R_ = cudf.DataFrame(self.R_)
+ except TypeError:
+ emb = cudf.DataFrame(emb.blocks[0].values)
+ self.R_ = cudf.DataFrame(self.R_.blocks[0].values)
+ else:
+ emb = pd.DataFrame(emb)
+ self.R_ = pd.DataFrame(self.R_)
+ res._xy = emb.join(self.R_)
return res
def _set_features( # noqa: E303
@@ -554,9 +551,10 @@ def umap(
logger.debug("umap_kwargs: %s", umap_kwargs)
# temporary until we have full cudf support in feature_utils.py
- has_cudf, _, cudf = lazy_cudf_import_has_dependancy()
+ self.has_cudf = deps.cudf
+ cudf = deps.cudf
- if has_cudf:
+ if self.has_cudf:
flag_nodes_cudf = isinstance(self._nodes, cudf.DataFrame)
flag_edges_cudf = isinstance(self._edges, cudf.DataFrame)
@@ -614,11 +612,11 @@ def umap(
logger.debug("data is type :: %s", (type(X_)))
if isinstance(X_, pd.DataFrame):
index_to_nodes_dict = dict(zip(range(len(nodes)), nodes))
- elif 'cudf.core.dataframe' in str(getmodule(X_)):
+ elif 'cudf' in str(getmodule(X_)):
index_to_nodes_dict = nodes # {}?
# add the safe coercion here
- X_, y_ = make_safe_gpu_dataframes(X_, y_, res.engine) # type: ignore
+ X_, y_ = make_safe_gpu_dataframes(X_, y_, res.engine, self.has_cudf) # type: ignore
res = res._process_umap(
res, X_, y_, kind, memoize, featurize_kwargs, verbose, **umap_kwargs
@@ -648,7 +646,7 @@ def umap(
)
# add the safe coercion here
- X_, y_ = make_safe_gpu_dataframes(X_, y_, res.engine) # type: ignore
+ X_, y_ = make_safe_gpu_dataframes(X_, y_, res.engine, self.has_cudf) # type: ignore
res = res._process_umap(
res, X_, y_, kind, memoize, featurize_kwargs, **umap_kwargs
@@ -721,11 +719,19 @@ def _bind_xy_from_umap(
emb = res._edge_embedding
if isinstance(df, type(emb)):
- df[x_name] = emb.values.T[0]
- df[y_name] = emb.values.T[1]
- elif isinstance(df, pd.DataFrame) and 'cudf.core.dataframe' in str(getmodule(emb)):
- df[x_name] = emb.to_numpy().T[0]
- df[y_name] = emb.to_numpy().T[1]
+ try:
+ df[x_name] = emb.values.T[0]
+ df[y_name] = emb.values.T[1]
+ except ValueError:
+ df[x_name] = emb.values[0]
+ df[y_name] = emb.values[1]
+ elif isinstance(df, pd.DataFrame) and 'cudf' in str(getmodule(emb)):
+ try:
+ df[x_name] = emb.to_numpy().T[0]
+ df[y_name] = emb.to_numpy().T[1]
+ except ValueError:
+ df[x_name] = emb.to_numpy()[0]
+ df[y_name] = emb.to_numpy()[1]
res = res.nodes(df) if kind == "nodes" else res.edges(df)
diff --git a/mypy.ini b/mypy.ini
index 898e001146..5b4403e91f 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -94,3 +94,6 @@ ignore_missing_imports = True
[mypy-cuml.*]
ignore_missing_imports = True
+
+[mypy-cu_cat.*]
+ignore_missing_imports = true
diff --git a/setup.py b/setup.py
index c81db1b09c..14bd79b731 100755
--- a/setup.py
+++ b/setup.py
@@ -27,7 +27,7 @@ def unique_flatten_dict(d):
dev_extras = {
'docs': ['sphinx==3.4.3', 'docutils==0.16', 'sphinx_autodoc_typehints==1.11.1', 'sphinx-rtd-theme==0.5.1', 'Jinja2<3.1'],
- 'test': ['flake8>=5.0', 'mock', 'mypy', 'pytest'] + stubs + test_workarounds,
+ 'test': ['flake8>=5.0', 'mock', 'mypy', 'pytest', 'parameterized'] + stubs + test_workarounds,
'testai': [
'numba>=0.57.1' # https://github.com/numba/numba/issues/8615
],
@@ -44,10 +44,11 @@ def unique_flatten_dict(d):
}
base_extras_heavy = {
- 'umap-learn': ['umap-learn', 'dirty-cat==0.2.0', 'scikit-learn>=1.0'],
+ 'umap-learn': ['umap-learn', 'dirty-cat', 'scikit-learn==1.3.2'],
}
# https://github.com/facebookresearch/faiss/issues/1589 for faiss-cpu 1.6.1, #'setuptools==67.4.0' removed
base_extras_heavy['ai'] = base_extras_heavy['umap-learn'] + ['scipy', 'dgl', 'torch<2', 'sentence-transformers', 'faiss-cpu', 'joblib']
+base_extras_heavy['cu-cat'] = ['cu-cat']
base_extras = {**base_extras_light, **base_extras_heavy}