Merge branch 'master' of https://github.com/dmlc/xgboost into rank_me…

…tric

python-package/xgboost/compat.py

@@ -79,6 +79,14 @@ def os_fspath(path):
 # END NUMPY PATHLIB ATTRIBUTION
 ###############################################################################
 
+
+def lazy_isinstance(instance, module, name):
+    '''Use string representation to identify a type.'''
+    module = type(instance).__module__ == module
+    name = type(instance).__name__ == name
+    return module and name
+
+
 # pandas
 try:
     from pandas import DataFrame, Series
@@ -95,27 +103,6 @@ def os_fspath(path):
     pandas_concat = None
     PANDAS_INSTALLED = False
 
-# dt
-try:
-    # Workaround for #4473, compatibility with dask
-    if sys.__stdin__ is not None and sys.__stdin__.closed:
-        sys.__stdin__ = None
-    import datatable
-
-    if hasattr(datatable, "Frame"):
-        DataTable = datatable.Frame
-    else:
-        DataTable = datatable.DataTable
-    DT_INSTALLED = True
-except ImportError:
-
-    # pylint: disable=too-few-public-methods
-    class DataTable(object):
-        """ dummy for datatable.DataTable """
-
-    DT_INSTALLED = False
-
-
 # cudf
 try:
     from cudf import DataFrame as CUDF_DataFrame

python-package/xgboost/core.py

@@ -19,9 +19,9 @@
 
 from .compat import (
     STRING_TYPES, DataFrame, MultiIndex, Int64Index, py_str,
-    PANDAS_INSTALLED, DataTable,
-    CUDF_INSTALLED, CUDF_DataFrame, CUDF_Series, CUDF_MultiIndex,
-    os_fspath, os_PathLike)
+    PANDAS_INSTALLED, CUDF_INSTALLED,
+    CUDF_DataFrame, CUDF_Series, CUDF_MultiIndex,
+    os_fspath, os_PathLike, lazy_isinstance)
 from .libpath import find_lib_path
 
 # c_bst_ulong corresponds to bst_ulong defined in xgboost/c_api.h
@@ -319,7 +319,8 @@ def _maybe_cudf_dataframe(data, feature_names, feature_types):
 def _maybe_dt_data(data, feature_names, feature_types,
                    meta=None, meta_type=None):
     """Validate feature names and types if data table"""
-    if not isinstance(data, DataTable):
+    if (not lazy_isinstance(data, 'datatable', 'Frame') and
+            not lazy_isinstance(data, 'datatable', 'DataTable')):
         return data, feature_names, feature_types
 
     if meta and data.shape[1] > 1:
@@ -470,7 +471,7 @@ def __init__(self, data, label=None, weight=None, base_margin=None,
             self._init_from_csc(data)
         elif isinstance(data, np.ndarray):
             self._init_from_npy2d(data, missing, nthread)
-        elif isinstance(data, DataTable):
+        elif lazy_isinstance(data, 'datatable', 'Frame'):
             self._init_from_dt(data, nthread)
         elif hasattr(data, "__cuda_array_interface__"):
             self._init_from_array_interface(data, missing, nthread)
@@ -1052,7 +1053,7 @@ def __init__(self, params=None, cache=(), model_file=None):
             _check_call(
                 _LIB.XGBoosterUnserializeFromBuffer(self.handle, ptr, length))
             self.__dict__.update(state)
-        elif isinstance(model_file, (STRING_TYPES, os_PathLike)):
+        elif isinstance(model_file, (STRING_TYPES, os_PathLike, bytearray)):
             self.load_model(model_file)
         elif model_file is None:
             pass
@@ -1512,7 +1513,8 @@ def save_raw(self):
         return ctypes2buffer(cptr, length.value)
 
     def load_model(self, fname):
-        """Load the model from a file, local or as URI.
+        """Load the model from a file or bytearray. Path to file can be local
+        or as an URI.
 
         The model is loaded from an XGBoost format which is universal among the
         various XGBoost interfaces. Auxiliary attributes of the Python Booster
@@ -1530,6 +1532,12 @@ def load_model(self, fname):
             # from URL.
             _check_call(_LIB.XGBoosterLoadModel(
                 self.handle, c_str(os_fspath(fname))))
+        elif isinstance(fname, bytearray):
+            buf = fname
+            length = c_bst_ulong(len(buf))
+            ptr = (ctypes.c_char * len(buf)).from_buffer(buf)
+            _check_call(_LIB.XGBoosterLoadModelFromBuffer(self.handle, ptr,
+                                                          length))
         else:
             raise TypeError('Unknown file type: ', fname)
 

tests/python/test_basic.py

@@ -36,6 +36,11 @@ def captured_output():
 
 
 class TestBasic(unittest.TestCase):
+    def test_compat(self):
+        from xgboost.compat import lazy_isinstance
+        a = np.array([1, 2, 3])
+        assert lazy_isinstance(a, 'numpy', 'ndarray')
+        assert not lazy_isinstance(a, 'numpy', 'dataframe')
 
     def test_basic(self):
         dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')

tests/python/test_basic_models.py

@@ -300,6 +300,13 @@ def test_model_binary_io(self):
         assert float(config['learner']['objective'][
             'reg_loss_param']['scale_pos_weight']) == 0.5
 
+        buf = bst.save_raw()
+        from_raw = xgb.Booster()
+        from_raw.load_model(buf)
+
+        buf_from_raw = from_raw.save_raw()
+        assert buf == buf_from_raw
+
     def test_model_json_io(self):
         loc = locale.getpreferredencoding(False)
         model_path = 'test_model_json_io.json'

tests/python/testing.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-from xgboost.compat import SKLEARN_INSTALLED, PANDAS_INSTALLED, DT_INSTALLED
+from xgboost.compat import SKLEARN_INSTALLED, PANDAS_INSTALLED
 from xgboost.compat import CUDF_INSTALLED, DASK_INSTALLED
 
 
@@ -19,7 +19,9 @@ def no_pandas():
 
 
 def no_dt():
-    return {'condition': not DT_INSTALLED,
+    import importlib.util
+    spec = importlib.util.find_spec('datatable')
+    return {'condition': spec is None,
             'reason': 'Datatable is not installed.'}