[jvm-packages] initial pyspark api (WIP)

jvm-packages/xgboost4j-spark/src/main/resources/ml/dmlc/xgboost4j/scala/spark/__init__.py

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2019 by Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+
+from sparkxgb import xgboost
+
+# Allows Pipeline()/PipelineModel() with XGBoost stages to be loaded from disk.
+# Needed because they try to import Python objects from their Java location.
+sys.modules['ml.dmlc.xgboost4j.scala.spark'] = xgboost

jvm-packages/xgboost4j-spark/src/main/resources/setup.py

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2019 by Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from codecs import open
+from os import path
+from setuptools import setup, find_packages
+
+# Read the long description from README.MD
+here = path.abspath(path.dirname(__file__))
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name='spark-xgboost',
+    version='0.90',
+    description='spark-xgboost is the PySpark package for XGBoost',
+
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    url='https://xgboost.ai/',
+    author='DMLC',
+    classifiers=[
+        # Project Maturity
+        'Development Status :: 5 - Production/Stable',
+
+        # Intended Users
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Build Tools',
+
+        # License
+        'License :: OSI Approved :: Apache Software License',
+
+        # Supported Python Versions
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+    ],
+    keywords='development spark xgboost',
+
+    packages=find_packages(),
+    include_package_data=False
+)

jvm-packages/xgboost4j-spark/src/main/resources/sparkxgb/__init__.py

@@ -0,0 +1,20 @@
+#
+# Copyright (c) 2019 by Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from sparkxgb import xgboost
+from sparkxgb.xgboost import XGBoostClassifier, XGBoostRegressor, XGBoostClassificationModel, XGBoostRegressionModel
+
+__all__ = ["XGBoostClassifier", "XGBoostRegressor", "XGBoostClassificationModel", "XGBoostRegressionModel"]
+__version__ = "0.90"

jvm-packages/xgboost4j-spark/src/main/resources/sparkxgb/common.py

@@ -0,0 +1,85 @@
+#
+# Copyright (c) 2019 by Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import re
+
+from pyspark.ml.param import Params
+from pyspark.ml.util import JavaMLWritable
+from pyspark.ml.wrapper import JavaModel, JavaEstimator
+
+from sparkxgb.util import XGBoostReadable
+
+
+class ParamGettersSetters(Params):
+    """
+    Mixin class used to generate the setters/getters for all params.
+    """
+
+    def _create_param_getters_and_setters(self):
+        for param in self.params:
+            param_name = param.name
+            fg_attr = "get" + re.sub(r"(?:^|_)(.)", lambda m: m.group(1).upper(), param_name)
+            fs_attr = "set" + re.sub(r"(?:^|_)(.)", lambda m: m.group(1).upper(), param_name)
+            # Generates getter and setter only if not exists
+            try:
+                getattr(self, fg_attr)
+            except AttributeError:
+                setattr(self, fg_attr, self._get_param_value(param_name))
+            try:
+                getattr(self, fs_attr)
+            except AttributeError:
+                setattr(self, fs_attr, self._set_param_value(param_name))
+
+    def _get_param_value(self, param_name):
+        def r():
+            try:
+                return self.getOrDefault(param_name)
+            except KeyError:
+                return None
+        return r
+
+    def _set_param_value(self, param_name):
+        def r(v):
+            self.set(self.getParam(param_name), v)
+            return self
+        return r
+
+
+class XGboostEstimator(JavaEstimator, XGBoostReadable, JavaMLWritable, ParamGettersSetters):
+    """
+    Mixin class for XGBoost estimators, like XGBoostClassifier and XGBoostRegressor.
+    """
+
+    def __init__(self, classname):
+        super(XGboostEstimator, self).__init__()
+        self.__class__._java_class_name = classname
+        self._java_obj = self._new_java_obj(classname, self.uid)
+        self._create_params_from_java()
+        self._create_param_getters_and_setters()
+
+
+class XGboostModel(JavaModel, XGBoostReadable, JavaMLWritable, ParamGettersSetters):
+    """
+    Mixin class for XGBoost models, like XGBoostClassificationModel and XGBoostRegressionModel.
+    """
+
+    def __init__(self, classname, java_model=None):
+        super(XGboostModel, self).__init__(java_model=java_model)
+        if classname and not java_model:
+            self.__class__._java_class_name = classname
+            self._java_obj = self._new_java_obj(classname, self.uid)
+        if java_model is not None:
+            self._transfer_params_from_java()
+        self._create_param_getters_and_setters()

jvm-packages/xgboost4j-spark/src/main/resources/sparkxgb/util.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2019 by Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from pyspark.ml.util import JavaMLReadable, JavaMLReader
+
+
+class XGBoostReadable(JavaMLReadable):
+    """
+    Mixin class that provides a read() method for XGBoostReader.
+    """
+
+    @classmethod
+    def read(cls):
+        """Returns an XGBoostReader instance for this class."""
+        return XGBoostReader(cls)
+
+
+class XGBoostReader(JavaMLReader):
+    """
+    A reader mixin class for XGBoost objects.
+    """
+
+    @classmethod
+    def _java_loader_class(cls, clazz):
+        if hasattr(clazz, '_java_class_name') and clazz._java_class_name is not None:
+            return clazz._java_class_name
+        else:
+            return JavaMLReader._java_loader_class(clazz)