diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py index d530723ca980..b548387e92db 100644 --- a/python/pyspark/__init__.py +++ b/python/pyspark/__init__.py @@ -36,6 +36,30 @@ Finer-grained cache persistence levels. """ +import os, re + + +def get_spark_version(): + """ + Detect version of Spark located under SPARK_HOME. + """ + libs_path = os.path.join(os.environ['SPARK_HOME'], 'lib') + for jar_file in os.listdir(libs_path): + if jar_file.startswith('spark-assembly'): + try: + return re.match('^spark-assembly-(.*)-', jar_file).group(1) + except IndexError: + raise ImportError("Can't detect Spark version by the assembly file") + + raise ImportError("Can't find file with Spark version") + + +__version__ = '2.0.0' +if __version__ != get_spark_version(): + raise ImportError("Incompatible versions of PySpark ({}) and Spark ({})".format( + __version__, get_spark_version() + )) + from pyspark.conf import SparkConf from pyspark.context import SparkContext diff --git a/python/setup.cfg b/python/setup.cfg new file mode 100644 index 000000000000..3c6e79cf31da --- /dev/null +++ b/python/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 000000000000..72afdcbec48c --- /dev/null +++ b/python/setup.py @@ -0,0 +1,42 @@ +from setuptools import setup + +import pyspark + + +setup( + name='pyspark', + version=pyspark.__version__, + packages=[ + 'pyspark', + 'pyspark.mllib', + 'pyspark.ml', + 'pyspark.sql', + 'pyspark.streaming', + ], + install_requires=[ + 'py4j==0.9', + ], + extras_require={ + 'ml': ['numpy>=1.7'], + 'sql': ['pandas'], + }, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Topic :: Software Development', + 'Intended Audience :: Developers', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'License :: OSI Approved :: Apache Software License', + ], + description='Apache Spark Python API', + keywords='spark pyspark', + author='Spark Developers', + author_email='dev@spark.apache.org', + url='https://github.com/apache/spark/tree/master/python', + license='http://www.apache.org/licenses/LICENSE-2.0', +)