Merge pull request #1 from christiandietze/main

Do not deliver Spark dependency

Makefile

@@ -1,11 +1,16 @@
-.PHONY: setup test
+.PHONY: setup-environment test
+PYTHON=$(shell brew --prefix)/opt/python@3.7/bin/python3
+
+install-python:
+ brew install python@3.7
 
 setup-environment:
- pip3 install virtualenv==20.0.15
- virtualenv env
- source env/bin/activate; \
- pip3 install -r requirements.txt
+ $(PYTHON) -m pip install virtualenv==20.0.15
+ $(PYTHON) -m virtualenv env
+ . env/bin/activate; \
+ pip3 install -r requirements.txt; \
+ pip3 install pyspark==2.4.4
 
-test:
- source env/bin/activate; \
+test: setup-environment
+ . env/bin/activate; \
  SPARK_LOCAL_IP=127.0.0.1 PYTHONPATH=./src python -m pytest $(target) -v

README.md

@@ -29,6 +29,12 @@ def main():
  Job(__package__).execute()
 ```
 
+## How to test
+
+The tests currently run against Spark 2.4, which does not work with Python versions > 3.7 ([reference](https://issues.apache.org/jira/browse/SPARK-29536)). For that reason we expect Python 3.7 to be installed with Homebrew.
+To run the tests, first install Python 3.7 with Homebrew (`make install-python`), then run the tests with `make test`. If you're not using Homebrew you can override the `PYTHON` variable as such: `make test PYTHON=/path/to/python3`.
+
+
 ## How to contribute
 
 If you want to contribute, please submit a Pull Request! Thank you :)

requirements.txt

@@ -1,4 +1,3 @@
-pyspark==3.0.1
 pytest==6.1.2
 pytest-spark==0.6.0
 testfixtures==6.15.0

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
  name="pyspark-core-utils",
- version="1.0.0",
+ version="1.0.1",
  author="ImmobilienScout24",
  description="PySpark core utils library",
  long_description=long_description,