From ba08c27f860d699715b3bbbd440b44bf0599eae5 Mon Sep 17 00:00:00 2001
From: Christian Dietze <christian.dietze@scout24.com>
Date: Wed, 5 May 2021 10:25:20 +0200
Subject: [PATCH] Do not deliver Spark dependency

* Remove explicit Spark dependency as we might have different Spark version in the environment using the utils
* Run tests against Spark 2.4.4
* Make sure tests are run with Python 3.7, as Spark 2.4 does not support newer Python versions
---
 Makefile         | 19 ++++++++++++-------
 README.md        |  6 ++++++
 requirements.txt |  1 -
 setup.py         |  2 +-
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 70943f2..9b63063 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,16 @@
-.PHONY: setup test
+.PHONY: setup-environment test
+PYTHON=$(shell brew --prefix)/opt/python@3.7/bin/python3
+
+install-python:
+	brew install python@3.7
 
 setup-environment:
-	pip3 install virtualenv==20.0.15
-	virtualenv env
-	source env/bin/activate; \
-	pip3 install -r requirements.txt
+	$(PYTHON) -m pip install virtualenv==20.0.15
+	$(PYTHON) -m virtualenv env
+	. env/bin/activate; \
+	pip3 install -r requirements.txt; \
+	pip3 install pyspark==2.4.4
 
-test:
-	source env/bin/activate; \
+test: setup-environment
+	. env/bin/activate; \
 	SPARK_LOCAL_IP=127.0.0.1 PYTHONPATH=./src python -m pytest $(target) -v
diff --git a/README.md b/README.md
index 7aa6680..b65d0c2 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,12 @@ def main():
     Job(__package__).execute()
 ```
 
+## How to test
+
+The tests currently run against Spark 2.4, which does not work with Python versions > 3.7 ([reference](https://issues.apache.org/jira/browse/SPARK-29536)). For that reason we expect Python 3.7 to be installed with Homebrew.
+To run the tests, first install Python 3.7 with Homebrew (`make install-python`), then run the tests with `make test`. If you're not using Homebrew you can override the `PYTHON` variable as such: `make test PYTHON=/path/to/python3`.
+
+
 ## How to contribute
 
 If you want to contribute, please submit a Pull Request! Thank you :)
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index bc34682..6300715 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-pyspark==3.0.1
 pytest==6.1.2
 pytest-spark==0.6.0
 testfixtures==6.15.0
diff --git a/setup.py b/setup.py
index 3b72885..be59f32 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pyspark-core-utils",
-    version="1.0.0",
+    version="1.0.1",
     author="ImmobilienScout24",
     description="PySpark core utils library",
     long_description=long_description,