TEDO-1157 upgrade to Spark3

Makefile

@@ -9,7 +9,7 @@ setup-environment:
  $(PYTHON) -m virtualenv env
  . env/bin/activate; \
  pip3 install -r requirements.txt; \
- pip3 install pyspark==2.4.4
+ pip3 install pyspark==3.1.2
 
 test: setup-environment
  . env/bin/activate; \

README.md

@@ -30,8 +30,6 @@ def main():
 ```
 
 ## How to test
-
-The tests currently run against Spark 2.4, which does not work with Python versions > 3.7 ([reference](https://issues.apache.org/jira/browse/SPARK-29536)). For that reason we expect Python 3.7 to be installed with Homebrew.
 To run the tests, first install Python 3.7 with Homebrew (`make install-python`), then run the tests with `make test`. If you're not using Homebrew you can override the `PYTHON` variable as such: `make test PYTHON=/path/to/python3`.
 
 

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
  name="pyspark-core-utils",
- version="1.0.1",
+ version="1.1.0",
  author="ImmobilienScout24",
  description="PySpark core utils library",
  long_description=long_description,
@@ -21,7 +21,7 @@
  ],
  package_dir={"": "src"},
  packages=setuptools.find_packages(where="src"),
- python_requires=">=3.6",
+ python_requires=">=3.7",
  install_requires=["importlib-resources==3.3.0",
  "PyYAML==5.4",
  "dotmap==1.3.23"]