Add support to joblib 1.3.0

.github/workflows/main.yml

@@ -25,11 +25,7 @@ jobs:
  - name: Install python packages
  run: |
  pip install joblib>=0.14.0 scikit-learn>=0.23.1 pytest pylint
- if [ "${{ matrix.PYSPARK_VERSION }}" = "3.4.0" ]; then
- pip install https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz
- else
- pip install pyspark==${{ matrix.PYSPARK_VERSION }}
- fi;
+ pip install pyspark==${{ matrix.PYSPARK_VERSION }}
  - name: Run pylint
  run: |
  ./run-pylint.sh

joblibspark/__init__.py

@@ -18,7 +18,7 @@
 """
 Joblib spark backend is a extension for joblib, which make joblib running on spark parallelly.
 """
-__version__ = '0.5.1'
+__version__ = '0.5.2'
 
 
 def register_spark():

joblibspark/backend.py

@@ -25,7 +25,12 @@
 
 from joblib.parallel \
  import AutoBatchingMixin, ParallelBackendBase, register_parallel_backend, SequentialBackend
-from joblib._parallel_backends import SafeFunction
+
+try:
+ from joblib._parallel_backends import SafeFunction
+except ImportError:
+ # joblib >= 1.3.0
+ SafeFunction = None
 
 from py4j.clientserver import ClientServer
 
@@ -202,11 +207,16 @@ def mapper_fn(_):
  except ImportError:
  pass
 
+ if SafeFunction is None:
+ return self._get_pool().apply_async(
+ run_on_worker_and_fetch_result, callback=callback
+ )
+
  return self._get_pool().apply_async(
- SafeFunction(run_on_worker_and_fetch_result),
- callback=callback
+ SafeFunction(run_on_worker_and_fetch_result), callback=callback
  )
 
+
  def get_nested_backend(self):
  """Backend instance to be used by nested Parallel calls.
  For nested backend, always use `SequentialBackend`