Merge pull request #904 from weixuanfu/fix_dask_ut

Fix dask installation
EpistasisLab · Aug 14, 2019 · 24b4374 · 24b4374
2 parents b0095b4 + 9317856
commit 24b4374
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 22 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -4,10 +4,7 @@ environment:
   matrix:
     - PYTHON_VERSION: 3.7
       MINICONDA: C:/Miniconda36-x64
-      DASK_ML_VERSION: 0.13.0
-    - PYTHON_VERSION: 2.7
-      MINICONDA: C:/Miniconda-x64
-      DASK_ML_VERSION: 0.12.0
+      DASK_ML_VERSION: 1.0.0
 
 platform:
   - x64
@@ -23,8 +20,7 @@ install:
   - conda info -a
   - conda create -q -n test-environment python=%PYTHON_VERSION% numpy scipy scikit-learn nose cython pandas pywin32 joblib
   - activate test-environment
-  - pip install deap tqdm update_checker stopit dask[delayed] cloudpickle==0.5.6
-  - pip install dask_ml==%DASK_ML_VERSION%
+  - pip install deap tqdm update_checker stopit xgboost dask[delayed] dask[dataframe] cloudpickle==0.5.6 fsspec>=0.3.3 dask_ml==%DASK_ML_VERSION%
 
 
 test_script:

diff --git a/.travis.yml b/.travis.yml
@@ -4,24 +4,19 @@ matrix:
   include:
   - name: "Python 3.7 on Xenial Linux"
     dist: xenial        # required for Python >= 3.7
-    env: PYTHON_VERSION="3.7"  DASK_ML_VERSION="0.13.0"
+    env: PYTHON_VERSION="3.7"  DASK_ML_VERSION="1.0.0"
     before_install:
       - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   - name: "Python 3.7 on Xenial Linux with coverage"
     dist: xenial        # required for Python >= 3.7
-    env: PYTHON_VERSION="3.7"  COVERAGE="true"  DASK_ML_VERSION="0.13.0"
-    before_install:
-      - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-  - name: "Python 2.7 on Xenial Linux"
-    dist: xenial
-    env: PYTHON_VERSION="2.7"  DASK_ML_VERSION="0.12.0"
+    env: PYTHON_VERSION="3.7"  COVERAGE="true"  DASK_ML_VERSION="1.0.0"
     before_install:
       - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   - name: "Python 3.7 on macOS"
     os: osx
     osx_image: xcode10.2  # Python 3.7.2 running on macOS 10.14.3
     language: shell       # 'language: python' is an error on Travis CI macOS
-    env: PYTHON_VERSION="3.7"  DASK_ML_VERSION="0.13.0"
+    env: PYTHON_VERSION="3.7"  DASK_ML_VERSION="1.0.0"
     before_install:
       - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh
 install: source ./ci/.travis_install.sh

diff --git a/ci/.travis_install.sh b/ci/.travis_install.sh
@@ -38,8 +38,7 @@ conda create -n testenv --yes python=$PYTHON_VERSION pip nose \
 source activate testenv
 
 pip install deap tqdm update_checker stopit \
-    dask[delayed] xgboost cloudpickle==0.5.6
-pip install dask_ml==$DASK_ML_VERSION
+    dask[delayed] dask[dataframe] xgboost cloudpickle==0.5.6 dask_ml==$DASK_ML_VERSION fsspec>=0.3.3
 
 if [[ "$COVERAGE" == "true" ]]; then
     pip install coverage coveralls

diff --git a/docs_sources/installing.md b/docs_sources/installing.md
@@ -46,10 +46,10 @@ pip install xgboost
 
 If you have issues installing XGBoost, check the [XGBoost installation documentation](http://xgboost.readthedocs.io/en/latest/build.html).
 
-If you plan to use [Dask](http://dask.pydata.org/en/latest/) for parallel training, make sure to install [dask[delay]](http://dask.pydata.org/en/latest/install.html) and [dask_ml](https://dask-ml.readthedocs.io/en/latest/install.html).
+If you plan to use [Dask](http://dask.pydata.org/en/latest/) for parallel training, make sure to install [dask[delay] and dask[dataframe]](https://docs.dask.org/en/latest/install.html) and [dask_ml](https://dask-ml.readthedocs.io/en/latest/install.html).
 
 ```Shell
-pip install dask[delayed] dask-ml
+pip install dask[delayed] dask[dataframe] dask-ml fsspec>=0.3.3
 ```
 
 If you plan to use the [TPOT-MDR configuration](https://arxiv.org/abs/1702.01780), make sure to install [scikit-mdr](https://github.com/EpistasisLab/scikit-mdr) and [scikit-rebate](https://github.com/EpistasisLab/scikit-rebate):

diff --git a/tests/test_dask_based.py b/tests/test_dask_based.py
@@ -27,15 +27,17 @@ def test_dask_matches(self):
                     cv=3,
                     random_state=42,
                     n_jobs=n_jobs,
-                    use_dask=False
+                    use_dask=False,
+                    verbosity=3
                 )
                 b = TPOTClassifier(
                     generations=0,
                     population_size=5,
                     cv=3,
                     random_state=42,
                     n_jobs=n_jobs,
-                    use_dask=True
+                    use_dask=True,
+                    verbosity=3
                 )
                 a.fit(X, y)
                 b.fit(X, y)

diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py
@@ -422,8 +422,8 @@ def _wrapped_cross_val_score(sklearn_pipeline, features, target,
             import dask_ml.model_selection  # noqa
             import dask  # noqa
             from dask.delayed import Delayed
-        except ImportError:
-            msg = "'use_dask' requires the optional dask and dask-ml depedencies."
+        except Exception as e:
+            msg = "'use_dask' requires the optional dask and dask-ml depedencies.\n{}".format(e)
             raise ImportError(msg)
 
         dsk, keys, n_splits = dask_ml.model_selection._search.build_graph(