From b170b2973a992652a57437937f95b87e6713a6e7 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Wed, 16 Dec 2020 21:07:32 +0100 Subject: [PATCH] CI: cleanup style check(#477) This PR clean up the style check output by capturing both stdout and stderr. Also `isort` line-length is now 88, which is what `black` and `flake8` use. Authors: - Mads R. B. Kristensen Approvers: - AJ Schmidt - Benjamin Zaitlen URL: https://github.com/rapidsai/dask-cuda/pull/477 --- .pre-commit-config.yaml | 6 +++--- ci/checks/style.sh | 19 ++++++++++++++----- dask_cuda/__init__.py | 2 +- dask_cuda/cuda_worker.py | 2 +- dask_cuda/explicit_comms/comms.py | 6 +----- dask_cuda/proxy_object.py | 5 +---- dask_cuda/tests/test_device_host_file.py | 6 +----- dask_cuda/tests/test_explicit_comms.py | 6 +----- dask_cuda/tests/test_local_cuda_cluster.py | 2 +- dask_cuda/tests/test_proxy.py | 7 ++++++- setup.cfg | 2 +- 11 files changed, 31 insertions(+), 32 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7fd2ac520..e5f44cd88 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,14 @@ repos: - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.0.7 hooks: - id: isort - repo: https://github.com/ambv/black - rev: stable + rev: 19.10b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.7 + rev: 3.8.3 hooks: - id: flake8 default_language_version: diff --git a/ci/checks/style.sh b/ci/checks/style.sh index ab026e5aa..48343273d 100755 --- a/ci/checks/style.sh +++ b/ci/checks/style.sh @@ -12,17 +12,23 @@ PATH=/opt/conda/bin:$PATH . /opt/conda/etc/profile.d/conda.sh conda activate rapids +# Print versions +echo -e "\nVersions:" +black --version +echo "isort, `isort --vn`" +echo "flake8, `flake8 --version`" + # Run isort and get results/return code -ISORT=`isort --recursive --check-only .` +ISORT=`isort --check-only dask_cuda 2>&1` ISORT_RETVAL=$? # Run black and get results/return code -BLACK=`black --check .` +BLACK=`black --check dask_cuda 2>&1` BLACK_RETVAL=$? # Run flake8 and get results/return code -FLAKE=`flake8 dask_cuda` -RETVAL=$? +FLAKE=`flake8 dask_cuda 2>&1` +FLAKE_RETVAL=$? # Output results if failure otherwise show pass if [ "$ISORT_RETVAL" != "0" ]; then @@ -41,7 +47,7 @@ else echo -e "\n\n>>>> PASSED: black style check\n\n" fi -if [ "$FLAKE" != "0" ]; then +if [ "$FLAKE_RETVAL" != "0" ]; then echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" echo -e "$FLAKE" echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" @@ -49,4 +55,7 @@ else echo -e "\n\n>>>> PASSED: flake8 style check\n\n" fi +RETVALS=($ISORT_RETVAL $BLACK_RETVAL $FLAKE_RETVAL) +IFS=$'\n' +RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1` exit $RETVAL diff --git a/dask_cuda/__init__.py b/dask_cuda/__init__.py index 91b049f73..d2fe16feb 100644 --- a/dask_cuda/__init__.py +++ b/dask_cuda/__init__.py @@ -1,6 +1,6 @@ from ._version import get_versions -from .local_cuda_cluster import LocalCUDACluster from .cuda_worker import CUDAWorker +from .local_cuda_cluster import LocalCUDACluster __version__ = get_versions()["version"] del get_versions diff --git a/dask_cuda/cuda_worker.py b/dask_cuda/cuda_worker.py index e32d9c53e..abf3e1996 100644 --- a/dask_cuda/cuda_worker.py +++ b/dask_cuda/cuda_worker.py @@ -10,11 +10,11 @@ import dask from distributed import Nanny +from distributed.deploy.cluster import Cluster from distributed.proctitle import ( enable_proctitle_on_children, enable_proctitle_on_current, ) -from distributed.deploy.cluster import Cluster from distributed.utils import parse_bytes from distributed.worker import parse_memory_limit diff --git a/dask_cuda/explicit_comms/comms.py b/dask_cuda/explicit_comms/comms.py index 72cfbbc48..e8bb0c11b 100644 --- a/dask_cuda/explicit_comms/comms.py +++ b/dask_cuda/explicit_comms/comms.py @@ -5,11 +5,7 @@ import distributed.comm from distributed import default_client, get_worker -from distributed.comm.addressing import ( - parse_address, - parse_host_port, - unparse_address, -) +from distributed.comm.addressing import parse_address, parse_host_port, unparse_address from . import utils diff --git a/dask_cuda/proxy_object.py b/dask_cuda/proxy_object.py index c7988f272..7171edff0 100644 --- a/dask_cuda/proxy_object.py +++ b/dask_cuda/proxy_object.py @@ -565,10 +565,7 @@ def obj_pxy_dask_deserialize(header, frames): subclass = ProxyObject else: subclass = pickle.loads(meta["subclass"]) - return subclass( - obj=(header["proxied-header"], frames), - **header["obj-pxy-meta"], - ) + return subclass(obj=(header["proxied-header"], frames), **header["obj-pxy-meta"],) @dask.dataframe.utils.hash_object_dispatch.register(ProxyObject) diff --git a/dask_cuda/tests/test_device_host_file.py b/dask_cuda/tests/test_device_host_file.py index 50c15ea76..9433e289c 100644 --- a/dask_cuda/tests/test_device_host_file.py +++ b/dask_cuda/tests/test_device_host_file.py @@ -13,11 +13,7 @@ ) from distributed.protocol.pickle import HIGHEST_PROTOCOL -from dask_cuda.device_host_file import ( - DeviceHostFile, - device_to_host, - host_to_device, -) +from dask_cuda.device_host_file import DeviceHostFile, device_to_host, host_to_device cupy = pytest.importorskip("cupy") diff --git a/dask_cuda/tests/test_explicit_comms.py b/dask_cuda/tests/test_explicit_comms.py index f1c66f6af..c1825d052 100644 --- a/dask_cuda/tests/test_explicit_comms.py +++ b/dask_cuda/tests/test_explicit_comms.py @@ -10,11 +10,7 @@ from distributed import Client from distributed.deploy.local import LocalCluster -from dask_cuda.explicit_comms import ( - CommsContext, - dataframe_merge, - dataframe_shuffle, -) +from dask_cuda.explicit_comms import CommsContext, dataframe_merge, dataframe_shuffle mp = mp.get_context("spawn") ucp = pytest.importorskip("ucp") diff --git a/dask_cuda/tests/test_local_cuda_cluster.py b/dask_cuda/tests/test_local_cuda_cluster.py index 6c0591fbb..092225497 100644 --- a/dask_cuda/tests/test_local_cuda_cluster.py +++ b/dask_cuda/tests/test_local_cuda_cluster.py @@ -6,7 +6,7 @@ from distributed.system import MEMORY_LIMIT from distributed.utils_test import gen_test -from dask_cuda import LocalCUDACluster, CUDAWorker, utils +from dask_cuda import CUDAWorker, LocalCUDACluster, utils from dask_cuda.initialize import initialize diff --git a/dask_cuda/tests/test_proxy.py b/dask_cuda/tests/test_proxy.py index f4eb83b0f..fc576058f 100644 --- a/dask_cuda/tests/test_proxy.py +++ b/dask_cuda/tests/test_proxy.py @@ -1,6 +1,7 @@ -import pickle import operator +import pickle +import pandas import pytest from pandas.testing import assert_frame_equal @@ -222,6 +223,10 @@ def task(x): ddf = dask_cudf.from_cudf(df, npartitions=1) ddf = ddf.map_partitions(task, meta=df.head()) got = ddf.compute() + if isinstance(got, pandas.Series): + pytest.xfail( + "BUG fixed by " + ) assert_frame_equal(got.to_pandas(), df.to_pandas()) diff --git a/setup.cfg b/setup.cfg index 76dd01bd2..143a9d62e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,7 @@ ignore = E231 [isort] -line_length=79 +line_length=88 multi_line_output=3 include_trailing_comma=True force_grid_wrap=0