diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e31c477477..88234e69e8 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,15 +2,9 @@ TODO: * [ ] Add unit tests and/or doctests in docstrings -* [ ] Unit tests and doctests pass locally under Python 3.6 (e.g., run ``tox -e py36`` or - ``pytest -v --doctest-modules zarr``) -* [ ] Unit tests pass locally under Python 2.7 (e.g., run ``tox -e py27`` or - ``pytest -v zarr``) -* [ ] PEP8 checks pass (e.g., run ``tox -e py36`` or ``flake8 --max-line-length=100 zarr``) * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions * [ ] New/modified features documented in docs/tutorial.rst -* [ ] Doctests in tutorial pass (e.g., run ``tox -e py36`` or ``python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst``) * [ ] Changes documented in docs/release.rst * [ ] Docs build locally (e.g., run ``tox -e docs``) * [ ] AppVeyor and Travis CI passes -* [ ] Test coverage to 100% (Coveralls passes) +* [ ] Test coverage is 100% (Coveralls passes) diff --git a/.travis.yml b/.travis.yml index d126fb755a..8a5e1fe521 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,15 +11,17 @@ addons: packages: - libdb-dev -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 +matrix: + include: + - python: 2.7 + - python: 3.5 + - python: 3.6 + - python: 3.7 + dist: xenial + sudo: true install: - - pip install -U pip setuptools wheel - - pip install -U tox-travis coveralls + - pip install -U pip setuptools wheel tox-travis coveralls script: - tox diff --git a/appveyor.yml b/appveyor.yml index ef94c37a54..987b51c1c4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,45 +14,36 @@ environment: - PYTHON: "C:\\Python27" PYTHON_VERSION: "2.7" - NUMPY_VERSION: "1.13.3" + NUMPY_VERSION: "1.15.2" - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7" - NUMPY_VERSION: "1.13.3" - DISTUTILS_USE_SDK: "1" - - - PYTHON: "C:\\Python34" - NUMPY_VERSION: "1.13.3" - PYTHON_VERSION: "3.4" - - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4" - NUMPY_VERSION: "1.13.3" + NUMPY_VERSION: "1.15.2" DISTUTILS_USE_SDK: "1" - PYTHON: "C:\\Python35" PYTHON_VERSION: "3.5" - NUMPY_VERSION: "1.13.3" + NUMPY_VERSION: "1.15.2" - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5" - NUMPY_VERSION: "1.13.3" + NUMPY_VERSION: "1.15.2" - PYTHON: "C:\\Python36" PYTHON_VERSION: "3.6" - NUMPY_VERSION: "1.13.3" + NUMPY_VERSION: "1.15.2" - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6" - NUMPY_VERSION: "1.13.3" + NUMPY_VERSION: "1.15.2" - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6" - NUMPY_VERSION: "1.14.0" + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7" + NUMPY_VERSION: "1.15.2" - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6" - NUMPY_VERSION: "1.14.0" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" + NUMPY_VERSION: "1.15.2" install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -62,7 +53,8 @@ build: off test_script: - "%CMD_IN_ENV% python -m pip install -U pip setuptools wheel" - "%CMD_IN_ENV% python -m pip install numpy==%NUMPY_VERSION%" + - "%CMD_IN_ENV% python -m pip install cython==0.29" + - "%CMD_IN_ENV% python -m pip install -v --no-binary=numcodecs numcodecs==0.5.5" - "%CMD_IN_ENV% python -m pip install -rrequirements_dev.txt" - "%CMD_IN_ENV% python setup.py install" - "%CMD_IN_ENV% python -m pytest -v --pyargs zarr" - diff --git a/docs/release.rst b/docs/release.rst index fdcd3cb0e2..9acab25fde 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,6 +1,18 @@ Release notes ============= +.. _release_2.3.0: + +2.3.0 (Work in Progress) +------------------------ + +Maintenance +~~~~~~~~~~~ + +* CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and + upgrade all package requirements. :issue:`308`. + + .. _release_2.2.0: 2.2.0 diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c174a57ae5..5c090669ce 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1331,7 +1331,7 @@ internal threads. The number of Blosc threads can be changed to increase or decrease this number, e.g.:: >>> from zarr import blosc - >>> blosc.set_nthreads(2) + >>> blosc.set_nthreads(2) # doctest: +SKIP 8 When a Zarr array is being used within a multi-threaded program, Zarr diff --git a/requirements_dev.txt b/requirements_dev.txt index 95e4a556b4..d495e04bfd 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,38 +1,60 @@ -appdirs==1.4.3 -args==0.1.0 asciitree==0.3.3 -certifi==2017.7.27.1 +asn1crypto==0.24.0 +atomicwrites==1.2.1 +attrs==18.2.0 +bleach==3.0.2 +boto3==1.9.26 +botocore==1.12.26 +certifi==2018.10.15 +cffi==1.11.5 chardet==3.0.4 -clint==0.5.1 -coverage==4.4.1 -coveralls==1.2.0 -Cython==0.27.2 +cmarkgfm==0.4.2 +configparser==3.5.0 +coverage==4.5.1 +coveralls==1.5.1 +cryptography==2.3.1 +Cython==0.29 docopt==0.6.2 +docutils==0.14 +enum34==1.1.6 fasteners==0.14.1 +filelock==3.0.9 flake8==3.5.0 -h5py==2.7.1 -idna==2.6 +funcsigs==1.0.2 +future==0.16.0 +h5py==2.8.0 +idna==2.7 +ipaddress==1.0.22 +jmespath==0.9.3 mccabe==0.6.1 -monotonic==1.3 -msgpack-python==0.4.8 -numcodecs==0.5.4 -packaging==16.8 -pkginfo==1.4.1 -pluggy==0.5.2 -py==1.4.34 -py-cpuinfo==3.3.0 +monotonic==1.5 +more-itertools==4.3.0 +msgpack-python==0.5.6 +numcodecs==0.5.5 +pathlib2==2.3.2 +pkginfo==1.4.2 +pluggy==0.8.0 +py==1.7.0 pycodestyle==2.3.1 +pycparser==2.19 pyflakes==1.6.0 -pyparsing==2.2.0 -pytest==3.2.3 -pytest-cov==2.5.1 -requests==2.18.4 +Pygments==2.2.0 +pyOpenSSL==18.0.0 +pytest==3.9.1 +pytest-cov==2.6.0 +python-dateutil==2.7.3 +readme-renderer==22.0 +requests==2.19.1 requests-toolbelt==0.8.0 -setuptools-scm==1.15.6 -s3fs==0.1.2 -tox==2.9.1 -tox-travis==0.8 -tqdm==4.19.4 -twine==1.9.1 -urllib3==1.22 -virtualenv==15.1.0 +s3fs==0.1.6 +s3transfer==0.1.13 +scandir==1.9.0 +six==1.11.0 +toml==0.10.0 +tox==3.5.2 +tox-travis==0.11 +tqdm==4.27.0 +twine==1.12.1 +urllib3==1.23 +virtualenv==16.0.0 +webencodings==0.5.1 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index ad6f7064c6..a4e7c2a6bd 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -1,2 +1,2 @@ -bsddb3==6.2.5 -lmdb==0.93 +bsddb3==6.2.6 +lmdb==0.94 diff --git a/setup.py b/setup.py index ae89a66ceb..a5e8334e43 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], maintainer='Alistair Miles', maintainer_email='alimanfoo@googlemail.com', diff --git a/tox.ini b/tox.ini index 21e869df54..7435b01f45 100644 --- a/tox.ini +++ b/tox.ini @@ -4,27 +4,37 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py34, py35, py36-npy{113,114}, docs +# N.B., test different versions of numpy under py36 rather than py37 +# because wheels for npy113 not available for py37 +envlist = py27, py35, py36-npy{113,114,115}, py37, docs [testenv] +install_command = pip install --no-binary=numcodecs {opts} {packages} setenv = PYTHONHASHSEED = 42 # hooks for coverage exclusions based on Python major version - py34,py35,py36: PY_MAJOR_VERSION = py3 + py35,py36,py37: PY_MAJOR_VERSION = py3 py27: PY_MAJOR_VERSION = py2 commands = + # clear out any data files generated during tests python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]' - py27,py34,py35: pytest -v --cov=zarr zarr - # don't run py36-npy114 with coverage because it is run together with py35-npy113 on travis + # main unit test runner + # N.B., don't run npy113 or npy114 with coverage because it is run together npy115 on travis + py27,py35,py36-npy115: pytest -v --cov=zarr --cov-config=.coveragerc zarr + py36-npy113: pytest -v zarr py36-npy114: pytest -v zarr - py36-npy113: pytest -v --cov=zarr --doctest-modules zarr - py27,py34,py35,py36-npy113: coverage report -m - py36-npy113: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst - py36-npy113: flake8 --max-line-length=100 zarr + py37: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-modules zarr + # generate a coverate report + py27,py35,py36-npy115,py37: coverage report -m + # run doctests in the tutorial and spec + py37: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst + # pep8 checks + py37: flake8 --max-line-length=100 zarr deps = py27: backports.lzma - py27,py34,py35,py36-npy113: numpy==1.13.3 - py36-npy114: numpy==1.14.0 + py36-npy113: numpy==1.13.3 + py36-npy114: numpy==1.14.6 + py27,py35,py36-npy115,py37: numpy==1.15.2 -rrequirements_dev.txt # linux only -rrequirements_dev_optional.txt diff --git a/zarr/core.py b/zarr/core.py index 03d9bdc667..262ae5d4ef 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2157,7 +2157,7 @@ def view(self, shape=None, chunks=None, dtype=None, array([0, 0, 1, ..., 1, 0, 0], dtype=uint8) >>> v = a.view(dtype=bool) >>> v[:] - array([False, False, True, ..., True, False, False], dtype=bool) + array([False, False, True, ..., True, False, False]) >>> np.all(a[:].view(dtype=bool) == v[:]) True diff --git a/zarr/creation.py b/zarr/creation.py index 004b0e4ad1..49b4a9d2ea 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -224,8 +224,8 @@ def zeros(shape, **kwargs): >>> z >>> z[:2, :2] - array([[ 0., 0.], - [ 0., 0.]]) + array([[0., 0.], + [0., 0.]]) """ @@ -245,8 +245,8 @@ def ones(shape, **kwargs): >>> z >>> z[:2, :2] - array([[ 1., 1.], - [ 1., 1.]]) + array([[1., 1.], + [1., 1.]]) """ @@ -266,8 +266,8 @@ def full(shape, fill_value, **kwargs): >>> z >>> z[:2, :2] - array([[ 42., 42.], - [ 42., 42.]]) + array([[42., 42.], + [42., 42.]]) """ diff --git a/zarr/storage.py b/zarr/storage.py index 39a497d08b..173325e23a 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1437,7 +1437,11 @@ def __init__(self, path, flag='c', mode=0o666, open=None, write_lock=True, self.open_kwargs = open_kwargs def __getstate__(self): - self.flush() # needed for py2 and ndbm + try: + self.flush() # needed for py2 and ndbm + except Exception: + # flush may fail if db has already been closed + pass return (self.path, self.flag, self.mode, self.open, self.write_lock, self.open_kwargs) @@ -1631,7 +1635,11 @@ def __init__(self, path, buffers=True, **kwargs): self.kwargs = kwargs def __getstate__(self): - self.flush() # just in case + try: + self.flush() # just in case + except Exception: + # flush may fail if db has already been closed + pass return self.path, self.buffers, self.kwargs def __setstate__(self, state): diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 390f888287..374b298c22 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -656,19 +656,39 @@ def test_read_only(self): def test_pickle(self): + # setup array z = self.create_array(shape=1000, chunks=100, dtype=int, cache_metadata=False, cache_attrs=False) - z[:] = np.random.randint(0, 1000, 1000) - z2 = pickle.loads(pickle.dumps(z)) - assert z.shape == z2.shape - assert z.chunks == z2.chunks - assert z.dtype == z2.dtype + shape = z.shape + chunks = z.chunks + dtype = z.dtype + compressor_config = None if z.compressor: - assert z.compressor.get_config() == z2.compressor.get_config() - assert z.fill_value == z2.fill_value - assert z._cache_metadata == z2._cache_metadata - assert z.attrs.cache == z2.attrs.cache - assert_array_equal(z[:], z2[:]) + compressor_config = z.compressor.get_config() + fill_value = z.fill_value + cache_metadata = z._cache_metadata + attrs_cache = z.attrs.cache + a = np.random.randint(0, 1000, 1000) + z[:] = a + + # round trip through pickle + dump = pickle.dumps(z) + # some stores cannot be opened twice at the same time, need to close + # store before can round-trip through pickle + if hasattr(z.store, 'close'): + z.store.close() + z2 = pickle.loads(dump) + + # verify + assert shape == z2.shape + assert chunks == z2.chunks + assert dtype == z2.dtype + if z2.compressor: + assert compressor_config == z2.compressor.get_config() + assert fill_value == z2.fill_value + assert cache_metadata == z2._cache_metadata + assert attrs_cache == z2.attrs.cache + assert_array_equal(a, z2[:]) def test_np_ufuncs(self): z = self.create_array(shape=(100, 100), chunks=(10, 10)) diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index eb437706f0..304714991e 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -3,7 +3,6 @@ import tempfile import shutil import atexit -import warnings import numpy as np @@ -22,12 +21,6 @@ from zarr.compat import PY2 -# needed for PY2/PY3 consistent behaviour -if PY2: # pragma: py3 no cover - warnings.resetwarnings() - warnings.simplefilter('always') - - # something bcolz-like class MockBcolzArray(object): @@ -457,12 +450,15 @@ def test_compression_args(): assert 'zlib' == z.compressor.codec_id assert 9 == z.compressor.level - with pytest.warns(UserWarning): - # 'compressor' overrides 'compression' - create(100, compressor=Zlib(9), compression='bz2', compression_opts=1) - with pytest.warns(UserWarning): - # 'compressor' ignores 'compression_opts' - create(100, compressor=Zlib(9), compression_opts=1) + # cannot get warning tests to work on PY2 + if not PY2: # pragma: py2 no cover + + with pytest.warns(UserWarning): + # 'compressor' overrides 'compression' + create(100, compressor=Zlib(9), compression='bz2', compression_opts=1) + with pytest.warns(UserWarning): + # 'compressor' ignores 'compression_opts' + create(100, compressor=Zlib(9), compression_opts=1) def test_create_read_only(): diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index f47012cf88..7758976c8c 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -820,23 +820,31 @@ def test_paths(self): g1['foo/../bar'] def test_pickle(self): - # setup + + # setup group g = self.create_group() d = g.create_dataset('foo/bar', shape=100, chunks=10) d[:] = np.arange(100) - - # needed for zip store - if hasattr(g.store, 'flush'): - g.store.flush() - - # pickle round trip - g2 = pickle.loads(pickle.dumps(g)) - assert g.path == g2.path - assert g.name == g2.name - assert len(g) == len(g2) - assert list(g) == list(g2) - assert g['foo'] == g2['foo'] - assert g['foo/bar'] == g2['foo/bar'] + path = g.path + name = g.name + n = len(g) + keys = list(g) + + # round-trip through pickle + dump = pickle.dumps(g) + # some stores cannot be opened twice at the same time, need to close + # store before can round-trip through pickle + if hasattr(g.store, 'close'): + g.store.close() + g2 = pickle.loads(dump) + + # verify + assert path == g2.path + assert name == g2.name + assert n == len(g2) + assert keys == list(g2) + assert isinstance(g2['foo'], Group) + assert isinstance(g2['foo/bar'], Array) class TestGroupWithDictStore(TestGroup): diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index f68f8a6ed6..79e6adaeac 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -128,12 +128,27 @@ def test_iterators(self): set(store.items())) def test_pickle(self): + + # setup store store = self.create_store() store['foo'] = b'bar' store['baz'] = b'quux' - store2 = pickle.loads(pickle.dumps(store)) - assert len(store) == len(store2) - assert sorted(store.keys()) == sorted(store2.keys()) + n = len(store) + keys = sorted(store.keys()) + + # round-trip through pickle + dump = pickle.dumps(store) + # some stores cannot be opened twice at the same time, need to close + # store before can round-trip through pickle + if hasattr(store, 'close'): + store.close() + # check can still pickle after close + assert dump == pickle.dumps(store) + store2 = pickle.loads(dump) + + # verify + assert n == len(store2) + assert keys == sorted(store2.keys()) assert b'bar' == store2['foo'] assert b'quux' == store2['baz'] @@ -745,6 +760,7 @@ class TestDBMStore(StoreTests, unittest.TestCase): def create_store(self): path = tempfile.mktemp(suffix='.anydbm') atexit.register(atexit_rmglob, path + '*') + # create store using default dbm implementation store = DBMStore(path, flag='n') return store