Test and CI upgrades and maintenance

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,15 +2,9 @@
 
 TODO:
 * [ ] Add unit tests and/or doctests in docstrings
-* [ ] Unit tests and doctests pass locally under Python 3.6 (e.g., run ``tox -e py36`` or 
-  ``pytest -v --doctest-modules zarr``)
-* [ ] Unit tests pass locally under Python 2.7 (e.g., run ``tox -e py27`` or
-  ``pytest -v zarr``)
-* [ ] PEP8 checks pass (e.g., run ``tox -e py36`` or ``flake8 --max-line-length=100 zarr``)
 * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions
 * [ ] New/modified features documented in docs/tutorial.rst
-* [ ] Doctests in tutorial pass (e.g., run ``tox -e py36`` or ``python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst``)
 * [ ] Changes documented in docs/release.rst
 * [ ] Docs build locally (e.g., run ``tox -e docs``)
 * [ ] AppVeyor and Travis CI passes
-* [ ] Test coverage to 100% (Coveralls passes)
+* [ ] Test coverage is 100% (Coveralls passes)

.travis.yml

@@ -11,15 +11,17 @@ addons:
     packages:
     - libdb-dev
 
-python:
-  - 2.7
-  - 3.4
-  - 3.5
-  - 3.6
+matrix:
+  include:
+    - python: 2.7
+    - python: 3.5
+    - python: 3.6
+    - python: 3.7
+      dist: xenial
+      sudo: true
 
 install:
-  - pip install -U pip setuptools wheel
-  - pip install -U tox-travis coveralls
+  - pip install -U pip setuptools wheel tox-travis coveralls
 
 script:
   - tox

appveyor.yml

@@ -14,45 +14,36 @@ environment:
 
     - PYTHON: "C:\\Python27"
       PYTHON_VERSION: "2.7"
-      NUMPY_VERSION: "1.13.3"
+      NUMPY_VERSION: "1.15.2"
 
     - PYTHON: "C:\\Python27-x64"
       PYTHON_VERSION: "2.7"
-      NUMPY_VERSION: "1.13.3"
-      DISTUTILS_USE_SDK: "1"
-
-    - PYTHON: "C:\\Python34"
-      NUMPY_VERSION: "1.13.3"
-      PYTHON_VERSION: "3.4"
-
-    - PYTHON: "C:\\Python34-x64"
-      PYTHON_VERSION: "3.4"
-      NUMPY_VERSION: "1.13.3"
+      NUMPY_VERSION: "1.15.2"
       DISTUTILS_USE_SDK: "1"
 
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "3.5"
-      NUMPY_VERSION: "1.13.3"
+      NUMPY_VERSION: "1.15.2"
 
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5"
-      NUMPY_VERSION: "1.13.3"
+      NUMPY_VERSION: "1.15.2"
 
     - PYTHON: "C:\\Python36"
       PYTHON_VERSION: "3.6"
-      NUMPY_VERSION: "1.13.3"
+      NUMPY_VERSION: "1.15.2"
 
     - PYTHON: "C:\\Python36-x64"
       PYTHON_VERSION: "3.6"
-      NUMPY_VERSION: "1.13.3"
+      NUMPY_VERSION: "1.15.2"
 
-    - PYTHON: "C:\\Python36"
-      PYTHON_VERSION: "3.6"
-      NUMPY_VERSION: "1.14.0"
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7"
+      NUMPY_VERSION: "1.15.2"
 
-    - PYTHON: "C:\\Python36-x64"
-      PYTHON_VERSION: "3.6"
-      NUMPY_VERSION: "1.14.0"
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7"
+      NUMPY_VERSION: "1.15.2"
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -62,7 +53,7 @@ build: off
 test_script:
   - "%CMD_IN_ENV% python -m pip install -U pip setuptools wheel"
   - "%CMD_IN_ENV% python -m pip install numpy==%NUMPY_VERSION%"
+  - "%CMD_IN_ENV% python -m pip install -v --no-binary=numcodecs numcodecs==0.5.5"
   - "%CMD_IN_ENV% python -m pip install -rrequirements_dev.txt"
   - "%CMD_IN_ENV% python setup.py install"
   - "%CMD_IN_ENV% python -m pytest -v --pyargs zarr"
-

docs/release.rst

@@ -1,6 +1,18 @@
 Release notes
 =============
 
+.. _release_2.3.0:
+
+2.3.0 (Work in Progress)
+------------------------
+
+Maintenance
+~~~~~~~~~~~
+
+* CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and
+  upgrade all package requirements. :issue:`308`.
+
+
 .. _release_2.2.0:
 
 2.2.0

docs/tutorial.rst

@@ -1331,7 +1331,7 @@ internal threads. The number of Blosc threads can be changed to increase or
 decrease this number, e.g.::
 
     >>> from zarr import blosc
-    >>> blosc.set_nthreads(2)
+    >>> blosc.set_nthreads(2)  # doctest: +SKIP
     8
 
 When a Zarr array is being used within a multi-threaded program, Zarr

requirements_dev.txt

@@ -1,38 +1,60 @@
-appdirs==1.4.3
-args==0.1.0
 asciitree==0.3.3
-certifi==2017.7.27.1
+asn1crypto==0.24.0
+atomicwrites==1.2.1
+attrs==18.2.0
+bleach==3.0.2
+boto3==1.9.26
+botocore==1.12.26
+certifi==2018.10.15
+cffi==1.11.5
 chardet==3.0.4
-clint==0.5.1
-coverage==4.4.1
-coveralls==1.2.0
-Cython==0.27.2
+cmarkgfm==0.4.2
+configparser==3.5.0
+coverage==4.5.1
+coveralls==1.5.1
+cryptography==2.3.1
+Cython==0.29
 docopt==0.6.2
+docutils==0.14
+enum34==1.1.6
 fasteners==0.14.1
+filelock==3.0.9
 flake8==3.5.0
-h5py==2.7.1
-idna==2.6
+funcsigs==1.0.2
+future==0.16.0
+h5py==2.8.0
+idna==2.7
+ipaddress==1.0.22
+jmespath==0.9.3
 mccabe==0.6.1
-monotonic==1.3
-msgpack-python==0.4.8
-numcodecs==0.5.4
-packaging==16.8
-pkginfo==1.4.1
-pluggy==0.5.2
-py==1.4.34
-py-cpuinfo==3.3.0
+monotonic==1.5
+more-itertools==4.3.0
+msgpack-python==0.5.6
+numcodecs==0.5.5
+pathlib2==2.3.2
+pkginfo==1.4.2
+pluggy==0.8.0
+py==1.7.0
 pycodestyle==2.3.1
+pycparser==2.19
 pyflakes==1.6.0
-pyparsing==2.2.0
-pytest==3.2.3
-pytest-cov==2.5.1
-requests==2.18.4
+Pygments==2.2.0
+pyOpenSSL==18.0.0
+pytest==3.9.1
+pytest-cov==2.6.0
+python-dateutil==2.7.3
+readme-renderer==22.0
+requests==2.19.1
 requests-toolbelt==0.8.0
-setuptools-scm==1.15.6
-s3fs==0.1.2
-tox==2.9.1
-tox-travis==0.8
-tqdm==4.19.4
-twine==1.9.1
-urllib3==1.22
-virtualenv==15.1.0
+s3fs==0.1.6
+s3transfer==0.1.13
+scandir==1.9.0
+six==1.11.0
+toml==0.10.0
+tox==3.5.2
+tox-travis==0.11
+tqdm==4.27.0
+twine==1.12.1
+urllib3==1.23
+virtualenv==16.0.0
+webencodings==0.5.1

requirements_dev_optional.txt

@@ -1,2 +1,2 @@
-bsddb3==6.2.5
-lmdb==0.93
+bsddb3==6.2.6
+lmdb==0.94

setup.py

@@ -45,6 +45,7 @@
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
     ],
     maintainer='Alistair Miles',
     maintainer_email='alimanfoo@googlemail.com',

tox.ini

@@ -4,27 +4,37 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, py34, py35, py36-npy{113,114}, docs
+# N.B., test different versions of numpy under py36 rather than py37
+# because wheels for npy113 not available for py37
+envlist = py27, py35, py36-npy{113,114,115}, py37, docs
 
 [testenv]
+install_command = pip install --no-binary=numcodecs {opts} {packages}
 setenv =
     PYTHONHASHSEED = 42
     # hooks for coverage exclusions based on Python major version
-    py34,py35,py36: PY_MAJOR_VERSION = py3
+    py35,py36,py37: PY_MAJOR_VERSION = py3
     py27: PY_MAJOR_VERSION = py2
 commands =
+    # clear out any data files generated during tests
     python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]'
-    py27,py34,py35: pytest -v --cov=zarr zarr
-    # don't run py36-npy114 with coverage because it is run together with py35-npy113 on travis
+    # main unit test runner
+    # N.B., don't run npy113 or npy114 with coverage because it is run together npy115 on travis
+    py27,py35,py36-npy115: pytest -v --cov=zarr --cov-config=.coveragerc zarr
+    py36-npy113: pytest -v zarr
     py36-npy114: pytest -v zarr
-    py36-npy113: pytest -v --cov=zarr --doctest-modules zarr
-    py27,py34,py35,py36-npy113: coverage report -m
-    py36-npy113: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst
-    py36-npy113: flake8 --max-line-length=100 zarr
+    py37: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-modules zarr
+    # generate a coverate report
+    py27,py35,py36-npy115,py37: coverage report -m
+    # run doctests in the tutorial and spec
+    py37: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst
+    # pep8 checks
+    py37: flake8 --max-line-length=100 zarr
 deps =
     py27: backports.lzma
-    py27,py34,py35,py36-npy113: numpy==1.13.3
-    py36-npy114: numpy==1.14.0
+    py36-npy113: numpy==1.13.3
+    py36-npy114: numpy==1.14.6
+    py27,py35,py36-npy115,py37: numpy==1.15.2
     -rrequirements_dev.txt
     # linux only
     -rrequirements_dev_optional.txt

zarr/core.py

@@ -2157,7 +2157,7 @@ def view(self, shape=None, chunks=None, dtype=None,
             array([0, 0, 1, ..., 1, 0, 0], dtype=uint8)
             >>> v = a.view(dtype=bool)
             >>> v[:]
-            array([False, False,  True, ...,  True, False, False], dtype=bool)
+            array([False, False,  True, ...,  True, False, False])
             >>> np.all(a[:].view(dtype=bool) == v[:])
             True
 

zarr/creation.py

@@ -224,8 +224,8 @@ def zeros(shape, **kwargs):
     >>> z
     <zarr.core.Array (10000, 10000) float64>
     >>> z[:2, :2]
-    array([[ 0.,  0.],
-           [ 0.,  0.]])
+    array([[0., 0.],
+           [0., 0.]])
 
     """
 
@@ -245,8 +245,8 @@ def ones(shape, **kwargs):
     >>> z
     <zarr.core.Array (10000, 10000) float64>
     >>> z[:2, :2]
-    array([[ 1.,  1.],
-           [ 1.,  1.]])
+    array([[1., 1.],
+           [1., 1.]])
 
     """
 
@@ -266,8 +266,8 @@ def full(shape, fill_value, **kwargs):
     >>> z
     <zarr.core.Array (10000, 10000) float64>
     >>> z[:2, :2]
-    array([[ 42.,  42.],
-           [ 42.,  42.]])
+    array([[42., 42.],
+           [42., 42.]])
 
     """
 

zarr/storage.py

@@ -1437,7 +1437,11 @@ def __init__(self, path, flag='c', mode=0o666, open=None, write_lock=True,
         self.open_kwargs = open_kwargs
 
     def __getstate__(self):
-        self.flush()  # needed for py2 and ndbm
+        try:
+            self.flush()  # needed for py2 and ndbm
+        except Exception:
+            # flush may fail if db has already been closed
+            pass
         return (self.path, self.flag, self.mode, self.open, self.write_lock,
                 self.open_kwargs)
 
@@ -1631,7 +1635,11 @@ def __init__(self, path, buffers=True, **kwargs):
         self.kwargs = kwargs
 
     def __getstate__(self):
-        self.flush()  # just in case
+        try:
+            self.flush()  # just in case
+        except Exception:
+            # flush may fail if db has already been closed
+            pass
         return self.path, self.buffers, self.kwargs
 
     def __setstate__(self, state):