From b6080e1dcdd902bfc3876278e8c490f75067c36c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 5 Apr 2016 16:34:52 -0700 Subject: [PATCH 01/14] rasterio checkin --- ci/requirements-py27-cdat+pynio.yml | 2 +- xarray/backends/__init__.py | 1 + xarray/backends/api.py | 4 +- xarray/backends/rasterio_.py | 132 ++++++++++++++++++++++++++++ xarray/test/__init__.py | 11 +++ xarray/test/test_backends.py | 31 ++++++- 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 xarray/backends/rasterio_.py diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml index feedb684cc8..6cba8567a22 100644 --- a/ci/requirements-py27-cdat+pynio.yml +++ b/ci/requirements-py27-cdat+pynio.yml @@ -4,7 +4,7 @@ channels: - dbrown # pynio dependencies: - python=2.7 - - cdat-lite + # - cdat-lite - dask - pytest - numpy diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index a082bd53e5e..192a3c57db2 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -10,3 +10,4 @@ from .pynio_ import NioDataStore from .scipy_ import ScipyDataStore from .h5netcdf_ import H5NetCDFStore +from .rasterio_ import RasterioDataStore diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 7afd796fafa..c4fec12f9d0 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -117,7 +117,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, decode_coords : bool, optional If True, decode the 'coordinates' attribute to identify coordinates in the resulting dataset. - engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio'}, optional + engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', 'rasterio'}, optional Engine to use when reading files. If not provided, the default engine is chosen based on available dependencies, with a preference for 'netcdf4'. @@ -218,6 +218,8 @@ def maybe_decode_store(store, lock=False): store = backends.H5NetCDFStore(filename_or_obj, group=group) elif engine == 'pynio': store = backends.NioDataStore(filename_or_obj) + elif engine == 'rasterio': + store = backends.RasterioDataStore(filename_or_obj) else: raise ValueError('unrecognized engine for open_dataset: %r' % engine) diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py new file mode 100644 index 00000000000..8f2491e858c --- /dev/null +++ b/xarray/backends/rasterio_.py @@ -0,0 +1,132 @@ +import numpy as np + +try: + import rasterio +except ImportError: + rasterio = False + +from .. import Variable, DataArray +from ..core.utils import FrozenOrderedDict, Frozen, NDArrayMixin +from ..core import indexing + +from .common import AbstractDataStore + +_VARNAME = 'raster' + + +class RasterioArrayWrapper(NDArrayMixin): + def __init__(self, array, ds): + self.array = array + self._ds = ds # make an explicit reference because pynio uses weakrefs + + @property + def dtype(self): + return np.dtype(self.array.typecode()) + + def __getitem__(self, key): + if key == () and self.ndim == 0: + return self.array.get_value() + return self.array[key] + + +class RasterioDataStore(AbstractDataStore): + """Store for accessing datasets via Rasterio + """ + def __init__(self, filename, mode='r'): + + with rasterio.drivers(): + self.ds = rasterio.open(filename, mode=mode, ) + + # Get coords + nx, ny = self.ds.width, self.ds.height + x0, y0 = self.ds.bounds.left, self.ds.bounds.top + dx, dy = self.ds.res[0], -self.ds.res[1] + + coords = {'y': np.arange(start=y0, stop=(y0 + ny * dy), step=dy), + 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} + + # Get dims + if self.ds.count == 3: + self.dims = ('band', 'y', 'x') + coords['band'] = self.ds.indexes + elif self.ds.count == 2: + self.dims = ('y', 'x') + else: + raise ValueError('unknown dims') + + attrs = {} + for attr_name in ['crs', 'affine', 'proj']: + try: + attrs[attr_name] = getattr(self.ds, attr_name) + except AttributeError: + pass + + def get_vardata(self, var_id=1): + """Read the geotiff band. + Parameters + ---------- + var_id: the variable name (here the band number) + """ + wx = (self.sub_x[0], self.sub_x[1] + 1) + wy = (self.sub_y[0], self.sub_y[1] + 1) + with rasterio.drivers(): + band = self.ds.read(var_id, window=(wy, wx)) + return band + + def open_store_variable(self, var): + if var != _VARNAME: + raise ValueError('Rasterio variables are all named %s' % _VARNAME) + data = indexing.LazilyIndexedArray(RasterioArrayWrapper(var, self.ds)) + return Variable(var.dimensions, data, var.attributes) + + def get_variables(self): + return FrozenOrderedDict((k, self.open_store_variable(v)) + for k, v in self.ds.variables.items()) + + def get_attrs(self): + return Frozen(self.ds.attributes) + + def get_dimensions(self): + return Frozen(self.ds.dimensions) + + def close(self): + self.ds.close() + + +def _transform_proj(p1, p2, x, y, nocopy=False): + """Wrapper around the pyproj transform. + When two projections are equal, this function avoids quite a bunch of + useless calculations. See https://github.com/jswhit/pyproj/issues/15 + """ + import pyproj + import copy + + if p1.srs == p2.srs: + if nocopy: + return x, y + else: + return copy.deepcopy(x), copy.deepcopy(y) + + return pyproj.transform(p1, p2, x, y) + + +def _try_to_get_latlon_coords(da): + import pyproj + if 'crs' in da.attrs: + proj = pyproj.Proj(da.attrs['crs']) + x, y = np.meshgrid(da['x'], da['y']) + proj_out = pyproj.Proj("+init=EPSG:4326", preserve_units=True) + xc, yc = _transform_proj(proj, proj_out, x, y) + coords = dict(y=da['y'], x=da['x']) + dims = ('y', 'x') + + da.coords['latitude'] = \ + DataArray(yc, coords=coords, dims=dims, name='latitude', + attrs={'units': 'degrees_north', 'long_name': 'latitude', + 'standard_name': 'latitude'}) + da.coords['longitude'] = DataArray(xc, coords=coords, dims=dims, name='latitude', + attrs={'units': 'degrees_east', + 'long_name': 'longitude', + 'standard_name': 'longitude'}) + + return da diff --git a/xarray/test/__init__.py b/xarray/test/__init__.py index c43c7208504..8c9cce62df4 100644 --- a/xarray/test/__init__.py +++ b/xarray/test/__init__.py @@ -47,6 +47,13 @@ has_pynio = False +try: + import rasterio + has_rasterioio = True +except ImportError: + has_rasterioio = False + + try: import dask.array import dask @@ -90,6 +97,10 @@ def requires_pynio(test): return test if has_pynio else unittest.skip('requires pynio')(test) +def requires_rasterio(test): + return test if has_rasterio else unittest.skip('requires rasterio')(test) + + def requires_scipy_or_netCDF4(test): return (test if has_scipy or has_netCDF4 else unittest.skip('requires scipy or netCDF4')(test)) diff --git a/xarray/test/test_backends.py b/xarray/test/test_backends.py index 6a3327e12a2..f5d45ed630a 100644 --- a/xarray/test/test_backends.py +++ b/xarray/test/test_backends.py @@ -21,7 +21,7 @@ from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap, requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf, - requires_pynio, has_netCDF4, has_scipy) + requires_pynio, requires_rasterio, has_netCDF4, has_scipy) from .test_dataset import create_test_data try: @@ -1034,6 +1034,35 @@ def test_weakrefs(self): self.assertDatasetIdentical(actual, expected) +@requires_rasterio +class TestRasterIO(CFEncodedDataTest, Only32BitTypes, TestCase): + def test_write_store(self): + # rasterio is read-only for now + pass + + def test_orthogonal_indexing(self): + # rasterio also does not support list-like indexing + pass + + @contextlib.contextmanager + def roundtrip(self, data, save_kwargs={}, open_kwargs={}): + with create_tmp_file() as tmp_file: + data.to_netcdf(tmp_file, engine='scipy', **save_kwargs) + with open_dataset(tmp_file, engine='pynio', **open_kwargs) as ds: + yield ds + + def test_weakrefs(self): + example = Dataset({'foo': ('x', np.arange(5.0))}) + expected = example.rename({'foo': 'bar', 'x': 'y'}) + + with create_tmp_file() as tmp_file: + example.to_netcdf(tmp_file, engine='scipy') + on_disk = open_dataset(tmp_file, engine='pynio') + actual = on_disk.rename({'foo': 'bar', 'x': 'y'}) + del on_disk # trigger garbage collection + self.assertDatasetIdentical(actual, expected) + + class TestEncodingInvalid(TestCase): def test_extract_nc4_encoding(self): From abeff8103f6f213efa72c4b177657ee828b6a296 Mon Sep 17 00:00:00 2001 From: NicWayand Date: Thu, 27 Oct 2016 15:55:56 -0600 Subject: [PATCH 02/14] temp fixes --- xarray/backends/rasterio_.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 8f2491e858c..7d5c53c67f1 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -34,7 +34,7 @@ class RasterioDataStore(AbstractDataStore): """ def __init__(self, filename, mode='r'): - with rasterio.drivers(): + with rasterio.Env(): self.ds = rasterio.open(filename, mode=mode, ) # Get coords @@ -46,10 +46,10 @@ def __init__(self, filename, mode='r'): 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} # Get dims - if self.ds.count == 3: + if self.ds.count >= 2: self.dims = ('band', 'y', 'x') coords['band'] = self.ds.indexes - elif self.ds.count == 2: + elif self.ds.count == 1: self.dims = ('y', 'x') else: raise ValueError('unknown dims') @@ -69,7 +69,7 @@ def get_vardata(self, var_id=1): """ wx = (self.sub_x[0], self.sub_x[1] + 1) wy = (self.sub_y[0], self.sub_y[1] + 1) - with rasterio.drivers(): + with rasterio.Env(): band = self.ds.read(var_id, window=(wy, wx)) return band From be6094e76fbc72902ab10306de4464b00498e828 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 28 Oct 2016 10:42:33 -0700 Subject: [PATCH 03/14] update rasterio reader, no lazy loading, no decoding of coords --- ci/requirements-py35.yml | 1 + setup.cfg | 2 +- xarray/backends/api.py | 2 +- xarray/backends/rasterio_.py | 100 +++++++++++++++++++---------------- xarray/test/__init__.py | 4 +- xarray/test/test_backends.py | 4 +- 6 files changed, 62 insertions(+), 51 deletions(-) diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index 0f3b005ea6a..f926cb54398 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -11,6 +11,7 @@ dependencies: - pandas - seaborn - scipy + - rasterio - pip: - coveralls - pytest-cov diff --git a/setup.cfg b/setup.cfg index 6770e9c807f..44b0d881cc2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [wheel] universal = 1 -[pytest] +[tool:pytest] python_files=test_*.py diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 4ea83535df2..8797be8b25a 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -1,4 +1,3 @@ -import sys import gzip import os.path import threading @@ -18,6 +17,7 @@ DATAARRAY_NAME = '__xarray_dataarray_name__' DATAARRAY_VARIABLE = '__xarray_dataarray_variable__' + def _get_default_engine(path, allow_remote=False): if allow_remote and is_remote_uri(path): # pragma: no cover try: diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 7d5c53c67f1..521ca85d212 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -8,20 +8,21 @@ from .. import Variable, DataArray from ..core.utils import FrozenOrderedDict, Frozen, NDArrayMixin from ..core import indexing +from ..core.pycompat import OrderedDict from .common import AbstractDataStore -_VARNAME = 'raster' +__rio_varname__ = 'raster' class RasterioArrayWrapper(NDArrayMixin): - def __init__(self, array, ds): - self.array = array - self._ds = ds # make an explicit reference because pynio uses weakrefs + def __init__(self, ds): + self._ds = ds + self.array = ds.read() @property def dtype(self): - return np.dtype(self.array.typecode()) + return np.dtype(self._ds.dtypes[0]) def __getitem__(self, key): if key == () and self.ndim == 0: @@ -42,52 +43,58 @@ def __init__(self, filename, mode='r'): x0, y0 = self.ds.bounds.left, self.ds.bounds.top dx, dy = self.ds.res[0], -self.ds.res[1] - coords = {'y': np.arange(start=y0, stop=(y0 + ny * dy), step=dy), - 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} + self.coords = {'y': np.arange(start=y0, stop=(y0 + ny * dy), step=dy), + 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} # Get dims if self.ds.count >= 2: self.dims = ('band', 'y', 'x') - coords['band'] = self.ds.indexes + self.coords['band'] = self.ds.indexes elif self.ds.count == 1: self.dims = ('y', 'x') else: raise ValueError('unknown dims') - attrs = {} - for attr_name in ['crs', 'affine', 'proj']: + self._attrs = OrderedDict() + for attr_name in ['crs', 'transform', 'proj']: try: - attrs[attr_name] = getattr(self.ds, attr_name) + self._attrs[attr_name] = getattr(self.ds, attr_name) except AttributeError: pass - def get_vardata(self, var_id=1): - """Read the geotiff band. - Parameters - ---------- - var_id: the variable name (here the band number) - """ - wx = (self.sub_x[0], self.sub_x[1] + 1) - wy = (self.sub_y[0], self.sub_y[1] + 1) - with rasterio.Env(): - band = self.ds.read(var_id, window=(wy, wx)) - return band + self.coords = _try_to_get_latlon_coords(self.coords, self._attrs) + + + + # def get_vardata(self, var_id=1): + # """Read the geotiff band. + # Parameters + # ---------- + # var_id: the variable name (here the band number) + # """ + # # wx = (self.sub_x[0], self.sub_x[1] + 1) + # # wy = (self.sub_y[0], self.sub_y[1] + 1) + # with rasterio.Env(): + # band = self.ds.read() # var_id, window=(wy, wx)) + # return band def open_store_variable(self, var): - if var != _VARNAME: - raise ValueError('Rasterio variables are all named %s' % _VARNAME) - data = indexing.LazilyIndexedArray(RasterioArrayWrapper(var, self.ds)) - return Variable(var.dimensions, data, var.attributes) + if var != __rio_varname__: + raise ValueError( + 'Rasterio variables are all named %s' % __rio_varname__) + data = indexing.LazilyIndexedArray( + RasterioArrayWrapper(self.ds)) + return Variable(self.dims, data, self._attrs) def get_variables(self): - return FrozenOrderedDict((k, self.open_store_variable(v)) - for k, v in self.ds.variables.items()) + return FrozenOrderedDict( + {__rio_varname__: self.open_store_variable(__rio_varname__)}) def get_attrs(self): - return Frozen(self.ds.attributes) + return Frozen(self._attrs) def get_dimensions(self): - return Frozen(self.ds.dimensions) + return Frozen(self.ds.dims) def close(self): self.ds.close() @@ -110,23 +117,26 @@ def _transform_proj(p1, p2, x, y, nocopy=False): return pyproj.transform(p1, p2, x, y) -def _try_to_get_latlon_coords(da): - import pyproj - if 'crs' in da.attrs: - proj = pyproj.Proj(da.attrs['crs']) - x, y = np.meshgrid(da['x'], da['y']) +def _try_to_get_latlon_coords(coords, attrs): + try: + import pyproj + except ImportError: + pyproj = False + if 'crs' in attrs and pyproj: + proj = pyproj.Proj(attrs['crs']) + x, y = np.meshgrid(coords['x'], coords['y']) proj_out = pyproj.Proj("+init=EPSG:4326", preserve_units=True) xc, yc = _transform_proj(proj, proj_out, x, y) - coords = dict(y=da['y'], x=da['x']) + coords = dict(y=coords['y'], x=coords['x']) dims = ('y', 'x') - da.coords['latitude'] = \ - DataArray(yc, coords=coords, dims=dims, name='latitude', - attrs={'units': 'degrees_north', 'long_name': 'latitude', - 'standard_name': 'latitude'}) - da.coords['longitude'] = DataArray(xc, coords=coords, dims=dims, name='latitude', - attrs={'units': 'degrees_east', - 'long_name': 'longitude', - 'standard_name': 'longitude'}) + coords['latitude'] = DataArray( + data=yc, coords=coords, dims=dims, name='latitude', + attrs={'units': 'degrees_north', 'long_name': 'latitude', + 'standard_name': 'latitude'}) + coords['longitude'] = DataArray( + data=xc, coords=coords, dims=dims, name='latitude', + attrs={'units': 'degrees_east', 'long_name': 'longitude', + 'standard_name': 'longitude'}) - return da + return coords diff --git a/xarray/test/__init__.py b/xarray/test/__init__.py index e75d7268e50..2cff7ab495f 100644 --- a/xarray/test/__init__.py +++ b/xarray/test/__init__.py @@ -49,9 +49,9 @@ try: import rasterio - has_rasterioio = True + has_rasterio = True except ImportError: - has_rasterioio = False + has_rasterio = False try: diff --git a/xarray/test/test_backends.py b/xarray/test/test_backends.py index 141db285d63..480bf645f38 100644 --- a/xarray/test/test_backends.py +++ b/xarray/test/test_backends.py @@ -1077,7 +1077,7 @@ def test_orthogonal_indexing(self): def roundtrip(self, data, save_kwargs={}, open_kwargs={}): with create_tmp_file() as tmp_file: data.to_netcdf(tmp_file, engine='scipy', **save_kwargs) - with open_dataset(tmp_file, engine='pynio', **open_kwargs) as ds: + with open_dataset(tmp_file, engine='rasterio', **open_kwargs) as ds: yield ds def test_weakrefs(self): @@ -1086,7 +1086,7 @@ def test_weakrefs(self): with create_tmp_file() as tmp_file: example.to_netcdf(tmp_file, engine='scipy') - on_disk = open_dataset(tmp_file, engine='pynio') + on_disk = open_dataset(tmp_file, engine='rasterio') actual = on_disk.rename({'foo': 'bar', 'x': 'y'}) del on_disk # trigger garbage collection self.assertDatasetIdentical(actual, expected) From 9855d32a937ddd5537b047aebbb2ff4a02aad7a1 Mon Sep 17 00:00:00 2001 From: NicWayand Date: Sat, 29 Oct 2016 22:06:26 -0600 Subject: [PATCH 04/14] keep band dim even for single band. Fix longitude typo --- xarray/backends/rasterio_.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 521ca85d212..13038fe35e6 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -47,11 +47,9 @@ def __init__(self, filename, mode='r'): 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} # Get dims - if self.ds.count >= 2: + if self.ds.count >= 1: self.dims = ('band', 'y', 'x') self.coords['band'] = self.ds.indexes - elif self.ds.count == 1: - self.dims = ('y', 'x') else: raise ValueError('unknown dims') @@ -65,7 +63,6 @@ def __init__(self, filename, mode='r'): self.coords = _try_to_get_latlon_coords(self.coords, self._attrs) - # def get_vardata(self, var_id=1): # """Read the geotiff band. # Parameters @@ -135,8 +132,7 @@ def _try_to_get_latlon_coords(coords, attrs): attrs={'units': 'degrees_north', 'long_name': 'latitude', 'standard_name': 'latitude'}) coords['longitude'] = DataArray( - data=xc, coords=coords, dims=dims, name='latitude', + data=xc, coords=coords, dims=dims, name='longitude', attrs={'units': 'degrees_east', 'long_name': 'longitude', 'standard_name': 'longitude'}) - return coords From 2a94fa4e8e0ae2e5faa016f3adeea19757cb2cfe Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Mon, 31 Oct 2016 11:45:15 -0600 Subject: [PATCH 05/14] Fix lat/lon decoding. Remove requirment comment --- ci/requirements-py27-cdat+pynio.yml | 2 +- xarray/backends/rasterio_.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml index 9dd17b7d6ae..53aafb058e1 100644 --- a/ci/requirements-py27-cdat+pynio.yml +++ b/ci/requirements-py27-cdat+pynio.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - python=2.7 - # - cdat-lite + - cdat-lite - dask - pytest - numpy diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index 13038fe35e6..b77274f1f4b 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -60,9 +60,6 @@ def __init__(self, filename, mode='r'): except AttributeError: pass - self.coords = _try_to_get_latlon_coords(self.coords, self._attrs) - - # def get_vardata(self, var_id=1): # """Read the geotiff band. # Parameters @@ -84,8 +81,11 @@ def open_store_variable(self, var): return Variable(self.dims, data, self._attrs) def get_variables(self): - return FrozenOrderedDict( - {__rio_varname__: self.open_store_variable(__rio_varname__)}) + # Get lat lon coordinates + coords = _try_to_get_latlon_coords(self.coords, self._attrs) + vars = {__rio_varname__: self.open_store_variable(__rio_varname__)} + vars.update(coords) + return FrozenOrderedDict(vars) def get_attrs(self): return Frozen(self._attrs) @@ -115,6 +115,7 @@ def _transform_proj(p1, p2, x, y, nocopy=False): def _try_to_get_latlon_coords(coords, attrs): + coords_out = {} try: import pyproj except ImportError: @@ -127,12 +128,12 @@ def _try_to_get_latlon_coords(coords, attrs): coords = dict(y=coords['y'], x=coords['x']) dims = ('y', 'x') - coords['latitude'] = DataArray( - data=yc, coords=coords, dims=dims, name='latitude', + coords_out['lat'] = DataArray( + data=yc, coords=coords, dims=dims, name='lat', attrs={'units': 'degrees_north', 'long_name': 'latitude', 'standard_name': 'latitude'}) - coords['longitude'] = DataArray( - data=xc, coords=coords, dims=dims, name='longitude', + coords_out['lon'] = DataArray( + data=xc, coords=coords, dims=dims, name='lon', attrs={'units': 'degrees_east', 'long_name': 'longitude', 'standard_name': 'longitude'}) - return coords + return coords_out From dd71a457a65b2665398df5aa515f0aaba37737cc Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Mon, 31 Oct 2016 12:31:16 -0600 Subject: [PATCH 06/14] Attr error suppression. DataArray to Variable objects. CI requirment updates --- ci/requirements-py27-cdat+pynio.yml | 1 + ci/requirements-py27-min.yml | 1 + ci/requirements-py27-netcdf4-dev.yml | 1 + ci/requirements-py27-pydap.yml | 1 + ci/requirements-py33.yml | 1 + ci/requirements-py34.yml | 1 + ci/requirements-py35-dask-dev.yml | 1 + ci/requirements-py35.yml | 2 +- xarray/backends/rasterio_.py | 31 ++++++---------------------- 9 files changed, 14 insertions(+), 26 deletions(-) diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml index 53aafb058e1..32eb1ddf267 100644 --- a/ci/requirements-py27-cdat+pynio.yml +++ b/ci/requirements-py27-cdat+pynio.yml @@ -12,5 +12,6 @@ dependencies: - scipy - pytest-cov - cyordereddict + - rasterio>=1.0 - pip: - coveralls diff --git a/ci/requirements-py27-min.yml b/ci/requirements-py27-min.yml index 7499157dbe9..2be01f1fa1f 100644 --- a/ci/requirements-py27-min.yml +++ b/ci/requirements-py27-min.yml @@ -4,6 +4,7 @@ dependencies: - pytest - numpy==1.9.3 - pandas==0.15.0 + - rasterio>=1.0 - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py27-netcdf4-dev.yml b/ci/requirements-py27-netcdf4-dev.yml index a64782de235..f67c238b26f 100644 --- a/ci/requirements-py27-netcdf4-dev.yml +++ b/ci/requirements-py27-netcdf4-dev.yml @@ -8,6 +8,7 @@ dependencies: - numpy - pandas - scipy + - rasterio>=1.0 - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py27-pydap.yml b/ci/requirements-py27-pydap.yml index 459f049c76a..2fce8ed927c 100644 --- a/ci/requirements-py27-pydap.yml +++ b/ci/requirements-py27-pydap.yml @@ -8,6 +8,7 @@ dependencies: - numpy - pandas - scipy + - rasterio>=1.0 - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py33.yml b/ci/requirements-py33.yml index 7ff08a5794f..5d7757a8e63 100644 --- a/ci/requirements-py33.yml +++ b/ci/requirements-py33.yml @@ -3,6 +3,7 @@ dependencies: - python=3.3 - pytest - pandas + - rasterio>=1.0 - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py34.yml b/ci/requirements-py34.yml index a49611751ca..047021ff193 100644 --- a/ci/requirements-py34.yml +++ b/ci/requirements-py34.yml @@ -4,6 +4,7 @@ dependencies: - bottleneck - pytest - pandas + - rasterio>=1.0 - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py35-dask-dev.yml b/ci/requirements-py35-dask-dev.yml index eab9ad21877..01e13ce4a64 100644 --- a/ci/requirements-py35-dask-dev.yml +++ b/ci/requirements-py35-dask-dev.yml @@ -7,6 +7,7 @@ dependencies: - pandas - scipy - toolz + - rasterio>=1.0 - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index f926cb54398..b085c6bf2cb 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -11,7 +11,7 @@ dependencies: - pandas - seaborn - scipy - - rasterio + - rasterio>=1.0 - pip: - coveralls - pytest-cov diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index b77274f1f4b..f8fa628458e 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -8,7 +8,7 @@ from .. import Variable, DataArray from ..core.utils import FrozenOrderedDict, Frozen, NDArrayMixin from ..core import indexing -from ..core.pycompat import OrderedDict +from ..core.pycompat import OrderedDict, suppress from .common import AbstractDataStore @@ -54,23 +54,9 @@ def __init__(self, filename, mode='r'): raise ValueError('unknown dims') self._attrs = OrderedDict() - for attr_name in ['crs', 'transform', 'proj']: - try: - self._attrs[attr_name] = getattr(self.ds, attr_name) - except AttributeError: - pass - - # def get_vardata(self, var_id=1): - # """Read the geotiff band. - # Parameters - # ---------- - # var_id: the variable name (here the band number) - # """ - # # wx = (self.sub_x[0], self.sub_x[1] + 1) - # # wy = (self.sub_y[0], self.sub_y[1] + 1) - # with rasterio.Env(): - # band = self.ds.read() # var_id, window=(wy, wx)) - # return band + with suppress(AttributeError): + for attr_name in ['crs', 'transform', 'proj']: + self._attrs[attr_name] = getattr(self.ds, attr_name) def open_store_variable(self, var): if var != __rio_varname__: @@ -125,15 +111,10 @@ def _try_to_get_latlon_coords(coords, attrs): x, y = np.meshgrid(coords['x'], coords['y']) proj_out = pyproj.Proj("+init=EPSG:4326", preserve_units=True) xc, yc = _transform_proj(proj, proj_out, x, y) - coords = dict(y=coords['y'], x=coords['x']) dims = ('y', 'x') - coords_out['lat'] = DataArray( - data=yc, coords=coords, dims=dims, name='lat', - attrs={'units': 'degrees_north', 'long_name': 'latitude', + coords_out['lat'] = Variable(dims,yc,attrs={'units': 'degrees_north', 'long_name': 'latitude', 'standard_name': 'latitude'}) - coords_out['lon'] = DataArray( - data=xc, coords=coords, dims=dims, name='lon', - attrs={'units': 'degrees_east', 'long_name': 'longitude', + coords_out['lon'] = Variable(dims,xc,attrs={'units': 'degrees_east', 'long_name': 'longitude', 'standard_name': 'longitude'}) return coords_out From 1db053337a607f60ce541f46f04b5650ffc1c679 Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Mon, 31 Oct 2016 12:46:23 -0600 Subject: [PATCH 07/14] missed ci/requirements-py35-pandas-dev.yml --- ci/requirements-py35-pandas-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements-py35-pandas-dev.yml b/ci/requirements-py35-pandas-dev.yml index 74f8abe84f0..4a28ffe6dda 100644 --- a/ci/requirements-py35-pandas-dev.yml +++ b/ci/requirements-py35-pandas-dev.yml @@ -7,6 +7,7 @@ dependencies: - netcdf4=1.1.9 - scipy=0.16.0 - toolz + - rasterio>=1.0 - pip: - coveralls - pytest-cov From e094becf637b10fcdc3725455657186e1c6feadb Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Mon, 31 Oct 2016 12:57:51 -0600 Subject: [PATCH 08/14] remove >= requirment --- ci/requirements-py27-cdat+pynio.yml | 2 +- ci/requirements-py27-min.yml | 2 +- ci/requirements-py27-netcdf4-dev.yml | 2 +- ci/requirements-py27-pydap.yml | 2 +- ci/requirements-py33.yml | 2 +- ci/requirements-py34.yml | 2 +- ci/requirements-py35-dask-dev.yml | 2 +- ci/requirements-py35-pandas-dev.yml | 2 +- ci/requirements-py35.yml | 2 +- doc/_static/dataset-diagram-build.sh | 0 10 files changed, 9 insertions(+), 9 deletions(-) mode change 100755 => 100644 doc/_static/dataset-diagram-build.sh diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml index 32eb1ddf267..46e3c0689d8 100644 --- a/ci/requirements-py27-cdat+pynio.yml +++ b/ci/requirements-py27-cdat+pynio.yml @@ -12,6 +12,6 @@ dependencies: - scipy - pytest-cov - cyordereddict - - rasterio>=1.0 + - rasterio - pip: - coveralls diff --git a/ci/requirements-py27-min.yml b/ci/requirements-py27-min.yml index 2be01f1fa1f..7d795ce580c 100644 --- a/ci/requirements-py27-min.yml +++ b/ci/requirements-py27-min.yml @@ -4,7 +4,7 @@ dependencies: - pytest - numpy==1.9.3 - pandas==0.15.0 - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py27-netcdf4-dev.yml b/ci/requirements-py27-netcdf4-dev.yml index f67c238b26f..ca570862e49 100644 --- a/ci/requirements-py27-netcdf4-dev.yml +++ b/ci/requirements-py27-netcdf4-dev.yml @@ -8,7 +8,7 @@ dependencies: - numpy - pandas - scipy - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py27-pydap.yml b/ci/requirements-py27-pydap.yml index 2fce8ed927c..faafddd6764 100644 --- a/ci/requirements-py27-pydap.yml +++ b/ci/requirements-py27-pydap.yml @@ -8,7 +8,7 @@ dependencies: - numpy - pandas - scipy - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py33.yml b/ci/requirements-py33.yml index 5d7757a8e63..212a8191361 100644 --- a/ci/requirements-py33.yml +++ b/ci/requirements-py33.yml @@ -3,7 +3,7 @@ dependencies: - python=3.3 - pytest - pandas - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py34.yml b/ci/requirements-py34.yml index 047021ff193..1f41be029d3 100644 --- a/ci/requirements-py34.yml +++ b/ci/requirements-py34.yml @@ -4,7 +4,7 @@ dependencies: - bottleneck - pytest - pandas - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py35-dask-dev.yml b/ci/requirements-py35-dask-dev.yml index 01e13ce4a64..d56f74afd9a 100644 --- a/ci/requirements-py35-dask-dev.yml +++ b/ci/requirements-py35-dask-dev.yml @@ -7,7 +7,7 @@ dependencies: - pandas - scipy - toolz - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py35-pandas-dev.yml b/ci/requirements-py35-pandas-dev.yml index 4a28ffe6dda..2fe30a3d027 100644 --- a/ci/requirements-py35-pandas-dev.yml +++ b/ci/requirements-py35-pandas-dev.yml @@ -7,7 +7,7 @@ dependencies: - netcdf4=1.1.9 - scipy=0.16.0 - toolz - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index b085c6bf2cb..f926cb54398 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -11,7 +11,7 @@ dependencies: - pandas - seaborn - scipy - - rasterio>=1.0 + - rasterio - pip: - coveralls - pytest-cov diff --git a/doc/_static/dataset-diagram-build.sh b/doc/_static/dataset-diagram-build.sh old mode 100755 new mode 100644 From 21f4343fd53fef3fd8fa11088ed96f2423f16828 Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Mon, 31 Oct 2016 13:00:16 -0600 Subject: [PATCH 09/14] added conda-forge channel to CI check --- ci/requirements-py27-min.yml | 2 ++ ci/requirements-py27-netcdf4-dev.yml | 2 ++ ci/requirements-py27-pydap.yml | 2 ++ ci/requirements-py33.yml | 2 ++ ci/requirements-py34.yml | 2 ++ ci/requirements-py35-dask-dev.yml | 2 ++ ci/requirements-py35-pandas-dev.yml | 2 ++ ci/requirements-py35.yml | 2 ++ 8 files changed, 16 insertions(+) diff --git a/ci/requirements-py27-min.yml b/ci/requirements-py27-min.yml index 7d795ce580c..c82354ff5bc 100644 --- a/ci/requirements-py27-min.yml +++ b/ci/requirements-py27-min.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=2.7 - pytest diff --git a/ci/requirements-py27-netcdf4-dev.yml b/ci/requirements-py27-netcdf4-dev.yml index ca570862e49..b8c3d8f9e94 100644 --- a/ci/requirements-py27-netcdf4-dev.yml +++ b/ci/requirements-py27-netcdf4-dev.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=2.7 - cython diff --git a/ci/requirements-py27-pydap.yml b/ci/requirements-py27-pydap.yml index faafddd6764..a79c64a6e17 100644 --- a/ci/requirements-py27-pydap.yml +++ b/ci/requirements-py27-pydap.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=2.7 - dask diff --git a/ci/requirements-py33.yml b/ci/requirements-py33.yml index 212a8191361..5b6a4c3d65f 100644 --- a/ci/requirements-py33.yml +++ b/ci/requirements-py33.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=3.3 - pytest diff --git a/ci/requirements-py34.yml b/ci/requirements-py34.yml index 1f41be029d3..b04a12c243c 100644 --- a/ci/requirements-py34.yml +++ b/ci/requirements-py34.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=3.4 - bottleneck diff --git a/ci/requirements-py35-dask-dev.yml b/ci/requirements-py35-dask-dev.yml index d56f74afd9a..f7416cc3215 100644 --- a/ci/requirements-py35-dask-dev.yml +++ b/ci/requirements-py35-dask-dev.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=3.5 - cython diff --git a/ci/requirements-py35-pandas-dev.yml b/ci/requirements-py35-pandas-dev.yml index 2fe30a3d027..1af4209f753 100644 --- a/ci/requirements-py35-pandas-dev.yml +++ b/ci/requirements-py35-pandas-dev.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=3.5 - cython=0.23.4 diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index f926cb54398..ddb08f03790 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -1,4 +1,6 @@ name: test_env +channels: + - conda-forge dependencies: - python=3.5 - cython From b11afaac70691d26deb81cfc5ac13442d08b7d2d Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Mon, 31 Oct 2016 13:35:37 -0600 Subject: [PATCH 10/14] add scipy requirement --- ci/requirements-py34.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements-py34.yml b/ci/requirements-py34.yml index b04a12c243c..3ad45660b80 100644 --- a/ci/requirements-py34.yml +++ b/ci/requirements-py34.yml @@ -7,6 +7,7 @@ dependencies: - pytest - pandas - rasterio + - scipy - pip: - coveralls - pytest-cov From af902f0111422e019aebe443cf0964ade0527c89 Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Mon, 31 Oct 2016 14:22:00 -0600 Subject: [PATCH 11/14] temp add verbose output --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e887ec60467..2d7629a7e51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,7 @@ install: - python setup.py install script: - - py.test xarray --cov=xarray --cov-report term-missing + - py.test xarray --cov=xarray --cov-report term-missing -v after_success: - coveralls From 0369ef891f1e8f12f5406c44dbc9c0753442ec4a Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Wed, 2 Nov 2016 13:07:26 -0400 Subject: [PATCH 12/14] roll back ci requirements. Rename vars --- ci/requirements-py27-cdat+pynio.yml | 1 - ci/requirements-py27-min.yml | 3 --- ci/requirements-py27-netcdf4-dev.yml | 3 --- ci/requirements-py27-pydap.yml | 3 --- ci/requirements-py33.yml | 3 --- ci/requirements-py35-pandas-dev.yml | 3 --- ci/requirements-py35.yml | 3 --- xarray/backends/rasterio_.py | 10 +++++----- 8 files changed, 5 insertions(+), 24 deletions(-) diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml index 46e3c0689d8..53aafb058e1 100644 --- a/ci/requirements-py27-cdat+pynio.yml +++ b/ci/requirements-py27-cdat+pynio.yml @@ -12,6 +12,5 @@ dependencies: - scipy - pytest-cov - cyordereddict - - rasterio - pip: - coveralls diff --git a/ci/requirements-py27-min.yml b/ci/requirements-py27-min.yml index c82354ff5bc..7499157dbe9 100644 --- a/ci/requirements-py27-min.yml +++ b/ci/requirements-py27-min.yml @@ -1,12 +1,9 @@ name: test_env -channels: - - conda-forge dependencies: - python=2.7 - pytest - numpy==1.9.3 - pandas==0.15.0 - - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py27-netcdf4-dev.yml b/ci/requirements-py27-netcdf4-dev.yml index b8c3d8f9e94..a64782de235 100644 --- a/ci/requirements-py27-netcdf4-dev.yml +++ b/ci/requirements-py27-netcdf4-dev.yml @@ -1,6 +1,4 @@ name: test_env -channels: - - conda-forge dependencies: - python=2.7 - cython @@ -10,7 +8,6 @@ dependencies: - numpy - pandas - scipy - - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py27-pydap.yml b/ci/requirements-py27-pydap.yml index a79c64a6e17..459f049c76a 100644 --- a/ci/requirements-py27-pydap.yml +++ b/ci/requirements-py27-pydap.yml @@ -1,6 +1,4 @@ name: test_env -channels: - - conda-forge dependencies: - python=2.7 - dask @@ -10,7 +8,6 @@ dependencies: - numpy - pandas - scipy - - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py33.yml b/ci/requirements-py33.yml index 5b6a4c3d65f..7ff08a5794f 100644 --- a/ci/requirements-py33.yml +++ b/ci/requirements-py33.yml @@ -1,11 +1,8 @@ name: test_env -channels: - - conda-forge dependencies: - python=3.3 - pytest - pandas - - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py35-pandas-dev.yml b/ci/requirements-py35-pandas-dev.yml index 1af4209f753..74f8abe84f0 100644 --- a/ci/requirements-py35-pandas-dev.yml +++ b/ci/requirements-py35-pandas-dev.yml @@ -1,6 +1,4 @@ name: test_env -channels: - - conda-forge dependencies: - python=3.5 - cython=0.23.4 @@ -9,7 +7,6 @@ dependencies: - netcdf4=1.1.9 - scipy=0.16.0 - toolz - - rasterio - pip: - coveralls - pytest-cov diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index ddb08f03790..0f3b005ea6a 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -1,6 +1,4 @@ name: test_env -channels: - - conda-forge dependencies: - python=3.5 - cython @@ -13,7 +11,6 @@ dependencies: - pandas - seaborn - scipy - - rasterio - pip: - coveralls - pytest-cov diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index f8fa628458e..5d32c25cb2c 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -44,7 +44,7 @@ def __init__(self, filename, mode='r'): dx, dy = self.ds.res[0], -self.ds.res[1] self.coords = {'y': np.arange(start=y0, stop=(y0 + ny * dy), step=dy), - 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} + 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} # Get dims if self.ds.count >= 1: @@ -56,7 +56,7 @@ def __init__(self, filename, mode='r'): self._attrs = OrderedDict() with suppress(AttributeError): for attr_name in ['crs', 'transform', 'proj']: - self._attrs[attr_name] = getattr(self.ds, attr_name) + self._attrs[attr_name] = getattr(self.ds, attr_name) def open_store_variable(self, var): if var != __rio_varname__: @@ -69,9 +69,9 @@ def open_store_variable(self, var): def get_variables(self): # Get lat lon coordinates coords = _try_to_get_latlon_coords(self.coords, self._attrs) - vars = {__rio_varname__: self.open_store_variable(__rio_varname__)} - vars.update(coords) - return FrozenOrderedDict(vars) + rio_vars = {__rio_varname__: self.open_store_variable(__rio_varname__)} + rio_vars.update(coords) + return FrozenOrderedDict(rio_vars) def get_attrs(self): return Frozen(self._attrs) From bb7342905c1d3b43d454e9aeb033ebd85f3ba6c9 Mon Sep 17 00:00:00 2001 From: Nicolas Wayand Date: Wed, 2 Nov 2016 13:11:47 -0400 Subject: [PATCH 13/14] roll back --- ci/requirements-py35-dask-dev.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/requirements-py35-dask-dev.yml b/ci/requirements-py35-dask-dev.yml index f7416cc3215..eab9ad21877 100644 --- a/ci/requirements-py35-dask-dev.yml +++ b/ci/requirements-py35-dask-dev.yml @@ -1,6 +1,4 @@ name: test_env -channels: - - conda-forge dependencies: - python=3.5 - cython @@ -9,7 +7,6 @@ dependencies: - pandas - scipy - toolz - - rasterio - pip: - coveralls - pytest-cov From 7037e9d80f761dd7e35aa85ea1535818bd6d3a6c Mon Sep 17 00:00:00 2001 From: NicWayand Date: Tue, 8 Nov 2016 14:54:21 -0600 Subject: [PATCH 14/14] fixed coord spacing bug where x and y were +1 dim than raster. Uses np.linspace now. --- xarray/backends/rasterio_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index f8fa628458e..0abd10d0342 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -43,8 +43,8 @@ def __init__(self, filename, mode='r'): x0, y0 = self.ds.bounds.left, self.ds.bounds.top dx, dy = self.ds.res[0], -self.ds.res[1] - self.coords = {'y': np.arange(start=y0, stop=(y0 + ny * dy), step=dy), - 'x': np.arange(start=x0, stop=(x0 + nx * dx), step=dx)} + self.coords = {'y': np.linspace(start=y0, num=ny, stop=(y0 + (ny-1) * dy)), + 'x': np.linspace(start=x0, num=nx, stop=(x0 + (nx-1) * dx))} # Get dims if self.ds.count >= 1: