Add python opener from rasterio 1.4 (#1331)

* Add rasterio's python opener from 1.4a1 and supporting things

* Modern _path module and updated usage
* A setup.cfg for editable installs
* A Dockerfile and Makefile for testing

Resolves #1328

* Add two more virtual filesystem tests

Dockerfile

@@ -0,0 +1,30 @@
+ARG GDAL=ubuntu-small-3.6.4
+FROM ghcr.io/osgeo/gdal:${GDAL} AS gdal
+ARG PYTHON_VERSION=3.9
+ENV LANG="C.UTF-8" LC_ALL="C.UTF-8"
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository -y ppa:deadsnakes/ppa
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ g++ \
+ gdb \
+ make \
+ python3-pip \
+ python${PYTHON_VERSION} \
+ python${PYTHON_VERSION}-dev \
+ python${PYTHON_VERSION}-venv \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements*.txt ./
+RUN python${PYTHON_VERSION} -m venv /venv && \
+ /venv/bin/python -m pip install -U build pip && \
+ /venv/bin/python -m pip install -r requirements-dev.txt && \
+ /venv/bin/python -m pip list
+
+FROM gdal
+COPY . .
+RUN /venv/bin/python -m build -o wheels
+RUN /venv/bin/python -m pip install --no-index -f wheels fiona[test]
+ENTRYPOINT ["/venv/bin/fio"]
+CMD ["--help"]

Makefile

@@ -0,0 +1,54 @@
+PYTHON_VERSION ?= 3.9
+GDAL ?= ubuntu-small-3.6.4
+all: deps clean install test
+
+.PHONY: docs
+
+install:
+ python setup.py build_ext
+ pip install -e .[all]
+
+deps:
+ pip install -r requirements-dev.txt
+
+clean:
+ pip uninstall -y fiona || echo "no need to uninstall"
+ python setup.py clean --all
+ find . -name '__pycache__' -delete -print -o -name '*.pyc' -delete -print
+ touch fiona/*.pyx
+
+sdist:
+ python setup.py sdist
+
+test:
+ python -m pytest --maxfail 1 -v --cov fiona --cov-report html --pdb tests
+
+docs:
+ cd docs && make apidocs && make html
+
+doctest:
+ py.test --doctest-modules fiona --doctest-glob='*.rst' docs/*.rst
+
+dockertestimage:
+ docker build --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona:$(GDAL)-py$(PYTHON_VERSION) .
+
+dockertest: dockertestimage
+ docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)'
+
+dockershell: dockertestimage
+ docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /bin/bash'
+
+dockersdist: dockertestimage
+ docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m build --sdist'
+
+dockergdb: dockertestimage
+ docker run -it -v $(shell pwd):/app --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && gdb -ex=r --args /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)'
+
+dockerdocs: dockertestimage
+ docker run -it -v $(shell pwd):/app --entrypoint=/bin/bash fiona:$(GDAL)-py$(PYTHON_VERSION) -c 'source /venv/bin/activate && cd docs && make clean && make html'
+
+dockertestimage-amd64:
+ docker build --platform linux/amd64 --target gdal --build-arg GDAL=$(GDAL) --build-arg PYTHON_VERSION=$(PYTHON_VERSION) -t fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) .
+
+dockertest-amd64: dockertestimage-amd64
+ docker run -it -v $(shell pwd):/app -v /tmp:/tmp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --entrypoint=/bin/bash fiona-amd64:$(GDAL)-py$(PYTHON_VERSION) -c '/venv/bin/python -m pip install --editable . --no-build-isolation && /venv/bin/python -B -m pytest -m "not wheel" --cov fiona --cov-report term-missing $(OPTS)'

fiona/__init__.py

@@ -20,6 +20,7 @@
 
 """
 
+from contextlib import ExitStack
 import glob
 import logging
 import os
@@ -46,6 +47,7 @@
 )
 from fiona._env import driver_count
 from fiona._show_versions import show_versions
+from fiona._vsiopener import _opener_registration
 from fiona.collection import BytesCollection, Collection
 from fiona.drvsupport import supported_drivers
 from fiona.env import ensure_env_with_credentials, Env
@@ -60,7 +62,7 @@
  _remove,
  _remove_layer,
 )
-from fiona.path import ParsedPath, parse_path, vsi_path
+from fiona._path import _ParsedPath, _UnparsedPath, _parse_path, _vsi_path
 from fiona.vfs import parse_paths as vfs_parse_paths
 
 # These modules are imported by fiona.ogrext, but are also import here to
@@ -82,7 +84,7 @@
  "remove",
 ]
 
-__version__ = "2.0dev"
+__version__ = "1.10dev"
 __gdal_version__ = get_gdal_release_name()
 
 gdal_version = get_gdal_version_tuple()
@@ -104,6 +106,7 @@ def open(
  enabled_drivers=None,
  crs_wkt=None,
  allow_unsupported_drivers=False,
+ opener=None,
  **kwargs
 ):
  """Open a collection for read, append, or write
@@ -191,6 +194,19 @@ def open(
  Defaults to GDAL's default (WKT1_GDAL for GDAL 3).
  allow_unsupported_drivers : bool
  If set to true do not limit GDAL drivers to set set of known working.
+ opener : callable or obj, optional
+ A custom dataset opener which can serve GDAL's virtual
+ filesystem machinery via Python file-like objects. The
+ underlying file-like object is obtained by calling *opener* with
+ (*fp*, *mode*) or (*fp*, *mode* + "b") depending on the format
+ driver's native mode. *opener* must return a Python file-like
+ object that provides read, seek, tell, and close methods. Note:
+ only one opener at a time per fp, mode pair is allowed.
+
+ Alternatively, opener may be a filesystem object from a package
+ like fsspec that provides the following methods: isdir(),
+ isfile(), ls(), mtime(), open(), and size(). The exact interface
+ is defined in the fiona._vsiopener._AbstractOpener class.
  kwargs : mapping
  Other driver-specific parameters that will be interpreted by
  the OGR library as layer creation or opening options.
@@ -273,49 +289,67 @@ def func(*args, **kwds):
  # At this point, the fp argument is a string or path-like object
  # which can be converted to a string.
  else:
- # If a pathlib.Path instance is given, convert it to a string path.
- if isinstance(fp, Path):
- fp = str(fp)
+ stack = ExitStack()
 
- if vfs:
- warnings.warn(
- "The vfs keyword argument is deprecated and will be removed in version 2.0.0. Instead, pass a URL that uses a zip or tar (for example) scheme.",
- FionaDeprecationWarning,
- stacklevel=2,
- )
- path, scheme, archive = vfs_parse_paths(fp, vfs=vfs)
- path = ParsedPath(path, archive, scheme)
+ if hasattr(fp, "path") and hasattr(fp, "fs"):
+ log.debug("Detected fp is an OpenFile: fp=%r", fp)
+ raw_dataset_path = fp.path
+ opener = fp.fs.open
  else:
- path = parse_path(fp)
-
- if mode in ("a", "r"):
- colxn = Collection(
- path,
- mode,
- driver=driver,
- encoding=encoding,
- layer=layer,
- enabled_drivers=enabled_drivers,
- allow_unsupported_drivers=allow_unsupported_drivers,
- **kwargs
- )
- elif mode == "w":
- colxn = Collection(
- path,
- mode,
- crs=crs,
- driver=driver,
- schema=schema,
- encoding=encoding,
- layer=layer,
- enabled_drivers=enabled_drivers,
- crs_wkt=crs_wkt,
- allow_unsupported_drivers=allow_unsupported_drivers,
- **kwargs
- )
- else:
- raise ValueError("mode string must be one of {'r', 'w', 'a'}")
-
+ raw_dataset_path = os.fspath(fp)
+
+ try:
+ if opener:
+ log.debug("Registering opener: raw_dataset_path=%r, mode=%r, opener=%r", raw_dataset_path, mode, opener)
+ vsi_path_ctx = _opener_registration(raw_dataset_path, mode[0], opener)
+ registered_vsi_path = stack.enter_context(vsi_path_ctx)
+ log.debug("Registered vsi path: registered_vsi_path%r", registered_vsi_path)
+ path = _UnparsedPath(registered_vsi_path)
+ else:
+ if vfs:
+ warnings.warn(
+ "The vfs keyword argument is deprecated and will be removed in version 2.0.0. Instead, pass a URL that uses a zip or tar (for example) scheme.",
+ FionaDeprecationWarning,
+ stacklevel=2,
+ )
+ path, scheme, archive = vfs_parse_paths(fp, vfs=vfs)
+ path = _ParsedPath(path, archive, scheme)
+ else:
+ path = _parse_path(fp)
+
+ if mode in ("a", "r"):
+ colxn = Collection(
+ path,
+ mode,
+ driver=driver,
+ encoding=encoding,
+ layer=layer,
+ enabled_drivers=enabled_drivers,
+ allow_unsupported_drivers=allow_unsupported_drivers,
+ **kwargs
+ )
+ elif mode == "w":
+ colxn = Collection(
+ path,
+ mode,
+ crs=crs,
+ driver=driver,
+ schema=schema,
+ encoding=encoding,
+ layer=layer,
+ enabled_drivers=enabled_drivers,
+ crs_wkt=crs_wkt,
+ allow_unsupported_drivers=allow_unsupported_drivers,
+ **kwargs
+ )
+ else:
+ raise ValueError("mode string must be one of {'r', 'w', 'a'}")
+
+ except Exception:
+ stack.close()
+ raise
+
+ colxn._env = stack
  return colxn
 
 
@@ -392,8 +426,8 @@ def listdir(fp):
  if not isinstance(fp, str):
  raise TypeError("invalid path: %r" % fp)
 
- pobj = parse_path(fp)
- return _listdir(vsi_path(pobj))
+ pobj = _parse_path(fp)
+ return _listdir(_vsi_path(pobj))
 
 
 @ensure_env_with_credentials
@@ -442,13 +476,13 @@ def listlayers(fp, vfs=None, **kwargs):
  FionaDeprecationWarning,
  stacklevel=2,
  )
- pobj_vfs = parse_path(vfs)
- pobj_path = parse_path(fp)
- pobj = ParsedPath(pobj_path.path, pobj_vfs.path, pobj_vfs.scheme)
+ pobj_vfs = _parse_path(vfs)
+ pobj_path = _parse_path(fp)
+ pobj = _ParsedPath(pobj_path.path, pobj_vfs.path, pobj_vfs.scheme)
  else:
- pobj = parse_path(fp)
+ pobj = _parse_path(fp)
 
- return _listlayers(vsi_path(pobj), **kwargs)
+ return _listlayers(_vsi_path(pobj), **kwargs)
 
 
 def prop_width(val):

fiona/_env.pyx

@@ -17,6 +17,7 @@ import threading
 
 from fiona._err cimport exc_wrap_int, exc_wrap_ogrerr
 from fiona._err import CPLE_BaseError
+from fiona._vsiopener cimport install_pyopener_plugin
 from fiona.errors import EnvError
 
 level_map = {
@@ -60,7 +61,7 @@ except ImportError:
  pass
 
 
-
+cdef VSIFilesystemPluginCallbacksStruct* pyopener_plugin = NULL
 cdef bint is_64bit = sys.maxsize > 2 ** 32
 
 cdef void set_proj_search_path(object path):
@@ -408,6 +409,7 @@ cdef class GDALEnv(ConfigEnv):
 
  GDALAllRegister()
  OGRRegisterAll()
+ install_pyopener_plugin(pyopener_plugin)
 
  if 'GDAL_DATA' in os.environ:
  log.debug("GDAL_DATA found in environment.")