Merge branch 'master' into feature/django

debian/control

@@ -23,7 +23,7 @@ Depends: ${python3:Depends},
          python3-unicodecsv,
          python3-yaml,
          ${misc:Depends}
-Suggests: python3-babel, python3-elasticsearch, python3-fiona, python3-geojson, python3-pydantic, python3-pygeometa, python3-pyproj, python3-rasterio
+Suggests: python3-babel, python3-elasticsearch (<8), python3-fiona, python3-geojson, python3-pydantic, python3-pygeometa, python3-pyproj, python3-rasterio
 Description: pygeoapi provides an API to geospatial data.
  .
  This package the pygeoapi module for Python 3.

docker/examples/elastic/ES/Dockerfile

@@ -54,7 +54,7 @@ RUN echo "discovery.type: single-node" >> config/elasticsearch.yml
 
 RUN yum --enablerepo=extras -y install epel-release \
         && yum install -y python3 python3-pip python3-setuptools python-typing \
-        && pip3 install --upgrade pip elasticsearch elasticsearch-dsl \
+        && pip3 install --upgrade pip elasticsearch==7.17.1 elasticsearch-dsl \
 	&& yum clean packages
 
 USER elasticsearch

docs/source/data-publishing/ogcapi-coverages.rst

@@ -15,7 +15,7 @@ pygeoapi core feature providers are listed below, along with a matrix of support
 parameters.
 
 .. csv-table::
-   :header: Provider, range-subset, subset, bbox, datetime
+   :header: Provider, properties, subset, bbox, datetime
    :align: left
 
    rasterio,✅,✅,✅,
@@ -96,8 +96,8 @@ Data access examples
   * http://localhost:5000/collections/foo/coverage?f=json
 * coverage access via native format (as defined in ``provider.format.name``)
   * http://localhost:5000/collections/foo/coverage?f=GRIB
-* coverage access with comma-separated range-subset
-  * http://localhost:5000/collections/foo/coverage?range-subset=1,3
+* coverage access with comma-separated properties
+  * http://localhost:5000/collections/foo/coverage?properties=1,3
 * coverage access with subsetting
   * http://localhost:5000/collections/foo/coverage?subset=lat(10,20)&subset=long(10,20)
 

docs/source/data-publishing/ogcapi-features.rst

@@ -281,7 +281,7 @@ Data access examples
 * browse features
   * http://localhost:5000/collections/foo/items
 * paging
-  * http://localhost:5000/collections/foo/items?startIndex=10&limit=10
+  * http://localhost:5000/collections/foo/items?offset=10&limit=10
 * CSV outputs
   * http://localhost:5000/collections/foo/items?f=csv
 * query features (spatial)

docs/source/data-publishing/ogcapi-records.rst

@@ -79,7 +79,7 @@ Metadata search examples
 * browse records
   * http://localhost:5000/collections/foo/items
 * paging
-  * http://localhost:5000/collections/foo/items?startIndex=10&limit=10
+  * http://localhost:5000/collections/foo/items?offset=10&limit=10
 * CSV outputs
   * http://localhost:5000/collections/foo/items?f=csv
 * query records (spatial)

docs/source/language.rst

@@ -229,7 +229,7 @@ An example Python code block for a custom provider with a language-aware ``query
    def __init__(self, provider_def):
        super().__init__(provider_def)
 
-   def query(self, startindex=0, limit=10, resulttype='results', bbox=[],
+   def query(self, offset=0, limit=10, resulttype='results', bbox=[],
              datetime_=None, properties=[], sortby=[], select_properties=[],
              skip_geometry=False, q=None, language=None):
        LOGGER.debug(f'Provider queried in {language.english_name} language')

docs/source/plugins.rst

@@ -68,7 +68,7 @@ The below template provides a minimal example (let's call the file ``mycoolvecto
                'field2': 'string'
            }
 
-       def query(self,startindex=0, limit=10, resulttype='results',
+       def query(self, offset=0, limit=10, resulttype='results',
                  bbox=[], datetime_=None, properties=[], sortby=[],
                  select_properties=[], skip_geometry=False, **kwargs):
 

pygeoapi/api.py

@@ -105,10 +105,14 @@
     'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30',
     'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html',
     'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson',
-    'http://www.opengis.net/spec/ogcapi_coverages-1/1.0/conf/core',
+    'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core',
     'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30',
     'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html',
     'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage',
+    'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset',
+    'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset',  # noqa
+    'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox',
+    'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime',  # noqa
     'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core',
     'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core',
     'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting',
@@ -1249,7 +1253,7 @@ def get_collection_items(
         headers = request.get_response_headers(SYSTEM_LOCALE)
 
         properties = []
-        reserved_fieldnames = ['bbox', 'f', 'lang', 'limit', 'startindex',
+        reserved_fieldnames = ['bbox', 'f', 'lang', 'limit', 'offset',
                                'resulttype', 'datetime', 'sortby',
                                'properties', 'skipGeometry', 'q']
 
@@ -1263,18 +1267,18 @@ def get_collection_items(
 
         LOGGER.debug('Processing query parameters')
 
-        LOGGER.debug('Processing startindex parameter')
+        LOGGER.debug('Processing offset parameter')
         try:
-            startindex = int(request.params.get('startindex'))
-            if startindex < 0:
-                msg = 'startindex value should be positive or zero'
+            offset = int(request.params.get('offset'))
+            if offset < 0:
+                msg = 'offset value should be positive or zero'
                 return self.get_exception(
                     400, headers, request.format, 'InvalidParameterValue', msg)
         except TypeError as err:
             LOGGER.warning(err)
-            startindex = 0
+            offset = 0
         except ValueError:
-            msg = 'startindex value should be an integer'
+            msg = 'offset value should be an integer'
             return self.get_exception(
                 400, headers, request.format, 'InvalidParameterValue', msg)
 
@@ -1407,7 +1411,7 @@ def get_collection_items(
         prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale)
 
         LOGGER.debug('Querying provider')
-        LOGGER.debug('startindex: {}'.format(startindex))
+        LOGGER.debug('offset: {}'.format(offset))
         LOGGER.debug('limit: {}'.format(limit))
         LOGGER.debug('resulttype: {}'.format(resulttype))
         LOGGER.debug('sortby: {}'.format(sortby))
@@ -1420,7 +1424,7 @@ def get_collection_items(
         LOGGER.debug('q: {}'.format(q))
 
         try:
-            content = p.query(startindex=startindex, limit=limit,
+            content = p.query(offset=offset, limit=limit,
                               resulttype=resulttype, bbox=bbox,
                               datetime_=datetime_, properties=properties,
                               sortby=sortby,
@@ -1445,7 +1449,7 @@ def get_collection_items(
 
         serialized_query_params = ''
         for k, v in request.params.items():
-            if k not in ('f', 'startindex'):
+            if k not in ('f', 'offset'):
                 serialized_query_params += '&'
                 serialized_query_params += urllib.parse.quote(k, safe='')
                 serialized_query_params += '='
@@ -1475,26 +1479,26 @@ def get_collection_items(
                 serialized_query_params)
         }]
 
-        if startindex > 0:
-            prev = max(0, startindex - limit)
+        if offset > 0:
+            prev = max(0, offset - limit)
             content['links'].append(
                 {
                     'type': 'application/geo+json',
                     'rel': 'prev',
                     'title': 'items (prev)',
-                    'href': '{}/collections/{}/items?startindex={}{}'
+                    'href': '{}/collections/{}/items?offset={}{}'
                     .format(self.config['server']['url'], dataset, prev,
                             serialized_query_params)
                 })
 
         if len(content['features']) == limit:
-            next_ = startindex + limit
+            next_ = offset + limit
             content['links'].append(
                 {
                     'type': 'application/geo+json',
                     'rel': 'next',
                     'title': 'items (next)',
-                    'href': '{}/collections/{}/items?startindex={}{}'
+                    'href': '{}/collections/{}/items?offset={}{}'
                     .format(
                         self.config['server']['url'], dataset, next_,
                         serialized_query_params)
@@ -1529,7 +1533,7 @@ def get_collection_items(
             content['items_path'] = path_info
             content['dataset_path'] = '/'.join(path_info.split('/')[:-1])
             content['collections_path'] = '/'.join(path_info.split('/')[:-2])
-            content['startindex'] = startindex
+            content['offset'] = offset
 
             content['id_field'] = p.id_field
             if p.uri_field is not None:
@@ -1605,7 +1609,7 @@ def post_collection_items(
         headers = request.get_response_headers(SYSTEM_LOCALE)
 
         properties = []
-        reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex',
+        reserved_fieldnames = ['bbox', 'f', 'limit', 'offset',
                                'resulttype', 'datetime', 'sortby',
                                'properties', 'skipGeometry', 'q',
                                'filter-lang']
@@ -1620,18 +1624,18 @@ def post_collection_items(
 
         LOGGER.debug('Processing query parameters')
 
-        LOGGER.debug('Processing startindex parameter')
+        LOGGER.debug('Processing offset parameter')
         try:
-            startindex = int(request.params.get('startindex'))
-            if startindex < 0:
-                msg = 'startindex value should be positive or zero'
+            offset = int(request.params.get('offset'))
+            if offset < 0:
+                msg = 'offset value should be positive or zero'
                 return self.get_exception(
                     400, headers, request.format, 'InvalidParameterValue', msg)
         except TypeError as err:
             LOGGER.warning(err)
-            startindex = 0
+            offset = 0
         except ValueError:
-            msg = 'startindex value should be an integer'
+            msg = 'offset value should be an integer'
             return self.get_exception(
                 400, headers, request.format, 'InvalidParameterValue', msg)
 
@@ -1772,7 +1776,7 @@ def post_collection_items(
                 400, headers, request.format, 'InvalidParameterValue', msg)
 
         LOGGER.debug('Querying provider')
-        LOGGER.debug('startindex: {}'.format(startindex))
+        LOGGER.debug('offset: {}'.format(offset))
         LOGGER.debug('limit: {}'.format(limit))
         LOGGER.debug('resulttype: {}'.format(resulttype))
         LOGGER.debug('sortby: {}'.format(sortby))
@@ -1808,7 +1812,7 @@ def post_collection_items(
             filter_ = None
             if val:
                 filter_ = CQLModel.parse_raw(data)
-            content = p.query(startindex=startindex, limit=limit,
+            content = p.query(offset=offset, limit=limit,
                               resulttype=resulttype, bbox=bbox,
                               datetime_=datetime_, properties=properties,
                               sortby=sortby,
@@ -2043,14 +2047,14 @@ def get_collection_coverage(self, request: Union[APIRequest, Any],
             # Format explicitly set using a query parameter
             query_args['format_'] = format_ = request.format
 
-        range_subset = request.params.get('range-subset')
-        if range_subset:
-            LOGGER.debug('Processing range-subset parameter')
-            query_args['range_subset'] = [rs for
-                                          rs in range_subset.split(',') if rs]
-            LOGGER.debug('Fields: {}'.format(query_args['range_subset']))
+        properties = request.params.get('properties')
+        if properties:
+            LOGGER.debug('Processing properties parameter')
+            query_args['properties'] = [rs for
+                                        rs in properties.split(',') if rs]
+            LOGGER.debug('Fields: {}'.format(query_args['properties']))
 
-            for a in query_args['range_subset']:
+            for a in query_args['properties']:
                 if a not in p.fields:
                     msg = 'Invalid field specified'
                     return self.get_exception(

pygeoapi/openapi.py

@@ -334,10 +334,10 @@ def get_oas_30(cfg):
                     'default': False
                 }
             },
-            'startindex': {
-                'name': 'startindex',
+            'offset': {
+                'name': 'offset',
                 'in': 'query',
-                'description': 'The optional startindex parameter indicates the index within the result set from which the server shall begin presenting results in the response document.  The first element has an index of 0 (default).',  # noqa
+                'description': 'The optional offset parameter indicates the index within the result set from which the server shall begin presenting results in the response document.  The first element has an index of 0 (default).',  # noqa
                 'required': False,
                 'schema': {
                     'type': 'integer',
@@ -484,7 +484,7 @@ def get_oas_30(cfg):
                         coll_properties,
                         {'$ref': '#/components/parameters/skipGeometry'},
                         {'$ref': '{}/parameters/sortby.yaml'.format(OPENAPI_YAML['oapir'])},  # noqa
-                        {'$ref': '#/components/parameters/startindex'},
+                        {'$ref': '#/components/parameters/offset'},
                     ],
                     'responses': {
                         '200': {'$ref': '{}#/components/responses/Features'.format(OPENAPI_YAML['oapif'])},  # noqa

pygeoapi/process/manager/tinydb_.py

@@ -27,6 +27,7 @@
 #
 # =================================================================
 
+import fcntl
 import io
 import json
 import logging
@@ -55,15 +56,18 @@ def __init__(self, manager_def):
         super().__init__(manager_def)
         self.is_async = True
 
-    def _connect(self):
-
+    def _connect(self, mode='r'):
         """
         connect to manager
 
         :returns: `bool` of status of result
         """
 
         self.db = tinydb.TinyDB(self.connection)
+
+        if mode == 'w':
+            fcntl.lockf(self.db.storage._handle, fcntl.LOCK_EX)
+
         return True
 
     def destroy(self):
@@ -102,7 +106,7 @@ def add_job(self, job_metadata):
         :returns: identifier of added job
         """
 
-        self._connect()
+        self._connect(mode='w')
         doc_id = self.db.insert(job_metadata)
         self.db.close()
 
@@ -118,7 +122,7 @@ def update_job(self, job_id, update_dict):
         :returns: `bool` of status result
         """
 
-        self._connect()
+        self._connect(mode='w')
         self.db.update(update_dict, tinydb.where('identifier') == job_id)
         self.db.close()
 
@@ -139,7 +143,7 @@ def delete_job(self, job_id):
             if location and self.output_dir is not None:
                 os.remove(location)
 
-        self._connect()
+        self._connect(mode='w')
         removed = bool(self.db.remove(tinydb.where('identifier') == job_id))
         self.db.close()
 

pygeoapi/provider/csv_.py

@@ -71,13 +71,13 @@ def get_fields(self):
                 fields[f] = {'type': 'string'}
             return fields
 
-    def _load(self, startindex=0, limit=10, resulttype='results',
+    def _load(self, offset=0, limit=10, resulttype='results',
               identifier=None, bbox=[], datetime_=None, properties=[],
               select_properties=[], skip_geometry=False, q=None):
         """
         Load CSV data
 
-        :param startindex: starting record to return (default 0)
+        :param offset: starting record to return (default 0)
         :param limit: number of records to return (default 10)
         :param datetime_: temporal (datestamp or extent)
         :param resulttype: return results or hit limit (default results)
@@ -109,7 +109,7 @@ def _load(self, startindex=0, limit=10, resulttype='results',
                 feature_collection['numberMatched'] = len(list(data_))
                 return feature_collection
             LOGGER.debug('Slicing CSV rows')
-            for row in itertools.islice(data_, startindex, startindex+limit):
+            for row in itertools.islice(data_, offset, offset+limit):
                 feature = {'type': 'Feature'}
                 feature['id'] = row.pop(self.id_field)
                 if not skip_geometry:
@@ -150,13 +150,13 @@ def _load(self, startindex=0, limit=10, resulttype='results',
 
         return feature_collection
 
-    def query(self, startindex=0, limit=10, resulttype='results',
+    def query(self, offset=0, limit=10, resulttype='results',
               bbox=[], datetime_=None, properties=[], sortby=[],
               select_properties=[], skip_geometry=False, q=None, **kwargs):
         """
         CSV query
 
-        :param startindex: starting record to return (default 0)
+        :param offset: starting record to return (default 0)
         :param limit: number of records to return (default 10)
         :param resulttype: return results or hit limit (default results)
         :param bbox: bounding box [minx,miny,maxx,maxy]
@@ -170,7 +170,7 @@ def query(self, startindex=0, limit=10, resulttype='results',
         :returns: dict of GeoJSON FeatureCollection
         """
 
-        return self._load(startindex, limit, resulttype,
+        return self._load(offset, limit, resulttype,
                           properties=properties,
                           select_properties=select_properties,
                           skip_geometry=skip_geometry)