Add extent based filtering for SensorThings layers

python/PyQt6/core/auto_generated/providers/sensorthings/qgssensorthingsutils.sip.in

@@ -61,6 +61,25 @@ Returns ``True`` if the specified entity ``type`` can have geometry attached.
 %Docstring
 Returns a filter string which restricts results to those matching the specified
 ``entityType`` and ``wkbType``.
+%End
+
+    static QString filterForExtent( const QString &geometryField, const QgsRectangle &extent );
+%Docstring
+Returns a filter string which restricts results to those within the specified
+``extent``.
+
+The ``extent`` should always be specified in EPSG:4326.
+
+.. versionadded:: 3.38
+%End
+
+    static QString combineFilters( const QStringList &filters );
+%Docstring
+Combines a set of SensorThings API filter operators.
+
+See https://docs.ogc.org/is/18-088/18-088.html#requirement-request-data-filter
+
+.. versionadded:: 3.38
 %End
 
     static QList< Qgis::GeometryType > availableGeometryTypes( const QString &uri, Qgis::SensorThingsEntity type, QgsFeedback *feedback = 0, const QString &authCfg = QString() );

python/core/auto_generated/providers/sensorthings/qgssensorthingsutils.sip.in

@@ -61,6 +61,25 @@ Returns ``True`` if the specified entity ``type`` can have geometry attached.
 %Docstring
 Returns a filter string which restricts results to those matching the specified
 ``entityType`` and ``wkbType``.
+%End
+
+    static QString filterForExtent( const QString &geometryField, const QgsRectangle &extent );
+%Docstring
+Returns a filter string which restricts results to those within the specified
+``extent``.
+
+The ``extent`` should always be specified in EPSG:4326.
+
+.. versionadded:: 3.38
+%End
+
+    static QString combineFilters( const QStringList &filters );
+%Docstring
+Combines a set of SensorThings API filter operators.
+
+See https://docs.ogc.org/is/18-088/18-088.html#requirement-request-data-filter
+
+.. versionadded:: 3.38
 %End
 
     static QList< Qgis::GeometryType > availableGeometryTypes( const QString &uri, Qgis::SensorThingsEntity type, QgsFeedback *feedback = 0, const QString &authCfg = QString() );

src/core/providers/sensorthings/qgssensorthingsprovider.cpp

@@ -183,6 +183,13 @@ QString QgsSensorThingsProvider::htmlMetadata() const
   return metadata;
 }
 
+Qgis::DataProviderFlags QgsSensorThingsProvider::flags() const
+{
+  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
+
+  return Qgis::DataProviderFlag::FastExtent2D;
+}
+
 QgsVectorDataProvider::Capabilities QgsSensorThingsProvider::capabilities() const
 {
   QGIS_PROTECT_QOBJECT_THREAD_ACCESS
@@ -196,6 +203,8 @@ QgsVectorDataProvider::Capabilities QgsSensorThingsProvider::capabilities() cons
 
 void QgsSensorThingsProvider::setDataSourceUri( const QString &uri )
 {
+  QGIS_PROTECT_QOBJECT_THREAD_ACCESS
+
   mSharedData = std::make_shared< QgsSensorThingsSharedData >( uri );
   QgsDataProvider::setDataSourceUri( uri );
 }
@@ -210,12 +219,7 @@ QgsCoordinateReferenceSystem QgsSensorThingsProvider::crs() const
 QgsRectangle QgsSensorThingsProvider::extent() const
 {
   QGIS_PROTECT_QOBJECT_THREAD_ACCESS
-
-#if 0
   return mSharedData->extent();
-#endif
-
-  return QgsRectangle();
 }
 
 QString QgsSensorThingsProvider::name() const
@@ -342,6 +346,22 @@ QVariantMap QgsSensorThingsProviderMetadata::decodeUri( const QString &uri ) con
       break;
   }
 
+  const QStringList bbox = dsUri.param( QStringLiteral( "bbox" ) ).split( ',' );
+  if ( bbox.size() == 4 )
+  {
+    QgsRectangle r;
+    bool xminOk = false;
+    bool yminOk = false;
+    bool xmaxOk = false;
+    bool ymaxOk = false;
+    r.setXMinimum( bbox[0].toDouble( &xminOk ) );
+    r.setYMinimum( bbox[1].toDouble( &yminOk ) );
+    r.setXMaximum( bbox[2].toDouble( &xmaxOk ) );
+    r.setYMaximum( bbox[3].toDouble( &ymaxOk ) );
+    if ( xminOk && yminOk && xmaxOk && ymaxOk )
+      components.insert( QStringLiteral( "bounds" ), r );
+  }
+
   return components;
 }
 
@@ -403,6 +423,12 @@ QString QgsSensorThingsProviderMetadata::encodeUri( const QVariantMap &parts ) c
     dsUri.setWkbType( Qgis::WkbType::MultiPolygonZ );
   }
 
+  if ( parts.contains( QStringLiteral( "bounds" ) ) && parts.value( QStringLiteral( "bounds" ) ).userType() == QMetaType::type( "QgsRectangle" ) )
+  {
+    const QgsRectangle bBox = parts.value( QStringLiteral( "bounds" ) ).value< QgsRectangle >();
+    dsUri.setParam( QStringLiteral( "bbox" ), QStringLiteral( "%1,%2,%3,%4" ).arg( bBox.xMinimum() ).arg( bBox.yMinimum() ).arg( bBox.xMaximum() ).arg( bBox.yMaximum() ) );
+  }
+
   return dsUri.uri( false );
 }
 

src/core/providers/sensorthings/qgssensorthingsprovider.h

@@ -30,7 +30,7 @@
  *
  * \since QGIS 3.36
  */
-class QgsSensorThingsProvider : public QgsVectorDataProvider
+class QgsSensorThingsProvider final : public QgsVectorDataProvider
 {
     Q_OBJECT
 
@@ -41,58 +41,59 @@ class QgsSensorThingsProvider : public QgsVectorDataProvider
 
     QgsSensorThingsProvider( const QString &uri, const QgsDataProvider::ProviderOptions &providerOptions, QgsDataProvider::ReadFlags flags = QgsDataProvider::ReadFlags() );
 
-    QgsAbstractFeatureSource *featureSource() const override;
-    QString storageType() const override;
-    QgsFeatureIterator getFeatures( const QgsFeatureRequest &request = QgsFeatureRequest() ) const override;
-    Qgis::WkbType wkbType() const override;
-    long long featureCount() const override;
-    QgsFields fields() const override;
-    QgsLayerMetadata layerMetadata() const override;
-    QString htmlMetadata() const override;
+    QgsAbstractFeatureSource *featureSource() const final;
+    QString storageType() const final;
+    QgsFeatureIterator getFeatures( const QgsFeatureRequest &request = QgsFeatureRequest() ) const final;
+    Qgis::WkbType wkbType() const final;
+    long long featureCount() const final;
+    QgsFields fields() const final;
+    QgsLayerMetadata layerMetadata() const final;
+    QString htmlMetadata() const final;
 
-    QgsVectorDataProvider::Capabilities capabilities() const override;
+    Qgis::DataProviderFlags flags() const final;
+    QgsVectorDataProvider::Capabilities capabilities() const final;
 
-    QgsCoordinateReferenceSystem crs() const override;
-    void setDataSourceUri( const QString &uri ) override;
-    QgsRectangle extent() const override;
-    bool isValid() const override { return mValid; }
+    QgsCoordinateReferenceSystem crs() const final;
+    void setDataSourceUri( const QString &uri ) final;
+    QgsRectangle extent() const final;
+    bool isValid() const final { return mValid; }
 
-    QString name() const override;
-    QString description() const override;
-    bool renderInPreview( const QgsDataProvider::PreviewContext &context ) override;
+    QString name() const final;
+    QString description() const final;
+    bool renderInPreview( const QgsDataProvider::PreviewContext &context ) final;
 
     static QString providerKey();
 
-    void handlePostCloneOperations( QgsVectorDataProvider *source ) override;
+    void handlePostCloneOperations( QgsVectorDataProvider *source ) final;
 
   private:
     bool mValid = false;
     std::shared_ptr<QgsSensorThingsSharedData> mSharedData;
 
     QgsLayerMetadata mLayerMetadata;
 
-    void reloadProviderData() override;
+    void reloadProviderData() final;
 };
 
-class QgsSensorThingsProviderMetadata: public QgsProviderMetadata
+class QgsSensorThingsProviderMetadata final: public QgsProviderMetadata
 {
     Q_OBJECT
 
   public:
     QgsSensorThingsProviderMetadata();
-    QIcon icon() const override;
-    QList<QgsDataItemProvider *> dataItemProviders() const override;
-    QVariantMap decodeUri( const QString &uri ) const override;
-    QString encodeUri( const QVariantMap &parts ) const override;
-    QgsSensorThingsProvider *createProvider( const QString &uri, const QgsDataProvider::ProviderOptions &options, QgsDataProvider::ReadFlags flags = QgsDataProvider::ReadFlags() ) override;
-    QList< Qgis::LayerType > supportedLayerTypes() const override;
+    QIcon icon() const final;
+    QList<QgsDataItemProvider *> dataItemProviders() const final;
+    QVariantMap decodeUri( const QString &uri ) const final;
+    QString encodeUri( const QVariantMap &parts ) const final;
+    QgsSensorThingsProvider *createProvider( const QString &uri, const QgsDataProvider::ProviderOptions &options, QgsDataProvider::ReadFlags flags = QgsDataProvider::ReadFlags() ) final;
+    QList< Qgis::LayerType > supportedLayerTypes() const final;
 
     // handling of stored connections
 
-    QMap<QString, QgsAbstractProviderConnection *> connections( bool cached ) override;
-    QgsAbstractProviderConnection *createConnection( const QString &name ) override;
-    void deleteConnection( const QString &name ) override;
-    void saveConnection( const QgsAbstractProviderConnection *connection, const QString &name ) override;
+    QMap<QString, QgsAbstractProviderConnection *> connections( bool cached ) final;
+    QgsAbstractProviderConnection *createConnection( const QString &name ) final;
+    void deleteConnection( const QString &name ) final;
+    void saveConnection( const QgsAbstractProviderConnection *connection, const QString &name ) final;
 
 };
 

src/core/providers/sensorthings/qgssensorthingsshareddata.cpp

@@ -37,6 +37,7 @@ QgsSensorThingsSharedData::QgsSensorThingsSharedData( const QString &uri )
   mGeometryField = QgsSensorThingsUtils::geometryFieldForEntityType( mEntityType );
   // use initial value of maximum page size as default
   mMaximumPageSize = uriParts.value( QStringLiteral( "pageSize" ), mMaximumPageSize ).toInt();
+  mFilterExtent = uriParts.value( QStringLiteral( "bounds" ) ).value< QgsRectangle >();
 
   if ( QgsSensorThingsUtils::entityTypeHasGeometry( mEntityType ) )
   {
@@ -130,6 +131,16 @@ QUrl QgsSensorThingsSharedData::parseUrl( const QUrl &url, bool *isTestEndpoint
   return modifiedUrl;
 }
 
+QgsRectangle QgsSensorThingsSharedData::extent() const
+{
+  QgsReadWriteLocker locker( mReadWriteLock, QgsReadWriteLocker::Read );
+
+  // Since we can't retrieve the actual layer extent via SensorThings API, we use a pessimistic
+  // global extent until we've retrieved all the features from the layer
+  return hasCachedAllFeatures() ? mFetchedFeatureExtent
+         : ( !mFilterExtent.isNull() ? mFilterExtent : QgsRectangle( -180, -90, 180, 90 ) );
+}
+
 long long QgsSensorThingsSharedData::featureCount( QgsFeedback *feedback ) const
 {
   QgsReadWriteLocker locker( mReadWriteLock, QgsReadWriteLocker::Read );
@@ -142,8 +153,12 @@ long long QgsSensorThingsSharedData::featureCount( QgsFeedback *feedback ) const
   // return no features, just the total count
   QString countUri = QStringLiteral( "%1?$top=0&$count=true" ).arg( mEntityBaseUri );
   const QString typeFilter = QgsSensorThingsUtils::filterForWkbType( mEntityType, mGeometryType );
-  if ( !typeFilter.isEmpty() )
-    countUri += QStringLiteral( "&$filter=" ) + typeFilter;
+  const QString extentFilter = QgsSensorThingsUtils::filterForExtent( mGeometryField, mFilterExtent );
+  QString filterString = QgsSensorThingsUtils::combineFilters( { typeFilter, extentFilter } );
+  if ( !filterString.isEmpty() )
+    filterString = QStringLiteral( "&$filter=" ) + filterString;
+  if ( !filterString.isEmpty() )
+    countUri += filterString;
 
   const QUrl url = parseUrl( QUrl( countUri ) );
 
@@ -215,8 +230,10 @@ bool QgsSensorThingsSharedData::getFeature( QgsFeatureId id, QgsFeature &f, QgsF
     locker.changeMode( QgsReadWriteLocker::Write );
     mNextPage = QStringLiteral( "%1?$top=%2&$count=false" ).arg( mEntityBaseUri ).arg( mMaximumPageSize );
     const QString typeFilter = QgsSensorThingsUtils::filterForWkbType( mEntityType, mGeometryType );
-    if ( !typeFilter.isEmpty() )
-      mNextPage += QStringLiteral( "&$filter=" ) + typeFilter;
+    const QString extentFilter = QgsSensorThingsUtils::filterForExtent( mGeometryField, mFilterExtent );
+    const QString filterString = QgsSensorThingsUtils::combineFilters( { typeFilter, extentFilter } );
+    if ( !filterString.isEmpty() )
+      mNextPage += QStringLiteral( "&$filter=" ) + filterString;
   }
 
   locker.unlock();
@@ -243,26 +260,30 @@ bool QgsSensorThingsSharedData::getFeature( QgsFeatureId id, QgsFeature &f, QgsF
 
 QgsFeatureIds QgsSensorThingsSharedData::getFeatureIdsInExtent( const QgsRectangle &extent, QgsFeedback *feedback, const QString &thisPage, QString &nextPage, const QgsFeatureIds &alreadyFetchedIds )
 {
-  const QgsGeometry extentGeom = QgsGeometry::fromRect( extent );
+  const QgsRectangle requestExtent = mFilterExtent.isNull() ? extent : extent.intersect( mFilterExtent );
+  const QgsGeometry extentGeom = QgsGeometry::fromRect( requestExtent );
   QgsReadWriteLocker locker( mReadWriteLock, QgsReadWriteLocker::Read );
 
   if ( hasCachedAllFeatures() || mCachedExtent.contains( extentGeom ) )
   {
     // all features cached locally, rely on local spatial index
-    return qgis::listToSet( mSpatialIndex.intersects( extent ) );
+    return qgis::listToSet( mSpatialIndex.intersects( requestExtent ) );
   }
 
-  // TODO -- is using 'geography' always correct here?
   const QString typeFilter = QgsSensorThingsUtils::filterForWkbType( mEntityType, mGeometryType );
-  QString queryUrl = !thisPage.isEmpty() ? thisPage : QStringLiteral( "%1?$top=%2&$count=false&$filter=geo.intersects(%3, geography'%4')%5" ).arg( mEntityBaseUri ).arg( mMaximumPageSize ).arg( mGeometryField, extent.asWktPolygon(), typeFilter.isEmpty() ? QString() : ( QStringLiteral( " and " ) + typeFilter ) );
+  const QString extentFilter = QgsSensorThingsUtils::filterForExtent( mGeometryField, requestExtent );
+  QString filterString = QgsSensorThingsUtils::combineFilters( { extentFilter, typeFilter } );
+  if ( !filterString.isEmpty() )
+    filterString = QStringLiteral( "&$filter=" ) + filterString;
+  QString queryUrl = !thisPage.isEmpty() ? thisPage : QStringLiteral( "%1?$top=%2&$count=false%3" ).arg( mEntityBaseUri ).arg( mMaximumPageSize ).arg( filterString );
 
   if ( thisPage.isEmpty() && mCachedExtent.intersects( extentGeom ) )
   {
     // we have SOME of the results from this extent cached. Let's return those first.
     // This is slightly nicer from a rendering point of view, because panning the map won't see features
     // previously visible disappear temporarily while we wait for them to be included in the service's result set...
     nextPage = queryUrl;
-    return qgis::listToSet( mSpatialIndex.intersects( extent ) );
+    return qgis::listToSet( mSpatialIndex.intersects( requestExtent ) );
   }
 
   locker.unlock();
@@ -306,6 +327,7 @@ void QgsSensorThingsSharedData::clearCache()
   mCachedFeatures.clear();
   mIotIdToFeatureId.clear();
   mSpatialIndex = QgsSpatialIndex();
+  mFetchedFeatureExtent = QgsRectangle();
 }
 
 bool QgsSensorThingsSharedData::processFeatureRequest( QString &nextPage, QgsFeedback *feedback, const std::function< void( const QgsFeature & ) > &fetchedFeatureCallback, const std::function<bool ()> &continueFetchingCallback, const std::function<void ()> &onNoMoreFeaturesCallback )
@@ -614,6 +636,7 @@ bool QgsSensorThingsSharedData::processFeatureRequest( QString &nextPage, QgsFee
               mCachedFeatures.insert( feature.id(), feature );
               mIotIdToFeatureId.insert( iotId, feature.id() );
               mSpatialIndex.addFeature( feature );
+              mFetchedFeatureExtent.combineExtentWith( feature.geometry().boundingBox() );
 
               fetchedFeatureCallback( feature );
             }

src/core/providers/sensorthings/qgssensorthingsshareddata.h

@@ -50,6 +50,7 @@ class QgsSensorThingsSharedData
     QString error() const { return mError; }
 
     QgsCoordinateReferenceSystem crs() const { return mSourceCRS; }
+    QgsRectangle extent() const;
     long long featureCount( QgsFeedback *feedback = nullptr ) const;
 
     bool hasCachedAllFeatures() const;
@@ -82,6 +83,12 @@ class QgsSensorThingsSharedData
     Qgis::WkbType mGeometryType = Qgis::WkbType::Unknown;
     QString mGeometryField;
     QgsFields mFields;
+
+    QgsRectangle mFilterExtent;
+
+    //! Extent calculated from features actually fetched so far
+    QgsRectangle mFetchedFeatureExtent;
+
     QgsCoordinateReferenceSystem mSourceCRS;
 
     mutable long long mFeatureCount = static_cast< long long >( Qgis::FeatureCountState::Uncounted );

src/core/providers/sensorthings/qgssensorthingsutils.cpp

@@ -20,6 +20,7 @@
 #include "qgsnetworkaccessmanager.h"
 #include "qgsblockingnetworkrequest.h"
 #include "qgslogger.h"
+#include "qgsrectangle.h"
 #include <QUrl>
 #include <QNetworkRequest>
 #include <nlohmann/json.hpp>
@@ -264,6 +265,30 @@ QString QgsSensorThingsUtils::filterForWkbType( Qgis::SensorThingsEntity entityT
   return QString();
 }
 
+QString QgsSensorThingsUtils::filterForExtent( const QString &geometryField, const QgsRectangle &extent )
+{
+  // TODO -- confirm using 'geography' is always correct here
+  return ( extent.isNull() || geometryField.isEmpty() )
+         ? QString()
+         : QStringLiteral( "geo.intersects(%1, geography'%2')" ).arg( geometryField, extent.asWktPolygon() );
+}
+
+QString QgsSensorThingsUtils::combineFilters( const QStringList &filters )
+{
+  QStringList nonEmptyFilters;
+  for ( const QString &filter : filters )
+  {
+    if ( !filter.isEmpty() )
+      nonEmptyFilters.append( filter );
+  }
+  if ( nonEmptyFilters.empty() )
+    return QString();
+  if ( nonEmptyFilters.size() == 1 )
+    return nonEmptyFilters.at( 0 );
+
+  return QStringLiteral( "(" ) + nonEmptyFilters.join( QStringLiteral( ") and (" ) ) + QStringLiteral( ")" );
+}
+
 QList<Qgis::GeometryType> QgsSensorThingsUtils::availableGeometryTypes( const QString &uri, Qgis::SensorThingsEntity type, QgsFeedback *feedback, const QString &authCfg )
 {
   QNetworkRequest request = QNetworkRequest( QUrl( uri ) );