python/plugins: turn on GDAL exceptions to avoid deprecation warning

doc/api_break.dox

@@ -17,6 +17,17 @@ https://github.com/qgis/qgis4.0_api/issues
 
 This page maintains a list of incompatible changes that happened in previous releases.
 
+QGIS 3.38        {#qgis_api_break_3_38}
+=========
+
+Enabling GDAL Python exceptions
+-------------------------------
+
+- QGIS now turns on exceptions in the osgeo.gdal, osgeo.ogr and osgeo.osr modules.
+  This may affect plugins that use the GDAL Python API. Typically gdal.Open,
+  gdal.OpenEx or ogr.Open() (non exclusive list) will now throw an exception
+  if the dataset cannot be opened.
+
 QGIS 3.24        {#qgis_api_break_3_24}
 =========
 

python/plugins/db_manager/db_plugins/gpkg/connector.py

@@ -39,6 +39,8 @@
 import sqlite3
 
 from osgeo import gdal, ogr, osr
+gdal.UseExceptions()
+ogr.UseExceptions()
 
 
 def classFactory():
@@ -75,11 +77,13 @@ def _opendb(self):
         # Keep this explicit assignment to None to make sure the file is
         # properly closed before being re-opened
         self.gdal_ds = None
-        self.gdal_ds = gdal.OpenEx(self.dbname, gdal.OF_UPDATE)
-        if self.gdal_ds is None:
-            self.gdal_ds = gdal.OpenEx(self.dbname)
-        if self.gdal_ds is None:
-            raise ConnectionError(QApplication.translate("DBManagerPlugin", '"{0}" not found').format(self.dbname))
+        try:
+            self.gdal_ds = gdal.OpenEx(self.dbname, gdal.OF_UPDATE)
+        except Exception:
+            try:
+                self.gdal_ds = gdal.OpenEx(self.dbname)
+            except Exception:
+                raise ConnectionError(QApplication.translate("DBManagerPlugin", '"{0}" not found').format(self.dbname))
         if self.gdal_ds.GetDriver().ShortName != 'GPKG':
             raise ConnectionError(QApplication.translate("DBManagerPlugin", '"{dbname}" not recognized as GPKG ({shortname} reported instead.)').format(dbname=self.dbname, shortname=self.gdal_ds.GetDriver().ShortName))
         self.has_raster = self.gdal_ds.RasterCount != 0 or self.gdal_ds.GetMetadata('SUBDATASETS') is not None
@@ -186,15 +190,11 @@ def cancel(self):
 
     @classmethod
     def isValidDatabase(cls, path):
-        if hasattr(gdal, 'OpenEx'):
+        try:
             ds = gdal.OpenEx(path)
-            if ds is None or ds.GetDriver().ShortName != 'GPKG':
-                return False
-        else:
-            ds = ogr.Open(path)
-            if ds is None or ds.GetDriver().GetName() != 'GPKG':
-                return False
-        return True
+        except Exception:
+            return False
+        return ds.GetDriver().ShortName == 'GPKG'
 
     def getInfo(self):
         return None
@@ -442,7 +442,10 @@ def getTableExtent(self, table, geom, force=False):
                 ds = self.gdal_ds
             else:
                 subdataset_name = 'GPKG:%s:%s' % (self.gdal_ds.GetDescription(), tablename)
-                ds = gdal.Open(subdataset_name)
+                try:
+                    ds = gdal.Open(subdataset_name)
+                except Exception:
+                    ds = None
             if ds is None:
                 return None
             gt = ds.GetGeoTransform()

python/plugins/grassprovider/tests/AlgorithmsTestBase.py

@@ -53,6 +53,8 @@
 
 import processing
 
+gdal.UseExceptions()
+
 
 def processingTestDataPath():
     return os.path.join(os.path.dirname(__file__), 'testdata')

python/plugins/processing/algs/gdal/GdalAlgorithmProvider.py

@@ -93,6 +93,8 @@
 pluginPath = os.path.normpath(os.path.join(
     os.path.split(os.path.dirname(__file__))[0], os.pardir))
 
+gdal.UseExceptions()
+
 
 class GdalAlgorithmProvider(QgsProcessingProvider):
 

python/plugins/processing/algs/gdal/GdalUtils.py

@@ -27,10 +27,6 @@
 
 import psycopg2
 
-with warnings.catch_warnings():
-    warnings.filterwarnings("ignore", category=DeprecationWarning)
-    from osgeo import ogr
-
 from qgis.core import (Qgis,
                        QgsBlockingProcess,
                        QgsRunProcess,
@@ -55,9 +51,9 @@
 from processing.tools.system import isWindows, isMac
 
 try:
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", category=DeprecationWarning)
-        from osgeo import gdal  # NOQA
+    from osgeo import gdal, ogr
+    gdal.UseExceptions()
+    ogr.UseExceptions()
 
     gdalAvailable = True
 except:
@@ -436,8 +432,9 @@ def ogrLayerName(uri):
             if f.startswith('layerid='):
                 layerid = int(f.split('=')[1])
 
-        ds = ogr.Open(basePath)
-        if not ds:
+        try:
+            ds = gdal.OpenEx(basePath, gdal.OF_VECTOR)
+        except Exception:
             return None
 
         ly = ds.GetLayer(layerid)

python/plugins/processing/algs/gdal/extractprojection.py

@@ -33,6 +33,9 @@
 
 pluginPath = os.path.split(os.path.split(os.path.dirname(__file__))[0])[0]
 
+gdal.UseExceptions()
+osr.UseExceptions()
+
 
 class ExtractProjection(GdalAlgorithm):
     INPUT = 'INPUT'

python/plugins/processing/algs/qgis/HypsometricCurves.py

@@ -24,6 +24,10 @@
 
 from osgeo import gdal, ogr, osr
 
+gdal.UseExceptions()
+ogr.UseExceptions()
+osr.UseExceptions()
+
 from qgis.core import (QgsRectangle,
                        QgsGeometry,
                        QgsFeatureRequest,

python/plugins/processing/algs/qgis/PointsFromLines.py

@@ -37,6 +37,8 @@
 from processing.tools import raster
 from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm
 
+gdal.UseExceptions()
+
 
 class PointsFromLines(QgisAlgorithm):
     INPUT_RASTER = 'INPUT_RASTER'

python/plugins/processing/gui/TestTools.py

@@ -53,6 +53,8 @@
 from qgis.PyQt.QtCore import QCoreApplication, QMetaObject
 from qgis.PyQt.QtWidgets import QDialog, QVBoxLayout, QTextEdit, QMessageBox
 
+gdal.UseExceptions()
+
 
 def extractSchemaPath(filepath):
     """
@@ -264,8 +266,9 @@ def createTest(text):
                                        'files'))
                 return
 
-            dataset = gdal.Open(token, GA_ReadOnly)
-            if dataset is None:
+            try:
+                dataset = gdal.Open(token, GA_ReadOnly)
+            except Exception:
                 QMessageBox.warning(None,
                                     tr('Error'),
                                     tr('Seems some outputs are temporary '

python/plugins/processing/tests/AlgorithmsTestBase.py

@@ -54,6 +54,8 @@
 
 import processing
 
+gdal.UseExceptions()
+
 
 def GDAL_COMPUTE_VERSION(maj, min, rev):
     return ((maj) * 1000000 + (min) * 10000 + (rev) * 100)

python/plugins/processing/tools/raster.py

@@ -25,6 +25,8 @@
 
 from qgis.core import QgsProcessingException
 
+gdal.UseExceptions()
+
 
 def scanraster(layer, feedback, band_number=1):
     filename = str(layer.source())

src/app/qgisapp.cpp

@@ -12523,6 +12523,7 @@ void QgisApp::loadPythonSupport()
     // init python runner
     QgsPythonRunner::setInstance( new QgsPythonRunnerImpl( mPythonUtils ) );
 
+    mPythonUtils->initGDAL();
     // QgsMessageLog::logMessage( tr( "Python support ENABLED :-) " ), QString(), Qgis::MessageLevel::Info );
   }
 #endif

src/python/qgspythonutils.h

@@ -218,6 +218,11 @@ class PYTHON_EXPORT QgsPythonUtils
      */
     virtual bool unloadPlugin( const QString &packageName ) = 0;
 
+    /**
+     * Initialize GDAL Python, turning on its exceptions.
+     * \since QGIS 3.38
+     */
+    virtual void initGDAL() = 0;
 };
 
 #endif

src/python/qgspythonutilsimpl.cpp

@@ -755,3 +755,12 @@ QStringList QgsPythonUtilsImpl::listActivePlugins()
   evalString( QStringLiteral( "'\\n'.join(qgis.utils.active_plugins)" ), output );
   return output.split( QChar( '\n' ), Qt::SkipEmptyParts );
 }
+
+void QgsPythonUtilsImpl::initGDAL()
+{
+  runString("from osgeo import gdal, ogr, osr");
+  // To avoid FutureWarning with GDAL >= 3.7.0
+  runString("gdal.UseExceptions()");
+  runString("ogr.UseExceptions()");
+  runString("osr.UseExceptions()");
+}

src/python/qgspythonutilsimpl.h

@@ -92,6 +92,7 @@ class QgsPythonUtilsImpl : public QgsPythonUtils
     bool canUninstallPlugin( const QString &packageName ) final;
     bool unloadPlugin( const QString &packageName ) final;
     bool isPluginEnabled( const QString &packageName ) const final;
+    void initGDAL() final;
 
   protected:
 

tests/src/python/test_provider_ogr_gpkg.py

@@ -2323,7 +2323,8 @@ def _testVectorLayerExporterDeferredSpatialIndex(self, layerOptions, expectSpati
 
         tmpfile = os.path.join(
             self.basetestpath, 'testVectorLayerExporterDeferredSpatialIndex.gpkg')
-        gdal.Unlink(tmpfile)
+        if os.path.exists(tmpfile):
+            os.unlink(tmpfile)
         options = {}
         options['driverName'] = 'GPKG'
         options['layerName'] = 'table1'
@@ -2442,7 +2443,10 @@ def testFixWrongMetadataReferenceColumnNameUpdate(self):
 
         ds = ogr.Open(tmpfile, update=1)
         gdal.PushErrorHandler()
-        ds.ExecuteSQL('DROP TRIGGER gpkg_metadata_reference_column_name_update')
+        try:
+            ds.ExecuteSQL('DROP TRIGGER gpkg_metadata_reference_column_name_update')
+        except Exception:
+            pass
         gdal.PopErrorHandler()
         # inject wrong trigger on purpose
         wrong_trigger = "CREATE TRIGGER 'gpkg_metadata_reference_column_name_update' " + \

tests/src/python/test_provider_ogr_sqlite.py

@@ -566,35 +566,10 @@ def testExtentSqlite(self):
         f = None
         ds = None
 
-        # create bad 3D dataset (declared ZM but with 2.5d data)
-        tmpfile = os.path.join(self.basetestpath, 'points_with_bad_z.sqlite')
+        # create empty 3D dataset
+        tmpfile = os.path.join(self.basetestpath, 'points_z_empty.sqlite')
         ds = ogr.GetDriverByName('SQLite').CreateDataSource(tmpfile, options=['SPATIALITE=YES'])
         lyr = ds.CreateLayer('test', geom_type=ogr.wkbPointZM, options=['FID=fid'])
-        f = ogr.Feature(lyr.GetLayerDefn())
-        f.SetFID(0)
-        f.SetGeometry(ogr.CreateGeometryFromWkt('Point Z (0 0 -5)'))
-        lyr.CreateFeature(f)
-        f = ogr.Feature(lyr.GetLayerDefn())
-        f.SetFID(1)
-        f.SetGeometry(ogr.CreateGeometryFromWkt('Point Z (1 1 -10)'))
-        lyr.CreateFeature(f)
-        f = ogr.Feature(lyr.GetLayerDefn())
-        f.SetFID(2)
-        f.SetGeometry(ogr.CreateGeometryFromWkt('Point Z (2 2 -15)'))
-        lyr.CreateFeature(f)
-        f = ogr.Feature(lyr.GetLayerDefn())
-        f.SetFID(3)
-        f.SetGeometry(ogr.CreateGeometryFromWkt('Point Z (3 3 5)'))
-        lyr.CreateFeature(f)
-        f = ogr.Feature(lyr.GetLayerDefn())
-        f.SetFID(4)
-        f.SetGeometry(ogr.CreateGeometryFromWkt('Point Z (4 4 10)'))
-        lyr.CreateFeature(f)
-        f = ogr.Feature(lyr.GetLayerDefn())
-        f.SetFID(5)
-        f.SetGeometry(ogr.CreateGeometryFromWkt('Point Z (5 5 15)'))
-        lyr.CreateFeature(f)
-        f = None
         ds = None
 
         # 2D points
@@ -647,7 +622,7 @@ def testExtentSqlite(self):
         self.assertAlmostEqual(vl.extent3D().zMaximum(), 15.0, places=3)
         del vl
 
-        vl = QgsVectorLayer(os.path.join(self.basetestpath, 'points_with_bad_z.sqlite'), 'test', 'ogr')
+        vl = QgsVectorLayer(os.path.join(self.basetestpath, 'points_z_empty.sqlite'), 'test', 'ogr')
         self.assertTrue(vl.isValid())
 
         self.assertTrue(math.isnan(vl.extent().xMinimum()))

tests/src/python/test_provider_shapefile.py

@@ -594,7 +594,7 @@ def testRepackAtFirstSave(self):
             shutil.copy(os.path.join(srcpath, file), tmpdir)
         datasource = os.path.join(tmpdir, 'shapefile.shp')
 
-        ds = osgeo.ogr.Open(datasource)
+        ds = osgeo.ogr.Open(datasource, update=1)
         lyr = ds.GetLayer(0)
         original_feature_count = lyr.GetFeatureCount()
         lyr.DeleteFeature(2)