Add gdal 3.5 and 3.6 to CI, update proj/gdal install scripts to use cmake, update drvsupport

.github/workflows/ci_linux.yml

@@ -30,39 +30,34 @@ jobs:
               PROJVERSION: "7.2.1",
               allow_failure: "false",
             }
-          - {
-              python: 3.8,
-              GDALVERSION: "3.4.2",
-              PROJVERSION: "8.2.1",
-              allow_failure: "false",
-            }
           - {
               python: 3.9,
-              GDALVERSION: "3.4.2",
+              GDALVERSION: "3.4.3",
               PROJVERSION: "8.2.1",
               allow_failure: "false",
             }
           - {
               python: "3.10",
-              GDALVERSION: "3.4.2",
-              PROJVERSION: "8.2.1",
+              GDALVERSION: "3.5.3",
+              PROJVERSION: "9.0.1",
               allow_failure: "false",
             }
           - {
-              python: "3.11-dev",
-              GDALVERSION: "3.4.2",
-              PROJVERSION: "8.2.1",
+              python: "3.11",
+              GDALVERSION: "3.6.0",
+              PROJVERSION: "9.1.0",
               allow_failure: "false",
             }
 
           # Test GDAL master
           - {
-              python: "3.10",
+              python: "3.11",
               GDALVERSION: "master",
-              PROJVERSION: "8.2.1",
+              PROJVERSION: "9.1.0",
               allow_failure: "true",
             }
 
+
     env:
       CYTHON_COVERAGE: "true"
       MAKEFLAGS: "-j 4 -s"
@@ -123,7 +118,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: gdalinstall
-          key: ${{ runner.os }}-gdal-${{ matrix.GDALVERSION }}-proj-${{ matrix.PROJVERSION }}-${{ hashFiles('**/apt_list') }}
+          key: ${{ runner.os }}-gdal-${{ matrix.GDALVERSION }}-proj-${{ matrix.PROJVERSION }}-${{ hashFiles('**/apt_list') }}-${{ hashFiles('scripts/travis_gdal_install.sh', 'scripts/travis_proj_install.sh', 'scripts/travis_filegdb_install.sh') }}
 
       - name: Cache pip
         uses: actions/cache@v3
@@ -182,7 +177,7 @@ jobs:
       - name: pytest
         continue-on-error: ${{ matrix.allow_failure == 'true' }}
         run: |
-          GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -m "not wheel" -k "not [FlatGeobuf]" --cov fiona --cov-report term-missing
+          GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -m "not wheel or gdal" --cov fiona --cov-report term-missing
 
       - name: Coveralls
         continue-on-error: ${{ matrix.allow_failure == 'true' }}

fiona/drvsupport.py

@@ -44,19 +44,19 @@
     # ESRI FileGDB 	FileGDB 	Yes 	Yes 	No, needs FileGDB API library
     # multi-layer
     ("FileGDB", "raw"),
-    ("OpenFileGDB", "r"),
+    ("OpenFileGDB", "raw"),
     # ESRI Personal GeoDatabase 	PGeo 	No 	Yes 	No, needs ODBC library
     # ESRI ArcSDE 	SDE 	No 	Yes 	No, needs ESRI SDE
     # ESRIJSON 	ESRIJSON 	No 	Yes 	Yes 
     ("ESRIJSON", "r"),
     # ESRI Shapefile 	ESRI Shapefile 	Yes 	Yes 	Yes
     ("ESRI Shapefile", "raw"),
     # FMEObjects Gateway 	FMEObjects Gateway 	No 	Yes 	No, needs FME
-    ("FlatGeobuf", "rw"),
+    ("FlatGeobuf", "raw"),
     # GeoJSON 	GeoJSON 	Yes 	Yes 	Yes
     ("GeoJSON", "raw"),
     # GeoJSONSeq 	GeoJSON sequences 	Yes 	Yes 	Yes 
-    ("GeoJSONSeq", "rw"),
+    ("GeoJSONSeq", "raw"),
     # Géoconcept Export 	Geoconcept 	Yes 	Yes 	Yes
     # multi-layers
     #   ("Geoconcept", "raw"),
@@ -160,12 +160,16 @@
     'w': {'GPKG': (1, 11, 0),
           'PCIDSK': (2, 0, 0),
           'GeoJSONSeq': (2, 4, 0),
-          'FlatGeobuf': (3, 1, 3)},
+          'FlatGeobuf': (3, 1, 3),
+          'OpenFileGDB': (3, 6, 0)},
 
     'a': {'GPKG': (1, 11, 0),
           'PCIDSK': (2, 0, 0),
           'GeoJSON': (2, 1, 0),
-          'MapInfo File': (2, 0, 0)}
+          'GeoJSONSeq': (3, 6, 0),
+          'MapInfo File': (2, 0, 0),
+          'FlatGeobuf': (3, 5, 1),
+          'OpenFileGDB': (3, 6, 0)}
 }
 
 
@@ -303,8 +307,9 @@ def _driver_converts_field_type_silently_to_str(driver, field_type):
         'BNA': None,
         'DXF': None,
         'PCIDSK': (2, 1, 0),
-        'FileGDB': None,
-        'FlatGeobuf': None
+        'FileGDB': (3, 5, 0),
+        'FlatGeobuf': None,
+        'OpenFileGDB': None
     },
     'datetime': {
         'ESRI Shapefile': None,
@@ -321,8 +326,9 @@ def _driver_converts_field_type_silently_to_str(driver, field_type):
         'BNA': None,
         'DXF': None,
         'PCIDSK': (2, 1, 0),
-        'FileGDB': None,
-        'FlatGeobuf': None
+        'FileGDB': (3, 5, 0),
+        'FlatGeobuf': None,
+        'OpenFileGDB': None
     }
 }
 
@@ -381,7 +387,8 @@ def _driver_supports_timezones(driver, field_type):
 # None: driver never supports timezones, (2, 0, 0): driver supports timezones with GDAL 2.0.0
 _drivers_not_supporting_milliseconds = {
     'GPSTrackMaker': None,
-    'FileGDB': None
+    'FileGDB': None,
+    'OpenFileGDB': None
 }
 
 

pytest.ini

@@ -8,6 +8,7 @@ filterwarnings =
 markers = 
     iconv: marks tests that require gdal to be compiled with iconv
     network: marks tests that require a network connection
-    wheel: marks test that only works when installed from wheel
+    wheel: marks tests that only works when installed from wheel
+    gdal: marks tests that are only dependent on GDAL functionality (e.g. for drvsupport)
 
 testpaths = tests

requirements-dev.txt

@@ -2,7 +2,7 @@
 boto3>=1.3.1
 coverage==5.2.1
 cython~=0.29
-pytest==6.2.5
-pytest-cov==2.12.1
+pytest==7.1.2
+pytest-cov==3.0.0
 pytz==2020.1
 wheel

scripts/travis_filegdb_install.sh

@@ -4,9 +4,9 @@
 if [ ! -d "$FILEGDB" ]; then
   mkdir -p $FILEGDB
   cd $FILEGDB
-  wget -q https://github.com/Esri/file-geodatabase-api/raw/master/FileGDB_API_1.5.1/FileGDB_API_1_5_1-64gcc51.tar.gz
-  tar -xzf FileGDB_API_1_5_1-64gcc51.tar.gz --strip=1 FileGDB_API-64gcc51
-  rm FileGDB_API_1_5_1-64gcc51.tar.gz
+  wget -q https://github.com/Esri/file-geodatabase-api/raw/master/FileGDB_API_1.5.2/FileGDB_API-RHEL7-64gcc83.tar.gz
+  tar -xzf FileGDB_API-RHEL7-64gcc83.tar.gz --strip=1 FileGDB_API-RHEL7-64gcc83
+  rm FileGDB_API-RHEL7-64gcc83.tar.gz
   rm -rf samples
   rm -rf doc
 fi

scripts/travis_gdal_install.sh

@@ -47,6 +47,159 @@ GDALOPTS="  --with-ogr \
             --without-mrf \
             --with-webp=no"
 
+GDAL_CMAKE_OPTS=" -DBUILD_APPS=OFF \
+        -DBUILD_CSHARP_BINDINGS=OFF \
+        -DBUILD_JAVA_BINDINGS=OFF \
+        -DBUILD_PYTHON_BINDINGS=OFF \
+        -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_AAIGRID=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ADRG=OFF \ 
+        -DGDAL_ENABLE_DRIVER_AIRSAR=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ARG=OFF \ 
+        -DGDAL_ENABLE_DRIVER_BLX=OFF \ 
+        -DGDAL_ENABLE_DRIVER_BMP=OFF \ 
+        -DGDAL_ENABLE_DRIVER_BSB=OFF \ 
+        -DGDAL_ENABLE_DRIVER_CALS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_CEOS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_COASP=OFF \ 
+        -DGDAL_ENABLE_DRIVER_COSAR=OFF \ 
+        -DGDAL_ENABLE_DRIVER_CTG=OFF \ 
+        -DGDAL_ENABLE_DRIVER_DAAS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_DIMAP=OFF \ 
+        -DGDAL_ENABLE_DRIVER_DTED=OFF \ 
+        -DGDAL_ENABLE_DRIVER_EEDA=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ELAS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ENVISAT=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ERS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ESRIC=OFF \ 
+        -DGDAL_ENABLE_DRIVER_EXR=OFF \ 
+        -DGDAL_ENABLE_DRIVER_FIT=OFF \ 
+        -DGDAL_ENABLE_DRIVER_FITS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_GFF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_GIF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_GRIB=OFF \ 
+        -DGDAL_ENABLE_DRIVER_GSG=OFF \ 
+        -DGDAL_ENABLE_DRIVER_GTIFF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_GXF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_HDF5=OFF \ 
+        -DGDAL_ENABLE_DRIVER_HEIF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_HF2=OFF \ 
+        -DGDAL_ENABLE_DRIVER_HFA=OFF \ 
+        -DGDAL_ENABLE_DRIVER_HTTP=OFF \ 
+        -DGDAL_ENABLE_DRIVER_IDRISI=ON \ 
+        -DGDAL_ENABLE_DRIVER_ILWIS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_IRIS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_JAXAPALSAR=OFF \ 
+        -DGDAL_ENABLE_DRIVER_JDEM=OFF \ 
+        -DGDAL_ENABLE_DRIVER_JP2OPENJPEG=OFF \ 
+        -DGDAL_ENABLE_DRIVER_JPEG=OFF \ 
+        -DGDAL_ENABLE_DRIVER_KMLSUPEROVERLAY=OFF \ 
+        -DGDAL_ENABLE_DRIVER_L1B=OFF \ 
+        -DGDAL_ENABLE_DRIVER_LEVELLER=OFF \ 
+        -DGDAL_ENABLE_DRIVER_MAP=OFF \ 
+        -DGDAL_ENABLE_DRIVER_MBTILES=OFF \ 
+        -DGDAL_ENABLE_DRIVER_MEM=OFF \ 
+        -DGDAL_ENABLE_DRIVER_MRF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_MSGN=OFF \ 
+        -DGDAL_ENABLE_DRIVER_NETCDF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_NGSGEOID=OFF \ 
+        -DGDAL_ENABLE_DRIVER_NITF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_NORTHWOOD=OFF \ 
+        -DGDAL_ENABLE_DRIVER_OGCAPI=OFF \ 
+        -DGDAL_ENABLE_DRIVER_OZI=OFF \ 
+        -DGDAL_ENABLE_DRIVER_PCIDSK=ON \ 
+        -DGDAL_ENABLE_DRIVER_PCRASTER=OFF \ 
+        -DGDAL_ENABLE_DRIVER_PDF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_PDS=ON \ 
+        -DGDAL_ENABLE_DRIVER_PLMOSAIC=OFF \ 
+        -DGDAL_ENABLE_DRIVER_PNG=OFF \ 
+        -DGDAL_ENABLE_DRIVER_POSTGISRASTER=OFF \ 
+        -DGDAL_ENABLE_DRIVER_PRF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_R=OFF \ 
+        -DGDAL_ENABLE_DRIVER_RASTERLITE=OFF \ 
+        -DGDAL_ENABLE_DRIVER_RAW=OFF \ 
+        -DGDAL_ENABLE_DRIVER_RIK=OFF \ 
+        -DGDAL_ENABLE_DRIVER_RMF=OFF \ 
+        -DGDAL_ENABLE_DRIVER_RS2=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SAFE=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SAGA=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SAR_CEOS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SDTS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SENTINEL2=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SGI=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SIGDEM=OFF \ 
+        -DGDAL_ENABLE_DRIVER_SRTMHGT=OFF \ 
+        -DGDAL_ENABLE_DRIVER_STACIT=OFF \ 
+        -DGDAL_ENABLE_DRIVER_STACTA=OFF \ 
+        -DGDAL_ENABLE_DRIVER_TERRAGEN=OFF \ 
+        -DGDAL_ENABLE_DRIVER_TGA=OFF \ 
+        -DGDAL_ENABLE_DRIVER_TIL=OFF \ 
+        -DGDAL_ENABLE_DRIVER_TSX=OFF \ 
+        -DGDAL_ENABLE_DRIVER_USGSDEM=OFF \  
+        -DGDAL_ENABLE_DRIVER_VRT=OFF \ 
+        -DGDAL_ENABLE_DRIVER_WCS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_WEBP=OFF \ 
+        -DGDAL_ENABLE_DRIVER_WMS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_WMTS=OFF \ 
+        -DGDAL_ENABLE_DRIVER_XPM=OFF \ 
+        -DGDAL_ENABLE_DRIVER_XYZ=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ZARR=OFF \ 
+        -DGDAL_ENABLE_DRIVER_ZMAP=OFF \ 
+        -DOGR_BUILD_OPTIONAL_DRIVERS=OFF \ 
+        -DOGR_ENABLE_DRIVER_AVC=OFF \ 
+        -DOGR_ENABLE_DRIVER_CAD=OFF \ 
+        -DOGR_ENABLE_DRIVER_CSV=ON \ 
+        -DOGR_ENABLE_DRIVER_CSW=OFF \ 
+        -DOGR_ENABLE_DRIVER_DGN=ON \ 
+        -DOGR_ENABLE_DRIVER_DXF=ON \ 
+        -DOGR_ENABLE_DRIVER_EDIGEO=OFF \ 
+        -DOGR_ENABLE_DRIVER_ELASTIC=OFF \ 
+        -DOGR_ENABLE_DRIVER_FLATGEOBUF=ON \ 
+        -DOGR_ENABLE_DRIVER_GEOCONCEPT=OFF \ 
+        -DOGR_ENABLE_DRIVER_GEOJSON=ON \ 
+        -DOGR_ENABLE_DRIVER_GEORSS=OFF \ 
+        -DOGR_ENABLE_DRIVER_GML=ON \ 
+        -DOGR_ENABLE_DRIVER_GMT=ON \ 
+        -DOGR_ENABLE_DRIVER_GPKG=ON \ 
+        -DOGR_ENABLE_DRIVER_GPSBABEL=OFF \ 
+        -DOGR_ENABLE_DRIVER_GPX=ON \ 
+        -DOGR_ENABLE_DRIVER_IDRISI=OFF \ 
+        -DOGR_ENABLE_DRIVER_ILI=OFF \ 
+        -DOGR_ENABLE_DRIVER_JML=OFF \ 
+        -DOGR_ENABLE_DRIVER_KML=OFF \ 
+        -DOGR_ENABLE_DRIVER_LIBKML=OFF \ 
+        -DOGR_ENABLE_DRIVER_LVBAG=OFF \ 
+        -DOGR_ENABLE_DRIVER_MAPML=OFF \ 
+        -DOGR_ENABLE_DRIVER_MEM=OFF \ 
+        -DOGR_ENABLE_DRIVER_MSSQLSPATIAL=OFF \ 
+        -DOGR_ENABLE_DRIVER_MYSQL=OFF \ 
+        -DOGR_ENABLE_DRIVER_NAS=OFF \ 
+        -DOGR_ENABLE_DRIVER_NGW=OFF \ 
+        -DOGR_ENABLE_DRIVER_NTF=OFF \ 
+        -DOGR_ENABLE_DRIVER_ODBC=OFF \ 
+        -DOGR_ENABLE_DRIVER_ODS=OFF \ 
+        -DOGR_ENABLE_DRIVER_OPENFILEGDB=ON \ 
+        -DOGR_ENABLE_DRIVER_OSM=OFF \ 
+        -DOGR_ENABLE_DRIVER_PGDUMP=OFF \ 
+        -DOGR_ENABLE_DRIVER_PGEO=OFF \ 
+        -DOGR_ENABLE_DRIVER_PLSCENES=OFF \ 
+        -DOGR_ENABLE_DRIVER_S57=ON \ 
+        -DOGR_ENABLE_DRIVER_SDTS=OFF \ 
+        -DOGR_ENABLE_DRIVER_SELAFIN=OFF \ 
+        -DOGR_ENABLE_DRIVER_SHAPE=ON \ 
+        -DOGR_ENABLE_DRIVER_SQLITE=ON \ 
+        -DOGR_ENABLE_DRIVER_SVG=OFF \ 
+        -DOGR_ENABLE_DRIVER_SXF=OFF \ 
+        -DOGR_ENABLE_DRIVER_TAB=ON \ 
+        -DOGR_ENABLE_DRIVER_TIGER=OFF \ 
+        -DOGR_ENABLE_DRIVER_VDV=OFF \ 
+        -DOGR_ENABLE_DRIVER_VFK=OFF \ 
+        -DOGR_ENABLE_DRIVER_VRT=ON \ 
+        -DOGR_ENABLE_DRIVER_WASP=OFF \ 
+        -DOGR_ENABLE_DRIVER_WFS=OFF \ 
+        -DOGR_ENABLE_DRIVER_XLS=OFF \ 
+        -DOGR_ENABLE_DRIVER_XLSX=OFF "
+
 if [ -d "$FILEGDB" ]; then
   GDALOPTS="$GDALOPTS --with-fgdb=$FILEGDB"
 fi
@@ -66,7 +219,7 @@ if [ "$GDALVERSION" = "master" ]; then
     PROJOPT="--with-proj=$GDALINST/gdal-$GDALVERSION"
     cd $GDALBUILD
     git clone --depth 1 https://github.com/OSGeo/gdal gdal-$GDALVERSION
-    cd gdal-$GDALVERSION/gdal
+    cd gdal-$GDALVERSION
     echo $PROJVERSION > newproj.txt
     git rev-parse HEAD > newrev.txt
     BUILD=no
@@ -80,9 +233,12 @@ if [ "$GDALVERSION" = "master" ]; then
         mkdir -p $GDALINST/gdal-$GDALVERSION
         cp newrev.txt $GDALINST/gdal-$GDALVERSION/rev.txt
         cp newproj.txt $GDALINST/gdal-$GDALVERSION/newproj.txt
-        ./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT
-        make
-        make install
+        mkdir build
+        cd build
+        echo "cmake -DCMAKE_INSTALL_PREFIX=$GDALINST/gdal-$GDALVERSION -DPROJ_INCLUDE_DIR=$GDALINST/gdal-$GDALVERSION $GDAL_CMAKE_OPTS -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF .."
+        cmake -DCMAKE_INSTALL_PREFIX=$GDALINST/gdal-$GDALVERSION "-DPROJ_INCLUDE_DIR=$GDALINST/gdal-$GDALVERSION" $GDAL_CMAKE_OPTS -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF ..
+        cmake --build .
+        cmake --build . --target install
     fi
 
 else
@@ -120,9 +276,18 @@ else
         wget -q https://download.osgeo.org/gdal/$gdalver/gdal-$GDALVERSION.tar.gz
         tar -xzf gdal-$GDALVERSION.tar.gz
         cd gdal-$gdalver
-        ./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT
-        make
-        make install
+        if [ -f "CMakeLists.txt" ]; then
+            mkdir build
+            cd build
+            echo "cmake -DCMAKE_INSTALL_PREFIX=$GDALINST/gdal-$GDALVERSION -DPROJ_INCLUDE_DIR=$GDALINST/gdal-$GDALVERSION $GDAL_CMAKE_OPTS -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF .."
+            cmake -DCMAKE_INSTALL_PREFIX=$GDALINST/gdal-$GDALVERSION "-DPROJ_INCLUDE_DIR=$GDALINST/gdal-$GDALVERSION" $GDAL_CMAKE_OPTS -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF ..
+            cmake --build .
+            cmake --build . --target install
+        else
+            ./configure --prefix=$GDALINST/gdal-$GDALVERSION $GDALOPTS $PROJOPT
+            make
+            make install
+        fi
     fi
 fi