From a9199a619f99175ac01da433c9ffdfae11f4381c Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 11:19:48 -0700 Subject: [PATCH 1/7] Updating exception handling for rasters in validation. RE:#1645 --- HISTORY.rst | 5 +++++ src/natcap/invest/validation.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index c72a0147ea..ddcd9f5005 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,6 +37,11 @@ Unreleased Changes ------------------ +* General + * Updating validation to handle a change in exceptions raised by GDAL in + ``pygeoprocessing.get_raster_info`` and + ``pygeoprocessing.get_vector_info``. + https://github.com/natcap/invest/issues/1645 * Urban Nature Access * The model now works as expected when the user provides an LULC raster that does not have a nodata value defined. diff --git a/src/natcap/invest/validation.py b/src/natcap/invest/validation.py index 01d22526bf..c9c64cf085 100644 --- a/src/natcap/invest/validation.py +++ b/src/natcap/invest/validation.py @@ -799,7 +799,11 @@ def check_spatial_overlap(spatial_filepaths_list, for filepath in spatial_filepaths_list: try: info = pygeoprocessing.get_raster_info(filepath) - except ValueError: + except (ValueError, RuntimeError): + # ValueError is raised by PyGeoprocessing < 3.4.4 when the file is + # not a raster. + # RuntimeError is raised by GDAL in PyGeoprocessing >= 3.4.4 when + # the file is not a raster. info = pygeoprocessing.get_vector_info(filepath) if info['projection_wkt'] is None: From f3137f68dcf7a4c3317d8c9b6014edd6cd27ec41 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 11:53:31 -0700 Subject: [PATCH 2/7] Adding a context manager for GDAL configuration options. RE:#1645 --- .../invest/forest_carbon_edge_effect.py | 8 +++++--- src/natcap/invest/utils.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/natcap/invest/forest_carbon_edge_effect.py b/src/natcap/invest/forest_carbon_edge_effect.py index bb0fb399d0..a7569250b3 100644 --- a/src/natcap/invest/forest_carbon_edge_effect.py +++ b/src/natcap/invest/forest_carbon_edge_effect.py @@ -765,9 +765,11 @@ def _build_spatial_index( local_model_dir, 'local_carbon_shape.shp') lulc_projection_wkt = pygeoprocessing.get_raster_info( base_raster_path)['projection_wkt'] - pygeoprocessing.reproject_vector( - tropical_forest_edge_carbon_model_vector_path, lulc_projection_wkt, - carbon_model_reproject_path) + + with utils._set_gdal_configuration('OGR_ENABLE_PARTIAL_REPROJECTION', 'TRUE'): + pygeoprocessing.reproject_vector( + tropical_forest_edge_carbon_model_vector_path, lulc_projection_wkt, + carbon_model_reproject_path) model_vector = gdal.OpenEx(carbon_model_reproject_path) model_layer = model_vector.GetLayer() diff --git a/src/natcap/invest/utils.py b/src/natcap/invest/utils.py index f34c25d9e5..dcae5734dd 100644 --- a/src/natcap/invest/utils.py +++ b/src/natcap/invest/utils.py @@ -121,6 +121,25 @@ def capture_gdal_logging(): gdal.PopErrorHandler() +@contextlib.contextmanager +def _set_gdal_configuration(opt, value): + """Temporarily set a GDAL configuration option. + + Args: + opt (string): The GDAL configuration option to set. + value (string): The value to set the option to. + + Returns: + ``None`` + """ + prior_value = gdal.GetConfigOption(opt) + gdal.SetConfigOption(opt, value) + try: + yield + finally: + gdal.SetConfigOption(opt, prior_value) + + def _format_time(seconds): """Render the integer number of seconds as a string. Returns a string.""" hours, remainder = divmod(seconds, 3600) From 1d9d6deba0f43f0139d3dbbd6cb273c0ffb54446 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 11:54:00 -0700 Subject: [PATCH 3/7] Adding some debugging in SWY. RE:#1645 --- .../invest/seasonal_water_yield/seasonal_water_yield.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py index 25228b3db4..78ec0d12d6 100644 --- a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py +++ b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py @@ -632,6 +632,7 @@ def execute(args): # ValueError when n_workers is an empty string. # TypeError when n_workers is None. n_workers = -1 # Synchronous mode. + LOGGER.debug('n_workers: %s', n_workers) task_graph = taskgraph.TaskGraph( os.path.join(args['workspace_dir'], 'taskgraph_cache'), n_workers, reporting_interval=5) @@ -642,6 +643,9 @@ def execute(args): (_INTERMEDIATE_BASE_FILES, intermediate_output_dir)], file_suffix) LOGGER.info('Checking that the AOI is not the output aggregate vector') + LOGGER.debug("aoi_path: %s", args['aoi_path']) + LOGGER.debug("aggregate_vector_path: %s", + os.path.normpath(file_registry['aggregate_vector_path'])) if (os.path.normpath(args['aoi_path']) == os.path.normpath(file_registry['aggregate_vector_path'])): raise ValueError( From ce88ef16dc55cb32e4414b0d24d062d545606721 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 12:07:59 -0700 Subject: [PATCH 4/7] Correcting a test filepath in SWY. RE:#1645 --- tests/test_seasonal_water_yield_regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_seasonal_water_yield_regression.py b/tests/test_seasonal_water_yield_regression.py index 899fc1446b..e5eafa0e98 100644 --- a/tests/test_seasonal_water_yield_regression.py +++ b/tests/test_seasonal_water_yield_regression.py @@ -495,7 +495,7 @@ def test_duplicate_aoi_assertion(self): args = { 'workspace_dir': self.workspace_dir, 'aoi_path': os.path.join( - self.workspace_dir, 'aggregated_results_foo.shp'), + self.workspace_dir, 'aggregated_results_swy_foo.shp'), 'results_suffix': 'foo', 'alpha_m': '1/12', 'beta_i': '1.0', From 2246ecd3a0c6b282d4b3c973bc4bd6c5bf7fac0b Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 12:50:15 -0700 Subject: [PATCH 5/7] Noting the change to FCEE in HISTORY. RE:#1645 --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index ddcd9f5005..bab7cba932 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,9 @@ Unreleased Changes ``pygeoprocessing.get_raster_info`` and ``pygeoprocessing.get_vector_info``. https://github.com/natcap/invest/issues/1645 +* Forest Carbon Edge Effects + * Updating vector reprojection to allow partial reprojection. Related to + https://github.com/natcap/invest/issues/1645 * Urban Nature Access * The model now works as expected when the user provides an LULC raster that does not have a nodata value defined. From b978f36f97a5db7a8887fabb5686e5f7c8477843 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 12:56:05 -0700 Subject: [PATCH 6/7] Fixing RST syntax. RE:#1645 --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index bab7cba932..a88c1f709b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -44,7 +44,7 @@ Unreleased Changes https://github.com/natcap/invest/issues/1645 * Forest Carbon Edge Effects * Updating vector reprojection to allow partial reprojection. Related to - https://github.com/natcap/invest/issues/1645 + https://github.com/natcap/invest/issues/1645 * Urban Nature Access * The model now works as expected when the user provides an LULC raster that does not have a nodata value defined. From 20f179ca26ba1c60729ffea040962793a5d2bbf8 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 15 Oct 2024 12:24:31 -0700 Subject: [PATCH 7/7] Bumping pygeoprocessing requirement to at least 2.4.6 RE:#1645 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf2d3aeb76..43ce18bc31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ numpy>=1.11.0,!=1.16.0,<2.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 scipy>=1.9.0,!=1.12.* -pygeoprocessing>=2.4.2 # pip-only +pygeoprocessing>=2.4.6 # pip-only taskgraph>=0.11.0 psutil>=5.6.6 chardet>=3.0.4