From a6645e9d843d29605e73e875ccc65e18a5f11a64 Mon Sep 17 00:00:00 2001 From: sheecegardezi Date: Tue, 5 May 2020 17:02:37 +1000 Subject: [PATCH 01/23] add pixel latlon converter and refpixel metadata update functionality --- input_parameters.conf | 4 ++-- pyrate/core/refpixel.py | 46 +++++++++++++++++++++++++++++++++++++++++ pyrate/core/shared.py | 4 ++-- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/input_parameters.conf b/input_parameters.conf index 56e2a7ab5..579fab772 100644 --- a/input_parameters.conf +++ b/input_parameters.conf @@ -85,8 +85,8 @@ nan_conversion: 1 # refnx/y: number of search grid points in x/y image dimensions # refchipsize: size of the data window at each search grid point # refminfrac: minimum fraction of valid (non-NaN) pixels in the data window -refx: -refy: +refx: 150.94211 +refy: -34.21916 refnx: 5 refny: 5 refchipsize: 5 diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index 8539d33a6..df8d8b539 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -31,6 +31,52 @@ from pyrate.core.logger import pyratelogger as log +def convert_pixel_value_to_geographic_coordinate(refx, refy, transform): + """ + Converts a pixel coordinate to a latitude/longitude coordinate given the + geotransform of the image. + Args: + refx: The pixel x coordinate. + refy: The pixel ye coordinate. + transform: The geotransform array of the image. + Returns: + Tuple of lon, lat geographic coordinate. + """ + + xOrigin = transform[0] + yOrigin = transform[3] + pixelWidth = transform[1] + pixelHeight = -transform[5] + + lon = refx*pixelWidth + xOrigin + lat = yOrigin - refy*pixelHeight + + return lon, lat + + +def convert_geographic_coordinate_to_pixel_value(lon, lat, transform): + """ + Converts a latitude/longitude coordinate to a pixel coordinate given the + geotransform of the image. + Args: + lon: Pixel longitude. + lat: Pixel latitude. + transform: The geotransform array of the image. + Returns: + Tuple of refx, refy pixel coordinates. + """ + + xOrigin = transform[0] + yOrigin = transform[3] + pixelWidth = transform[1] + pixelHeight = -transform[5] + + refx = int((lon - xOrigin) / pixelWidth) + refy = int((yOrigin - lat) / pixelHeight) + + return int(refx), int(refy) + + # TODO: move error checking to config step (for fail fast) # TODO: this function is not used. Plan removal def ref_pixel(ifgs, params): diff --git a/pyrate/core/shared.py b/pyrate/core/shared.py index d8383c868..59da1ad5a 100644 --- a/pyrate/core/shared.py +++ b/pyrate/core/shared.py @@ -955,8 +955,8 @@ def write_output_geotiff(md, gt, wkt, data, dest, nodata): ds.SetMetadataItem("PYRATE_REFPIX_Y", str(md["PYRATE_REFPIX_Y"])) if "PYRATE_MEAN_REF_AREA" in md: ds.SetMetadataItem("PYRATE_MEAN_REF_AREA", str(md["PYRATE_MEAN_REF_AREA"])) - if "STANDARD_DEVIATION_REF_AREA" in md: - ds.SetMetadataItem("PYRATE_STANDARD_DEVIATION_REF_AREA", str(md["PYRATE_STANDARD_DEVIATION_REF_AREA"])) + if "PYRATE_STDDEV_REF_AREA" in md: + ds.SetMetadataItem("PYRATE_STDDEV_REF_AREA", str(md["PYRATE_STDDEV_REF_AREA"])) # write data to geotiff band = ds.GetRasterBand(1) From aeffcbcaf5e4c8212a9f5fec000eb0e515ea9478 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Fri, 12 Jun 2020 06:51:58 +1000 Subject: [PATCH 02/23] lat/lon ref pixel setup complete --- pyrate/core/ifgconstants.py | 8 +++++++ pyrate/core/refpixel.py | 48 +++++++++++++++++++++++++++++++++++++ pyrate/core/shared.py | 20 +++++----------- pyrate/process.py | 6 ++++- 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/pyrate/core/ifgconstants.py b/pyrate/core/ifgconstants.py index 0da624adc..0bf699cea 100644 --- a/pyrate/core/ifgconstants.py +++ b/pyrate/core/ifgconstants.py @@ -64,6 +64,14 @@ DATA_UNITS = 'DATA_UNITS' INPUT_TYPE = 'INPUT_TYPE' +PYRATE_REFPIX_X = 'PYRATE_REFPIX_X' +PYRATE_REFPIX_Y = 'PYRATE_REFPIX_Y' +PYRATE_REFPIX_LAT = 'PYRATE_REFPIX_LAT' +PYRATE_REFPIX_LON = 'PYRATE_REFPIX_LON' +PYRATE_MEAN_REF_AREA = 'PYRATE_MEAN_REF_AREA' +PYRATE_STDDEV_REF_AREA = 'PYRATE_STDDEV_REF_AREA' +SEQUENCE_POSITION = 'SEQUENCE_POSITION' + DAYS_PER_YEAR = 365.25 # span of year, not a calendar year YEARS_PER_DAY = 1 / DAYS_PER_YEAR SPEED_OF_LIGHT_METRES_PER_SECOND = 3e8 diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index df8d8b539..f12682369 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -25,12 +25,60 @@ from numpy import isnan, std, mean, sum as nsum from joblib import Parallel, delayed +from osgeo import gdal + import pyrate.core.config as cf +from pyrate.core import ifgconstants as ifc +from pyrate.core import mpiops from pyrate.core.shared import Ifg from pyrate.core.shared import joblib_log_level from pyrate.core.logger import pyratelogger as log +def update_refpix_metadata(ifg_paths, pyrate_refpix_x, pyrate_refpix_y, pyrate_refpix_lat, pyrate_refpix_lon, params): + """ + Function that adds metadata about the chosen reference pixel to each interferogram. + """ + process_ifgs_paths = mpiops.array_split(ifg_paths) + + for ifg_file in process_ifgs_paths: + log.debug("Updating metadata for: "+ifg_file) + ifg = Ifg(ifg_file) + log.debug("Open dataset") + ifg.open(readonly=True) + log.debug("Set no data value") + ifg.nodata_value = params["noDataValue"] + log.debug("Update no data values in dataset") + ifg.convert_to_nans() + log.debug("Convert mm") + ifg.convert_to_mm() + half_patch_size = params["refchipsize"] // 2 + x, y = pyrate_refpix_x, pyrate_refpix_y + log.debug("Extract reference pixel windows") + data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1, + x - half_patch_size: x + half_patch_size + 1] + log.debug("Calculate standard deviation for reference window") + stddev_ref_area = np.std(data[~np.isnan(data)]) + log.debug("Calculate mean for reference window") + mean_ref_area = np.mean(data[~np.isnan(data)]) + ifg.close() + + dataset = gdal.Open(ifg_file, gdal.GA_Update) + metadata = dataset.GetMetadata() + metadata.update({ + ifc.PYRATE_REFPIX_X: str(pyrate_refpix_x), + ifc.PYRATE_REFPIX_Y: str(pyrate_refpix_y), + ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), + ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon), + ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), + ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area) + }) + dataset.SetMetadata(metadata) + + # manual close dataset + dataset = None + + def convert_pixel_value_to_geographic_coordinate(refx, refy, transform): """ Converts a pixel coordinate to a latitude/longitude coordinate given the diff --git a/pyrate/core/shared.py b/pyrate/core/shared.py index 59da1ad5a..ac00f3c27 100644 --- a/pyrate/core/shared.py +++ b/pyrate/core/shared.py @@ -942,21 +942,13 @@ def write_output_geotiff(md, gt, wkt, data, dest, nodata): # set other metadata ds.SetMetadataItem('DATA_TYPE', str(md['DATA_TYPE'])) + # sequence position for time series products - if "SEQUENCE_POSITION" in md: - ds.SetMetadataItem("SEQUENCE_POSITION", str(md["SEQUENCE_POSITION"])) - if "PYRATE_REFPIX_LAT" in md: - ds.SetMetadataItem("PYRATE_REFPIX_LAT", str(md["PYRATE_REFPIX_LAT"])) - if "PYRATE_REFPIX_LON" in md: - ds.SetMetadataItem("PYRATE_REFPIX_LON", str(md["PYRATE_REFPIX_LON"])) - if "PYRATE_REFPIX_X" in md: - ds.SetMetadataItem("PYRATE_REFPIX_X", str(md["PYRATE_REFPIX_X"])) - if "PYRATE_REFPIX_Y" in md: - ds.SetMetadataItem("PYRATE_REFPIX_Y", str(md["PYRATE_REFPIX_Y"])) - if "PYRATE_MEAN_REF_AREA" in md: - ds.SetMetadataItem("PYRATE_MEAN_REF_AREA", str(md["PYRATE_MEAN_REF_AREA"])) - if "PYRATE_STDDEV_REF_AREA" in md: - ds.SetMetadataItem("PYRATE_STDDEV_REF_AREA", str(md["PYRATE_STDDEV_REF_AREA"])) + + for k in [ifc.SEQUENCE_POSITION, ifc.PYRATE_REFPIX_X, ifc.PYRATE_REFPIX_Y, ifc.PYRATE_REFPIX_LAT, + ifc.PYRATE_REFPIX_LON, ifc.PYRATE_MEAN_REF_AREA, ifc.PYRATE_STDDEV_REF_AREA]: + if k in md: + ds.SetMetadataItem(str(md[k])) # write data to geotiff band = ds.GetRasterBand(1) diff --git a/pyrate/process.py b/pyrate/process.py index 21d29cc7c..90db87de2 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -129,6 +129,8 @@ def _ref_pixel_calc(ifg_paths, params): ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) + # assume all interferograms have same projection and will share the same transform + transform = ifg.dataset.GetGeoTransform() if refx == -1 or refy == -1: @@ -150,12 +152,14 @@ def _ref_pixel_calc(ifg_paths, params): "Reference pixel calculation returned an all nan slice!\n" "Cannot continue downstream computation. Please change reference pixel algorithm used before " "continuing.") - refy, refx = refpixel_returned log.info('Selected reference pixel coordinate: ({}, {})'.format(refx, refy)) else: + refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(refx, refy, transform) log.info('Reusing reference pixel from config file: ({}, {})'.format(refx, refy)) + log.warn("Ensure reference pixel values are in lat/lon") + log.debug("refpx, refpy: "+str(refx) + " " + str(refy)) ifg.close() return int(refx), int(refy) From 195a15a93524b9314149a00dea492f6f44e21502 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Fri, 12 Jun 2020 07:33:39 +1000 Subject: [PATCH 03/23] lat/lon reference pixel supported --- pyrate/core/refpixel.py | 18 +++++++++++------- pyrate/core/shared.py | 2 +- pyrate/process.py | 17 ++++++++++------- .../small_test/conf/pyrate_gamma_test.conf | 4 ++-- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index f12682369..2a8fb9a3b 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -35,10 +35,14 @@ from pyrate.core.logger import pyratelogger as log -def update_refpix_metadata(ifg_paths, pyrate_refpix_x, pyrate_refpix_y, pyrate_refpix_lat, pyrate_refpix_lon, params): +def update_refpix_metadata(ifg_paths, refx, refy, transform, params): """ Function that adds metadata about the chosen reference pixel to each interferogram. """ + + pyrate_refpix_lat, pyrate_refpix_lon = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, + refx, refy, transform) + process_ifgs_paths = mpiops.array_split(ifg_paths) for ifg_file in process_ifgs_paths: @@ -53,21 +57,21 @@ def update_refpix_metadata(ifg_paths, pyrate_refpix_x, pyrate_refpix_y, pyrate_r log.debug("Convert mm") ifg.convert_to_mm() half_patch_size = params["refchipsize"] // 2 - x, y = pyrate_refpix_x, pyrate_refpix_y + x, y = refx, refy log.debug("Extract reference pixel windows") data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1, x - half_patch_size: x + half_patch_size + 1] log.debug("Calculate standard deviation for reference window") - stddev_ref_area = np.std(data[~np.isnan(data)]) + stddev_ref_area = np.nanstd(data) log.debug("Calculate mean for reference window") - mean_ref_area = np.mean(data[~np.isnan(data)]) + mean_ref_area = np.nanmean(data) ifg.close() dataset = gdal.Open(ifg_file, gdal.GA_Update) metadata = dataset.GetMetadata() metadata.update({ - ifc.PYRATE_REFPIX_X: str(pyrate_refpix_x), - ifc.PYRATE_REFPIX_Y: str(pyrate_refpix_y), + ifc.PYRATE_REFPIX_X: str(refx), + ifc.PYRATE_REFPIX_Y: str(refy), ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon), ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), @@ -122,7 +126,7 @@ def convert_geographic_coordinate_to_pixel_value(lon, lat, transform): refx = int((lon - xOrigin) / pixelWidth) refy = int((yOrigin - lat) / pixelHeight) - return int(refx), int(refy) + return refx, refy # TODO: move error checking to config step (for fail fast) diff --git a/pyrate/core/shared.py b/pyrate/core/shared.py index ac00f3c27..77c295a13 100644 --- a/pyrate/core/shared.py +++ b/pyrate/core/shared.py @@ -948,7 +948,7 @@ def write_output_geotiff(md, gt, wkt, data, dest, nodata): for k in [ifc.SEQUENCE_POSITION, ifc.PYRATE_REFPIX_X, ifc.PYRATE_REFPIX_Y, ifc.PYRATE_REFPIX_LAT, ifc.PYRATE_REFPIX_LON, ifc.PYRATE_MEAN_REF_AREA, ifc.PYRATE_STDDEV_REF_AREA]: if k in md: - ds.SetMetadataItem(str(md[k])) + ds.SetMetadataItem(k, str(md[k])) # write data to geotiff band = ds.GetRasterBand(1) diff --git a/pyrate/process.py b/pyrate/process.py index 90db87de2..e69143497 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -124,15 +124,15 @@ def _ref_pixel_calc(ifg_paths, params): """ Wrapper for reference pixel calculation """ - refx = params[cf.REFX] - refy = params[cf.REFY] + lon = params[cf.REFX] + lat = params[cf.REFY] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) # assume all interferograms have same projection and will share the same transform transform = ifg.dataset.GetGeoTransform() - if refx == -1 or refy == -1: + if lon == -1 or lat == -1: log.info('Searching for best reference pixel location') @@ -152,14 +152,17 @@ def _ref_pixel_calc(ifg_paths, params): "Reference pixel calculation returned an all nan slice!\n" "Cannot continue downstream computation. Please change reference pixel algorithm used before " "continuing.") - refy, refx = refpixel_returned + refy, refx = refpixel_returned # row first means first value is latitude - log.info('Selected reference pixel coordinate: ({}, {})'.format(refx, refy)) else: - refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(refx, refy, transform) - log.info('Reusing reference pixel from config file: ({}, {})'.format(refx, refy)) + refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(lon, lat, transform) + log.info('Using reference pixel from config file (lat, lon): ({}, {})'.format(lat, lon)) log.warn("Ensure reference pixel values are in lat/lon") + log.info('Selected reference pixel coordinate: ({}, {})'.format(refx, refy)) + + refpixel.update_refpix_metadata(ifg_paths, refx, refy, transform, params) + log.debug("refpx, refpy: "+str(refx) + " " + str(refy)) ifg.close() return int(refx), int(refy) diff --git a/tests/test_data/small_test/conf/pyrate_gamma_test.conf b/tests/test_data/small_test/conf/pyrate_gamma_test.conf index a541a50bb..79f6fcdc4 100644 --- a/tests/test_data/small_test/conf/pyrate_gamma_test.conf +++ b/tests/test_data/small_test/conf/pyrate_gamma_test.conf @@ -62,8 +62,8 @@ ifgylast: -34.22 # refnx/y: number of search grid points in x/y direction # refchipsize: chip size of the data window at each search grid point # refminfrac: minimum fraction of valid (non-NaN) pixels in the data window -refx: -refy: +refx: 150.94211 +refy: -34.21916 refnx: 5 refny: 5 refchipsize: 5 From a79c3c8705626b2a620928eee204c1d4aa3ec042 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Fri, 12 Jun 2020 08:17:51 +1000 Subject: [PATCH 04/23] fixed refpixel tests --- pyrate/process.py | 2 +- tests/common.py | 9 +++++++++ tests/test_refpixel.py | 12 ++++-------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pyrate/process.py b/pyrate/process.py index e69143497..0d1835bca 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -157,7 +157,7 @@ def _ref_pixel_calc(ifg_paths, params): else: refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(lon, lat, transform) log.info('Using reference pixel from config file (lat, lon): ({}, {})'.format(lat, lon)) - log.warn("Ensure reference pixel values are in lat/lon") + log.warning("Ensure reference pixel values are in lat/lon") log.info('Selected reference pixel coordinate: ({}, {})'.format(refx, refy)) diff --git a/tests/common.py b/tests/common.py index 88fe8621d..602d190ab 100644 --- a/tests/common.py +++ b/tests/common.py @@ -160,6 +160,15 @@ def assert_tifs_equal(tif1, tif2): sds = None +def copy_small_ifg_file_list(): + temp_dir = tempfile.mkdtemp() + move_files(SML_TEST_TIF, temp_dir, file_type='*.tif', copy=True) + datafiles = glob.glob(join(temp_dir, "*.tif")) + for d in datafiles: + Path(d).chmod(0o664) # assign write permission as conv2tif output is readonly + return temp_dir, datafiles + + def small_ifg_file_list(datafiles=None): """Returns the file list of all the .tif files after prepifg conversion input phase data is in radians; these ifgs are in radians - not converted to mm""" diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 19aa0cf63..c8998c6ee 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -26,7 +26,7 @@ from pyrate.core.refpixel import ref_pixel, _step from pyrate import process from tests.common import TEST_CONF_ROIPAC -from tests.common import small_data_setup, MockIfg, small_ifg_file_list +from tests.common import small_data_setup, MockIfg, small_ifg_file_list, copy_small_ifg_file_list # TODO: figure out how editing resource.setrlimit fixes the error @@ -223,9 +223,9 @@ def _expected_ref_pixel(ifgs, cs): class LegacyEqualityTest(unittest.TestCase): def setUp(self): - self.ifg_paths = small_ifg_file_list() self.params = cf.get_config_params(TEST_CONF_ROIPAC) self.params[cf.PARALLEL] = False + self.params[cf.OUT_DIR], self.ifg_paths = copy_small_ifg_file_list() self.params[cf.OUT_DIR] = tempfile.mkdtemp() self.params_alt_ref_frac = copy.copy(self.params) self.params_alt_ref_frac[cf.REF_MIN_FRAC] = 0.5 @@ -256,9 +256,7 @@ def test_small_test_data_ref_chipsize_15(self): self.assertAlmostEqual(0.5, self.params_alt_ref_frac[cf.REF_MIN_FRAC]) def test_small_test_data_ref_all_1(self): - - refx, refy = process._ref_pixel_calc(self.ifg_paths, - self.params_all_1s) + refx, refy = process._ref_pixel_calc(self.ifg_paths, self.params_all_1s) self.assertAlmostEqual(0.7, self.params_all_1s[cf.REF_MIN_FRAC]) self.assertEqual(1, self.params_all_1s[cf.REFNX]) @@ -270,11 +268,9 @@ def test_small_test_data_ref_all_1(self): class LegacyEqualityTestMultiprocessParallel(unittest.TestCase): def setUp(self): - self.ifg_paths = small_ifg_file_list() self.params = cf.get_config_params(TEST_CONF_ROIPAC) self.params[cf.PARALLEL] = True - self.params[cf.OUT_DIR] = tempfile.mkdtemp() - + self.params[cf.OUT_DIR], self.ifg_paths = copy_small_ifg_file_list() self.params_alt_ref_frac = copy.copy(self.params) self.params_alt_ref_frac[cf.REF_MIN_FRAC] = 0.5 self.params_all_2s = copy.copy(self.params) From 6e817b9f834e0f6b4d5270b2e1a8bfc9dda9a10e Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 13 Jun 2020 06:15:48 +1000 Subject: [PATCH 05/23] stop overwriting test files during ref pixel tests --- tests/common.py | 11 +++++++++++ tests/test_refpixel.py | 14 +++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/common.py b/tests/common.py index 602d190ab..90d4240ce 100644 --- a/tests/common.py +++ b/tests/common.py @@ -169,6 +169,17 @@ def copy_small_ifg_file_list(): return temp_dir, datafiles +def copy_and_setup_small_data(): + temp_dir, datafiles = copy_small_ifg_file_list() + datafiles.sort() + ifgs = [dem_or_ifg(i) for i in datafiles] + + for i in ifgs: + i.open() + i.nodata_value = 0 + return temp_dir, ifgs + + def small_ifg_file_list(datafiles=None): """Returns the file list of all the .tif files after prepifg conversion input phase data is in radians; these ifgs are in radians - not converted to mm""" diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index c8998c6ee..6f671e4d4 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -20,13 +20,16 @@ import unittest import tempfile import shutil +from pathlib import Path from numpy import nan, mean, std, isnan from pyrate.core import config as cf from pyrate.core.refpixel import ref_pixel, _step from pyrate import process +from pyrate.configuration import Configuration from tests.common import TEST_CONF_ROIPAC -from tests.common import small_data_setup, MockIfg, small_ifg_file_list, copy_small_ifg_file_list +from tests.common import small_data_setup, MockIfg, small_ifg_file_list, copy_small_ifg_file_list, \ + copy_and_setup_small_data # TODO: figure out how editing resource.setrlimit fixes the error @@ -100,8 +103,8 @@ class ReferencePixelTests(unittest.TestCase): """ def setUp(self): - self.ifgs = small_data_setup() self.params = cf.get_config_params(TEST_CONF_ROIPAC) + self.params[cf.OUT_DIR], self.ifgs = copy_and_setup_small_data() self.params[cf.REFNX] = REFNX self.params[cf.REFNY] = REFNY self.params[cf.REF_CHIP_SIZE] = CHIPSIZE @@ -224,9 +227,11 @@ class LegacyEqualityTest(unittest.TestCase): def setUp(self): self.params = cf.get_config_params(TEST_CONF_ROIPAC) - self.params[cf.PARALLEL] = False + self.params[cf.PARALLEL] = 0 self.params[cf.OUT_DIR], self.ifg_paths = copy_small_ifg_file_list() - self.params[cf.OUT_DIR] = tempfile.mkdtemp() + conf_file = Path(self.params[cf.OUT_DIR], 'conf_file.conf') + cf.write_config_file(params=self.params, output_conf_file=conf_file) + self.params = Configuration(conf_file).__dict__ self.params_alt_ref_frac = copy.copy(self.params) self.params_alt_ref_frac[cf.REF_MIN_FRAC] = 0.5 self.params_all_2s = copy.copy(self.params) @@ -257,7 +262,6 @@ def test_small_test_data_ref_chipsize_15(self): def test_small_test_data_ref_all_1(self): refx, refy = process._ref_pixel_calc(self.ifg_paths, self.params_all_1s) - self.assertAlmostEqual(0.7, self.params_all_1s[cf.REF_MIN_FRAC]) self.assertEqual(1, self.params_all_1s[cf.REFNX]) self.assertEqual(1, self.params_all_1s[cf.REFNY]) From 4f654c829f68f7675320e890000a6ea718d57c9d Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 13 Jun 2020 06:30:06 +1000 Subject: [PATCH 06/23] add refpixel metadata tests --- pyrate/core/refpixel.py | 2 +- tests/test_refpixel.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index 2a8fb9a3b..453b9483d 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -40,7 +40,7 @@ def update_refpix_metadata(ifg_paths, refx, refy, transform, params): Function that adds metadata about the chosen reference pixel to each interferogram. """ - pyrate_refpix_lat, pyrate_refpix_lon = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, + pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, refx, refy, transform) process_ifgs_paths = mpiops.array_split(ifg_paths) diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 6f671e4d4..5ce683714 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -25,6 +25,8 @@ from pyrate.core import config as cf from pyrate.core.refpixel import ref_pixel, _step +from pyrate.core import shared, ifgconstants as ifc + from pyrate import process from pyrate.configuration import Configuration from tests.common import TEST_CONF_ROIPAC @@ -260,6 +262,18 @@ def test_small_test_data_ref_chipsize_15(self): self.assertEqual(refy, 7) self.assertAlmostEqual(0.5, self.params_alt_ref_frac[cf.REF_MIN_FRAC]) + def test_metadata(self): + refx, refy = process._ref_pixel_calc(self.ifg_paths, self.params_chipsize_15) + for i in self.ifg_paths: + ifg = shared.Ifg(i) + ifg.open(readonly=True) + md = ifg.meta_data + for k, v in zip([ifc.PYRATE_REFPIX_X, ifc.PYRATE_REFPIX_Y, ifc.PYRATE_REFPIX_LAT, + ifc.PYRATE_REFPIX_LON, ifc.PYRATE_MEAN_REF_AREA, ifc.PYRATE_STDDEV_REF_AREA], + [str(refx), str(refy), 0, 0, 0, 0]): + assert k in md # metadata present + # assert values + def test_small_test_data_ref_all_1(self): refx, refy = process._ref_pixel_calc(self.ifg_paths, self.params_all_1s) self.assertAlmostEqual(0.7, self.params_all_1s[cf.REF_MIN_FRAC]) From a28dcbed7b6a04b347282926b1e4f8e7d43e97e6 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 13 Jun 2020 06:50:12 +1000 Subject: [PATCH 07/23] new Ifg function add metadata to update metadata of interferograms --- pyrate/core/refpixel.py | 10 ++-------- pyrate/core/shared.py | 11 +++++++++++ tests/test_refpixel.py | 1 + 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index 453b9483d..f7bba6535 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -65,11 +65,7 @@ def update_refpix_metadata(ifg_paths, refx, refy, transform, params): stddev_ref_area = np.nanstd(data) log.debug("Calculate mean for reference window") mean_ref_area = np.nanmean(data) - ifg.close() - - dataset = gdal.Open(ifg_file, gdal.GA_Update) - metadata = dataset.GetMetadata() - metadata.update({ + ifg.add_metadata(**{ ifc.PYRATE_REFPIX_X: str(refx), ifc.PYRATE_REFPIX_Y: str(refy), ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), @@ -77,10 +73,8 @@ def update_refpix_metadata(ifg_paths, refx, refy, transform, params): ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area) }) - dataset.SetMetadata(metadata) - # manual close dataset - dataset = None + ifg.close() def convert_pixel_value_to_geographic_coordinate(refx, refy, transform): diff --git a/pyrate/core/shared.py b/pyrate/core/shared.py index 77c295a13..8bb25b75f 100644 --- a/pyrate/core/shared.py +++ b/pyrate/core/shared.py @@ -479,6 +479,17 @@ def write_modified_phase(self, data=None): self.dataset.SetMetadataItem(k, v) self.dataset.FlushCache() + def add_metadata(self, **kwargs): + if self.is_read_only: + raise IOError("Cannot write to read only Ifg") + + if self.meta_data is None: + raise IOError("Ifg not open") + + for k, v in kwargs.items(): + self.dataset.SetMetadataItem(k, v) + self.dataset.FlushCache() # write to disc + class IfgPart(object): """ diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 5ce683714..2b6443f30 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -273,6 +273,7 @@ def test_metadata(self): [str(refx), str(refy), 0, 0, 0, 0]): assert k in md # metadata present # assert values + ifg.close() def test_small_test_data_ref_all_1(self): refx, refy = process._ref_pixel_calc(self.ifg_paths, self.params_all_1s) From 8cf8f1cc4f047d7a5def545db88f8a2aa222e0b2 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 13 Jun 2020 07:03:38 +1000 Subject: [PATCH 08/23] pass through lat/lon provided by user to the refpixel output --- pyrate/core/refpixel.py | 9 +++------ pyrate/core/shared.py | 7 ++----- pyrate/process.py | 4 +++- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index f7bba6535..8a74d52f9 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -35,14 +35,11 @@ from pyrate.core.logger import pyratelogger as log -def update_refpix_metadata(ifg_paths, refx, refy, transform, params): +def update_refpix_metadata(ifg_paths, refx, refy, lon, lat, params): """ Function that adds metadata about the chosen reference pixel to each interferogram. """ - pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, - refx, refy, transform) - process_ifgs_paths = mpiops.array_split(ifg_paths) for ifg_file in process_ifgs_paths: @@ -68,8 +65,8 @@ def update_refpix_metadata(ifg_paths, refx, refy, transform, params): ifg.add_metadata(**{ ifc.PYRATE_REFPIX_X: str(refx), ifc.PYRATE_REFPIX_Y: str(refy), - ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), - ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon), + ifc.PYRATE_REFPIX_LAT: str(lat), + ifc.PYRATE_REFPIX_LON: str(lon), ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area) }) diff --git a/pyrate/core/shared.py b/pyrate/core/shared.py index 8bb25b75f..642b21165 100644 --- a/pyrate/core/shared.py +++ b/pyrate/core/shared.py @@ -480,11 +480,8 @@ def write_modified_phase(self, data=None): self.dataset.FlushCache() def add_metadata(self, **kwargs): - if self.is_read_only: - raise IOError("Cannot write to read only Ifg") - - if self.meta_data is None: - raise IOError("Ifg not open") + if (not self.is_open) or self.is_read_only: + raise IOError("Ifg not open or readonly. Cannot write!") for k, v in kwargs.items(): self.dataset.SetMetadataItem(k, v) diff --git a/pyrate/process.py b/pyrate/process.py index 0d1835bca..9ff524266 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -154,6 +154,8 @@ def _ref_pixel_calc(ifg_paths, params): "continuing.") refy, refx = refpixel_returned # row first means first value is latitude + # get the actual lon/lat + lon, lat = mpiops.run_once(refpixel.convert_pixel_value_to_geographic_coordinate, refx, refy, transform) else: refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(lon, lat, transform) log.info('Using reference pixel from config file (lat, lon): ({}, {})'.format(lat, lon)) @@ -161,7 +163,7 @@ def _ref_pixel_calc(ifg_paths, params): log.info('Selected reference pixel coordinate: ({}, {})'.format(refx, refy)) - refpixel.update_refpix_metadata(ifg_paths, refx, refy, transform, params) + refpixel.update_refpix_metadata(ifg_paths, refx, refy, lon, lat, params) log.debug("refpx, refpy: "+str(refx) + " " + str(refy)) ifg.close() From a3af6241f72e1b6f47fa34a0823abd0ac999d870 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 13 Jun 2020 08:31:55 +1000 Subject: [PATCH 09/23] bugfix and more tests for refpixel --- pyrate/core/refpixel.py | 11 +++++--- pyrate/process.py | 4 +-- ...t_mpi_vs_multiprocess_vs_single_process.py | 16 +++-------- tests/test_refpixel.py | 27 ++++++++++++++++--- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index 8a74d52f9..07f993f0a 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -35,11 +35,14 @@ from pyrate.core.logger import pyratelogger as log -def update_refpix_metadata(ifg_paths, refx, refy, lon, lat, params): +def update_refpix_metadata(ifg_paths, refx, refy, transform, params): """ Function that adds metadata about the chosen reference pixel to each interferogram. """ + pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, + refx, refy, transform) + process_ifgs_paths = mpiops.array_split(ifg_paths) for ifg_file in process_ifgs_paths: @@ -65,8 +68,8 @@ def update_refpix_metadata(ifg_paths, refx, refy, lon, lat, params): ifg.add_metadata(**{ ifc.PYRATE_REFPIX_X: str(refx), ifc.PYRATE_REFPIX_Y: str(refy), - ifc.PYRATE_REFPIX_LAT: str(lat), - ifc.PYRATE_REFPIX_LON: str(lon), + ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), + ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon), ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area) }) @@ -117,7 +120,7 @@ def convert_geographic_coordinate_to_pixel_value(lon, lat, transform): refx = int((lon - xOrigin) / pixelWidth) refy = int((yOrigin - lat) / pixelHeight) - return refx, refy + return refx + 1, refy + 1 # TODO: move error checking to config step (for fail fast) diff --git a/pyrate/process.py b/pyrate/process.py index 9ff524266..0d1835bca 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -154,8 +154,6 @@ def _ref_pixel_calc(ifg_paths, params): "continuing.") refy, refx = refpixel_returned # row first means first value is latitude - # get the actual lon/lat - lon, lat = mpiops.run_once(refpixel.convert_pixel_value_to_geographic_coordinate, refx, refy, transform) else: refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(lon, lat, transform) log.info('Using reference pixel from config file (lat, lon): ({}, {})'.format(lat, lon)) @@ -163,7 +161,7 @@ def _ref_pixel_calc(ifg_paths, params): log.info('Selected reference pixel coordinate: ({}, {})'.format(refx, refy)) - refpixel.update_refpix_metadata(ifg_paths, refx, refy, lon, lat, params) + refpixel.update_refpix_metadata(ifg_paths, refx, refy, transform, params) log.debug("refpx, refpy: "+str(refx) + " " + str(refy)) ifg.close() diff --git a/tests/test_mpi_vs_multiprocess_vs_single_process.py b/tests/test_mpi_vs_multiprocess_vs_single_process.py index 5928aac38..40ee02f32 100644 --- a/tests/test_mpi_vs_multiprocess_vs_single_process.py +++ b/tests/test_mpi_vs_multiprocess_vs_single_process.py @@ -87,17 +87,11 @@ def test_pipeline_parallel_vs_mpi(modified_config, gamma_conf): mr_conf, params_m = modified_config(gamma_conf, 1, 'multiprocess_conf.conf') - check_call(f"pyrate conv2tif -f {mr_conf}", shell=True) - check_call(f"pyrate prepifg -f {mr_conf}", shell=True) - check_call(f"pyrate process -f {mr_conf}", shell=True) - check_call(f"pyrate merge -f {mr_conf}", shell=True) + check_call(f"pyrate workflow -f {mr_conf}", shell=True) sr_conf, params_s = modified_config(gamma_conf, 0, 'singleprocess_conf.conf') - check_call(f"pyrate conv2tif -f {sr_conf}", shell=True) - check_call(f"pyrate prepifg -f {sr_conf}", shell=True) - check_call(f"pyrate process -f {sr_conf}", shell=True) - check_call(f"pyrate merge -f {sr_conf}", shell=True) + check_call(f"pyrate workflow -f {sr_conf}", shell=True) # convert2tif tests, 17 interferograms assert_same_files_produced(params[cf.OUT_DIR], params_m[cf.OUT_DIR], params_s[cf.OUT_DIR], "*_unw_ifg.tif", 17) @@ -229,11 +223,7 @@ def test_stack_and_ts_mpi_vs_parallel_vs_serial(modified_config_short, gamma_con sr_conf, params_p = modified_config_short(gamma_conf, parallel, 'parallel_conf.conf', 0) - check_call(f"pyrate conv2tif -f {sr_conf}", shell=True) - check_call(f"pyrate prepifg -f {sr_conf}", shell=True) - check_call(f"pyrate process -f {sr_conf}", shell=True) - check_call(f"pyrate merge -f {sr_conf}", shell=True) - + check_call(f"pyrate workflow -f {sr_conf}", shell=True) # convert2tif tests, 17 interferograms assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_unw_ifg.tif", 17) diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 2b6443f30..4cdb3d19a 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -20,6 +20,7 @@ import unittest import tempfile import shutil +from subprocess import check_call from pathlib import Path from numpy import nan, mean, std, isnan @@ -30,8 +31,8 @@ from pyrate import process from pyrate.configuration import Configuration from tests.common import TEST_CONF_ROIPAC -from tests.common import small_data_setup, MockIfg, small_ifg_file_list, copy_small_ifg_file_list, \ - copy_and_setup_small_data +from tests.common import small_data_setup, MockIfg, copy_small_ifg_file_list, \ + copy_and_setup_small_data, manipulate_test_conf, assert_two_dirs_equal # TODO: figure out how editing resource.setrlimit fixes the error @@ -344,5 +345,23 @@ def test_small_test_data_ref_all_1(self): self.assertEqual(refy, 2) -if __name__ == "__main__": - unittest.main() +def test_gamma_ref_pixel_search_vs_lat_lon(tempdir, gamma_conf): + # refx, refy: 38 58 + # refx: 150.94211 + # refy: -34.21916 + params_1 = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=-1, refy=-1) + params_2 = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=150.941666654, refy=-34.218333314) + assert_two_dirs_equal(params_1[cf.OUT_DIR], params_2[cf.OUT_DIR], f"*{params_1[cf.IFG_CROP_OPT]}cr.tif", 18) + + +def _get_mlooked_files(gamma_conf, tdir, refx, refy): + params = manipulate_test_conf(gamma_conf, tdir) + params[cf.REFX] = refx + params[cf.REFY] = refy + output_conf_file = 'config.conf' + output_conf = tdir.joinpath(output_conf_file) + cf.write_config_file(params=params, output_conf_file=output_conf) + check_call(f"pyrate conv2tif -f {output_conf}", shell=True) + check_call(f"pyrate prepifg -f {output_conf}", shell=True) + check_call(f"pyrate process -f {output_conf}", shell=True) + return params From f3e8d7213dd8a742344bc758df09736282af7d6b Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 13 Jun 2020 08:34:22 +1000 Subject: [PATCH 10/23] more refpixel test --- tests/test_refpixel.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 4cdb3d19a..3b07abab7 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -250,6 +250,13 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.params[cf.OUT_DIR]) + def test_small_test_data_ref_pixel_lat_lon_provided(self): + self.params[cf.REFX], self.params[cf.REFY] = 150.941666654, -34.218333314 + refx, refy = process._ref_pixel_calc(self.ifg_paths, self.params) + self.assertEqual(refx, 38) + self.assertEqual(refy, 58) + self.assertAlmostEqual(0.8, self.params[cf.REF_MIN_FRAC]) + def test_small_test_data_ref_pixel(self): refx, refy = process._ref_pixel_calc(self.ifg_paths, self.params) self.assertEqual(refx, 38) From c8ea6c783e6751d3d8676c8ca7ef8d2ab824fe2b Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Tue, 16 Jun 2020 04:00:23 +1000 Subject: [PATCH 11/23] need more resolution for correct refpixel and pytest slow marker for refpixel lat/lon integration test --- tests/test_data/small_test/conf/pyrate_gamma_test.conf | 4 ++-- tests/test_refpixel.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_data/small_test/conf/pyrate_gamma_test.conf b/tests/test_data/small_test/conf/pyrate_gamma_test.conf index 79f6fcdc4..e5f431490 100644 --- a/tests/test_data/small_test/conf/pyrate_gamma_test.conf +++ b/tests/test_data/small_test/conf/pyrate_gamma_test.conf @@ -62,8 +62,8 @@ ifgylast: -34.22 # refnx/y: number of search grid points in x/y direction # refchipsize: chip size of the data window at each search grid point # refminfrac: minimum fraction of valid (non-NaN) pixels in the data window -refx: 150.94211 -refy: -34.21916 +refx: 150.941666654 +refy: -34.218333314 refnx: 5 refny: 5 refchipsize: 5 diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 3b07abab7..d98b4fe47 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -18,10 +18,10 @@ """ import copy import unittest -import tempfile import shutil from subprocess import check_call from pathlib import Path +import pytest from numpy import nan, mean, std, isnan from pyrate.core import config as cf @@ -352,6 +352,7 @@ def test_small_test_data_ref_all_1(self): self.assertEqual(refy, 2) +@pytest.mark.slow def test_gamma_ref_pixel_search_vs_lat_lon(tempdir, gamma_conf): # refx, refy: 38 58 # refx: 150.94211 From 3e53979f16a002eec609028c834b065ffe92e16e Mon Sep 17 00:00:00 2001 From: Matt Garthwaite Date: Tue, 16 Jun 2020 21:59:49 +1000 Subject: [PATCH 12/23] change metadata tag entries for refpixel --- pyrate/core/ifgconstants.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyrate/core/ifgconstants.py b/pyrate/core/ifgconstants.py index 0bf699cea..a5ece2b91 100644 --- a/pyrate/core/ifgconstants.py +++ b/pyrate/core/ifgconstants.py @@ -64,12 +64,12 @@ DATA_UNITS = 'DATA_UNITS' INPUT_TYPE = 'INPUT_TYPE' -PYRATE_REFPIX_X = 'PYRATE_REFPIX_X' -PYRATE_REFPIX_Y = 'PYRATE_REFPIX_Y' -PYRATE_REFPIX_LAT = 'PYRATE_REFPIX_LAT' -PYRATE_REFPIX_LON = 'PYRATE_REFPIX_LON' -PYRATE_MEAN_REF_AREA = 'PYRATE_MEAN_REF_AREA' -PYRATE_STDDEV_REF_AREA = 'PYRATE_STDDEV_REF_AREA' +PYRATE_REFPIX_X = 'REF_PIX_X' +PYRATE_REFPIX_Y = 'REF_PIX_Y' +PYRATE_REFPIX_LAT = 'REF_PIX_LAT' +PYRATE_REFPIX_LON = 'REF_PIX_LON' +PYRATE_MEAN_REF_AREA = 'REF_AREA_MEAN' +PYRATE_STDDEV_REF_AREA = 'REF_AREA_STDDEV' SEQUENCE_POSITION = 'SEQUENCE_POSITION' DAYS_PER_YEAR = 365.25 # span of year, not a calendar year From 2341d544a092b953b0d40c720fbdcdb4c6ce8665 Mon Sep 17 00:00:00 2001 From: Matt Garthwaite Date: Tue, 16 Jun 2020 22:29:24 +1000 Subject: [PATCH 13/23] improve console messages for x/y and lon/lat refpixel coords --- input_parameters.conf | 6 +++--- pyrate/process.py | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/input_parameters.conf b/input_parameters.conf index fabf41d42..c10f55628 100644 --- a/input_parameters.conf +++ b/input_parameters.conf @@ -16,7 +16,7 @@ apsest: 0 tscal: 1 # Optional save of numpy array files for output products (MERGE) -savenpy: 1 +savenpy: 0 #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # Multi-threading parameters used by stacking/timeseries/prepifg @@ -88,8 +88,8 @@ nan_conversion: 1 # refnx/y: number of search grid points in x/y image dimensions # refchipsize: size of the data window at each search grid point # refminfrac: minimum fraction of valid (non-NaN) pixels in the data window -refx: 150.94211 -refy: -34.21916 +refx: 150.941666654 +refy: -34.218333314 refnx: 5 refny: 5 refchipsize: 5 diff --git a/pyrate/process.py b/pyrate/process.py index 0d1835bca..56e16533b 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -153,13 +153,15 @@ def _ref_pixel_calc(ifg_paths, params): "Cannot continue downstream computation. Please change reference pixel algorithm used before " "continuing.") refy, refx = refpixel_returned # row first means first value is latitude + log.info('Selected reference pixel coordinate (x, y): ({}, {})'.format(refx, refy)) + lon, lat = refpixel.convert_pixel_value_to_geographic_coordinate(refx, refy, transform) + log.info('Selected reference pixel coordinate (lon, lat): ({}, {})'.format(lon, lat)) else: + log.info('Using reference pixel from config file (lon, lat): ({}, {})'.format(lon, lat)) + log.warning("Ensure user supplied reference pixel values are in lon/lat") refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(lon, lat, transform) - log.info('Using reference pixel from config file (lat, lon): ({}, {})'.format(lat, lon)) - log.warning("Ensure reference pixel values are in lat/lon") - - log.info('Selected reference pixel coordinate: ({}, {})'.format(refx, refy)) + log.info('Converted reference pixel coordinate (x, y): ({}, {})'.format(refx, refy)) refpixel.update_refpix_metadata(ifg_paths, refx, refy, transform, params) From d4955726532290a5139d4db997098ecc0c7dfbac Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Wed, 17 Jun 2020 05:53:54 +1000 Subject: [PATCH 14/23] minor changes --- tests/test_mpi_vs_multiprocess_vs_single_process.py | 5 ++--- tests/test_prepifg.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_mpi_vs_multiprocess_vs_single_process.py b/tests/test_mpi_vs_multiprocess_vs_single_process.py index 2c3ea276b..9a4f61d06 100644 --- a/tests/test_mpi_vs_multiprocess_vs_single_process.py +++ b/tests/test_mpi_vs_multiprocess_vs_single_process.py @@ -236,8 +236,7 @@ def test_stack_and_ts_mpi_vs_parallel_vs_serial(modified_config_short, gamma_con print("coherence files compared") # prepifg + process steps that overwrite tifs test - - # 17 ifgs + 1 dem + # 17 mlooked ifgs + 1 dem + 17 mlooked coherence files if params[cf.COH_MASK]: assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], f"*{params[cf.IFG_CROP_OPT]}cr.tif", 35) else: @@ -252,7 +251,7 @@ def test_stack_and_ts_mpi_vs_parallel_vs_serial(modified_config_short, gamma_con # compare merge step assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "stack*.tif", 3) - #assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "stack*.npy", 3) not saved by default + assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "stack*.npy", 3) assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "tscuml*.tif") print("==========================xxx===========================") diff --git a/tests/test_prepifg.py b/tests/test_prepifg.py index 39399c68f..b666dec3c 100644 --- a/tests/test_prepifg.py +++ b/tests/test_prepifg.py @@ -87,6 +87,7 @@ def test_prepifg_file_types(tempdir, gamma_conf, coh_mask): cf.write_config_file(params=params, output_conf_file=output_conf) params_s = Configuration(output_conf).__dict__ conv2tif.main(params_s) + # reread params from config params_s = Configuration(output_conf).__dict__ prepifg.main(params_s) ifg_files = list(Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif')) @@ -133,7 +134,7 @@ def test_prepifg_file_types(tempdir, gamma_conf, coh_mask): dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM - # shutil.rmtree(tdir) + shutil.rmtree(tdir) # convenience ifg creation funcs From e99411038609da5b8ce1ba4da7dc18a5113dea17 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Fri, 19 Jun 2020 04:32:39 +1000 Subject: [PATCH 15/23] rounding may work better for refpixel --- pyrate/core/refpixel.py | 6 +++--- pyrate/process.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index 07f993f0a..04ed097f8 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -117,10 +117,10 @@ def convert_geographic_coordinate_to_pixel_value(lon, lat, transform): pixelWidth = transform[1] pixelHeight = -transform[5] - refx = int((lon - xOrigin) / pixelWidth) - refy = int((yOrigin - lat) / pixelHeight) + refx = round((lon - xOrigin) / pixelWidth) + refy = round((yOrigin - lat) / pixelHeight) - return refx + 1, refy + 1 + return refx, refy # TODO: move error checking to config step (for fail fast) diff --git a/pyrate/process.py b/pyrate/process.py index 56e16533b..364fefabf 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -21,7 +21,7 @@ from os.path import join import pickle as cp from collections import OrderedDict -from typing import List +from typing import List, Tuple import numpy as np from pyrate.core import (shared, algorithm, orbital, ref_phs_est as rpe, @@ -120,7 +120,7 @@ def _save_mst_tile(tile, i, preread_ifgs): mpiops.comm.barrier() -def _ref_pixel_calc(ifg_paths, params): +def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: """ Wrapper for reference pixel calculation """ From 2c8f20fb66f787267e6bf7e4d65bc63467d450cf Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 20 Jun 2020 06:37:35 +1000 Subject: [PATCH 16/23] simpler prepfig functions and type hints --- pyrate/core/prepifg_helper.py | 13 ++++++----- pyrate/prepifg.py | 43 ++++++++++++++++------------------- pyrate/process.py | 10 ++++++++ tests/test_pyrate.py | 11 +++++---- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/pyrate/core/prepifg_helper.py b/pyrate/core/prepifg_helper.py index 6d178aa23..7435607b1 100644 --- a/pyrate/core/prepifg_helper.py +++ b/pyrate/core/prepifg_helper.py @@ -24,13 +24,13 @@ from math import modf from numbers import Number from decimal import Decimal - +from typing import List, Tuple, Union from numpy import array, nan, isnan, nanmean, float32, zeros, sum as nsum from osgeo import gdal from pyrate.core.gdal_python import crop_resample_average from pyrate.core import config as cf -from pyrate.core.shared import output_tiff_filename, dem_or_ifg +from pyrate.core.shared import output_tiff_filename, dem_or_ifg, Ifg, DEM CustomExts = namedtuple('CustExtents', ['xfirst', 'yfirst', 'xlast', 'ylast']) @@ -45,7 +45,8 @@ GRID_TOL = 1e-6 -def get_analysis_extent(crop_opt, rasters, xlooks, ylooks, user_exts): +def get_analysis_extent(crop_opt: int, rasters: List[Union[Ifg, DEM]], xlooks: int, ylooks: int, + user_exts: Tuple[float, float, float, float]) -> Tuple[float, float, float, float]: """ Function checks prepifg parameters and returns extents/bounding box. @@ -267,7 +268,7 @@ def _resample(data, xscale, yscale, thresh): return dest -def _min_bounds(ifgs): +def _min_bounds(ifgs: List[Ifg]) -> Tuple[float, float, float, float]: """ Returns bounds for overlapping area of the given interferograms. """ @@ -279,7 +280,7 @@ def _min_bounds(ifgs): return xmin, ymin, xmax, ymax -def _max_bounds(ifgs): +def _max_bounds(ifgs: List[Ifg]) -> Tuple[float, float, float, float]: """ Returns bounds for the total area covered by the given interferograms. """ @@ -291,7 +292,7 @@ def _max_bounds(ifgs): return xmin, ymin, xmax, ymax -def _get_same_bounds(ifgs): +def _get_same_bounds(ifgs: List[Ifg]) -> Tuple[float, float, float, float]: """ Check and return bounding box for ALREADY_SAME_SIZE option. """ diff --git a/pyrate/prepifg.py b/pyrate/prepifg.py index aff72ff7c..49426d2b0 100644 --- a/pyrate/prepifg.py +++ b/pyrate/prepifg.py @@ -20,10 +20,9 @@ # -*- coding: utf-8 -*- import os from subprocess import check_call -from typing import List +from typing import List, Tuple from pathlib import Path from joblib import Parallel, delayed -from typing import Union import numpy as np from osgeo import gdal from pyrate.core import shared, mpiops, config as cf, prepifg_helper, gamma, roipac, ifgconstants as ifc @@ -60,19 +59,21 @@ def main(params): shared.mkdir_p(params[cf.OUT_DIR]) # create output dir - process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank] + user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) + xlooks, ylooks, crop = cf.transform_params(params) + ifgs = [prepifg_helper.dem_or_ifg(p.converted_path) for p in ifg_paths] + exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts) - do_prepifg(process_ifgs_paths, params) + process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank] + do_prepifg(process_ifgs_paths, exts, params) mpiops.comm.barrier() log.info("Finished prepifg") -def do_prepifg(multi_paths: List[MultiplePaths], params: dict) -> None: +def do_prepifg(multi_paths: List[MultiplePaths], exts: Tuple[float, float, float, float], params: dict) -> None: """ Prepare interferograms by applying multilooking/cropping operations. - :param list multi_paths: List of full-res geotiffs - :param dict params: Parameters dictionary corresponding to config file """ # pylint: disable=expression-not-assigned parallel = params[cf.PARALLEL] @@ -84,33 +85,26 @@ def do_prepifg(multi_paths: List[MultiplePaths], params: dict) -> None: raise FileNotFoundError("Can not find geotiff: " + str(f) + ". Ensure you have converted your " "interferograms to geotiffs.") - ifgs = [prepifg_helper.dem_or_ifg(p.converted_path) for p in multi_paths] - xlooks, ylooks, crop = cf.transform_params(params) - user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) - exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts) - thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] - if params[cf.LARGE_TIFS]: log.info("Using gdal system calls to process prepifg") - ifg = ifgs[0] + ifg = prepifg_helper.dem_or_ifg(multi_paths[0].converted_path) + xlooks, ylooks = params[cf.IFG_LKSX], params[cf.IFG_LKSY] res_str = [xlooks * ifg.x_step, ylooks * ifg.y_step] res_str = ' '.join([str(e) for e in res_str]) if parallel: Parallel(n_jobs=params[cf.PROCESSES], verbose=50)( - delayed(__prepifg_system)( - crop, exts, gtiff_path, params, res_str, thresh, xlooks, ylooks) for gtiff_path in multi_paths - ) + delayed(__prepifg_system)(exts, gtiff_path, params, res_str) for gtiff_path in multi_paths) else: for m_path in multi_paths: - __prepifg_system(crop, exts, m_path, params, res_str, thresh, xlooks, ylooks) + __prepifg_system(exts, m_path, params, res_str) else: if parallel: Parallel(n_jobs=params[cf.PROCESSES], verbose=50)( - delayed(_prepifg_multiprocessing)(p, xlooks, ylooks, exts, thresh, crop, params) for p in multi_paths + delayed(_prepifg_multiprocessing)(p, exts, params) for p in multi_paths ) else: for m_path in multi_paths: - _prepifg_multiprocessing(m_path, xlooks, ylooks, exts, thresh, crop, params) + _prepifg_multiprocessing(m_path, exts, params) COMMON_OPTIONS = "-co BLOCKXSIZE=256 -co BLOCKYSIZE=256 -co TILED=YES --config GDAL_CACHEMAX=64 -q" @@ -118,8 +112,9 @@ def do_prepifg(multi_paths: List[MultiplePaths], params: dict) -> None: GDAL_CALC = 'gdal_calc_local.py' -def __prepifg_system(crop, exts, gtiff, params, res, thresh, xlooks, ylooks): - p, c, l = _prepifg_multiprocessing(gtiff, xlooks, ylooks, exts, thresh, crop, params) +def __prepifg_system(exts, gtiff, params, res): + thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] + p, c, l = _prepifg_multiprocessing(gtiff, exts, params) log.info("Multilooking {p} into {l}".format(p=p, l=l)) extents = ' '.join([str(e) for e in exts]) @@ -208,10 +203,12 @@ def __update_meta_data(p_unset, c, l): ds = None -def _prepifg_multiprocessing(m_path: MultiplePaths, xlooks, ylooks, exts, thresh, crop, params): +def _prepifg_multiprocessing(m_path: MultiplePaths, exts: Tuple[float, float, float, float], params: dict): """ Multiprocessing wrapper for prepifg """ + xlooks, ylooks, crop = cf.transform_params(params) + thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] header = find_header(m_path, params) header[ifc.INPUT_TYPE] = m_path.input_type diff --git a/pyrate/process.py b/pyrate/process.py index 364fefabf..3de6f2af6 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -120,6 +120,16 @@ def _save_mst_tile(tile, i, preread_ifgs): mpiops.comm.barrier() +def __validate_ref_pixel(params): + from pyrate.core import prepifg_helper + extents = prepifg_helper.get_analysis_extent( + crop_opt=params[cf.IFG_CROP_OPT], + rasters=[prepifg_helper.dem_or_ifg(p.converted_path) for p in params[cf.INTERFEROGRAM_FILES]], + xlooks=params[cf.IFG_LKSX], ylooks=params[cf.IFG_LKSY], + user_exts=(params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) + ) + + def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: """ Wrapper for reference pixel calculation diff --git a/tests/test_pyrate.py b/tests/test_pyrate.py index 142eee911..1ff65f1d3 100644 --- a/tests/test_pyrate.py +++ b/tests/test_pyrate.py @@ -222,8 +222,10 @@ def setUpClass(cls): # dest_paths are tifs that have been geotif converted and multilooked cls.converted_paths = [b.converted_path for b in multi_paths] cls.sampled_paths = [b.sampled_path for b in multi_paths] - conv2tif.do_geotiff(multi_paths, params) - prepifg.do_prepifg(multi_paths, params) + from copy import copy + orig_params = copy(params) + conv2tif.main(params) + prepifg.main(orig_params) tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols) ifgs = common.small_data_setup() params[cf.INTERFEROGRAM_FILES] = multi_paths @@ -246,8 +248,9 @@ def setUpClass(cls): cls.converted_paths_s = [b.converted_path for b in multi_paths] cls.sampled_paths_s = [b.sampled_path for b in multi_paths] - conv2tif.do_geotiff(multi_paths, params) - prepifg.do_prepifg(multi_paths, params) + orig_params = copy(params) + conv2tif.main(params) + prepifg.main(orig_params) params[cf.INTERFEROGRAM_FILES] = multi_paths cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs(cls.sampled_paths_s, params, rows, cols) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) From 5d0edbb15879fa0e02202ad12344bfe06f76d6b3 Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Sat, 20 Jun 2020 07:02:04 +1000 Subject: [PATCH 17/23] refpixel validation --- pyrate/process.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/pyrate/process.py b/pyrate/process.py index 3de6f2af6..447015b39 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -120,20 +120,29 @@ def _save_mst_tile(tile, i, preread_ifgs): mpiops.comm.barrier() -def __validate_ref_pixel(params): - from pyrate.core import prepifg_helper - extents = prepifg_helper.get_analysis_extent( - crop_opt=params[cf.IFG_CROP_OPT], - rasters=[prepifg_helper.dem_or_ifg(p.converted_path) for p in params[cf.INTERFEROGRAM_FILES]], - xlooks=params[cf.IFG_LKSX], ylooks=params[cf.IFG_LKSY], - user_exts=(params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) - ) +def __validate_ref_pixel(params: dict) -> None: + lon, lat = params[cf.REFX], params[cf.REFY] + if lon == -1 or lat == -1: + return + from pyrate.core import prepifg_helper + xmin, ymin, xmax, ymax = prepifg_helper.get_analysis_extent( + crop_opt=params[cf.IFG_CROP_OPT], + rasters=[prepifg_helper.dem_or_ifg(p.sampled_path) for p in params[cf.INTERFEROGRAM_FILES]], + xlooks=params[cf.IFG_LKSX], ylooks=params[cf.IFG_LKSY], + user_exts=(params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) + ) + msg = f"Supplied reference pixel is out of bounds for the ananlysis extents" + if (lon < xmin) or (lon > xmax): + raise ValueError(msg) + if (lat < ymin) or (lat > ymax): + raise ValueError(msg) def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: """ Wrapper for reference pixel calculation """ + lon = params[cf.REFX] lat = params[cf.REFY] @@ -300,6 +309,8 @@ def process_ifgs(ifg_paths, params, rows, cols): :return: vcmt: Variance-covariance matrix array :rtype: ndarray """ + # validate refpixel + __validate_ref_pixel(params) if mpiops.size > 1: # turn of multiprocessing during mpi jobs params[cf.PARALLEL] = False From 2757b686716630beff2c1ad501380466167cfef1 Mon Sep 17 00:00:00 2001 From: Matt Garthwaite Date: Mon, 22 Jun 2020 16:37:27 +1000 Subject: [PATCH 18/23] refactor supplied lat/lon value validation; improve error messages --- pyrate/core/refpixel.py | 45 ++++++++++++++++++++++++++++++++--------- pyrate/process.py | 23 ++------------------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index 04ed097f8..fccb51cb6 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -33,6 +33,7 @@ from pyrate.core.shared import Ifg from pyrate.core.shared import joblib_log_level from pyrate.core.logger import pyratelogger as log +from pyrate.core import prepifg_helper def update_refpix_metadata(ifg_paths, refx, refy, transform, params): @@ -158,8 +159,8 @@ def ref_pixel(ifgs, params): mean_sds.append(_ref_pixel_multi(g, half_patch_size, phase_data, thresh, params)) refxy = find_min_mean(mean_sds, grid) - if isinstance(refxy, ValueError): - raise ValueError('Refpixel calculation not possible!') + if isinstance(refxy, RefPixelError): + raise RefPixelError('Refpixel calculation not possible!') refy, refx = refxy @@ -184,7 +185,7 @@ def find_min_mean(mean_sds, grid): try: refp_index = np.nanargmin(mean_sds) return grid[refp_index] - except ValueError as v: + except RefPixelError as v: log.error(v) return v @@ -342,7 +343,7 @@ def _validate_chipsize(chipsize, head): if chipsize < 3 or chipsize > head.ncols or (chipsize % 2 == 0): msg = "Chipsize setting must be >=3 and at least <= grid width" - raise ValueError(msg) + raise RefPixelError(msg) log.debug('Chipsize validation successful') @@ -354,7 +355,7 @@ def _validate_minimum_fraction(min_frac): raise cf.ConfigException('Minimum fraction is None') if min_frac < 0.0 or min_frac > 1.0: - raise ValueError("Minimum fraction setting must be >= 0.0 and <= 1.0 ") + raise RefPixelError("Minimum fraction setting must be >= 0.0 and <= 1.0 ") def _validate_search_win(refnx, refny, chipsize, head): @@ -367,7 +368,7 @@ def _validate_search_win(refnx, refny, chipsize, head): max_width = (head.ncols - (chipsize-1)) if refnx < 1 or refnx > max_width: msg = "Invalid refnx setting, must be > 0 and <= %s" - raise ValueError(msg % max_width) + raise RefPixelError(msg % max_width) if refny is None: raise cf.ConfigException('refny is None') @@ -375,10 +376,36 @@ def _validate_search_win(refnx, refny, chipsize, head): max_rows = (head.nrows - (chipsize-1)) if refny < 1 or refny > max_rows: msg = "Invalid refny setting, must be > 0 and <= %s" - raise ValueError(msg % max_rows) + raise RefPixelError(msg % max_rows) + + +def _validate_ref_pixel(params: dict) -> None: + """ + Sanity check that supplied lat/lon values sit within image bounds + """ + lon, lat = params[cf.REFX], params[cf.REFY] + if lon == -1 or lat == -1: + return + xmin, ymin, xmax, ymax = prepifg_helper.get_analysis_extent( + crop_opt=params[cf.IFG_CROP_OPT], + rasters=[prepifg_helper.dem_or_ifg(p.sampled_path) for p in params[cf.INTERFEROGRAM_FILES]], + xlooks=params[cf.IFG_LKSX], ylooks=params[cf.IFG_LKSY], + user_exts=(params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) + ) + err = 0 + if (lon < xmin) or (lon > xmax): err += 1 + if (lat < ymin) or (lat > ymax): err += 2 + if err == 1: + msg = f"Supplied longitude value is outside the bounds of the interferogram data" + elif err == 2: + msg = f"Supplied latitude value is outside the bounds of the interferogram data" + elif err == 3: + msg = f"Supplied latitude and longitude values are outside the bounds of the interferogram data" + + if err != 0: raise RefPixelError(msg) class RefPixelError(Exception): - ''' + """ Generic exception for reference pixel errors. - ''' + """ diff --git a/pyrate/process.py b/pyrate/process.py index 447015b39..2e57b7f0f 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -120,24 +120,6 @@ def _save_mst_tile(tile, i, preread_ifgs): mpiops.comm.barrier() -def __validate_ref_pixel(params: dict) -> None: - lon, lat = params[cf.REFX], params[cf.REFY] - if lon == -1 or lat == -1: - return - from pyrate.core import prepifg_helper - xmin, ymin, xmax, ymax = prepifg_helper.get_analysis_extent( - crop_opt=params[cf.IFG_CROP_OPT], - rasters=[prepifg_helper.dem_or_ifg(p.sampled_path) for p in params[cf.INTERFEROGRAM_FILES]], - xlooks=params[cf.IFG_LKSX], ylooks=params[cf.IFG_LKSY], - user_exts=(params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) - ) - msg = f"Supplied reference pixel is out of bounds for the ananlysis extents" - if (lon < xmin) or (lon > xmax): - raise ValueError(msg) - if (lat < ymin) or (lat > ymax): - raise ValueError(msg) - - def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: """ Wrapper for reference pixel calculation @@ -154,7 +136,6 @@ def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: if lon == -1 or lat == -1: log.info('Searching for best reference pixel location') - half_patch_size, thresh, grid = refpixel.ref_pixel_setup(ifg_paths, params) process_grid = mpiops.array_split(grid) refpixel.save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params) @@ -177,6 +158,8 @@ def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: log.info('Selected reference pixel coordinate (lon, lat): ({}, {})'.format(lon, lat)) else: + # validate user supplied ref pixel + refpixel._validate_ref_pixel(params) log.info('Using reference pixel from config file (lon, lat): ({}, {})'.format(lon, lat)) log.warning("Ensure user supplied reference pixel values are in lon/lat") refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(lon, lat, transform) @@ -309,8 +292,6 @@ def process_ifgs(ifg_paths, params, rows, cols): :return: vcmt: Variance-covariance matrix array :rtype: ndarray """ - # validate refpixel - __validate_ref_pixel(params) if mpiops.size > 1: # turn of multiprocessing during mpi jobs params[cf.PARALLEL] = False From 52d226f3f3d02d808437d16550491ca701384e30 Mon Sep 17 00:00:00 2001 From: Matt Garthwaite Date: Tue, 23 Jun 2020 20:55:39 +1000 Subject: [PATCH 19/23] minor edit - fix up tests --- pyrate/core/refpixel.py | 4 ++-- pyrate/process.py | 4 ++-- tests/test_refpixel.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index fccb51cb6..2a517263c 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -379,9 +379,9 @@ def _validate_search_win(refnx, refny, chipsize, head): raise RefPixelError(msg % max_rows) -def _validate_ref_pixel(params: dict) -> None: +def validate_supplied_lat_lon(params: dict) -> None: """ - Sanity check that supplied lat/lon values sit within image bounds + Function to validate that the user supplied lat/lon values sit within image bounds """ lon, lat = params[cf.REFX], params[cf.REFY] if lon == -1 or lat == -1: diff --git a/pyrate/process.py b/pyrate/process.py index 2e57b7f0f..8cbfd8e28 100644 --- a/pyrate/process.py +++ b/pyrate/process.py @@ -158,8 +158,6 @@ def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: log.info('Selected reference pixel coordinate (lon, lat): ({}, {})'.format(lon, lat)) else: - # validate user supplied ref pixel - refpixel._validate_ref_pixel(params) log.info('Using reference pixel from config file (lon, lat): ({}, {})'.format(lon, lat)) log.warning("Ensure user supplied reference pixel values are in lon/lat") refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(lon, lat, transform) @@ -303,6 +301,8 @@ def process_ifgs(ifg_paths, params, rows, cols): preread_ifgs = _create_ifg_dict(ifg_paths, params=params) + # validate user supplied ref pixel + refpixel.validate_supplied_lat_lon(params) refpx, refpy = _ref_pixel_calc(ifg_paths, params) # remove non ifg keys diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index d98b4fe47..4a6019f02 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -25,7 +25,7 @@ from numpy import nan, mean, std, isnan from pyrate.core import config as cf -from pyrate.core.refpixel import ref_pixel, _step +from pyrate.core.refpixel import ref_pixel, _step, RefPixelError from pyrate.core import shared, ifgconstants as ifc from pyrate import process @@ -68,7 +68,7 @@ def test_missing_chipsize(self): def test_chipsize_valid(self): for illegal in [0, -1, -15, 1, 2, self.ifgs[0].ncols+1, 4, 6, 10, 20]: self.params[cf.REF_CHIP_SIZE] = illegal - self.assertRaises(ValueError, ref_pixel, self.ifgs, self.params) + self.assertRaises(RefPixelError, ref_pixel, self.ifgs, self.params) def test_minimum_fraction_missing(self): self.params[cf.REF_MIN_FRAC] = None @@ -77,18 +77,18 @@ def test_minimum_fraction_missing(self): def test_minimum_fraction_threshold(self): for illegal in [-0.1, 1.1, 1.000001, -0.0000001]: self.params[cf.REF_MIN_FRAC] = illegal - self.assertRaises(ValueError, ref_pixel, self.ifgs, self.params) + self.assertRaises(RefPixelError, ref_pixel, self.ifgs, self.params) def test_search_windows(self): # 45 is max # cells a width 3 sliding window can iterate over for illegal in [-5, -1, 0, 46, 50, 100]: self.params[cf.REFNX] = illegal - self.assertRaises(ValueError, ref_pixel, self.ifgs, self.params) + self.assertRaises(RefPixelError, ref_pixel, self.ifgs, self.params) # 40 is max # cells a width 3 sliding window can iterate over for illegal in [-5, -1, 0, 71, 85, 100]: self.params[cf.REFNY] = illegal - self.assertRaises(ValueError, ref_pixel, self.ifgs, self.params) + self.assertRaises(RefPixelError, ref_pixel, self.ifgs, self.params) def test_missing_search_windows(self): self.params[cf.REFNX] = None From 9c743c846803bddb57e0d5c2a0858275ebe9de4c Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Wed, 24 Jun 2020 05:01:54 +1000 Subject: [PATCH 20/23] minor fix --- pyrate/prepifg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyrate/prepifg.py b/pyrate/prepifg.py index 49426d2b0..a9803df24 100644 --- a/pyrate/prepifg.py +++ b/pyrate/prepifg.py @@ -88,6 +88,7 @@ def do_prepifg(multi_paths: List[MultiplePaths], exts: Tuple[float, float, float if params[cf.LARGE_TIFS]: log.info("Using gdal system calls to process prepifg") ifg = prepifg_helper.dem_or_ifg(multi_paths[0].converted_path) + ifg.open() xlooks, ylooks = params[cf.IFG_LKSX], params[cf.IFG_LKSY] res_str = [xlooks * ifg.x_step, ylooks * ifg.y_step] res_str = ' '.join([str(e) for e in res_str]) From fbab1348f9411ff083a48d99493aded5719299fb Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Wed, 24 Jun 2020 06:00:45 +1000 Subject: [PATCH 21/23] update refpixel tests and validation --- pyrate/core/refpixel.py | 19 ++++++++----------- tests/conftest.py | 2 +- ...t_mpi_vs_multiprocess_vs_single_process.py | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/pyrate/core/refpixel.py b/pyrate/core/refpixel.py index 2a517263c..d25fbafba 100644 --- a/pyrate/core/refpixel.py +++ b/pyrate/core/refpixel.py @@ -392,17 +392,14 @@ def validate_supplied_lat_lon(params: dict) -> None: xlooks=params[cf.IFG_LKSX], ylooks=params[cf.IFG_LKSY], user_exts=(params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) ) - err = 0 - if (lon < xmin) or (lon > xmax): err += 1 - if (lat < ymin) or (lat > ymax): err += 2 - if err == 1: - msg = f"Supplied longitude value is outside the bounds of the interferogram data" - elif err == 2: - msg = f"Supplied latitude value is outside the bounds of the interferogram data" - elif err == 3: - msg = f"Supplied latitude and longitude values are outside the bounds of the interferogram data" - - if err != 0: raise RefPixelError(msg) + msg = "Supplied {} value is outside the bounds of the interferogram data" + lat_lon_txt = '' + if (lon < xmin) or (lon > xmax): + lat_lon_txt += 'longitude' + if (lat < ymin) or (lat > ymax): + lat_lon_txt += ' and latitude' if lat_lon_txt else 'latitude' + if lat_lon_txt: + raise RefPixelError(msg.format(lat_lon_txt)) class RefPixelError(Exception): diff --git a/tests/conftest.py b/tests/conftest.py index 64a53668a..19240a3ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,7 +56,7 @@ def ref_est_method(request): return request.param -@pytest.fixture(params=[(-1, -1), (38, 58)]) +@pytest.fixture(params=[(-1, -1), (150.941666654, -34.218333314)]) def ref_pixel(request): return request.param diff --git a/tests/test_mpi_vs_multiprocess_vs_single_process.py b/tests/test_mpi_vs_multiprocess_vs_single_process.py index 9a4f61d06..330f3d7c8 100644 --- a/tests/test_mpi_vs_multiprocess_vs_single_process.py +++ b/tests/test_mpi_vs_multiprocess_vs_single_process.py @@ -23,7 +23,7 @@ def parallel(request): return request.param -@pytest.fixture(params=[1, 2, 3, 4]) +@pytest.fixture(params=[1, 2, 4]) def local_crop(request): return request.param From 3f25d5015f261db3620e672f78ee210fbe6839ba Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Wed, 24 Jun 2020 06:17:13 +1000 Subject: [PATCH 22/23] update refpixel tests --- tests/test_refpixel.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 4a6019f02..06c39880c 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -19,7 +19,7 @@ import copy import unittest import shutil -from subprocess import check_call +from subprocess import check_call, run from pathlib import Path import pytest from numpy import nan, mean, std, isnan @@ -352,13 +352,20 @@ def test_small_test_data_ref_all_1(self): self.assertEqual(refy, 2) +@pytest.mark.slow +def test_error_msg_refpixel_out_out_bounds(tempdir, gamma_conf): + "check correct latitude/longitude refpixel error is raised when specified refpixel is out of bounds" + for x, (refx, refy) in zip(['longitude', 'latitude', 'longitude and latitude'], + [(150., -34.218333314), (150.941666654, -34.), (150, -34)]): + _, out = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=refx, refy=refy) + msg = "Supplied {} value is outside the bounds of the interferogram data" + assert msg.format(x) in out.stderr + + @pytest.mark.slow def test_gamma_ref_pixel_search_vs_lat_lon(tempdir, gamma_conf): - # refx, refy: 38 58 - # refx: 150.94211 - # refy: -34.21916 - params_1 = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=-1, refy=-1) - params_2 = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=150.941666654, refy=-34.218333314) + params_1, _ = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=-1, refy=-1) + params_2, _ = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=150.941666654, refy=-34.218333314) assert_two_dirs_equal(params_1[cf.OUT_DIR], params_2[cf.OUT_DIR], f"*{params_1[cf.IFG_CROP_OPT]}cr.tif", 18) @@ -371,5 +378,6 @@ def _get_mlooked_files(gamma_conf, tdir, refx, refy): cf.write_config_file(params=params, output_conf_file=output_conf) check_call(f"pyrate conv2tif -f {output_conf}", shell=True) check_call(f"pyrate prepifg -f {output_conf}", shell=True) - check_call(f"pyrate process -f {output_conf}", shell=True) - return params + stdout = run(f"pyrate process -f {output_conf}", shell=True, capture_output=True, text=True) + print("============================================", stdout) + return params, stdout From 226cd598d965a11ce53d07964189ddd291f8416a Mon Sep 17 00:00:00 2001 From: Sudipta Basak Date: Wed, 24 Jun 2020 08:35:37 +1000 Subject: [PATCH 23/23] update tests --- tests/common.py | 8 ++++++++ ...st_mpi_vs_multiprocess_vs_single_process.py | 16 ++++++++++------ tests/test_prepifg_system_vs_python.py | 18 +++++++++--------- tests/test_refpixel.py | 4 +++- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/tests/common.py b/tests/common.py index 90d4240ce..9fd8ba3d3 100644 --- a/tests/common.py +++ b/tests/common.py @@ -24,6 +24,7 @@ import stat import tempfile from os.path import join +from subprocess import check_output from pathlib import Path import numpy as np @@ -36,6 +37,13 @@ from pyrate.constants import PYRATEPATH from pyrate.configuration import Configuration +TRAVIS = True if 'TRAVIS' in os.environ else False +PYTHON3P6 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.6') else False +PYTHON3P7 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.7') else False +PYTHON3P8 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.8') else False +GDAL_VERSION = check_output(["gdal-config", "--version"]).decode(encoding="utf-8").split('\n')[0] + + TEMPDIR = tempfile.gettempdir() TESTDIR = join(PYRATEPATH, 'tests') BASE_TEST = join(PYRATEPATH, "tests", "test_data") diff --git a/tests/test_mpi_vs_multiprocess_vs_single_process.py b/tests/test_mpi_vs_multiprocess_vs_single_process.py index 330f3d7c8..32fd0d755 100644 --- a/tests/test_mpi_vs_multiprocess_vs_single_process.py +++ b/tests/test_mpi_vs_multiprocess_vs_single_process.py @@ -5,13 +5,17 @@ from subprocess import check_call, check_output, CalledProcessError import numpy as np from pyrate.core import config as cf -from tests.common import assert_same_files_produced, assert_two_dirs_equal, manipulate_test_conf +from tests.common import ( + assert_same_files_produced, + assert_two_dirs_equal, + manipulate_test_conf, + TRAVIS, + PYTHON3P6, + PYTHON3P7, + PYTHON3P8, + GDAL_VERSION +) -TRAVIS = True if 'TRAVIS' in os.environ else False -PYTHON3P6 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.6') else False -PYTHON3P7 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.7') else False -PYTHON3P8 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.8') else False -GDAL_VERSION = check_output(["gdal-config", "--version"]).decode(encoding="utf-8").split('\n')[0] # python3.7 and gdal3.0.4 REGRESSION = PYTHON3P7 and (GDAL_VERSION == '3.0.4') # python3.7 and gdal3.0.2 diff --git a/tests/test_prepifg_system_vs_python.py b/tests/test_prepifg_system_vs_python.py index 49a9c1f3b..c6772cdc0 100644 --- a/tests/test_prepifg_system_vs_python.py +++ b/tests/test_prepifg_system_vs_python.py @@ -1,17 +1,17 @@ -import os import shutil import pytest -import numpy as np from pathlib import Path -from subprocess import check_call, check_output, CalledProcessError +from subprocess import check_call import numpy as np from pyrate.core import config as cf -from tests.common import assert_two_dirs_equal, manipulate_test_conf -TRAVIS = True if 'TRAVIS' in os.environ else False -PYTHON3P6 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.6') else False -PYTHON3P7 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.7') else False -PYTHON3P8 = True if ('TRAVIS_PYTHON_VERSION' in os.environ and os.environ['TRAVIS_PYTHON_VERSION'] == '3.8') else False -GDAL_VERSION = check_output(["gdal-config", "--version"]).decode(encoding="utf-8").split('\n')[0] +from tests.common import ( + assert_two_dirs_equal, + manipulate_test_conf, + TRAVIS, + PYTHON3P6, + PYTHON3P7, + GDAL_VERSION +) # python3.7 and gdal3.0.4 REGRESSION = PYTHON3P7 and (GDAL_VERSION == '3.0.4') # python3.7 and gdal3.0.2 diff --git a/tests/test_refpixel.py b/tests/test_refpixel.py index 06c39880c..17839b420 100644 --- a/tests/test_refpixel.py +++ b/tests/test_refpixel.py @@ -32,7 +32,7 @@ from pyrate.configuration import Configuration from tests.common import TEST_CONF_ROIPAC from tests.common import small_data_setup, MockIfg, copy_small_ifg_file_list, \ - copy_and_setup_small_data, manipulate_test_conf, assert_two_dirs_equal + copy_and_setup_small_data, manipulate_test_conf, assert_two_dirs_equal, PYTHON3P6 # TODO: figure out how editing resource.setrlimit fixes the error @@ -353,6 +353,7 @@ def test_small_test_data_ref_all_1(self): @pytest.mark.slow +@pytest.mark.skip(PYTHON3P6, reason='Skipped in python3p6') def test_error_msg_refpixel_out_out_bounds(tempdir, gamma_conf): "check correct latitude/longitude refpixel error is raised when specified refpixel is out of bounds" for x, (refx, refy) in zip(['longitude', 'latitude', 'longitude and latitude'], @@ -363,6 +364,7 @@ def test_error_msg_refpixel_out_out_bounds(tempdir, gamma_conf): @pytest.mark.slow +@pytest.mark.skip(PYTHON3P6, reason='Skipped in python3p6') def test_gamma_ref_pixel_search_vs_lat_lon(tempdir, gamma_conf): params_1, _ = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=-1, refy=-1) params_2, _ = _get_mlooked_files(gamma_conf, Path(tempdir()), refx=150.941666654, refy=-34.218333314)