From 23b57ce23cd10c3528f9dbea007014a85500f97b Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Sun, 17 Mar 2024 14:59:26 -0700 Subject: [PATCH 1/4] Setting cloud score masking default to True --- openet/sims/image.py | 4 ++-- pyproject.toml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openet/sims/image.py b/openet/sims/image.py index 49b32c5..9448bfc 100644 --- a/openet/sims/image.py +++ b/openet/sims/image.py @@ -444,11 +444,11 @@ def from_landsat_c2_sr(cls, sr_image, cloudmask_args={}, **kwargs): if 'snow_flag' not in cloudmask_args.keys(): cloudmask_args['snow_flag'] = True if 'cloud_score_flag' not in cloudmask_args.keys(): - cloudmask_args['cloud_score_flag'] = False + cloudmask_args['cloud_score_flag'] = True if 'cloud_score_pct' not in cloudmask_args.keys(): cloudmask_args['cloud_score_pct'] = 100 if 'filter_flag' not in cloudmask_args.keys(): - cloudmask_args['filter_flag'] = False + cloudmask_args['filter_flag'] = True # QA_RADSAT band will need to be added above if applying saturated masking # if 'saturated_flag' not in cloudmask_args.keys(): # cloudmask_args['saturated_flag'] = False diff --git a/pyproject.toml b/pyproject.toml index 90be05f..b0cbfbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openet-sims" -version = "0.2.0" +version = "0.2.1" authors = [ { name = "Alberto Guzman", email = "aguzman@csumb.edu" }, ] @@ -16,8 +16,8 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "earthengine-api>=0.1.364", - "openet-core>=0.4.0", + "earthengine-api >= 0.1.364", + "openet-core >= 0.4.0", ] [project.urls] From 02d3e6d1baca0c0a7ecc154f28a5d7854c6e9818 Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Tue, 19 Mar 2024 05:43:10 -0700 Subject: [PATCH 2/4] Adding QA_RADSAT masking back in --- openet/sims/image.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/openet/sims/image.py b/openet/sims/image.py index 9448bfc..b234829 100644 --- a/openet/sims/image.py +++ b/openet/sims/image.py @@ -419,18 +419,18 @@ def from_landsat_c2_sr(cls, sr_image, cloudmask_args={}, **kwargs): # Rename bands to generic names input_bands = ee.Dictionary({ - 'LANDSAT_4': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL'], - 'LANDSAT_5': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL'], - 'LANDSAT_7': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL'], - 'LANDSAT_8': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', 'ST_B10', 'QA_PIXEL'], - 'LANDSAT_9': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', 'ST_B10', 'QA_PIXEL'], + 'LANDSAT_4': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_5': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_7': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_8': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', 'ST_B10', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_9': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', 'ST_B10', 'QA_PIXEL', 'QA_RADSAT'], }) - output_bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'tir', 'QA_PIXEL'] + output_bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'tir', 'QA_PIXEL', 'QA_RADSAT'] prep_image = ( sr_image .select(input_bands.get(spacecraft_id), output_bands) - .multiply([0.0000275, 0.0000275, 0.0000275, 0.0000275, 0.0000275, 0.0000275, 0.00341802, 1]) - .add([-0.2, -0.2, -0.2, -0.2, -0.2, -0.2, 149.0, 0]) + .multiply([0.0000275, 0.0000275, 0.0000275, 0.0000275, 0.0000275, 0.0000275, 0.00341802, 1, 1]) + .add([-0.2, -0.2, -0.2, -0.2, -0.2, -0.2, 149.0, 0, 0]) ) # Default the cloudmask flags to True if they were not @@ -449,9 +449,8 @@ def from_landsat_c2_sr(cls, sr_image, cloudmask_args={}, **kwargs): cloudmask_args['cloud_score_pct'] = 100 if 'filter_flag' not in cloudmask_args.keys(): cloudmask_args['filter_flag'] = True - # QA_RADSAT band will need to be added above if applying saturated masking - # if 'saturated_flag' not in cloudmask_args.keys(): - # cloudmask_args['saturated_flag'] = False + if 'saturated_flag' not in cloudmask_args.keys(): + cloudmask_args['saturated_flag'] = True cloud_mask = openet.core.common.landsat_c2_sr_cloud_mask(sr_image, **cloudmask_args) From 857f6f402e853d34ad7bc4661614ba995f6c3e18 Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Wed, 20 Mar 2024 10:09:17 -0700 Subject: [PATCH 3/4] Added support for QA_RADSAT masking back Until we make minor version update keep defaults as False --- openet/sims/image.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/openet/sims/image.py b/openet/sims/image.py index b234829..7335b2c 100644 --- a/openet/sims/image.py +++ b/openet/sims/image.py @@ -419,13 +419,19 @@ def from_landsat_c2_sr(cls, sr_image, cloudmask_args={}, **kwargs): # Rename bands to generic names input_bands = ee.Dictionary({ - 'LANDSAT_4': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], - 'LANDSAT_5': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], - 'LANDSAT_7': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], - 'LANDSAT_8': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', 'ST_B10', 'QA_PIXEL', 'QA_RADSAT'], - 'LANDSAT_9': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', 'ST_B10', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_4': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', + 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_5': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', + 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_7': ['SR_B1', 'SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B7', + 'ST_B6', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_8': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', + 'ST_B10', 'QA_PIXEL', 'QA_RADSAT'], + 'LANDSAT_9': ['SR_B2', 'SR_B3', 'SR_B4', 'SR_B5', 'SR_B6', 'SR_B7', + 'ST_B10', 'QA_PIXEL', 'QA_RADSAT'], }) - output_bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', 'tir', 'QA_PIXEL', 'QA_RADSAT'] + output_bands = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2', + 'lst', 'QA_PIXEL', 'QA_RADSAT'] prep_image = ( sr_image .select(input_bands.get(spacecraft_id), output_bands) @@ -444,13 +450,13 @@ def from_landsat_c2_sr(cls, sr_image, cloudmask_args={}, **kwargs): if 'snow_flag' not in cloudmask_args.keys(): cloudmask_args['snow_flag'] = True if 'cloud_score_flag' not in cloudmask_args.keys(): - cloudmask_args['cloud_score_flag'] = True + cloudmask_args['cloud_score_flag'] = False if 'cloud_score_pct' not in cloudmask_args.keys(): cloudmask_args['cloud_score_pct'] = 100 if 'filter_flag' not in cloudmask_args.keys(): - cloudmask_args['filter_flag'] = True + cloudmask_args['filter_flag'] = False if 'saturated_flag' not in cloudmask_args.keys(): - cloudmask_args['saturated_flag'] = True + cloudmask_args['saturated_flag'] = False cloud_mask = openet.core.common.landsat_c2_sr_cloud_mask(sr_image, **cloudmask_args) From cad9b1febb0f2fad2eb93580e7b58ac9bcfe4a31 Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Sat, 23 Mar 2024 11:52:34 -0700 Subject: [PATCH 4/4] Updated point image and point collection value tests to match openet-core changes --- openet/sims/tests/test_a_utils.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/openet/sims/tests/test_a_utils.py b/openet/sims/tests/test_a_utils.py index fe9d280..ff2a79a 100644 --- a/openet/sims/tests/test_a_utils.py +++ b/openet/sims/tests/test_a_utils.py @@ -42,29 +42,34 @@ def test_constant_image_value_multiband_bands(expected=10.123456789, tol=0.00000 @pytest.mark.parametrize( 'image_id, xy, scale, expected, tol', [ - ['USGS/NED', [-106.03249, 37.17777], 10, 2364.351, 0.001], - ['USGS/NED', [-106.03249, 37.17777], 1, 2364.351, 0.001], + ['USGS/3DEP/10m', [-106.03249, 37.17777], 30, 2364.169, 0.001], + ['USGS/3DEP/10m', [-106.03249, 37.17777], 10, 2364.138, 0.001], + ['USGS/3DEP/10m', [-106.03249, 37.17777], 1, 2364.138, 0.001], + ['NASA/NASADEM_HGT/001', [-106.03249, 37.17777], 30, 2361, 0.001], ] ) def test_point_image_value(image_id, xy, scale, expected, tol): - output = utils.point_image_value(ee.Image(image_id).rename('output'), xy) + output = utils.point_image_value( + ee.Image(image_id).select(['elevation'], ['output']), xy, scale + ) assert abs(output['output'] - expected) <= tol @pytest.mark.parametrize( 'image_id, image_date, xy, scale, expected, tol', [ - # CGM - This test stopped working for a scale of 1 and returns a different - # value for a scale of 10 than the point_image_value() function above. - # This function uses getRegion() instead of a reduceRegion() call, - # so there might have been some sort of change in getRegion(). - ['USGS/NED', '2012-04-04', [-106.03249, 37.17777], 10, 2364.286, 0.001], - # CGM - The default scale of 1 now returns None/Null for some reason - # ['USGS/NED', '2012-04-04', [-106.03249, 37.17777], 1, 2364.351, 0.001], + ['USGS/3DEP/10m', '2012-04-04', [-106.03249, 37.17777], 30, 2364.169, 0.001], + ['USGS/3DEP/10m', '2012-04-04', [-106.03249, 37.17777], 10, 2364.097, 0.001], + ['USGS/3DEP/10m', '2012-04-04', [-106.03249, 37.17777], 1, 2364.138, 0.001], + ['NASA/NASADEM_HGT/001', '2012-04-04', [-106.03249, 37.17777], 30, 2361, 0.001], ] ) def test_point_coll_value(image_id, image_date, xy, scale, expected, tol): - input_img = ee.Image(image_id).rename(['output']) + # The image must have a system:time_start for this function to work correctly + input_img = ( + ee.Image(image_id).select(['elevation'], ['output']) + .set({'system:time_start': ee.Date(image_date).millis()}) + ) output = utils.point_coll_value(ee.ImageCollection([input_img]), xy, scale) assert abs(output['output'][image_date] - expected) <= tol