Skip to content

Commit 42d1340

Browse files
wphyojplwphyojpl
authored
fix: Check bbox in Granules STAC (#88)
* feat: add AUDIT log level for upload * chore: update outdated tests * fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val * fix: check invalid bbocx when applicable
1 parent fedb70f commit 42d1340

File tree

3 files changed

+122
-14
lines changed

3 files changed

+122
-14
lines changed

‎mdps_ds_lib/lib/cumulus_stac/granules_catalog.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,22 @@ def get_child_link_hrefs(self, catalog_file_path: str, rel_name: str = 'item'):
5252
new_child_links.append(os.path.join(catalog_dir, each_link))
5353
return new_child_links
5454

55+
@staticmethod
56+
def is_bbox_valid(bbox_array):
57+
if len(bbox_array) != 4:
58+
return False
59+
lon_a, lat_a, lon_b, lat_b = bbox_array
60+
if lon_a == lon_b or lat_a == lat_b:
61+
return False
62+
return True
63+
5564
def get_granules_item(self, granule_stac_json) -> Item:
5665
if not FileUtils.file_exist(granule_stac_json):
5766
raise ValueError(f'missing file: {granule_stac_json}')
5867
granules_stac = FileUtils.read_json(granule_stac_json)
5968
granules_stac = Item.from_dict(granules_stac)
69+
if granules_stac.bbox is not None and not self.is_bbox_valid(granules_stac.bbox):
70+
raise ValueError(f'invalid BBOX. It may be a point or a line: {granules_stac.bbox}')
6071
return granules_stac
6172

6273
def extract_assets_href(self, granules_stac: Item, dir_name: str = '') -> dict:

‎mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def __exec_actual_job(self, each_child, lock) -> bool:
6161
"type": "Point",
6262
"coordinates": [0.0, 0.0]
6363
},
64-
bbox=[0.0, 0.0, 0.0, 0.0],
64+
bbox=[-180, -90, 180, 90],
6565
datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(),
6666
collection='unknown')
6767
self.__error_list.put(error_item.to_dict(False, False))

‎tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py‎

Lines changed: 110 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from unittest import TestCase
55

66
import pystac
7+
from pystac import Item
78

89
from mdps_ds_lib.lib.cumulus_stac.granules_catalog import GranulesCatalog
910
from mdps_ds_lib.lib.utils.file_utils import FileUtils
@@ -134,20 +135,32 @@ def test_get_granules_item(self):
134135
"bbox": [
135136
0.0,
136137
0.0,
137-
0.0,
138-
0.0
138+
10.0,
139+
10.0
139140
],
140141
"stac_extensions": [],
141142
"collection": "SNDR_SNPP_ATMS_L1A___1"
142143
}
144+
with tempfile.TemporaryDirectory() as tmp_dir_name:
145+
granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json')
146+
FileUtils.write_json(granules_catalog_path, sample_granules)
147+
pystac_catalog = GranulesCatalog().get_granules_item(granules_catalog_path)
148+
self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12')
149+
sample_granules['bbox'] = [10, 10, 0, 0]
150+
with tempfile.TemporaryDirectory() as tmp_dir_name:
151+
granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json')
152+
FileUtils.write_json(granules_catalog_path, sample_granules)
153+
pystac_catalog = GranulesCatalog().get_granules_item(granules_catalog_path)
154+
self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12')
155+
sample_granules['bbox'] = [0, 10, 10, 0]
143156
with tempfile.TemporaryDirectory() as tmp_dir_name:
144157
granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json')
145158
FileUtils.write_json(granules_catalog_path, sample_granules)
146159
pystac_catalog = GranulesCatalog().get_granules_item(granules_catalog_path)
147160
self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12')
148161
return
149162

150-
def test_extract_assets_href(self):
163+
def test_get_granules_item_invalid_bbox(self):
151164
sample_granules = {
152165
"type": "Feature",
153166
"stac_version": "1.0.0",
@@ -179,6 +192,12 @@ def test_extract_assets_href(self):
179192
"description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc",
180193
"roles": ["data"],
181194
},
195+
"SNDR.SNPP.ATMS.L1A.nominal2.12.1.nc": {
196+
"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.1.nc",
197+
"title": "SNDR.SNPP.ATMS.L1A.nominal2.12.1.nc",
198+
"description": "SNDR.SNPP.ATMS.L1A.nominal2.12.1.nc",
199+
"roles": ["data"],
200+
},
182201
"SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas": {
183202
"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas",
184203
"title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas",
@@ -201,6 +220,82 @@ def test_extract_assets_href(self):
201220
"stac_extensions": [],
202221
"collection": "SNDR_SNPP_ATMS_L1A___1"
203222
}
223+
with tempfile.TemporaryDirectory() as tmp_dir_name:
224+
granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json')
225+
FileUtils.write_json(granules_catalog_path, sample_granules)
226+
with self.assertRaises(ValueError) as context:
227+
GranulesCatalog().get_granules_item(granules_catalog_path)
228+
self.assertTrue(str(context.exception).startswith('invalid BBOX. It may be a point or a line'))
229+
sample_granules['bbox'] = [0, 1, 0, 1]
230+
with tempfile.TemporaryDirectory() as tmp_dir_name:
231+
granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json')
232+
FileUtils.write_json(granules_catalog_path, sample_granules)
233+
with self.assertRaises(ValueError) as context:
234+
GranulesCatalog().get_granules_item(granules_catalog_path)
235+
self.assertTrue(str(context.exception).startswith('invalid BBOX. It may be a point or a line'))
236+
sample_granules['bbox'] = [1, 0, 1, 0]
237+
with tempfile.TemporaryDirectory() as tmp_dir_name:
238+
granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json')
239+
FileUtils.write_json(granules_catalog_path, sample_granules)
240+
with self.assertRaises(ValueError) as context:
241+
GranulesCatalog().get_granules_item(granules_catalog_path)
242+
self.assertTrue(str(context.exception).startswith('invalid BBOX. It may be a point or a line'))
243+
return
244+
245+
def test_extract_assets_href(self):
246+
sample_granules = {
247+
"type": "Feature",
248+
"stac_version": "1.0.0",
249+
"id": "SNDR.SNPP.ATMS.L1A.nominal2.12",
250+
"properties": {
251+
"start_datetime": "2016-01-14T11:00:00Z",
252+
"end_datetime": "2016-01-14T11:06:00Z",
253+
"created": "2020-12-14T13:50:00Z",
254+
"updated": "2022-08-15T06:26:25.344000Z",
255+
"datetime": "2022-08-15T06:26:17.938000Z"
256+
},
257+
"geometry": {
258+
"type": "Point",
259+
"coordinates": [
260+
0.0,
261+
0.0
262+
]
263+
},
264+
"links": [
265+
{
266+
"rel": "collection",
267+
"href": "."
268+
}
269+
],
270+
"assets": {
271+
"SNDR.SNPP.ATMS.L1A.nominal2.12.nc": {
272+
"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc",
273+
"title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc",
274+
"description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc",
275+
"roles": ["data"],
276+
},
277+
"SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas": {
278+
"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas",
279+
"title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas",
280+
"description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas",
281+
"roles": ["metadata__data"],
282+
},
283+
"SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml": {
284+
"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml",
285+
"title": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml",
286+
"description": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml",
287+
"roles": ["metadata__cmr"],
288+
}
289+
},
290+
"bbox": [
291+
0.0,
292+
0.0,
293+
10.0,
294+
10.0
295+
],
296+
"stac_extensions": [],
297+
"collection": "SNDR_SNPP_ATMS_L1A___1"
298+
}
204299
with tempfile.TemporaryDirectory() as tmp_dir_name:
205300
granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json')
206301
FileUtils.write_json(granules_catalog_path, sample_granules)
@@ -263,8 +358,8 @@ def test_extract_assets_relative_href_01(self):
263358
"bbox": [
264359
0.0,
265360
0.0,
266-
0.0,
267-
0.0
361+
10.0,
362+
10.0
268363
],
269364
"stac_extensions": [],
270365
"collection": "SNDR_SNPP_ATMS_L1A___1"
@@ -337,8 +432,8 @@ def test_extract_assets_relative_href_02(self):
337432
"bbox": [
338433
0.0,
339434
0.0,
340-
0.0,
341-
0.0
435+
10.0,
436+
10.0
342437
],
343438
"stac_extensions": [],
344439
"collection": "SNDR_SNPP_ATMS_L1A___1"
@@ -416,8 +511,8 @@ def test_extract_assets_relative_href_03(self):
416511
"bbox": [
417512
0.0,
418513
0.0,
419-
0.0,
420-
0.0
514+
10.0,
515+
10.0
421516
],
422517
"stac_extensions": [],
423518
"collection": "SNDR_SNPP_ATMS_L1A___1"
@@ -485,8 +580,8 @@ def test_update_assets_href(self):
485580
"bbox": [
486581
0.0,
487582
0.0,
488-
0.0,
489-
0.0
583+
10.0,
584+
10.0
490585
],
491586
"stac_extensions": [],
492587
"collection": "SNDR_SNPP_ATMS_L1A___1"
@@ -570,8 +665,8 @@ def test_update_assets_href_02(self):
570665
"bbox": [
571666
0.0,
572667
0.0,
573-
0.0,
574-
0.0
668+
10.0,
669+
10.0
575670
],
576671
"stac_extensions": [],
577672
"collection": "SNDR_SNPP_ATMS_L1A___1"
@@ -721,6 +816,8 @@ def test_manual_validdate_stac(self):
721816
"collection": "URN:NASA:UNITY:UDS_BLACK:DEV:UDS_UNIT_COLLECTION___2410010608"
722817
}'''
723818
raw_json = json.loads(raw_json)
819+
granules_stac = Item.from_dict(raw_json)
820+
print(granules_stac.bbox)
724821
schema1 = {
725822
"$schema": "http://json-schema.org/draft-07/schema#",
726823
"$id": "https://stac-extensions.github.io/file/v2.1.0/schema.json#",

0 commit comments

Comments
 (0)