From e0276794cc120a5b6d5973eaca8a6e2075f6eb09 Mon Sep 17 00:00:00 2001 From: R Date: Tue, 8 Oct 2024 11:54:38 -0400 Subject: [PATCH 1/9] Interim commit --- view/justSrl.py | 37 ++++++++ view/justSrlPlus.py | 90 +++++++++++++++++++ view/vhrToolkit.py | 209 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 336 insertions(+) create mode 100755 view/justSrl.py create mode 100755 view/justSrlPlus.py create mode 100755 view/vhrToolkit.py diff --git a/view/justSrl.py b/view/justSrl.py new file mode 100755 index 0000000..2e8c2a6 --- /dev/null +++ b/view/justSrl.py @@ -0,0 +1,37 @@ +#!/usr/bin/python + +from pathlib import Path +import sys + +from srlite.model.SrliteWorkflow import SrliteWorkflow + + +# ----------------------------------------------------------------------------- +# main +# +# vhr-toolkit/view/justSrl.py +# ----------------------------------------------------------------------------- +def main(): + + srlDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/8-srl') + + toaDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/5-toas') + + ccdcDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/7-ccdc') + + cMaskDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/6-masks') + + srl = SrliteWorkflow(output_dir=srlDir, + toa_src=toaDir, + target_dir=ccdcDir, + cloudmask_dir=cMaskDir) + +# ----------------------------------------------------------------------------- +# Invoke the main +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + sys.exit(main()) diff --git a/view/justSrlPlus.py b/view/justSrlPlus.py new file mode 100755 index 0000000..158f1ad --- /dev/null +++ b/view/justSrlPlus.py @@ -0,0 +1,90 @@ +#!/usr/bin/python + +import csv +import logging +from pathlib import Path +import sys + +from core.model.DgFile import DgFile +from evhr.model.EvhrToA import EvhrToA +from pyCCDC.model.CCDCPipeline import CCDCPipeline +from srlite.model.SrliteWorkflow import SrliteWorkflow + +from vhr_cloudmask.model.pipelines.cloudmask_cnn_pipeline import \ + CloudMaskPipeline + + +# ----------------------------------------------------------------------------- +# main +# +# vhr-toolkit/view/justSrlPlus.py +# ----------------------------------------------------------------------------- +def main(): + + outDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK') + + srlDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/8-srl') + + toaDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/5-toas') + + ccdcDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/7-ccdc') + + cMaskDir = \ + Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/6-masks') + + scenesFile = Path('/explore/nobackup/projects/ilab/projects/vhr-toolkit/' + + 'testing/evhr/toa_clv_test_alaska_cloud.csv') + + # Scenes + with open(scenesFile, newline='') as csvFile: + + reader = csv.reader(csvFile) + scenes = [Path(scene[0]) for scene in reader] + + dgScenes = [] + + for s in scenes: + + if not s.exists(): + logger.warning('Scene, ' + str(s) + ' does not exist.') + + else: + dgScenes.append(DgFile(str(s))) + + # --- + # Logging + # --- + logger = logging.getLogger() + logger.setLevel(logging.INFO) + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + logger.addHandler(ch) + + # EVHR + toa = EvhrToA(outDir) + toa.run(dgScenes) + + # Cloud Mask + toaRegex = [str(toaDir / '*-toa.tif')] + cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) + cmpl.predict() + + # CCDC + ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) + ccdc.run() + + # SRL + srl = SrliteWorkflow(output_dir=srlDir, + toa_src=toaDir, + target_dir=ccdcDir, + cloudmask_dir=cMaskDir) + +# ----------------------------------------------------------------------------- +# Invoke the main +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + sys.exit(main()) diff --git a/view/vhrToolkit.py b/view/vhrToolkit.py new file mode 100755 index 0000000..6f247cb --- /dev/null +++ b/view/vhrToolkit.py @@ -0,0 +1,209 @@ +#!/usr/bin/python + +import argparse +import csv +import logging +from pathlib import Path +import sys + +from core.model.DgFile import DgFile + +from evhr.model.EvhrToA import EvhrToA +from pyCCDC.model.CCDCPipeline import CCDCPipeline +from srlite.model.SrliteWorkflow import SrliteWorkflow + +from vhr_cloudmask.model.pipelines.cloudmask_cnn_pipeline import \ + CloudMaskPipeline + + +# ----------------------------------------------------------------------------- +# main +# +# vhr-toolkit/view/vhrToolkit.py -o /explore/nobackup/people/rlgill/SystemTesting/testILTK --scenes_in_file /explore/nobackup/projects/ilab/projects/vhr-toolkit/testing/evhr/toa_clv_test_alaska_cloud.csv +# +# TODO: EvhrToA needs an accessor for _outDir, _toaDir. +# TODO: EvhrToA.__init__() outDir s/b of type Path. +# TODO: EvhrToA.run() return list of ToAs produced? +# ----------------------------------------------------------------------------- +def main(): + + # --- + # Parse input arguments. + # --- + args = parseArgs() + + # --- + # Logging + # --- + logger = logging.getLogger() + logger.setLevel(logging.INFO) + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + logger.addHandler(ch) + + # --- + # Make DgFiles from the input scenes. + # --- + scenes = args.scenes + + if args.scenes_in_file: + + with open(args.scenes_in_file, newline='') as csvFile: + + reader = csv.reader(csvFile) + scenes = [Path(scene[0]) for scene in reader] + + dgScenes = scenesToDgFiles(scenes, logger) + + # --- + # EVHR + # --- + logger.info('Running EVHR.') + toa = EvhrToA(args.o, args.dem, args.pan_res, args.pan_sharpen, logger) + toa.run(dgScenes) + toaDir = Path(toa._toaDir) + toaDirNum = int(toaDir.name.split('-')[0]) + toas = toaDir.glob('*-toa.tif') + + # --- + # EVHR -> CloudMaskPipeline + # --- + logger.info('Running CloudMaskPipeline.') + cMaskDirNum = toaDirNum + 1 + cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks') + cMaskDir.mkdir(exist_ok=True) + toaRegex = [str(toaDir / '*-toa.tif')] + cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) + cmpl.predict() + + # --- + # EHVR -> CCDC + # --- + logger.info('Running CCDC.') + ccdcDirNum = cMaskDirNum + 1 + ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') + ccdcDir.mkdir(exist_ok=True) + expCcdc = [ccdcDir / f.name.replace('-toa', '-toa_ccdc') for f in toas] + + if not all([f.exists() for f in expCcdc]): + + ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) + ccdc.run() + + # --- + # EVHR + Cloud Mask + CCDC -> SRL + # --- + logger.info('Running SR Lite.') + srlDirNum = ccdcDirNum + 1 + srlDir = Path(args.o) / (str(srlDirNum) + '-srl') + srlDir.mkdir(exist_ok=True) + + import pdb + pdb.set_trace() + + srl = SrliteWorkflow(output_dir=srlDir, + toa_src=toaDir, + target_dir=ccdcDir, + cloudmask_dir=cMaskDir) #, + # regressor='rma', + # debug=1, + # pmask='True', + # cloudmask='True', + # csv='True', + # band8='True', + # clean='True', + # # cloudmask_suffix=cloudmask_suffix, + # target_suffix='-ccdc.tif', + # logger=logger) + + # srl.processToas() + +# ----------------------------------------------------------------------------- +# parseArgs +# ----------------------------------------------------------------------------- +def parseArgs() -> argparse.Namespace: + + desc = 'Use this application to run the VHR Toolkit.' + parser = argparse.ArgumentParser(description=desc) + + # --- + # Universal Parameters + # --- + parser.add_argument('-o', + default='.', + help='Path to output directory') + + # --- + # EVHR Parameters + # --- + parser.add_argument('--dem', + type=Path, + required=False, + help='Fully-qualified path to DEM footprints shape ' + ' file.') + + parser.add_argument('--pan_res', + type=float, + default=1, + choices=[0.5, 1], + help='The resolution, in meters, of panchromatic ' + 'output images') + + parser.add_argument('--pan_sharpen', + action='store_true', + help='Apply panchromatic sharpening to the output ' + 'ToA images.') + + group = parser.add_mutually_exclusive_group(required=True) + + group.add_argument('--scenes', + type=Path, + nargs='*', + help='Fully-qualified path to scene files') + + group.add_argument('--scenes_in_file', + type=Path, + help='Fully-qualified path to CSV file containing a ' + 'list of scene files') + + # --- + # VHR Cloud Mask Parameters + # --- + + # --- + # CCDC Parameters + # --- + + # --- + # SR-lite Parameters + # --- + + # --- + # Parse + # --- + args = parser.parse_args() + + return args + +# ----------------------------------------------------------------------------- +# scenesToDgFiles +# ----------------------------------------------------------------------------- +def scenesToDgFiles(scenes: list, logger: logging.RootLogger) -> list: + + dgScenes = [] + + for s in scenes: + + if not s.exists(): + logger.warning('Scene, ' + str(s) + ' does not exist.') + + else: + dgScenes.append(DgFile(str(s))) + + return dgScenes + +# ----------------------------------------------------------------------------- +# Invoke the main +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + sys.exit(main()) From a254e4571e58ca362203454d7e8e2991f5eddee1 Mon Sep 17 00:00:00 2001 From: gtamkin Date: Wed, 9 Oct 2024 16:13:23 -0400 Subject: [PATCH 2/9] Glenn's attempt (to be discarded eventually) --- view/vhrToolkitGT.py | 210 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100755 view/vhrToolkitGT.py diff --git a/view/vhrToolkitGT.py b/view/vhrToolkitGT.py new file mode 100755 index 0000000..113869c --- /dev/null +++ b/view/vhrToolkitGT.py @@ -0,0 +1,210 @@ +#!/usr/bin/python + +import argparse +import csv +import logging +from pathlib import Path +import sys + +from core.model.DgFile import DgFile + +from evhr.model.EvhrToA import EvhrToA +# from pyCCDC.model.CCDCPipeline import CCDCPipeline +from srlite.model.SrliteWorkflow import SrliteWorkflow + +# from vhr_cloudmask.model.pipelines.cloudmask_cnn_pipeline import \ +# CloudMaskPipeline + + +# ----------------------------------------------------------------------------- +# main +# +# vhr-toolkit/view/vhrToolkit.py -o /explore/nobackup/people/rlgill/SystemTesting/testILTK --scenes_in_file /explore/nobackup/projects/ilab/projects/vhr-toolkit/testing/evhr/toa_clv_test_alaska_cloud.csv +# +# TODO: EvhrToA needs an accessor for _outDir, _toaDir. +# TODO: EvhrToA.__init__() outDir s/b of type Path. +# TODO: EvhrToA.run() return list of ToAs produced? +# ----------------------------------------------------------------------------- +def main(): + + # --- + # Parse input arguments. + # --- + args = parseArgs() + + # --- + # Logging + # --- + logger = logging.getLogger() + logger.setLevel(logging.INFO) + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + logger.addHandler(ch) + + # --- + # Make DgFiles from the input scenes. + # --- + scenes = args.scenes + + if args.scenes_in_file: + + with open(args.scenes_in_file, newline='') as csvFile: + + reader = csv.reader(csvFile) + scenes = [Path(scene[0]) for scene in reader] + + dgScenes = scenesToDgFiles(scenes, logger) + + # --- + # EVHR + # --- + logger.info('Running EVHR.') + toa = EvhrToA(args.o, args.dem, args.pan_res, args.pan_sharpen, logger) + toa.run(dgScenes) + toaDir = Path(toa._toaDir) + toaDirNum = int(toaDir.name.split('-')[0]) + toas = toaDir.glob('*-toa.tif') + + # --- + # EVHR -> CloudMaskPipeline + # --- + logger.info('Running CloudMaskPipeline.') + cMaskDirNum = toaDirNum + 1 + cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks') + cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks/5-toas') + cMaskDir.mkdir(exist_ok=True) + toaRegex = [str(toaDir / '*-toa.tif')] + # cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) + # cmpl.predict() + + # --- + # EHVR -> CCDC + # --- + logger.info('Running CCDC.') + ccdcDirNum = cMaskDirNum + 1 + ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') + ccdcDir.mkdir(exist_ok=True) + expCcdc = [ccdcDir / f.name.replace('-toa', '-toa_ccdc') for f in toas] + + # if not all([f.exists() for f in expCcdc]): + + # ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) + # ccdc.run() + + # --- + # EVHR + Cloud Mask + CCDC -> SRL + # --- + logger.info('Running SR Lite.') + srlDirNum = ccdcDirNum + 1 + srlDir = Path(args.o) / (str(srlDirNum) + '-srl') + srlDir.mkdir(exist_ok=True) + + # import pdb + # pdb.set_trace() + srl = SrliteWorkflow(output_dir=srlDir, + toa_src=toaDir, + target_dir=ccdcDir, + cloudmask_dir=cMaskDir, + regressor='rma', + debug=1, + pmask='True', + cloudmask='True', + csv='True', + band8='True', + clean='True', + toa_suffix='-toa.tif', + target_suffix='-toa_ccdc.tif', + cloudmask_suffix='-toa.cloudmask.tif', + logger=logger) + + srl.processToas() + +# ----------------------------------------------------------------------------- +# parseArgs +# ----------------------------------------------------------------------------- +def parseArgs() -> argparse.Namespace: + + desc = 'Use this application to run the VHR Toolkit.' + parser = argparse.ArgumentParser(description=desc) + + # --- + # Universal Parameters + # --- + parser.add_argument('-o', + default='.', + help='Path to output directory') + + # --- + # EVHR Parameters + # --- + parser.add_argument('--dem', + type=Path, + required=False, + help='Fully-qualified path to DEM footprints shape ' + ' file.') + + parser.add_argument('--pan_res', + type=float, + default=1, + choices=[0.5, 1], + help='The resolution, in meters, of panchromatic ' + 'output images') + + parser.add_argument('--pan_sharpen', + action='store_true', + help='Apply panchromatic sharpening to the output ' + 'ToA images.') + + group = parser.add_mutually_exclusive_group(required=True) + + group.add_argument('--scenes', + type=Path, + nargs='*', + help='Fully-qualified path to scene files') + + group.add_argument('--scenes_in_file', + type=Path, + help='Fully-qualified path to CSV file containing a ' + 'list of scene files') + + # --- + # VHR Cloud Mask Parameters + # --- + + # --- + # CCDC Parameters + # --- + + # --- + # SR-lite Parameters + # --- + + # --- + # Parse + # --- + args = parser.parse_args() + + return args + +# ----------------------------------------------------------------------------- +# scenesToDgFiles +# ----------------------------------------------------------------------------- +def scenesToDgFiles(scenes: list, logger: logging.RootLogger) -> list: + + dgScenes = [] + + for s in scenes: + + if not s.exists(): + logger.warning('Scene, ' + str(s) + ' does not exist.') + + else: + dgScenes.append(DgFile(str(s))) + + return dgScenes + +# ----------------------------------------------------------------------------- +# Invoke the main +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + sys.exit(main()) From 0f5d3a6e027d00f1067b26268a71d0973441542c Mon Sep 17 00:00:00 2001 From: gtamkin Date: Thu, 10 Oct 2024 10:42:19 -0400 Subject: [PATCH 3/9] reprojected CCDC --- view/vhrToolkitGT.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/view/vhrToolkitGT.py b/view/vhrToolkitGT.py index 113869c..91ff483 100755 --- a/view/vhrToolkitGT.py +++ b/view/vhrToolkitGT.py @@ -82,7 +82,9 @@ def main(): # --- logger.info('Running CCDC.') ccdcDirNum = cMaskDirNum + 1 - ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') + # Toggle these two to cause 'Invalid Coordinate' error (Mel reprojected the original) + ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc/reprojected') + # ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') ccdcDir.mkdir(exist_ok=True) expCcdc = [ccdcDir / f.name.replace('-toa', '-toa_ccdc') for f in toas] From 38ea18d430e3a95f592d2b3d7e3ca4f93ddd5bd4 Mon Sep 17 00:00:00 2001 From: gtamkin Date: Thu, 10 Oct 2024 10:47:15 -0400 Subject: [PATCH 4/9] turned off auto-clean --- view/vhrToolkitGT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/view/vhrToolkitGT.py b/view/vhrToolkitGT.py index 91ff483..94f89ec 100755 --- a/view/vhrToolkitGT.py +++ b/view/vhrToolkitGT.py @@ -113,7 +113,7 @@ def main(): cloudmask='True', csv='True', band8='True', - clean='True', + clean='False', toa_suffix='-toa.tif', target_suffix='-toa_ccdc.tif', cloudmask_suffix='-toa.cloudmask.tif', From 5d89029ac9f71e834988a174c5b40a4fe45f6ad7 Mon Sep 17 00:00:00 2001 From: gtamkin Date: Thu, 10 Oct 2024 11:12:08 -0400 Subject: [PATCH 5/9] turned off EVHR --- .vscode/launch.json | 22 ++++++++++++++++++++++ view/vhrToolkitGT.py | 12 ++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..320bffb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "vhr-toolkit/view/vhrToolkit.py", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-o", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#27_Add_API_class/20241010-csv-band8-default-bandpairs", + "--scenes_in_file", + "/explore/nobackup/projects/ilab/projects/vhr-toolkit/testing/evhr/toa_clv_test_alaska_cloud.csv", + ] + } + ] +} \ No newline at end of file diff --git a/view/vhrToolkitGT.py b/view/vhrToolkitGT.py index 94f89ec..8fa5000 100755 --- a/view/vhrToolkitGT.py +++ b/view/vhrToolkitGT.py @@ -46,21 +46,21 @@ def main(): # --- scenes = args.scenes - if args.scenes_in_file: + # if args.scenes_in_file: - with open(args.scenes_in_file, newline='') as csvFile: + # with open(args.scenes_in_file, newline='') as csvFile: - reader = csv.reader(csvFile) - scenes = [Path(scene[0]) for scene in reader] + # reader = csv.reader(csvFile) + # scenes = [Path(scene[0]) for scene in reader] - dgScenes = scenesToDgFiles(scenes, logger) + # dgScenes = scenesToDgFiles(scenes, logger) # --- # EVHR # --- logger.info('Running EVHR.') toa = EvhrToA(args.o, args.dem, args.pan_res, args.pan_sharpen, logger) - toa.run(dgScenes) + # toa.run(dgScenes) toaDir = Path(toa._toaDir) toaDirNum = int(toaDir.name.split('-')[0]) toas = toaDir.glob('*-toa.tif') From ed2008a755b14fc57499f4738076e188f5541b16 Mon Sep 17 00:00:00 2001 From: gtamkin Date: Fri, 18 Oct 2024 15:35:34 -0400 Subject: [PATCH 6/9] first end-to-end test --- view/vhrToolkitGT.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/view/vhrToolkitGT.py b/view/vhrToolkitGT.py index 8fa5000..3d13f0e 100755 --- a/view/vhrToolkitGT.py +++ b/view/vhrToolkitGT.py @@ -9,11 +9,11 @@ from core.model.DgFile import DgFile from evhr.model.EvhrToA import EvhrToA -# from pyCCDC.model.CCDCPipeline import CCDCPipeline +from pyCCDC.model.CCDCPipeline import CCDCPipeline from srlite.model.SrliteWorkflow import SrliteWorkflow -# from vhr_cloudmask.model.pipelines.cloudmask_cnn_pipeline import \ -# CloudMaskPipeline +from vhr_cloudmask.model.pipelines.cloudmask_cnn_pipeline import \ + CloudMaskPipeline # ----------------------------------------------------------------------------- @@ -46,21 +46,21 @@ def main(): # --- scenes = args.scenes - # if args.scenes_in_file: + if args.scenes_in_file: - # with open(args.scenes_in_file, newline='') as csvFile: + with open(args.scenes_in_file, newline='') as csvFile: - # reader = csv.reader(csvFile) - # scenes = [Path(scene[0]) for scene in reader] + reader = csv.reader(csvFile) + scenes = [Path(scene[0]) for scene in reader] - # dgScenes = scenesToDgFiles(scenes, logger) + dgScenes = scenesToDgFiles(scenes, logger) # --- # EVHR # --- logger.info('Running EVHR.') toa = EvhrToA(args.o, args.dem, args.pan_res, args.pan_sharpen, logger) - # toa.run(dgScenes) + toa.run(dgScenes) toaDir = Path(toa._toaDir) toaDirNum = int(toaDir.name.split('-')[0]) toas = toaDir.glob('*-toa.tif') @@ -71,11 +71,11 @@ def main(): logger.info('Running CloudMaskPipeline.') cMaskDirNum = toaDirNum + 1 cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks') - cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks/5-toas') +# cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks/5-toas') cMaskDir.mkdir(exist_ok=True) toaRegex = [str(toaDir / '*-toa.tif')] - # cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) - # cmpl.predict() + cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) + cmpl.predict() # --- # EHVR -> CCDC @@ -83,15 +83,15 @@ def main(): logger.info('Running CCDC.') ccdcDirNum = cMaskDirNum + 1 # Toggle these two to cause 'Invalid Coordinate' error (Mel reprojected the original) - ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc/reprojected') - # ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') + # ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc/reprojected') + ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') ccdcDir.mkdir(exist_ok=True) expCcdc = [ccdcDir / f.name.replace('-toa', '-toa_ccdc') for f in toas] - # if not all([f.exists() for f in expCcdc]): + if not all([f.exists() for f in expCcdc]): - # ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) - # ccdc.run() + ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) + ccdc.run() # --- # EVHR + Cloud Mask + CCDC -> SRL @@ -106,7 +106,7 @@ def main(): srl = SrliteWorkflow(output_dir=srlDir, toa_src=toaDir, target_dir=ccdcDir, - cloudmask_dir=cMaskDir, + cloudmask_dir=Path(str(cMaskDir) + '/5-toas'), regressor='rma', debug=1, pmask='True', From 2e5a1110bec943e5e821a05f36dc12b7475d5f94 Mon Sep 17 00:00:00 2001 From: R Date: Thu, 24 Oct 2024 09:24:06 -0400 Subject: [PATCH 7/9] Misc --- view/justSrl.py | 37 -------- view/justSrlPlus.py | 90 ------------------ view/vhrToolkitGT.py | 212 ------------------------------------------- 3 files changed, 339 deletions(-) delete mode 100755 view/justSrl.py delete mode 100755 view/justSrlPlus.py delete mode 100755 view/vhrToolkitGT.py diff --git a/view/justSrl.py b/view/justSrl.py deleted file mode 100755 index 2e8c2a6..0000000 --- a/view/justSrl.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/python - -from pathlib import Path -import sys - -from srlite.model.SrliteWorkflow import SrliteWorkflow - - -# ----------------------------------------------------------------------------- -# main -# -# vhr-toolkit/view/justSrl.py -# ----------------------------------------------------------------------------- -def main(): - - srlDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/8-srl') - - toaDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/5-toas') - - ccdcDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/7-ccdc') - - cMaskDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/6-masks') - - srl = SrliteWorkflow(output_dir=srlDir, - toa_src=toaDir, - target_dir=ccdcDir, - cloudmask_dir=cMaskDir) - -# ----------------------------------------------------------------------------- -# Invoke the main -# ----------------------------------------------------------------------------- -if __name__ == "__main__": - sys.exit(main()) diff --git a/view/justSrlPlus.py b/view/justSrlPlus.py deleted file mode 100755 index 158f1ad..0000000 --- a/view/justSrlPlus.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/python - -import csv -import logging -from pathlib import Path -import sys - -from core.model.DgFile import DgFile -from evhr.model.EvhrToA import EvhrToA -from pyCCDC.model.CCDCPipeline import CCDCPipeline -from srlite.model.SrliteWorkflow import SrliteWorkflow - -from vhr_cloudmask.model.pipelines.cloudmask_cnn_pipeline import \ - CloudMaskPipeline - - -# ----------------------------------------------------------------------------- -# main -# -# vhr-toolkit/view/justSrlPlus.py -# ----------------------------------------------------------------------------- -def main(): - - outDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK') - - srlDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/8-srl') - - toaDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/5-toas') - - ccdcDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/7-ccdc') - - cMaskDir = \ - Path('/explore/nobackup/people/rlgill/SystemTesting/testILTK/6-masks') - - scenesFile = Path('/explore/nobackup/projects/ilab/projects/vhr-toolkit/' + - 'testing/evhr/toa_clv_test_alaska_cloud.csv') - - # Scenes - with open(scenesFile, newline='') as csvFile: - - reader = csv.reader(csvFile) - scenes = [Path(scene[0]) for scene in reader] - - dgScenes = [] - - for s in scenes: - - if not s.exists(): - logger.warning('Scene, ' + str(s) + ' does not exist.') - - else: - dgScenes.append(DgFile(str(s))) - - # --- - # Logging - # --- - logger = logging.getLogger() - logger.setLevel(logging.INFO) - ch = logging.StreamHandler(sys.stdout) - ch.setLevel(logging.INFO) - logger.addHandler(ch) - - # EVHR - toa = EvhrToA(outDir) - toa.run(dgScenes) - - # Cloud Mask - toaRegex = [str(toaDir / '*-toa.tif')] - cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) - cmpl.predict() - - # CCDC - ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) - ccdc.run() - - # SRL - srl = SrliteWorkflow(output_dir=srlDir, - toa_src=toaDir, - target_dir=ccdcDir, - cloudmask_dir=cMaskDir) - -# ----------------------------------------------------------------------------- -# Invoke the main -# ----------------------------------------------------------------------------- -if __name__ == "__main__": - sys.exit(main()) diff --git a/view/vhrToolkitGT.py b/view/vhrToolkitGT.py deleted file mode 100755 index 3d13f0e..0000000 --- a/view/vhrToolkitGT.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/python - -import argparse -import csv -import logging -from pathlib import Path -import sys - -from core.model.DgFile import DgFile - -from evhr.model.EvhrToA import EvhrToA -from pyCCDC.model.CCDCPipeline import CCDCPipeline -from srlite.model.SrliteWorkflow import SrliteWorkflow - -from vhr_cloudmask.model.pipelines.cloudmask_cnn_pipeline import \ - CloudMaskPipeline - - -# ----------------------------------------------------------------------------- -# main -# -# vhr-toolkit/view/vhrToolkit.py -o /explore/nobackup/people/rlgill/SystemTesting/testILTK --scenes_in_file /explore/nobackup/projects/ilab/projects/vhr-toolkit/testing/evhr/toa_clv_test_alaska_cloud.csv -# -# TODO: EvhrToA needs an accessor for _outDir, _toaDir. -# TODO: EvhrToA.__init__() outDir s/b of type Path. -# TODO: EvhrToA.run() return list of ToAs produced? -# ----------------------------------------------------------------------------- -def main(): - - # --- - # Parse input arguments. - # --- - args = parseArgs() - - # --- - # Logging - # --- - logger = logging.getLogger() - logger.setLevel(logging.INFO) - ch = logging.StreamHandler(sys.stdout) - ch.setLevel(logging.INFO) - logger.addHandler(ch) - - # --- - # Make DgFiles from the input scenes. - # --- - scenes = args.scenes - - if args.scenes_in_file: - - with open(args.scenes_in_file, newline='') as csvFile: - - reader = csv.reader(csvFile) - scenes = [Path(scene[0]) for scene in reader] - - dgScenes = scenesToDgFiles(scenes, logger) - - # --- - # EVHR - # --- - logger.info('Running EVHR.') - toa = EvhrToA(args.o, args.dem, args.pan_res, args.pan_sharpen, logger) - toa.run(dgScenes) - toaDir = Path(toa._toaDir) - toaDirNum = int(toaDir.name.split('-')[0]) - toas = toaDir.glob('*-toa.tif') - - # --- - # EVHR -> CloudMaskPipeline - # --- - logger.info('Running CloudMaskPipeline.') - cMaskDirNum = toaDirNum + 1 - cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks') -# cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks/5-toas') - cMaskDir.mkdir(exist_ok=True) - toaRegex = [str(toaDir / '*-toa.tif')] - cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) - cmpl.predict() - - # --- - # EHVR -> CCDC - # --- - logger.info('Running CCDC.') - ccdcDirNum = cMaskDirNum + 1 - # Toggle these two to cause 'Invalid Coordinate' error (Mel reprojected the original) - # ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc/reprojected') - ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') - ccdcDir.mkdir(exist_ok=True) - expCcdc = [ccdcDir / f.name.replace('-toa', '-toa_ccdc') for f in toas] - - if not all([f.exists() for f in expCcdc]): - - ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) - ccdc.run() - - # --- - # EVHR + Cloud Mask + CCDC -> SRL - # --- - logger.info('Running SR Lite.') - srlDirNum = ccdcDirNum + 1 - srlDir = Path(args.o) / (str(srlDirNum) + '-srl') - srlDir.mkdir(exist_ok=True) - - # import pdb - # pdb.set_trace() - srl = SrliteWorkflow(output_dir=srlDir, - toa_src=toaDir, - target_dir=ccdcDir, - cloudmask_dir=Path(str(cMaskDir) + '/5-toas'), - regressor='rma', - debug=1, - pmask='True', - cloudmask='True', - csv='True', - band8='True', - clean='False', - toa_suffix='-toa.tif', - target_suffix='-toa_ccdc.tif', - cloudmask_suffix='-toa.cloudmask.tif', - logger=logger) - - srl.processToas() - -# ----------------------------------------------------------------------------- -# parseArgs -# ----------------------------------------------------------------------------- -def parseArgs() -> argparse.Namespace: - - desc = 'Use this application to run the VHR Toolkit.' - parser = argparse.ArgumentParser(description=desc) - - # --- - # Universal Parameters - # --- - parser.add_argument('-o', - default='.', - help='Path to output directory') - - # --- - # EVHR Parameters - # --- - parser.add_argument('--dem', - type=Path, - required=False, - help='Fully-qualified path to DEM footprints shape ' - ' file.') - - parser.add_argument('--pan_res', - type=float, - default=1, - choices=[0.5, 1], - help='The resolution, in meters, of panchromatic ' - 'output images') - - parser.add_argument('--pan_sharpen', - action='store_true', - help='Apply panchromatic sharpening to the output ' - 'ToA images.') - - group = parser.add_mutually_exclusive_group(required=True) - - group.add_argument('--scenes', - type=Path, - nargs='*', - help='Fully-qualified path to scene files') - - group.add_argument('--scenes_in_file', - type=Path, - help='Fully-qualified path to CSV file containing a ' - 'list of scene files') - - # --- - # VHR Cloud Mask Parameters - # --- - - # --- - # CCDC Parameters - # --- - - # --- - # SR-lite Parameters - # --- - - # --- - # Parse - # --- - args = parser.parse_args() - - return args - -# ----------------------------------------------------------------------------- -# scenesToDgFiles -# ----------------------------------------------------------------------------- -def scenesToDgFiles(scenes: list, logger: logging.RootLogger) -> list: - - dgScenes = [] - - for s in scenes: - - if not s.exists(): - logger.warning('Scene, ' + str(s) + ' does not exist.') - - else: - dgScenes.append(DgFile(str(s))) - - return dgScenes - -# ----------------------------------------------------------------------------- -# Invoke the main -# ----------------------------------------------------------------------------- -if __name__ == "__main__": - sys.exit(main()) From 0d12d79973eee107637d080e9efccdc2876ea65f Mon Sep 17 00:00:00 2001 From: R Date: Thu, 24 Oct 2024 14:32:39 -0400 Subject: [PATCH 8/9] Misc --- view/vhrToolkit.py | 58 ++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/view/vhrToolkit.py b/view/vhrToolkit.py index 6f247cb..f6541de 100755 --- a/view/vhrToolkit.py +++ b/view/vhrToolkit.py @@ -26,7 +26,7 @@ # TODO: EvhrToA.run() return list of ToAs produced? # ----------------------------------------------------------------------------- def main(): - + # --- # Parse input arguments. # --- @@ -42,7 +42,7 @@ def main(): logger.addHandler(ch) # --- - # Make DgFiles from the input scenes. + # Make DgFiles from the input scenes. # --- scenes = args.scenes @@ -54,7 +54,7 @@ def main(): scenes = [Path(scene[0]) for scene in reader] dgScenes = scenesToDgFiles(scenes, logger) - + # --- # EVHR # --- @@ -72,6 +72,7 @@ def main(): cMaskDirNum = toaDirNum + 1 cMaskDir = Path(args.o) / (str(cMaskDirNum) + '-masks') cMaskDir.mkdir(exist_ok=True) + cMaskActualOutDir = cMaskDir / '5-toas' toaRegex = [str(toaDir / '*-toa.tif')] cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) cmpl.predict() @@ -84,9 +85,9 @@ def main(): ccdcDir = Path(args.o) / (str(ccdcDirNum) + '-ccdc') ccdcDir.mkdir(exist_ok=True) expCcdc = [ccdcDir / f.name.replace('-toa', '-toa_ccdc') for f in toas] - + if not all([f.exists() for f in expCcdc]): - + ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir) ccdc.run() @@ -98,31 +99,28 @@ def main(): srlDir = Path(args.o) / (str(srlDirNum) + '-srl') srlDir.mkdir(exist_ok=True) - import pdb - pdb.set_trace() - srl = SrliteWorkflow(output_dir=srlDir, toa_src=toaDir, target_dir=ccdcDir, - cloudmask_dir=cMaskDir) #, - # regressor='rma', - # debug=1, - # pmask='True', - # cloudmask='True', - # csv='True', - # band8='True', - # clean='True', - # # cloudmask_suffix=cloudmask_suffix, - # target_suffix='-ccdc.tif', - # logger=logger) - - # srl.processToas() - + cloudmask_dir=cMaskActualOutDir, + regressor='rma', + debug=1, + pmask='True', + cloudmask='True', + csv='True', + band8='True', + clean='True', + cloudmask_suffix='-toa.cloudmask.tif', + target_suffix='-toa_ccdc.tif', + logger=logger) + + srl.processToas() + # ----------------------------------------------------------------------------- # parseArgs # ----------------------------------------------------------------------------- def parseArgs() -> argparse.Namespace: - + desc = 'Use this application to run the VHR Toolkit.' parser = argparse.ArgumentParser(description=desc) @@ -169,27 +167,27 @@ def parseArgs() -> argparse.Namespace: # --- # VHR Cloud Mask Parameters # --- - + # --- # CCDC Parameters # --- - + # --- # SR-lite Parameters # --- - + # --- # Parse # --- args = parser.parse_args() - + return args - + # ----------------------------------------------------------------------------- # scenesToDgFiles # ----------------------------------------------------------------------------- def scenesToDgFiles(scenes: list, logger: logging.RootLogger) -> list: - + dgScenes = [] for s in scenes: @@ -199,7 +197,7 @@ def scenesToDgFiles(scenes: list, logger: logging.RootLogger) -> list: else: dgScenes.append(DgFile(str(s))) - + return dgScenes # ----------------------------------------------------------------------------- From 4772693f9ef0e7bf5a075eb670f0fb0a78879c78 Mon Sep 17 00:00:00 2001 From: R Date: Thu, 24 Oct 2024 15:38:44 -0400 Subject: [PATCH 9/9] Review modifications. --- view/vhrToolkit.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/view/vhrToolkit.py b/view/vhrToolkit.py index f6541de..c3f62dd 100755 --- a/view/vhrToolkit.py +++ b/view/vhrToolkit.py @@ -19,7 +19,7 @@ # ----------------------------------------------------------------------------- # main # -# vhr-toolkit/view/vhrToolkit.py -o /explore/nobackup/people/rlgill/SystemTesting/testILTK --scenes_in_file /explore/nobackup/projects/ilab/projects/vhr-toolkit/testing/evhr/toa_clv_test_alaska_cloud.csv +# vhr-toolkit/view/vhrToolkit.py -o /explore/nobackup/people/rlgill/SystemTesting/testILTK --scenes_in_file /explore/nobackup/projects/ilab/projects/vhr-toolkit/testing/evhr/toa_clv_test_alaska_cloud.csv # noqa: E501 # # TODO: EvhrToA needs an accessor for _outDir, _toaDir. # TODO: EvhrToA.__init__() outDir s/b of type Path. @@ -74,7 +74,10 @@ def main(): cMaskDir.mkdir(exist_ok=True) cMaskActualOutDir = cMaskDir / '5-toas' toaRegex = [str(toaDir / '*-toa.tif')] - cmpl = CloudMaskPipeline(output_dir=cMaskDir, inference_regex_list=toaRegex) + + cmpl = CloudMaskPipeline(output_dir=cMaskDir, + inference_regex_list=toaRegex) + cmpl.predict() # --- @@ -116,6 +119,7 @@ def main(): srl.processToas() + # ----------------------------------------------------------------------------- # parseArgs # ----------------------------------------------------------------------------- @@ -183,6 +187,7 @@ def parseArgs() -> argparse.Namespace: return args + # ----------------------------------------------------------------------------- # scenesToDgFiles # ----------------------------------------------------------------------------- @@ -200,6 +205,7 @@ def scenesToDgFiles(scenes: list, logger: logging.RootLogger) -> list: return dgScenes + # ----------------------------------------------------------------------------- # Invoke the main # -----------------------------------------------------------------------------