Merge pull request #1720 from pierotofy/autorerun

Feat: Auto rerun-from

VERSION

@@ -1 +1 @@
-3.3.1
+3.3.2

opendm/arghelpers.py

@@ -0,0 +1,76 @@
+from opendm import log
+from shlex import _find_unsafe
+import json
+import os
+
+def double_quote(s):
+    """Return a shell-escaped version of the string *s*."""
+    if not s:
+        return '""'
+    if _find_unsafe(s) is None:
+        return s
+
+    # use double quotes, and prefix double quotes with a \
+    # the string $"b is then quoted as "$\"b"
+    return '"' + s.replace('"', '\\\"') + '"'
+
+def args_to_dict(args):
+    args_dict = vars(args)
+    result = {}
+    for k in sorted(args_dict.keys()):
+        # Skip _is_set keys
+        if k.endswith("_is_set"):
+            continue
+
+        # Don't leak token
+        if k == 'sm_cluster' and args_dict[k] is not None:
+            result[k] = True
+        else:
+            result[k] = args_dict[k]
+
+    return result
+
+def save_opts(opts_json, args):
+    try:
+        with open(opts_json, "w", encoding='utf-8') as f:
+            f.write(json.dumps(args_to_dict(args)))
+    except Exception as e:
+        log.ODM_WARNING("Cannot save options to %s: %s" % (opts_json, str(e)))
+
+def compare_args(opts_json, args, rerun_stages):
+    if not os.path.isfile(opts_json):
+        return {}
+
+    try:
+        diff = {}
+
+        with open(opts_json, "r", encoding="utf-8") as f:
+            prev_args = json.loads(f.read())
+        cur_args = args_to_dict(args)
+
+        for opt in cur_args:
+            cur_value = cur_args[opt]
+            prev_value = prev_args.get(opt, None)
+            stage = rerun_stages.get(opt, None)
+
+            if stage is not None and cur_value != prev_value:
+                diff[opt] = prev_value
+
+        return diff
+    except:
+        return {}
+
+def find_rerun_stage(opts_json, args, rerun_stages, processopts):
+    # Find the proper rerun stage if one is not explicitly set
+    if not ('rerun_is_set' in args or 'rerun_from_is_set' in args or 'rerun_all_is_set' in args):
+        args_diff = compare_args(opts_json, args, rerun_stages)
+        if args_diff:
+            if 'split_is_set' in args:
+                return processopts[processopts.index('dataset'):], args_diff
+
+            try:
+                stage_idxs = [processopts.index(rerun_stages[opt]) for opt in args_diff.keys() if rerun_stages[opt] is not None]
+                return processopts[min(stage_idxs):], args_diff
+            except ValueError as e:
+                print(str(e))
+    return None, {}

opendm/config.py

@@ -13,6 +13,101 @@
                'odm_meshing', 'mvs_texturing', 'odm_georeferencing',
                'odm_dem', 'odm_orthophoto', 'odm_report', 'odm_postprocess']
 
+rerun_stages = {
+    '3d_tiles': 'odm_postprocess',
+    'align': 'odm_georeferencing',
+    'auto_boundary': 'odm_filterpoints',
+    'auto_boundary_distance': 'odm_filterpoints',
+    'bg_removal': 'dataset',
+    'boundary': 'odm_filterpoints',
+    'build_overviews': 'odm_orthophoto',
+    'camera_lens': 'dataset',
+    'cameras': 'dataset',
+    'cog': 'odm_dem',
+    'copy_to': 'odm_postprocess',
+    'crop': 'odm_georeferencing',
+    'dem_decimation': 'odm_dem',
+    'dem_euclidean_map': 'odm_dem',
+    'dem_gapfill_steps': 'odm_dem',
+    'dem_resolution': 'odm_dem',
+    'dsm': 'odm_dem',
+    'dtm': 'odm_dem',
+    'end_with': None,
+    'fast_orthophoto': 'odm_filterpoints',
+    'feature_quality': 'opensfm',
+    'feature_type': 'opensfm',
+    'force_gps': 'opensfm',
+    'gcp': 'dataset',
+    'geo': 'dataset',
+    'gltf': 'mvs_texturing',
+    'gps_accuracy': 'dataset',
+    'help': None,
+    'ignore_gsd': 'opensfm',
+    'matcher_neighbors': 'opensfm',
+    'matcher_order': 'opensfm',
+    'matcher_type': 'opensfm',
+    'max_concurrency': None,
+    'merge': 'Merge',
+    'mesh_octree_depth': 'odm_meshing',
+    'mesh_size': 'odm_meshing',
+    'min_num_features': 'opensfm',
+    'name': None,
+    'no_gpu': None,
+    'optimize_disk_space': None,
+    'orthophoto_compression': 'odm_orthophoto',
+    'orthophoto_cutline': 'odm_orthophoto',
+    'orthophoto_kmz': 'odm_orthophoto',
+    'orthophoto_no_tiled': 'odm_orthophoto',
+    'orthophoto_png': 'odm_orthophoto',
+    'orthophoto_resolution': 'odm_orthophoto',
+    'pc_classify': 'odm_dem',
+    'pc_copc': 'odm_georeferencing',
+    'pc_csv': 'odm_georeferencing',
+    'pc_ept': 'odm_georeferencing',
+    'pc_filter': 'openmvs',
+    'pc_las': 'odm_georeferencing',
+    'pc_quality': 'opensfm',
+    'pc_rectify': 'odm_dem',
+    'pc_sample': 'odm_filterpoints',
+    'pc_skip_geometric': 'openmvs',
+    'primary_band': 'dataset',
+    'project_path': None,
+    'radiometric_calibration': 'opensfm',
+    'rerun': None,
+    'rerun_all': None,
+    'rerun_from': None,
+    'rolling_shutter': 'opensfm',
+    'rolling_shutter_readout': 'opensfm',
+    'sfm_algorithm': 'opensfm',
+    'sfm_no_partial': 'opensfm',
+    'skip_3dmodel': 'odm_meshing',
+    'skip_band_alignment': 'opensfm',
+    'skip_orthophoto': 'odm_orthophoto',
+    'skip_report': 'odm_report',
+    'sky_removal': 'dataset',
+    'sm_cluster': 'split',
+    'sm_no_align': 'split',
+    'smrf_scalar': 'odm_dem',
+    'smrf_slope': 'odm_dem',
+    'smrf_threshold': 'odm_dem',
+    'smrf_window': 'odm_dem',
+    'split': 'split',
+    'split_image_groups': 'split',
+    'split_overlap': 'split',
+    'texturing_keep_unseen_faces': 'mvs_texturing',
+    'texturing_single_material': 'mvs_texturing',
+    'texturing_skip_global_seam_leveling': 'mvs_texturing',
+    'texturing_skip_local_seam_leveling': 'mvs_texturing',
+    'tiles': 'odm_dem',
+    'use_3dmesh': 'mvs_texturing',
+    'use_exif': 'dataset',
+    'use_fixed_camera_params': 'opensfm',
+    'use_hybrid_bundle_adjustment': 'opensfm',
+    'version': None,
+    'video_limit': 'dataset',
+    'video_resolution': 'dataset',
+}
+
 with open(os.path.join(context.root_path, 'VERSION')) as version_file:
     __version__ = version_file.read().strip()
 

opendm/log.py

@@ -7,7 +7,7 @@
 import shutil
 import multiprocessing
 
-from opendm.loghelpers import double_quote, args_to_dict
+from opendm.arghelpers import double_quote, args_to_dict
 from vmem import virtual_memory
 
 if sys.platform == 'win32' or os.getenv('no_ansiesc'):

opendm/utils.py

@@ -4,7 +4,7 @@
 from opendm import log
 from opendm.photo import find_largest_photo_dims
 from osgeo import gdal
-from opendm.loghelpers import double_quote
+from opendm.arghelpers import double_quote
 
 class NumpyEncoder(json.JSONEncoder):
     def default(self, obj):

run.py

@@ -13,7 +13,7 @@
 from opendm import io
 from opendm.progress import progressbc
 from opendm.utils import get_processing_results_paths, rm_r
-from opendm.loghelpers import args_to_dict
+from opendm.arghelpers import args_to_dict, save_opts, compare_args, find_rerun_stage
 
 from stages.odm_app import ODMApp
 
@@ -29,20 +29,26 @@ def odm_version():
 
     log.ODM_INFO('Initializing ODM %s - %s' % (odm_version(), system.now()))
 
+    progressbc.set_project_name(args.name)
+    args.project_path = os.path.join(args.project_path, args.name)
+
+    if not io.dir_exists(args.project_path):
+        log.ODM_ERROR('Directory %s does not exist.' % args.name)
+        exit(1)
+
+    opts_json = os.path.join(args.project_path, "options.json")
+    auto_rerun_stage, opts_diff = find_rerun_stage(opts_json, args, config.rerun_stages, config.processopts)
+    if auto_rerun_stage is not None and len(auto_rerun_stage) > 0:
+        log.ODM_INFO("Rerunning from: %s" % auto_rerun_stage[0])
+        args.rerun_from = auto_rerun_stage
+
     # Print args
     args_dict = args_to_dict(args)
     log.ODM_INFO('==============')
     for k in args_dict.keys():
-        log.ODM_INFO('%s: %s' % (k, args_dict[k]))
+        log.ODM_INFO('%s: %s%s' % (k, args_dict[k], ' [changed]' if k in opts_diff else ''))
     log.ODM_INFO('==============')
-
-    progressbc.set_project_name(args.name)
-
-    # Add project dir if doesn't exist
-    args.project_path = os.path.join(args.project_path, args.name)
-    if not io.dir_exists(args.project_path):
-        log.ODM_WARNING('Directory %s does not exist. Creating it now.' % args.name)
-        system.mkdir_p(os.path.abspath(args.project_path))
+
 
     # If user asks to rerun everything, delete all of the existing progress directories.
     if args.rerun_all:
@@ -57,6 +63,9 @@ def odm_version():
 
     app = ODMApp(args)
     retcode = app.execute()
+
+    if retcode == 0:
+        save_opts(opts_json, args)
 
     # Do not show ASCII art for local submodels runs
     if retcode == 0 and not "submodels" in args.project_path:

stages/odm_app.py

@@ -27,6 +27,7 @@ def __init__(self, args):
         Initializes the application and defines the ODM application pipeline stages
         """
         json_log_paths = [os.path.join(args.project_path, "log.json")]
+
         if args.copy_to:
             json_log_paths.append(args.copy_to)