Science portal run

example/cfis/config_MaMa_onthefly.ini

@@ -44,7 +44,7 @@ OUTPUT_DIR = $SP_RUN/output
 [JOB]
 
 # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
-SMP_BATCH_SIZE = 8
+SMP_BATCH_SIZE = 16
 
 # Timeout value (optional), default is None, i.e. no timeout limit applied
 TIMEOUT = 96:00:00

example/cfis/config_exp_psfex.ini

@@ -9,7 +9,8 @@
 VERBOSE = True
 
 # Name of run (optional) default: shapepipe_run
-RUN_NAME = run_sp_exp_SxSePsfPi
+#RUN_NAME = run_sp_exp_SxSePsfPi
+RUN_NAME = run_sp_exp_SxSePsf
 
 # Add date and time to RUN_NAME, optional, default: True
 ; RUN_DATETIME = False
@@ -19,7 +20,8 @@ RUN_NAME = run_sp_exp_SxSePsfPi
 [EXECUTION]
 
 # Module name, single string or comma-separated list of valid module runner names
-MODULE = sextractor_runner, setools_runner, psfex_runner,psfex_interp_runner
+MODULE = sextractor_runner, setools_runner, psfex_runner
+#psfex_interp_runner
 
 
 # Run mode, SMP or MPI
@@ -57,7 +59,8 @@ TIMEOUT = 96:00:00
 [SEXTRACTOR_RUNNER]
 
 # Input from two modules
-INPUT_MODULE = split_exp_runner, mask_runner
+INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2
+#INPUT_MODULE = split_exp_runner, mask_runner_run_2
 
 # Read pipeline flag files created by mask module
 FILE_PATTERN = image, weight, pipeline_flag

example/cfis/config_tile_PiViSmVi_canfar.ini

@@ -183,6 +183,6 @@ PREFIX =
 
 # Additional parameters for path and file pattern corresponding to single-exposure
 # run outputs
-ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner
+ME_IMAGE_DIR = last:split_exp_runner, last:split_exp_runner, last:split_exp_runner, all:sextractor_runner
 ME_IMAGE_PATTERN = flag, image, weight, background
 ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite

scripts/python/link_to_exp_for_tile.py

@@ -165,10 +165,13 @@ def matching_subdirs(base_dir, pattern):
 
     # Find all matching subdirectories
     subdirs = []
-    for entry in os.listdir(base_dir):
-        full_path = os.path.join(base_dir, entry)
-        if os.path.isdir(full_path) and entry.startswith(pattern):
-            subdirs.append(full_path)
+    if os.path.exists(base_dir):
+        for entry in os.listdir(base_dir):
+            full_path = os.path.join(base_dir, entry)
+            if os.path.isdir(full_path) and entry.startswith(pattern):
+                subdirs.append(full_path)
+    else:
+        print(f"Warning: {base_dir} does not exist, continuing...")
 
     # Sort according to creation date
     subdirs.sort(key=os.path.getctime)
@@ -187,14 +190,25 @@ def get_exp_IDs(tile_base_dir, tile_ID, verbose=False):
 
     tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID)
 
-    subdirs = matching_subdirs(tile_out_dir, "run_sp_GitFeGie")
+    pattern = "run_sp_GitFeGie"
+    subdirs = matching_subdirs(tile_out_dir, pattern)
 
+    if len(subdirs) == 0:
+        raise IOError(
+            f"No matching directory '{pattern}' in {tile_out_dir} found"
+        )
     if len(subdirs) != 1:
-        raise IOError(f"Exactly one matching directory in {tile_out_dir} expected, not {len(subdirs)}")
+        raise IOError(
+            f"Exactly one directory natching {pattern} in {tile_out_dir} "
+            + f"expected, not {len(subdirs)}"
+        )
 
     # Replace dot with dash in tile ID
     tile_ID_sp = re.sub(r"\.", "-", tile_ID)
-    exp_ID_file = f"{subdirs[0]}/find_exposures_runner/output/exp_numbers-{tile_ID_sp}.txt"
+    exp_ID_file = (
+        f"{subdirs[0]}/find_exposures_runner/output/"
+        + f"exp_numbers-{tile_ID_sp}.txt"
+    )
 
     exp_IDs = []
     with open(exp_ID_file) as f_in:
@@ -234,7 +248,10 @@ def get_paths(exp_base_dir, exp_shdu_IDs, pattern):
             number[n_subdirs] += 1
 
         if n_subdirs != 1:
-            msg = f"Exactly one matching directory in {path} expected, not {n_subdirs}"
+            msg = (
+                f"Exactly on directory matching {pattern} in {path} expected,"
+                + f"  not {n_subdirs}"
+            )
             print(msg)
             if n_subdirs == 0:
                 continue
@@ -253,9 +270,26 @@ def create_links_paths(tile_base_dir, tile_ID, paths):
         src = path
         dst = f"{tile_out_dir}/{tail}"
         if os.path.exists(dst):
-            print(f"Warning: {dst} already exists, no link created")
+            src_existing = os.readlink(dst)
+            if src_existing == src:
+                print(src, dst)
+                    #f"Warning: {src} <- {dst} already exists, no link created"
+                #)
+                continue
+            else:
+                idx = 1
+                dst_orig = dst
+                while True:
+                    dst = f"{dst_orig}_{idx}"
+                    if os.path.exists(dst):
+                        idx += 1
+                    else:
+                        #print("MKDEBUG new ", dst)
+                        #print(f"ln -s {src} {dst}")
+                        os.symlink(src, dst)
+                        break
         else:
-            print(f"ln -s {src} {dst}")
+            #print(f"ln -s {src} {dst}")
             os.symlink(src, dst)
 
 
@@ -285,6 +319,10 @@ def main(argv=None):
     for pattern in patterns:
         paths, number = get_paths(exp_base_dir, exp_shdu_IDs, pattern)
         print(number)
+        if "2159307-31" in paths:
+            print("found")
+        else:
+            print("not found")
 
         create_links_paths(tile_base_dir, tile_ID, paths)