changes corresponding to mode 3 cvmfsexec PR

creation/lib/cgWConsts.py

@@ -18,8 +18,9 @@
 from . import cWConsts
 
 # these are in the stage dir, so they need to be renamed if changed
-AFTER_FILE_LISTFILE = "after_%s" % cWConsts.FILE_LISTFILE
-AT_FILE_LISTFILE = "at_%s" % cWConsts.FILE_LISTFILE
+AFTER_FILE_LISTFILE = f"after_{cWConsts.FILE_LISTFILE}"
+AT_FILE_LISTFILE = f"at_{cWConsts.FILE_LISTFILE}"
+PRECVMFS_FILE_LISTFILE = f"precvmfs_{cWConsts.FILE_LISTFILE}"
 
 CONDOR_FILE = "condor_bin_%s.tgz"
 CONDOR_DIR = "condor"
@@ -32,6 +33,8 @@
 CVMFSEXEC_DIR = "cvmfsexec"
 CVMFSEXEC_ATTR = "CVMFSEXEC_DIR"
 
+# constant that defines the priority values used by the factory/glidein
+PRIORITY_SETTINGS = ["file_list", "precvmfs_file_list", "at_file_list", "after_file_list"]
 
 # these are in the submit dir, so they can be changed
 PARAMS_FILE = "params.cfg"

creation/lib/cgWDictFile.py

@@ -244,6 +244,9 @@ def get_main_dicts(submit_dir, stage_dir):
  main_dicts["build_cvmfsexec"] = cWDictFile.ReprDictFile(submit_dir, cgWConsts.CVMFSEXEC_BUILD_FILE)
  main_dicts["frontend_descript"] = cWDictFile.ReprDictFile(submit_dir, cgWConsts.FRONTEND_DESCRIPT_FILE)
  main_dicts["gridmap"] = cWDictFile.GridMapDict(stage_dir, cWConsts.insert_timestr(cWConsts.GRIDMAP_FILE))
+ main_dicts["precvmfs_file_list"] = cWDictFile.FileDictFile(
+ stage_dir, cWConsts.insert_timestr(cgWConsts.PRECVMFS_FILE_LISTFILE), fname_idx=cgWConsts.PRECVMFS_FILE_LISTFILE
+ )
  main_dicts["at_file_list"] = cWDictFile.FileDictFile(
  stage_dir, cWConsts.insert_timestr(cgWConsts.AT_FILE_LISTFILE), fname_idx=cgWConsts.AT_FILE_LISTFILE
  )
@@ -300,10 +303,15 @@ def load_main_dicts(main_dicts): # update in place
  # print "\ndebug %s main_dicts['description'].keys2 = %s" % (__file__, main_dicts['description'].keys2)
  # print "\ndebug %s dir(main_dicts['description']) = %s" % (__file__, dir(main_dicts['description']))
  # TODO: To remove if upgrade from older versions is not a problem
+ try:
+ main_dicts["precvmfs_file_list"].load(fname=main_dicts["description"].vals2["precvmfs_file_list"])
+ except KeyError:
+ # when upgrading from older version the new precvmfs_file_list may not be in the description
+ main_dicts["precvmfs_file_list"].load()
  try:
  main_dicts["at_file_list"].load(fname=main_dicts["description"].vals2["at_file_list"])
  except KeyError:
- # when upgrading form older version the new at_file_list may not be in the description
+ # when upgrading from older version the new at_file_list may not be in the description
  main_dicts["at_file_list"].load()
  main_dicts["after_file_list"].load(fname=main_dicts["description"].vals2["after_file_list"])
  load_common_dicts(main_dicts, main_dicts["description"])
@@ -331,7 +339,7 @@ def load_entry_dicts(entry_dicts, entry_name, summary_signature): # update in p
 def refresh_description(dicts): # update in place
  description_dict = dicts["description"]
  description_dict.add(dicts["signature"].get_fname(), "signature", allow_overwrite=True)
- for k in ("file_list", "at_file_list", "after_file_list"):
+ for k in cgWConsts.PRIORITY_SETTINGS:
  if k in dicts:
  description_dict.add(dicts[k].get_fname(), k, allow_overwrite=True)
 
@@ -377,11 +385,21 @@ def refresh_file_list(dicts, is_main, files_set_readonly=True, files_reset_chang
 # dictionaries must have been written to disk before using this
 def refresh_signature(dicts): # update in place
  signature_dict = dicts["signature"]
- for k in ("consts", "vars", "untar_cfg", "gridmap", "file_list", "at_file_list", "after_file_list", "description"):
+ for k in (
+ "consts",
+ "vars",
+ "untar_cfg",
+ "gridmap",
+ "file_list",
+ "precvmfs_file_list",
+ "at_file_list",
+ "after_file_list",
+ "description",
+ ):
  if k in dicts:
  signature_dict.add_from_file(dicts[k].get_filepath(), allow_overwrite=True)
  # add signatures of all the files linked in the lists
- for k in ("file_list", "at_file_list", "after_file_list"):
+ for k in cgWConsts.PRIORITY_SETTINGS:
  if k in dicts:
  filedict = dicts[k]
  for fname in filedict.get_immutable_files():
@@ -406,11 +424,11 @@ def save_common_dicts(dicts, is_main, set_readonly=True): # will update in plac
  # 'consts','untar_cfg','vars' will be loaded
  refresh_file_list(dicts, is_main)
  # save files in the file lists
- for k in ("file_list", "at_file_list", "after_file_list"):
+ for k in cgWConsts.PRIORITY_SETTINGS:
  if k in dicts:
  dicts[k].save_files(allow_overwrite=True)
  # then save the lists
- for k in ("file_list", "at_file_list", "after_file_list"):
+ for k in cgWConsts.PRIORITY_SETTINGS:
  if k in dicts:
  dicts[k].save(set_readonly=set_readonly)
  # calc and save the signatues
@@ -484,7 +502,7 @@ def reuse_common_dicts(dicts, other_dicts, is_main, all_reused):
  # since the file names may have changed, refresh the file_list
  refresh_file_list(dicts, is_main)
  # check file-based dictionaries
- for k in ("file_list", "at_file_list", "after_file_list"):
+ for k in cgWConsts.PRIORITY_SETTINGS:
  if k in dicts:
  all_reused = reuse_file_dict(dicts, other_dicts, k) and all_reused
 

creation/lib/cgWParamDict.py

@@ -11,6 +11,8 @@
 import os.path
 import shutil
 
+from collections import Counter
+
 from glideinwms.lib import pubCrypto, subprocessSupport
 from glideinwms.lib.util import str2bool
 
@@ -220,15 +222,26 @@ def populate(self, other=None):
  self.dicts["params"].add("GLIDEIN_Factory_Collector", str(factory_monitoring_collector))
  populate_gridmap(self.conf, self.dicts["gridmap"])
 
+ # the following list will be a megalist containing all the scripts; used for duplication check logic subsequently
+ all_scripts = list()
  # NOTE that all the files in these _scripts lists are added as executables (i.e. must report with error_gen)
  file_list_scripts = [
  "collector_setup.sh",
  "create_temp_mapfile.sh",
  "gwms-python",
  cgWConsts.CONDOR_STARTUP_FILE,
  ]
+ # add the above list to the megalist created before
+ all_scripts.extend(file_list_scripts)
+
+ # singularity_setup should be performed after cvmfs_setup; condor_chirp's order does not matter
+ precvmfs_file_list_scripts = ["cvmfs_setup.sh"]
+ all_scripts.extend(precvmfs_file_list_scripts) # add this list to the megalist
+
  # These are right after the entry, before some VO scripts. The order in the following list is important
  at_file_list_scripts = ["singularity_setup.sh", "condor_chirp", "gconfig.py"]
+ all_scripts.extend(at_file_list_scripts) # adding the above list to the megalist as before
+
  # The order in the following list is important
  after_file_list_scripts = [
  "check_proxy.sh",
@@ -241,15 +254,14 @@ def populate(self, other=None):
  "glidein_sitewms_setup.sh",
  "script_wrapper.sh",
  "smart_partitionable.sh",
- "cvmfs_setup.sh",
  "cvmfs_umount.sh",
  ]
- # Only execute scripts once
- duplicate_scripts = list(set(file_list_scripts).intersection(after_file_list_scripts))
- duplicate_scripts += list(set(file_list_scripts).intersection(at_file_list_scripts))
- duplicate_scripts += list(set(at_file_list_scripts).intersection(after_file_list_scripts))
+ all_scripts.extend(after_file_list_scripts) # adding the above list to the megalist as before
+ # Scripts need to be only executed once, so check for duplicates
+ count_duplicates = Counter(all_scripts)
+ duplicate_scripts = [scr for scr, cnt in count_duplicates.items() if cnt > 1]
  if duplicate_scripts:
- raise RuntimeError("Duplicates found in the list of files to execute '%s'" % ",".join(duplicate_scripts))
+ raise RuntimeError(f"Duplicates found in the list of files to execute: {', '.join(duplicate_scripts)}")
 
  # Load more system scripts
  for script_name in file_list_scripts:
@@ -405,6 +417,12 @@ def populate(self, other=None):
  )
 
  # add additional system scripts
+ for script_name in precvmfs_file_list_scripts:
+ self.dicts["precvmfs_file_list"].add_from_file(
+ script_name,
+ cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec:r"),
+ os.path.join(cgWConsts.WEB_BASE_DIR, script_name),
+ )
  for script_name in at_file_list_scripts:
  self.dicts["at_file_list"].add_from_file(
  script_name,