From 5a6164812535707f6940051ecd55f40940d980ad Mon Sep 17 00:00:00 2001 From: Kevin Pedro Date: Sat, 14 Sep 2024 10:55:31 -0500 Subject: [PATCH] reorg for better full-fast separation, handle harvest --- .../python/upgradeWorkflowComponents.py | 120 +++++++++--------- 1 file changed, 62 insertions(+), 58 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index f15b7d716da84..5dad1ca0886a9 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -2420,29 +2420,12 @@ def setup_(self, step, stepName, stepDict, k, properties): # just copy steps stepDict[stepName][k] = merge([stepDict[step][k]]) def setupPU_(self, step, stepName, stepDict, k, properties): - # setup for stage 1 - if "GenSim" in stepName: - stepNamePmx = stepName.replace('GenSim','Premix') - if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {} - # avoid overwriting fastsim alternative - if not k in stepDict[stepNamePmx]: - stepDict[stepNamePmx][k] = merge([ - { - '-s': 'GEN,SIM,DIGI:pdigi_valid', - '--datatier': 'PREMIX', - '--eventcontent': 'PREMIX', - '--procModifiers': 'premix_stage1' - }, - stepDict[stepName][k] - ]) - if "ProdLike" in self.suffix: - stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]]) - # setup for stage 1 fastsim - elif "Gen" in stepName: - stepNamePmx = stepName.replace('Gen','Premix') - if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {} - # avoid overwriting fullsim alternative - if not k in stepDict[stepNamePmx]: + # fastsim version + if 'FS' in k: + # setup for stage 1 fastsim + if "Gen" in stepName: + stepNamePmx = stepName.replace('Gen','Premix') + if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {} stepDict[stepNamePmx][k] = merge([ { '-s': 'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid', @@ -2456,12 +2439,11 @@ def setupPU_(self, step, stepName, stepDict, k, properties): if "ProdLike" in self.suffix: # todo pass - # setup for stage 2 - elif "Digi" in step or "Reco" in step: - # go back to non-PU step version - d = merge([stepDict[self.getStepName(step)][k]]) - if d is None: return - if "Digi" in step: + # setup for stage 2 fastsim + elif "FastSimRun3" in step: + # go back to non-PU step version + d = merge([stepDict[self.getStepName(step)][k]]) + if d is None: return tmpsteps = [] for s in d["-s"].split(","): if s == "DIGI" or "DIGI:" in s: @@ -2469,40 +2451,60 @@ def setupPU_(self, step, stepName, stepDict, k, properties): else: tmpsteps.append(s) d = merge([{"-s" : ",".join(tmpsteps), - "--datamix" : "PreMix", - "--procModifiers": "premix_stage2"}, + "--datamix" : "PreMix"}, d]) - # for combined stage1+stage2 - if "_PMXS1S2" in self.suffix: - d = merge([digiPremixLocalPileup, d]) - elif "Reco" in step: if "--procModifiers" in d: d["--procModifiers"] += ",premix_stage2" else: d["--procModifiers"] = "premix_stage2" - stepDict[stepName][k] = d - # setup for stage 2 fastsim - elif "FastSimRun3" in step: - # go back to non-PU step version - d = merge([stepDict[self.getStepName(step)][k]]) - if d is None: return - tmpsteps = [] - for s in d["-s"].split(","): - if s == "DIGI" or "DIGI:" in s: - tmpsteps.extend([s, "DATAMIX"]) - else: - tmpsteps.append(s) - d = merge([{"-s" : ",".join(tmpsteps), - "--datamix" : "PreMix"}, - d]) - if "--procModifiers" in d: - d["--procModifiers"] += ",premix_stage2" - else: - d["--procModifiers"] = "premix_stage2" - # for combined stage1+stage2 - if "_PMXS1S2" in self.suffix: - d = merge([digiPremixLocalPileup, d]) - stepDict[stepName][k] = d + # for combined stage1+stage2 + if "_PMXS1S2" in self.suffix: + d = merge([digiPremixLocalPileup, d]) + stepDict[stepName][k] = d + elif "HARVESTFastRun3" in step: + # increment input step number + stepDict[stepName][k] = merge([{'--filein':'file:step3_inDQM.root'},stepDict[stepName][k]]) + else: + # setup for stage 1 + if "GenSim" in stepName: + stepNamePmx = stepName.replace('GenSim','Premix') + if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {} + stepDict[stepNamePmx][k] = merge([ + { + '-s': 'GEN,SIM,DIGI:pdigi_valid', + '--datatier': 'PREMIX', + '--eventcontent': 'PREMIX', + '--procModifiers': 'premix_stage1' + }, + stepDict[stepName][k] + ]) + if "ProdLike" in self.suffix: + stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]]) + # setup for stage 2 + elif "Digi" in step or "Reco" in step: + # go back to non-PU step version + d = merge([stepDict[self.getStepName(step)][k]]) + if d is None: return + if "Digi" in step: + tmpsteps = [] + for s in d["-s"].split(","): + if s == "DIGI" or "DIGI:" in s: + tmpsteps.extend([s, "DATAMIX"]) + else: + tmpsteps.append(s) + d = merge([{"-s" : ",".join(tmpsteps), + "--datamix" : "PreMix", + "--procModifiers": "premix_stage2"}, + d]) + # for combined stage1+stage2 + if "_PMXS1S2" in self.suffix: + d = merge([digiPremixLocalPileup, d]) + elif "Reco" in step: + if "--procModifiers" in d: + d["--procModifiers"] += ",premix_stage2" + else: + d["--procModifiers"] = "premix_stage2" + stepDict[stepName][k] = d def condition(self, fragment, stepList, key, hasHarvest): if not 'PU' in key: return False @@ -2543,6 +2545,7 @@ def workflow_(self, workflows, num, fragment, stepList, key): 'RecoNanoFakeHLT', 'Nano', 'FastSimRun3', + 'HARVESTFastRun3', ], suffix = '_PMXS2', offset = 0.98, @@ -2565,6 +2568,7 @@ def workflow_(self, workflows, num, fragment, stepList, key): 'RecoNanoFakeHLT', 'Nano', 'FastSimRun3', + 'HARVESTFastRun3', ], suffix = '_PMXS1S2', offset = 0.99,