Skip to content

Commit

Permalink
Merge pull request #46007 from kpedro88/Phase2-WF82
Browse files Browse the repository at this point in the history 
FastSim premixing workflows and matrix cleanup
  • Loading branch information
@cmsbuild
cmsbuild authored Oct 4, 2024
2 parents 14a12c2 + 1ca6072 commit 1edffb7
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 59 deletions.
4 changes: 2 additions & 2 deletions Configuration/PyReleaseValidation/python/relval_upgrade.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ def notForGenOnly(key,specialType):
stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','HLT75e33'),specialWF.suffix))
# similar hacks for premixing
if 'PMX' in specialType:
if 'GenSim' in step:
s = step.replace('GenSim','Premix')+'PU' # later processing requires to have PU here
if 'GenSim' in step or 'Gen' in step:
s = step.replace('GenSim','Premix').replace('Gen','Premix')+'PU' # later processing requires to have PU here
if step in specialWF.PU:
stepMade = stepMaker(key,'PREMIX',s,specialWF.suffix)
# append for combined
Expand Down
158 changes: 101 additions & 57 deletions Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ def workflow(self, workflows, num, fragment, stepList, key, hasHarvest):
def workflow_(self, workflows, num, fragment, stepList, key):
fragmentTmp = [fragment, key]
if len(self.suffix)>0: fragmentTmp.append(self.suffix)
workflows[num+self.offset] = [ fragmentTmp, stepList ]
# avoid spurious workflows (no steps modified)
if self.offset==0 or workflows[num][1]!=stepList:
workflows[num+self.offset] = [ fragmentTmp, stepList ]
def condition(self, fragment, stepList, key, hasHarvest):
return False
def preventReuse(self, stepName, stepDict, k):
Expand Down Expand Up @@ -562,7 +564,7 @@ def condition(self, fragment, stepList, key, hasHarvest):

# WeightedMeanFitter vertexing workflows
class UpgradeWorkflow_weightedVertex(UpgradeWorkflow):
def __init__(self, reco = {}, harvest = {}, **kwargs):
def __init__(self, **kwargs):
# adapt the parameters for the UpgradeWorkflow init method
super(UpgradeWorkflow_weightedVertex, self).__init__(
steps = [
Expand Down Expand Up @@ -590,8 +592,6 @@ def __init__(self, reco = {}, harvest = {}, **kwargs):
'HARVESTNanoFakeHLT',
],
**kwargs)
self.__reco = reco
self.__harvest = harvest

def setup_(self, step, stepName, stepDict, k, properties):
# temporarily remove trigger & downstream steps
Expand Down Expand Up @@ -2264,7 +2264,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
stepDict[stepName][k] = None

def condition(self, fragment, stepList, key, hasHarvest):
return ('2021' in key or '2023' in key or '2026' in key)
return fragment=="TTbar_14TeV" and ('2021' in key or '2023' in key or '2026' in key)

upgradeWFs['ECALComponent'] = UpgradeWorkflow_ECalComponent(
suffix = '_ecalComponent',
Expand Down Expand Up @@ -2520,58 +2520,103 @@ def setup_(self, step, stepName, stepDict, k, properties):
# just copy steps
stepDict[stepName][k] = merge([stepDict[step][k]])
def setupPU_(self, step, stepName, stepDict, k, properties):
# setup for stage 1
if "GenSim" in stepName:
stepNamePmx = stepName.replace('GenSim','Premix')
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
stepDict[stepNamePmx][k] = merge([
{
'-s': 'GEN,SIM,DIGI:pdigi_valid',
'--datatier': 'PREMIX',
'--eventcontent': 'PREMIX',
'--procModifiers': 'premix_stage1'
},
stepDict[stepName][k]
])
if "ProdLike" in self.suffix:
stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]])
# setup for stage 2
elif "Digi" in step or "Reco" in step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if d is None: return
if "Digi" in step:
# fastsim version
if 'FS' in k:
# setup for stage 1 fastsim
if "Gen" in stepName:
stepNamePmx = stepName.replace('Gen','Premix')
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
stepDict[stepNamePmx][k] = merge([
{
'-s': 'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid',
'--fast':'',
'--datatier': 'PREMIX',
'--eventcontent': 'PREMIX',
'--procModifiers': 'premix_stage1'
},
stepDict[stepName][k]
])
if "ProdLike" in self.suffix:
# todo
pass
# setup for stage 2 fastsim
elif "FastSimRun3" in step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if d is None: return
tmpsteps = []
for s in d["-s"].split(","):
if s == "DIGI" or "DIGI:" in s:
tmpsteps.extend([s, "DATAMIX"])
else:
tmpsteps.append(s)
d = merge([{"-s" : ",".join(tmpsteps),
"--datamix" : "PreMix",
"--procModifiers": "premix_stage2"},
"--datamix" : "PreMix"},
d])
# for combined stage1+stage2
if "_PMXS1S2" in self.suffix:
d = merge([digiPremixLocalPileup, d])
elif "Reco" in step:
if "--procModifiers" in d:
d["--procModifiers"] += ",premix_stage2"
else:
d["--procModifiers"] = "premix_stage2"
stepDict[stepName][k] = d
# Increase the input file step number by one for Nano in combined stage1+stage2
elif "Nano"==step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if "--filein" in d:
filein = d["--filein"]
m = re.search("step(?P<ind>\\d+)_", filein)
if m:
d["--filein"] = filein.replace(m.group(), "step%d_"%(int(m.group("ind"))+1))
stepDict[stepName][k] = d
# run2/3 WFs use Nano (not NanoPU) in PU WF
stepDict[self.getStepName(step)][k] = merge([d])
# for combined stage1+stage2
if "_PMXS1S2" in self.suffix:
d = merge([digiPremixLocalPileup, d])
stepDict[stepName][k] = d
elif "HARVESTFastRun3" in step:
# increment input step number
stepDict[stepName][k] = merge([{'--filein':'file:step3_inDQM.root'},stepDict[stepName][k]])
else:
# setup for stage 1
if "GenSim" in stepName:
stepNamePmx = stepName.replace('GenSim','Premix')
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
stepDict[stepNamePmx][k] = merge([
{
'-s': 'GEN,SIM,DIGI:pdigi_valid',
'--datatier': 'PREMIX',
'--eventcontent': 'PREMIX',
'--procModifiers': 'premix_stage1'
},
stepDict[stepName][k]
])
if "ProdLike" in self.suffix:
stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]])
# setup for stage 2
elif "Digi" in step or "Reco" in step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if d is None: return
if "Digi" in step:
tmpsteps = []
for s in d["-s"].split(","):
if s == "DIGI" or "DIGI:" in s:
tmpsteps.extend([s, "DATAMIX"])
else:
tmpsteps.append(s)
d = merge([{"-s" : ",".join(tmpsteps),
"--datamix" : "PreMix",
"--procModifiers": "premix_stage2"},
d])
# for combined stage1+stage2
if "_PMXS1S2" in self.suffix:
d = merge([digiPremixLocalPileup, d])
elif "Reco" in step:
if "--procModifiers" in d:
d["--procModifiers"] += ",premix_stage2"
else:
d["--procModifiers"] = "premix_stage2"
stepDict[stepName][k] = d
# separate nano step now only used in ProdLike workflows for Run3/Phase2
elif "Nano"==step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if "_PMXS1S2" in self.suffix and "--filein" in d:
filein = d["--filein"]
m = re.search("step(?P<ind>\\d+)", filein)
if m:
d["--filein"] = filein.replace(m.group(), "step%d"%(int(m.group("ind"))+1))
stepDict[stepName][k] = d
# run2/3 WFs use Nano (not NanoPU) in PU WF
stepDict[self.getStepName(step)][k] = merge([d])
def condition(self, fragment, stepList, key, hasHarvest):
if not 'PU' in key:
return False
Expand All @@ -2590,6 +2635,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
steps = [
],
PU = [
'Gen',
'GenSim',
'GenSimHLBeamSpot',
'GenSimHLBeamSpot14',
Expand All @@ -2610,6 +2656,8 @@ def workflow_(self, workflows, num, fragment, stepList, key):
'RecoNano',
'RecoNanoFakeHLT',
'Nano',
'FastSimRun3',
'HARVESTFastRun3',
],
suffix = '_PMXS2',
offset = 0.98,
Expand All @@ -2618,6 +2666,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
upgradeWFs['PMXS1S2'] = UpgradeWorkflowPremix(
steps = [],
PU = [
'Gen',
'GenSim',
'GenSimHLBeamSpot',
'GenSimHLBeamSpot14',
Expand All @@ -2630,6 +2679,8 @@ def workflow_(self, workflows, num, fragment, stepList, key):
'RecoNano',
'RecoNanoFakeHLT',
'Nano',
'FastSimRun3',
'HARVESTFastRun3',
],
suffix = '_PMXS1S2',
offset = 0.99,
Expand Down Expand Up @@ -2683,8 +2734,6 @@ def setup_(self, step, stepName, stepDict, k, properties):
"--eventcontent": "PREMIXRAW"},
d])
stepDict[stepName][k] = d
if 'Nano'==step:
stepDict[stepName][k] = merge([{'--filein':'file:step5.root','-s':'NANO','--datatier':'NANOAODSIM','--eventcontent':'NANOEDMAODSIM'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
# use both conditions
return UpgradeWorkflowPremix.condition(self, fragment, stepList, key, hasHarvest) and UpgradeWorkflow_ProdLike.condition(self, fragment, stepList, key, hasHarvest)
Expand Down Expand Up @@ -2750,7 +2799,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
else:
stepDict[stepName][k] = merge([stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return ('FS' in key)
return fragment=="TTbar_14TeV" and ('FS' in key)
upgradeWFs['Run3FStrackingOnly'] = UpgradeWorkflow_Run3FStrackingOnly(
steps = [
'Gen',
Expand Down Expand Up @@ -2792,17 +2841,12 @@ def condition(self, fragment, stepList, key, hasHarvest):

class UpgradeWorkflow_DD4hep(UpgradeWorkflow):
def setup_(self, step, stepName, stepDict, k, properties):
if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']:
if '2023' in stepDict[step][k]['--conditions']:
stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2023'}, stepDict[step][k]])
else:
stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2021'}, stepDict[step][k]])
elif 'Phase2' in stepDict[step][k]['--era']:
if 'Phase2' in stepDict[step][k]['--era']:
dd4hepGeom="DD4hep"
dd4hepGeom+=stepDict[step][k]['--geometry']
stepDict[stepName][k] = merge([{'--geometry' : dd4hepGeom, '--procModifiers': 'dd4hep'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return ('2021' in key or '2023' in key or '2026' in key) and ('FS' not in key)
return ('2026' in key) and ('FS' not in key)
upgradeWFs['DD4hep'] = UpgradeWorkflow_DD4hep(
steps = [
'GenSim',
Expand Down Expand Up @@ -2835,7 +2879,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']:
stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic', '--geometry': 'DB:Extended'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return '2021' in key and 'FS' not in key
return fragment=="TTbar_14TeV" and '2021' in key and 'FS' not in key
upgradeWFs['DD4hepDB'] = UpgradeWorkflow_DD4hepDB(
steps = [
'GenSim',
Expand Down Expand Up @@ -2871,7 +2915,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
tmp_eras = ','.join(tmp_eras)
stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic_ddd', '--geometry': 'DB:Extended', '--era': tmp_eras}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return '2021' in key and 'FS' not in key
return fragment=="TTbar_14TeV" and '2021' in key and 'FS' not in key
upgradeWFs['DDDDB'] = UpgradeWorkflow_DDDDB(
steps = [
'GenSim',
Expand Down

0 comments on commit 1edffb7

Please sign in to comment.