Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FastSim premixing workflows and matrix cleanup #46007

Merged
merged 8 commits into from
Oct 4, 2024
Merged

FastSim premixing workflows and matrix cleanup #46007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
18fa2e6
avoid spurious workflows
kpedro88 Sep 13, 2024
741e106
remove excessive workflows
kpedro88 Sep 13, 2024
9c43dd6
remove unused parameters
kpedro88 Sep 13, 2024
1862a94
extend premix workflows to fastsim (run 3)
kpedro88 Sep 13, 2024
acb7e14
reorg for better full-fast separation, handle harvest
kpedro88 Sep 14, 2024
809035e
remove redundant step number change
kpedro88 Sep 27, 2024
ad685bb
fix mistake in regex
kpedro88 Sep 27, 2024
1ca6072
restrict dd4hep special workflow to Phase 2; restore for all fragments
kpedro88 Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Configuration/PyReleaseValidation/python/relval_upgrade.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ def notForGenOnly(key,specialType):
stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','HLT75e33'),specialWF.suffix))
# similar hacks for premixing
if 'PMX' in specialType:
if 'GenSim' in step:
s = step.replace('GenSim','Premix')+'PU' # later processing requires to have PU here
if 'GenSim' in step or 'Gen' in step:
s = step.replace('GenSim','Premix').replace('Gen','Premix')+'PU' # later processing requires to have PU here
if step in specialWF.PU:
stepMade = stepMaker(key,'PREMIX',s,specialWF.suffix)
# append for combined
Expand Down
158 changes: 101 additions & 57 deletions Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ def workflow(self, workflows, num, fragment, stepList, key, hasHarvest):
def workflow_(self, workflows, num, fragment, stepList, key):
fragmentTmp = [fragment, key]
if len(self.suffix)>0: fragmentTmp.append(self.suffix)
workflows[num+self.offset] = [ fragmentTmp, stepList ]
# avoid spurious workflows (no steps modified)
if self.offset==0 or workflows[num][1]!=stepList:
workflows[num+self.offset] = [ fragmentTmp, stepList ]
def condition(self, fragment, stepList, key, hasHarvest):
return False
def preventReuse(self, stepName, stepDict, k):
Expand Down Expand Up @@ -562,7 +564,7 @@ def condition(self, fragment, stepList, key, hasHarvest):

# WeightedMeanFitter vertexing workflows
class UpgradeWorkflow_weightedVertex(UpgradeWorkflow):
def __init__(self, reco = {}, harvest = {}, **kwargs):
def __init__(self, **kwargs):
# adapt the parameters for the UpgradeWorkflow init method
super(UpgradeWorkflow_weightedVertex, self).__init__(
steps = [
Expand Down Expand Up @@ -590,8 +592,6 @@ def __init__(self, reco = {}, harvest = {}, **kwargs):
'HARVESTNanoFakeHLT',
],
**kwargs)
self.__reco = reco
self.__harvest = harvest

def setup_(self, step, stepName, stepDict, k, properties):
# temporarily remove trigger & downstream steps
Expand Down Expand Up @@ -2264,7 +2264,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
stepDict[stepName][k] = None

def condition(self, fragment, stepList, key, hasHarvest):
return ('2021' in key or '2023' in key or '2026' in key)
return fragment=="TTbar_14TeV" and ('2021' in key or '2023' in key or '2026' in key)

upgradeWFs['ECALComponent'] = UpgradeWorkflow_ECalComponent(
suffix = '_ecalComponent',
Expand Down Expand Up @@ -2520,58 +2520,103 @@ def setup_(self, step, stepName, stepDict, k, properties):
# just copy steps
stepDict[stepName][k] = merge([stepDict[step][k]])
def setupPU_(self, step, stepName, stepDict, k, properties):
# setup for stage 1
if "GenSim" in stepName:
stepNamePmx = stepName.replace('GenSim','Premix')
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
stepDict[stepNamePmx][k] = merge([
{
'-s': 'GEN,SIM,DIGI:pdigi_valid',
'--datatier': 'PREMIX',
'--eventcontent': 'PREMIX',
'--procModifiers': 'premix_stage1'
},
stepDict[stepName][k]
])
if "ProdLike" in self.suffix:
stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]])
# setup for stage 2
elif "Digi" in step or "Reco" in step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if d is None: return
if "Digi" in step:
# fastsim version
if 'FS' in k:
Copy link
Contributor

@AdrianoDee AdrianoDee Oct 1, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Having the if not 'FS' in k: branch first would definitely simplified the diff here :-)

# setup for stage 1 fastsim
if "Gen" in stepName:
stepNamePmx = stepName.replace('Gen','Premix')
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
stepDict[stepNamePmx][k] = merge([
{
'-s': 'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid',
'--fast':'',
'--datatier': 'PREMIX',
'--eventcontent': 'PREMIX',
'--procModifiers': 'premix_stage1'
},
stepDict[stepName][k]
])
if "ProdLike" in self.suffix:
# todo
pass
# setup for stage 2 fastsim
elif "FastSimRun3" in step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if d is None: return
tmpsteps = []
for s in d["-s"].split(","):
if s == "DIGI" or "DIGI:" in s:
tmpsteps.extend([s, "DATAMIX"])
else:
tmpsteps.append(s)
d = merge([{"-s" : ",".join(tmpsteps),
"--datamix" : "PreMix",
"--procModifiers": "premix_stage2"},
"--datamix" : "PreMix"},
d])
# for combined stage1+stage2
if "_PMXS1S2" in self.suffix:
d = merge([digiPremixLocalPileup, d])
elif "Reco" in step:
if "--procModifiers" in d:
d["--procModifiers"] += ",premix_stage2"
else:
d["--procModifiers"] = "premix_stage2"
stepDict[stepName][k] = d
# Increase the input file step number by one for Nano in combined stage1+stage2
elif "Nano"==step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if "--filein" in d:
filein = d["--filein"]
m = re.search("step(?P<ind>\\d+)_", filein)
if m:
d["--filein"] = filein.replace(m.group(), "step%d_"%(int(m.group("ind"))+1))
stepDict[stepName][k] = d
# run2/3 WFs use Nano (not NanoPU) in PU WF
stepDict[self.getStepName(step)][k] = merge([d])
# for combined stage1+stage2
if "_PMXS1S2" in self.suffix:
d = merge([digiPremixLocalPileup, d])
stepDict[stepName][k] = d
elif "HARVESTFastRun3" in step:
# increment input step number
stepDict[stepName][k] = merge([{'--filein':'file:step3_inDQM.root'},stepDict[stepName][k]])
else:
# setup for stage 1
if "GenSim" in stepName:
stepNamePmx = stepName.replace('GenSim','Premix')
if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {}
stepDict[stepNamePmx][k] = merge([
{
'-s': 'GEN,SIM,DIGI:pdigi_valid',
'--datatier': 'PREMIX',
'--eventcontent': 'PREMIX',
'--procModifiers': 'premix_stage1'
},
stepDict[stepName][k]
])
if "ProdLike" in self.suffix:
stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]])
# setup for stage 2
elif "Digi" in step or "Reco" in step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if d is None: return
if "Digi" in step:
tmpsteps = []
for s in d["-s"].split(","):
if s == "DIGI" or "DIGI:" in s:
tmpsteps.extend([s, "DATAMIX"])
else:
tmpsteps.append(s)
d = merge([{"-s" : ",".join(tmpsteps),
"--datamix" : "PreMix",
"--procModifiers": "premix_stage2"},
d])
# for combined stage1+stage2
if "_PMXS1S2" in self.suffix:
d = merge([digiPremixLocalPileup, d])
elif "Reco" in step:
if "--procModifiers" in d:
d["--procModifiers"] += ",premix_stage2"
else:
d["--procModifiers"] = "premix_stage2"
stepDict[stepName][k] = d
# separate nano step now only used in ProdLike workflows for Run3/Phase2
elif "Nano"==step:
# go back to non-PU step version
d = merge([stepDict[self.getStepName(step)][k]])
if "_PMXS1S2" in self.suffix and "--filein" in d:
filein = d["--filein"]
m = re.search("step(?P<ind>\\d+)", filein)
if m:
d["--filein"] = filein.replace(m.group(), "step%d"%(int(m.group("ind"))+1))
stepDict[stepName][k] = d
# run2/3 WFs use Nano (not NanoPU) in PU WF
stepDict[self.getStepName(step)][k] = merge([d])
def condition(self, fragment, stepList, key, hasHarvest):
if not 'PU' in key:
return False
Expand All @@ -2590,6 +2635,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
steps = [
],
PU = [
'Gen',
'GenSim',
'GenSimHLBeamSpot',
'GenSimHLBeamSpot14',
Expand All @@ -2610,6 +2656,8 @@ def workflow_(self, workflows, num, fragment, stepList, key):
'RecoNano',
'RecoNanoFakeHLT',
'Nano',
'FastSimRun3',
'HARVESTFastRun3',
],
suffix = '_PMXS2',
offset = 0.98,
Expand All @@ -2618,6 +2666,7 @@ def workflow_(self, workflows, num, fragment, stepList, key):
upgradeWFs['PMXS1S2'] = UpgradeWorkflowPremix(
steps = [],
PU = [
'Gen',
'GenSim',
'GenSimHLBeamSpot',
'GenSimHLBeamSpot14',
Expand All @@ -2630,6 +2679,8 @@ def workflow_(self, workflows, num, fragment, stepList, key):
'RecoNano',
'RecoNanoFakeHLT',
'Nano',
'FastSimRun3',
'HARVESTFastRun3',
],
suffix = '_PMXS1S2',
offset = 0.99,
Expand Down Expand Up @@ -2683,8 +2734,6 @@ def setup_(self, step, stepName, stepDict, k, properties):
"--eventcontent": "PREMIXRAW"},
d])
stepDict[stepName][k] = d
if 'Nano'==step:
stepDict[stepName][k] = merge([{'--filein':'file:step5.root','-s':'NANO','--datatier':'NANOAODSIM','--eventcontent':'NANOEDMAODSIM'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
# use both conditions
return UpgradeWorkflowPremix.condition(self, fragment, stepList, key, hasHarvest) and UpgradeWorkflow_ProdLike.condition(self, fragment, stepList, key, hasHarvest)
Expand Down Expand Up @@ -2750,7 +2799,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
else:
stepDict[stepName][k] = merge([stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return ('FS' in key)
return fragment=="TTbar_14TeV" and ('FS' in key)
upgradeWFs['Run3FStrackingOnly'] = UpgradeWorkflow_Run3FStrackingOnly(
steps = [
'Gen',
Expand Down Expand Up @@ -2792,17 +2841,12 @@ def condition(self, fragment, stepList, key, hasHarvest):

class UpgradeWorkflow_DD4hep(UpgradeWorkflow):
def setup_(self, step, stepName, stepDict, k, properties):
if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']:
if '2023' in stepDict[step][k]['--conditions']:
stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2023'}, stepDict[step][k]])
else:
stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2021'}, stepDict[step][k]])
elif 'Phase2' in stepDict[step][k]['--era']:
if 'Phase2' in stepDict[step][k]['--era']:
dd4hepGeom="DD4hep"
dd4hepGeom+=stepDict[step][k]['--geometry']
stepDict[stepName][k] = merge([{'--geometry' : dd4hepGeom, '--procModifiers': 'dd4hep'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return ('2021' in key or '2023' in key or '2026' in key) and ('FS' not in key)
return ('2026' in key) and ('FS' not in key)
upgradeWFs['DD4hep'] = UpgradeWorkflow_DD4hep(
steps = [
'GenSim',
Expand Down Expand Up @@ -2835,7 +2879,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']:
stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic', '--geometry': 'DB:Extended'}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return '2021' in key and 'FS' not in key
return fragment=="TTbar_14TeV" and '2021' in key and 'FS' not in key
upgradeWFs['DD4hepDB'] = UpgradeWorkflow_DD4hepDB(
steps = [
'GenSim',
Expand Down Expand Up @@ -2871,7 +2915,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
tmp_eras = ','.join(tmp_eras)
stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic_ddd', '--geometry': 'DB:Extended', '--era': tmp_eras}, stepDict[step][k]])
def condition(self, fragment, stepList, key, hasHarvest):
return '2021' in key and 'FS' not in key
return fragment=="TTbar_14TeV" and '2021' in key and 'FS' not in key
upgradeWFs['DDDDB'] = UpgradeWorkflow_DDDDB(
steps = [
'GenSim',
Expand Down