Skip to content

Commit

Permalink
Merge pull request cms-sw#49 from hatakeyamak/pfvalidation-10_6_X-master
Browse files Browse the repository at this point in the history 
Pfvalidation 10 6 x master
  • Loading branch information
@hatakeyamak
hatakeyamak authored Jun 4, 2019
2 parents 4fd49af + c7b36aa commit 3085648
Show file tree
Hide file tree
Showing 13 changed files with 270 additions and 118,064 deletions.
41 changes: 12 additions & 29 deletions Validation/RecoParticleFlow/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@ TMPDIR=tmp
RELVALCMD=${CMSSW_BASE}/src/Validation/RecoParticleFlow/test/run_relval.sh
DQM_MC=DQM_V0001_R000000001__Global__CMSSW_X_Y_Z__RECO

conf:
cd ${TMPDIR} && ${RELVALCMD} conf reco 0

dumpconf:
cd ${TMPDIR}/conf && python -c 'import step3; print step3.process.dumpPython()' > step3_dump.py
cp ${TMPDIR}/conf/step3.py crab/
cp ${TMPDIR}/conf/step3_dump.py crab/

QCD: QCD_reco QCD_dqm

QCD_reco:
Expand All @@ -21,8 +29,11 @@ ZMM_reco:
MinBias_reco:
cd ${TMPDIR} && ${RELVALCMD} MinBias reco 0

NuGunPU_reco:
cd ${TMPDIR} && ${RELVALCMD} NuGunPU reco 0

#dqm: QCD_dqm QCDPU_dqm ZMM_dqm MinBias_dqm SN_dqm
dqm: QCD_dqm QCDPU_dqm ZMM_dqm NuGunPU_dqm
dqm: QCD_dqm QCDPU_dqm NuGunPU_dqm

QCD_dqm:
rm -f ${TMPDIR}/QCD/DQM*.root
Expand All @@ -44,34 +55,6 @@ NuGunPU_dqm:
rm -f ${TMPDIR}/NuGunPU/DQM*.root
cd ${TMPDIR} && ${RELVALCMD} NuGunPU dqm 0

#post: QCD_post QCDPU_post ZMM_post MinBias_post SN_post
#post: QCD_post QCDPU_post NuGunPU_post

#QCD_post:
# rm -f ${TMPDIR}/QCD/*withResponse*.root ${TMPDIR}/QCD/DQMTotal.root
# cd ${TMPDIR}/QCD && python ${CMSSW_BASE}/src/Validation/RecoParticleFlow/test/addResponseAndResolution.py DQM*.root
# hadd -f ${TMPDIR}/QCD/DQMTotal.root ${TMPDIR}/QCD/DQM_*.root

#QCDPU_post:
# rm -f ${TMPDIR}/QCDPU/*withResponse*.root ${TMPDIR}/QCDPU/DQMTotal.root
# cd ${TMPDIR}/QCDPU && python ${CMSSW_BASE}/src/Validation/RecoParticleFlow/test/addResponseAndResolution.py DQM*.root
# hadd -f ${TMPDIR}/QCDPU/DQMTotal.root ${TMPDIR}/QCDPU/DQM_*.root

#ZMM_post:
# rm -f ${TMPDIR}/ZMM/*withResponse*.root ${TMPDIR}/ZMM/DQMTotal.root
# cd ${TMPDIR}/ZMM && python ${CMSSW_BASE}/src/Validation/RecoParticleFlow/test/addResponseAndResolution.py DQM*.root
# hadd -f ${TMPDIR}/ZMM/DQMTotal.root ${TMPDIR}/ZMM/DQM_*.root

#MinBias_post:
# rm -f ${TMPDIR}/MinBias/*withResponse*.root ${TMPDIR}/MinBias/DQMTotal.root
# cd ${TMPDIR}/MinBias && python ${CMSSW_BASE}/src/Validation/RecoParticleFlow/test/addResponseAndResolution.py DQM*.root
# hadd -f ${TMPDIR}/MinBias/DQMTotal.root ${TMPDIR}/MinBias/DQM_*.root

#NuGunPU_post:
# rm -f ${TMPDIR}/NoGunPU/*withResponse*.root ${TMPDIR}/NuGunPU/DQMTotal.root
# cd ${TMPDIR}/NuGunPU && python ${CMSSW_BASE}/src/Validation/RecoParticleFlow/test/addResponseAndResolution.py DQM*.root
# hadd -f ${TMPDIR}/NuGunPU/DQMTotal.root ${TMPDIR}/NuGunPU/DQM_*.root

.PHONY: plots # Enable re-running plots

plots:
Expand Down
36 changes: 20 additions & 16 deletions Validation/RecoParticleFlow/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ cmsenv
#get the code
git cms-checkout-topic jpata:pfvalidation-10_6_X-master
# 'git cms-rebase-topic jpata:pfvalidation-10_6_X-master' might be better when using 10_6_0_patchX or higher version
scram b -j4
cd $CMSSW_BASE/src/Validation/RecoParticleFlow
Expand All @@ -21,20 +22,21 @@ voms-proxy-init -voms cms
#RECO step, about 30 minutes
#Necessary if you need to re-reco events to test introduced changes to PF reco.
#Note that the default era & condition is now set to 2018. Change CONDITIONS and ERA in test/run_relval.sh when trying other era, before trying the above commands.
make QCD_reco
#DQM step, a few minutes
make QCD_dqm
# Repeat for QCDPU & NuGunPU (make QCDPU_reco etc.) or edit the 'make plots' part of
# Makefile for successfully running 'make plots'
# Repeat for QCDPU & NuGunPU (make QCDPU_reco, make QCDPU_dqm etc.) or use crab for reco and run dqm steps as indicated below.
# Or, edit the 'make plots' part of Makefile for successfully running 'make plots'
#Do final HTML plots
#Do final HTML plots (this just plot two identical results in tmp/{QCD,QCDPU,NuGunPU} twice)
make plots
# or if you have tmp/QCD_ref, tmp/QCDPU_ref, tmp/NuGunPU_ref (i.e. reference results) etc under _tmp area
#
# or if you have reference DQM results in tmp/QCD_ref, tmp/QCDPU_ref, tmp/NuGunPU_ref (i.e. reference results) etc under _tmp area:
make pltos_with_ref
# The 'plots' directory can be viewed from a web browser once it is moved to e.g. /afs/cern.ch/user/f/foo/www/.
# In this case the URL for the directory is 'http://cern.ch/foo/plots', where 'foo' is the username
~~~
Expand All @@ -44,10 +46,13 @@ make pltos_with_ref

The reco step can also be run via Crab, using
~~~
make conf
make dumpconf
cd crab
python multicrab.py
~~~

Note that the default era, condition, and samples are now set to 2018. Change CONDITIONS and ERA in test/run_relval.sh when trying other era, before trying the above commands. Also check (and if necessary, update) input samples and conf.Site.storageSite specified in $CMSSW_BASE/src/Validation/RecoParticleFlow/crab/multicrab.py.
Take note that the CMSSW python configuration for running the RECO sequence is dumped into `crab/step3_dump.py`.


Expand All @@ -56,15 +61,11 @@ Take note that the CMSSW python configuration for running the RECO sequence is d
~~~
# For example (default for 2018):
#CONDITIONS=auto:phase1_2017_realistic
#ERA=Run2_2017
CONDITIONS=auto:phase1_2018_realistic
ERA=Run2_2018
#CONDITIONS=auto:phase2_realistic
#ERA=Phase2C8_timing_layer_bar
#CONDITIONS=auto:phase1_2017_realistic ERA=Run2_2017 # for 2017 scenarios
CONDITIONS=auto:phase1_2018_realistic ERA=Run2_2018 # for 2018 scenarios
#CONDITIONS=auto:phase2_realistic ERA=Phase2C8_timing_layer_bar # for phase2
#Running with 2 threads allows to use more memory on grid
NTHREADS=2
TMPDIR=tmp
NTHREADS=2 TMPDIR=tmp
cd $CMSSW_BASE/src/Validation/RecoParticleFlow
make -p tmp/QCD; cd tmp/QCD
Expand All @@ -73,16 +74,19 @@ make -p tmp/QCDPU; cd tmp/QCDPU
make -p tmp/NuGunPU; cd tmp/NuGunPU
#)
# make a text file for input files
# make a text file for input files. For example:
dasgoclient --query="file dataset=/RelValQCD_FlatPt_15_3000HS_13/CMSSW_10_6_0-106X_upgrade2018_realistic_v4-v1/MINIAODSIM" > step3_filelist.txt
#(or
dasgoclient --query="file dataset=/RelValQCD_FlatPt_15_3000HS_13/CMSSW_10_6_0-PU25ns_106X_upgrade2018_realistic_v4-v1/MINIAODSIM" > step3_filelist.txt
dasgoclient --query="file dataset=/RelValNuGun/CMSSW_10_6_0-PU25ns_106X_upgrade2018_realistic_v4-v1/MINIAODSIM" > step3_filelist.txt
dasgoclient --query="file dataset=/RelValQCD_Pt-15To7000_Flat_14TeV/CMSSW_10_6_0-106X_upgrade2023_realistic_v2_2023D41noPU-v2/MINIAODSIM" > step3_filelist.txt
dasgoclient --query="file dataset=/RelValQCD_Pt-15To7000_Flat_14TeV/CMSSW_10_6_0-PU25ns_106X_upgrade2023_realistic_v2_2023D41PU200-v1/MINIAODSIM" > step3_filelist.txt
dasgoclient --query="file dataset=/RelValNuGun/CMSSW_10_6_0-PU25ns_106X_upgrade2023_realistic_v2_2023D41PU200-v1/MINIAODSIM" > step3_filelist.txt
or using the list of files from your crab output areas.
#)
cat step3_filelist.txt
cmsDriver.py step5 --conditions $CONDITIONS -s DQM:@pfDQM --datatier DQMIO --nThreads $NTHREADS --era $ERA --eventcontent DQM --filein filelist:step3_filelist.txt --fileout file:step5.root -n -1 2>&1 | tee step5.log
cmsDriver.py step6 --conditions $CONDITIONS -s HARVESTING:@pfDQM --era $ERA --filetype DQM --filein file:step5.root --fileout file:step6.root
cmsDriver.py step5 --conditions $CONDITIONS -s DQM:@pfDQM --datatier DQMIO --nThreads $NTHREADS --era $ERA --eventcontent DQM --filein filelist:step3_filelist.txt --fileout file:step5.root -n -1 >& step5.log &
# after step5 is completed:
cmsDriver.py step6 --conditions $CONDITIONS -s HARVESTING:@pfDQM --era $ERA --filetype DQM --filein file:step5.root --fileout file:step6.root >& step6.log &
52 changes: 52 additions & 0 deletions Validation/RecoParticleFlow/crab/multicrab_phase2.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
from CRABAPI.RawCommand import crabCommand
from CRABClient.UserUtilities import getUsernameFromSiteDB
from CRABClient.UserUtilities import config
from copy import deepcopy
import os

def submit(config):
res = crabCommand('submit', config = config)
#save crab config for the future
with open(config.General.workArea + "/crab_" + config.General.requestName + "/crab_config.py", "w") as fi:
fi.write(config.pythonise_())

samples = [
("/RelValQCD_Pt-15To7000_Flat_14TeV/CMSSW_10_6_0-106X_upgrade2023_realistic_v2_2023D41noPU-v2/GEN-SIM-DIGI-RAW", "QCD_noPU_phase2"),
(" /RelValQCD_Pt-15To7000_Flat_14TeV/CMSSW_10_6_0-PU25ns_106X_upgrade2023_realistic_v2_2023D41PU200-v1/GEN-SIM-DIGI-RAW", "QCD_PU_phase2"),
#("/RelValZMM_14/CMSSW_10_6_0-106X_upgrade2023_realistic_v2_2023D41noPU-v2/GEN-SIM-DIGI-RAW", "ZMM_phase2"),
("/RelValNuGun/CMSSW_10_6_0-PU25ns_106X_upgrade2023_realistic_v2_2023D41PU200-v1/GEN-SIM-DIGI-RAW", "NuGun_PU_phase2"),
]

if __name__ == "__main__":
for dataset, name in samples:

if os.path.isfile("step3_dump.pyc"):
os.remove("step3_dump.pyc")

conf = config()

conf.General.requestName = name
conf.General.transferLogs = True
conf.General.workArea = 'crab_projects'
conf.JobType.pluginName = 'Analysis'
conf.JobType.psetName = 'step3_dump.py'
conf.JobType.maxJobRuntimeMin = 4*60
conf.JobType.allowUndistributedCMSSW = True
conf.JobType.outputFiles = ["step3_inMINIAODSIM.root"]
conf.JobType.maxMemoryMB = 4000
conf.JobType.numCores = 2

conf.Data.inputDataset = dataset
conf.Data.splitting = 'LumiBased'
conf.Data.unitsPerJob = 10
#conf.Data.totalUnits = 50
conf.Data.publication = False
conf.Data.outputDatasetTag = 'pfvalidation'
#conf.Data.ignoreLocality = True

# Where the output files will be transmitted to
conf.Site.storageSite = 'T3_US_Baylor'
#conf.Site.storageSite = 'T2_US_Caltech'
#conf.Site.whitelist = ["T2_US_Caltech", "T2_CH_CERN"]

submit(conf)
Loading

0 comments on commit 3085648

Please sign in to comment.