Merge jpata/master into master

.github/workflows/test.yml

@@ -9,32 +9,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  delphes-tf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install python deps
-        run: |
-          sudo apt install python3 python3-pip wget
-          sudo python3 -m pip install --upgrade pip
-          sudo python3 -m pip install --upgrade setuptools
-          sudo python3 -m pip install tensorflow==2.4 setGPU sklearn matplotlib mplhep pandas scipy uproot3 uproot3-methods awkward0 keras-tuner networkx tensorflow-probability==0.12.2 tensorflow-addons==0.13.0 tqdm
-      - name: Run delphes TF model
-        run: ./scripts/local_test_delphes_tf.sh
-
-  cms-tf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install python deps
-        run: |
-          sudo apt install python3 python3-pip wget
-          sudo python3 -m pip install --upgrade pip
-          sudo python3 -m pip install --upgrade setuptools
-          sudo python3 -m pip install tensorflow==2.4 setGPU sklearn matplotlib mplhep pandas scipy uproot3 uproot3-methods awkward0 keras-tuner networkx tensorflow-probability==0.12.2 tensorflow-addons==0.13.0 tqdm
-      - name: Run CMS TF model
-        run: ./scripts/local_test_cms_tf.sh
-
   delphes-pipeline:
     runs-on: ubuntu-latest
     steps:
@@ -44,7 +18,7 @@ jobs:
           sudo apt install python3 python3-pip wget
           sudo python3 -m pip install --upgrade pip
           sudo python3 -m pip install --upgrade setuptools
-          sudo python3 -m pip install tensorflow==2.4 setGPU sklearn matplotlib mplhep pandas scipy uproot3 uproot3-methods awkward0 keras-tuner networkx tensorflow-probability==0.12.2 tensorflow-addons==0.13.0 tqdm click
+          sudo python3 -m pip install tensorflow==2.4 setGPU sklearn matplotlib mplhep pandas scipy uproot3 uproot3-methods awkward0 keras-tuner networkx tensorflow-probability==0.12.2 tensorflow-addons==0.13.0 tqdm click 'ray[default]' 'ray[tune]'
       - name: Run delphes TF model
         run: ./scripts/local_test_delphes_pipeline.sh
 
@@ -57,7 +31,7 @@ jobs:
           sudo apt install python3 python3-pip wget
           sudo python3 -m pip install --upgrade pip
           sudo python3 -m pip install --upgrade setuptools
-          sudo python3 -m pip install tensorflow==2.4 setGPU sklearn matplotlib mplhep pandas scipy uproot3 uproot3-methods awkward0 keras-tuner networkx tensorflow-probability==0.12.2 tensorflow-addons==0.13.0 tqdm click
+          sudo python3 -m pip install tensorflow==2.4 setGPU sklearn matplotlib mplhep pandas scipy uproot3 uproot3-methods awkward0 keras-tuner networkx tensorflow-probability==0.12.2 tensorflow-addons==0.13.0 tqdm click 'ray[default]' 'ray[tune]'
       - name: Run CMS TF model using the pipeline
         run: ./scripts/local_test_cms_pipeline.sh
 

.gitignore

@@ -13,3 +13,12 @@ mlpf/pytorch/data
 test_tmp/
 test_tmp_delphes/
 .DS_Store
+
+prp
+*.pyc
+*.pyo
+
+mlpf/updated/LRP/pid*
+mlpf/updated/LRP/class*
+
+*.ipynb_checkpoints

README.md

@@ -16,7 +16,7 @@
 Short instructions with a single test file in [notebooks/delphes-tf-mlpf-quickstart.ipynb](notebooks/delphes-tf-mlpf-quickstart.ipynb).
 
 Long instructions for reproducing the full training from scratch in [README_delphes.md](README_delphes.md).
-The plots can be generated using the notebook [delphes/resolution_checks.ipynb](delphes/resolution_checks.ipynb).
+The plots can be generated using the notebook [delphes/delphes_model_analysis.ipynb](delphes/delphes_model_analysis.ipynb).
 
 ### Delphes dataset
 The dataset is available from zenodo: https://doi.org/10.5281/zenodo.4559324.

README_cms.md

@@ -33,5 +33,5 @@ git clone https://github.com/jpata/particleflow.git
 cd particleflow
 
 #run a small local test including data prep and training
-./scripts/local_test_cms_tf.sh
+./scripts/local_test_cms_pipeline.sh
 ```

README_delphes.md

@@ -2,28 +2,55 @@
 
 The following instructions use singularity, but you may have a different local setup.
 
+```bash
+#Download all pkl.bz2 files from https://zenodo.org/record/4559324
+
+#now move the data into the right place
+mv *pythia8_qcd*.pkl.bz2 data/pythia8_qcd/val
+mv *pythia8_ttbar*.pkl.bz2 data/pythia8_qcd/raw
+mv data/pythia8_qcd/raw/*pythia8_ttbar_9_*.pkl.bz2 data/pythia8_qcd/val
+
+# Generate the TFRecord datasets needed for larger-than-RAM training
+python3 mlpf/pipeline.py data -c parameters/delphes.yaml
+
+# Run the training of the base GNN model using e.g. 5 GPUs in a data-parallel mode
+CUDA_VISIBLE_DEVICES=0,1,2,3,4 python3 mlpf/pipeline.py train -c parameters/delphes.yaml
+
+#Run the validation to produce the predictions file
+python3 mlpf/pipeline.py evaluate -c parameters/delphes.yaml -t experiments/delphes_* -v "data/pythia8_qcd/val/*.pkl.bz2" -e evaluate_qcd
+python3 mlpf/pipeline.py evaluate -c parameters/delphes.yaml -t experiments/delphes_* -v "data/pythia8_ttbar/val/*.pkl.bz2" -e evaluate_ttbar
+```
+
+## Recipe for generation
+The Delphes AngularSmearing module has been modified to correctly take into account the smearing for tracks, see [delphes/install.sh](delphes/install.sh).
+
+```bash
+wget http://atlaswww.hep.anl.gov/hepsim/soft/centos7hepsim.img
+sudo singularity build --sandbox centos7hepsim.sandbox centos7hepsim.img
+sudo singularity exec -B /home --writable centos7hepsim.sandbox ./install.sh
+sudo singularity build centos7hepsim.sif centos7hepsim.sandbox
+sudo rm -Rf centos7hepsim.sandbox
+```
+
 ```bash
 cd delphes
 
 # Run the simulation step
 # Generate events with pythia, mix them with PU and run a detector simulation using Delphes
-singularity exec http://jpata.web.cern.ch/jpata/centos7hepsim.sif ./run_sim.sh
+singularity exec centos7hepsim.sif ./run_sim.sh
 
 # Run the ntuplization step
 # generate X,y input matrices for NN training in out/pythia8_ttbar/*.pkl.bz2
-singularity exec http://jpata.web.cern.ch/jpata/centos7hepsim.sif ./run_ntuple.sh
-singularity exec http://jpata.web.cern.ch/jpata/centos7hepsim.sif ./run_ntuple_qcd.sh
+singularity exec centos7hepsim.sif ./run_ntuple.sh
+singularity exec centos7hepsim.sif ./run_ntuple_qcd.sh
 
-#Alternatively, to skip run_sim.sh and run_ntuple.sh, download everything from https://doi.org/10.5281/zenodo.4452283 and put into out/pythia8_ttbar
-
-#now move the data into the right place
 mv out/pythia8_ttbar ../data/
 cd ../data/pythia8_ttbar
 mkdir raw
 mkdir val
 mkdir root
 mv *.root root/
-mb *.promc root/
+mv *.promc root/
 mv *.pkl.bz2 raw/
 cd ../..
 
@@ -35,26 +62,4 @@ mv *.root root/
 mv *.promc root/
 mv *.pkl.bz2 val/
 cd ../..
-
-# Generate the TFRecord datasets needed for larger-than-RAM training
-singularity exec --nv http://jpata.web.cern.ch/jpata/base.simg python3 mlpf/launcher.py --action data --model-spec parameters/delphes-gnn-skipconn.yaml
-
-# Run the training of the base GNN model using e.g. 5 GPUs in a data-parallel mode
-CUDA_VISIBLE_DEVICES=0,1,2,3,4 singularity exec --nv http://jpata.web.cern.ch/jpata/base.simg python3 mlpf/launcher.py --action train --model-spec parameters/delphes-gnn-skipconn.yaml
-
-#Run the validation to produce the predictions file
-singularity exec --nv http://jpata.web.cern.ch/jpata/base.simg python3 mlpf/launcher.py --action eval --model-spec parameters/delphes-gnn-skipconn.yaml --weights ./experiments/delphes-gnn-skipconn-*/weights-300-*.hdf5
-
-singularity exec --nv http://jpata.web.cern.ch/jpata/base.simg python3 mlpf/launcher.py --action time --model-spec parameters/delphes-gnn-skipconn.yaml --weights ./experiments/delphes-gnn-skipconn-*/weights-300-*.hdf5
-```
-
-## Recipe to prepare Delphes singularity image
-NB: The Delphes AngularSmearing module has been modified to correctly take into account the smearing for tracks, see [delphes/install.sh](delphes/install.sh)
-
-```bash
-wget http://atlaswww.hep.anl.gov/hepsim/soft/centos7hepsim.img
-sudo singularity build --sandbox centos7hepsim.sandbox centos7hepsim.img
-sudo singularity exec -B /home --writable centos7hepsim.sandbox ./install.sh
-sudo singularity build centos7hepsim.sif centos7hepsim.sandbox
-sudo rm -Rf centos7hepsim.sandbox
 ```

clic/dumper.py

@@ -61,7 +61,8 @@ def pfParticleToDict(par):
         "px": mom[0],
         "py": mom[1],
         "pz": mom[2],
-        "energy": par.getEnergy()
+        "energy": par.getEnergy(),
+        "charge": par.getCharge()
     }
     return vec
 
@@ -210,6 +211,7 @@ def caloHitToDict(par, calohit_to_cluster, genparticle_dict, calohit_recotosim):
         nPF=colPF.getNumberOfElements()
         nCl=colCl.getNumberOfElements()
         nTr=colTr.getNumberOfElements()
+        nHit=simTrackHits.getNumberOfElements()
         nHCB=colHCB.getNumberOfElements()
         nHCE=colHCE.getNumberOfElements()
         nECB=colECB.getNumberOfElements()
@@ -223,7 +225,7 @@ def caloHitToDict(par, calohit_to_cluster, genparticle_dict, calohit_recotosim):
             assert(not (recohit in calohit_recotosim))
             calohit_recotosim[recohit] = simhit
 
-        print "Event %d, nGen=%d, nPF=%d, nClusters=%d, nTracks=%d, nHCAL=%d, nECAL=%d" % (nEvent, nMc, nPF, nCl, nTr, nHCB+nHCE, nECB+nECE)
+        print "Event %d, nGen=%d, nPF=%d, nClusters=%d, nTracks=%d, nHCAL=%d, nECAL=%d, nHits=%d" % (nEvent, nMc, nPF, nCl, nTr, nHCB+nHCE, nECB+nECE, nHit)
 
         genparticles = []
         genparticle_dict = {}