ci: Add CKF pull diagnostics to physics perf monitoring (#2061)

CI/physmon/phys_perf_mon.sh

@@ -76,6 +76,26 @@ function full_chain() {
         --title "IVF ${suffix}" \
         -o $outdir/ivf_${suffix}.html \
         -p $outdir/ivf_${suffix}_plots
+
+    Examples/Scripts/generic_plotter.py \
+        $outdir/tracksummary_ckf_${suffix}.root \
+        tracksummary \
+        $outdir/tracksummary_ckf_${suffix}_hist.root \
+        --silent \
+        --config CI/physmon/tracksummary_ckf_config.yml
+    ec=$(($ec | $?))
+
+    # remove ntuple file because it's large
+    rm $outdir/tracksummary_ckf_${suffix}.root
+
+    run \
+        $outdir/tracksummary_ckf_${suffix}_hist.root \
+        $refdir/tracksummary_ckf_${suffix}_hist.root \
+        --title "Track Summary CKF ${suffix}" \
+        -o $outdir/tracksummary_ckf_${suffix}.html \
+        -p $outdir/tracksummary_ckf_${suffix}_plots
+
+
 }
 
 full_chain truth_smeared

CI/physmon/physmon.py

@@ -228,7 +228,11 @@ def run_ckf_tracking(truthSmearedSeeded, truthEstimatedSeeded, label):
         s.run()
         del s
 
-        for stem in ["performance_ckf", "performance_vertexing"] + (
+        for stem in [
+            "performance_ckf",
+            "tracksummary_ckf",
+            "performance_vertexing",
+        ] + (
             ["performance_seeding", "performance_ambi"]
             if label in ["seeded", "orthogonal"]
             else ["performance_seeding"]

CI/physmon/tracksummary_ckf_config.yml

@@ -0,0 +1,173 @@
+histograms:
+  "chi2Sum":
+    min: 0
+    max: 100
+
+  "pull_.*":
+    nbins: 50
+    min: -6
+    max: 6
+
+  "eLOC0_fit":
+    min: -5
+    max: 5
+
+  eLOC1_fit:
+    min: -200
+    max: 200
+
+  eQOP_fit:
+    min: -2
+    max: 2
+
+  eTHETA_fit:
+    min: 0
+    max: 3.14
+
+  eT_fit:
+    min: -150
+    max: 150
+
+  err_eLOC0_fit:
+    min: 0
+    max: 0.2
+
+  err_eLOC1_fit:
+    min: 0
+    max: 0.8
+
+  err_ePHI_fit:
+    min: 0
+    max: 0.008
+
+  err_eQOP_fit:
+    min: 0
+    max: 0.1
+
+  err_eT_fit:
+    min: 0
+    max: 1
+
+  err_eTHETA_fit:
+    min: 0
+    max: 0.0004
+
+  ".*Chi2":
+    min: 0
+    max: 10
+
+  measurementLayer|measurementVolume|outlierLayer|outlierVolume:
+    nbins: 30
+    min: 0
+    max: 30
+
+  measurementVolume:
+    nbins: 30
+    min: 0
+    max: 30
+
+  NDF:
+    nbins: 50
+    min: 0
+    max: 50
+
+  nHoles|nMajorityHits|nMeasurements|nOutliers|nSharedHits|nStates:
+    nbins: 10
+    min: 0
+    max: 10
+
+
+  "res_eLOC0_fit|res_eLOC1_fit":
+    nbins: 80
+    min: -2
+    max: 2
+
+  "res_ePHI_fit|res_eTHETA_fit":
+    nbins: 80
+    min: -0.01
+    max: 0.01
+
+  res_eTHETA_fit:
+    nbins: 80
+    min: -0.001
+    max: 0.001
+
+  res_eQOP_fit:
+    nbins: 80
+    min: -0.01
+    max: 0.01
+
+  res_eT_fit:
+    nbins: 80
+    min: -100.0
+    max: 100
+
+  t_charge:
+    nbins: 100
+    min: -1.0
+    max: 1.0
+
+  t_time:
+    nbins: 100
+    min: -1.0
+    max: 1.0
+
+  "t_vx|t_vy":
+    nbins: 100
+    min: -0.04
+    max: 0.04
+
+  t_vz:
+    nbins: 100
+    min: -200.0
+    max: 200
+
+  "t_px|t_py|t_pz":
+    nbins: 100
+    min: -10.0
+    max: 10
+
+  t_theta:
+    nbins: 100
+    min: 0
+    max: 3.14
+
+  t_phi:
+    nbins: 100
+    min: -3.14
+    max: 3.14
+
+  t_eta:
+    nbins: 100
+    min: -2.7
+    max: 2.7
+
+  "t_p$":
+    nbins: 100
+    min: 1
+    max: 10
+
+  t_pT:
+    nbins: 100
+    min: 0
+    max: 10
+
+  t_d0:
+    nbins: 100
+    min: -0.05
+    max: 0.05
+
+  t_z0:
+    nbins: 100
+    min: -200.0
+    max: 200
+
+exclude:
+  - event_nr
+  - hasFittedParams
+  - majorityParticleId
+  - multiTraj_nr
+  - subTraj_nr
+
+
+

Examples/Scripts/generic_plotter.py

@@ -3,6 +3,7 @@
 from typing import Optional, Dict, List
 import re
 import enum
+import sys
 
 import uproot
 import typer
@@ -34,6 +35,7 @@ class Extra(HistConfig):
 class Config(Model):
     histograms: Dict[str, HistConfig] = pydantic.Field(default_factory=dict)
     extra_histograms: List[Extra] = pydantic.Field(default_factory=list)
+    exclude: List[str] = pydantic.Field(default_factory=list)
 
 
 class Mode(str, enum.Enum):
@@ -71,6 +73,7 @@ def main(
     silent: bool = typer.Option(
         False, "--silent", "-s", help="Do not print any output"
     ),
+    dump_yml: bool = typer.Option(False, help="Print axis ranges as yml"),
 ):
     """
     Script to plot all branches in a TTree from a ROOT file, with optional configurable binning and ranges.
@@ -91,10 +94,12 @@ def main(
     histograms = {}
 
     if not silent:
-        print(config.extra_histograms)
+        print(config.extra_histograms, file=sys.stderr)
 
     for df in tree.iterate(library="ak", how=dict):
         for col in df.keys():
+            if any([re.match(ex, col) for ex in config.exclude]):
+                continue
             h = histograms.get(col)
             values = awkward.flatten(df[col], axis=None)
 
@@ -104,6 +109,15 @@ def main(
                 for ex, data in config.histograms.items():
                     if re.match(ex, col):
                         found = data.copy()
+                        print(
+                            "Found HistConfig",
+                            ex,
+                            "for",
+                            col,
+                            ":",
+                            found,
+                            file=sys.stderr,
+                        )
 
                 if found is None:
                     found = HistConfig()
@@ -159,16 +173,29 @@ def main(
 
     for k, h in histograms.items():
         if not silent:
-            print(k, h.axes[0])
+            if dump_yml:
+                ax = h.axes[0]
+                s = """
+{k}:
+  nbins: {b}
+  min: {min}
+  max: {max}
+                """.format(
+                    k=k, b=len(ax.edges) - 1, min=ax.edges[0], max=ax.edges[-1]
+                )
+                print(s)
+            else:
+                print(k, h.axes[0])
         outfile[k] = h
 
         if plots is not None:
             fig, ax = matplotlib.pyplot.subplots()
 
-            h.plot(ax=ax)
+            h.plot(ax=ax, flow=None)
 
             fig.tight_layout()
             fig.savefig(str(plots / f"{k}.{plot_format}"))
+            matplotlib.pyplot.close()
 
 
 if __name__ == "__main__":