Updated default behavior of run-by-run QA according to agreement in QA meeting on 5/19

StRoot/PWGTools/BadRunQA/QA.py

@@ -14,7 +14,8 @@
 
 def segmentAndReject(runs, x, xerr, pen=1, min_size=10, gamma=None, stdRange=5, maxIter=100, 
                      useJMLR=False, useMAD=False, weights=None, segmentOnce=False, 
-                     merge=False, reCalculateNormal=False, legacy=False, globalRejection=False, **kwargs):
+                     merge=False, reCalculateNormal=False, legacy=False, globalRejection=False, 
+                     quadRange=False, **kwargs):
     if useJMLR:
         print('Execution with JMLR')
     else:
@@ -29,7 +30,7 @@ def segmentAndReject(runs, x, xerr, pen=1, min_size=10, gamma=None, stdRange=5,
     edgeRuns = []
     i = 0
     runRj, reasonRj, mean, std = outlierDetector(runs_copy, x_copy, xerr_copy, edgeRuns, stdRange=stdRange, 
-                                                 useMAD=useMAD, weights=weights, legacy=legacy, seqRej=False)
+                                                 useMAD=useMAD, weights=weights, legacy=legacy, seqRej=False, quadRange=quadRange)
     if globalRejection and runRj.shape[0] > 0:
         runsRejected.append(runRj)
         reasonsRejected.append(reasonRj)
@@ -58,7 +59,7 @@ def segmentAndReject(runs, x, xerr, pen=1, min_size=10, gamma=None, stdRange=5,
             edgeRunsCand = runs_copy[idValid][result]
             result = np.searchsorted(runs_copy, edgeRunsCand).tolist()
         runRj, reasonRj, meanCand, stdCand = outlierDetector(runs_copy, x_copy, xerr_copy, result, stdRange=stdRange, 
-                                                              useMAD=useMAD, weights=weights, legacy=legacy, seqRej=(legacy and i > 0))
+                                                              useMAD=useMAD, weights=weights, legacy=legacy, seqRej=(legacy and i > 0), quadRange=quadRange)
 
         if runRj.shape[0] == 0:
             break
@@ -139,10 +140,11 @@ def printBanner():
     parser.add_argument('-pg', '--plotGood', action='store_true', help='Plot QA plots again, but only with good runs')
     parser.add_argument('-m', '--mapping', help='If x-axis of TProfile does not corresponds to STAR run ID, you can supply a file that translate bin low edge to STAR ID')
     parser.add_argument('-so', '--segmentOnce', action='store_true', help='Only run segmentation algorithm once. You can still iterate, but the segment edges will remain unchanged in each iteration')
-    parser.add_argument('-ei', '--excludeInvalid', action='store_false', help='Do not load any runs where uncertainty of any observables is zero from the get go, don\'t even count towards total number of runs.')
-    parser.add_argument('-rn', '--reCalculateNormal', action='store_true', help='mean and standard deviation of data set is re-calculated in each iteration.')
+    parser.add_argument('-ei', '--excludeInvalid', action='store_true', help='Do not load any runs where uncertainty of any observables is zero from the get go, don\'t even count towards total number of runs.')
+    parser.add_argument('-rn', '--reCalculateNormal', action='store_false', help='mean and standard deviation of data set is re-calculated in each iteration. (default: %(default)s)')
     parser.add_argument('-mi', '--mergeID', action='store_true', help='Merge nearby segments if their means are too close to each other, like within 5 SDs.')
-    parser.add_argument('-g', '--globalRejection', action='store_true', help='Run outliner rejection once before segmentation iteration.')
+    parser.add_argument('-g', '--globalRejection', action='store_false', help='Run outliner rejection once before segmentation iteration. (default: %(default)s)')
+    parser.add_argument('-q', '--quadRange', action='store_false', help='Reject runs by adding uncertainty in quaduature instead of absolute value. (default: %(default)s)')
     parser.add_argument('-lg', '--legacy', action='store_true', help='Use legacy mode to emulate run-by-run v2')
 
 
@@ -166,6 +168,7 @@ def printBanner():
         args.reCalculateNormal = True
         args.mergeID = True
         args.globalRejection = False
+        args.quadRange = True
 
     # read data from file
     print('Reading TProfile from %s' % (args.input))
@@ -213,7 +216,7 @@ def printBanner():
         for ruj, rej, me, st, ed, pe, i in pool.imap(partial(segmentAndReject, runs, x, xerr, useJMLR=args.JMLR, useMAD=args.MAD,
                                                               min_size=args.minSize, stdRange=args.rejectionRange, maxIter=args.maxIter,
                                                               weights=weights, segmentOnce=args.segmentOnce, merge=args.mergeID, 
-                                                              reCalculateNormal=args.reCalculateNormal, legacy=args.legacy, globalRejection=args.globalRejection), 
+                                                              reCalculateNormal=args.reCalculateNormal, legacy=args.legacy, globalRejection=args.globalRejection, quadRange=args.quadRange), 
                                                         args.pen): 
             # choose penalty that rejectes the most number of runs
             print('%d runs rejected when pen = %f' % (len(ruj), pe))

StRoot/PWGTools/BadRunQA/README.md

@@ -8,11 +8,17 @@ This code works on both Linux and Windows. It should work in Mac but it has not
 ## Installation
 
 To install the software, follow these steps:
-The EASY way (Linux only. For Windows and Mac, use the recommended way):
+The EASY way (Linux with python version >= 3.11 only. For Windows, Mac or Linux with older python, use the recommended way):
 1. Run the following command: `pip3 install -r requirements.txt`
 
 The RECOMMENDED way (works across platforms):
 1.      Install miniconda from https://docs.conda.io/en/latest/miniconda.html
+2.      Create d dedicated environment using environment.yml called "QA":
+
+`conda env create -n "QA" --file enviornment.yml`
+
+That should be it. IF it doesn't work, it means the recommended package versions doesn't work in your machine. Do the following 3 steps instead,
+
 2.      Create a dedicated environment by running the following command to create an environment called “QA”: 
 
 `conda config --add channels conda-forge`
@@ -21,7 +27,11 @@ The RECOMMENDED way (works across platforms):
 
 3.      Activate the environment by running `conda activate QA`
 4.      Install the remining packages with `pip install pyfiglet`
-The environment is called “QA” in the example, but you can call it whatever you want. You need to run step 3 EVERY TIME you restart your terminal if you use the recommended way.
+
+
+The environment is called “QA” in the example, but you can call it whatever you want. You need to activate the environment EVERY TIME you restart your terminal if you use the recommended way by
+
+`conda activate QA`
 
 If you are using "conda" or any other virtual environment, you can replace all "python3" with just "python" in the following sections. I just want user with older OS to be sure that they are using python3 instead of python2. 
 

StRoot/PWGTools/BadRunQA/environment.yml

@@ -0,0 +1,19 @@
+name: QA
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - pip=23.0.1
+  - matplotlib=3.7.1
+  - matplotlib-base=3.7.1
+  - ruptures=1.1.7
+  - pandas=2.0.1
+  - numpy=1.24.3
+  - scipy=1.10.1
+  - uproot=5.0.7
+  - uproot-base=5.0.7
+  - scikit-learn=1.2.2
+  - pip:
+      - pyfiglet==0.8.post1
+prefix: /star/u/ctsang/miniconda3/envs/QA
+

StRoot/PWGTools/BadRunQA/outlierDetector.py

@@ -61,7 +61,7 @@ def outlierSegment(runs, values, uncert, stdRange=3, weights=None, meanAlg=weigh
     return runs[idRejected], idRejectedReason[idRejected], mean, stdRange*std
 
 
-def outlierDetector(runs, values, uncert, idSegments, useMAD, weights, legacy=False, seqRej=False, **kwargs):
+def outlierDetector(runs, values, uncert, idSegments, useMAD, weights, legacy=False, seqRej=False, quadRange=False, **kwargs):
     runsRejected = np.array([])
     idRejected = []
     stdRange = []
@@ -73,7 +73,7 @@ def outlierDetector(runs, values, uncert, idSegments, useMAD, weights, legacy=Fa
         meanAlg = weightedMean
         stdAlg = weightedStd
 
-    if legacy:
+    if quadRange:
         gt = greaterlegacy
     else:
         gt = greater

StRoot/PWGTools/BadRunQA/requirements.txt

@@ -0,0 +1,30 @@
+awkward==2.1.4
+awkward-cpp==15
+certifi==2022.12.7
+charset-normalizer==3.1.0
+contourpy==1.0.7
+cycler==0.11.0
+docopt==0.6.2
+fonttools==4.39.3
+idna==3.4
+joblib==1.2.0
+kiwisolver==1.4.4
+matplotlib==3.7.1
+numpy==1.24.3
+packaging==23.1
+pandas==2.0.1
+Pillow==9.5.0
+pyfiglet==0.8.post1
+pyparsing==3.0.9
+python-dateutil==2.8.2
+pytz==2023.3
+requests==2.30.0
+ruptures==1.1.7
+scikit-learn==1.2.2
+scipy==1.10.1
+six==1.16.0
+threadpoolctl==3.1.0
+tzdata==2023.3
+uproot==5.0.7
+urllib3==2.0.1
+yarg==0.1.9