Add additional Eventdisplay integration tests

.github/workflows/v2dl3-codeChecking.yml

@@ -15,7 +15,7 @@ on:
       - 'setup.py'
 
 jobs:
-  tests:
+  lint:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -31,17 +31,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install conda dependencies
-        uses: mamba-org/setup-micromamba@v1
+      - name: flake8 Lint
+        uses: py-actions/flake8@v2
         with:
-          environment-file: environment-eventdisplay.yml
-          create-args: python=${{ matrix.python-version }}
-          init-shell: bash
-
-      - name: flake8
-        shell: bash -leo pipefail {0}
-        run: |
-          flake8 --count --select=E9,F63,F7,F82 --ignore=E203,W503 \
-                   --show-source --statistics .
-          flake8 --count --max-complexity=25 --ignore=E203,W503 \
-                 --max-line-length=127 --statistics .
+          ignore: "E203,W503"
+          max-line-length: "127"
+          path: "."

.github/workflows/v2dl3-eventdisplay.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
 
     steps:
       - uses: actions/checkout@v3
@@ -49,7 +49,7 @@ jobs:
         run: |
           ./utils/download_Eventdisplay_test_data.sh
 
-      - name: V2Dl3-Eventdisplay
+      - name: Eventdisplay-pointlike
         shell: bash -leo pipefail {0}
         env:
           PYTHONPATH: .
@@ -58,19 +58,81 @@ jobs:
           python pyV2DL3/script/v2dl3_for_Eventdisplay.py \
                -f ./64080.anasum.root \
                   ./effectiveArea.root \
+                  --logfile test-pointlike.fits.log \
                   test-pointlike-CI.fits.gz
           python pyV2DL3/script/compareFitsFiles.py \
                 -f ED-pointlike-CI.fits.gz test-pointlike-CI.fits.gz \
                 -d test-pointlike.log
           cat test-pointlike.log
           grep -q "No differences found" test-pointlike.log
+
+      - name: Eventdisplay-full-enclosure
+        shell: bash -leo pipefail {0}
+        env:
+          PYTHONPATH: .
+        run: |
           # full-enclosure tests
           python pyV2DL3/script/v2dl3_for_Eventdisplay.py --full-enclosure \
                -f ./64080.anasum.root \
                   ./effectiveArea.root \
+                  --logfile test-full-enclosure.fits.log \
                   test-full-enclosure-CI.fits.gz
           python pyV2DL3/script/compareFitsFiles.py \
                 -f ED-fullenclosure-CI.fits.gz test-full-enclosure-CI.fits.gz \
                 -d test-full-enclosure.log
           cat test-full-enclosure.log
           grep -q "No differences found" test-full-enclosure.log
+
+      - name: Eventdisplay-pointlike-DB
+        shell: bash -leo pipefail {0}
+        env:
+          PYTHONPATH: .
+        run: |
+          # point-like tests (with DB)
+          python pyV2DL3/script/v2dl3_for_Eventdisplay.py \
+               -f ./64080.anasum.root \
+                  ./effectiveArea.root \
+                  --db_fits_file ./64080.db.fits.gz \
+                  --logfile test-pointlike-db.fits.log \
+                  test-pointlike-db-CI.fits.gz
+          python pyV2DL3/script/compareFitsFiles.py \
+                -f ED-pointlike-db-CI.fits.gz test-pointlike-db-CI.fits.gz \
+                -d test-pointlike-db.log
+          cat test-pointlike-db.log
+          grep -q "No differences found" test-pointlike-db.log
+
+      - name: Eventdisplay-pointlike-select
+        shell: bash -leo pipefail {0}
+        env:
+          PYTHONPATH: .
+        run: |
+          # point-like tests with event selection
+          python pyV2DL3/script/v2dl3_for_Eventdisplay.py \
+               -f ./64080.allevents.anasum.root \
+                  ./effectiveArea.root \
+                  --evt_filter eventfilter.yml \
+                  --logfile test-pointlike-all.fits.log \
+                  test-pointlike-all-CI.fits.gz
+          python pyV2DL3/script/compareFitsFiles.py \
+                -f ED-pointlike-all-CI.fits.gz test-pointlike-all-CI.fits.gz \
+                -d test-pointlike-all.log
+          cat test-pointlike-all.log
+          grep -q "No differences found" test-pointlike-all.log
+
+      - name: Eventdisplay-full-enclosure-select
+        shell: bash -leo pipefail {0}
+        env:
+          PYTHONPATH: .
+        run: |
+          # full-enclosure tests with event selection
+          python pyV2DL3/script/v2dl3_for_Eventdisplay.py --full-enclosure \
+               -f ./64080.allevents.anasum.root \
+                  ./effectiveArea.root \
+                  --evt_filter eventfilter.yml \
+                  --logfile test-full-enclosure-all.fits.log \
+                  test-full-enclosure-all-CI.fits.gz
+          python pyV2DL3/script/compareFitsFiles.py \
+                -f ED-full-enclosure-all-CI.fits.gz test-full-enclosure-all-CI.fits.gz \
+                -d test-full-enclosure-all.log
+          cat test-full-enclosure-all.log
+          grep -q "No differences found" test-full-enclosure-all.log

environment-eventdisplay.yml

@@ -14,7 +14,7 @@ dependencies:
   - pre-commit
   - pyflakes
   - pytest
-  - python>=3.8
+  - python>=3.11
   - pyyaml
   - scipy
   - uproot

pyV2DL3/constant.py

@@ -1,6 +1,6 @@
 # Global setting
 
-VERSION = "0.3"
+VERSION = "0.5"
 
 # Feb 1 2005
 VTS_REFERENCE_MJD = 53402.0

pyV2DL3/generateObsHduIndex.py

@@ -3,8 +3,7 @@
 
 from astropy.io import fits
 from astropy.io.fits import table_to_hdu
-from astropy.table import Table
-from astropy.table import vstack
+from astropy.table import Table, vstack
 
 from pyV2DL3.addHDUClassKeyword import addHDUClassKeyword
 
@@ -99,7 +98,7 @@ def get_unit_string_from_comment(comment_string):
     Return unit string from FITS comment
 
     Examples: 'average pointing azimuth [deg]'
-    
+
     """
     bstart = comment_string.find("[")
     bstopp = comment_string.find("]")

utils/compare_fits_dirs.sh

@@ -2,7 +2,7 @@
 
 # Prints fitsdiff outputs that detect a difference.
 #
-# Takes 'test' and 'control' directories as arguments. 
+# Takes 'test' and 'control' directories as arguments.
 DIR1=$1
 DIR2=$2
 
@@ -11,15 +11,15 @@ DIR2=$2
 # e.g `test/batch1/name1.fits` matches to `control/batch1/name1.fits`
 #
 # Exit codes:
-#   0 = no differences, 
+#   0 = no differences,
 #   1 = at least one file contained differences
 
 DIFF_FOUND=false
 
 # Works recursively
 for FILE in $(find $DIR1 -name \*.fits); do
-    fitsdiff $FILE ${FILE/$DIR1/$DIR2} > $FILE.txt
-    
+    fitsdiff -k CREATOR $FILE ${FILE/$DIR1/$DIR2} > $FILE.txt
+
     if ! grep -q "No differences found" $FILE.txt; then
         cat $FILE.txt
         DIFF_FOUND=true
@@ -30,4 +30,4 @@ if [ "$DIFF_FOUND" = true ] ; then
     exit 1
 fi
 
-echo "Fits files contain no differences."
+echo "Fits files contain no differences."

utils/download_Eventdisplay_test_data.sh

@@ -1,6 +1,18 @@
 #!/bin/bash
+#
+# Expected files in tar ball:
+# - 64080.anasum.root
+# - EffectiveArea.root
+# - 64080.db.fits.gz
+# - ED-pointlike-CI.fits.gz
+# - ED-pointlike-CI.fits.gz
+# - ED-pointlike-db-CI.fits.gz
+# - ED-pointlike-all-CI.fits.gz
+# - ED-full-enclosure-all-CI.fits.gz
+# - eventfilter.yml
+# (plus corresponding log files)
 
 set -e
 
-wget --no-verbose https://syncandshare.desy.de/index.php/s/S8KAyRAmNdF9qEX/download/github-CI.tar.gz
+wget --no-verbose https://syncandshare.desy.de/index.php/s/Pj4KBZ9E9cjLfk7/download/github-CI.tar.gz
 tar -xvzf github-CI.tar.gz

utils/prepare_Eventdisplay_CI.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Prepare Eventdisplay CI files for integration tests
+
+# point-like tests
+python pyV2DL3/script/v2dl3_for_Eventdisplay.py \
+    -f ./64080.anasum.root \
+        ./effectiveArea.root \
+        --logfile ED-pointlike-CI.log \
+        ED-pointlike-CI.fits.gz
+
+# full-enclosure tests
+python pyV2DL3/script/v2dl3_for_Eventdisplay.py --full-enclosure \
+    -f ./64080.anasum.root \
+       ./effectiveArea.root \
+       --logfile ED-fullenclosure-CI.log \
+       ED-fullenclosure-CI.fits.gz
+
+# point-like tests with DB
+python pyV2DL3/script/v2dl3_for_Eventdisplay.py \
+    -f ./64080.anasum.root \
+        ./effectiveArea.root \
+        --db_fits_file ./64080.db.fits.gz \
+        --logfile ED-pointlike-db-CI.log \
+        ED-pointlike-db-CI.fits.gz
+
+# point-like tests with event filter
+python pyV2DL3/script/v2dl3_for_Eventdisplay.py \
+    -f ./64080.allevents.anasum.root \
+       ./effectiveArea.root \
+       --evt_filter eventfilter.yml \
+       --logfile ED-pointlike-all-CI.log \
+       ED-pointlike-all-CI.fits.gz
+
+# full-enclosure tests with event selection
+python pyV2DL3/script/v2dl3_for_Eventdisplay.py --full-enclosure \
+     -f ./64080.allevents.anasum.root \
+        ./effectiveArea.root \
+        --evt_filter eventfilter.yml \
+        --logfile ED-full-enclosure-all-CI.log \
+        ED-full-enclosure-all-CI.fits.gz