David mts

.travis.yml

@@ -46,7 +46,7 @@ script:
 - pytest digicampipe
 - make -C docs/ html
 - |
-  if [ "${TRAVIS_EVENT_TYPE}" == "cron" ]; then
+  if [ "${TRAVIS_EVENT_TYPE}" == "cron" ] || [ "${TRAVIS_EVENT_TYPE}" == "api" ]; then
     travis_wait 30 pytest -c all_tests.ini digicampipe
   fi
 env:

bash/0_main.sh

@@ -0,0 +1,83 @@
+#!/bin/env bash
+
+source $HOME/.bashrc
+$HOME/.miniconda3/bin/activate digicampipe
+
+export MATPLOTLIBRC='../matplotlibrc'
+
+
+### INPUT FILES ###
+
+export DIGICAM_GHV_OFF_FILES=($(echo /sst1m/raw/2018/06/27/SST1M_01/SST1M_01_20180627_{1292..1299}.fits.fz))
+export DIGICAM_GHV_ON_FILES=($(echo /sst1m/raw/2018/06/27/SST1M_01/SST1M_01_20180627_{1286..1292}.fits.fz))
+export DIGICAM_DARK_FILES=($(echo /sst1m/raw/2018/06/27/SST1M_01/SST1M_01_20180627_{1276..1279}.fits.fz))
+export DIGICAM_AC_FILES=($(echo /sst1m/raw/2018/06/28/SST1M_01/SST1M_01_20180628_{1350..1454}.fits.fz))
+export DIGICAM_DC_FILES=($(echo /sst1m/raw/2018/06/28/SST1M_01/SST1M_01_20180628_{1455..1504}.fits.fz))
+export DIGICAM_AC_DC_FILES_1=($(echo /sst1m/raw/2018/06/28/SST1M_01/SST1M_01_20180628_{1505..2087}.fits.fz))
+export DIGICAM_AC_DC_FILES_2=($(echo /sst1m/raw/2018/07/01/SST1M_01/SST1M_01_20180701_{434..497}.fits.fz))
+
+
+export DIGICAM_AC_LEVEL=({0..19..1} {20..35..5} {45..445..5})
+export DIGICAM_AC_LEVEL_1=({0..18..2} {20..440..10})
+export DIGICAM_AC_LEVEL_2=({0..18..2} {20..440..20})
+export DIGICAM_DC_LEVEL=({200..445..5})
+export DIGICAM_DC_LEVEL_1=({200..440..10})
+export DIGICAM_DC_LEVEL_2=(${DIGICAM_DC_LEVEL_1[@]:12:2})
+export DIGICAM_DC_LEVEL_1=(${DIGICAM_DC_LEVEL_1[@]:0:11})
+
+PIXEL=$1
+
+if [ -z "$PIXEL" ]; then
+    export DIGICAM_PIXELS=({0..1295..1})
+else
+    export DIGICAM_PIXELS=($PIXEL)
+fi
+
+
+export DIGICAM_INTEGRAL_WIDTH=7
+export DIGICAM_INTEGRAL_SHIFT=0
+export DIGICAM_N_SAMPLES=50
+export DIGICAM_GAIN_APPROX=20
+export DIGICAM_LSB_MIN=-10
+export DIGICAM_LSB_MAX=3000
+export DIGICAM_LSB_BIN=1
+export DIGICAM_SATURATION_THRESHOLD=3000
+
+function tolist () {
+    local array="$@"
+    array=$(echo ${array[@]} | tr -s ' ' ',')
+    echo $array
+    }
+
+export -p tolist
+# echo ${DIGICAM_PIXELS[@]}
+# a=$(tolist "${DIGICAM_PIXELS[@]}")
+
+### OUTPUT FILES ###
+
+# DATE=`date +%Y%m%d`
+DATE=20190123
+export DIGICAM_FOLDER='/sst1m/analyzed/calib/'$DATE'/'
+mkdir -p $DIGICAM_FOLDER
+mkdir -p $DIGICAM_FOLDER'figures/'
+
+export GHV_OFF_RAW_HISTO=$DIGICAM_FOLDER'ghv_off_raw_histo.fits'
+export GHV_OFF_RATE_SCAN=$DIGICAM_FOLDER'ghv_off_rate_scan.fits'
+export GHV_ON_RAW_HISTO=$DIGICAM_FOLDER'ghv_on_raw_histo.fits'
+export GHV_ON_RATE_SCAN=$DIGICAM_FOLDER'ghv_on_rate_scan_histo.fits'
+export DARK_RAW_HISTO=$DIGICAM_FOLDER'dark_raw_histo.fits'
+export DARK_RATE_SCAN=$DIGICAM_FOLDER'dark_rate_scan.fits'
+export DARK_MAX_HISTO=$DIGICAM_FOLDER'dark_max_histo.fits'
+export DARK_CHARGE_HISTO=$DIGICAM_FOLDER'dark_charge_histo.fits'
+export SPE_RESULTS=$DIGICAM_FOLDER'spe_results.fits'
+export TIMING_HISTO=$DIGICAM_FOLDER'timing_histo.fits'
+export TIMING_RESULTS=$DIGICAM_FOLDER'timing_results.fits'
+export FMPE_CHARGE_HISTO=$DIGICAM_FOLDER'fmpe_charge_histo.fits'
+export FMPE_AMPLITUDE_HISTO=$DIGICAM_FOLDER'fmpe_amplitude_histo.fits'
+export FMPE_RESULTS=$DIGICAM_FOLDER'fmpe_results.fits'
+export MPE_CHARGE_HISTO=$DIGICAM_FOLDER'mpe_charge_histo.fits'
+export MPE_RESULTS=$DIGICAM_FOLDER'mpe_results.fits'
+export AC_LED_FILE=$DIGCAM_FOLDER'ac_led.fits'
+export TEMPLATE_FILE=$DIGICAM_FOLDER'pulse_shape.fits'
+export BASELINE_SHIFT_RESULTS=$DIGICAM_FOLDER'baseline_shift_results.fits'
+export CALIBRATION_FILE=$DIGICAM_FOLDER'calibration_results.fits'

bash/10_templates.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+output_path=/sst1m/analyzed/template_scan/20180702
+digicam_path=~/ctasoft/digicampipe
+DATA_BASE_DIR=/sst1m/raw/2018/
+
+source 0_main.sh
+
+rm $TEMPLATE_FILE
+digicam-pulse-template --output=$TEMPLATE_FILE --per_pixel ${DIGICAM_AC_FILES[@]:20:10}

bash/1_raw_ghv_off.sh

@@ -0,0 +1,12 @@
+#!/bin/env bash
+#SBATCH --time=01:00:00
+#SBATCH --partition=mono
+#SBATCH --mem=4G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='ghv-off'
+
+source 0_main.sh
+
+digicam-raw compute --output=$GHV_OFF_RAW_HISTO ${DIGICAM_GHV_OFF_FILES[@]}
+digicam-raw save_figure --output=$DIGICAM_FOLDER'figures/raw_ghv_off_histo.pdf' $GHV_OFF_RAW_HISTO
+digicam-rate-scan --compute --output=$GHV_OFF_RATE_SCAN --threshold_step=1 ${DIGICAM_GHV_OFF_FILES[@]}

bash/2_raw_ghv_on.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+#SBATCH --time=01:00:00
+#SBATCH --partition=mono
+#SBATCH --mem=4G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='ghv-on'
+
+source 0_main.sh
+
+
+pixels=$(echo ${DIGICAM_PIXELS[@]} | tr -s ' ' ',')
+
+digicam-raw compute --output=$GHV_ON_RAW_HISTO --pixel=$pixels ${DIGICAM_GHV_ON_FILES[@]}
+digicam-rate-scan --compute --output=$GHV_ON_RATE_SCAN --threshold_step=1 ${DIGICAM_GHV_ON_FILES[@]}
+digicam-raw save_figure --output=$DIGICAM_FOLDER'figures/raw_ghv_on_histo.pdf' $GHV_ON_RAW_HISTO

bash/3_raw_dark.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#SBATCH --time=06:00:00
+#SBATCH --partition=mono
+#SBATCH --mem=4G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='dark'
+
+source 0_main.sh
+
+pixels=$(echo ${DIGICAM_PIXELS[@]} | tr -s ' ' ',')
+
+digicam-raw compute --output=$DARK_RAW_HISTO --pixel=$pixels ${DIGICAM_DARK_FILES[@]}
+digicam-rate-scan --compute --output=$DARK_RATE_SCAN --threshold_step=10 --threshold_end=100 ${DIGICAM_DARK_FILES[@]}
+digicam-raw save_figure --output=$DIGICAM_FOLDER'figures/raw_histo.pdf' $DARK_RAW_HISTO

bash/4_spe.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#SBATCH --time=04:00:00
+#SBATCH --partition=mono-shared
+#SBATCH --mem=12G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='spe'
+
+source 0_main.sh
+
+pixel=$(echo ${DIGICAM_PIXELS[@]} | tr -s ' ' ',')
+
+digicam-spe --compute --pixel=$pixel --max_histo_filename=$DARK_MAX_HISTO --charge_histo_filename=$DARK_CHARGE_HISTO --raw_histo_filename=$DARK_RAW_HISTO --output=$CALIBRATION_FILE --integral_width=$DIGICAM_INTEGRAL_WIDTH --shift=$DIGICAM_INTEGRAL_SHIFT --n_samples=$DIGICAM_N_SAMPLES ${DIGICAM_DARK_FILES[@]}
+digicam-spe --fit --pixel=$pixel --max_histo_filename=$DARK_MAX_HISTO --charge_histo_filename=$DARK_CHARGE_HISTO --raw_histo_filename=$DARK_RAW_HISTO --output=$CALIBRATION_FILE --integral_width=$DIGICAM_INTEGRAL_WIDTH --shift=$DIGICAM_INTEGRAL_SHIFT --n_samples=$DIGICAM_N_SAMPLES ${DIGICAM_DARK_FILES[@]}

bash/5_timing.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#SBATCH --time=02:00:00
+#SBATCH --partition=mono
+#SBATCH --mem=4G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='timing'
+
+source 0_main.sh
+
+pixels=$(tolist "${DIGICAM_PIXELS[@]}")
+ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}")
+
+digicam-timing --compute --pixels=$pixels --output=$CALIBRATION_FILE --timing_histo_filename=$TIMING_HISTO --n_samples=$DIGICAM_N_SAMPLES --ac_levels=$ac_levels ${DIGICAM_AC_FILES[@]}
+digicam-timing --fit --pixels=$pixels --output=$CALIBRATION_FILE --timing_histo_filename=$TIMING_HISTO --n_samples=$DIGICAM_N_SAMPLES --ac_levels=$ac_levels ${DIGICAM_AC_FILES[@]}
+digicam-timing --display --output=$CALIBRATION_FILE --timing_histo_filename=$TIMING_HISTO --n_samples=$DIGICAM_N_SAMPLES --ac_levels=$ac_levels ${DIGICAM_AC_FILES[@]}

bash/6_fmpe.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#SBATCH --time=06:00:00
+#SBATCH --partition=mono
+#SBATCH --mem=8G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='fmpe'
+
+source 0_main.sh
+
+digicam-fmpe --compute --charge_histo_filename=$FMPE_CHARGE_HISTO --amplitude_histo_filename=$FMPE_AMPLITUDE_HISTO --output=$CALIBRATION_FILE --pixels=$(tolist "${DIGICAM_PIXELS[@]}") --estimated_gain=$DIGICAM_GAIN_APPROX --n_samples=$DIGICAM_N_SAMPLES --shift=$DIGICAM_INTEGRAL_SHIFT --integral_width=$DIGICAM_INTEGRAL_WIDTH ${DIGICAM_AC_FILES[@]}
+digicam-fmpe --fit --charge_histo_filename=$FMPE_CHARGE_HISTO --amplitude_histo_filename=$FMPE_AMPLITUDE_HISTO --output=$CALIBRATION_FILE --pixels=$(tolist "${DIGICAM_PIXELS[@]}") --estimated_gain=$DIGICAM_GAIN_APPROX --n_samples=$DIGICAM_N_SAMPLES --shift=$DIGICAM_INTEGRAL_SHIFT --integral_width=$DIGICAM_INTEGRAL_WIDTH ${DIGICAM_AC_FILES[@]}
+digicam-fmpe --charge_histo_filename=$FMPE_CHARGE_HISTO --amplitude_histo_filename=$FMPE_AMPLITUDE_HISTO --output=$CALIBRATION_FILE --pixels=$(tolist "${DIGICAM_PIXELS[@]}") --figure_path=$DIGICAM_FOLDER'figures/fmpe.pdf' --estimated_gain=$DIGICAM_GAIN_APPROX --n_samples=$DIGICAM_N_SAMPLES --shift=$DIGICAM_INTEGRAL_SHIFT --integral_width=$DIGICAM_INTEGRAL_WIDTH ${DIGICAM_AC_FILES[@]}

bash/7_mpe.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#SBATCH --time=06:00:00
+#SBATCH --partition=mono-shared
+#SBATCH --mem=4G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='mpe'
+
+source 0_main.sh ${SLURM_ARRAY_TASK_ID} $1 $2
+
+
+DIGICAM_INTEGRAL_WIDTH=$1
+DIGICAM_INTEGRAL_SHIFT=$2
+
+PIXEL=${DIGICAM_PIXELS[0]}
+output=$(printf $DIGICAM_FOLDER'calibration_results_window_%03d_shift_%03d_pixel_%04d.fits' $DIGICAM_INTEGRAL_WIDTH $DIGICAM_INTEGRAL_SHIFT $PIXEL)
+figure_path=$(printf $DIGICAM_FOLDER'figures/mpe_fit_window_%03d_shift_%03d_pixel_%04d.fits' $DIGICAM_INTEGRAL_WIDTH $DIGICAM_INTEGRAL_SHIFT $PIXEL)
+
+
+exit
+if [[ -z "${SLURM_ARRAY_TASK_ID}" ]];
+then
+    digicam-mpe compute --output=$MPE_CHARGE_HISTO --ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}") --calib=$CALIBRATION_FILE --shift=$DIGICAM_INTEGRAL_SHIFT --integral_width=$DIGICAM_INTEGRAL_WIDTH --adc_min=$DIGICAM_LSB_MIN --adc_max=$DIGICAM_LSB_MAX --pixel=$(tolist "${DIGICAM_PIXELS[@]}") ${DIGICAM_AC_FILES[@]}
+    # digicam-mpe fit summed --output=$CALIBRATION_FILE --ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}") --pixel=$(tolist "${DIGICAM_PIXELS[@]}") $MPE_CHARGE_HISTO
+    # digicam-mpe fit combined --output=$CALIBRATION_FILE --ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}") --pixel=$(tolist "${DIGICAM_PIXELS[@]}") $MPE_CHARGE_HISTO
+    # digicam-mpe fit single --output=$CALIBRATION_FILE --ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}") --pixel=$(tolist "${DIGICAM_PIXELS[@]}") $MPE_CHARGE_HISTO
+else
+
+    digicam-mpe fit combined --output=$output --ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}") --pixel=$PIXEL $MPE_CHARGE_HISTO
+    #digicam-mpe save_figure --output=$figure_path --calib=$output --ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}") $MPE_CHARGE_HISTO
+fi
+# digicam-mpe --display --compute_output=$MPE_CHARGE_HISTO --fit_output=$MPE_RESULTS --shift=$DIGICAM_INTEGRAL_SHIFT --integral_width=$DIGICAM_INTEGRAL_WIDTH --timing=$TIMING_RESULTS --gain=$FMPE_RESULTS --adc_min=$DIGICAM_LSB_MIN --adc_max=$DIGICAM_LSB_MAX --pixel=$(tolist "${DIGICAM_PIXELS[@]}") --ac_levels=$(tolist "${DIGICAM_AC_LEVEL[@]}") ${DIGICAM_AC_FILES[@]}

bash/8_baseline_shift.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#SBATCH --time=04:00:00
+#SBATCH --partition=mono
+#SBATCH --mem=8G
+#SBATCH --output=/home/%u/job_logs/%x-%A_%a.out
+#SBATCH --job-name='baseline-shift'
+
+source 0_main.sh
+
+
+
+i=0
+
+histo_files=()
+
+for file in "${DIGICAM_DC_FILES[@]}"
+do
+    output=$DIGICAM_FOLDER'histo_dc_level_'${DIGICAM_DC_LEVEL[$i]}'.fits'
+    histo_files+=($output)
+    # echo digicam-raw --compute --output=$output $file
+    i=$((i + 1))
+done
+# digicam-baseline-shift --compute --output=$BASELINE_SHIFT_RESULTS --integral_width=$DIGICAM_INTEGRAL_WIDTH --dark=${histo_files[0]} --dc_levels=$(tolist "${DIGICAM_DC_LEVEL[@]}") ${histo_files[@]}
+# digicam-baseline-shift --fit --output=$BASELINE_SHIFT_RESULTS --integral_width=$DIGICAM_INTEGRAL_WIDTH --template=$TEMPLATE_FILE --gain=$FMPE_RESULTS --dark=${histo_files[0]} --dc_levels=$(tolist "${DIGICAM_DC_LEVEL[@]}") ${histo_files[@]}
+# digicam-baseline-shift --display --output=$BASELINE_SHIFT_RESULTS --integral_width=$DIGICAM_INTEGRAL_WIDTH --template=$TEMPLATE_FILE --gain=$FMPE_RESULTS --dark=${histo_files[0]} --dc_levels=$(tolist "${DIGICAM_DC_LEVEL[@]}") ${histo_files[@]}
+
+
+i=0
+k=0
+N_AC=${#DIGICAM_AC_LEVEL_1[@]}
+histo_files=()
+for file in "${DIGICAM_AC_DC_FILES_1[@]}"
+do
+    j=$(($i % $N_AC))
+
+    if [[ "$j" -eq "0" ]]; then
+
+        output=$DIGICAM_FOLDER'histo_dc_level_'${DIGICAM_DC_LEVEL_1[$k]}'_from_acdc_scan.fits'
+        histo_files+=($output)
+        # echo digicam-raw --compute --output=$output $file
+        k=$(($k + 1))
+    fi
+
+    i=$((i + 1))
+done
+
+i=0
+k=0
+N_AC=${#DIGICAM_AC_LEVEL_2[@]}
+for file in "${DIGICAM_AC_DC_FILES_2[@]}"
+do
+    j=$(($i % $N_AC))
+
+    if [[ "$j" -eq "0" ]]; then
+
+        output=$DIGICAM_FOLDER'histo_dc_level_'${DIGICAM_DC_LEVEL_2[$k]}'_from_acdc_scan.fits'
+        histo_files+=($output)
+        digicam-raw --compute --output=$output $file
+        k=$(($k + 1))
+    fi
+
+    i=$((i + 1))
+done
+
+dc_levels=("${DIGICAM_DC_LEVEL_1[@]}" "${DIGICAM_DC_LEVEL_2[@]}")
+digicam-baseline-shift --compute --output=$DIGICAM_FOLDER'baseline_shift_acdc_scan.fits' --integral_width=$DIGICAM_INTEGRAL_WIDTH --dark=${histo_files[0]} --dc_levels=$(tolist "${dc_levels[@]}") ${histo_files[@]}
+# digicam-baseline-shift --fit --output=$DIGICAM_FOLDER'baseline_shift_acdc_scan.fits' --integral_width=$DIGICAM_INTEGRAL_WIDTH --template=$TEMPLATE_FILE --gain=$FMPE_RESULTS --dark=${histo_files[0]} --dc_levels=$(tolist "${DIGICAM_DC_LEVEL_1[@]}") ${histo_files[@]}
+# digicam-baseline-shift --display --output=$DIGICAM_FOLDER'baseline_shift_acdc_scan.fits' --integral_width=$DIGICAM_INTEGRAL_WIDTH --template=$TEMPLATE_FILE --gain=$FMPE_RESULTS --dark=${histo_files[0]} --dc_levels=$(tolist "${DIGICAM_DC_LEVEL_1[@]}") ${histo_files[@]}

bash/baobab_submit_calibration.sh

@@ -0,0 +1,23 @@
+#!/bin/env bash
+
+# sbatch 1_raw_ghv_off.sh
+# sbatch 2_raw_ghv_on.sh
+# sbatch 3_raw_dark.sh
+# sbatch 4_spe.sh
+# job1=$(sbatch --parsable 5_timing.sh);
+
+# job3=$(sbatch --parsable 7_mpe.sh);
+# sbatch --dependency=afterok:${job3} --array=0-1295 7_mpe.sh
+# sbatch --array=558 7_mpe.sh
+
+for width in {1..20};
+do
+    for shift in {-4..4};
+        do
+
+            sbatch 9_charge_linearity.sh $width $shift
+        done
+done
+
+# sbatch --dependency=afterany:${job2} 7_mpe.sh
+# sbatch 8_baseline_shift.sh