Screen GEFS grid2obs Aeronet file before ASCII2NC call Feature/aeronetasciifix (#593)

* add python check file code

* add checking aeronet record columns

* incremental changes 10/17/2024 1

* updated script to use post-screen aeronet file for metplus ascii2nc

* fix fcst missing sen email

* incremental changes 10/22/2024

* PR changes #1

* fix number comparison

Author: Ho-ChunHuang-NOAA

parm/metplus_config/prep/global_ens/chem_grid2obs/ASCII2NC_obsAERONET.conf

@@ -60,7 +60,7 @@ ASCII2NC_TIME_SUMMARY_VALID_THRESH = 0.0
 
 [dir]
 
-ASCII2NC_INPUT_DIR = {ENV[DCOMINaeronet]}
+ASCII2NC_INPUT_DIR = {ENV[DATA]}
 
 ASCII2NC_OUTPUT_DIR = {ENV[finalprep]}
 
@@ -69,7 +69,7 @@ LOG_DIR = {OUTPUT_BASE}/logs/{ENV[COMPONENT]}
 
 [filename_templates]
 
-ASCII2NC_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/validation_data/aq/{ENV[OBTTYPE]}/{valid?fmt=%Y%m%d}.lev15
+ASCII2NC_INPUT_TEMPLATE = checked_{ENV[OBTTYPE]}_{valid?fmt=%Y%m%d}.lev15
 
 ASCII2NC_OUTPUT_TEMPLATE = {ENV[OBTTYPE]}_All_{valid?fmt=%Y%m%d}_lev15.nc
 

scripts/prep/global_ens/exevs_global_ens_gefs_chem_grid2obs_prep.sh

@@ -39,17 +39,31 @@ for OBTTYPE in ${obstype}; do
     if [ "${OBTTYPE}" == "aeronet" ]; then
         checkfile=${DCOMINaeronet}/${INITDATE}/validation_data/aq/${OBTTYPE}/${INITDATE}.lev15
         if [ -s ${checkfile} ]; then
-            if [ -s ${prep_config_file} ]; then
-                run_metplus.py ${prep_config_file} ${config_common}
-                export err=$?; err_chk
-                if [ ${SENDCOM} = "YES" ]; then
-                    cpfile=${finalprep}/${OBTTYPE}_All_${INITDATE}_lev15.nc
-                    if [ -e ${cpfile} ]; then
-                        cp -v ${cpfile} ${COMOUTprep}
+            screen_file=${DATA}/checked_${OBTTYPE}_${INITDATE}.lev15
+	    python ${USHevs}/${COMPONENT}/screen_aeronet_aod_lev15.py ${checkfile} ${screen_file}
+	    number_of_record=$(wc -l ${screen_file} | awk -F" " '{print $1}')
+	    ## There is 6 comment and header lines 
+            if [ ${number_of_record} -gt 6 ]; then
+		if [ -s ${prep_config_file} ]; then
+                    run_metplus.py ${prep_config_file} ${config_common}
+                    export err=$?; err_chk
+                    if [ ${SENDCOM} = "YES" ]; then
+                        cpfile=${finalprep}/${OBTTYPE}_All_${INITDATE}_lev15.nc
+                        if [ -e ${cpfile} ]; then
+                            cp -v ${cpfile} ${COMOUTprep}
+                        fi
                     fi
+                else
+                    echo "WARNING: can not find ${prep_config_file}"
                 fi
             else
-                echo "WARNING: can not find ${prep_config_file}"
+                if [ ${SENDMAIL} = "YES" ]; then
+                    echo "DEBUG : There is no valid record to be processed for ${checkfile}" >> mailmsg
+                    echo "File in question is ${checkfile}" >> mailmsg
+                    echo "==============" >> mailmsg
+                    flag_send_message=YES
+                fi
+                echo "DEBUG : There is no valid record to be processed for ${checkfile}"
             fi
         else
             if [ ${SENDMAIL} = "YES" ]; then

scripts/stats/global_ens/exevs_global_ens_chem_grid2obs_stats.sh

@@ -128,6 +128,7 @@ for ObsType in ${grid2obs_list}; do
               echo "WARNING: No ${model1} ${obs_var} forecast was available for ${aday} t${acyc}z" > mailmsg
               echo "Missing file is ${fcst_file}" >> mailmsg
               echo "==============" >> mailmsg
+              flag_send_message=YES
             fi
 
             echo "WARNING: No ${model1} ${obs_var} forecast was available for ${aday} t${acyc}z"

ush/global_ens/screen_aeronet_aod_lev15.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+###############################################################################
+# Name of Script: 
+# Contact(s):     Ho-Chun Huang (ho-chun.huang@noaa.gov)
+# Purpose of Script: Read AERONET AOD file and remove bad records with
+#                    inconsistent columns number as header
+#
+# History Log:
+###############################################################################
+
+import os
+import sys
+
+print(f'Python Script: {sys.argv[0]}')
+# input and output file specified on the command line
+
+if len(sys.argv) < 2:
+    script_name = os.path.basename(sys.argv[0])
+    print(f"FATAL ERROR: {script_name} -> Must specify input and output files.")
+    sys.exit(1)
+
+# Read the input file as the first argument
+input_file = os.path.expandvars(sys.argv[1])
+print(f'Input Original Aeronet File:  {input_file}')
+
+# Read the Output file as the second argument
+output_file = os.path.expandvars(sys.argv[2])
+print(f'Output screened Aeronet File: {output_file}')
+
+if not os.path.exists(input_file):
+    print(f"DEBUG :: Can not find input Aeronet file - {input_file}")
+    print(f"DEBUG :: Check the existence of input file before calling {sys.argv[0]}")
+    sys.exit()
+
+rfile=open(input_file, 'r')
+wfile=open(output_file,'w')
+
+count=0
+flag_data=False
+for line in rfile:
+    if not flag_data:
+        if line[0:4] != "Date":
+            wfile.write(line)
+        else:
+            wfile.write(line)
+            line=line.rstrip("\n")
+            hdr=line.split(",")
+            num_hdr=len(hdr)
+            flag_data=True
+        count += 1
+    else:
+        count += 1
+        line=line.rstrip("\n")
+        var=[]
+        var=line.split(",")
+        num_var=len(var)
+        if num_var == num_hdr:
+            wfile.write(line+"\n")
+        else:
+            print(f"DEBUG :: Line {count} has different columns number {num_var} vs standard {num_hdr}")
+wfile.close()