Merge pull request #338 from EdwardSafford-NOAA/master

GitHub Issue NOAA-EMC/GSI#313. Store data files for global sources in tar files.
AndrewEichmann-NOAA · Mar 23, 2022 · 9256850 · 9256850
2 parents 3d87350 + 35cf5f9
commit 9256850
Show file tree

Hide file tree

Showing 41 changed files with 762 additions and 644 deletions.
diff --git a/util/Radiance_Monitor/CMakeLists.txt b/util/Radiance_Monitor/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 2.8.12)
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     # I am top-level project.
     if( NOT DEFINED ENV{CC} )
@@ -66,6 +66,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     endif()
 
     cmake_policy(SET CMP0009 NEW)
+    cmake_policy(SET CMP0074 NEW)
     find_package(OpenMP)
     message("found openmp with flag ${OPENMP_Fortran_FLAGS}")
 
@@ -84,8 +85,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     find_package( W3NCO )
 
     set(BUILD_NCDIAG ON)
-    set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag/include")
-    add_subdirectory(src/ncdiag)
+    set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag")
+    add_subdirectory(${PROJECT_SOURCE_DIR}/../../src/ncdiag ${PROJECT_BINARY_DIR}/src/ncdiag)
     set(NCDIAG_LIBRARIES ncdiag )
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 endif()

diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_MK_BASE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_MK_BASE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_mk_base )

diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_VALIDATE_TM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_VALIDATE_TM_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_validate_tm.x )

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh
@@ -192,8 +192,8 @@ if [[  -d ${DATA_LOCATION} ]]; then
         -o ${logfile} --ntasks=1 --mem=5g ${job}
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-           -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/CP.${PDY}.${CYC}.err \
+	   -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
    fi
 else
    echo "Unable to locate DATA_LOCATION: ${DATA_LOCATION}"

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh
@@ -248,8 +248,8 @@ if [[ -e ${radstat} && -e ${biascr} ]]; then
         -o ${logfile} --ntasks=1 --mem=5g ${job} 
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-        -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${pdy}.${cyc}.err \
+        -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
    fi
 
 else  # radstat and/or biascr not found

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh
@@ -306,8 +306,8 @@ elif [[ $MY_MACHINE = "hera" ]]; then
         -j oe -o ${logfile} ${job}
 
 elif [[ $MY_MACHINE = "wcoss2" ]]; then
-   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-        -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${PDY}.${cyc}.err \
+        -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
 fi
 
 

diff --git a/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl b/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl
@@ -88,8 +88,8 @@
       $search_string = $run;
    }
 
-   my @mmdirs = grep { /$search_string/ } @alldirs;
-
+   my @mmdirs = grep {/$search_string/} @alldirs;
+   
    #-----------------------------------------------------------------------   
    #  If there are no $run.yyyymmdd subdirectories, then exit without 
    #    returning any date string.
@@ -157,13 +157,26 @@
          $hr_ctr = $hr_ctr - 1;
 
          $newdir = "${dirpath}/${sortmm[$ctr]}/${hrs[$hr_ctr]}/${lcm}";
-#         print " newdir = $newdir \n";
-
 
          if( -d $newdir ) {
             opendir DIR, $newdir or die "Cannot open the current directory: $!";
 
-            my @timefiles = grep { /ieee_d/ } readdir DIR;
+            my @dirfiles = readdir DIR;
+            my @timefiles = grep { /ieee_d/ } @dirfiles;
+
+            #  If no *ieee_d* files were found then look for the compressed format of
+            #  radmon_*.tar files instead.  If found then use the parent directories
+            #  to establish the cycle time.
+            #
+            if( $#timefiles < 0 ) {       
+               my @tarfiles = grep { /radmon_time.tar/ } @dirfiles;
+
+               if( $#tarfiles >= 0 ) { 
+                  my $cycle = "${sortmm[$ctr]}${hrs[$hr_ctr]}";
+                  push( @timefiles, $cycle );
+               }
+            } 
+
 
             if( $#timefiles >= 0 ) {
                my @sorttime = sort( @timefiles );
@@ -201,18 +214,13 @@
                   }
                }
             }
-
          } 
 
       } while $hr_ctr > 0 && $found_cycle == 0;
 
-#      print " found_cycle, ctr, end_ctr = $found_cycle, $ctr, $end_ctr \n";
-
       if( $cyc == 0 && $ctr >= $end_ctr ){  
-#         print " exiting from if\n";
          $exit_flag = 1;
       } elsif( $cyc == 1 && $ctr <= $end_ctr ){
-#         print " exiting from elsif\n";
          $exit_flag = 1;
       }
 

diff --git a/util/Radiance_Monitor/image_gen/html/Install_html.sh b/util/Radiance_Monitor/image_gen/html/Install_html.sh
@@ -51,12 +51,6 @@ else
 fi
 
 
-#--------------------------------------------------------------
-#  source plot_rad_conf to get WEB_SVR, WEB_USER, WEBDIR
-#
-. ${RADMON_IMAGE_GEN}/parm/plot_rad_conf
-
-
 #--------------------------------------------------------------
 #  call the appropriate child script for glb or rgn
 #

diff --git a/util/Radiance_Monitor/image_gen/html/install_glb.sh b/util/Radiance_Monitor/image_gen/html/install_glb.sh
@@ -33,35 +33,8 @@ RAD_AREA="glb"
 this_file=`basename $0`
 this_dir=`dirname $0`
 
-#top_parm=${this_dir}/../../parm
-#
-#if [[ -s ${top_parm}/RadMon_config ]]; then
-#   . ${top_parm}/RadMon_config
-#else
-#   echo "ERROR:  Unable to source ${top_parm}/RadMon_config"
-#   exit
-#fi
-#
-#if [[ -s ${top_parm}/RadMon_user_settings ]]; then
-#   . ${top_parm}/RadMon_user_settings
-#else
-#   echo "ERROR:  Unable to source ${top_parm}/RadMon_user_settings"
-#   exit
-#fi
-
-
-#--------------------------------------------------------------
-#  source plot_rad_conf to get WEB_SVR, WEB_USER, WEBDIR
-#
-. ${RADMON_IMAGE_GEN}/parm/plot_rad_conf
-
-
-#--------------------------------------------------------------
-#  Get the area for this SUFFIX from the data_map file
-#
 
 new_webdir=${WEBDIR}/${SUFFIX}
-. ${RADMON_IMAGE_GEN}/parm/glbl_conf
 
 echo RAD_AREA = $RAD_AREA
 echo TANKverf = $TANKverf
@@ -85,7 +58,7 @@ cd $workdir
 #  Find the first date with data.  Start at today and work
 #  backwards.  Stop after 90 days and exit.
 #
-PDATE=`${IG_SCRIPTS}/find_cycle.pl --dir ${TANKverf} --cyc 1`
+PDATE=`${IG_SCRIPTS}/nu_find_cycle.pl --dir ${TANKverf} --cyc 1`
 echo PDATE= $PDATE
 
 limit=`$NDATE -2160 $PDATE`		# 90 days
@@ -98,35 +71,30 @@ echo limit, PDATE = $limit, $PDATE
 data_found=0
 while [[ data_found -eq 0 && $PDATE -ge $limit ]]; do
    PDY=`echo $PDATE|cut -c1-8`
+   CYC=`echo $PDATE|cut -c9-10`
 
-   test_dir=${TANKverf}/${RUN}.${PDY}/${MONITOR}
+   test_dir=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR}
    if [[ ! -d ${test_dir} ]]; then
-      test_dir=${TANKverf}/${RUN}.${PDY}
+      test_dir=${TANKverf}/${RUN}.${PDY}/${MONITOR}
    fi
    if [[ ! -d ${test_dir} ]]; then
-      test_dir=${TANKverf}/${MONITOR}.${PDY}
+      test_dir=${TANKverf}/${RUN}.${PDY}
    fi
 
    echo "test_dir = ${test_dir}"
 
    if [[ -d ${test_dir} ]]; then
       echo " test_dir is GO "
-      test00=`ls ${test_dir}/angle.*${PDY}00*.ieee_d* | wc -l`
-      test06=`ls ${test_dir}/angle.*${PDY}06*.ieee_d* | wc -l`
-      test12=`ls ${test_dir}/angle.*${PDY}12*.ieee_d* | wc -l`
-      test18=`ls ${test_dir}/angle.*${PDY}18*.ieee_d* | wc -l`
-      if [[ $test00 -gt 0 ]]; then
-         test_list=`ls ${test_dir}/angle.*${PDY}00*.ieee_d*`
-         data_found=1
-      elif [[ $test06 -gt 0 ]]; then
-         test_list=`ls ${test_dir}/angle.*${PDY}06*.ieee_d*`
-         data_found=1
-      elif [[ $test12 -gt 0 ]]; then
-         test_list=`ls ${test_dir}/angle.*${PDY}12*.ieee_d*`
-         data_found=1
-      elif [[ $test18 -gt 0 ]]; then
-         test_list=`ls ${test_dir}/angle.*${PDY}18*.ieee_d*`
-         data_found=1
+
+      if [[ -e ${test_dir}/angle.tar ]]; then
+         test_list=`tar -tv ${test_dir}/angle.tar`
+	 data_found=1 
+      else
+         test=`ls ${test_dir}/angle.*${PDATE}*.ieee_d* | wc -l`
+         if [[ $test -gt 0 ]]; then
+            test_list=`ls ${test_dir}/angle.*${PDATE}*.ieee_d*`
+            data_found=1
+	 fi
       fi
    else
       echo "test_dir is NOGO"

diff --git a/util/Radiance_Monitor/image_gen/html/install_rgn.sh b/util/Radiance_Monitor/image_gen/html/install_rgn.sh
@@ -50,18 +50,10 @@ else
 fi
 
 
-#--------------------------------------------------------------
-#  source plot_rad_conf to get WEB_SVR, WEB_USER, WEBDIR
-#
-. ${RADMON_IMAGE_GEN}/parm/plot_rad_conf
-
-
 #--------------------------------------------------------------
 #  Get the area for this SUFFIX from the data_map file
 #
-
 new_webdir=${WEBDIR}/${SUFFIX}
-. ${RADMON_IMAGE_GEN}/parm/rgnl_conf
 
 echo RAD_AREA = $RAD_AREA
 echo TANKverf = $TANKverf

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_ANGLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_ANGLE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_angle )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_BCOEF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_BCOEF_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_bcoef )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_HORIZ_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_HORIZ_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_horiz )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_SUMMARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_SUMMARY_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_summary )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_TIME_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_TIME_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_time )

diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh
@@ -109,6 +109,10 @@ if [[ $? -ne 0 ]]; then
 fi
 
 
+if [[ ! -d ${IMGNDIR} ]]; then
+   mkdir -p ${IMGNDIR}
+fi
+
 #--------------------------------------------------------------------
 # Determine cycle to plot.  Exit if cycle is > last available
 # data.
@@ -183,7 +187,6 @@ export PDY=`echo $PDATE|cut -c1-8`
 #--------------------------------------------------------------------
 #  Locate ieee_src in $TANKverf and verify data files are present
 #
-
 ieee_src=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR}
 
 if [[ ! -d ${ieee_src} ]]; then
@@ -199,7 +202,16 @@ if [[ ! -d ${ieee_src} ]]; then
    exit 5
 fi
 
+#-----------------------------------------------------
+# check $ieee_src for data files.  If none are found
+# check contents of the radmon_angle.tar file.
+#
 nfile_src=`ls -l ${ieee_src}/*${PDATE}*ieee_d* | egrep -c '^-'`
+if [[ $nfile_src -le 0 ]]; then
+   if [[ -e ${ieee_src}/radmon_angle.tar ]]; then
+      nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d | wc -l`
+   fi
+fi
 
 if [[ $nfile_src -le 0 ]]; then
    echo " Missing ieee_src files, nfile_src = ${nfile_src}, aborting plot"
@@ -238,6 +250,9 @@ fi
 #  the $satype_file can't be found.
 #
 test_list=`ls ${ieee_src}/angle.*${PDATE}.ieee_d*`
+if [[ $test_list = "" ]]; then
+   test_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d` 
+fi
 
 for test in ${test_list}; do
    this_file=`basename $test`
@@ -332,11 +347,13 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then
       export WEBDIR=${WEBDIR}/${RADMON_SUFFIX}/${RUN}/pngs
 
       if [[ $MY_MACHINE = "wcoss2" ]]; then 
-         cmdfile=transfer_cmd
+         cmdfile="${PLOT_WORK_DIR}/transfer_cmd"
          echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile
+         chmod 755 $cmdfile
 
-         $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \
-              -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
+         run_time="$rhr$cmin"	# HHMM format for qsub
+         $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err \
+	      -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} -a ${run_time} ${cmdfile}
       else
          $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \
               -R affinity[core] -J ${jobname} -cwd ${PWD} -b $run_time ${job}

diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh
@@ -286,8 +286,8 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then
 	    cmdfile=transfer_cmd
 	    echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile
 
-	    $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \
-	         -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
+	    $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err 
+	         -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
 
 	 else
             $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \