Skip to content

Commit

Permalink
Github issue NOAA-EMC#373. Compress RadMon data files. Completes NOAA…
Browse files Browse the repository at this point in the history 
…-EMC#373.
  • Loading branch information
@EdwardSafford-NOAA
EdwardSafford-NOAA committed May 5, 2022
1 parent e7f8c88 commit 8d0050f
Show file tree
Hide file tree
Showing 15 changed files with 47 additions and 43 deletions.
2 changes: 1 addition & 1 deletion util/Radiance_Monitor/data_extract/ush/radmon_copy.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ if [[ ${exit_value} -eq 0 ]]; then

for type in ${type_list}; do

file_list=`ls ${DATA_LOCATION}/${type}.*${PDATE}*ieee_d* `
file_list=`ls ${DATA_LOCATION}/${type}.*${PDATE}*ieee_d* ${type}*tar* `

for file in ${file_list}; do
bfile=`basename ${file}`
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,8 @@ fi
#
nfile_src=`ls -l ${ieee_src}/*${PDATE}*ieee_d* | egrep -c '^-'`
if [[ $nfile_src -le 0 ]]; then
if [[ -e ${ieee_src}/radmon_angle.tar ]]; then
nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d | wc -l`
if [[ -e ${ieee_src}/radmon_angle.tar || -e ${ieee_src}/radmon_angle.tar.${Z} ]]; then
nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ieee_d | wc -l`
fi
fi

Expand Down Expand Up @@ -251,7 +251,7 @@ fi
#
test_list=`ls ${ieee_src}/angle.*${PDATE}.ieee_d*`
if [[ $test_list = "" ]]; then
test_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d`
test_list=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ieee_d`
fi

for test in ${test_list}; do
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,13 @@ for type in ${SATYPE}; do
# Determine if the angle files are in a tar file
# and, if so, extract the ctl files for this $type.
#
if [[ -s ${ieee_src}/radmon_angle.tar ]]; then
if [[ -s ${ieee_src}/radmon_angle.tar || -s ${ieee_src}/radmon_angle.tar.${Z} ]]; then
using_tar=1
ctl_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ctl`
ctl_list=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ${type} | grep ctl`
if [[ ${ctl_list} != "" ]]; then
cwd=`pwd`
cd ${ieee_src}
tar -xf ./radmon_angle.tar ${ctl_list}
tar -xf ./radmon_angle.tar* ${ctl_list}
cd ${cwd}
fi
fi
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,13 @@ for type in ${SATYPE}; do
# Determine if the bcoef files are in an tar file.
# if so extract the ctl files for this $type.
#
if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then
if [[ -s ${ieee_src}/radmon_bcoef.tar || -s ${ieee_src}/radmon_bcoef.tar.${Z} ]]; then
using_tar=1
ctl_list=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ctl`
ctl_list=`tar -tf ${ieee_src}/radmon_bcoef.tar* | grep ${type} | grep ctl`
if [[ ${ctl_list} != "" ]]; then
cwd=`pwd`
cd ${ieee_src}
tar -xf ./radmon_bcoef.tar ${ctl_list}
tar -xf ./radmon_bcoef.tar* ${ctl_list}
cd ${cwd}
fi
fi
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,13 @@ for type in ${SATYPE}; do
# so the code block that follows will work with both
# tarred and non-tarred storage schemes.
#
if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then
if [[ -s ${ieee_src}/radmon_bcor.tar || -s ${ieee_src}/radmon_bcor.tar.${Z} ]]; then
using_tar=1
ctl_list=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ctl`
ctl_list=`tar -tf ${ieee_src}/radmon_bcor.tar* | grep ${type} | grep ctl`
if [[ ${ctl_list} != "" ]]; then
cwd=`pwd`
cd ${ieee_src}
tar -xf ./radmon_bcor.tar ${ctl_list}
tar -xf ./radmon_bcor.tar* ${ctl_list}
cd ${cwd}
fi
fi
Expand Down
8 changes: 4 additions & 4 deletions util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,14 @@ for type in ${SATYPE}; do
# Determine if the time files are in an tar file.
# if so extract the ctl files for this $type.
#
if [[ -s ${ieee_src}/radmon_time.tar ]]; then
if [[ -s ${ieee_src}/radmon_time.tar || -s ${ieee_src}/radmon_time.tar.${Z} ]]; then
using_tar=1
ctl_list=`tar -tf ${ieee_src}/radmon_time.tar | grep $type | grep ctl`
ctl_list=`tar -tf ${ieee_src}/radmon_time.tar* | grep $type | grep ctl`
if [[ ${ctl_list} != "" ]]; then
cwd=`pwd`
cd ${ieee_src}
ctl_list=`tar -tf ./radmon_time.tar | grep $type | grep ctl`
tar -xf ${ieee_src}/radmon_time.tar ${ctl_list}
ctl_list=`tar -tf ./radmon_time.tar* | grep $type | grep ctl`
tar -xf ${ieee_src}/radmon_time.tar* ${ctl_list}
cd ${cwd}
fi
fi
Expand Down
4 changes: 2 additions & 2 deletions util/Radiance_Monitor/image_gen/ush/plot_angle.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,9 @@ for type in ${SATYPE2}; do
# and copy them locally.
#
if [[ -s ${ieee_src}/radmon_angle.tar ]]; then
files=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ieee_d`
files=`tar -tf ${ieee_src}/radmon_angle.tar* | grep ${type} | grep ieee_d`
if [[ ${files} != "" ]]; then
tar -xf ${ieee_src}/radmon_angle.tar ${files}
tar -xf ${ieee_src}/radmon_angle.tar* ${files}
fi
else
files=`ls ${ieee_src}/angle.*${type}*ieee_d*`
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,10 @@ for type in ${SATYPE}; do
# Locate the ieee_d files, first checking for a tar file,
# and copy them locally.
#
if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then
files=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ieee_d`
if [[ -s ${ieee_src}/radmon_bcoef.tar || -s ${ieee_src}/radmon_bcoef.tar ]]; then
files=`tar -tf ${ieee_src}/radmon_bcoef.tar* | grep ${type} | grep ieee_d`
if [[ ${files} != "" ]]; then
tar -xf ${ieee_src}/radmon_bcoef.tar ${files}
tar -xf ${ieee_src}/radmon_bcoef.tar* ${files}
fi
else
files=`ls ${ieee_src}/bcoef.*${type}*ieee_d*`
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/plot_bcor.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ for type in ${SATYPE2}; do
# Now locate this cycle's data files, first checking for
# a tar file, and copy them locally.
#
if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then
files=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ieee_d`
tar -xf ${ieee_src}/radmon_bcor.tar ${files}
if [[ -s ${ieee_src}/radmon_bcor.tar || -s ${ieee_src}/radmon_bcor.tar.${Z} ]]; then
files=`tar -tf ${ieee_src}/radmon_bcor.tar* | grep ${type} | grep ieee_d`
tar -xf ${ieee_src}/radmon_bcor.tar* ${files}

else
files=`ls ${ieee_src}/bcor.*${type}*ieee_d*`
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/plot_summary.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ for type in ${SATYPE2}; do
# Locate the data files, first checking for a tar file,
# and copy them locally.
#
if [[ -s ${ieee_src}/radmon_time.tar ]]; then
files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d`
if [[ -s ${ieee_src}/radmon_time.tar || -s ${ieee_src}/radmon_time.tar.{Z} ]]; then
files=`tar -tf ${ieee_src}/radmon_time.tar* | grep ${type} | grep ieee_d`
if [[ ${files} != "" ]]; then
tar -xf ${ieee_src}/radmon_time.tar ${files}
tar -xf ${ieee_src}/radmon_time.tar* ${files}
fi
else
files=`ls ${ieee_src}/time.*${type}*ieee_d*`
Expand Down
6 changes: 3 additions & 3 deletions util/Radiance_Monitor/image_gen/ush/plot_time.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,10 @@ for type in ${SATYPE2}; do
# Locate the data files, first checking for a tar file,
# and copy them locally.
#
if [[ -s ${ieee_src}/radmon_time.tar ]]; then
files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d`
if [[ -s ${ieee_src}/radmon_time.tar || -s ${ieee_src}/radmon_time.tar.${Z} ]]; then
files=`tar -tf ${ieee_src}/radmon_time.tar* | grep ${type} | grep ieee_d`
if [[ ${files} != "" ]]; then
tar -xf ${ieee_src}/radmon_time.tar ${files}
tar -xf ${ieee_src}/radmon_time.tar* ${files}
fi
else
files=`ls ${ieee_src}/time.*${type}*ieee_d*`
Expand Down
7 changes: 4 additions & 3 deletions util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,13 +229,14 @@ EOF

tar_file=radmon_angle.tar
tar -cf $tar_file angle*.ieee_d* angle*.ctl*
mv $tar_file ${TANKverf_rad}
${COMPRESS} ${tar_file}
mv $tar_file.${Z} ${TANKverf_rad}/.

if [[ $RAD_AREA = "rgn" ]]; then
cwd=`pwd`
cd ${TANKverf_rad}
tar -xf ${tar_file}
rm ${tar_file}
tar -xf ${tar_file}.${Z}
rm ${tar_file}.${Z}
cd ${cwd}
fi

Expand Down
7 changes: 4 additions & 3 deletions util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,13 +214,14 @@ EOF

tar_file=radmon_bcoef.tar
tar -cf $tar_file bcoef*.ieee_d* bcoef*.ctl*
mv $tar_file ${TANKverf_rad}
${COMPRESS} ${tar_file}
mv $tar_file.${Z} ${TANKverf_rad}

if [[ $RAD_AREA = "rgn" ]]; then
cwd=`pwd`
cd ${TANKverf_rad}
tar -xf ${tar_file}
rm ${tar_file}
tar -xf ${tar_file}.${Z}
rm ${tar_file}.${Z}
cd ${cwd}
fi

Expand Down
7 changes: 4 additions & 3 deletions util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,13 +207,14 @@ EOF
tar_file=radmon_bcor.tar

tar -cf $tar_file bcor*.ieee_d* bcor*.ctl*
mv $tar_file ${TANKverf_rad}
${COMPRESS} ${tar_file}
mv $tar_file.${Z} ${TANKverf_rad}/.

if [[ $RAD_AREA = "rgn" ]]; then
cwd=`pwd`
cd ${TANKverf_rad}
tar -xf ${tar_file}
rm ${tar_file}
tar -xf ${tar_file}.${Z}
rm ${tar_file}.${Z}
cd ${cwd}
fi

Expand Down
7 changes: 4 additions & 3 deletions util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,13 +264,14 @@ EOF

tar_file=radmon_time.tar
tar -cf $tar_file time*.ieee_d* time*.ctl*
mv $tar_file ${TANKverf_rad}
${COMPRESS} ${tar_file}
mv $tar_file.${Z} ${TANKverf_rad}/.

if [[ $RAD_AREA = "rgn" ]]; then
cwd=`pwd`
cd ${TANKverf_rad}
tar -xf ${tar_file}
rm ${tar_file}
tar -xf ${tar_file}.${Z}
rm ${tar_file}.${Z}
cd ${cwd}
fi

Expand Down

0 comments on commit 8d0050f

Please sign in to comment.