-
Notifications
You must be signed in to change notification settings - Fork 0
/
monitor_SEASONAL.sh
executable file
·202 lines (182 loc) · 7.02 KB
/
monitor_SEASONAL.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
##########################################################################################
#### CHECK THE SEASONAL FORECAST EXECUTION ###############################################
#### ENSEMBLE MEMBER IS ALWAYS 1 FOR HINDCAST ############################################
##########################################################################################
# assumptions: 10 months , 9 collections:
# assumptions: 10 months ,19 collections:
# available on tape: 9 monthly (10 months) 9 daily (10 months) 2 diurnal (10 months)
# available on tape: 19 monthly (10 months) 19 daily (10 months) 2 diurnal (10 months)
# available on disk: 19 monthly (10 months) 2 daily (10 months) 2 diurnal (10 months) 2 daily (3 months jul25)
############################################################################################
# PROVIDE YEAR AND EXPERIMENT ID
# YEAR: YYYY (1999-2015)
# ID : MMMDD (jan-dec 1-31)
############################################################################################
set YEAR = $1
set ENSN=$2
if ( ( ${ENSN} > 1 ) & ( ${ENSN} < 6 ) ) then
echo "ENSEMBLE NUMBER $ENSN IS RESERVED FOR subX, EXIT"
exit
else if ( ${ENSN} < 1 ) then
echo "ENSEMBLE NOT PROVIDED, CHECKING ENSEMBLE NUMBER 1"
set ENSN=1
endif
set npreops = `qme | grep preops | wc -l`
set ngmaodev = `qme | grep gmaodev | wc -l`
set ngmaofcst = `qme | grep gmaofcst | grep compute | wc -l`
echo "TOTAL PREOPS QUEUE JOBS NUMBER: $npreops"
echo "TOTAL GMAODEV QUEUE JOBS NUMBER: $ngmaodev"
echo "TOTAL GMAOFCST QUEUE JOBS NUMBER: $ngmaofcst"
echo
echo "================= ALL RUNS FOR YEAR AND ENSEMBLE #======="
qme | grep ${YEAR} | grep R${ENSN}
qme | grep ${YEAR} | grep P${ENSN}
qme | grep ${YEAR} | grep A${ENSN}
echo "========================================================="
set ncollections=21
ls -1 $GEOSS2S/util/submitted/ens${ENSN}gcm_setup${YEAR}*
set exists = $status
if ($exists > 0 ) then
echo "$YEAR FOR ENSEMBLE $ENSN is not available for monitoring- either completed or not submitted, EXIT"
exit
endif
set datesIC = `ls -1 $GEOSS2S/util/submitted/ens${ENSN}gcm_setup${YEAR}* | cut -c111-118`
if ( $ENSN > 9 ) set datesIC = `ls -1 $GEOSS2S/util/submitted/ens${ENSN}gcm_setup${YEAR}* | cut -c112-119`
echo "PROCESSING $datesIC"
@ arccount = 0
foreach dateIC ( $datesIC )
# get year
set yyIC = `echo $dateIC | cut -c1-4`
# get month
set mmIC = `echo $dateIC | cut -c5-6`
# get day
set ddIC = `echo $dateIC | cut -c7-8`
# get month name
if ($mmIC == '01') set mn = 'jan'
if ($mmIC == '02') set mn = 'feb'
if ($mmIC == '03') set mn = 'mar'
if ($mmIC == '04') set mn = 'apr'
if ($mmIC == '05') set mn = 'may'
if ($mmIC == '06') set mn = 'jun'
if ($mmIC == '07') set mn = 'jul'
if ($mmIC == '08') set mn = 'aug'
if ($mmIC == '09') set mn = 'sep'
if ($mmIC == '10') set mn = 'oct'
if ($mmIC == '11') set mn = 'nov'
if ($mmIC == '12') set mn = 'dec'
set ID=${mn}${ddIC}
set DD = `echo $ID | cut -c4-5`
#set JOBNAME = "${ID}_${YEAR}R$ENSN"
set JOBNAME = "${ID}_${YEAR}"
set NCOLLS=21
if ( $NCOLLS == 19 ) set ntape=400
if ( $NCOLLS == 19 ) set ndisk=236
if ( $NCOLLS == 21 ) set ntape=450
if ( $NCOLLS == 21) set ndisk=266
set MM = 2
if ( $DD > 14 ) then
set ndisk = 269
set MM = 3
endif
if ( -e $GEOSS2S/runh/$YEAR/${ID}/ens$ENSN/cap_restartIC ) then
set cap_sta1 = `cat $GEOSS2S/runh/$YEAR/${ID}/ens$ENSN/cap_restartIC`
else
echo "NO EXPERIMENT SUBMITTED: $YEAR $ID $ENSN "
goto NEXTIC
endif
if ( -e $GEOSS2S/runh/$YEAR/${ID}/ens$ENSN/DONE ) then
echo "DONE - NO ACTION NECESSARY FOR $YEAR $ID"
goto NEXTIC
endif
echo
echo "________________________________________________________________________________________"
echo JOB QUEUE STATUS:
qme | grep ${JOBNAME} | grep R${ENSN}
qme | grep ${JOBNAME} | grep P${ENSN}
qme | grep ${JOBNAME} | grep A${ENSN}
set cap_sta2 = `cat $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/cap_restart`
set cap_sta3 = `cat $GEOSS2S/runh/$YEAR/${ID}/ens$ENSN/CAP.rc | grep END_DATE | cut -c15-29`
echo
echo "_______________________________________________"
echo "____________DATE $YEAR $ID $ENSN"
echo "____________DATE START ${cap_sta1}"
echo "____________DATE DONE ${cap_sta2}"
echo "____________DATE LAST ${cap_sta3}"
echo "_______________________________________________"
@ nn = 0
@ afn = 0
echo 1- CHECK:
echo 1- IF ARCHIVE FILE NUMBER = 400: monthly 19x10+ diurnal 2x10 + daily 19x10
echo 1- IF ARCHIVE FILE NUMBER = ${ntape}: monthly 21x10+ diurnal 3x10 + daily 21x10
set afn1 = `ls -1 $ARCHIVE/GEOS_S2S/seasonal/Y$YEAR/${ID}/ens$ENSN/geosgcm_*/*nc4 | wc -l`
set afn2 = `ls -1 $ARCHIVE/GEOS_S2S/seasonal/Y$YEAR/${ID}/ens$ENSN/geosgcm*/*tar | wc -l`
@ afnn = $afn1 + $afn2
if ($afnn == $ntape) then
@ nn = $nn + 1
echo OK
else
echo " NOT READY: NUMBER FILES IN ARCHIVE IS: $afnn = $afn1 + $afn2"
endif
echo 2- CHECK:
echo "2- IF DISK FILE NUMBER == ${ndisk}: monthly 19x10+ diurnal 2x10 + daily subx 3x${MM}+ daily vis 2x10"
echo "2- IF DISK FILE NUMBER == ${ndisk}: monthly 21x10+ diurnal 3x10 + daily subx 3x${MM}+ daily vis 2x10"
set mfnu1 = `ls -1 $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/geosgcm_*/*monthly*nc4 | wc -l`
set mfnu2 = `ls -1 $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/geosgcm_*/diurnal/*diurnal*nc4 | wc -l`
set mfnu3 = `ls -1 $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/geosgcm_*/*tar | wc -l`
@ mfnu = $mfnu1 + $mfnu2
@ mfnu = $mfnu + $mfnu3
if ($mfnu == $ndisk) then
@ nn = $nn + 1
echo OK
else
echo " NOT READY: NUMBER FILES ON DISK IS: $mfnu = $mfnu1 + $mfnu2 + $mfnu3"
endif
echo 3- CHECK:
echo 3- IF RUN IS FINISHED BASED ON cap_restart
set lcap = `cat $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/cap_restart | cut -c1-8`
set endd = `cat $GEOSS2S/runh/$YEAR/${ID}/ens$ENSN/CAP.rc | grep END_DATE | cut -c15-22`
set endh = `cat $GEOSS2S/runh/$YEAR/${ID}/ens$ENSN/CAP.rc | grep END_DATE | cut -c15-20`
if ($lcap == $endd) then
@ nn = $nn + 1
echo OK
else
echo " NOT READY: cap_restart VALUE NOT YET LAST: $lcap VS $endd"
set runtag="${ID}_${YEAR}R$ENSN"
qme | grep $runtag
set qstat = $status
if ($qstat != 0) echo " JOB $runtag IS NOT RUNNING"
endif
echo 4- CHECK:
echo "4- IF HOLDING CONTENT IS ONLY IN $endh"
set nhld1 = `ls -1 $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/holding/geos*/*/*nc4 | wc -l`
set qstat = $status
set nhld2 = `ls -1 $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/holding/geos*/$endh/*nc4 | wc -l`
if ( ! -e $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/holding) then
echo " NO HOLDING - NO ACTION NECESSARY"
#exit
endif
if ( ($qstat == 0) & ($nhld1 == $nhld2) ) then
@ nn = $nn + 1
echo OK
else
echo " NOT READY: HOLDING CONTENT: $nhld1 $nhld2"
endif
echo 5- CHECK:
echo 5- IF MOM_Output HAS 9 MONTHLY FILES
set nmom1 = `ls -1 $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/MOM_Output/ocean_monthly.*nc | wc -l`
ls -1 $GEOSS2S/runx/$YEAR/${ID}/ens$ENSN/MOM_Output/ | grep NOTMOM
set mstat = $status
if ( ( ${nmom1} == 9 ) & ( $mstat != 0 ) ) then
@ nn = $nn + 1
echo OK
else
echo " NOT READY: NUMBER MONTHLIES ${nmom1} OR MOM TRANSFER ERROR $mstat"
endif
echo SUMMARY:
if ($nn == 5) then
echo " COMPLETED: ALL CONDITIONS MET TO CLEAN DISK"
else
echo " NOT READY: $nn CONDITIONS MET FOR CLEANING"
endif
NEXTIC:
end