diff --git a/fix b/fix index 6720dce3c1..384204259f 160000 --- a/fix +++ b/fix @@ -1 +1 @@ -Subproject commit 6720dce3c168ab3a211c265d2a870c20bd220e35 +Subproject commit 384204259f72b55ae8dd1b3e6959b5bc735ce904 diff --git a/src/gsi/gsimod.F90 b/src/gsi/gsimod.F90 index 95ff7f79b4..7177719fde 100644 --- a/src/gsi/gsimod.F90 +++ b/src/gsi/gsimod.F90 @@ -35,6 +35,7 @@ module gsimod use obsmod, only: luse_obsdiag use obsmod, only: netcdf_diag, binary_diag use obsmod, only: l_wcp_cwm,ompslp_mult_fact + use obsmod, only: l_obsprvdiag use obsmod, only: aircraft_recon, & ! The following variables are the coefficients that describe @@ -480,6 +481,9 @@ module gsimod ! option for checking and adjusting the profile of Qr/Qs/Qg/Qnr ! retrieved through cloud analysis to reduce the background ! reflectivity ghost in analysis. (default is 0) +! 2021-11-16 Zhao - add option l_obsprvdiag (if true) to trigger the output of +! observation provider and sub-provider information into +! obsdiags files (used for AutoObsQC) ! 01-07-2022 Hu Add fv3_io_layout_y to let fv3lam interface read/write subdomain restart ! files. The fv3_io_layout_y needs to match fv3lam model ! option io_layout(2). @@ -699,6 +703,8 @@ module gsimod ! (.TRUE.: on; .FALSE.: off) / Inputfile: l2rwbufr_cltl (bufr format) ! l_use_dbz_directDA - option to assimilate radar reflectivity obs directly in GSI ! (.TRUE.: on; .FALSE.: off) / Inputfile: dbzbufr (bufr format) +! l_obsprvdiag - trigger (if true) writing out observation provider and sub-provider +! information into obsdiags files (used for AutoObsQC) ! ! NOTE: for now, if in regional mode, then iguess=-1 is forced internally. ! add use of guess file later for regional mode. @@ -743,7 +749,7 @@ module gsimod if_model_dbz,imp_physics,lupp,netcdf_diag,binary_diag,l_wcp_cwm,aircraft_recon,diag_version,& write_fv3_incr,incvars_to_zero,incvars_zero_strat,incvars_efold,diag_version,& cao_check,lcalc_gfdl_cfrac,tau_fcst,efsoi_order,lupdqc,lqcoef,cnvw_option,l2rwthin,hurricane_radar,& - l_reg_update_hydro_delz, & + l_reg_update_hydro_delz, l_obsprvdiag,& l_use_dbz_directDA, l_use_rw_columntilt ! GRIDOPTS (grid setup variables,including regional specific variables): diff --git a/src/gsi/obsmod.F90 b/src/gsi/obsmod.F90 index 2d9209e774..40dc3cc0dc 100644 --- a/src/gsi/obsmod.F90 +++ b/src/gsi/obsmod.F90 @@ -157,6 +157,9 @@ module obsmod ! GSI namelist level. ! 2020-09-15 Wu - add option tcp_posmatch to mitigate possibility of erroneous TC initialization ! 2020-09-19 CAPS(J. Park) - add 'vad_near_analtime' flag to assimilate newvad obs around analysis time only +! 2021-11-16 Zhao - add option l_obsprvdiag (if true) to trigger the output of +! observation provider and sub-provider information into +! obsdiags files (used for AutoObsQC) ! ! Subroutines Included: ! sub init_obsmod_dflts - initialize obs related variables to default values @@ -398,6 +401,7 @@ module obsmod ! (nobs_type,npe) ! def binary_diag - trigger binary diag-file output (being phased out) ! def netcdf_diag - trigger netcdf diag-file output +! def l_obsprvdiag - trigger obs provider info output into obsdiags files ! def l_wcp_cwm - namelist logical whether to use operator that ! includes cwm for both swcp and lwcp or not ! def neutral_stability_windfact_2dvar - logical, if .true., then use simple formula representing @@ -484,6 +488,7 @@ module obsmod public :: nobs_sub public :: netcdf_diag, binary_diag + public :: l_obsprvdiag public :: l_wcp_cwm public :: aircraft_recon @@ -553,6 +558,7 @@ module obsmod logical luse_obsdiag logical binary_diag, netcdf_diag + logical l_obsprvdiag ! Declare types @@ -719,6 +725,7 @@ subroutine init_obsmod_dflts ! 2015-07-10 pondeca - add cldch ! 2015-10-27 todling - default to luse_obsdiag is true now ! 2016-03-07 pondeca - add uwnd10m,vwnd10m +! 2021-11-16 zhao - add initialization of l_obsprvdiag (.FALSE. as default) ! ! input argument list: ! @@ -911,6 +918,9 @@ subroutine init_obsmod_dflts netcdf_diag = .false. ! by default, do not write netcdf_diag binary_diag = .true. ! by default, do write binary diag +! set default on triggering the output of obs provider info into obsdiags file + l_obsprvdiag = .false. ! by default, do not write obs provider info + l_wcp_cwm = .false. ! .true. = use operator that involves cwm aircraft_recon = .false. ! .true. = use DOE for aircraft data hurricane_radar = .false. ! .true. = use radar data for hurricane application diff --git a/src/gsi/setupps.f90 b/src/gsi/setupps.f90 index 69379f15f7..04344a2f7e 100644 --- a/src/gsi/setupps.f90 +++ b/src/gsi/setupps.f90 @@ -81,6 +81,11 @@ subroutine setupps(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsa ! 2017-03-31 Hu - addd option l_closeobs to use closest obs to analysis ! time in analysis ! 2019-09-20 Su - remove current VQC part and add subroutine call on VQC and add new VQC option +! 2021-10-xx pondeca/morris/zhao - added observation provider/subprovider +! information in diagonostic file, which is used +! in offline observation quality control program (AutoObsQC) +! for 3D-RTMA (if l_obsprvdiag is true). +! ! ! input argument list: ! lunin - unit from which to read observations @@ -118,6 +123,7 @@ subroutine setupps(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsa use gsi_4dvar, only: nobs_bins,hr_obsbin,min_offset use oneobmod, only: magoberr,maginnov,oneobtest use obsmod, only: netcdf_diag, binary_diag, dirname + use obsmod, only: l_obsprvdiag use nc_diag_write_mod, only: nc_diag_init, nc_diag_header, nc_diag_metadata, & nc_diag_write, nc_diag_data2d use nc_diag_read_mod, only: nc_diag_read_init, nc_diag_read_get_dim, nc_diag_read_close @@ -321,7 +327,10 @@ subroutine setupps(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsa ioff0=20 nreal=ioff0 if (lobsdiagsave) nreal=nreal+4*miter+1 - if (twodvar_regional) then; nreal=nreal+2; allocate(cprvstg(nobs),csprvstg(nobs)); endif + if (twodvar_regional .or. l_obsprvdiag) then + nreal=nreal+2 !account for idomsfc,izz + allocate(cprvstg(nobs),csprvstg(nobs)) !obs provider info + endif if (save_jacobian) then nnz = 1 ! number of non-zero elements in dH(x)/dx profile nind = 1 @@ -667,13 +676,13 @@ subroutine setupps(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsa if(binary_diag .and. ii>0)then write(7)' ps',nchar,nreal,ii,mype,ioff0 write(7)cdiagbuf(1:ii),rdiagbuf(:,1:ii) - deallocate(cdiagbuf,rdiagbuf) - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then write(7)cprvstg(1:ii),csprvstg(1:ii) deallocate(cprvstg,csprvstg) endif end if + deallocate(cdiagbuf,rdiagbuf) end if ! End of routine @@ -856,7 +865,7 @@ subroutine contents_binary_diag_(odiag) enddo endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then ioff = ioff + 1 rdiagbuf(ioff,ii) = data(idomsfc,i) ! dominate surface type ioff = ioff + 1 @@ -904,6 +913,8 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_metadata("Observation", sngl(pob) ) call nc_diag_metadata("Obs_Minus_Forecast_adjusted", sngl(pob-pges) ) call nc_diag_metadata("Obs_Minus_Forecast_unadjusted", sngl(pob-pgesorig)) + call nc_diag_metadata("Forecast_adjusted", sngl(pges) ) + call nc_diag_metadata("Forecast_unadjusted", sngl(pgesorig) ) if (lobsdiagsave) then @@ -921,7 +932,7 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_data2d("ObsDiagSave_obssen", odiag%obssen ) endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then call nc_diag_metadata("Dominant_Sfc_Type", data(idomsfc,i) ) call nc_diag_metadata("Model_Terrain", data(izz,i) ) r_prvstg = data(iprvd,i) @@ -936,6 +947,8 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_data2d("Observation_Operator_Jacobian_val", real(dhx_dx%val,r_single)) endif + call nc_diag_data2d("atmosphere_pressure_coordinate", sngl(exp(prsltmp)*r1000)) + end subroutine contents_netcdf_diag_ subroutine final_vars_ diff --git a/src/gsi/setupq.f90 b/src/gsi/setupq.f90 index 8189b0bcb5..719b14f750 100644 --- a/src/gsi/setupq.f90 +++ b/src/gsi/setupq.f90 @@ -107,6 +107,10 @@ subroutine setupq(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav ! error (DOE) calculation to the namelist ! level; they are now loaded by ! aircraftinfo. +! 2021-10-xx pondeca/morris/zhao - added observation provider/subprovider +! information in diagonostic file, which is used +! in offline observation quality control program (AutoObsQC) +! for 3D-RTMA (if l_obsprvdiag is true). ! ! ! input argument list: @@ -144,6 +148,7 @@ subroutine setupq(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav use m_obsLList, only: obsLList use obsmod, only: luse_obsdiag,ianldate use obsmod, only: netcdf_diag, binary_diag, dirname + use obsmod, only: l_obsprvdiag use nc_diag_write_mod, only: nc_diag_init, nc_diag_header, nc_diag_metadata, & nc_diag_write, nc_diag_data2d use nc_diag_read_mod, only: nc_diag_read_init, nc_diag_read_get_dim, nc_diag_read_close @@ -245,6 +250,7 @@ subroutine setupq(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav character(8) station_id character(8),allocatable,dimension(:):: cdiagbuf,cdiagbufp character(8),allocatable,dimension(:):: cprvstg,csprvstg + character(8),allocatable,dimension(:):: cprvstgp,csprvstgp ! <-- provider info array for pseudo obs character(8) c_prvstg,c_sprvstg real(r_double) r_prvstg,r_sprvstg @@ -394,7 +400,11 @@ subroutine setupq(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav ioff0=21 nreal=ioff0 if (lobsdiagsave) nreal=nreal+4*miter+1 - if (twodvar_regional) then; nreal=nreal+2; allocate(cprvstg(nobs),csprvstg(nobs)); endif + if (twodvar_regional .or. l_obsprvdiag) then + nreal=nreal+2 ! account for idomsfc,izz + allocate(cprvstg(nobs),csprvstg(nobs)) ! obs provider info + if(l_pbl_pseudo_surfobsq) allocate(cprvstgp(nobs*3),csprvstgp(nobs*3)) ! obs provider info for pseudo obs + endif if (save_jacobian) then nnz = 2 ! number of non-zero elements in dH(x)/dx profile nind = 1 @@ -442,6 +452,7 @@ subroutine setupq(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav dpres=data(ipres,i) rmaxerr=data(iqmax,i) + rstation_id = data(id,i) error=data(ier2,i) prest=r10*exp(dpres) ! in mb var_jb=data(ijb,i) @@ -920,11 +931,11 @@ subroutine setupq(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav if (err_adjst>tiny_r_kind) errinv_adjst = one/err_adjst if (err_final>tiny_r_kind) errinv_final = one/err_final - if(binary_diag) call contents_binary_diagp_() + if(binary_diag) call contents_binary_diagp_(my_diag_pbl) else iip=3*nobs endif - if(netcdf_diag) call contents_netcdf_diagp_() + if(netcdf_diag) call contents_netcdf_diagp_(my_diag_pbl) endif !conv_diagsave .and. luse(i)) prest = prest - pps_press_incr @@ -945,20 +956,25 @@ subroutine setupq(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav if(conv_diagsave) then if(netcdf_diag) call nc_diag_write if(binary_diag .and. ii>0)then - write(7)' q',nchar,nreal,ii+iip,mype,ioff0 + write(7)' q',nchar,nreal,ii+iip,mype,ioff0,iip if(l_pbl_pseudo_surfobsq .and. iip>0) then write(7)cdiagbuf(1:ii),cdiagbufp(1:iip),rdiagbuf(:,1:ii),rdiagbufp(:,1:iip) - deallocate(cdiagbufp,rdiagbufp) else write(7)cdiagbuf(1:ii),rdiagbuf(:,1:ii) endif - deallocate(cdiagbuf,rdiagbuf) - if (twodvar_regional) then - write(7)cprvstg(1:ii),csprvstg(1:ii) + if (twodvar_regional .or. l_obsprvdiag) then + if(l_pbl_pseudo_surfobsq .and. iip>0) then + write(7)cprvstg(1:ii),cprvstgp(1:iip),csprvstg(1:ii),csprvstgp(1:iip) + else + write(7)cprvstg(1:ii),csprvstg(1:ii) + endif deallocate(cprvstg,csprvstg) + if(l_pbl_pseudo_surfobsq) deallocate(cprvstgp,csprvstgp) endif endif + deallocate(cdiagbuf,rdiagbuf) + if(l_pbl_pseudo_surfobsq) deallocate(cdiagbufp,rdiagbufp) end if ! End of routine @@ -1148,7 +1164,7 @@ subroutine contents_binary_diag_(odiag) enddo endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then ioff = ioff + 1 rdiagbuf(ioff,ii) = data(idomsfc,i) ! dominate surface type ioff = ioff + 1 @@ -1165,12 +1181,13 @@ subroutine contents_binary_diag_(odiag) end subroutine contents_binary_diag_ - subroutine contents_binary_diagp_ + subroutine contents_binary_diagp_(odiag) + type(obs_diag),pointer,intent(in):: odiag cdiagbufp(iip) = station_id ! station id rdiagbufp(1,iip) = ictype(ikx) ! observation type - rdiagbufp(2,iip) = icsubtype(ikx) ! observation subtype + rdiagbufp(2,iip) = -1 ! observation subtype (-1 for pseudo obs sub-type) rdiagbufp(3,iip) = data(ilate,i) ! observation latitude (degrees) rdiagbufp(4,iip) = data(ilone,i) ! observation longitude (degrees) @@ -1201,8 +1218,43 @@ subroutine contents_binary_diagp_ rdiagbufp(21,iip) = 1e+10_r_single ! spread (filled in by EnKF) ioff=ioff0 +!---- + if (lobsdiagsave) then + do jj=1,miter + ioff=ioff+1 + if (odiag%muse(jj)) then + rdiagbufp(ioff,iip) = one + else + rdiagbufp(ioff,iip) = -one + endif + enddo + do jj=1,miter+1 + ioff=ioff+1 + rdiagbufp(ioff,iip) = odiag%nldepart(jj) + enddo + do jj=1,miter + ioff=ioff+1 + rdiagbufp(ioff,iip) = odiag%tldepart(jj) + enddo + do jj=1,miter + ioff=ioff+1 + rdiagbufp(ioff,iip) = odiag%obssen(jj) + enddo + endif + + if (twodvar_regional .or. l_obsprvdiag) then + ioff = ioff + 1 + rdiagbufp(ioff,iip) = -9999._r_single ! data(idomsfc,i) ! dominate surface type + ioff = ioff + 1 + rdiagbufp(ioff,iip) = -9999._r_single ! data(izz,i) ! model terrain at ob location + r_prvstg = data(iprvd,i) + cprvstgp(iip) = '88888888' !c_prvstg ! provider name + r_sprvstg = data(isprvd,i) + csprvstgp(iip) = '88888888' !c_sprvstg ! subprovider name + endif +!---- if (save_jacobian) then - call writearray(dhx_dx, rdiagbuf(ioff+1:nreal,ii)) + call writearray(dhx_dx, rdiagbufp(ioff+1:nreal,iip)) ioff = ioff + size(dhx_dx) endif @@ -1240,6 +1292,8 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_metadata("Observation", sngl(data(iqob,i))) call nc_diag_metadata("Obs_Minus_Forecast_adjusted", sngl(ddiff) ) call nc_diag_metadata("Obs_Minus_Forecast_unadjusted", sngl(qob-qges) ) + call nc_diag_metadata("Forecast_adjusted", sngl(data(iqob,i)-ddiff)) + call nc_diag_metadata("Forecast_unadjusted", sngl(qges)) call nc_diag_metadata("Forecast_Saturation_Spec_Hum", sngl(qsges) ) if (lobsdiagsave) then do jj=1,miter @@ -1256,7 +1310,7 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_data2d("ObsDiagSave_obssen", odiag%obssen ) endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then call nc_diag_metadata("Dominant_Sfc_Type", data(idomsfc,i) ) call nc_diag_metadata("Model_Terrain", data(izz,i) ) r_prvstg = data(iprvd,i) @@ -1270,16 +1324,20 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_data2d("Observation_Operator_Jacobian_val", real(dhx_dx%val,r_single)) endif + call nc_diag_data2d("atmosphere_pressure_coordinate",exp(prsltmp)*r1000) + end subroutine contents_netcdf_diag_ - subroutine contents_netcdf_diagp_ + subroutine contents_netcdf_diagp_(odiag) + type(obs_diag),pointer,intent(in):: odiag ! Observation class character(7),parameter :: obsclass = ' q' + real(r_kind),dimension(miter) :: obsdiag_iuse call nc_diag_metadata("Station_ID", station_id ) call nc_diag_metadata("Observation_Class", obsclass ) call nc_diag_metadata("Observation_Type", ictype(ikx) ) - call nc_diag_metadata("Observation_Subtype", icsubtype(ikx) ) + call nc_diag_metadata("Observation_Subtype", -1 ) ! (-1 for pseudo obs sub-type) call nc_diag_metadata("Latitude", sngl(data(ilate,i)) ) call nc_diag_metadata("Longitude", sngl(data(ilone,i)) ) call nc_diag_metadata("Station_Elevation", sngl(data(istnelv,i)) ) @@ -1302,13 +1360,42 @@ subroutine contents_netcdf_diagp_ call nc_diag_metadata("Observation", sngl(data(iqob,i))) call nc_diag_metadata("Obs_Minus_Forecast_adjusted", sngl(ddiff) ) call nc_diag_metadata("Obs_Minus_Forecast_unadjusted", sngl(ddiff) ) + call nc_diag_metadata("Forecast_adjusted", sngl(data(iqob,i)-ddiff)) + call nc_diag_metadata("Forecast_unadjusted", sngl(data(iqob,i)-ddiff)) call nc_diag_metadata("Forecast_Saturation_Spec_Hum", sngl(qsges) ) +!---- + if (lobsdiagsave) then + do jj=1,miter + if (odiag%muse(jj)) then + obsdiag_iuse(jj) = one + else + obsdiag_iuse(jj) = -one + endif + enddo + call nc_diag_data2d("ObsDiagSave_iuse", obsdiag_iuse ) + call nc_diag_data2d("ObsDiagSave_nldepart", odiag%nldepart ) + call nc_diag_data2d("ObsDiagSave_tldepart", odiag%tldepart ) + call nc_diag_data2d("ObsDiagSave_obssen", odiag%obssen ) + endif + + if (twodvar_regional .or. l_obsprvdiag) then + call nc_diag_metadata("Dominant_Sfc_Type", data(idomsfc,i) ) + call nc_diag_metadata("Model_Terrain", data(izz,i) ) + r_prvstg = data(iprvd,i) + call nc_diag_metadata("Provider_Name", "88888888" ) + r_sprvstg = data(isprvd,i) + call nc_diag_metadata("Subprovider_Name", "88888888" ) + endif +!---- if (save_jacobian) then call nc_diag_data2d("Observation_Operator_Jacobian_stind", dhx_dx%st_ind) call nc_diag_data2d("Observation_Operator_Jacobian_endind", dhx_dx%end_ind) call nc_diag_data2d("Observation_Operator_Jacobian_val", real(dhx_dx%val,r_single)) endif + + call nc_diag_data2d("atmosphere_pressure_coordinate",exp(prsltmp)*r1000) + end subroutine contents_netcdf_diagp_ subroutine final_vars_ diff --git a/src/gsi/setupspd.f90 b/src/gsi/setupspd.f90 index 1e740fdd25..91b2467bf3 100644 --- a/src/gsi/setupspd.f90 +++ b/src/gsi/setupspd.f90 @@ -79,6 +79,10 @@ subroutine setupspd(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diags ! error (DOE) calculation to the namelist ! level; they are now loaded by ! aircraftinfo. +! 2021-10-xx pondeca/morris/zhao - added observation provider/subprovider +! information in diagonostic file, which is used +! in offline observation quality control program (AutoObsQC) +! for 3D-RTMA (if l_obsprvdiag is true). ! ! input argument list: ! lunin - unit from which to read observations @@ -108,6 +112,7 @@ subroutine setupspd(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diags lobsdiagsave,nobskeep,lobsdiag_allocated,time_offset,& lobsdiag_forenkf,aircraft_recon use obsmod, only: netcdf_diag, binary_diag, dirname, ianldate + use obsmod, only: l_obsprvdiag use nc_diag_write_mod, only: nc_diag_init, nc_diag_header, nc_diag_metadata, & nc_diag_write, nc_diag_data2d use nc_diag_read_mod, only: nc_diag_read_init, nc_diag_read_get_dim, nc_diag_read_close @@ -280,7 +285,10 @@ subroutine setupspd(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diags ioff0=21 nreal=ioff0 if (lobsdiagsave) nreal=nreal+4*miter+1 - if (twodvar_regional) then; nreal=nreal+2; allocate(cprvstg(nobs),csprvstg(nobs)); endif + if (twodvar_regional .or. l_obsprvdiag) then + nreal=nreal+2 ! account for idomsfc,izz + allocate(cprvstg(nobs),csprvstg(nobs)) ! obs provider info + endif if (save_jacobian) then nnz = 4 ! number of non-zero elements in dH(x)/dx profile nind = 2 @@ -684,13 +692,13 @@ subroutine setupspd(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diags if(binary_diag .and. ii>0)then write(7)'spd',nchar,nreal,ii,mype,ioff0 write(7)cdiagbuf(1:ii),rdiagbuf(:,1:ii) - deallocate(cdiagbuf,rdiagbuf) - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then write(7)cprvstg(1:ii),csprvstg(1:ii) deallocate(cprvstg,csprvstg) endif end if + deallocate(cdiagbuf,rdiagbuf) end if ! End of routine @@ -915,7 +923,7 @@ subroutine contents_binary_diag_(odiag) enddo endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then ioff = ioff + 1 rdiagbuf(ioff,ii) = data(idomsfc,i) ! dominate surface type ioff = ioff + 1 @@ -980,7 +988,7 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_data2d("ObsDiagSave_obssen", odiag%obssen ) endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then call nc_diag_metadata("Dominant_Sfc_Type", data(idomsfc,i) ) call nc_diag_metadata("Model_Terrain", data(izz,i) ) r_prvstg = data(iprvd,i) diff --git a/src/gsi/setupt.f90 b/src/gsi/setupt.f90 index d94d1f26dc..e7e73d82c3 100644 --- a/src/gsi/setupt.f90 +++ b/src/gsi/setupt.f90 @@ -40,6 +40,7 @@ subroutine setupt(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav use gsi_4dvar, only: nobs_bins,hr_obsbin,min_offset use obsmod, only: netcdf_diag, binary_diag, dirname + use obsmod, only: l_obsprvdiag use nc_diag_write_mod, only: nc_diag_init, nc_diag_header, nc_diag_metadata, & nc_diag_write, nc_diag_data2d use nc_diag_read_mod, only: nc_diag_read_init, nc_diag_read_get_dim, nc_diag_read_close @@ -69,7 +70,7 @@ subroutine setupt(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav use converr, only: ptabl use rapidrefresh_cldsurf_mod, only: l_gsd_terrain_match_surftobs,l_sfcobserror_ramp_t use rapidrefresh_cldsurf_mod, only: l_pbl_pseudo_surfobst, pblh_ration,pps_press_incr - use rapidrefresh_cldsurf_mod, only: i_use_2mt4b,i_sfct_gross,l_closeobs,i_coastline + use rapidrefresh_cldsurf_mod, only: i_use_2mt4b,i_sfct_gross,l_closeobs,i_coastline use aircraftinfo, only: npredt,predt,aircraft_t_bc_pof,aircraft_t_bc, & aircraft_t_bc_ext,ostats_t,rstats_t,upd_pred_t @@ -221,6 +222,10 @@ subroutine setupt(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav ! observation error (DOE) calculation to ! the namelist level; they are now ! loaded by obsmod. +! 2021-10-xx pondeca/morris/zhao - added observation provider/subprovider +! information in diagonostic file, which is used +! in offline observation quality control program (AutoObsQC) +! for 3D-RTMA (if l_obsprvdiag is true). ! ! !REMARKS: ! language: f90 @@ -296,6 +301,7 @@ subroutine setupt(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav character(8) station_id character(8),allocatable,dimension(:):: cdiagbuf,cdiagbufp character(8),allocatable,dimension(:):: cprvstg,csprvstg + character(8),allocatable,dimension(:):: cprvstgp,csprvstgp ! <-- provider info array for pseudo obs character(8) c_prvstg,c_sprvstg real(r_double) r_prvstg,r_sprvstg @@ -469,7 +475,11 @@ subroutine setupt(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav nreal=nreal+npredt+2 idia0=nreal if (lobsdiagsave) nreal=nreal+4*miter+1 - if (twodvar_regional) then; nreal=nreal+2; allocate(cprvstg(nobs),csprvstg(nobs)); endif + if (twodvar_regional .or. l_obsprvdiag) then + nreal=nreal+2 ! account for idomsfc, izz used in diag for RTMA + allocate(cprvstg(nobs),csprvstg(nobs)) ! provider/subprovider info + if(l_pbl_pseudo_surfobst) allocate(cprvstgp(nobs*3),csprvstgp(nobs*3)) ! provider of pseudo obs + endif if (save_jacobian) then nnz = 2 ! number of non-zero elements in dH(x)/dx profile nind = 1 @@ -1233,12 +1243,12 @@ subroutine setupt(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav if (err_adjst>tiny_r_kind) errinv_adjst=one/err_adjst if (err_final>tiny_r_kind) errinv_final=one/err_final - if(binary_diag) call contents_binary_diagp_ + if(binary_diag) call contents_binary_diagp_(my_diag_pbl) else iip=nobs endif - if(netcdf_diag) call contents_netcdf_diagp_ + if(netcdf_diag) call contents_netcdf_diagp_(my_diag_pbl) end if prest = prest - pps_press_incr @@ -1260,20 +1270,25 @@ subroutine setupt(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav if(conv_diagsave)then if(netcdf_diag) call nc_diag_write if(binary_diag .and. ii>0)then - write(7)' t',nchar,nreal,ii+iip,mype,idia0 + write(7)' t',nchar,nreal,ii+iip,mype,idia0,iip if(l_pbl_pseudo_surfobst .and. iip>0) then write(7)cdiagbuf(1:ii),cdiagbufp(1:iip),rdiagbuf(:,1:ii),rdiagbufp(:,1:iip) - deallocate(cdiagbufp,rdiagbufp) else write(7)cdiagbuf(1:ii),rdiagbuf(:,1:ii) endif - deallocate(cdiagbuf,rdiagbuf) - if (twodvar_regional) then - write(7)cprvstg(1:ii),csprvstg(1:ii) + if (twodvar_regional .or. l_obsprvdiag) then + if(l_pbl_pseudo_surfobst .and. iip>0) then + write(7)cprvstg(1:ii),cprvstgp(1:iip),csprvstg(1:ii),csprvstgp(1:iip) + else + write(7)cprvstg(1:ii),csprvstg(1:ii) + endif deallocate(cprvstg,csprvstg) + if(l_pbl_pseudo_surfobst) deallocate(cprvstgp,csprvstgp) endif end if + deallocate(cdiagbuf,rdiagbuf) + if(l_pbl_pseudo_surfobst) deallocate(cdiagbufp,rdiagbufp) end if @@ -1550,7 +1565,7 @@ subroutine contents_binary_diag_(odiag) enddo endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then idia = idia + 1 rdiagbuf(idia,ii) = data(idomsfc,i) ! dominate surface type idia = idia + 1 @@ -1568,12 +1583,13 @@ subroutine contents_binary_diag_(odiag) end subroutine contents_binary_diag_ - subroutine contents_binary_diagp_ + subroutine contents_binary_diagp_(odiag) + type(obs_diag),pointer,intent(in):: odiag cdiagbufp(iip) = station_id ! station id rdiagbufp(1,iip) = ictype(ikx) ! observation type - rdiagbufp(2,iip) = icsubtype(ikx) ! observation subtype + rdiagbufp(2,iip) = -1 ! observation subtype (-1 for pseudo obs sub-type) rdiagbufp(3,iip) = data(ilate,i) ! observation latitude (degrees) rdiagbufp(4,iip) = data(ilone,i) ! observation longitude (degrees) @@ -1604,8 +1620,44 @@ subroutine contents_binary_diagp_ rdiagbufp(20,iip) = 1.e10_r_single ! spread (filled in by EnKF) idia=idia0 +!---- + if (lobsdiagsave) then + do jj=1,miter + idia=idia+1 + if (odiag%muse(jj)) then + rdiagbufp(idia,iip) = one + else + rdiagbufp(idia,iip) = -one + endif + enddo + do jj=1,miter+1 + idia=idia+1 + rdiagbufp(idia,iip) = odiag%nldepart(jj) + enddo + do jj=1,miter + idia=idia+1 + rdiagbufp(idia,iip) = odiag%tldepart(jj) + enddo + do jj=1,miter + idia=idia+1 + rdiagbufp(idia,iip) = odiag%obssen(jj) + enddo + endif + + if (twodvar_regional .or. l_obsprvdiag) then + idia = idia + 1 + rdiagbufp(idia,iip) = -9999._r_single ! data(idomsfc,i) ! dominate surface type + idia = idia + 1 + rdiagbufp(idia,iip) = -9999._r_single ! data(izz,i) ! model terrain at observation location +! r_prvstg = data(iprvd,i) + cprvstgp(iip) = '88888888' !c_prvstg ! provider name +! r_sprvstg = data(isprvd,i) + csprvstgp(iip) = '88888888' !c_sprvstg ! subprovider name + endif +!---- + if (save_jacobian) then - call writearray(dhx_dx, rdiagbuf(idia+1:nreal,ii)) + call writearray(dhx_dx, rdiagbufp(idia+1:nreal,iip)) idia = idia + size(dhx_dx) endif @@ -1645,6 +1697,9 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_metadata("Observation", sngl(data(itob,i)) ) call nc_diag_metadata("Obs_Minus_Forecast_adjusted", sngl(ddiff) ) call nc_diag_metadata("Obs_Minus_Forecast_unadjusted", sngl(tob-tges) ) + call nc_diag_metadata("Forecast_unadjusted", sngl(tges)) + call nc_diag_metadata("Forecast_adjusted", sngl(data(itob,i)-ddiff)) + if (aircraft_t_bc_pof .or. aircraft_t_bc .or. aircraft_t_bc_ext) then call nc_diag_metadata("Data_Pof", sngl(data(ipof,i)) ) call nc_diag_metadata("Data_Vertical_Velocity", sngl(data(ivvlc,i)) ) @@ -1681,7 +1736,7 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_data2d("ObsDiagSave_obssen", odiag%obssen ) endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then call nc_diag_metadata("Dominant_Sfc_Type", data(idomsfc,i) ) call nc_diag_metadata("Model_Terrain", data(izz,i) ) r_prvstg = data(iprvd,i) @@ -1696,17 +1751,23 @@ subroutine contents_netcdf_diag_(odiag) call nc_diag_data2d("Observation_Operator_Jacobian_val", real(dhx_dx%val,r_single)) endif + ! need additional arrays for GeoVaLs for T2 + call nc_diag_data2d("atmosphere_pressure_coordinate", sngl(exp(prsltmp)*r1000)) + end subroutine contents_netcdf_diag_ - subroutine contents_netcdf_diagp_ + subroutine contents_netcdf_diagp_(odiag) + type(obs_diag),pointer,intent(in):: odiag ! Observation class character(7),parameter :: obsclass = ' t' real(r_single),parameter:: missing = -9.99e9_r_single + real(r_kind),dimension(miter) :: obsdiag_iuse + call nc_diag_metadata("Station_ID", station_id ) call nc_diag_metadata("Observation_Class", obsclass ) call nc_diag_metadata("Observation_Type", ictype(ikx) ) - call nc_diag_metadata("Observation_Subtype", icsubtype(ikx) ) + call nc_diag_metadata("Observation_Subtype", -1 ) ! (-1 for pseudo obs sub-type) call nc_diag_metadata("Latitude", sngl(data(ilate,i)) ) call nc_diag_metadata("Longitude", sngl(data(ilone,i)) ) call nc_diag_metadata("Station_Elevation", sngl(data(istnelv,i)) ) @@ -1729,13 +1790,42 @@ subroutine contents_netcdf_diagp_ call nc_diag_metadata("Observation", sngl(data(itob,i)) ) call nc_diag_metadata("Obs_Minus_Forecast_adjusted", sngl(ddiff) ) call nc_diag_metadata("Obs_Minus_Forecast_unadjusted", sngl(ddiff) ) + call nc_diag_metadata("Forecast_unadjusted", sngl(data(itob,i)-ddiff)) + call nc_diag_metadata("Forecast_adjusted", sngl(data(itob,i)-ddiff)) + +!---- + if (lobsdiagsave) then + do jj=1,miter + if (odiag%muse(jj)) then + obsdiag_iuse(jj) = one + else + obsdiag_iuse(jj) = -one + endif + enddo + call nc_diag_data2d("ObsDiagSave_iuse", obsdiag_iuse ) + call nc_diag_data2d("ObsDiagSave_nldepart", odiag%nldepart ) + call nc_diag_data2d("ObsDiagSave_tldepart", odiag%tldepart ) + call nc_diag_data2d("ObsDiagSave_obssen", odiag%obssen ) + endif + + if (twodvar_regional .or. l_obsprvdiag) then + call nc_diag_metadata("Dominant_Sfc_Type", data(idomsfc,i) ) + call nc_diag_metadata("Model_Terrain", data(izz,i) ) + call nc_diag_metadata("Provider_Name", "88888888" ) + call nc_diag_metadata("Subprovider_Name", "88888888" ) + endif + +!---- if (save_jacobian) then call nc_diag_data2d("Observation_Operator_Jacobian_stind", dhx_dx%st_ind) call nc_diag_data2d("Observation_Operator_Jacobian_endind", dhx_dx%end_ind) call nc_diag_data2d("Observation_Operator_Jacobian_val", real(dhx_dx%val,r_single)) endif + ! need additional arrays for GeoVaLs for T2 + call nc_diag_data2d("atmosphere_pressure_coordinate", sngl(exp(prsltmp)*r1000)) + end subroutine contents_netcdf_diagp_ subroutine final_vars_ diff --git a/src/gsi/setupw.f90 b/src/gsi/setupw.f90 index 33f0a62957..55e498fe68 100644 --- a/src/gsi/setupw.f90 +++ b/src/gsi/setupw.f90 @@ -41,6 +41,7 @@ subroutine setupw(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav use obsmod, only: luse_obsdiag use obsmod, only: netcdf_diag, binary_diag, dirname + use obsmod, only: l_obsprvdiag use obsmod, only: neutral_stability_windfact_2dvar,use_similarity_2dvar use nc_diag_write_mod, only: nc_diag_init, nc_diag_header, nc_diag_metadata, & nc_diag_write, nc_diag_data2d @@ -218,6 +219,10 @@ subroutine setupw(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav ! level; they are now loaded by ! aircraftinfo. ! 2020-05-04 wu - no rotate_wind for fv3_regional +! 2021-10-xx pondeca/morris/zhao - added observation provider/subprovider +! information in diagonostic file, which is used +! in offline observation quality control program (AutoObsQC) +! for 3D-RTMA (if l_obsprvdiag is true). ! ! REMARKS: ! language: f90 @@ -398,7 +403,10 @@ subroutine setupw(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav ioff0=25 nreal=ioff0 if (lobsdiagsave) nreal=nreal+7*miter+2 - if (twodvar_regional) then; nreal=nreal+2; allocate(cprvstg(nobs),csprvstg(nobs)); endif + if (twodvar_regional .or. l_obsprvdiag) then + nreal=nreal+2 ! account for idomsfc,izz + allocate(cprvstg(nobs),csprvstg(nobs)) ! obs provider info + endif if (save_jacobian) then nnz = 2 ! number of non-zero elements in dH(x)/dx profile nind = 1 @@ -1485,13 +1493,13 @@ subroutine setupw(obsLL,odiagLL,lunin,mype,bwork,awork,nele,nobs,is,conv_diagsav if(binary_diag .and. ii>0)then write(7)' uv',nchar,nreal,ii,mype,ioff0 write(7)cdiagbuf(1:ii),rdiagbuf(:,1:ii) - deallocate(cdiagbuf,rdiagbuf) - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then write(7)cprvstg(1:ii),csprvstg(1:ii) deallocate(cprvstg,csprvstg) endif end if + deallocate(cdiagbuf,rdiagbuf) end if @@ -1752,7 +1760,7 @@ subroutine contents_binary_diag_(udiag,vdiag) enddo endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then ioff = ioff + 1 rdiagbuf(ioff,ii) = data(idomsfc,i) ! dominate surface type ioff = ioff + 1 @@ -1826,10 +1834,14 @@ subroutine contents_netcdf_diag_(udiag,vdiag) call nc_diag_metadata("u_Observation", sngl(uob_e) ) call nc_diag_metadata("u_Obs_Minus_Forecast_adjusted", sngl(dudiff_e) ) call nc_diag_metadata("u_Obs_Minus_Forecast_unadjusted", sngl(uob_e-uges_e) ) + call nc_diag_metadata("u_Forecast_adjusted", sngl(uob_e-dudiff_e)) + call nc_diag_metadata("u_Forecast_unadjusted", sngl(uges_e)) call nc_diag_metadata("v_Observation", sngl(vob_e) ) call nc_diag_metadata("v_Obs_Minus_Forecast_adjusted", sngl(dvdiff_e) ) call nc_diag_metadata("v_Obs_Minus_Forecast_unadjusted", sngl(vob_e-vges_e) ) + call nc_diag_metadata("v_Forecast_adjusted", sngl(vob_e-dvdiff_e)) + call nc_diag_metadata("v_Forecast_unadjusted", sngl(vges_e)) endif if (lobsdiagsave) then @@ -1852,7 +1864,7 @@ subroutine contents_netcdf_diag_(udiag,vdiag) !++ call nc_diag_data2d("ObsDiagSave_obssen", vdiag%obssen ) endif - if (twodvar_regional) then + if (twodvar_regional .or. l_obsprvdiag) then call nc_diag_metadata("Dominant_Sfc_Type", data(idomsfc,i) ) call nc_diag_metadata("Model_Terrain", data(izz,i) ) r_prvstg = data(iprvd,i) @@ -1869,6 +1881,7 @@ subroutine contents_netcdf_diag_(udiag,vdiag) call nc_diag_data2d("v_Observation_Operator_Jacobian_val", real(dhx_dx_v%val,r_single)) endif + call nc_diag_data2d("atmosphere_pressure_coordinate", exp(prsltmp)*r1000) end subroutine contents_netcdf_diag_ diff --git a/util/Analysis_Utilities/read_diag/read_diag_conv.f90 b/util/Analysis_Utilities/read_diag/read_diag_conv.f90 index b7c13f9b2d..42618e2566 100644 --- a/util/Analysis_Utilities/read_diag/read_diag_conv.f90 +++ b/util/Analysis_Utilities/read_diag/read_diag_conv.f90 @@ -48,35 +48,55 @@ PROGRAM read_diag_conv ! read in variables ! character(8),allocatable,dimension(:):: cdiagbuf + character(8),allocatable,dimension(:):: cprvstg,csprvstg ! obs provider/sub-provider + character(8)::cprovider,csubprovider real(r_single),allocatable,dimension(:,:)::rdiagbuf - integer(i_kind) nchar,nreal,ii,mype + integer(i_kind) nchar,nreal,ii,mype,iip ! iip: number of pseudo-obs if existing integer(i_kind) idate ! ! namelist files ! character(180) :: infilename ! file from GSI running directory character(180) :: outfilename ! file name saving results - namelist/iosetup/ infilename, outfilename + logical :: l_obsprvdiag ! if true, read/write obs provider/sub-provider info + logical :: dump_pseudo_obs_too !if true write out pseudo obs as well + namelist/iosetup/ infilename, outfilename, l_obsprvdiag, dump_pseudo_obs_too ! ! output variables ! character(len=3) :: var - real :: rlat,rlon,rprs,robs1,rdpt1,robs2,rdpt2,ruse,rerr - real :: rdhr, ddiff + real(r_single) :: rlat,rlon,rprs,rhgt,robs1,rdpt1,robs2,rdpt2,ruse,rerr + real(r_single) :: rdhr,iusev,ddiff character(8) :: stationID - integer :: itype,iuse,iusev + integer(i_kind) :: itype,iuse + integer(i_kind) :: isubtype,isubtype0 ! ! misc. ! character :: ch - integer :: i,j,k,ios - integer :: ic, iflg + integer(i_kind) :: i,j,k,ios + integer(i_kind) :: ic, iflg + + logical fexist ! - outfilename='diag_results' - open(11,file='namelist.conv') - read(11,iosetup) - close(11) +! initialization of variables in namelist + data infilename /'diag_conv.dat'/ + data outfilename /'diag_results'/ + data l_obsprvdiag /.false./ ! no obs provider info (by default) + data dump_pseudo_obs_too /.false./ +! outfilename='diag_results' + + inquire(file='namelist.conv',exist=fexist) + if(fexist) then + open(11,file='namelist.conv') + read(11,iosetup) + close(11) + else + write(6,*) "no reading from namelist file." + endif + write(6,*) "checking the input/output setup info:" + write(6,iosetup) ! open(42, file=trim(outfilename),IOSTAT=ios) if(ios > 0 ) then @@ -100,25 +120,41 @@ PROGRAM read_diag_conv write(*,*) var, nchar,nreal,ii,mype if (ii > 0) then allocate(cdiagbuf(ii),rdiagbuf(nreal,ii)) + if (l_obsprvdiag) allocate(cprvstg(ii),csprvstg(ii)) read(17,ERR=999,end=110) cdiagbuf, rdiagbuf + if (l_obsprvdiag) then + cprvstg='XXXXXXXX' + csprvstg='XXXXXXXX' + if (var(1:3)==' t' .or. var(1:3)==' q' .or. var(2:3)=='ps' .or. & + var(2:3)=='uv' .or. var(1:3)=='spd') read(17)cprvstg,csprvstg + endif do i=1,ii + if (l_obsprvdiag) then + cprovider=cprvstg(i) + csubprovider=csprvstg(i) + endif itype=rdiagbuf(1,i) ! observation type + isubtype=rdiagbuf(2,i) ! observation subtype rlat=rdiagbuf(3,i) ! observation latitude (degrees) rlon=rdiagbuf(4,i) ! observation longitude (degrees) rprs=rdiagbuf(6,i) ! observation pressure (hPa) + rhgt=rdiagbuf(7,i) ! observation height (meters) rdhr=rdiagbuf(8,i) ! obs time (hours relative to analysis time) iuse=int(rdiagbuf(12,i)) ! analysis usage flag (1=use, -1=monitoring ) iusev=int(rdiagbuf(11,i)) ! analysis usage flag ( value ) ddiff=rdiagbuf(18,i) ! obs-ges used in analysis (K) - rerr = 0 - if (rdiagbuf(16,i) > 0) then ! final inverse observation error (K**-1) - rerr=1.0/rdiagbuf(16,i) - end if + rerr = 0._r_single + if (rdiagbuf(16,i) > 1.0E-12_r_single) then ! final inverse observation error (K**-1) + rerr = 1.0/rdiagbuf(16,i) + else + rerr = 0._r_single + end if robs1=rdiagbuf(17,i) ! observation (K) rdpt1=rdiagbuf(18,i) ! obs-ges used in analysis ! get station ID stationID = cdiagbuf(i) +! Remove odd spaces in the station ID iflg = 0 do ic=8,1,-1 ch = stationID(ic:ic) @@ -131,12 +167,42 @@ PROGRAM read_diag_conv stationID(ic:ic) = '_' endif enddo + +! Remove odd spaces in the obs provider, and subprovider names + if (l_obsprvdiag) then + iflg = 0 + do ic=8,1,-1 + ch = cprovider(ic:ic) + if (ch > ' ' .and. ch <= 'z') then + iflg = 1 + else + cprovider(ic:ic) = ' ' + end if + if (ch == ' ' .and. iflg == 1) then + cprovider(ic:ic) = '_' + endif + enddo + + iflg = 0 + do ic=8,1,-1 + ch = csubprovider(ic:ic) + if (ch > ' ' .and. ch <= 'z') then + iflg = 1 + else + csubprovider(ic:ic) = ' ' + end if + if (ch == ' ' .and. iflg == 1) then + csubprovider(ic:ic) = '_' + endif + enddo + endif ! ! When the data is q, unit convert kg/kg -> g/kg **/ if (var == " q") then robs1 = robs1 * 1000.0 rdpt1 = rdpt1 * 1000.0 rerr = rerr * 1000.0 + ddiff = ddiff * 1000.0 end if ! When the data is pw, replase the rprs to -999.0 **/ if (var == " pw") rprs=-999.0 @@ -145,24 +211,74 @@ PROGRAM read_diag_conv robs1=-99999.9 ddiff=-99999.9 endif + +! check up the information in the obs provider, and subprovider names + if (l_obsprvdiag) then + if (cprovider(1:4)=='B7Hv' .or. cprovider(5:8)=='vH7B') then !this provider name comes with strange characters + cprovider(1:4)='B7Hv' + cprovider(5:8)=' ' + endif + + if (csubprovider(1:4)=='B7Hv' .or. csubprovider(5:8)=='vH7B') then + csubprovider(1:4)='B7Hv' + csubprovider(5:8)=' ' + endif + + if (itype==154 .and. trim(adjustl(var))=='tca') then + stationID='GOESSKY' + cprovider='GOESSKY' + csubprovider='GOESSKY' + end if + + if (trim(cprovider)=='') cprovider='EMPTY' + if (trim(csubprovider)=='') csubprovider='EMPTY' + endif + +! special treatment to obs tca/cei/vis/gst +! If we have ceiling or total cloud amount obs less than 0, +! set to missing + if ((trim(adjustl(var))=="tca" .or. trim(adjustl(var))=="cei" .or. & + trim(adjustl(var))=="vis" .or. trim(adjustl(var))=="gst") .and. & + robs1 < 0.) then + robs1=0.10000E+10 + ddiff=0.10000E+10 + end if + + if (dump_pseudo_obs_too) then + isubtype0=0 ! reset subtype (esp.for pseudo-obs) to be zero in order to print it our + else + isubtype0=isubtype + endif ! ! write out result for one variable on one pitch - if (var .ne. " uv") then - write (42,'(A3," @ ",A8," : ",I3,F10.2,F8.2,F8.2,F8.2,I5,2F10.2)') & + if (l_obsprvdiag) then ! write out obs provider info with other info together + if ( var .ne. " uv" .and. isubtype0 >=0 ) then + write (42,'(A3,1x,A8,1x,A8,1x,A8,1x,I3,1x,F10.2,F8.2,F8.2,2F20.5,I5,2E15.5,1x,"NaN NaN ",E15.5,F10.3)') & + var,stationID,cprovider,csubprovider,itype,rdhr,rlat,rlon,rprs,rhgt,iuse,robs1,ddiff,rerr,iusev + else if ( var .eq. " uv" .and. isubtype0 >=0 ) then +! ** When the data is uv, additional output is needed **/ + robs2=rdiagbuf(20,i) + rdpt2=rdiagbuf(21,i) + write (42,'(A3,1x,A8,1x,A8,1x,A8,1x,I3,1x,F10.2,F8.2,F8.2,2F20.5,I5,4E15.5,1x,E15.5,F10.3)') & + var,stationID,cprovider,csubprovider,itype,rdhr,rlat,rlon,rprs,rhgt,iuse,robs1,ddiff,robs2,rdpt2,rerr,iusev + endif + else ! if no need to write out obs provider info + if (var .ne. " uv") then + write (42,'(A3," @ ",A8," : ",I3,F10.2,F8.2,F8.2,F8.2,I5,2F10.2)') & var,stationID,itype,rdhr,rlat,rlon,rprs,iuse,robs1,ddiff - else + else ! ** When the data is uv, additional output is needed **/ - robs2=rdiagbuf(20,i) - rdpt2=rdiagbuf(21,i) - write (42,'(A3," @ ",A8," : ",I3,F10.2,F8.2,F8.2,F8.2,I5,4F10.2)') & + robs2=rdiagbuf(20,i) + rdpt2=rdiagbuf(21,i) + write (42,'(A3," @ ",A8," : ",I3,F10.2,F8.2,F8.2,F8.2,I5,4F10.2)') & var,stationID,itype,rdhr,rlat,rlon,rprs,iuse,robs1,ddiff,robs2, rdpt2 + endif endif - - enddo ! i end for one station deallocate(cdiagbuf,rdiagbuf) + if (l_obsprvdiag) deallocate(cprvstg,csprvstg) else read(17) endif diff --git a/util/Radiance_Monitor/CMakeLists.txt b/util/Radiance_Monitor/CMakeLists.txt index 1bc451ea68..e11f186d0a 100644 --- a/util/Radiance_Monitor/CMakeLists.txt +++ b/util/Radiance_Monitor/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) # I am top-level project. if( NOT DEFINED ENV{CC} ) @@ -66,6 +66,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) endif() cmake_policy(SET CMP0009 NEW) + cmake_policy(SET CMP0074 NEW) find_package(OpenMP) message("found openmp with flag ${OPENMP_Fortran_FLAGS}") @@ -84,8 +85,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) find_package( W3NCO ) set(BUILD_NCDIAG ON) - set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag/include") - add_subdirectory(src/ncdiag) + set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag") + add_subdirectory(${PROJECT_SOURCE_DIR}/../../src/ncdiag ${PROJECT_BINARY_DIR}/src/ncdiag) set(NCDIAG_LIBRARIES ncdiag ) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) endif() diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt index 7089ff1134..603d4acfbc 100644 --- a/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_MK_BASE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_MK_BASE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_mk_base ) diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt index be546a9601..071e872e8f 100644 --- a/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_VALIDATE_TM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_VALIDATE_TM_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_validate_tm.x ) diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh index b24927322a..a5c62347b6 100755 --- a/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh +++ b/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh @@ -192,8 +192,8 @@ if [[ -d ${DATA_LOCATION} ]]; then -o ${logfile} --ntasks=1 --mem=5g ${job} elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/CP.${PDY}.${CYC}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} fi else echo "Unable to locate DATA_LOCATION: ${DATA_LOCATION}" diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh index ef689e36a6..2a8ca89a45 100755 --- a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh +++ b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh @@ -248,8 +248,8 @@ if [[ -e ${radstat} && -e ${biascr} ]]; then -o ${logfile} --ntasks=1 --mem=5g ${job} elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${pdy}.${cyc}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} fi else # radstat and/or biascr not found diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh index 97b0e2499d..0eac566ac6 100755 --- a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh +++ b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh @@ -306,8 +306,8 @@ elif [[ $MY_MACHINE = "hera" ]]; then -j oe -o ${logfile} ${job} elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${PDY}.${cyc}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} fi diff --git a/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl b/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl index af13f64d87..f304279a0e 100755 --- a/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl +++ b/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl @@ -88,8 +88,8 @@ $search_string = $run; } - my @mmdirs = grep { /$search_string/ } @alldirs; - + my @mmdirs = grep {/$search_string/} @alldirs; + #----------------------------------------------------------------------- # If there are no $run.yyyymmdd subdirectories, then exit without # returning any date string. @@ -157,13 +157,26 @@ $hr_ctr = $hr_ctr - 1; $newdir = "${dirpath}/${sortmm[$ctr]}/${hrs[$hr_ctr]}/${lcm}"; -# print " newdir = $newdir \n"; - if( -d $newdir ) { opendir DIR, $newdir or die "Cannot open the current directory: $!"; - my @timefiles = grep { /ieee_d/ } readdir DIR; + my @dirfiles = readdir DIR; + my @timefiles = grep { /ieee_d/ } @dirfiles; + + # If no *ieee_d* files were found then look for the compressed format of + # radmon_*.tar files instead. If found then use the parent directories + # to establish the cycle time. + # + if( $#timefiles < 0 ) { + my @tarfiles = grep { /radmon_time.tar/ } @dirfiles; + + if( $#tarfiles >= 0 ) { + my $cycle = "${sortmm[$ctr]}${hrs[$hr_ctr]}"; + push( @timefiles, $cycle ); + } + } + if( $#timefiles >= 0 ) { my @sorttime = sort( @timefiles ); @@ -201,18 +214,13 @@ } } } - } } while $hr_ctr > 0 && $found_cycle == 0; -# print " found_cycle, ctr, end_ctr = $found_cycle, $ctr, $end_ctr \n"; - if( $cyc == 0 && $ctr >= $end_ctr ){ -# print " exiting from if\n"; $exit_flag = 1; } elsif( $cyc == 1 && $ctr <= $end_ctr ){ -# print " exiting from elsif\n"; $exit_flag = 1; } diff --git a/util/Radiance_Monitor/image_gen/html/Install_html.sh b/util/Radiance_Monitor/image_gen/html/Install_html.sh index ec81991e62..8ecb71d1da 100755 --- a/util/Radiance_Monitor/image_gen/html/Install_html.sh +++ b/util/Radiance_Monitor/image_gen/html/Install_html.sh @@ -51,12 +51,6 @@ else fi -#-------------------------------------------------------------- -# source plot_rad_conf to get WEB_SVR, WEB_USER, WEBDIR -# -. ${RADMON_IMAGE_GEN}/parm/plot_rad_conf - - #-------------------------------------------------------------- # call the appropriate child script for glb or rgn # diff --git a/util/Radiance_Monitor/image_gen/html/install_glb.sh b/util/Radiance_Monitor/image_gen/html/install_glb.sh index eb55e7c201..4c70951df9 100755 --- a/util/Radiance_Monitor/image_gen/html/install_glb.sh +++ b/util/Radiance_Monitor/image_gen/html/install_glb.sh @@ -33,35 +33,8 @@ RAD_AREA="glb" this_file=`basename $0` this_dir=`dirname $0` -#top_parm=${this_dir}/../../parm -# -#if [[ -s ${top_parm}/RadMon_config ]]; then -# . ${top_parm}/RadMon_config -#else -# echo "ERROR: Unable to source ${top_parm}/RadMon_config" -# exit -#fi -# -#if [[ -s ${top_parm}/RadMon_user_settings ]]; then -# . ${top_parm}/RadMon_user_settings -#else -# echo "ERROR: Unable to source ${top_parm}/RadMon_user_settings" -# exit -#fi - - -#-------------------------------------------------------------- -# source plot_rad_conf to get WEB_SVR, WEB_USER, WEBDIR -# -. ${RADMON_IMAGE_GEN}/parm/plot_rad_conf - - -#-------------------------------------------------------------- -# Get the area for this SUFFIX from the data_map file -# new_webdir=${WEBDIR}/${SUFFIX} -. ${RADMON_IMAGE_GEN}/parm/glbl_conf echo RAD_AREA = $RAD_AREA echo TANKverf = $TANKverf @@ -85,7 +58,7 @@ cd $workdir # Find the first date with data. Start at today and work # backwards. Stop after 90 days and exit. # -PDATE=`${IG_SCRIPTS}/find_cycle.pl --dir ${TANKverf} --cyc 1` +PDATE=`${IG_SCRIPTS}/nu_find_cycle.pl --dir ${TANKverf} --cyc 1` echo PDATE= $PDATE limit=`$NDATE -2160 $PDATE` # 90 days @@ -98,35 +71,30 @@ echo limit, PDATE = $limit, $PDATE data_found=0 while [[ data_found -eq 0 && $PDATE -ge $limit ]]; do PDY=`echo $PDATE|cut -c1-8` + CYC=`echo $PDATE|cut -c9-10` - test_dir=${TANKverf}/${RUN}.${PDY}/${MONITOR} + test_dir=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR} if [[ ! -d ${test_dir} ]]; then - test_dir=${TANKverf}/${RUN}.${PDY} + test_dir=${TANKverf}/${RUN}.${PDY}/${MONITOR} fi if [[ ! -d ${test_dir} ]]; then - test_dir=${TANKverf}/${MONITOR}.${PDY} + test_dir=${TANKverf}/${RUN}.${PDY} fi echo "test_dir = ${test_dir}" if [[ -d ${test_dir} ]]; then echo " test_dir is GO " - test00=`ls ${test_dir}/angle.*${PDY}00*.ieee_d* | wc -l` - test06=`ls ${test_dir}/angle.*${PDY}06*.ieee_d* | wc -l` - test12=`ls ${test_dir}/angle.*${PDY}12*.ieee_d* | wc -l` - test18=`ls ${test_dir}/angle.*${PDY}18*.ieee_d* | wc -l` - if [[ $test00 -gt 0 ]]; then - test_list=`ls ${test_dir}/angle.*${PDY}00*.ieee_d*` - data_found=1 - elif [[ $test06 -gt 0 ]]; then - test_list=`ls ${test_dir}/angle.*${PDY}06*.ieee_d*` - data_found=1 - elif [[ $test12 -gt 0 ]]; then - test_list=`ls ${test_dir}/angle.*${PDY}12*.ieee_d*` - data_found=1 - elif [[ $test18 -gt 0 ]]; then - test_list=`ls ${test_dir}/angle.*${PDY}18*.ieee_d*` - data_found=1 + + if [[ -e ${test_dir}/angle.tar ]]; then + test_list=`tar -tv ${test_dir}/angle.tar` + data_found=1 + else + test=`ls ${test_dir}/angle.*${PDATE}*.ieee_d* | wc -l` + if [[ $test -gt 0 ]]; then + test_list=`ls ${test_dir}/angle.*${PDATE}*.ieee_d*` + data_found=1 + fi fi else echo "test_dir is NOGO" diff --git a/util/Radiance_Monitor/image_gen/html/install_rgn.sh b/util/Radiance_Monitor/image_gen/html/install_rgn.sh index ce2a4de31b..6ceb1a4caf 100755 --- a/util/Radiance_Monitor/image_gen/html/install_rgn.sh +++ b/util/Radiance_Monitor/image_gen/html/install_rgn.sh @@ -50,18 +50,10 @@ else fi -#-------------------------------------------------------------- -# source plot_rad_conf to get WEB_SVR, WEB_USER, WEBDIR -# -. ${RADMON_IMAGE_GEN}/parm/plot_rad_conf - - #-------------------------------------------------------------- # Get the area for this SUFFIX from the data_map file # - new_webdir=${WEBDIR}/${SUFFIX} -. ${RADMON_IMAGE_GEN}/parm/rgnl_conf echo RAD_AREA = $RAD_AREA echo TANKverf = $TANKverf diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt index 44fcad0750..e585be7a32 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_ANGLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_ANGLE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_angle ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt index ca50c08216..06e447e3d5 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_BCOEF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_BCOEF_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_bcoef ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt index d74885b276..07e20110d9 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_HORIZ_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_HORIZ_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_horiz ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt index dde3b66137..98dd52d74d 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_SUMMARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_SUMMARY_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_summary ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt index a14d8566e6..3f4bfa4cc8 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_TIME_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_TIME_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_time ) diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh index c5b2f87d44..644653df39 100755 --- a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh +++ b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh @@ -109,6 +109,10 @@ if [[ $? -ne 0 ]]; then fi +if [[ ! -d ${IMGNDIR} ]]; then + mkdir -p ${IMGNDIR} +fi + #-------------------------------------------------------------------- # Determine cycle to plot. Exit if cycle is > last available # data. @@ -183,7 +187,6 @@ export PDY=`echo $PDATE|cut -c1-8` #-------------------------------------------------------------------- # Locate ieee_src in $TANKverf and verify data files are present # - ieee_src=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR} if [[ ! -d ${ieee_src} ]]; then @@ -199,7 +202,16 @@ if [[ ! -d ${ieee_src} ]]; then exit 5 fi +#----------------------------------------------------- +# check $ieee_src for data files. If none are found +# check contents of the radmon_angle.tar file. +# nfile_src=`ls -l ${ieee_src}/*${PDATE}*ieee_d* | egrep -c '^-'` +if [[ $nfile_src -le 0 ]]; then + if [[ -e ${ieee_src}/radmon_angle.tar ]]; then + nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d | wc -l` + fi +fi if [[ $nfile_src -le 0 ]]; then echo " Missing ieee_src files, nfile_src = ${nfile_src}, aborting plot" @@ -238,6 +250,9 @@ fi # the $satype_file can't be found. # test_list=`ls ${ieee_src}/angle.*${PDATE}.ieee_d*` +if [[ $test_list = "" ]]; then + test_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d` +fi for test in ${test_list}; do this_file=`basename $test` @@ -332,11 +347,13 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then export WEBDIR=${WEBDIR}/${RADMON_SUFFIX}/${RUN}/pngs if [[ $MY_MACHINE = "wcoss2" ]]; then - cmdfile=transfer_cmd + cmdfile="${PLOT_WORK_DIR}/transfer_cmd" echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile + chmod 755 $cmdfile - $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \ - -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile} + run_time="$rhr$cmin" # HHMM format for qsub + $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err \ + -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} -a ${run_time} ${cmdfile} else $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \ -R affinity[core] -J ${jobname} -cwd ${PWD} -b $run_time ${job} diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh index c8eec4d35d..6654861505 100755 --- a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh +++ b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh @@ -286,8 +286,8 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then cmdfile=transfer_cmd echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile - $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \ - -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile} + $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err + -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile} else $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \ diff --git a/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh index 72b9348e85..a46eaf64ad 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh @@ -39,54 +39,92 @@ allmissing=1 cycdy=$((24/$CYCLE_INTERVAL)) # number cycles per day ndays=$(($NUM_CYCLES/$cycdy)) # number of days in plot period -echo SATYPE=$SATYPE - for type in ${SATYPE}; do found=0 - finished=0 - ctr=$ndays test_day=$PDATE - - while [[ $found -eq 0 && $finished -ne 1 ]]; do - - if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next + ctr=$ndays + + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do + + if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` + else pdy=`echo $test_day|cut -c1-8` - ieee_src=${TANKverf}/radmon.${pdy} + cyc=`echo $test_day|cut -c9-10` + fi + + + #--------------------------------------------------- + # Check to see if the *ctl* files are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || + ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then + found=1 + else - pdy=`echo $test_day|cut -c1-8` - ieee_src=${TANKverf}/${RUN}.${pdy}/${CYC}/${MONITOR} + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} fi if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${pdy} fi - fi - - echo "ieee_src = with pdy = $pdy" - - if [[ -s ${ieee_src}/angle.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/angle.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/angle.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/angle.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} - fi - found=1 + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi + + using_tar=0 + #---------------------------------------------------- + # Determine if the angle files are in a tar file + # and, if so, extract the ctl files for this $type. + # + if [[ -s ${ieee_src}/radmon_angle.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + tar -xf ./radmon_angle.tar ${ctl_list} + cd ${cwd} + fi + fi + + #------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'angle' from the file name. + # + ctl_files=`ls $ieee_src/angle.$type*.ctl*` + prefix='angle.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #---------------------------------------------------------------- + # If there's a radmon_angle.tar archive in ${ieee_src} then + # delete the extracted *ctl* files to leave just the tar files. + # + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/angle.${type}.ctl* + rm -f ${ieee_src}/angle.${type}_anl.ctl* + fi - elif [[ -s ${ieee_src}/angle.${type}.ctl ]]; then - $NCP ${ieee_src}/angle.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/angle.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/angle.${type}_anl.ctl ${imgndir}/${type}_anl.ctl - fi - found=1 fi - - if [[ $found -eq 0 ]]; then + + if [[ ${found} -eq 0 ]]; then + #------------------------------------------ + # Step to the previous day and try again. + # if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` - ctr=$(($ctr-1)) - else - finished=1 + ctr=$(($ctr-1)) fi fi done @@ -95,8 +133,10 @@ for type in ${SATYPE}; do allmissing=0 found=1 fi + done + if [[ $allmissing = 1 ]]; then echo ERROR: Unable to plot. All angle control files are missing from ${TANKverf} for requested date range. exit 2 @@ -107,26 +147,26 @@ fi # Update the time definition (tdef) line in the angle control # files. # -for type in ${SATYPE}; do - if [[ -s ${imgndir}/${type}.ctl.${Z} ]]; then - ${UNCOMPRESS} ${imgndir}/${type}.ctl.${Z} +for sat in ${SATYPE}; do + if [[ -s ${imgndir}/${sat}.ctl.${Z} ]]; then + ${UNCOMPRESS} ${imgndir}/${sat}.ctl.${Z} fi - ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${type}.ctl ${START_DATE} ${NUM_CYCLES} -done + ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${sat}.ctl ${START_DATE} ${NUM_CYCLES} - -#------------------------------------------------------------------- -# Separate the sources with a large number of channels. These will -# be submitted in dedicated jobs, while the sources with a smaller -# number of channels will be submitted together. -# -for sat in ${SATYPE}; do + #------------------------------------------------------------------- + # Separate the sources with a large number of channels. These will + # be submitted in dedicated jobs, while the sources with a smaller + # number of channels will be submitted together. + # nchanl=`cat ${imgndir}/${sat}.ctl | gawk '/title/{print $NF}'` + if [[ $nchanl -lt 100 ]]; then satlist=" $sat $satlist " else big_satlist=" $sat $big_satlist " fi + + ${COMPRESS} ${imgndir}/${sat}.ctl done echo "" @@ -180,9 +220,7 @@ done chmod 755 ${cmdfile} echo "CMDFILE: ${cmdfile}" -ntasks=`cat $cmdfile|wc -l ` wall_tm="0:20" - if [[ ${MY_MACHINE} = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -M 500 -W ${wall_tm} \ -R "affinity[core]" -J ${jobname} -cwd ${PWD} $cmdfile @@ -200,8 +238,8 @@ elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then -J ${jobname} -cwd ${PWD} $cmdfile elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_angle_${suffix}.err \ + -V -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile} fi @@ -218,7 +256,7 @@ echo "starting big_satlist" for sat in ${big_satlist}; do echo processing $sat in $big_satlist - if [[ ${MY_MACHINE} = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then + if [[ ${MY_MACHINE} = "wcoss_d" || $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss2" ]]; then cmdfile=${PLOT_WORK_DIR}/cmdfile_pangle_${sat} if [[ -e ${cmdfile} ]]; then @@ -229,23 +267,32 @@ for sat in ${big_satlist}; do jobname=plot_${RADMON_SUFFIX}_ang_${sat} logfile=${LOGdir}/plot_angle_${sat}.log - ntasks=`cat $cmdfile|wc -l ` - echo "ntasks = $ntasks" + if [[ -e ${logfile} ]]; then + rm ${logfile} + fi wall_tm="0:30" if [[ $MY_MACHINE = "wcoss_d" ]]; then - mem="12000" - $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -M ${mem} -W ${wall_tm} \ - -R "affinity[core]" -J ${jobname} -cwd ${PWD} $cmdfile + $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -W ${wall_tm} \ + -R "affinity[core]" -R "rusage[mem=10000]" -J ${jobname} -cwd ${PWD} $cmdfile elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -M 600 -W ${wall_tm} \ -J ${jobname} -cwd ${PWD} $cmdfile - fi + elif [[ $MY_MACHINE = "wcoss2" ]]; then + errfile=${LOGdir}/plot_angle_${sat}.err + if [[ -e ${errfile} ]]; then + rm ${errfile} + fi + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_angle_${sat}.err \ + -V -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile} + fi - elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "jet" || $MY_MACHINE = "s4" ]]; then # hera|jet|s4, submit 1 job for each sat/list item + #--------------------------------------------------- + # hera|jet|s4, submit 1 job for each sat/list item + elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "jet" || $MY_MACHINE = "s4" ]]; then ii=0 logfile=${LOGdir}/plot_angle_${sat}.log @@ -260,7 +307,10 @@ for sat in ${big_satlist}; do (( ii=ii+1 )) done - if [[ ! $MY_MACHINE = "jet" ]]; then + if [[ $MY_MACHINE = "hera" ]]; then + $SUB --account ${ACCOUNT} -n $ii -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ + --mem=0 --wrap "srun -l --multi-prog ${cmdfile}" + elif [[ $MY_MACHINE = "s4" ]]; then $SUB --account ${ACCOUNT} -n $ii -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ --wrap "srun -l --multi-prog ${cmdfile}" else diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh index 05e175470b..8eedb783ad 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh @@ -38,50 +38,83 @@ test_day=$PDATE for type in ${SATYPE}; do found=0 - finished=0 test_day=$PDATE ctr=$ndays - while [[ $found -eq 0 && $finished -ne 1 ]]; do + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory - pdy=`echo $test_day|cut -c1-8` + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` else pdy=`echo $test_day|cut -c1-8` + cyc=`echo $test_day|cut -c9-10` fi - ieee_src=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${PDY}/${MONITOR} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${PDY} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${MONITOR}.${PDY} - fi - + #--------------------------------------------------- + # Check to see if the *ctl* files are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || + ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then + found=1 + else + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi - if [[ -s ${ieee_src}/bcoef.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcoef.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/bcoef.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcoef.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} + using_tar=0 + #---------------------------------------------------- + # Determine if the bcoef files are in an tar file. + # if so extract the ctl files for this $type. + # + if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + tar -xf ./radmon_bcoef.tar ${ctl_list} + cd ${cwd} + fi fi - found=1 - elif [[ -s ${ieee_src}/bcoef.${type}.ctl ]]; then - $NCP ${ieee_src}/bcoef.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/bcoef.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/bcoef.${type}_anl.ctl ${imgndir}/${type}_anl.ctl + + #-------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'bcoef.' from the file name. + # + ctl_files=`ls $ieee_src/bcoef.$type*.ctl*` + prefix='bcoef.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #------------------------------------------------------- + # If there's a radmon_bcoef.tar archive in ${ieee_src} + # then delete the extracted *ctl* files. + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/bcoef.${type}.ctl* + rm -f ${ieee_src}/bcoef.${type}_anl.ctl* fi - found=1 + fi - if [[ $found -eq 0 ]]; then + if [[ $found -eq 0 ]]; then if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` ctr=$(($ctr-1)) - else - finished=1 fi fi done @@ -135,7 +168,7 @@ elif [[ $MY_MACHINE = "jet" ]]; then -p ${RADMON_PARTITION} -o ${logfile} -D . $IG_SCRIPTS/plot_bcoef.sh elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e $LOGdir/plot_bcoef.err -V \ -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} $IG_SCRIPTS/plot_bcoef.sh fi diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh index ba41fe9d9a..2751d4501c 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh @@ -29,7 +29,6 @@ fi # are found or we run out of dates to check. Report an error to # the log file and exit if no ctl files are found. # -allmissing=1 pdy=`echo $PDATE|cut -c1-8` cyc=`echo $PDATE|cut -c9-10` @@ -40,65 +39,96 @@ test_day=$PDATE for type in ${SATYPE}; do found=0 - finished=0 test_day=$PDATE ctr=$ndays - while [[ $found -eq 0 && $finished -ne 1 ]]; do - if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do + + if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory - pdy=`echo $test_day|cut -c1-8` + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` else pdy=`echo $test_day|cut -c1-8` + cyc=`echo $test_day|cut -c9-10` fi - ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${MONITOR}.${pdy} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy} - fi - if [[ ! -d ${ieee_src} ]]; then - echo "Unable to locate ieee_src directory, exiting mk_time_plots.sh." - exit 12 - fi - - - if [[ -s ${ieee_src}/bcor.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcor.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/bcor.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcor.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} - fi + #------------------------------------------------------------------ + # Check to see if the *ctl* files for this $type are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || + ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then found=1 - elif [[ -s ${ieee_src}/bcor.${type}.ctl ]]; then - $NCP ${ieee_src}/bcor.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/bcor.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/bcor.${type}_anl.ctl ${imgndir}/${type}_anl.ctl + else + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} fi - found=1 + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi + + using_tar=0 + #--------------------------------------------------------- + # Determine if the bcor files are in a tar file. + # if so extract the ctl files for this $type. + # + # Note that the ctl files are moved back to ${ieee_src} + # so the code block that follows will work with both + # tarred and non-tarred storage schemes. + # + if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + tar -xf ./radmon_bcor.tar ${ctl_list} + cd ${cwd} + fi + fi + + #-------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'bcor.' from the file name. + # + ctl_files=`ls $ieee_src/bcor.$type*.ctl*` + prefix='bcor.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #------------------------------------------------------ + # If there's a radmon_bcor.tar archive in ${ieee_src} + # then delete the extracted *ctl* files. + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/bcor.${type}*.ctl* + fi + fi - if [[ $found -eq 0 ]]; then + if [[ ${found} -eq 0 ]]; then # if not found try previous day if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` ctr=$(($ctr-1)) - else - finished=1 fi fi - done - if [[ -s ${imgndir}/${type}.ctl.${Z} || -s ${imgndir}/${type}.ctl ]]; then - allmissing=0 - found=1 - fi + done + done -if [[ $allmissing = 1 ]]; then +nctl=`ls ${imgndir}/*ctl* -1 | wc -l` +if [[ $nctl -le 0 ]]; then echo ERROR: Unable to plot. All bcor control files are missing. exit 14 fi @@ -112,8 +142,8 @@ for type in ${SATYPE}; do if [[ -s ${imgndir}/${type}.ctl.${Z} ]]; then ${UNCOMPRESS} ${imgndir}/${type}.ctl.${Z} fi - ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${type}.ctl ${START_DATE} ${NUM_CYCLES} + ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${type}.ctl ${START_DATE} ${NUM_CYCLES} done for sat in ${SATYPE}; do @@ -142,9 +172,8 @@ cd ${PLOT_WORK_DIR} #------------------------------------------------------------------------- -# Loop over satellite/instruments. Submit poe job to make plots. Each task handles -# a single satellite/insrument. - +# Loop over satellite/instruments and submit job. +# suffix=a cmdfile=cmdfile_pbcor_${suffix} jobname=plot_${RADMON_SUFFIX}_bcor_${suffix} @@ -189,6 +218,10 @@ elif [[ $MY_MACHINE = "jet" ]]; then $SUB --account ${ACCOUNT} -n $ctr -o ${logfile} -D . -J ${jobname} \ -p ${RADMON_PARTITION} --time=2:00:00 --wrap "srun -l --multi-prog ${cmdfile}" +elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_bcor_${suffix}.err \ + -V -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} + fi @@ -246,8 +279,8 @@ for sat in ${bigSATLIST}; do -p ${RADMON_PARTITION} --time=1:00:00 --wrap "srun -l --multi-prog ${cmdfile}" elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_bcor_${suffix}.err \ + -V -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} fi echo "submitted $sat" @@ -255,4 +288,4 @@ done echo "end mk_bcor_plots.sh" -exit 0 +exit diff --git a/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh b/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh index 9da4218795..1d3af5aa4c 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh @@ -91,8 +91,7 @@ EOF #------------------------------ # mv output files to IMGNDIR # - cp -f ${type}.*.bcoef.txt ${IMGNDIR}/bcoef/. -# cp -f ${type}.chan.txt ${IMGNDIR}/time/. + mv ${type}.*.bcoef.txt ${IMGNDIR}/bcoef/. done diff --git a/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh index 009adfed98..d6b5a0b536 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh @@ -160,7 +160,7 @@ done # submit the plot jobs # -if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then +if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss2" ]]; then cmdfile="./cmdfile_horiz_${RADMON_SUFFIX}_${PID}" logfile=${LOGdir}/horiz_${PID}.log rm -f $cmdfile @@ -177,9 +177,11 @@ if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then if [[ $MY_MACHINE = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \ -W 0:45 -J ${jobname} -cwd ${PWD} $cmdfile - else + + elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 0:45 \ -J ${jobname} -cwd ${PWD} $cmdfile + fi else # hera|jet|s4 @@ -225,9 +227,14 @@ for sat in ${bigSATLIST}; do if [[ $MY_MACHINE = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \ -W 2:45 -J ${jobname} -cwd ${PWD} $cmdfile + elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 2:45 \ -J ${jobname} -cwd ${PWD} $cmdfile + + elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/horiz_${PID}.err \ + -V -l select=1:mem=1g -l walltime=2:00:00 -N ${jobname} ${cmdfile} else $SUB -A $ACCOUNT -l procs=${ntasks},walltime=2:00:00 -N ${jobname} \ -V -j oe -o $LOGdir/horiz_${PID}.log $cmdfile @@ -248,9 +255,15 @@ for sat in ${bigSATLIST}; do if [[ $MY_MACHINE = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \ -W 2:45 -J ${jobname} -cwd ${PWD} $cmdfile + elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 2:45 \ -J ${jobname} -cwd ${PWD} $cmdfile + + elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/horiz_${PID}.err \ + -V -l select=1:mem=1g -l walltime=2:00:00 -N ${jobname} ${cmdfile} + else $SUB -A $ACCOUNT -l procs=${ntasks},walltime=2:00:00 -N ${jobname} \ -V -j oe -o $LOGdir/horiz_${PID}.log $cmdfile diff --git a/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh index c1ccf2c116..66d626c361 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh @@ -27,64 +27,104 @@ fi # then $TANKverf/radmon.$pdy. # allmissing=1 -pdy=`echo $PDATE|cut -c1-8` -cyc=`echo $PDATE|cut -c9-10` cycdy=$((24/$CYCLE_INTERVAL)) # number cycles per day ndays=$(($NUM_CYCLES/$cycdy)) # number days in plot period test_day=$PDATE +#-------------------------------------------------------- +# Verify there are control files available in $imgndir +# for everything in $SATYPE. +# for type in ${SATYPE}; do found=0 - finished=0 test_day=$PDATE ctr=$ndays - while [[ ${found} -eq 0 && $finished -ne 1 ]]; do + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory - pdy=`echo $test_day|cut -c1-8` + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` else pdy=`echo $test_day|cut -c1-8` + cyc=`echo $test_day|cut -c9-10` fi - ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${MONITOR}.${pdy} - fi - if [[ -s ${ieee_src}/time.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/time.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/time.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/time.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} - fi + #--------------------------------------------------- + # Check to see if the *ctl* files are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then found=1 - elif [[ -s ${ieee_src}/time.${type}.ctl ]]; then - $NCP ${ieee_src}/time.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/time.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/time.${type}_anl.ctl ${imgndir}/${type}_anl.ctl + + else + + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} fi - found=1 + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi + + using_tar=0 + #-------------------------------------------------- + # Determine if the time files are in an tar file. + # if so extract the ctl files for this $type. + # + if [[ -s ${ieee_src}/radmon_time.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_time.tar | grep $type | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + ctl_list=`tar -tf ./radmon_time.tar | grep $type | grep ctl` + tar -xf ${ieee_src}/radmon_time.tar ${ctl_list} + cd ${cwd} + fi + fi + + #-------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'time.' from the file name. + # + ctl_files=`ls $ieee_src/time.$type*.ctl*` + prefix='time.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #------------------------------------------------------ + # If there's a radmon_time.tar archive in ${ieee_src} + # then delete the extracted *ctl* files. + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/time.${type}.ctl* + rm -f ${ieee_src}/time.${type}_anl.ctl* + fi + fi if [[ ${found} -eq 0 ]]; then if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` ctr=$(($ctr-1)) - else - finished=1 fi fi done + if [[ -s ${imgndir}/${type}.ctl.${Z} || -s ${imgndir}/${type}.ctl ]]; then allmissing=0 found=1 @@ -152,7 +192,7 @@ elif [[ ${MY_MACHINE} = "jet" ]]; then --partition ${RADMON_PARTITION} -o ${logfile} ${IG_SCRIPTS}/plot_summary.sh elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_summary.err -V \ -l select=1:mem=1g -l walltime=10:00 -N ${jobname} ${IG_SCRIPTS}/plot_summary.sh fi @@ -182,58 +222,58 @@ list="count penalty omgnbc total omgbc" # Build command file and submit plot job for intruments not on # the bigSAT list. # +suffix=a +jobname=plot_${RADMON_SUFFIX}_tm_${suffix} - suffix=a - cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${suffix} - jobname=plot_${RADMON_SUFFIX}_tm_${suffix} - logfile=${LOGdir}/plot_time_${suffix}.log - +cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${suffix} +if [[ -e ${cmdfile} ]]; then rm -f $cmdfile +fi + +logfile=${LOGdir}/plot_time_${suffix}.log +if [[ -e ${logfile} ]]; then rm ${logfile} +fi >$cmdfile - ctr=0 - - for sat in ${SATLIST}; do - if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then - echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile - else - echo "$IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile - fi - ((ctr=ctr+1)) - done - - chmod 755 $cmdfile +ctr=0 - if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then - wall_tm="2:30" +for sat in ${SATLIST}; do + if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then + echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile else - wall_tm="0:45" + echo "$IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile fi + ((ctr=ctr+1)) +done +chmod 755 $cmdfile - if [[ $MY_MACHINE = "wcoss_d" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ - -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} +wall_tm="0:45" +if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then + wall_tm="2:30" +fi - elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then - echo "using ctr = ${ctr}" - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ - --wrap "srun -l --multi-prog ${cmdfile}" +if [[ $MY_MACHINE = "wcoss_d" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ + -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} - elif [[ $MY_MACHINE = "jet" ]]; then - echo "using ctr = ${ctr}" - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ - -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" +elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ + --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ - -J ${jobname} -cwd ${PWD} ${cmdfile} +elif [[ $MY_MACHINE = "jet" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ + -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} - fi +elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ + -J ${jobname} -cwd ${PWD} ${cmdfile} + +elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_time_${suffix}.err -V \ + -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} +fi @@ -244,54 +284,62 @@ list="count penalty omgnbc total omgbc" # data that a separate job for each provides a faster solution. # #--------------------------------------------------------------------------- - for sat in ${bigSATLIST}; do - - cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${sat} - jobname=plot_${RADMON_SUFFIX}_tm_${sat} - logfile=${LOGdir}/plot_time_${sat}.log +for sat in ${bigSATLIST}; do + jobname=plot_${RADMON_SUFFIX}_tm_${sat} - rm -f ${logfile} + cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${sat} + if [[ -e ${cmdfile} ]]; then rm -f ${cmdfile} + fi - ctr=0 - for var in $list; do - if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then - echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile - else - echo "$IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile - fi - ((ctr=ctr+1)) - done - chmod 755 $cmdfile + logfile=${LOGdir}/plot_time_${sat}.log + if [[ -e ${logfile} ]]; then + rm -f ${logfile} + fi - if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then - wall_tm="2:30" + ctr=0 + for var in $list; do + if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then + echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile else - wall_tm="1:00" + echo "$IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile fi + ((ctr=ctr+1)) + done + chmod 755 $cmdfile - if [[ $MY_MACHINE = "wcoss_d" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ - -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} + wall_tm="1:00" + if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then + wall_tm="2:30" + fi - elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ - -J ${jobname} -cwd ${PWD} ${cmdfile} + if [[ $MY_MACHINE = "wcoss_d" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ + -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} - elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ - --wrap "srun -l --multi-prog ${cmdfile}" + elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ + -J ${jobname} -cwd ${PWD} ${cmdfile} + + elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ + --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "jet" ]]; then - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ - -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" + elif [[ $MY_MACHINE = "jet" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ + -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=1:30:00 -N ${jobname} ${cmdfile} + elif [[ $MY_MACHINE = "wcoss2" ]]; then + logfile=${LOGdir}/plot_time_${sat}.log + if [[ -e ${logfile} ]]; then + rm ${logfile} fi - done + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_time_${sat}.err -V \ + -l select=1:mem=1g -l walltime=1:30:00 -N ${jobname} ${cmdfile} + fi + +done echo End mk_time_plots.sh diff --git a/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl b/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl index af13f64d87..f304279a0e 100755 --- a/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl +++ b/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl @@ -88,8 +88,8 @@ $search_string = $run; } - my @mmdirs = grep { /$search_string/ } @alldirs; - + my @mmdirs = grep {/$search_string/} @alldirs; + #----------------------------------------------------------------------- # If there are no $run.yyyymmdd subdirectories, then exit without # returning any date string. @@ -157,13 +157,26 @@ $hr_ctr = $hr_ctr - 1; $newdir = "${dirpath}/${sortmm[$ctr]}/${hrs[$hr_ctr]}/${lcm}"; -# print " newdir = $newdir \n"; - if( -d $newdir ) { opendir DIR, $newdir or die "Cannot open the current directory: $!"; - my @timefiles = grep { /ieee_d/ } readdir DIR; + my @dirfiles = readdir DIR; + my @timefiles = grep { /ieee_d/ } @dirfiles; + + # If no *ieee_d* files were found then look for the compressed format of + # radmon_*.tar files instead. If found then use the parent directories + # to establish the cycle time. + # + if( $#timefiles < 0 ) { + my @tarfiles = grep { /radmon_time.tar/ } @dirfiles; + + if( $#tarfiles >= 0 ) { + my $cycle = "${sortmm[$ctr]}${hrs[$hr_ctr]}"; + push( @timefiles, $cycle ); + } + } + if( $#timefiles >= 0 ) { my @sorttime = sort( @timefiles ); @@ -201,18 +214,13 @@ } } } - } } while $hr_ctr > 0 && $found_cycle == 0; -# print " found_cycle, ctr, end_ctr = $found_cycle, $ctr, $end_ctr \n"; - if( $cyc == 0 && $ctr >= $end_ctr ){ -# print " exiting from if\n"; $exit_flag = 1; } elsif( $cyc == 1 && $ctr <= $end_ctr ){ -# print " exiting from elsif\n"; $exit_flag = 1; } diff --git a/util/Radiance_Monitor/image_gen/ush/plot_angle.sh b/util/Radiance_Monitor/image_gen/ush/plot_angle.sh index 75d1baddb4..94dfd55e8c 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_angle.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_angle.sh @@ -52,18 +52,10 @@ edate0=`echo $edate|cut -c1-8` #-------------------------------------------------------------------- # Copy control files to $wrkdir - -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/angle -else - ctldir=$TANKverf/angle -fi - -echo ctldir = $ctldir +ctldir=$IMGNDIR/angle for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ + $NCP $ctldir/${type}*.ctl* ./ done ${UNCOMPRESS} *.ctl.${Z} @@ -74,20 +66,30 @@ ${UNCOMPRESS} *.ctl.${Z} # for type in ${SATYPE2}; do + $NCP $ctldir/${type}*.ctl* ./ + ${UNCOMPRESS} *.ctl.${Z} + cdate=$bdate - while [[ $cdate -le $edate ]] ; do + + #------------------------------------- + # Locate and copy data files. + # + while [[ $cdate -le $edate ]]; do + if [[ $REGIONAL_RR -eq 1 ]]; then tdate=`$NDATE +6 $cdate` - day=`echo $tdate | cut -c1-8 ` + day=`echo $tdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` . ${IG_SCRIPTS}/rr_set_tz.sh $cyc else - day=`echo $cdate | cut -c1-8 ` + day=`echo $cdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` fi + #---------------------------------------------------- + # Attempt to locate the extracted ieee data files. + # ieee_src=${TANKverf}/${RUN}.${day}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${day}/${MONITOR} fi @@ -98,46 +100,39 @@ for type in ${SATYPE2}; do ieee_src=${TANKverf}/${RUN}.${day} fi - - if [[ -d ${ieee_src} ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.angle.${type}.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/angle.${type}.${cdate}.ieee_d - else - test_file=${ieee_src}/angle.${type}.${cdate}.ieee_d - fi - - if [[ $USE_ANL = 1 ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.angle.${type}_anl.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/angle.${type}_anl.${cdate}.ieee_d - else - test_file2=${ieee_src}/angle.${type}_anl.${cdate}.ieee_d - fi - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the data files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_angle.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_angle.tar ${files} fi + else + files=`ls ${ieee_src}/angle.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi - - adate=`$NDATE +${CYCLE_INTERVAL} $cdate` + adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` cdate=$adate done + ${UNCOMPRESS} $wrkdir/*.ieee_d.${Z} + #----------------------------------------------- + # Remove 'angle.' from the *ieee_d file names. + # + prefix="angle." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + #----------------------------------------------------------------------- # mk_digital_ang.sh produces the text files used by the js/html files # to generate the interactive charts diff --git a/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh b/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh index 0a21b5a555..d9c783c1f9 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh @@ -33,46 +33,39 @@ edate0=`echo $edate|cut -c1-8` #-------------------------------------------------------------------- # Copy executable and control files to $tmpdir - -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/bcoef -else - ctldir=$TANKverf/bcoef -fi - -echo ctldir = $ctldir - +ctldir=$IMGNDIR/bcoef +$NCP ${IG_SCRIPTS}/mk_digital_bcoef.sh . #-------------------------------------------------------------------- # Loop over satellite types. Copy data files, create plots and # place on the web server. # -# Data file location may either be in angle, bcoef, bcor, and time -# subdirectories under $TANKverf, or in the Operational organization -# of radmon.YYYYMMDD directories under $TANKverf - - -$NCP ${IG_SCRIPTS}/mk_digital_bcoef.sh . - for type in ${SATYPE}; do - $NCP $ctldir/${type}.ctl* ./ - ${UNCOMPRESS} ${type}.ctl.${Z} + $NCP $ctldir/${type}*.ctl* ./ + ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate + + #------------------------------------- + # Locate and copy data files. + # while [[ $cdate -le $edate ]]; do + if [[ $REGIONAL_RR -eq 1 ]]; then tdate=`$NDATE +6 $cdate` - day=`echo $tdate | cut -c1-8 ` + day=`echo $tdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` . ${IG_SCRIPTS}/rr_set_tz.sh $cyc else - day=`echo $cdate | cut -c1-8 ` + day=`echo $cdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` fi + #---------------------------------------------------- + # Attempt to locate the extracted ieee data files. + # ieee_src=${TANKverf}/${RUN}.${day}/${cyc}/${MONITOR} if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${day}/${MONITOR} @@ -84,46 +77,39 @@ for type in ${SATYPE}; do ieee_src=${TANKverf}/${RUN}.${day} fi - if [[ -d ${ieee_src} ]]; then - - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.bcoef.${type}.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/bcoef.${type}.${cdate}.ieee_d - else - test_file=${ieee_src}/bcoef.${type}.${cdate}.ieee_d - fi - - if [[ $USE_ANL = 1 ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.bcoef.${type}_anl.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/bcoef.${type}_anl.${cdate}.ieee_d - else - test_file2=${ieee_src}/bcoef.${type}_anl.${cdate}.ieee_d - fi - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the ieee_d files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_bcoef.tar ${files} fi - + else + files=`ls ${ieee_src}/bcoef.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi - adate=`$NDATE +${CYCLE_INTERVAL} $cdate` + adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` cdate=$adate done + ${UNCOMPRESS} *.ieee_d.${Z} + #----------------------------------------------- + # Remove 'bcoef.' from the *ieee_d file names. + # + prefix="bcoef." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then list="mean atmpath clw lapse2 lapse cos_ssmis sin_ssmis emiss ordang4 ordang3 ordang2 ordang1" for var in $list; do @@ -140,23 +126,24 @@ EOF if [[ ! -d ${IMGNDIR}/bcoef ]]; then mkdir -p ${IMGNDIR}/bcoef fi - cp -f *.png ${IMGNDIR}/bcoef + mv *.png ${IMGNDIR}/bcoef fi #-------------------------------------------------------------------------- # mk_digital_bcoef.sh produces the data files needed by the js/html files # to generate the interactive charts. + # ./mk_digital_bcoef.sh ${type} - + rm -f ${type}*ieee_d done #-------------------------------------------------------------------- # Clean $tmpdir. Submit done job. - +# cd $tmpdir cd ../ rm -rf $tmpdir diff --git a/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh b/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh index 83702c9e1e..feb17576fd 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh @@ -17,7 +17,7 @@ echo "begin plot_bcor.sh" #------------------------------------------------------------------ # Set environment variables. - +# word_count=`echo $PTYPE | wc -w` echo word_count = $word_count @@ -43,47 +43,39 @@ edate=$PDATE bdate0=`echo $bdate|cut -c1-8` edate0=`echo $edate|cut -c1-8` - -#-------------------------------------------------------------------- -# Set ctldir to point to control file directory - -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/bcor -else - ctldir=$TANKverf/bcor -fi - -echo ctldir = $ctldir - +ctldir=$IMGNDIR/bcor #-------------------------------------------------------------------- # Loop over satellite types. Copy data files, create plots and # place on the web server. # -# Data file location may either be in angle, bcoef, bcor, and time -# subdirectories under $TANKverf, or in the Operational organization -# of radmon.YYYYMMDD directories under $TANKverf - for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ + $NCP $ctldir/${type}*.ctl* ./ ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate + + #------------------------------------- + # Locate and copy data files. + # while [[ $cdate -le $edate ]]; do + if [[ $REGIONAL_RR -eq 1 ]]; then tdate=`$NDATE +6 $cdate` - day=`echo $tdate | cut -c1-8 ` + day=`echo $tdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` . ${IG_SCRIPTS}/rr_set_tz.sh $cyc else - day=`echo $cdate | cut -c1-8 ` - cyc=`echo $cdate | cut -c9-10 ` + day=`echo $cdate | cut -c1-8` + cyc=`echo $cdate | cut -c9-10` fi + #---------------------------------------------------- + # Attempt to locate the parent directory for the + # extracted ieee data files. + # ieee_src=${TANKverf}/${RUN}.${day}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${day}/${MONITOR} fi @@ -94,39 +86,38 @@ for type in ${SATYPE2}; do ieee_src=${TANKverf}/${RUN}.${day} fi - nfile_src=`ls -l ${ieee_src}/*${cdate}*ieee_d* | egrep -c '^-'` - - echo "nfile_src = $nfile_src" - - if [[ -d ${ieee_src} ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then - test_file=${ieee_src}/${rgnHH}.bcor.${type}.${cdate}.ieee_d.${rgnTM} - else - test_file=${ieee_src}/bcor.${type}.${cdate}.ieee_d - fi - - if [[ $USE_ANL = 1 ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then - test_file=${ieee_src}/${rgnHH}.bcor.${type}_anl.${cdate}.ieee_d.${rgnTM} - else - test_file2=${ieee_src}/bcor.${type}_anl.${cdate}.ieee_d - fi - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi + + #----------------------------------------------------------- + # Now locate this cycle's data files, first checking for + # a tar file, and copy them locally. + # + if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ieee_d` + tar -xf ${ieee_src}/radmon_bcor.tar ${files} + + else + files=`ls ${ieee_src}/bcor.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi - adate=`$NDATE +${CYCLE_INTERVAL} $cdate` + adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` cdate=$adate done + ${UNCOMPRESS} *.ieee_d.${Z} + #----------------------------------------------- + # Remove 'bcor.' from the *ieee_d file names. + # + prefix="bcor." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + for var in ${PTYPE}; do echo $var if [ "$var" = 'count' ]; then @@ -147,47 +138,20 @@ EOF $GRADS -bpc "run ${tmpdir}/${type}_${var}.gs" done -#-------------------------------------------------------------------- -# Delete data files - -# rm -f ${type}.ieee_d -# rm -f ${type}.ctl - done #-------------------------------------------------------------------- # Copy image files to $IMGNDIR to set up for mirror to web server. -# Delete images and data files. - -if [[ ! -d ${IMGNDIR}/bcor ]]; then - mkdir -p ${IMGNDIR}/bcor -fi -cp -r *.png ${IMGNDIR}/bcor - +# +mv *.png ${IMGNDIR}/bcor/. #-------------------------------------------------------------------- # Clean $tmpdir - +# cd $tmpdir cd ../ rm -rf $tmpdir - -#-------------------------------------------------------------------- -# If this is the last bcor plot job to finish then rm PLOT_WORK_DIR. -# - -#count=`ls ${LOADLQ}/*plot*_${RADMON_SUFFIX}* | wc -l` -#complete=`grep "COMPLETED" ${LOADLQ}/*plot*_${RADMON_SUFFIX}* | wc -l` - -#running=`expr $count - $complete` - -#if [[ $running -eq 1 ]]; then -# cd ${PLOT_WORK_DIR} -# cd ../ -# rm -rf ${PLOT_WORK_DIR} -#fi - echo "end plot_bcor.sh" exit diff --git a/util/Radiance_Monitor/image_gen/ush/plot_summary.sh b/util/Radiance_Monitor/image_gen/ush/plot_summary.sh index bc394bb8e7..61455bd6bf 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_summary.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_summary.sh @@ -29,17 +29,7 @@ edate=$PDATE bdate0=`echo $bdate|cut -c1-8` edate0=`echo $edate|cut -c1-8` -#-------------------------------------------------------------------- -# Set ctldir to point to correct control file source -# -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/time -else - ctldir=$TANKverf/time -fi - - +ctldir=$IMGNDIR/time usef="use.txt" timesf="times.txt" chanf="chan.txt" @@ -52,7 +42,7 @@ chanf="chan.txt" # for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ + $NCP $ctldir/${type}*.ctl* ./ ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate @@ -88,24 +78,20 @@ for type in ${SATYPE2}; do echo "rgnHH, rgnTM = $rgnHH, $rgnTM" - test_file=${ieee_src}/time.${type}.${cdate}.ieee_d - - if [[ $USE_ANL = 1 ]]; then - test_file2=${ieee_src}/time.${type}_anl.${cdate}.ieee_d - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the data files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_time.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_time.tar ${files} + fi + else + files=`ls ${ieee_src}/time.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` @@ -114,17 +100,26 @@ for type in ${SATYPE2}; do ${UNCOMPRESS} *.ieee_d.${Z} + #----------------------------------------------- + # Remove 'time.' from the *ieee_d file names. + # + prefix="time." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done -#-------------------------------------------------------------------- -# Plotting is moving towards dynamic, interactive images drawn -# on-the-fly in the client browser. These images require small text -# files instead of static images produced by GrADS. The flag -# PLOT_STATIC_IMGS controls the conditional plotting of the older -# static images. -# -# At present this only affects the summary plots, but will eventually -# include most radiance images. -# + #-------------------------------------------------------------------- + # Plotting is moving towards dynamic, interactive images drawn + # on-the-fly in the client browser. These images require small text + # files instead of static images produced by GrADS. The flag + # PLOT_STATIC_IMGS controls the conditional plotting of the older + # static images. + # + # At present this only affects the summary plots, but will eventually + # include most radiance images. + # if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then outfile=${tmpdir}/${type}.gs @@ -204,7 +199,7 @@ EOF echo "END data file generation:" rm -f ${input} - + rm -f ${type}*ieee_d done @@ -215,13 +210,14 @@ done if [[ ! -d ${IMGNDIR}/summary ]]; then mkdir -p ${IMGNDIR}/summary fi -$NCP *summary.png ${IMGNDIR}/summary/. + +if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then + $NCP *summary.png ${IMGNDIR}/summary/. +fi for type in ${SATYPE2}; do - $NCP ${type}.sum.txt ${IMGNDIR}/summary/${type}.${PDATE}.sum.txt + mv ${type}.sum.txt ${IMGNDIR}/summary/${type}.${PDATE}.sum.txt done -$NCP *.sum.txt ${IMGNDIR}/summary/. - #-------------------------------------------------------------------- diff --git a/util/Radiance_Monitor/image_gen/ush/plot_time.sh b/util/Radiance_Monitor/image_gen/ush/plot_time.sh index e33528acdd..8d45e1b2ce 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_time.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_time.sh @@ -45,17 +45,7 @@ edate=$PDATE bdate0=`echo $bdate|cut -c1-8` edate0=`echo $edate|cut -c1-8` -#-------------------------------------------------------------------- -# Set ctldir to point to correct control file source -# -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/time -else - ctldir=$TANKverf/time -fi - -echo ctldir = $ctldir +ctldir=$IMGNDIR/time #-------------------------------------------------------------------- @@ -68,15 +58,8 @@ echo ctldir = $ctldir # for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ - if [[ -s ./${type}.ctl.${Z} ]]; then - ${UNCOMPRESS} ./${type}.ctl.${Z} - fi - - if [[ $USE_ANL = 1 ]]; then - $NCP $ctldir/${type}_anl.ctl* ./ - ${UNCOMPRESS} ./${type}_anl.ctl.${Z} - fi + $NCP $ctldir/${type}*.ctl* ./ + ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate while [[ $cdate -le $edate ]]; do @@ -102,27 +85,20 @@ for type in ${SATYPE2}; do ieee_src=${TANKverf}/${RUN}.${day} fi - - if [[ -d ${ieee_src} ]]; then - test_file=${ieee_src}/time.${type}.${cdate}.ieee_d - - if [[ $USE_ANL = 1 ]]; then - test_file2=${ieee_src}/time.${type}_anl.${cdate}.ieee_d - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the data files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_time.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_time.tar ${files} fi + else + files=`ls ${ieee_src}/time.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi adate=`$NDATE +${CYCLE_INTERVAL} $cdate` @@ -131,6 +107,17 @@ for type in ${SATYPE2}; do ${UNCOMPRESS} ./*.ieee_d.${Z} + #----------------------------------------------- + # Remove 'time.' from the *ieee_d file names. + # + prefix="time." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + + if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then for var in ${PTYPE}; do echo $var diff --git a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt index 30836d410c..b8c4d0aa90 100644 --- a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt +++ b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt @@ -1,2 +1,2 @@ -abi_g16 abi_g17 ahi_himawari8 amsua_metop-b amsua_metop-c amsua_n15 amsua_n18 amsua_n19 atms_npp atms_n20 avhrr_metop-b avhrr_n18 avhrr_n19 cris-fsr_npp cris-fsr_n20 hirs4_n19 iasi_metop-b iasi_metop-c mhs_metop-b mhs_metop-c mhs_n19 seviri_m08 seviri_m11 ssmis_f17 ssmis_f18 +abi_g16 ahi_himawari8 amsua_metop-b amsua_metop-c amsua_n15 amsua_n18 amsua_n19 atms_npp atms_n20 avhrr_metop-b avhrr_n18 avhrr_n19 cris-fsr_npp cris-fsr_n20 hirs4_n19 iasi_metop-b iasi_metop-c mhs_metop-b mhs_metop-c mhs_n19 seviri_m08 seviri_m11 ssmis_f17 ssmis_f18 diff --git a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt index ecb51f7b6d..122b828255 100644 --- a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt +++ b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt @@ -47,6 +47,7 @@ mhs_metop-c -49.444 1.111 90 iasi_metop-a -48.330 3.334 60 iasi_metop-b -48.330 3.334 60 + iasi_metop-c -48.330 3.334 60 cris_npp -48.330 3.333 30 cris-fsr_npp -48.330 3.333 30 cris-fsr_n20 -48.330 3.333 30 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt index 19aba63061..a881d15816 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_ANGLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_ANGLE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_angle ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt index 5068244458..116bcfe319 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_BCOEF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_BCOEF_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_bcoef ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt index 59ac8532da..7b2bfe03a0 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_BCOR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_BCOR_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_bcor ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt index 3bb4e4b2ec..27cac80252 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_TIME_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_TIME_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_time ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh index b592c76990..2b0bf1ecd4 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh @@ -162,10 +162,7 @@ else fi if [[ $REGIONAL_RR -eq 1 ]]; then -# angl_file=${rgnHH}.angle.${data_file}.${rgnTM} angl_file=${rgnHH}.${data_file}.${rgnTM} -# else -# angl_file=angle.${data_file} fi @@ -228,17 +225,19 @@ EOF done # for type in ${SATYPE} loop - cwd=`pwd` - tar_file=radmon_angle.tar + ${USHradmon}/rstprod.sh + tar_file=radmon_angle.tar tar -cf $tar_file angle*.ieee_d* angle*.ctl* - mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - cd ${cwd} + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then err=3 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh index 14a5905a81..bd4642d77b 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh @@ -210,17 +210,19 @@ EOF done # type in $SATYPE loop - cwd=`pwd` - tar_file=radmon_bcoef.tar + ${USHradmon}/rstprod.sh + tar_file=radmon_bcoef.tar tar -cf $tar_file bcoef*.ieee_d* bcoef*.ctl* mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - - cd $cwd + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then err=5 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh index 553cb8111f..6f2bb0388a 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh @@ -202,18 +202,20 @@ EOF done # dtype in $gesanl loop done # type in $SATYPE loop - cwd=`pwd` + + ${USHradmon}/rstprod.sh tar_file=radmon_bcor.tar tar -cf $tar_file bcor*.ieee_d* bcor*.ctl* mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - - cd $cwd - + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then err=7 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh index 4603dad218..97f53307ac 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh @@ -260,18 +260,19 @@ EOF done - cwd=`pwd` - tar_file=radmon_time.tar + ${USHradmon}/rstprod.sh + tar_file=radmon_time.tar tar -cf $tar_file time*.ieee_d* time*.ctl* mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - - cd $cwd - + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then echo "fail, ctr = $fail, $ctr" diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh new file mode 100755 index 0000000000..dd5a09ec02 --- /dev/null +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +#--------------------------------------------------------- +# rstprod.sh +# +# Restrict data from select sensors and satellites +#--------------------------------------------------------- + +# Restrict select sensors and satellites +echo "--> rstprod.sh" + +export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} +rlist="saphir abi_g16" +for rtype in $rlist; do + ${CHGRP_CMD} *${rtype}* +done + +echo "<-- rstprod.sh" diff --git a/util/Radiance_Monitor/parm/RadMon_config b/util/Radiance_Monitor/parm/RadMon_config index d411da3902..381c4767bb 100644 --- a/util/Radiance_Monitor/parm/RadMon_config +++ b/util/Radiance_Monitor/parm/RadMon_config @@ -222,6 +222,8 @@ elif [[ $MY_MACHINE = "wcoss2" ]]; then # module use /apps/test/modules module load GrADS/2.2.1-cce-11.0.4 + export GRADS=`which grads` + export STNMAP=`which stnmap` #------------------------ export SUB="qsub"