Update ATM var and ens to python based g-w (#388)

* add sondes to test_gdasapp_atm_jjob_var_run, replace geometric height with geopoential height in sondes.yaml (#357)

* update atm_jjob var and ens ctests to work with python UFS-DA ATM g-w (#357)

* correct yaml syntax error in config.yaml (#357)

* correct name of ens final j-job on Orion (#357)

* lgetkf updates: specify analysis variables, turn off posterior observer (#357)

* update yamls used in UFS-DA ATM prototype cycling to be consistent with g-w python (#357)

* remove RESTART from bkg path

* Update UFS DA ATM yaml and scripts to use g-w pygw (#357)

* remove 10-digit datetime string from radiance bias correction filenames (#357)

* replace JEDIVAREXE with JEDIEXE in config.atmanl (#357)

* remove extraneous yaml import (#357)

* add g-w pygw to PYTHONPATH in run_jedi_exe scripts (#357)

* move atm run_jedi_exe ctests into global-workflow (#357)

parm/atm/berror/hybvar_bump.yaml

@@ -10,7 +10,7 @@ components:
       active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr]
       bump:
         io:
-          data directory: !ENV &staticb_dir ${DATA}/berror
+          data directory: &staticb_dir $(DATA)/berror
         drivers:
           multivariate strategy: specific_univariate
           read local nicas: true
@@ -95,17 +95,17 @@ components:
     covariance model: ensemble
     members from template:
       template:
-        datetime: '{{BKG_ISOTIME}}'
+        datetime: '{{ current_cycle | to_isotime }}'
         filetype: fms restart
         state variables: *3dvars
         datapath: ens/mem%mem%/
-        filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc'
-        filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc'
-        filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc'
-        filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc'
-        filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res'
+        filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc'
+        filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc'
+        filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc'
+        filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
+        filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
       pattern: '%mem%'
-      nmembers: !ENV ${NMEM_ENKF}
+      nmembers: $(NMEM_ENKF)
       zero padding: 3
 #    localization:
 #      localization method: SABER

parm/atm/berror/hybvar_gsibec.yaml

@@ -8,10 +8,10 @@ components:
                                       specific_humidity,cloud_liquid_ice,cloud_liquid_water,
                                       mole_fraction_of_ozone_in_air]
       output variables: *control_vars
-      gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4
-      gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4
-#     gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77
-      gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml
+      gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4
+      gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4
+#     gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77
+      gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml
       processor layout x direction: &layout_gsib_x 3
       processor layout y direction: &layout_gsib_y 2
       debugging mode: false
@@ -35,17 +35,17 @@ components:
     covariance model: ensemble
     members from template:
       template:
-        datetime: '{{BKG_ISOTIME}}'
+        datetime: '{{ current_cycle | to_isotime }}'
         filetype: fms restart
         state variables: *3dvars_anal
         datapath: ens/mem%mem%
-        filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc'
-        filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc'
-        filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc'
-        filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc'
-        filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res'
+        filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc'
+        filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc'
+        filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc'
+        filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
+        filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
       pattern: '%mem%'
-      nmembers: !ENV ${NMEM_ENKF}
+      nmembers: $(NMEM_ENKF)
       zero padding: 3
 #    localization:
 #      localization method: SABER

parm/atm/berror/staticb_bump.yaml

@@ -8,7 +8,7 @@ saber blocks:
   active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr]
   bump:
     io:
-      data directory: !ENV &staticb_dir ${DATA}/berror
+      data directory: &staticb_dir $(DATA)/berror
     drivers:
       multivariate strategy: specific_univariate
       read local nicas: true

parm/atm/berror/staticb_gsibec.yaml

@@ -6,10 +6,10 @@ saber central block:
                            specific_humidity,cloud_liquid_ice,cloud_liquid_water,
                            mole_fraction_of_ozone_in_air]
   output variables: *control_vars
-  gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4
-  gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4
-# gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77
-  gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml
+  gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4
+  gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4
+# gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77
+  gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml
   processor layout x direction: &layout_gsib_x 3
   processor layout y direction: &layout_gsib_y 2
   debugging mode: false

parm/atm/hofx/hofx4d.yaml

@@ -1,35 +1,35 @@
-window begin: '{{ATM_WINDOW_BEGIN}}'
+window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
 window length: $(ATM_WINDOW_LENGTH)
 forecast length: $(ATM_WINDOW_LENGTH)
 geometry:
   fms initialization:
-    namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml
-    field table filename: !ENV ${DATA}/fv3jedi/field_table
-  akbk: !ENV ${DATA}/fv3jedi/akbk.nc4
+    namelist filename: $(DATA)/fv3jedi/fmsmpp.nml
+    field table filename: $(DATA)/fv3jedi/field_table
+  akbk: $(DATA)/fv3jedi/akbk.nc4
   layout:
-  - !ENV ${layout_x}
-  - !ENV ${layout_y}
+  - $(layout_x)
+  - $(layout_y)
   npx: $(npx_ges)
   npy: $(npy_ges)
   npz: $(npz_ges)
-  field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml
+  field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml
 initial condition:
-  datapath: !ENV ${DATA}/bkg
+  datapath: $(DATA)/bkg
   filetype: fms restart
-  datetime: '{{ATM_WINDOW_BEGIN}}'
-  filename_core: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_core.res.nc'
-  filename_trcr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_tracer.res.nc'
-  filename_sfcd: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.sfc_data.nc'
-  filename_sfcw: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc'
-  filename_cplr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.coupler.res'
+  datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+  filename_core: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_core.res.nc'
+  filename_trcr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_tracer.res.nc'
+  filename_sfcd: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.sfc_data.nc'
+  filename_sfcw: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_srf_wnd.res.nc'
+  filename_cplr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.coupler.res'
   state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis,
                     slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                     u_srf,v_srf,f10m]
 model:
   name: PSEUDO
-  datapath: !ENV ${DATA}/bkg
+  datapath: $(DATA)/bkg
   filetype: fms restart
-  datetime: '{{ATM_WINDOW_BEGIN}}'
+  datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
   filename is datetime templated: true
   filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc'
   filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc'

parm/atm/hofx/hofx_nomodel.yaml

@@ -1,26 +1,26 @@
-window begin: '{{ATM_WINDOW_BEGIN}}'
+window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
 window length: $(ATM_WINDOW_LENGTH)
 geometry:
   fms initialization:
-    namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml
-    field table filename: !ENV ${DATA}/fv3jedi/field_table
-  akbk: !ENV ${DATA}/fv3jedi/akbk.nc4
+    namelist filename: $(DATA)/fv3jedi/fmsmpp.nml
+    field table filename: $(DATA)/fv3jedi/field_table
+  akbk: $(DATA)/fv3jedi/akbk.nc4
   layout:
-  - !ENV ${layout_x}
-  - !ENV ${layout_y}
+  - $(layout_x)
+  - $(layout_y)
   npx: $(npx_ges)
   npy: $(npy_ges)
   npz: $(npz_ges)
-  field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml
+  field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml
 state:
-  datapath: !ENV ${DATA}/bkg
+  datapath: $(DATA)/bkg
   filetype: fms restart
-  datetime: '{{BKG_ISOTIME}}'
-  filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc'
-  filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc'
-  filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc'
-  filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc'
-  filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res'
+  datetime: '{{ current_cycle | to_isotime }}'
+  filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc'
+  filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc'
+  filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc'
+  filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
+  filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
   state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis,
                     slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                     u_srf,v_srf,f10m]

parm/atm/lgetkf/lgetkf.yaml

@@ -4,32 +4,33 @@ geometry:
     field table filename: ./fv3jedi/field_table
   akbk: ./fv3jedi/akbk.nc4
   layout:
-  - !ENV ${layout_x}
-  - !ENV ${layout_y}
+  - $(layout_x)
+  - $(layout_y)
   npx: $(npx_ges)
   npy: $(npy_ges)
   npz: $(npz_ges)
   field metadata override: ./fv3jedi/gfs-restart.yaml
-window begin: '$(ATM_WINDOW_BEGIN)'
-window length: PT6H
+window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+window length: $(ATM_WINDOW_LENGTH)
+increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr]
 
 background:
-   datetime: '{{BKG_ISOTIME}}'
+   datetime: '{{ current_cycle | to_isotime }}'
    members from template:
      template:
-       datetime: '{{BKG_ISOTIME}}'
+       datetime: '{{ current_cycle | to_isotime }}'
        filetype: fms restart
        state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis,
                         slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                         u_srf,v_srf,f10m]
        datapath: bkg/mem%mem%
-       filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc'
-       filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc'
-       filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc'
-       filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc'
-       filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res'
+       filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc'
+       filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc'
+       filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc'
+       filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc'
+       filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res'
      pattern: '%mem%'
-     nmembers: !ENV ${NMEM_ENKF}
+     nmembers: $(NMEM_ENKF)
      zero padding: 3
 
 observations: !INC ${OBS_LIST}
@@ -41,6 +42,7 @@ driver:
   save prior mean: false
   save posterior mean increment: false
   save posterior ensemble increments: true
+  do posterior observer: false
 
 local ensemble DA:
   solver: GETKF

parm/atm/obs/config/aircraft.yaml

@@ -3,15 +3,15 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs/${OPREFIX}aircraft.${CDATE}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)aircraft.${{ current_cycle | to_YMDH }}.nc4
     obsgrouping:
       group variables: ["stationIdentification"]
       sort variable: "pressure"
       sort order: "descending"
   obsdataout:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/diags/diag_aircraft_${CDATE}.nc4
+      obsfile: $(DATA)/diags/diag_aircraft_${{ current_cycle | to_YMDH }}.nc4
   io pool:
     max pool size: 1
   simulated variables: [windEastward, windNorthward, airTemperature, specificHumidity]

parm/atm/obs/config/amsua_n19.yaml

@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs/${OPREFIX}amsua_n19.${PDY}${cyc}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/diags/diag_amsua_n19_${PDY}${cyc}.nc4
+      obsfile: $(DATA)/diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4
   io pool:
     max pool size: 1
   simulated variables: [brightnessTemperature]
@@ -20,16 +20,16 @@ obs operator:
   obs options:
     Sensor_ID: amsua_n19
     EndianType: little_endian
-    CoefficientPath: !ENV ${DATA}/crtm/
+    CoefficientPath: $(DATA)/crtm/
 obs bias:
-  input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${gPDY}${gcyc}.nc4
-  output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${PDY}${cyc}.nc4
+  input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4
+  output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4
   variational bc:
     predictors:
     - name: constant
     - name: lapse_rate
       order: 2
-      tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${gPDY}${gcyc}.txt
+      tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.txt
     - name: lapse_rate
       tlapse: *amsua_n19_tlapse
     - name: emissivity
@@ -46,11 +46,11 @@ obs bias:
     step size: 1.0e-4
     largest analysis variance: 10000.0
     prior:
-      input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${gPDY}${gcyc}.nc4
+      input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.nc4
       inflation:
         ratio: 1.1
         ratio for small dataset: 2.0
-    output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${PDY}${cyc}.nc4
+    output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4
 obs filters:
 - filter: BlackList
   filter variables:

parm/atm/obs/config/atms_n20.yaml

@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs/${OPREFIX}atms_n20.${CDATE}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.${{ current_cycle | to_YMDH }}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/diags/diag_atms_n20_${CDATE}.nc4
+      obsfile: $(DATA)/diags/diag_atms_n20_${{ current_cycle | to_YMDH }}.nc4
   io pool:
     max pool size: 1
   simulated variables: [brightness_temperature]
@@ -20,16 +20,16 @@ obs operator:
   obs options:
     Sensor_ID: atms_n20
     EndianType: little_endian
-    CoefficientPath: !ENV ${DATA}/crtm/
+    CoefficientPath: $(DATA)/crtm/
 obs bias:
-  input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias.${GDATE}.nc4
-  output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias.${CDATE}.nc4
+  input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4
+  output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.nc4
   variational bc:
     predictors:
     - name: constant
     - name: lapse_rate
       order: 2
-      tlapse: &atms_n20_tlapse !ENV ${DATA}/obs/${GPREFIX}atms_n20.tlapse.${GDATE}.txt
+      tlapse: &atms_n20_tlapse $(DATA)/obs/$(GPREFIX)atms_n20.tlapse.txt
     - name: lapse_rate
       tlapse: *atms_n20_tlapse
     - name: emissivity
@@ -46,11 +46,11 @@ obs bias:
     step size: 1.0e-4
     largest analysis variance: 10000.0
     prior:
-      input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias_cov.${GDATE}.nc4
+      input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias_cov.nc4
       inflation:
         ratio: 1.1
         ratio for small dataset: 2.0
-    output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias_cov.${CDATE}.nc4
+    output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.nc4
 obs filters:
 - filter: BlackList
   filter variables: