diff --git a/CMakeLists.txt b/CMakeLists.txt index a822301fe..3281931f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ if(BUILD_GDASBUNDLE) # External (required) observation operators option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF - ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda/crtm.git" TAG v2.3-jedi.3 ) + ecbuild_bundle( PROJECT crtm GIT "https://github.com/ADCollard/crtm.git" TAG v2.3-jedi.3_fix ) # Build GSI-B option(BUILD_GSIBEC "Build GSI-B" OFF) @@ -75,7 +75,7 @@ if(BUILD_GDASBUNDLE) ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop ) ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop ) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop ) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH feature/gdasapp_ufo ) # FMS and FV3 dynamical core ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) diff --git a/parm/atm/obs/testing/amsua_metop-a.yaml b/parm/atm/obs/testing/amsua_metop-a.yaml new file mode 100644 index 000000000..d2951e032 --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-a.yaml @@ -0,0 +1,471 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-a + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_metop-a + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-a_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-a_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-15 +geovals: + filename: !ENV amsua_metop-a_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_metop-a_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-a_tlap !ENV amsua_metop-a_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-a_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *all_channels + options: + channels: *all_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *all_channels + - name: HofX/brightnessTemperature + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 93421 + diff --git a/parm/atm/obs/testing/amsua_metop-b.yaml b/parm/atm/obs/testing/amsua_metop-b.yaml new file mode 100644 index 000000000..3c666d8e9 --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-b.yaml @@ -0,0 +1,471 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-b + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_metop-b + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-b_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-b_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-15 +geovals: + filename: !ENV amsua_metop-b_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_metop-b_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-b_tlap !ENV amsua_metop-b_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-b_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *all_channels + options: + channels: *all_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *all_channels + - name: HofX/brightnessTemperature + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 71068 + diff --git a/parm/atm/obs/testing/amsua_metop-c.yaml b/parm/atm/obs/testing/amsua_metop-c.yaml new file mode 100644 index 000000000..0e9e82795 --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-c.yaml @@ -0,0 +1,470 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-c + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_metop-c + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-c_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-c_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-15 +geovals: + filename: !ENV amsua_metop-c_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_metop-c_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-c_tlap !ENV amsua_metop-c_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-c_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *all_channels + options: + channels: *all_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *all_channels + - name: HofX/brightnessTemperature + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 113345 diff --git a/parm/atm/obs/testing/amsua_n15.yaml b/parm/atm/obs/testing/amsua_n15.yaml new file mode 100644 index 000000000..973a2f26a --- /dev/null +++ b/parm/atm/obs/testing/amsua_n15.yaml @@ -0,0 +1,470 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n15 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_n15 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n15_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n15_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-15 +geovals: + filename: !ENV amsua_n15_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_n15_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n15_tlap !ENV amsua_n15_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n15_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *all_channels + options: + channels: *all_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *all_channels + - name: HofX/brightnessTemperature + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 91792 diff --git a/parm/atm/obs/testing/amsua_n18.yaml b/parm/atm/obs/testing/amsua_n18.yaml new file mode 100644 index 000000000..d11eba99b --- /dev/null +++ b/parm/atm/obs/testing/amsua_n18.yaml @@ -0,0 +1,471 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: amsua_n18 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n18_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n18_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-15 +geovals: + filename: !ENV amsua_n18_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV amsua_n18_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n18_tlap !ENV amsua_n18_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n18_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *all_channels + options: + channels: *all_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *all_channels + - name: HofX/brightnessTemperature + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 116999 + diff --git a/parm/atm/obs/testing/amsua_n19.yaml b/parm/atm/obs/testing/amsua_n19.yaml index 05048e5e0..f91b045c1 100644 --- a/parm/atm/obs/testing/amsua_n19.yaml +++ b/parm/atm/obs/testing/amsua_n19.yaml @@ -4,7 +4,7 @@ obs operator: Clouds: [Water, Ice] Cloud_Fraction: 1.0 obs options: - Sensor_ID: amsua_n19 + Sensor_ID: &Sensor_ID amsua_n19 EndianType: little_endian CoefficientPath: crtm/ obs space: @@ -18,7 +18,7 @@ obs space: type: H5File obsfile: !ENV amsua_n19_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: 1-15 + channels: &all_channels 1-15 geovals: filename: !ENV amsua_n19_geoval_${CDATE}.nc4 obs bias: @@ -39,3 +39,432 @@ obs bias: - name: scan_angle order: 2 - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *all_channels + options: + channels: *all_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *all_channels + - name: HofX/brightnessTemperature + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *all_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 107747 diff --git a/parm/atm/obs/testing/atms_n20.yaml b/parm/atm/obs/testing/atms_n20.yaml new file mode 100644 index 000000000..1eaf198ce --- /dev/null +++ b/parm/atm/obs/testing/atms_n20.yaml @@ -0,0 +1,491 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: atms_n20 + obsdatain: + engine: + type: H5File + obsfile: !ENV atms_n20_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV atms_n20_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-22 +geovals: + filename: !ENV atms_n20_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV atms_n20_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_n20_tlap !ENV atms_n20_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *atms_n20_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *all_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *all_channels + options: + channels: *all_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Calculate Innovation@DerivedMetaData +- filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *all_channels + - name: brightnessTemperature@HofX + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: HydrometeorCheckATMS@ObsFunction + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *all_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *all_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 171071 # GSI: 171076 +# Notes: one point difference in HydrometeorCheck for channel 7 + diff --git a/parm/atm/obs/testing/atms_npp.yaml b/parm/atm/obs/testing/atms_npp.yaml new file mode 100644 index 000000000..6558d79cc --- /dev/null +++ b/parm/atm/obs/testing/atms_npp.yaml @@ -0,0 +1,491 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: atms_npp + obsdatain: + engine: + type: H5File + obsfile: !ENV atms_npp_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV atms_npp_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-22 +geovals: + filename: !ENV atms_npp_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV atms_npp_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_npp_tlap !ENV atms_npp_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *atms_npp_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for CLW retrieval +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: CLWRetrievalReject + initial value: false + force reinitialization: true + +# Diagnostic flag for hydrometeor check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: HydrometeorCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for gross check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: GrossCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for inter-channel consistency check +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *all_channels + flags: + - name: InterChannelCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *all_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *all_channels + options: + channels: *all_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *all_channels + options: + channels: *all_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Calculate Innovation@DerivedMetaData +- filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *all_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *all_channels + - name: brightnessTemperature@HofX + channels: *all_channels + coefs: [1, -1] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + +# Step 2: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 3: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalReject + - name: reject + +# Step 4: Window channel sanity check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: HydrometeorCheckATMS@ObsFunction + channels: *all_channels + options: + channels: *all_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheckReject + ignore: rejected observations + - name: reject + +# Step 6: Observation error inflation based on topography check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *all_channels + +# Step 7: Obs error inflation based on TOA transmittancec check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *all_channels + options: + channels: *all_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *all_channels + +# Step 8: Observation error inflation based on surface jacobian check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *all_channels + +# Step 9: Situation dependent check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *all_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 3.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *all_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *all_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *all_channels + options: + channels: *all_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *all_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *all_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *all_channels + actions: + - name: set + flag: GrossCheckReject + ignore: rejected observations + - name: reject + +# Step 11: Inter-channel check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *all_channels + options: + channels: *all_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelCheckReject + ignore: rejected observations + - name: reject +passedBenchmark: 169997 # GSI: 169998 +# Notes: one point difference in HydrometeorCheck for channel 7 + diff --git a/parm/atm/obs/testing/avhrr3_metop-a.yaml b/parm/atm/obs/testing/avhrr3_metop-a.yaml new file mode 100644 index 000000000..da5b5689c --- /dev/null +++ b/parm/atm/obs/testing/avhrr3_metop-a.yaml @@ -0,0 +1,191 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID avhrr3_metop-a + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: avhrr3_metop-a + obsdatain: + engine: + type: H5File + obsfile: !ENV avhrr3_metop-a_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV avhrr3_metop-a_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 3-5 +geovals: + filename: !ENV avhrr3_metop-a_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV avhrr3_metop-a_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &avhrr3_metop-a_tlap !ENV avhrr3_metop-a_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *avhrr3_metop-a_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 1: Assign initial observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter vector: [0.600, 0.680, 0.720] + +# Step 2: Wavenumber Check +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 89.0 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1.0e-12 + +# Step 3: Observation error inflation based on wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: *all_channels + options: + channels: *all_channels + +# Step 4: Observation error inflation based on topography check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + minvalue: 0.00001 + maxvalue: 1000.0 + action: + name: reject + +# Step 6: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +# Step 7: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + use_flag_clddet: [ 1, 1, 1 ] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + error parameter vector: [0.600, 0.680, 0.720] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 8: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + error parameter vector: [0.600, 0.680, 0.720] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + +# Step 10: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + obserr_bound_max: [ 6.0, 6.0, 6.0 ] + error parameter vector: [0.600, 0.680, 0.720] + action: + name: reject +passedBenchmark: 8774 + + + diff --git a/parm/atm/obs/testing/avhrr3_metop-b.yaml b/parm/atm/obs/testing/avhrr3_metop-b.yaml new file mode 100644 index 000000000..dae03740e --- /dev/null +++ b/parm/atm/obs/testing/avhrr3_metop-b.yaml @@ -0,0 +1,191 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID avhrr3_metop-b + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: avhrr3_metop-b + obsdatain: + engine: + type: H5File + obsfile: !ENV avhrr3_metop-b_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV avhrr3_metop-b_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 3-5 +geovals: + filename: !ENV avhrr3_metop-b_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV avhrr3_metop-b_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &avhrr3_metop-b_tlap !ENV avhrr3_metop-b_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *avhrr3_metop-b_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 1: Assign initial observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter vector: [0.450, 0.550, 0.600] + +# Step 2: Wavenumber Check +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 89.0 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1.0e-12 + +# Step 3: Observation error inflation based on wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: *all_channels + options: + channels: *all_channels + +# Step 4: Observation error inflation based on topography check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + minvalue: 0.00001 + maxvalue: 1000.0 + action: + name: reject + +# Step 6: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +# Step 7: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + use_flag_clddet: [ 1, 1, 1 ] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + error parameter vector: [0.450, 0.550, 0.600] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 8: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + error parameter vector: [0.450, 0.550, 0.600] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + +# Step 10: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + obserr_bound_max: [ 6.0, 6.0, 6.0 ] + error parameter vector: [0.450, 0.550, 0.600] + action: + name: reject +passedBenchmark: 9150 + + + diff --git a/parm/atm/obs/testing/avhrr3_n18.yaml b/parm/atm/obs/testing/avhrr3_n18.yaml new file mode 100644 index 000000000..f07e2aba4 --- /dev/null +++ b/parm/atm/obs/testing/avhrr3_n18.yaml @@ -0,0 +1,191 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID avhrr3_n18 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: avhrr3_n18 + obsdatain: + engine: + type: H5File + obsfile: !ENV avhrr3_n18_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV avhrr3_n18_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 3-5 +geovals: + filename: !ENV avhrr3_n18_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV avhrr3_n18_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &avhrr3_n18_tlap !ENV avhrr3_n18_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *avhrr3_n18_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 1: Assign initial observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter vector: [0.600, 0.680, 0.720] + +# Step 2: Wavenumber Check +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 89.0 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1.0e-12 + +# Step 3: Observation error inflation based on wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: *all_channels + options: + channels: *all_channels + +# Step 4: Observation error inflation based on topography check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + minvalue: 0.00001 + maxvalue: 1000.0 + action: + name: reject + +# Step 6: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +# Step 7: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + use_flag_clddet: [ 1, 1, 1 ] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + error parameter vector: [0.600, 0.680, 0.720] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 8: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + error parameter vector: [0.600, 0.680, 0.720] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + +# Step 10: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + obserr_bound_max: [ 6.0, 6.0, 6.0 ] + error parameter vector: [0.600, 0.680, 0.720] + action: + name: reject +passedBenchmark: 10523 + + + diff --git a/parm/atm/obs/testing/avhrr3_n19.yaml b/parm/atm/obs/testing/avhrr3_n19.yaml new file mode 100644 index 000000000..6a4c2eee5 --- /dev/null +++ b/parm/atm/obs/testing/avhrr3_n19.yaml @@ -0,0 +1,191 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID avhrr3_n19 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: avhrr3_n19 + obsdatain: + engine: + type: H5File + obsfile: !ENV avhrr3_n19_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV avhrr3_n19_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 3-5 +geovals: + filename: !ENV avhrr3_n19_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV avhrr3_n19_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &avhrr3_n19_tlap !ENV avhrr3_n19_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *avhrr3_n19_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 1: Assign initial observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter vector: [0.600, 0.680, 0.720] + +# Step 2: Wavenumber Check +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 89.0 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1.0e-12 + +# Step 3: Observation error inflation based on wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: *all_channels + options: + channels: *all_channels + +# Step 4: Observation error inflation based on topography check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + minvalue: 0.00001 + maxvalue: 1000.0 + action: + name: reject + +# Step 6: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + +# Step 7: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + use_flag_clddet: [ 1, 1, 1 ] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + error parameter vector: [0.600, 0.680, 0.720] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 8: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ 1, 1, 1 ] + error parameter vector: [0.600, 0.680, 0.720] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + channels: *all_channels + sensor: *Sensor_ID + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + +# Step 10: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *all_channels + options: + channels: *all_channels + obserr_bound_max: [ 6.0, 6.0, 6.0 ] + error parameter vector: [0.600, 0.680, 0.720] + action: + name: reject +passedBenchmark: 9566 # GSI: 9567 + + + diff --git a/parm/atm/obs/testing/seviri_m08.yaml b/parm/atm/obs/testing/seviri_m08.yaml new file mode 100644 index 000000000..f09fccabc --- /dev/null +++ b/parm/atm/obs/testing/seviri_m08.yaml @@ -0,0 +1,329 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID seviri_m08 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: seviri_m08 + obsdatain: + engine: + type: H5File + obsfile: !ENV seviri_m08_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV seviri_m08_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &seviri_m08_channels 4-11 +geovals: + filename: !ENV seviri_m08_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV seviri_m08_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: !ENV &seviri_m08_tlap seviri_m08_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *seviri_m08_tlap + - name: emissivity + - name: scan_angle + var_name: sensorScanPosition + order: 4 + - name: scan_angle + var_name: sensorScanPosition + order: 3 + - name: scan_angle + var_name: sensorScanPosition + order: 2 + - name: scan_angle + var_name: sensorScanPosition + +obs prior filters: +# Step 1: Assign obs error for each channel +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + action: + name: assign error + error parameter vector: [1.80, 2.50, 2.25, 1.25, 1.25, 1.25, 1.45, 1.25] + +obs post filters: +# Step 2: Surface type check +# Reject channels 5-6 over land-dominant area +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 4, 7-11 + where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + +# Reject all channels over snow-dominant area +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + +# Reject all channels over ice-dominant area +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + +# Reject all channelsover mixed surface +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 3: Terrain Check: Do not use when height > 1km +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + where: + - variable: + name: GeoVaLs/surface_geopotential_height + maxvalue: 1000.0 + +# Step 4: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + minvalue: 0.0000 + maxvalue: 1000.0 + action: + name: reject + +# Step 5: Error Inflation based on topography +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *seviri_m08_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *seviri_m08_channels + options: + sensor: *Sensor_ID + channels: *seviri_m08_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *seviri_m08_channels + +# Step 6: Error Inflation based on TOA transmittance +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *seviri_m08_channels + options: + channels: *seviri_m08_channels + +# Step 7: Cloud detection check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *seviri_m08_channels + options: + channels: *seviri_m08_channels + use_flag: [ -1, 1, 1, -1, -1, -1, -1, -1 ] + use_flag_clddet: [ -2, -2, -2, -2, -2, -2, -2, -2 ] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + error parameter vector: [1.80, 2.50, 2.25, 1.25, 1.25, 1.25, 1.45, 1.25] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 8: Scene consistency check using channel 9 +# Reject channels 4, 6-11 if channel 9 if scene consistency is greated than 0.5 +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 4, 6-11 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature + channels: 9 + maxvalue: 0.5 + max_exclusive: true + +# Step 9: Gross check +# Reject channels 4, 6-11 if omf > 2 +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: 4, 6-11 + absolute threshold: 2.0 + action: + name: reject + +# Step 10: Error inflation for channels 3-4 based on scene consistency from channel 5 +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + maxvalue: 0.5 + minvalue: 0.4 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.14891 + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + maxvalue: 0.6 + max_exclusive: true + minvalue: 0.5 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.29228 + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + maxvalue: 0.7 + max_exclusive: true + minvalue: 0.6 + action: + name: inflate error + inflation factor: 1.49666 + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + minvalue: 0.7 + action: + name: inflate error + inflation factor: 1.51987 + +# Step 11: +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *seviri_m08_channels + options: + channels: *seviri_m08_channels + use_flag: [ -1, 1, 1, -1, -1, -1, -1, -1 ] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 12: Error inflation based on surface jacobian check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *seviri_m08_channels + options: + channels: *seviri_m08_channels + sensor: seviri_m08 + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + +# Step 13: Cloud fraction cehck +# Reject channels 4, 6-11 if Cloud fraction (percent) > 2 +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 4, 6-11 + where: + - variable: + name: MetaData/cloudAmount + maxvalue: 2 + +# Step14: Final gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m08_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *seviri_m08_channels + options: + sensor: *Sensor_ID + channels: *seviri_m08_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [0.0, 0.0, 0.0, 0.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *seviri_m08_channels + options: + channels: *seviri_m08_channels + obserr_bound_max: [ 2.0, 4.0, 3.5, 2.0, 2.0, 2.0, 2.0, 3.0 ] + error parameter vector: [1.80, 2.50, 2.25, 1.25, 1.25, 1.25, 1.45, 1.25] + action: + name: reject +passedBenchmark: 8792 diff --git a/parm/atm/obs/testing/seviri_m11.yaml b/parm/atm/obs/testing/seviri_m11.yaml new file mode 100644 index 000000000..51527b72c --- /dev/null +++ b/parm/atm/obs/testing/seviri_m11.yaml @@ -0,0 +1,329 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID seviri_m11 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: seviri_m11 + obsdatain: + engine: + type: H5File + obsfile: !ENV seviri_m11_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV seviri_m11_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &seviri_m11_channels 4-11 +geovals: + filename: !ENV seviri_m11_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV seviri_m11_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: !ENV &seviri_m11_tlap seviri_m11_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *seviri_m11_tlap + - name: emissivity + - name: scan_angle + var_name: sensorScanPosition + order: 4 + - name: scan_angle + var_name: sensorScanPosition + order: 3 + - name: scan_angle + var_name: sensorScanPosition + order: 2 + - name: scan_angle + var_name: sensorScanPosition + +obs prior filters: +# Step 1: Assign obs error for each channel +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + action: + name: assign error + error parameter vector: [0.75, 2.50, 2.25, 1.25, 1.25, 0.75, 0.80, 1.25] + +obs post filters: +# Step 2: Surface type check +# Reject channels 5-6 over land-dominant area +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 4, 7-11 + where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + +# Reject all channels over snow-dominant area +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + +# Reject all channels over ice-dominant area +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + +# Reject all channelsover mixed surface +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 3: Terrain Check: Do not use when height > 1km +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + where: + - variable: + name: GeoVaLs/surface_geopotential_height + maxvalue: 1000.0 + +# Step 4: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + minvalue: 0.0000 + maxvalue: 1000.0 + action: + name: reject + +# Step 5: Error Inflation based on topography +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *seviri_m11_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *seviri_m11_channels + options: + sensor: *Sensor_ID + channels: *seviri_m11_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *seviri_m11_channels + +# Step 6: Error Inflation based on TOA transmittance +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *seviri_m11_channels + options: + channels: *seviri_m11_channels + +# Step 7: Cloud detection check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *seviri_m11_channels + options: + channels: *seviri_m11_channels + use_flag: [ -1, 1, 1, -1, -1, -1, -1, -1 ] + use_flag_clddet: [ -1, -2, -2, -2, -2, -2, -2, -2 ] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + error parameter vector: [0.75, 2.50, 2.25, 1.25, 1.25, 0.75, 0.80, 1.25] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 8: Scene consistency check using channel 9 +# Reject channels 4, 6-11 if channel 9 if scene consistency is greated than 0.5 +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 4, 6-11 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature + channels: 9 + maxvalue: 0.5 + max_exclusive: true + +# Step 9: Gross check +# Reject channels 4, 6-11 if omf > 2 +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: 4, 6-11 + absolute threshold: 2.0 + action: + name: reject + +# Step 10: Error inflation for channels 3-4 based on scene consistency from channel 5 +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + maxvalue: 0.5 + minvalue: 0.4 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.14891 + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + maxvalue: 0.6 + max_exclusive: true + minvalue: 0.5 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.29228 + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + maxvalue: 0.7 + max_exclusive: true + minvalue: 0.6 + action: + name: inflate error + inflation factor: 1.49666 + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 6-7 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_5 + minvalue: 0.7 + action: + name: inflate error + inflation factor: 1.51987 + +# Step 11: +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *seviri_m11_channels + options: + channels: *seviri_m11_channels + use_flag: [ -1, 1, 1, -1, -1, -1, -1, -1 ] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 12: Error inflation based on surface jacobian check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *seviri_m11_channels + options: + channels: *seviri_m11_channels + sensor: seviri_m11 + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + +# Step 13: Cloud fraction cehck +# Reject channels 4, 6-11 if Cloud fraction (percent) > 2 +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 4, 6-11 + where: + - variable: + name: MetaData/cloudAmount + maxvalue: 2 + +# Step14: Final gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *seviri_m11_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *seviri_m11_channels + options: + sensor: *Sensor_ID + channels: *seviri_m11_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [0.0, 0.0, 0.0, 0.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *seviri_m11_channels + options: + channels: *seviri_m11_channels + obserr_bound_max: [ 2.0, 4.0, 3.5, 2.0, 2.0, 2.0, 2.0, 3.0 ] + error parameter vector: [0.75, 2.50, 2.25, 1.25, 1.25, 0.75, 0.80, 1.25] + action: + name: reject +passedBenchmark: 10714 diff --git a/parm/atm/obs/testing/ssmis_f17.yaml b/parm/atm/obs/testing/ssmis_f17.yaml index d3435a2f5..bfab30c20 100644 --- a/parm/atm/obs/testing/ssmis_f17.yaml +++ b/parm/atm/obs/testing/ssmis_f17.yaml @@ -2,7 +2,7 @@ obs operator: name: CRTM Absorbers: [H2O,O3,CO2] obs options: - Sensor_ID: ssmis_f17 + Sensor_ID: &Sensor_ID ssmis_f17 EndianType: little_endian CoefficientPath: crtm/ obs space: @@ -16,7 +16,7 @@ obs space: type: H5File obsfile: !ENV ssmis_f17_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: 1-24 + channels: &all_channels 1-24 geovals: filename: !ENV ssmis_f17_geoval_${CDATE}.nc4 obs bias: @@ -54,57 +54,48 @@ obs bias: var_name: sensorScanPosition obs prior filters: +# Step 1: Initial Observation Error Assignment - filter: Perform Action filter variables: - name: brightnessTemperature - channels: 1-24 + channels: *all_channels action: name: assign error - error parameter vector: [ 1.5, 0.5, 0.5, 0.5, 0.5, 1, 1, 3, 3, 3, 3, 2.4, 1.27, 1.44, 3, - 1.34, 1.74, 3.75, 3, 3, 2, 6.4, 1, 1] - + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] obs post filters: -#step1: Gross check (setuprad) +# Step 2: Gross check over water-dominant area - filter: Background Check filter variables: - name: brightnessTemperature - channels: 1-24 - threshold: 1.5 - action: - name: reject -#step1: Gross check(qcmod) -- filter: Background Check - filter variables: - - name: brightnessTemperature - channels: 1-24 + channels: *all_channels absolute threshold: 3.5 remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 action: name: reject -- filter: Difference Check +# Step 3: Reject all channels if surface height is greater than 2km +- filter: RejectList filter variables: - name: brightnessTemperature - channels: 1-2,12-16 - reference: ObsValue/brightnessTemperature_2 - value: HofX/brightnessTemperature_2 - minvalue: -1.5 - maxvalue: 1.5 + channels: *all_channels where: + - variable: + name: GeoVaLs/surface_geopotential_height + minvalue: 2000.0 + min_exclusive: true - variable: name: GeoVaLs/water_area_fraction - maxvalue: 0.99 -#QC_terrain: If seviri and terrain height > 2km. do not use -- filter: Domain Check - filter variables: - - name: brightnessTemperature - channels: 1-24 - where: - - variable: - name: MetaData/heightOfSurface - maxvalue: 2000.0 -#do not use over mixed surface -- filter: BlackList + maxvalue: 0.99 + max_exclusive: true + +# Step 4: Reject data over mixed surface type +- filter: RejectList filter variables: - name: brightnessTemperature channels: 1-3,8-18 @@ -112,42 +103,668 @@ obs post filters: - variable: name: GeoVaLs/land_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/water_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/ice_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/surface_snow_area_fraction maxvalue: 0.99 -#step4: Generate q.c. bounds and modified variances -- filter: BlackList + max_exclusive: true + +# Step 5: Channel 2 O-F check over non-water dominant area +- filter: Difference Check filter variables: - name: brightnessTemperature - channels: 1-24 + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 6: Gross check over non-water dominant area +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true action: - name: inflate error - inflation variable: -# Surface Jacobian check - name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: 1-24 + name: reject + +# Step 7: Scattering check for channels 9-11 using channels 8 and 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex9 + type: float + function: + name: ObsFunction/Arithmetic options: - channels: 1-24 - sensor: ssmis_f17 - obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] - obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] -# Useflag Check + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.485934, 0.485934, 0.473806, -0.473806] + intercept: 271.252327 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.413688, 0.413688, 0.361549, -0.361549] + intercept: 272.280341 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.400882, 0.400882, 0.270510, -0.270510] + intercept: 278.824902 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: DerivedMetaData/SSMISScatteringIndex9 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: DerivedMetaData/SSMISScatteringIndex10 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: DerivedMetaData/SSMISScatteringIndex11 + maxvalue: 2 + +# Step 8: NSST retrieval check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: 1-24 + channels: *all_channels test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: 1-24 + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels options: - channels: 1-24 - use_flag: [ 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1] - minvalue: 1.0e-12 + channels: *all_channels + use_flag: [ -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1] + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Error inflation based on surface jacobian +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] + obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 10: Final gross check +# Channel 1 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: [ 2.25 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + +# Channel 2 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + +# Channel 3 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + +# Channel 4 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 action: name: reject + +# Channel 5 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + +# Channel 6 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + +# Channel 7 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + +# Channel 8 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + +# Channel 9 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + +# Channel 10 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + +# Channel 11 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + +# Channel 12 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: [ 3.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + +# Channel 13 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: [ 1.905 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + +# Channel 14 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: [ 2.16 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + +# Channel 15 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + +# Channel 16 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: [ 2.01 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + +# Channel 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: [ 2.61 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + +# Channel 18 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: [ 5.625 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + +# Channel 19 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + +# Channel 20 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + +# Channel 21 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: [ 3.00 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + +# Channel 22 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: [ 9.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + +# Channel 23 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + +# Channel 24 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + +passedBenchmark: 126336 diff --git a/parm/atm/obs/testing/ssmis_f18.yaml b/parm/atm/obs/testing/ssmis_f18.yaml index bb310b2eb..87859a26c 100644 --- a/parm/atm/obs/testing/ssmis_f18.yaml +++ b/parm/atm/obs/testing/ssmis_f18.yaml @@ -2,7 +2,7 @@ obs operator: name: CRTM Absorbers: [H2O,O3,CO2] obs options: - Sensor_ID: ssmis_f18 + Sensor_ID: &Sensor_ID ssmis_f18 EndianType: little_endian CoefficientPath: crtm/ obs space: @@ -16,7 +16,7 @@ obs space: type: H5File obsfile: !ENV ssmis_f18_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: 1-24 + channels: &all_channels 1-24 geovals: filename: !ENV ssmis_f18_geoval_${CDATE}.nc4 obs bias: @@ -54,57 +54,48 @@ obs bias: var_name: sensorScanPosition obs prior filters: +# Step 1: Initial Observation Error Assignment - filter: Perform Action filter variables: - name: brightnessTemperature - channels: 1-24 + channels: *all_channels action: name: assign error - error parameter vector: [ 1.5, 0.5, 0.5, 0.5, 0.5, 1, 1, 3, 3, 3, 3, 2.4, 1.27, 1.44, 3, - 1.34, 1.74, 3.75, 3, 3, 2, 6.4, 1, 1] - + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] obs post filters: -#step1: Gross check (setuprad) +# Step 2: Gross check over water-dominant area - filter: Background Check filter variables: - name: brightnessTemperature - channels: 1-24 - threshold: 1.5 - action: - name: reject -#step1: Gross check(qcmod) -- filter: Background Check - filter variables: - - name: brightnessTemperature - channels: 1-24 + channels: *all_channels absolute threshold: 3.5 remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 action: name: reject -- filter: Difference Check +# Step 3: Reject all channels if surface height is greater than 2km +- filter: RejectList filter variables: - name: brightnessTemperature - channels: 1-2,12-16 - reference: ObsValue/brightnessTemperature_2 - value: HofX/brightnessTemperature_2 - minvalue: -1.5 - maxvalue: 1.5 + channels: *all_channels where: + - variable: + name: GeoVaLs/surface_geopotential_height + minvalue: 2000.0 + min_exclusive: true - variable: name: GeoVaLs/water_area_fraction - maxvalue: 0.99 -#QC_terrain: If seviri and terrain height > 2km. do not use -- filter: Domain Check - filter variables: - - name: brightnessTemperature - channels: 1-24 - where: - - variable: - name: MetaData/heightOfSurface - maxvalue: 2000.0 -#do not use over mixed surface -- filter: BlackList + maxvalue: 0.99 + max_exclusive: true + +# Step 4: Reject data over mixed surface type +- filter: RejectList filter variables: - name: brightnessTemperature channels: 1-3,8-18 @@ -112,42 +103,668 @@ obs post filters: - variable: name: GeoVaLs/land_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/water_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/ice_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/surface_snow_area_fraction maxvalue: 0.99 -#step4: Generate q.c. bounds and modified variances -- filter: BlackList + max_exclusive: true + +# Step 5: Channel 2 O-F check over non-water dominant area +- filter: Difference Check filter variables: - name: brightnessTemperature - channels: 1-24 + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 6: Gross check over non-water dominant area +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true action: - name: inflate error - inflation variable: -# Surface Jacobian check - name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: 1-24 + name: reject + +# Step 7: Scattering check for channels 9-11 using channels 8 and 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex9 + type: float + function: + name: ObsFunction/Arithmetic options: - channels: 1-24 - sensor: ssmis_f18 - obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] - obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] -# Useflag Check + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.485934, 0.485934, 0.473806, -0.473806] + intercept: 271.252327 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.413688, 0.413688, 0.361549, -0.361549] + intercept: 272.280341 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.400882, 0.400882, 0.270510, -0.270510] + intercept: 278.824902 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: DerivedMetaData/SSMISScatteringIndex9 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: DerivedMetaData/SSMISScatteringIndex10 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: DerivedMetaData/SSMISScatteringIndex11 + maxvalue: 2 + +# Step 8: NSST retrieval check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: 1-24 + channels: *all_channels test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: 1-24 + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels options: - channels: 1-24 - use_flag: [ 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1] - minvalue: 1.0e-12 + channels: *all_channels + use_flag: [ -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1] + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Error inflation based on surface jacobian +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] + obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 10: Final gross check +# Channel 1 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: [ 2.25 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + +# Channel 2 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + +# Channel 3 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + +# Channel 4 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 action: name: reject + +# Channel 5 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + +# Channel 6 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + +# Channel 7 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + +# Channel 8 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + +# Channel 9 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + +# Channel 10 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + +# Channel 11 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + +# Channel 12 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: [ 3.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + +# Channel 13 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: [ 1.905 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + +# Channel 14 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: [ 2.16 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + +# Channel 15 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + +# Channel 16 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: [ 2.01 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + +# Channel 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: [ 2.61 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + +# Channel 18 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: [ 5.625 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + +# Channel 19 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + +# Channel 20 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + +# Channel 21 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: [ 3.00 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + +# Channel 22 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: [ 9.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + +# Channel 23 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + +# Channel 24 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + +passedBenchmark: 72466 diff --git a/ush/eva/gen_eva_obs_yaml.py b/ush/eva/gen_eva_obs_yaml.py index a556349ee..18a7a7818 100755 --- a/ush/eva/gen_eva_obs_yaml.py +++ b/ush/eva/gen_eva_obs_yaml.py @@ -33,7 +33,7 @@ def gen_eva_obs_yaml(inputyaml, templateyaml, outputdir): tmp_os = obsspace['obs space'] tmp_dict = { 'name': tmp_os['name'], - 'diagfile': tmp_os['obsdataout']['engine']['obsfile'].replace('.nc4', '_0000.nc4'), + 'diagfile': tmp_os['obsdataout']['engine']['obsfile'], 'vars': tmp_os['simulated variables'], 'channels': tmp_os.get('channels', None), } diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index 5ce5e4b95..65690f180 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -76,7 +76,7 @@ exename=test_ObsFilters.x #-------------- Do not modify below this line ---------------- # paths that should only be changed by an expert user -dataprocdate=20230202 # Production date of test data +dataprocdate=20230215 # Production date of test data obtype_short=${obtype:0:4} if [ $obtype_short = "cris" ] || [ $obtype_short = "iasi" ] || [ $obtype_short = "hirs" ] || [ $obtype_short = "sevi" ] || \ @@ -88,10 +88,12 @@ else fi if [ $machine = orion ]; then - export Datapath='/work2/noaa/da/cmartin/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate +# export Datapath='/work2/noaa/da/cmartin/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate + export Datapath='/work2/noaa/da/eliu/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate FixDir=/work2/noaa/da/cmartin/GDASApp/fix elif [ $machine = hera ]; then - export Datapath='/scratch1/NCEPDEV/da/Cory.R.Martin/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate +# export Datapath='/scratch1/NCEPDEV/da/Cory.R.Martin/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate + export Datapath='/scratch1/NCEPDEV/da/Emily.Liu/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate FixDir=/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix else echo "Machine " $machine "not found"