From 9a638d619f261aeed3be2a8770150d3d59542b18 Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Fri, 11 Jun 2021 18:48:29 -0600 Subject: [PATCH 01/39] Update Blocking.py Added some comments --- .../s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py index e1ca2e3df6..d1231cf95d 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py @@ -44,12 +44,12 @@ def run_CBL(self,cblinfiles,cblyrs): weightf = np.repeat(way[:,np.newaxis],360,axis=1) ####Find latitude of maximum high-pass STD (CBL) - mstd = np.nanstd(z500_anom_4d,axis=1) - mhweight = mstd * weightf - cbli = np.argmax(mhweight,axis=1) + mstd = np.nanstd(z500_anom_4d,axis=1) # Standard deviation over latitudes + mhweight = mstd * weightf # Multiply standard deviation by weights + cbli = np.argmax(mhweight,axis=1) # Find the index of the max for each year (result is year, lon) CBL = np.zeros((len(z500_anom_4d[:,0,0,0]),len(lons))) for j in np.arange(0,len(yr),1): - CBL[j,:] = lats[cbli[j,:]] + CBL[j,:] = lats[cbli[j,:]] # Return the latitude that was the max (result is year, lon) ###Apply Moving Average to Smooth CBL Profiles lt = len(lons) From 231476a05ba0e420e837cda2b4dc08c0e05fca8e Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Fri, 11 Jun 2021 18:49:15 -0600 Subject: [PATCH 02/39] Update Blocking.py --- .../s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py index d1231cf95d..723ea6140d 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking.py @@ -46,10 +46,10 @@ def run_CBL(self,cblinfiles,cblyrs): ####Find latitude of maximum high-pass STD (CBL) mstd = np.nanstd(z500_anom_4d,axis=1) # Standard deviation over latitudes mhweight = mstd * weightf # Multiply standard deviation by weights - cbli = np.argmax(mhweight,axis=1) # Find the index of the max for each year (result is year, lon) + cbli = np.argmax(mhweight,axis=1) # Find the index of the max for each year (size is year, lon) CBL = np.zeros((len(z500_anom_4d[:,0,0,0]),len(lons))) for j in np.arange(0,len(yr),1): - CBL[j,:] = lats[cbli[j,:]] # Return the latitude that was the max (result is year, lon) + CBL[j,:] = lats[cbli[j,:]] # Return the latitude that was the max (size is year, lon) ###Apply Moving Average to Smooth CBL Profiles lt = len(lons) From 016c1fc8d5a764a12f54f37244fdefec8f5b52e2 Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Mon, 12 Jul 2021 09:48:28 -0600 Subject: [PATCH 03/39] Added mpr output --- .../save_input_files_txt_missing.py | 1 + .../Blocking_WeatherRegime_util.py | 150 ++++++++---------- .../save_input_files_txt_missing.py | 14 ++ ...erScript_obsERA_obsOnly_WeatherRegime.conf | 15 +- .../Blocking_WeatherRegime_util.py | 142 +---------------- .../WeatherRegime.py | 49 ++++-- .../WeatherRegime_driver.py | 102 ++++++++++-- .../WeatherRegime_obsERA_obsOnly.conf | 67 ++++---- .../save_input_files_txt_missing.py | 1 + 9 files changed, 262 insertions(+), 279 deletions(-) create mode 120000 parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py create mode 100755 parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/save_input_files_txt_missing.py mode change 100644 => 120000 parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/Blocking_WeatherRegime_util.py create mode 120000 parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/save_input_files_txt_missing.py diff --git a/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py b/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py new file mode 120000 index 0000000000..57c9f9549b --- /dev/null +++ b/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py @@ -0,0 +1 @@ +../UserScript_obsERA_obsOnly_Blocking/save_input_files_txt_missing.py \ No newline at end of file diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py index afaa39ce53..4197c55b12 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py @@ -1,8 +1,9 @@ import os import netCDF4 import numpy as np -from metplus.util import pre_run_setup, config_metplus, get_start_end_interval_times, get_lead_sequence -from metplus.util import get_skip_times, skip_time, is_loop_by_init, ti_calculate, do_string_sub +import datetime +from metplus.util import pre_run_setup, config_metplus + def parse_steps(config_list): @@ -33,73 +34,49 @@ def parse_steps(config_list): return steps_list_fcst, steps_list_obs, config_list -def find_input_files(inconfig, use_init, intemplate, secondtemplate=''): - loop_time, end_time, time_interval = get_start_end_interval_times(inconfig) - skip_times = get_skip_times(inconfig) - - start_mth = loop_time.strftime('%m') - template = inconfig.getraw('config',intemplate) - if secondtemplate: - template2 = inconfig.getraw('config',secondtemplate) - file_list2 = [] - - file_list = [] - yr_list = [] - mth_list = [] - day_list = [] - yr_full_list = [] - if use_init: - timname = 'init' - else: - timname = 'valid' - input_dict = {} - input_dict['loop_by'] = timname - pmth = start_mth - while loop_time <= end_time: - lead_seq = get_lead_sequence(inconfig) - for ls in lead_seq: - new_time = loop_time + ls - input_dict[timname] = loop_time - input_dict['lead'] = ls - - outtimestuff = ti_calculate(input_dict) - if skip_time(outtimestuff, skip_times): - continue - cmth = outtimestuff['valid'].strftime('%m') - filepath = do_string_sub(template, **outtimestuff) - mth_list.append(cmth) - day_list.append(outtimestuff['valid'].strftime('%d')) - yr_full_list.append(outtimestuff['valid'].strftime('%Y')) - if secondtemplate: - filepath2 = do_string_sub(template2, **outtimestuff) - if os.path.exists(filepath) and os.path.exists(filepath2): - file_list.append(filepath) - file_list2.append(filepath2) - else: - file_list.append('') - file_list2.append('') - else: - if os.path.exists(filepath): - file_list.append(filepath) - else: - file_list.append('') - - if (int(cmth) == int(start_mth)) and (int(pmth) != int(start_mth)): - yr_list.append(int(outtimestuff['valid'].strftime('%Y'))) - pmth = cmth - - loop_time += time_interval - - if secondtemplate: - file_list = [file_list,file_list2] - yr_list.append(int(outtimestuff['valid'].strftime('%Y'))) - - if all('' == fn for fn in file_list): - raise Exception('No input files found as given: '+template) - - return file_list, yr_list, mth_list, day_list, yr_full_list - -def read_nc_met(infiles,yrlist,invar): +def write_mpr_file(data_obs,data_fcst,lats_in,lons_in,time_obs,time_fcst,mname,fvar,flev,ovar,olev,maskname,obslev,outfile): + + dlength = len(lons_in) + bdims = data_obs.shape + + index_num = np.arange(0,dlength,1)+1 + + # Get the length of the model, FCST_VAR, FCST_LEV, OBS_VAR, OBS_LEV, VX_MASK + mname_len = str(max([5,len(mname)])+3) + mask_len = str(max([7,len(maskname)])+3) + fvar_len = str(max([8,len(fvar)])+3) + flev_len = str(max([8,len(flev)])+3) + ovar_len = str(max([7,len(ovar)])+3) + olev_len = str(max([7,len(olev)])+3) + + format_string = '%-10s %-'+mname_len+'s %-7s %-12s %-18s %-18s %-12s %-17s %-17s %-'+fvar_len+'s %-' \ + +flev_len+'s %-'+ovar_len+'s %-'+olev_len+'s %-10s %-'+mask_len+'s %-13s %-13s %-13s %-13s %-13s' \ + ' %-13s %-13s\n' + format_string2 = '%-10s %-'+mname_len+'s %-7s %-12s %-18s %-18s %-12s %-17s %-17s %-'+fvar_len+'s %-' \ + +flev_len+'s %-'+ovar_len+'s %-'+olev_len+'s %-10s %-'+mask_len+'s %-13s %-13s %-13s %-13s %-13s' \ + ' %-13s %-13s %-10s %-10s %-10s %-12.4f %-12.4f %-10s %-10s %-12.4f %-12.4f %-10s %-10s %-10s %-10s' + + # Write the file + for y in range(bdims[0]): + for dd in range(bdims[1]): + if time_fcst['valid'][y][dd]: + ft_stamp = time_fcst['lead'][y][dd]+'L_'+time_fcst['valid'][y][dd][0:8]+'_' \ + +time_fcst['valid'][y][dd][9:15]+'V' + mpr_outfile_name = outfile+'_'+ft_stamp+'.stat' + with open(mpr_outfile_name, 'w') as mf: + mf.write(format_string % ('VERSION', 'MODEL', 'DESC', 'FCST_LEAD', 'FCST_VALID_BEG', 'FCST_VALID_END', + 'OBS_LEAD', 'OBS_VALID_BEG', 'OBS_VALID_END', 'FCST_VAR', 'FCST_LEV', 'OBS_VAR', 'OBS_LEV', + 'OBTYPE', 'VX_MASK', 'INTERP_MTHD','INTERP_PNTS', 'FCST_THRESH', 'OBS_THRESH', 'COV_THRESH', + 'ALPHA', 'LINE_TYPE')) + for dpt in range(dlength): + mf.write(format_string2 % ('V9.1',mname,'NA',time_fcst['lead'][y][dd],time_fcst['valid'][y][dd], + time_fcst['valid'][y][dd],time_obs['lead'][y][dd],time_obs['valid'][y][dd], + time_obs['valid'][y][dd],fvar,flev,ovar,olev,'ADPUPA',maskname,'NEAREST','1','NA','NA', + 'NA','NA','MPR',str(dlength),str(index_num[dpt]),'NA',lats_in[dpt],lons_in[dpt],obslev, + 'NA',data_fcst[y,dd,dpt],data_obs[y,dd,dpt],'NA','NA','NA','NA')) + + +def read_nc_met(infiles,invar,nseasons,dseasons): print("Reading in Data") @@ -112,6 +89,9 @@ def read_nc_met(infiles,yrlist,invar): indata.close() var_3d = np.empty([len(infiles),len(invar_arr[:,0]),len(invar_arr[0,:])]) + init_list = [] + valid_list = [] + lead_list = [] for i in range(0,len(infiles)): @@ -119,23 +99,31 @@ def read_nc_met(infiles,yrlist,invar): if infiles[i]: indata = netCDF4.Dataset(infiles[i]) new_invar = indata.variables[invar][:] + init_time_str = indata.variables[invar].getncattr('init_time') valid_time_str = indata.variables[invar].getncattr('valid_time') + lead_dt = datetime.datetime.strptime(valid_time_str,'%Y%m%d_%H%M%S') - datetime.datetime.strptime(init_time_str,'%Y%m%d_%H%M%S') + leadmin,leadsec = divmod(lead_dt.total_seconds(), 60) + leadhr,leadmin = divmod(leadmin,60) + lead_str = str(int(leadhr)).zfill(2)+str(int(leadmin)).zfill(2)+str(int(leadsec)).zfill(2) indata.close() else: new_invar = np.empty((1,len(var_3d[0,:,0]),len(var_3d[0,0,:])),dtype=np.float) + init_time_str = '' + valid_time_str = '' + lead_str = '' new_invar[:] = np.nan + init_list.append(init_time_str) + valid_list.append(valid_time_str) + lead_list.append(lead_str) var_3d[i,:,:] = new_invar - yr = np.array(yrlist) - if len(var_3d[:,0,0])%float(len(yrlist)) != 0: - lowval = int(len(var_3d[:,0,0])/float(len(yrlist))) - newarrlen = (lowval+1) * float(len(yrlist)) - arrexp = int(newarrlen - len(var_3d[:,0,0])) - arrfill = np.empty((arrexp,len(var_3d[0,:,0]),len(var_3d[0,0,:])),dtype=np.float) - arrfill[:] = np.nan - var_3d = np.append(var_3d,arrfill,axis=0) - sdim = len(var_3d[:,0,0])/float(len(yrlist)) - var_4d = np.reshape(var_3d,[len(yrlist),int(sdim),len(var_3d[0,:,0]),len(var_3d[0,0,:])]) - - return var_4d,lats,lons,yr + var_4d = np.reshape(var_3d,[nseasons,dseasons,len(var_3d[0,:,0]),len(var_3d[0,0,:])]) + + # Reshape time arrays and store them in a dictionary + init_list_2d = np.reshape(init_list,[nseasons,dseasons]) + valid_list_2d = np.reshape(valid_list,[nseasons,dseasons]) + lead_list_2d = np.reshape(lead_list,[nseasons,dseasons]) + time_dict = {'init':init_list_2d,'valid':valid_list_2d,'lead':lead_list_2d} + + return var_4d,lats,lons,time_dict diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/save_input_files_txt_missing.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/save_input_files_txt_missing.py new file mode 100755 index 0000000000..4a9970c782 --- /dev/null +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/save_input_files_txt_missing.py @@ -0,0 +1,14 @@ +#! /usr/bin/env python + +import os +import sys + +input_file = sys.argv[1] +output_file = sys.argv[2] + +filelist = open(output_file,'a+') +if os.path.isfile(input_file): + filelist.write(input_file + '\n') +else: + filelist.write('\n') +filelist.close() diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf index ba76b11a68..61927fc304 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf @@ -2,7 +2,7 @@ [config] # All steps, including pre-processing: -# PROCESS_LIST = RegridDataPlane(regrid_obs), PcpCombine(daily_mean_obs), UserScript(script_wr) +# PROCESS_LIST = RegridDataPlane(regrid_obs), PcpCombine(daily_mean_obs), UserScript(obs_wr_filelist), UserScript(script_wr) # Weather Regime Analysis only: PROCESS_LIST = UserScript(script_wr) @@ -50,6 +50,11 @@ LOOP_ORDER = processes # location of configuration files used by MET applications CONFIG_DIR={PARM_BASE}/use_cases/model_applications/s2s +OBS_WR_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily +OBS_WR_INPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d}_NH.nc +OBS_WR_OUTPUT_DIR = {OBS_WR_INPUT_DIR} +OBS_WR_OUTPUT_TEMPLATE = ERA_daily_files_lead{lead?fmt=%HHH}.tx + # Regridding Pre-Processing Step [regrid_obs] @@ -138,6 +143,14 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/Z500_6hourly_{valid?fmt=%Y%m OBS_PCP_COMBINE_OUTPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d?shift=-64800}_NH.nc +# Get a File list for the OBS IBL data (daily mean files) +[obs_wr_filelist] +# Find the files for each time +USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_FOR_EACH + +USER_SCRIPT_COMMAND = {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py {OBS_WR_INPUT_DIR}/{OBS_WR_INPUT_TEMPLATE} {OBS_WR_OUTPUT_DIR}/{OBS_WR_OUTPUT_TEMPLATE} + + # Run the Weather Regime Script [script_wr] # User Script Commands diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/Blocking_WeatherRegime_util.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/Blocking_WeatherRegime_util.py deleted file mode 100644 index afaa39ce53..0000000000 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/Blocking_WeatherRegime_util.py +++ /dev/null @@ -1,141 +0,0 @@ -import os -import netCDF4 -import numpy as np -from metplus.util import pre_run_setup, config_metplus, get_start_end_interval_times, get_lead_sequence -from metplus.util import get_skip_times, skip_time, is_loop_by_init, ti_calculate, do_string_sub - -def parse_steps(config_list): - - steps_config_part_fcst = [s for s in config_list if "FCST_STEPS" in s] - steps_list_fcst = [] - - steps_config_part_obs = [s for s in config_list if "OBS_STEPS" in s] - steps_list_obs = [] - - # Setup the Steps - if steps_config_part_fcst: - steps_param_fcst = steps_config_part_fcst[0].split("=")[1] - steps_list_fcst = steps_param_fcst.split("+") - config_list.remove(steps_config_part_fcst[0]) - if steps_config_part_obs: - steps_param_obs = steps_config_part_obs[0].split("=")[1] - steps_list_obs = steps_param_obs.split("+") - config_list.remove(steps_config_part_obs[0]) - - config = pre_run_setup(config_list) - if not steps_config_part_fcst: - steps_param_fcst = config.getstr('config','FCST_STEPS','') - steps_list_fcst = steps_param_fcst.split("+") - if not steps_config_part_obs: - steps_param_obs = config.getstr('config','OBS_STEPS','') - steps_list_obs = steps_param_obs.split("+") - - return steps_list_fcst, steps_list_obs, config_list - - -def find_input_files(inconfig, use_init, intemplate, secondtemplate=''): - loop_time, end_time, time_interval = get_start_end_interval_times(inconfig) - skip_times = get_skip_times(inconfig) - - start_mth = loop_time.strftime('%m') - template = inconfig.getraw('config',intemplate) - if secondtemplate: - template2 = inconfig.getraw('config',secondtemplate) - file_list2 = [] - - file_list = [] - yr_list = [] - mth_list = [] - day_list = [] - yr_full_list = [] - if use_init: - timname = 'init' - else: - timname = 'valid' - input_dict = {} - input_dict['loop_by'] = timname - pmth = start_mth - while loop_time <= end_time: - lead_seq = get_lead_sequence(inconfig) - for ls in lead_seq: - new_time = loop_time + ls - input_dict[timname] = loop_time - input_dict['lead'] = ls - - outtimestuff = ti_calculate(input_dict) - if skip_time(outtimestuff, skip_times): - continue - cmth = outtimestuff['valid'].strftime('%m') - filepath = do_string_sub(template, **outtimestuff) - mth_list.append(cmth) - day_list.append(outtimestuff['valid'].strftime('%d')) - yr_full_list.append(outtimestuff['valid'].strftime('%Y')) - if secondtemplate: - filepath2 = do_string_sub(template2, **outtimestuff) - if os.path.exists(filepath) and os.path.exists(filepath2): - file_list.append(filepath) - file_list2.append(filepath2) - else: - file_list.append('') - file_list2.append('') - else: - if os.path.exists(filepath): - file_list.append(filepath) - else: - file_list.append('') - - if (int(cmth) == int(start_mth)) and (int(pmth) != int(start_mth)): - yr_list.append(int(outtimestuff['valid'].strftime('%Y'))) - pmth = cmth - - loop_time += time_interval - - if secondtemplate: - file_list = [file_list,file_list2] - yr_list.append(int(outtimestuff['valid'].strftime('%Y'))) - - if all('' == fn for fn in file_list): - raise Exception('No input files found as given: '+template) - - return file_list, yr_list, mth_list, day_list, yr_full_list - -def read_nc_met(infiles,yrlist,invar): - - print("Reading in Data") - - #Find the first non empty file name so I can get the variable sizes - locin = next(sub for sub in infiles if sub) - indata = netCDF4.Dataset(locin) - lats = indata.variables['lat'][:] - lons = indata.variables['lon'][:] - invar_arr = indata.variables[invar][:] - indata.close() - - var_3d = np.empty([len(infiles),len(invar_arr[:,0]),len(invar_arr[0,:])]) - - for i in range(0,len(infiles)): - - #Read in the data - if infiles[i]: - indata = netCDF4.Dataset(infiles[i]) - new_invar = indata.variables[invar][:] - init_time_str = indata.variables[invar].getncattr('init_time') - valid_time_str = indata.variables[invar].getncattr('valid_time') - indata.close() - else: - new_invar = np.empty((1,len(var_3d[0,:,0]),len(var_3d[0,0,:])),dtype=np.float) - new_invar[:] = np.nan - var_3d[i,:,:] = new_invar - - yr = np.array(yrlist) - if len(var_3d[:,0,0])%float(len(yrlist)) != 0: - lowval = int(len(var_3d[:,0,0])/float(len(yrlist))) - newarrlen = (lowval+1) * float(len(yrlist)) - arrexp = int(newarrlen - len(var_3d[:,0,0])) - arrfill = np.empty((arrexp,len(var_3d[0,:,0]),len(var_3d[0,0,:])),dtype=np.float) - arrfill[:] = np.nan - var_3d = np.append(var_3d,arrfill,axis=0) - sdim = len(var_3d[:,0,0])/float(len(yrlist)) - var_4d = np.reshape(var_3d,[len(yrlist),int(sdim),len(var_3d[0,:,0]),len(var_3d[0,0,:])]) - - return var_4d,lats,lons,yr diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/Blocking_WeatherRegime_util.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/Blocking_WeatherRegime_util.py new file mode 120000 index 0000000000..7839a88078 --- /dev/null +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/Blocking_WeatherRegime_util.py @@ -0,0 +1 @@ +../UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py \ No newline at end of file diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py index 6b06174732..7eeb0aa1c4 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py @@ -1,5 +1,6 @@ import os import numpy as np +#import pandas as pd from pylab import * from sklearn.cluster import KMeans import scipy @@ -8,8 +9,13 @@ from numpy import ones,vstack from numpy.linalg import lstsq from eofs.standard import Eof -from metplus.util import config_metplus, get_start_end_interval_times, get_lead_sequence -from metplus.util import get_skip_times, skip_time, is_loop_by_init, ti_calculate +from metplus.util import config_metplus +#, get_start_end_interval_times, get_lead_sequence +#from metplus.util import get_skip_times, skip_time, is_loop_by_init, ti_calculate + +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from mpl_toolkits.basemap import Basemap class WeatherRegimeCalculation(): @@ -20,6 +26,7 @@ def __init__(self,config,label): self.wrnum = config.getint('WeatherRegime',label+'_WR_NUMBER',6) self.numi = config.getint('WeatherRegime',label+'_NUM_CLUSTERS',20) self.NUMPCS = config.getint('WeatherRegime',label+'_NUM_PCS',10) + self.wr_tstep = config.getint('WeatherRegime',label+'_WR_FREQ',7) self.wr_outfile_type = config.getstr('WeatherRegime',label+'_WR_OUTPUT_FILE_TYPE','text') self.wr_outfile_dir = config.getstr('WeatherRegime','WR_OUTPUT_FILE_DIR','') self.wr_outfile = config.getstr('WeatherRegime',label+'_WR_OUTPUT_FILE',label+'_WeatherRegime') @@ -130,7 +137,7 @@ def reconstruct_heights(self,eof,pc,reshape_arr): return a1 - def run_K_means(self,a1,yr,mth,day,arr_shape): + def run_K_means(self,a1,timedict,arr_shape): arrdims = len(arr_shape) @@ -173,10 +180,16 @@ def run_K_means(self,a1,yr,mth,day,arr_shape): #Save Label data [YR,DAY] # Make some conversions first wrc_shape = wrc.shape - yr_1d = np.array(yr) - mth_1d = np.array(mth) - day_1d = np.array(day) - wrc_1d = np.reshape(wrc,len(mth)) + len1d = wrc.size + valid_time_1d = np.reshape(timedict['valid'],len1d) + yr_1d = [] + mth_1d = [] + day_1d = [] + for vt1 in valid_time_1d: + yr_1d.append(vt1[0:4]) + mth_1d.append(vt1[4:6]) + day_1d.append(vt1[6:8]) + wrc_1d = np.reshape(wrc,len1d) # netcdf file if self.wr_outfile_type=='netcdf': @@ -195,7 +208,7 @@ def run_K_means(self,a1,yr,mth,day,arr_shape): ymd_arr[dd] = yr_1d[dd]+mth_1d[dd]+day_1d[dd] nc = nc4.Dataset(wr_full_outfile, 'w') - nc.createDimension('time', len(mth)) + nc.createDimension('time', len(mth_1d)) nc.Conventions = "CF-1.7" nc.title = "Weather Regime Classification" nc.institution = "NCAR DTCenter" @@ -231,5 +244,23 @@ def run_K_means(self,a1,yr,mth,day,arr_shape): with open(wr_full_outfile, 'w+') as datafile_id: np.savetxt(datafile_id, otdata, fmt=['%6s','%3s','%4s','%6s'], header='Year Month Day WeatherRegime') + return input, self.wrnum, perc, wrc + + + def compute_wr_freq(self, WR): + + ######## Simple Count ########## + WRfreq = np.zeros((self.wrnum,len(WR[:,0]),len(WR[0,:])-self.wr_tstep)) + + for yy in np.arange(0,len(WRfreq[0,:,0]),1): + d1=0;d2=self.wr_tstep + for dd in np.arange(len(WRfreq[0,0,:])): + temp = WR[yy,d1:d2] + for ww in np.arange(self.wrnum): + WRfreq[ww,yy,dd] = len(np.where(temp==ww)[0]) + d1=d1+1;d2=d2+1 + + WRmean = np.nanmean(WRfreq,axis=1) + dlen_plot = len(WRfreq[0,0]) - return input, self.wrnum, perc + return WRmean, dlen_plot diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py index 28814fadf6..220e53ac98 100755 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py @@ -3,12 +3,28 @@ import os import numpy as np import netCDF4 +import warnings +import atexit from WeatherRegime import WeatherRegimeCalculation from metplus.util import pre_run_setup, config_metplus, get_start_end_interval_times, get_lead_sequence from metplus.util import get_skip_times, skip_time, is_loop_by_init, ti_calculate, do_string_sub, getlist from metplotpy.contributed.weather_regime import plot_weather_regime as pwr -from Blocking_WeatherRegime_util import find_input_files, parse_steps, read_nc_met +from Blocking_WeatherRegime_util import parse_steps, read_nc_met, write_mpr_file + + +def cleanup_daily_files(obs_dailyfile, fcst_dailyfile, keep_daily_files): + if keep_daily_files == 'false': + try: + os.remove(obs_anomfile) + except: + pass + + try: + os.remove(fcst_anomfile) + except: + pass + def main(): @@ -19,8 +35,8 @@ def main(): config = pre_run_setup(config_list) if not steps_list_obs and not steps_list_fcst: - print('No processing steps requested for either the model or observations,') - print('no data will be processed') + warnings.warn('No processing steps requested for either the model or observations,') + warnings.warn('no data will be processed') ###################################################################### @@ -38,6 +54,22 @@ def main(): if not os.path.exists(oplot_dir): os.makedirs(oplot_dir) + # Check to see if there is a mpr output directory + mpr_outdir = config.getstr('WeatherRegime','WR_MPR_OUTPUT_DIR','') + if not mpr_outdir: + obase = config.getstr('config','OUTPUT_BASE') + mpr_outdir = os.path.join(obase,'mpr') + + # Get number of seasons and days per season + nseasons = config.getint('WeatherRegime','NUM_SEASONS') + dseasons = config.getint('WeatherRegime','DAYS_PER_SEASON') + + # Grab the Daily (IBL) text files + obs_wr_filetxt = config.getstr('WeatherRegime','OBS_WR_INPUT_TEXTFILE','') + fcst_wr_filetxt = config.getstr('WeatherRegime','FCST_WR_INPUT_TEXTFILE','') + keep_wr_textfile = config.getstr('WeatherRegime','KEEP_WR_FILE_LISTING', 'False').lower() + atexit.register(cleanup_daily_files, obs_wr_filetxt, fcst_wr_filetxt, keep_wr_textfile) + elbow_config = config_metplus.replace_config_from_section(config,'WeatherRegime') elbow_config_init = config.find_section('WeatherRegime','INIT_BEG') elbow_config_valid = config.find_section('WeatherRegime','VALID_BEG') @@ -45,15 +77,21 @@ def main(): if ("ELBOW" in steps_list_obs) or ("EOF" in steps_list_obs) or ("KMEANS" in steps_list_obs): - obs_infiles,yr_obs,mth_obs,day_obs,yr_full_obs = find_input_files(elbow_config, use_init, 'OBS_WR_TEMPLATE') + with open(obs_wr_filetxt) as owl: + obs_infiles = owl.read().splitlines() + if len(obs_infiles) != (nseasons*dseasons): + raise Exception('Invalid Obs data; each year must contain the same date range to calculate seasonal averages.') obs_invar = config.getstr('WeatherRegime','OBS_WR_VAR','') - z500_obs,lats_obs,lons_obs,year_obs = read_nc_met(obs_infiles,yr_obs,obs_invar) + z500_obs,lats_obs,lons_obs,timedict_obs = read_nc_met(obs_infiles,obs_invar,nseasons,dseasons) z500_detrend_obs,z500_detrend_2d_obs = steps_obs.weights_detrend(lats_obs,lons_obs,z500_obs) if ("ELBOW" in steps_list_fcst) or ("EOF" in steps_list_fcst) or("KMEANS" in steps_list_fcst): - fcst_infiles,yr_fcst, mth_fcst,day_fcst,yr_full_fcst = find_input_files(elbow_config, use_init, 'FCST_WR_TEMPLATE') + with open(fcst_wr_filetxt) as fwl: + fcst_infiles = fwl.read().splitlines() + if len(fcst_infiles) != (nseasons*dseasons): + raise Exception('Invalid Obs data; each year must contain the same date range to calculate seasonal averages.') fcst_invar = config.getstr('WeatherRegime','FCST_WR_VAR','') - z500_fcst,lats_fcst,lons_fcst,year_fcst = read_nc_met(fcst_infiles,yr_fcst,fcst_invar) + z500_fcst,lats_fcst,lons_fcst,timedict_fcst = read_nc_met(fcst_infiles,fcst_invar,nseasons,dseasons) z500_detrend_fcst,z500_detrend_2d_fcst = steps_fcst.weights_detrend(lats_fcst,lons_fcst,z500_fcst) @@ -113,11 +151,24 @@ def main(): if ("KMEANS" in steps_list_obs): print('Running Obs K Means') - kmeans_obs,wrnum_obs,perc_obs = steps_obs.run_K_means(z500_detrend_2d_obs,yr_full_obs,mth_obs,day_obs,z500_obs.shape) + kmeans_obs,wrnum_obs,perc_obs,wrc_obs= steps_obs.run_K_means(z500_detrend_2d_obs,timedict_obs,z500_obs.shape) if ("KMEANS" in steps_list_fcst): print('Running Forecast K Means') - kmeans_fcst,wrnum_fcst,perc_fcst = steps_fcst.run_K_means(z500_detrend_2d_fcst,yr_full_fcst,mth_fcst,day_fcst,z500_fcst.shape) + kmeans_fcst,wrnum_fcst,perc_fcst,wrc_fcst = steps_fcst.run_K_means(z500_detrend_2d_fcst,timedict_fcst,z500_fcst.shape) + + #if ("KMEANS" in steps_list_obs) and ("KMEANS" in steps_list_fcst): + modname = config.getstr('WeatherRegime','MODEL_NAME','GFS') + maskname = config.getstr('WeatherRegime','MASK_NAME','FULL') + wr_outfile_prefix = os.path.join(mpr_outdir,'weather_regime_stat_'+modname) + wrc_fcst = wrc_obs + wrc_obs_mpr = wrc_obs[:,:,np.newaxis] + wrc_fcst_mpr = wrc_fcst[:,:,np.newaxis] + timedict_fcst = timedict_obs + if not os.path.exists(mpr_outdir): + os.makedirs(mpr_outdir) + write_mpr_file(wrc_obs_mpr,wrc_fcst_mpr,[0.0],[0.0],timedict_obs,timedict_fcst,modname, + 'WeatherRegimeClass','Z500','WeatherRegimeClass','Z500',maskname,'500',wr_outfile_prefix) if ("PLOTKMEANS" in steps_list_obs): if not ("KMEANS" in steps_list_obs): @@ -138,5 +189,38 @@ def main(): pwr.plot_K_means(kmeans_fcst,wrnum_fcst,lons_fcst,lats_fcst,perc_fcts,kmeans_plot_outname,pltlvls) + if ("TIMEFREQ" in steps_list_obs): + wrmean_obs,dlen_obs = steps_obs.compute_wr_freq(wrc_obs) + + if ("TIMEFREQ" in steps_list_fcst): + wrmean_fcst,dlen_fcst = steps_fcst.compute_wr_freq(wrc_fcst) + + #if ("TIMEFREQ" in steps_list_obs) and ("TIMEFREQ" in steps_list_fcst): + #modname = config.getstr('WeatherRegime','MODEL_NAME','GFS') + #maskname = config.getstr('WeatherRegime','MASK_NAME','FULL') + #wr_outfile_prefix = os.path.join(mpr_outdir,'weather_regime_freq_stat_'+modname) + #wrmean_fcst = wrmean_obs + #timedict_fcst = timedict_obs + #wrmean_obs_mpr = wrmean_obs[:,:,np.newaxis] + #wrmean_fcst_mpr wrmean_fcst[:,:,np.newaxis] + #if not os.path.exists(mpr_outdir): + # os.makedirs(mpr_outdir) + #write_mpr_file(wrmean_obs_mpr,wrmean_fcst_mpr,[0.0],[0.0],timedict_obs,timedict_fcst,modname, + # 'WeatherRegimeClass','Z500','WeatherRegimeClass','Z500',maskname,'500',wr_outfile_prefix) + + if ("PLOTFREQ" in steps_list_obs): + freq_plot_title = config.getstr('WeatherRegime','OBS_FREQ_PLOT_TITLE','Seasonal Cycle of WR Days/Week') + freq_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','OBS_FREQ_PLOT_OUTPUT_NAME','obs_freq') + pwr.plot_wr_frequency(wrmean_obs,wrnum_obs,dlen_obs,freq_plot_title,freq_plot_outname) + + if ("PLOTFREQ" in steps_list_fcst): + freq_plot_title = config.getstr('WeatherRegime','FCST_FREQ_PLOT_TITLE','Seasonal Cycle of WR Days/Week') + freq_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','FCST_FREQ_PLOT_OUTPUT_NAME','fcst_freq') + pwr.plot_wr_frequency(wrmean_fcst,wrnum_fcst,dlen_fcst,freq_plot_title,freq_plot_outname) + + + #if ("ANOMALY_CORR" in steps_list_obs): + + if __name__ == "__main__": main() diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf index 5ea236ddce..e51da98597 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf @@ -1,46 +1,23 @@ # Blocking METplus Configuration [config] # Steps to Run -OBS_STEPS = ELBOW+PLOTELBOW+EOF+PLOTEOF+KMEANS+PLOTKMEANS -#OBS_STEPS = ELBOW+EOF+KMEANS - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = VALID - -# Format of INIT_BEG and INIT_END -VALID_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -VALID_BEG = 1979120100 - -# End time for METplus run -VALID_END = 2017022800 - -# list of forecast leads to process -LEAD_SEQ = 0 - -# Increment between METplus runs (in seconds if no units are specified) -# Must be >= 60 seconds -VALID_INCREMENT = 86400 - -# Only Process DJF -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:0229" - -# Order of loops to process data - Options are times, processes -# Not relevant if only one item is in the PROCESS_LIST -# times = run all wrappers in the PROCESS_LIST for a single run time, then -# increment the run time and run all wrappers again until all times have -# been evaluated. -# processes = run the first wrapper in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST until all -# wrappers have been run -LOOP_ORDER = processes +OBS_STEPS = ELBOW+PLOTELBOW+EOF+PLOTEOF+KMEANS+PLOTKMEANS+TIMEFREQ+PLOTFREQ # Variables Specific to Weather Regime analysis [WeatherRegime] -# Directory for the Z500 data to read in to the blocking python code -OBS_WR_TEMPLATE = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily/Z500_daily_{valid?fmt=%Y%m%d}_NH.nc +# Number of Seasons and Days per season that should be available +# The code will fill missing data, but requires the same number of days per +# season for each year. You may need to omit leap days if February is part of +# the processing +NUM_SEASONS = 38 +DAYS_PER_SEASON = 90 + +# Text files containing a list of input files for the IBL code +OBS_WR_INPUT_TEXTFILE = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily/ERA_daily_files_lead000.txt + +# Boolean indicating whether to keep the above 2 text files or delete them at the end of the run +KEEP_WR_FILE_LISTING = True # Variable for the Z500 data OBS_WR_VAR = Z500 @@ -54,10 +31,16 @@ OBS_NUM_CLUSTERS = 20 # Number of principal components OBS_NUM_PCS = 10 +# Time (in timesteps) over which to compute weather regime frequencies +# i.e. if your data time step is days and you want to average over 7 +# days, input 7 +# Optional, only needed if you want to compute frequencies +OBS_WR_FREQ = 7 + # Type, name and directory of Output File for weather regime classification # Type options are text or netcdf -#OBS_WR_OUTPUT_FILE_TYPE = text -OBS_WR_OUTPUT_FILE_TYPE = netcdf +OBS_WR_OUTPUT_FILE_TYPE = text +#OBS_WR_OUTPUT_FILE_TYPE = netcdf OBS_WR_OUTPUT_FILE = obs_weather_regime_class WR_OUTPUT_FILE_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime @@ -75,3 +58,11 @@ EOF_PLOT_LEVELS = -50, -45, -40, -35, -30, -25, -20, -15, -10, -5, 0, 5, 10, 1 # K means Plot Output Name and contour levels OBS_KMEANS_PLOT_OUTPUT_NAME = obs_kmeans KMEANS_PLOT_LEVELS = -80, -70, -60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60, 70, 80 + +# Frequency Plot title and output file name +OBS_FREQ_PLOT_TITLE = Seasonal Cycle of WR Days/Week (1979-2017) +OBS_FREQ_PLOT_OUTPUT_NAME = obs_freq + +# MPR file information +MASK_NAME = FULL +WR_MPR_OUTPUT_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime/mpr diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/save_input_files_txt_missing.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/save_input_files_txt_missing.py new file mode 120000 index 0000000000..57c9f9549b --- /dev/null +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/save_input_files_txt_missing.py @@ -0,0 +1 @@ +../UserScript_obsERA_obsOnly_Blocking/save_input_files_txt_missing.py \ No newline at end of file From 4122ebb3e487e02c10b67311365f1c38f17fc6c9 Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Tue, 13 Jul 2021 10:27:51 -0600 Subject: [PATCH 04/39] Added MPR output --- ...UserScript_obsERA_obsOnly_WeatherRegime.py | 7 +- .../Blocking_WeatherRegime_util.py | 48 ++---- ...erScript_obsERA_obsOnly_WeatherRegime.conf | 75 ++++++++- .../WeatherRegime.py | 31 ++-- .../WeatherRegime_driver.py | 156 +++++++++--------- .../WeatherRegime_obsERA_obsOnly.conf | 68 -------- 6 files changed, 180 insertions(+), 205 deletions(-) delete mode 100644 parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf diff --git a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py index 1d22815f99..e8bf103583 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py +++ b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py @@ -59,7 +59,8 @@ # This use case runs the weather regime driver script which runs the steps the user # lists in STEPS_OBS. The possible steps are regridding, time averaging, computing the # elbow (ELBOW), plotting the elbow (PLOTELBOW), computing EOFs (EOF), plotting EOFs -# (PLOTEOF), computing K means (KMEANS), and plotting the K means (PLOTKMEANS). Regridding +# (PLOTEOF), computing K means (KMEANS), plotting the K means (PLOTKMEANS), computing a time +# frequency of weather regimes (TIMEFREQ) and plotting the time frequency (PLOTFREQ). Regridding # and time averaging are set up in the UserScript .conf file and are formatted as follows: # PROCESS_LIST = RegridDataPlane(regrid_obs), PcpCombine(daily_mean_obs), UserScript(script_wr) # @@ -73,8 +74,8 @@ # # The weather regime python code is run for each time for the forecast and observations # data. This example loops by valid time. This version is set to only process the weather -# regime steps (ELBOW, PLOTELBOW, EOF, PLOTEOF, KMEANS, PLOTKMEANS), omitting the REGRID -# and TIMEAVE pre-processing steps. However, the configurations for pre-processing are +# regime steps (ELBOW, PLOTELBOW, EOF, PLOTEOF, KMEANS, PLOTKMEANS, TIMEFREQ, PLOTFREQ), omitting +# the REGRID and TIMEAVE pre-processing steps. However, the configurations for pre-processing are # available for user reference. ############################################################################## diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py index 4197c55b12..f73d77318e 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py @@ -5,33 +5,15 @@ from metplus.util import pre_run_setup, config_metplus -def parse_steps(config_list): +def parse_steps(): - steps_config_part_fcst = [s for s in config_list if "FCST_STEPS" in s] - steps_list_fcst = [] + steps_param_fcst = os.environ.get('FCST_STEPS','') + steps_list_fcst = steps_param_fcst.split("+") - steps_config_part_obs = [s for s in config_list if "OBS_STEPS" in s] - steps_list_obs = [] + steps_param_obs = os.environ.get('OBS_STEPS','') + steps_list_obs = steps_param_obs.split("+") - # Setup the Steps - if steps_config_part_fcst: - steps_param_fcst = steps_config_part_fcst[0].split("=")[1] - steps_list_fcst = steps_param_fcst.split("+") - config_list.remove(steps_config_part_fcst[0]) - if steps_config_part_obs: - steps_param_obs = steps_config_part_obs[0].split("=")[1] - steps_list_obs = steps_param_obs.split("+") - config_list.remove(steps_config_part_obs[0]) - - config = pre_run_setup(config_list) - if not steps_config_part_fcst: - steps_param_fcst = config.getstr('config','FCST_STEPS','') - steps_list_fcst = steps_param_fcst.split("+") - if not steps_config_part_obs: - steps_param_obs = config.getstr('config','OBS_STEPS','') - steps_list_obs = steps_param_obs.split("+") - - return steps_list_fcst, steps_list_obs, config_list + return steps_list_fcst, steps_list_obs def write_mpr_file(data_obs,data_fcst,lats_in,lons_in,time_obs,time_fcst,mname,fvar,flev,ovar,olev,maskname,obslev,outfile): @@ -49,12 +31,12 @@ def write_mpr_file(data_obs,data_fcst,lats_in,lons_in,time_obs,time_fcst,mname,f ovar_len = str(max([7,len(ovar)])+3) olev_len = str(max([7,len(olev)])+3) - format_string = '%-10s %-'+mname_len+'s %-7s %-12s %-18s %-18s %-12s %-17s %-17s %-'+fvar_len+'s %-' \ - +flev_len+'s %-'+ovar_len+'s %-'+olev_len+'s %-10s %-'+mask_len+'s %-13s %-13s %-13s %-13s %-13s' \ - ' %-13s %-13s\n' - format_string2 = '%-10s %-'+mname_len+'s %-7s %-12s %-18s %-18s %-12s %-17s %-17s %-'+fvar_len+'s %-' \ - +flev_len+'s %-'+ovar_len+'s %-'+olev_len+'s %-10s %-'+mask_len+'s %-13s %-13s %-13s %-13s %-13s' \ - ' %-13s %-13s %-10s %-10s %-10s %-12.4f %-12.4f %-10s %-10s %-12.4f %-12.4f %-10s %-10s %-10s %-10s' + format_string = '%-10s %-'+mname_len+'s %-7s %-12s %-18s %-18s %-12s %-17s %-17s %-'+fvar_len+'s ' \ + '%-'+flev_len+'s %-'+ovar_len+'s %-'+olev_len+'s %-10s %-'+mask_len+'s %-13s %-13s %-13s %-13s ' \ + '%-13s %-13s %-13s\n' + format_string2 = '%-10s %-'+mname_len+'s %-7s %-12s %-18s %-18s %-12s %-17s %-17s %-'+fvar_len+'s ' \ + '%-'+flev_len+'s %-'+ovar_len+'s %-'+olev_len+'s %-10s %-'+mask_len+'s %-13s %-13s %-13s %-13s ' \ + '%-13s %-13s %-13s %-10s %-10s %-10s %-12.4f %-12.4f %-10s %-10s %-12.4f %-12.4f %-10s %-10s %-10s %-10s\n' # Write the file for y in range(bdims[0]): @@ -64,9 +46,9 @@ def write_mpr_file(data_obs,data_fcst,lats_in,lons_in,time_obs,time_fcst,mname,f +time_fcst['valid'][y][dd][9:15]+'V' mpr_outfile_name = outfile+'_'+ft_stamp+'.stat' with open(mpr_outfile_name, 'w') as mf: - mf.write(format_string % ('VERSION', 'MODEL', 'DESC', 'FCST_LEAD', 'FCST_VALID_BEG', 'FCST_VALID_END', - 'OBS_LEAD', 'OBS_VALID_BEG', 'OBS_VALID_END', 'FCST_VAR', 'FCST_LEV', 'OBS_VAR', 'OBS_LEV', - 'OBTYPE', 'VX_MASK', 'INTERP_MTHD','INTERP_PNTS', 'FCST_THRESH', 'OBS_THRESH', 'COV_THRESH', + mf.write(format_string % ('VERSION', 'MODEL', 'DESC', 'FCST_LEAD', 'FCST_VALID_BEG', 'FCST_VALID_END', + 'OBS_LEAD', 'OBS_VALID_BEG', 'OBS_VALID_END', 'FCST_VAR', 'FCST_LEV', 'OBS_VAR', 'OBS_LEV', + 'OBTYPE', 'VX_MASK', 'INTERP_MTHD','INTERP_PNTS', 'FCST_THRESH', 'OBS_THRESH', 'COV_THRESH', 'ALPHA', 'LINE_TYPE')) for dpt in range(dlength): mf.write(format_string2 % ('V9.1',mname,'NA',time_fcst['lead'][y][dd],time_fcst['valid'][y][dd], diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf index 61927fc304..7e295697fb 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf @@ -53,7 +53,7 @@ CONFIG_DIR={PARM_BASE}/use_cases/model_applications/s2s OBS_WR_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily OBS_WR_INPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d}_NH.nc OBS_WR_OUTPUT_DIR = {OBS_WR_INPUT_DIR} -OBS_WR_OUTPUT_TEMPLATE = ERA_daily_files_lead{lead?fmt=%HHH}.tx +OBS_WR_OUTPUT_TEMPLATE = ERA_daily_files_lead{lead?fmt=%HHH}.txt # Regridding Pre-Processing Step @@ -151,13 +151,82 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_FOR_EACH USER_SCRIPT_COMMAND = {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py {OBS_WR_INPUT_DIR}/{OBS_WR_INPUT_TEMPLATE} {OBS_WR_OUTPUT_DIR}/{OBS_WR_OUTPUT_TEMPLATE} +# Variables for the Weather Regime code +[user_env_vars] +# Steps to Run +OBS_STEPS = ELBOW+PLOTELBOW+EOF+PLOTEOF+KMEANS+PLOTKMEANS+TIMEFREQ+PLOTFREQ + +# Make OUTPUT_BASE Available to the script +SCRIPT_OUTPUT_BASE = {OUTPUT_BASE} + +# Number of Seasons and Days per season that should be available +# The code will fill missing data, but requires the same number of days per +# season for each year. You may need to omit leap days if February is part of +# the processing +NUM_SEASONS = 38 +DAYS_PER_SEASON = 90 + +# Text files containing a list of input files for the IBL code +OBS_WR_INPUT_TEXTFILE = {OBS_WR_OUTPUT_DIR}/{OBS_WR_OUTPUT_TEMPLATE} + +# Boolean indicating whether to keep the above text file or delete it at the end of the run +KEEP_WR_FILE_LISTING = True + +# Variable for the Z500 data +OBS_WR_VAR = Z500 + +# Weather Regime Number +OBS_WR_NUMBER = 6 + +# Number of clusters +OBS_NUM_CLUSTERS = 20 + +# Number of principal components +OBS_NUM_PCS = 10 + +# Time (in timesteps) over which to compute weather regime frequencies +# i.e. if your data time step is days and you want to average over 7 +# days, input 7 +# Optional, only needed if you want to compute frequencies +OBS_WR_FREQ = 7 + +# Type, name and directory of Output File for weather regime classification +# Type options are text or netcdf +OBS_WR_OUTPUT_FILE_TYPE = text +OBS_WR_OUTPUT_FILE = obs_weather_regime_class +WR_OUTPUT_FILE_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime + +# Directory to send output plots +WR_PLOT_OUTPUT_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime/plots/ + +# Elbow Plot Title and output file name +OBS_ELBOW_PLOT_TITLE = Elbow Method For Optimal k +OBS_ELBOW_PLOT_OUTPUT_NAME = obs_elbow + +# EOF plot output name and contour levels +OBS_EOF_PLOT_OUTPUT_NAME = obs_eof +EOF_PLOT_LEVELS = -50, -45, -40, -35, -30, -25, -20, -15, -10, -5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50 + +# K means Plot Output Name and contour levels +OBS_KMEANS_PLOT_OUTPUT_NAME = obs_kmeans +KMEANS_PLOT_LEVELS = -80, -70, -60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60, 70, 80 + +# Frequency Plot title and output file name +OBS_FREQ_PLOT_TITLE = Seasonal Cycle of WR Days/Week (1979-2017) +OBS_FREQ_PLOT_OUTPUT_NAME = obs_freq + +# MPR file information +MASK_NAME = FULL +WR_MPR_OUTPUT_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime/mpr + + # Run the Weather Regime Script [script_wr] # User Script Commands USER_SCRIPT_CUSTOM_LOOP_LIST = nc # Run the user script once -USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE +USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_LEAD # Command to run the user script with input configuration file -USER_SCRIPT_COMMAND = {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf dir.INPUT_BASE={INPUT_BASE} dir.OUTPUT_BASE={OUTPUT_BASE} dir.MET_INSTALL_DIR={MET_INSTALL_DIR} +USER_SCRIPT_COMMAND = {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py index 7eeb0aa1c4..83b010161f 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py @@ -1,6 +1,5 @@ import os import numpy as np -#import pandas as pd from pylab import * from sklearn.cluster import KMeans import scipy @@ -9,27 +8,20 @@ from numpy import ones,vstack from numpy.linalg import lstsq from eofs.standard import Eof -from metplus.util import config_metplus -#, get_start_end_interval_times, get_lead_sequence -#from metplus.util import get_skip_times, skip_time, is_loop_by_init, ti_calculate - -import matplotlib.pyplot as plt -import matplotlib.colors as colors -from mpl_toolkits.basemap import Basemap class WeatherRegimeCalculation(): """Contains the programs to perform a Weather Regime Analysis """ - def __init__(self,config,label): + def __init__(self,label): - self.wrnum = config.getint('WeatherRegime',label+'_WR_NUMBER',6) - self.numi = config.getint('WeatherRegime',label+'_NUM_CLUSTERS',20) - self.NUMPCS = config.getint('WeatherRegime',label+'_NUM_PCS',10) - self.wr_tstep = config.getint('WeatherRegime',label+'_WR_FREQ',7) - self.wr_outfile_type = config.getstr('WeatherRegime',label+'_WR_OUTPUT_FILE_TYPE','text') - self.wr_outfile_dir = config.getstr('WeatherRegime','WR_OUTPUT_FILE_DIR','') - self.wr_outfile = config.getstr('WeatherRegime',label+'_WR_OUTPUT_FILE',label+'_WeatherRegime') + self.wrnum = int(os.environ.get(label+'_WR_NUMBER',6)) + self.numi = int(os.environ.get(label+'_NUM_CLUSTERS',20)) + self.NUMPCS = int(os.environ.get(label+'_NUM_PCS',10)) + self.wr_tstep = int(os.environ.get(label+'_WR_FREQ',7)) + self.wr_outfile_type = os.environ.get(label+'_WR_OUTPUT_FILE_TYPE','text') + self.wr_outfile_dir = os.environ.get('WR_OUTPUT_FILE_DIR','') + self.wr_outfile = os.environ.get(label+'_WR_OUTPUT_FILE',label+'_WeatherRegime') def get_cluster_fraction(self, m, label): @@ -250,7 +242,7 @@ def run_K_means(self,a1,timedict,arr_shape): def compute_wr_freq(self, WR): ######## Simple Count ########## - WRfreq = np.zeros((self.wrnum,len(WR[:,0]),len(WR[0,:])-self.wr_tstep)) + WRfreq = np.zeros((self.wrnum,len(WR[:,0]),len(WR[0,:])-self.wr_tstep+1)) for yy in np.arange(0,len(WRfreq[0,:,0]),1): d1=0;d2=self.wr_tstep @@ -260,7 +252,6 @@ def compute_wr_freq(self, WR): WRfreq[ww,yy,dd] = len(np.where(temp==ww)[0]) d1=d1+1;d2=d2+1 - WRmean = np.nanmean(WRfreq,axis=1) - dlen_plot = len(WRfreq[0,0]) + dlen_plot = len(WRfreq[0,0,:]) - return WRmean, dlen_plot + return WRfreq, dlen_plot, self.wr_tstep diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py index 220e53ac98..ad9d503571 100755 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py @@ -7,8 +7,7 @@ import atexit from WeatherRegime import WeatherRegimeCalculation -from metplus.util import pre_run_setup, config_metplus, get_start_end_interval_times, get_lead_sequence -from metplus.util import get_skip_times, skip_time, is_loop_by_init, ti_calculate, do_string_sub, getlist +from metplus.util import getlist from metplotpy.contributed.weather_regime import plot_weather_regime as pwr from Blocking_WeatherRegime_util import parse_steps, read_nc_met, write_mpr_file @@ -28,60 +27,50 @@ def cleanup_daily_files(obs_dailyfile, fcst_dailyfile, keep_daily_files): def main(): - all_steps = ["ELBOW","PLOTELBOW","EOF","PLOTEOF","KMEANS","PLOTKMEANS"] - - inconfig_list = sys.argv[1:] - steps_list_fcst,steps_list_obs,config_list = parse_steps(inconfig_list) - config = pre_run_setup(config_list) + steps_list_fcst,steps_list_obs = parse_steps() if not steps_list_obs and not steps_list_fcst: warnings.warn('No processing steps requested for either the model or observations,') - warnings.warn('no data will be processed') + warnings.warn('No data will be processed') ###################################################################### # Blocking Calculation and Plotting ###################################################################### # Set up the data - steps_obs = WeatherRegimeCalculation(config,'OBS') - steps_fcst = WeatherRegimeCalculation(config,'FCST') + steps_obs = WeatherRegimeCalculation('OBS') + steps_fcst = WeatherRegimeCalculation('FCST') # Check to see if there is a plot directory - oplot_dir = config.getstr('WeatherRegime','WR_PLOT_OUTPUT_DIR','') + oplot_dir = os.environ.get('WR_PLOT_OUTPUT_DIR','') + obase = os.environ['SCRIPT_OUTPUT_BASE'] if not oplot_dir: - obase = config.getstr('config','OUTPUT_BASE') - oplot_dir = obase+'/'+'plots' + oplot_dir = os.path.join(obase,'plots') if not os.path.exists(oplot_dir): os.makedirs(oplot_dir) # Check to see if there is a mpr output directory - mpr_outdir = config.getstr('WeatherRegime','WR_MPR_OUTPUT_DIR','') + mpr_outdir = os.environ.get('WR_MPR_OUTPUT_DIR','') if not mpr_outdir: - obase = config.getstr('config','OUTPUT_BASE') mpr_outdir = os.path.join(obase,'mpr') # Get number of seasons and days per season - nseasons = config.getint('WeatherRegime','NUM_SEASONS') - dseasons = config.getint('WeatherRegime','DAYS_PER_SEASON') + nseasons = int(os.environ['NUM_SEASONS']) + dseasons = int(os.environ['DAYS_PER_SEASON']) - # Grab the Daily (IBL) text files - obs_wr_filetxt = config.getstr('WeatherRegime','OBS_WR_INPUT_TEXTFILE','') - fcst_wr_filetxt = config.getstr('WeatherRegime','FCST_WR_INPUT_TEXTFILE','') - keep_wr_textfile = config.getstr('WeatherRegime','KEEP_WR_FILE_LISTING', 'False').lower() + # Grab the Daily text files + obs_wr_filetxt = os.environ.get('OBS_WR_INPUT_TEXTFILE','') + fcst_wr_filetxt = os.environ.get('FCST_WR_INPUT_TEXTFILE','') + keep_wr_textfile = os.environ.get('KEEP_WR_FILE_LISTING', 'False').lower() atexit.register(cleanup_daily_files, obs_wr_filetxt, fcst_wr_filetxt, keep_wr_textfile) - elbow_config = config_metplus.replace_config_from_section(config,'WeatherRegime') - elbow_config_init = config.find_section('WeatherRegime','INIT_BEG') - elbow_config_valid = config.find_section('WeatherRegime','VALID_BEG') - use_init = is_loop_by_init(elbow_config) - if ("ELBOW" in steps_list_obs) or ("EOF" in steps_list_obs) or ("KMEANS" in steps_list_obs): with open(obs_wr_filetxt) as owl: obs_infiles = owl.read().splitlines() if len(obs_infiles) != (nseasons*dseasons): raise Exception('Invalid Obs data; each year must contain the same date range to calculate seasonal averages.') - obs_invar = config.getstr('WeatherRegime','OBS_WR_VAR','') + obs_invar = os.environ.get('OBS_WR_VAR','') z500_obs,lats_obs,lons_obs,timedict_obs = read_nc_met(obs_infiles,obs_invar,nseasons,dseasons) z500_detrend_obs,z500_detrend_2d_obs = steps_obs.weights_detrend(lats_obs,lons_obs,z500_obs) @@ -90,7 +79,7 @@ def main(): fcst_infiles = fwl.read().splitlines() if len(fcst_infiles) != (nseasons*dseasons): raise Exception('Invalid Obs data; each year must contain the same date range to calculate seasonal averages.') - fcst_invar = config.getstr('WeatherRegime','FCST_WR_VAR','') + fcst_invar = os.environ.get('FCST_WR_VAR','') z500_fcst,lats_fcst,lons_fcst,timedict_fcst = read_nc_met(fcst_infiles,fcst_invar,nseasons,dseasons) z500_detrend_fcst,z500_detrend_2d_fcst = steps_fcst.weights_detrend(lats_fcst,lons_fcst,z500_fcst) @@ -107,16 +96,16 @@ def main(): if not ("ELBOW" in steps_list_obs): raise Exception('Must run observed Elbow before plotting observed elbow.') print('Creating Obs Elbow plot') - elbow_plot_title = config.getstr('WeatherRegime','OBS_ELBOW_PLOT_TITLE','Elbow Method For Optimal k') - elbow_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','OBS_ELBOW_PLOT_OUTPUT_NAME','obs_elbow') + elbow_plot_title = os.environ.get('OBS_ELBOW_PLOT_TITLE','Elbow Method For Optimal k') + elbow_plot_outname = os.path.join(oplot_dir,os.environ.get('OBS_ELBOW_PLOT_OUTPUT_NAME','obs_elbow')) pwr.plot_elbow(K_obs,d_obs,mi_obs,line_obs,curve_obs,elbow_plot_title,elbow_plot_outname) if ("PLOTELBOW" in steps_list_fcst): if not ("ELBOW" in steps_list_fcst): raise Exception('Must run forecast Elbow before plotting forecast elbow.') print('Creating Forecast Elbow plot') - elbow_plot_title = config.getstr('WeatherRegime','FCST_ELBOW_PLOT_TITLE','Elbow Method For Optimal k') - elbow_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','FCST_ELBOW_PLOT_OUTPUT_NAME','fcst_elbow') + elbow_plot_title = os.environ.get('FCST_ELBOW_PLOT_TITLE','Elbow Method For Optimal k') + elbow_plot_outname = os.path.join(oplot_dir,os.environ.get('FCST_ELBOW_PLOT_OUTPUT_NAME','fcst_elbow')) pwr.plot_elbow(K_fcst,d_fcst,mi_fcst,line_fcst,curve_fcst,elbow_plot_title,elbow_plot_outname) @@ -134,18 +123,18 @@ def main(): if not ("EOF" in steps_list_obs): raise Exception('Must run observed EOFs before plotting observed EOFs.') print('Plotting Obs EOFs') - pltlvls_str = getlist(config.getstr('WeatherRegime','EOF_PLOT_LEVELS','')) + pltlvls_str = getlist(os.environ['EOF_PLOT_LEVELS']) pltlvls = [float(pp) for pp in pltlvls_str] - eof_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','OBS_EOF_PLOT_OUTPUT_NAME','obs_eof') + eof_plot_outname = os.path.join(oplot_dir,os.environ.get('OBS_EOF_PLOT_OUTPUT_NAME','obs_eof')) pwr.plot_eof(eof_obs,wrnum_obs,variance_fractions_obs,lons_obs,lats_obs,eof_plot_outname,pltlvls) if ("PLOTEOF" in steps_list_fcst): if not ("EOF" in steps_list_fcst): raise Exception('Must run forecast EOFs before plotting forecast EOFs.') print('Plotting Forecast EOFs') - pltlvls_str = getlist(config.getstr('WeatherRegime','EOF_PLOT_LEVELS','')) + pltlvls_str = getlist(os.environ['EOF_PLOT_LEVELS']) pltlvls = [float(pp) for pp in pltlvls_str] - eof_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','OBS_EOF_PLOT_OUTPUT_NAME','obs_eof') + eof_plot_outname = os.path.join(oplot_dir,os.environ.get('OBS_EOF_PLOT_OUTPUT_NAME','obs_eof')) pwr.plot_eof(eof_fcst,wrnum_fcst,variance_fractions_fcst,lons_fcst,lats_fcst,eof_plot_outname,pltlvls) @@ -155,71 +144,82 @@ def main(): if ("KMEANS" in steps_list_fcst): print('Running Forecast K Means') - kmeans_fcst,wrnum_fcst,perc_fcst,wrc_fcst = steps_fcst.run_K_means(z500_detrend_2d_fcst,timedict_fcst,z500_fcst.shape) - - #if ("KMEANS" in steps_list_obs) and ("KMEANS" in steps_list_fcst): - modname = config.getstr('WeatherRegime','MODEL_NAME','GFS') - maskname = config.getstr('WeatherRegime','MASK_NAME','FULL') - wr_outfile_prefix = os.path.join(mpr_outdir,'weather_regime_stat_'+modname) - wrc_fcst = wrc_obs - wrc_obs_mpr = wrc_obs[:,:,np.newaxis] - wrc_fcst_mpr = wrc_fcst[:,:,np.newaxis] - timedict_fcst = timedict_obs - if not os.path.exists(mpr_outdir): - os.makedirs(mpr_outdir) - write_mpr_file(wrc_obs_mpr,wrc_fcst_mpr,[0.0],[0.0],timedict_obs,timedict_fcst,modname, - 'WeatherRegimeClass','Z500','WeatherRegimeClass','Z500',maskname,'500',wr_outfile_prefix) + kmeans_fcst,wrnum_fcst,perc_fcst,wrc_fcst = steps_fcst.run_K_means(z500_detrend_2d_fcst,timedict_fcst, + z500_fcst.shape) + + if ("KMEANS" in steps_list_obs) and ("KMEANS" in steps_list_fcst): + modname = os.environ.get('MODEL_NAME','GFS') + maskname = os.environ.get('MASK_NAME','FULL') + mpr_full_outdir = os.path.join(mpr_outdir,'/WeatherRegime') + wr_outfile_prefix = os.path.join(mpr_full_outdir,'weather_regime_stat_'+modname) + wrc_obs_mpr = wrc_obs[:,:,np.newaxis] + wrc_fcst_mpr = wrc_fcst[:,:,np.newaxis] + if not os.path.exists(mpr_full_outdir): + os.makedirs(mpr_full_outdir) + write_mpr_file(wrc_obs_mpr,wrc_fcst_mpr,[0.0],[0.0],timedict_obs,timedict_fcst,modname, + 'WeatherRegimeClass','Z500','WeatherRegimeClass','Z500',maskname,'500',wr_outfile_prefix) if ("PLOTKMEANS" in steps_list_obs): if not ("KMEANS" in steps_list_obs): raise Exception('Must run observed Kmeans before plotting observed Kmeans.') print('Plotting Obs K Means') - pltlvls_str = getlist(config.getstr('WeatherRegime','KMEANS_PLOT_LEVELS','')) + pltlvls_str = getlist(os.environ['KMEANS_PLOT_LEVELS']) pltlvls = [float(pp) for pp in pltlvls_str] - kmeans_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','OBS_KMEANS_PLOT_OUTPUT_NAME','obs_kmeans') + kmeans_plot_outname = os.path.join(oplot_dir,os.environ.get('OBS_KMEANS_PLOT_OUTPUT_NAME','obs_kmeans')) pwr.plot_K_means(kmeans_obs,wrnum_obs,lons_obs,lats_obs,perc_obs,kmeans_plot_outname,pltlvls) if ("PLOTKMEANS" in steps_list_fcst): if not ("KMEANS" in steps_list_fcst): raise Exception('Must run forecast Kmeans before plotting forecast Kmeans.') print('Plotting Forecast K Means') - pltlvls_str = getlist(config.getstr('WeatherRegime','KMEANS_PLOT_LEVELS','')) + pltlvls_str = getlist(os.environ['KMEANS_PLOT_LEVELS']) pltlvls = [float(pp) for pp in pltlvls_str] - kmeans_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','FCST_KMEANS_PLOT_OUTPUT_NAME','fcst_kmeans') + kmeans_plot_outname = os.path.join(oplot_dir,os.environ.get('FCST_KMEANS_PLOT_OUTPUT_NAME','fcst_kmeans')) pwr.plot_K_means(kmeans_fcst,wrnum_fcst,lons_fcst,lats_fcst,perc_fcts,kmeans_plot_outname,pltlvls) if ("TIMEFREQ" in steps_list_obs): - wrmean_obs,dlen_obs = steps_obs.compute_wr_freq(wrc_obs) + wrfreq_obs,dlen_obs,ts_diff_obs = steps_obs.compute_wr_freq(wrc_obs) if ("TIMEFREQ" in steps_list_fcst): - wrmean_fcst,dlen_fcst = steps_fcst.compute_wr_freq(wrc_fcst) - - #if ("TIMEFREQ" in steps_list_obs) and ("TIMEFREQ" in steps_list_fcst): - #modname = config.getstr('WeatherRegime','MODEL_NAME','GFS') - #maskname = config.getstr('WeatherRegime','MASK_NAME','FULL') - #wr_outfile_prefix = os.path.join(mpr_outdir,'weather_regime_freq_stat_'+modname) - #wrmean_fcst = wrmean_obs - #timedict_fcst = timedict_obs - #wrmean_obs_mpr = wrmean_obs[:,:,np.newaxis] - #wrmean_fcst_mpr wrmean_fcst[:,:,np.newaxis] - #if not os.path.exists(mpr_outdir): - # os.makedirs(mpr_outdir) - #write_mpr_file(wrmean_obs_mpr,wrmean_fcst_mpr,[0.0],[0.0],timedict_obs,timedict_fcst,modname, - # 'WeatherRegimeClass','Z500','WeatherRegimeClass','Z500',maskname,'500',wr_outfile_prefix) + wrfreq_fcst,dlen_fcst,ts_diff_fcst = steps_fcst.compute_wr_freq(wrc_fcst) + + if ("TIMEFREQ" in steps_list_obs) and ("TIMEFREQ" in steps_list_fcst): + modname = os.environ.get('MODEL_NAME','GFS') + maskname = os.environ.get('MASK_NAME','FULL') + mpr_full_outdir = os.path.join(mpr_outdir,'freq') + ## remove ## + #wrfreq_fcst = wrfreq_obs + #timedict_fcst = timedict_obs + #ts_diff_fcst = ts_diff_obs + #### + timedict_obs_mpr = {'init':timedict_obs['init'][:,ts_diff_obs-1:], + 'valid':timedict_obs['valid'][:,ts_diff_obs-1:],'lead':timedict_obs['lead'][:,ts_diff_obs-1:]} + timedict_fcst_mpr = {'init':timedict_fcst['init'][:,ts_diff_fcst-1:], + 'valid':timedict_fcst['valid'][:,ts_diff_fcst-1:],'lead':timedict_fcst['lead'][:,ts_diff_fcst-1:]} + wrfreq_obs_mpr = wrfreq_obs[:,:,:,np.newaxis] + wrfreq_fcst_mpr = wrfreq_fcst[:,:,:,np.newaxis] + if not os.path.exists(mpr_full_outdir): + os.makedirs(mpr_full_outdir) + for wrn in np.arange(wrnum_obs): + wr_outfile_prefix = os.path.join(mpr_full_outdir,'weather_regime'+str(wrn+1).zfill(2)+'_freq_stat_'+modname) + write_mpr_file(wrfreq_obs_mpr[wrn,:,:,:],wrfreq_fcst_mpr[wrn,:,:,:],[0.0],[0.0],timedict_obs, + timedict_fcst,modname,'WeatherRegimeClass','Z500','WeatherRegimeClass','Z500',maskname,'500', + wr_outfile_prefix) if ("PLOTFREQ" in steps_list_obs): - freq_plot_title = config.getstr('WeatherRegime','OBS_FREQ_PLOT_TITLE','Seasonal Cycle of WR Days/Week') - freq_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','OBS_FREQ_PLOT_OUTPUT_NAME','obs_freq') - pwr.plot_wr_frequency(wrmean_obs,wrnum_obs,dlen_obs,freq_plot_title,freq_plot_outname) + if not ("TIMEFREQ" in steps_list_obs): + raise Exception('Must run observed Frequency calculation before plotting the frequencies.') + freq_plot_title = os.environ.get('OBS_FREQ_PLOT_TITLE','Seasonal Cycle of WR Days/Week') + freq_plot_outname = os.path.join(oplot_dir,os.environ.get('OBS_FREQ_PLOT_OUTPUT_NAME','obs_freq')) + pwr.plot_wr_frequency(wrfreq_obs,wrnum_obs,dlen_obs,freq_plot_title,freq_plot_outname) if ("PLOTFREQ" in steps_list_fcst): - freq_plot_title = config.getstr('WeatherRegime','FCST_FREQ_PLOT_TITLE','Seasonal Cycle of WR Days/Week') - freq_plot_outname = oplot_dir+'/'+config.getstr('WeatherRegime','FCST_FREQ_PLOT_OUTPUT_NAME','fcst_freq') - pwr.plot_wr_frequency(wrmean_fcst,wrnum_fcst,dlen_fcst,freq_plot_title,freq_plot_outname) - - - #if ("ANOMALY_CORR" in steps_list_obs): + if not ("TIMEFREQ" in steps_list_fcst): + raise Exception('Must run forecast Frequency calculation before plotting the frequencies.') + freq_plot_title = os.environ.get('FCST_FREQ_PLOT_TITLE','Seasonal Cycle of WR Days/Week') + freq_plot_outname = os.path.join(oplot_dir,os.environ.get('FCST_FREQ_PLOT_OUTPUT_NAME','fcst_freq')) + pwr.plot_wr_frequency(wrfreq_fcst,wrnum_fcst,dlen_fcst,freq_plot_title,freq_plot_outname) if __name__ == "__main__": diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf deleted file mode 100644 index e51da98597..0000000000 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_obsERA_obsOnly.conf +++ /dev/null @@ -1,68 +0,0 @@ -# Blocking METplus Configuration -[config] -# Steps to Run -OBS_STEPS = ELBOW+PLOTELBOW+EOF+PLOTEOF+KMEANS+PLOTKMEANS+TIMEFREQ+PLOTFREQ - - -# Variables Specific to Weather Regime analysis -[WeatherRegime] -# Number of Seasons and Days per season that should be available -# The code will fill missing data, but requires the same number of days per -# season for each year. You may need to omit leap days if February is part of -# the processing -NUM_SEASONS = 38 -DAYS_PER_SEASON = 90 - -# Text files containing a list of input files for the IBL code -OBS_WR_INPUT_TEXTFILE = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily/ERA_daily_files_lead000.txt - -# Boolean indicating whether to keep the above 2 text files or delete them at the end of the run -KEEP_WR_FILE_LISTING = True - -# Variable for the Z500 data -OBS_WR_VAR = Z500 - -# Weather Regime Number -OBS_WR_NUMBER = 6 - -# Number of clusters -OBS_NUM_CLUSTERS = 20 - -# Number of principal components -OBS_NUM_PCS = 10 - -# Time (in timesteps) over which to compute weather regime frequencies -# i.e. if your data time step is days and you want to average over 7 -# days, input 7 -# Optional, only needed if you want to compute frequencies -OBS_WR_FREQ = 7 - -# Type, name and directory of Output File for weather regime classification -# Type options are text or netcdf -OBS_WR_OUTPUT_FILE_TYPE = text -#OBS_WR_OUTPUT_FILE_TYPE = netcdf -OBS_WR_OUTPUT_FILE = obs_weather_regime_class -WR_OUTPUT_FILE_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime - -# Directory to send output plots -WR_PLOT_OUTPUT_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime/plots/ - -# Elbow Plot Title and output file name -OBS_ELBOW_PLOT_TITLE = Elbow Method For Optimal k -OBS_ELBOW_PLOT_OUTPUT_NAME = obs_elbow - -# EOF plot output name and contour levels -OBS_EOF_PLOT_OUTPUT_NAME = obs_eof -EOF_PLOT_LEVELS = -50, -45, -40, -35, -30, -25, -20, -15, -10, -5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50 - -# K means Plot Output Name and contour levels -OBS_KMEANS_PLOT_OUTPUT_NAME = obs_kmeans -KMEANS_PLOT_LEVELS = -80, -70, -60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60, 70, 80 - -# Frequency Plot title and output file name -OBS_FREQ_PLOT_TITLE = Seasonal Cycle of WR Days/Week (1979-2017) -OBS_FREQ_PLOT_OUTPUT_NAME = obs_freq - -# MPR file information -MASK_NAME = FULL -WR_MPR_OUTPUT_DIR = {OUTPUT_BASE}/s2s/UserScript_obsERA_obsOnly_WeatherRegime/mpr From f31487294d1807a4a87e1c936d8670292d2ba508 Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Tue, 13 Jul 2021 11:06:19 -0600 Subject: [PATCH 05/39] Updated documentation --- ...UserScript_obsERA_obsOnly_WeatherRegime.py | 55 +++++++++++-------- .../WeatherRegime.py | 4 +- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py index e8bf103583..e5d035abf4 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py +++ b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py @@ -13,15 +13,16 @@ # -------------------- # # To perform a weather regime analysis using 500 mb height data. There are 2 pre- -# processing steps, RegridDataPlane and PcpCombine, and 3 steps in the weather regime -# analysis, elbow, EOFs, and K means. The elbow and K means steps begin with K means -# clustering. Elbow then computes the sum of squared distances for clusters 1 - 14 -# and draws a straight line from the sum of squared distance for the clusters. This -# helps determine the optimal cluster number by examining the largest difference between -# the curve and the straight line. The EOFs step is optional. It computes an empirical -# orthogonal function analysis. The K means step uses clustering to compute the -# frequency of occurrernce and anomalies for each cluster to give the most common -# weather regimes. +# processing steps, RegridDataPlane and PcpCombine, and 4 steps in the weather regime +# analysis, elbow, EOFs, K means, and Time frequency. The elbow and K means steps begin +# with K means clustering. Elbow then computes the sum of squared distances for clusters +# 1 - 14 and draws a straight line from the sum of squared distance for the clusters. +# This helps determine the optimal cluster number by examining the largest difference +# between the curve and the straight line. The EOFs step is optional. It computes an +# empirical orthogonal function analysis. The K means step uses clustering to compute +# the frequency of occurrernce and anomalies for each cluster to give the most common +# weather regimes. Finally, the time frequency computes the frequency of each weather +# regime over a user specified time frame. ############################################################################## # Datasets @@ -57,16 +58,18 @@ # ------------------ # # This use case runs the weather regime driver script which runs the steps the user -# lists in STEPS_OBS. The possible steps are regridding, time averaging, computing the -# elbow (ELBOW), plotting the elbow (PLOTELBOW), computing EOFs (EOF), plotting EOFs -# (PLOTEOF), computing K means (KMEANS), plotting the K means (PLOTKMEANS), computing a time -# frequency of weather regimes (TIMEFREQ) and plotting the time frequency (PLOTFREQ). Regridding -# and time averaging are set up in the UserScript .conf file and are formatted as follows: -# PROCESS_LIST = RegridDataPlane(regrid_obs), PcpCombine(daily_mean_obs), UserScript(script_wr) -# -# The other steps are listed in the weather regime analsysis .conf file +# lists in STEPS_OBS. The possible steps are regridding, time averaging, creating a list of input +# files for the weather regime calculation, computing the elbow (ELBOW), plotting the elbow +# (PLOTELBOW), computing EOFs (EOF), plotting EOFs (PLOTEOF), computing K means (KMEANS), plotting +# the K means (PLOTKMEANS), computing a time frequency of weather regimes (TIMEFREQ) and plotting +# the time frequency (PLOTFREQ). Regridding and time averaging are set up in the UserScript .conf file +# and are formatted as follows: +# PROCESS_LIST = RegridDataPlane(regrid_obs), PcpCombine(daily_mean_obs), UserScript(obs_wr_filelist), UserScript(script_wr) +# +# The other steps are listed in the [user_env_vars] section of the UserScript .conf file # in the following format: # OBS_STEPS = ELBOW+PLOTELBOW+EOF+PLOTEOF+KMEANS+PLOTKMEANS +# ############################################################################## # METplus Workflow @@ -75,8 +78,9 @@ # The weather regime python code is run for each time for the forecast and observations # data. This example loops by valid time. This version is set to only process the weather # regime steps (ELBOW, PLOTELBOW, EOF, PLOTEOF, KMEANS, PLOTKMEANS, TIMEFREQ, PLOTFREQ), omitting -# the REGRID and TIMEAVE pre-processing steps. However, the configurations for pre-processing are -# available for user reference. +# the regridding, time averaging, and creating the file list pre-processing steps. However, the +# configurations for pre-processing are available for user reference. +# ############################################################################## # METplus Configuration @@ -163,10 +167,15 @@ # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. Output for this use # case will be found in model_applications/s2s/WeatherRegime (relative to **OUTPUT_BASE**) and will contain output -# for the steps requested. This may include the regridded data, daily averaged files, and a weather regime output -# file. In addition, output elbow, EOF, and Kmeans weather regime plots can be generated. The location -# of these output plots can be specified as WR_OUTPUT_DIR. If it is not specified, plots will be sent -# to model_applications/s2s/WeatherRegime/plots (relative to **OUTPUT_BASE**). +# for the steps requested. This may include the regridded data, daily averaged files, a text file containing the +# list of input files, aweather regime output, and MET matched pair files for the weather regime classification and +# time frequency (if KMEANS and TIMEFREQ are run for both the forecast and observation data). In addition, output +# elbow, EOF, and Kmeans weather regime plots can be generated. The location of these output plots can be specified +# as WR_OUTPUT_DIR. If it is not specified, plots will be sent to {OUTPUT_BASE}/plots. The output location for +# the matched pair files can be specified as WR_MPR_OUTPUT_DIR. If it is not specified, it will be sent to +# {OUTPUT_BASE}/mpr. The output weather regime text or netCDF file location is set in WR_OUTPUT_FILE_DIR. If this +# is not specified, the output text/netCDF file will be sent to {OUTPUT_BASE}. +# ############################################################################## # Keywords diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py index 83b010161f..2165cbada7 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py @@ -185,7 +185,7 @@ def run_K_means(self,a1,timedict,arr_shape): # netcdf file if self.wr_outfile_type=='netcdf': - wr_full_outfile = self.wr_outfile_dir+'/'+self.wr_outfile+'.nc' + wr_full_outfile = os.path.join(self.wr_outfile_dir,self.wr_outfile+'.nc') if os.path.isfile(wr_full_outfile): os.remove(wr_full_outfile) @@ -225,7 +225,7 @@ def run_K_means(self,a1,timedict,arr_shape): # text file if self.wr_outfile_type=='text': - wr_full_outfile = self.wr_outfile_dir+'/'+self.wr_outfile+'.txt' + wr_full_outfile = os.path.join(self.wr_outfile_dir,self.wr_outfile+'.txt') if os.path.isfile(wr_full_outfile): os.remove(wr_full_outfile) From c4cc2f70cd0802124f1b38b8293e5c7b12056ddb Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Tue, 13 Jul 2021 12:45:30 -0600 Subject: [PATCH 06/39] Fixed a path --- .../UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py index 2165cbada7..9c2d5c83fa 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime.py @@ -20,7 +20,7 @@ def __init__(self,label): self.NUMPCS = int(os.environ.get(label+'_NUM_PCS',10)) self.wr_tstep = int(os.environ.get(label+'_WR_FREQ',7)) self.wr_outfile_type = os.environ.get(label+'_WR_OUTPUT_FILE_TYPE','text') - self.wr_outfile_dir = os.environ.get('WR_OUTPUT_FILE_DIR','') + self.wr_outfile_dir = os.environ.get('WR_OUTPUT_FILE_DIR',os.environ['SCRIPT_OUTPUT_BASE']) self.wr_outfile = os.environ.get(label+'_WR_OUTPUT_FILE',label+'_WeatherRegime') From 7b680e685a8b5fe0140820a1bef822061224b781 Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Tue, 13 Jul 2021 15:38:59 -0600 Subject: [PATCH 07/39] Updates to documentation --- .../s2s/UserScript_obsERA_obsOnly_WeatherRegime.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py index e5d035abf4..8607ccc07e 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py +++ b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.py @@ -23,6 +23,7 @@ # the frequency of occurrernce and anomalies for each cluster to give the most common # weather regimes. Finally, the time frequency computes the frequency of each weather # regime over a user specified time frame. +# ############################################################################## # Datasets @@ -68,7 +69,7 @@ # # The other steps are listed in the [user_env_vars] section of the UserScript .conf file # in the following format: -# OBS_STEPS = ELBOW+PLOTELBOW+EOF+PLOTEOF+KMEANS+PLOTKMEANS +# OBS_STEPS = ELBOW+PLOTELBOW+EOF+PLOTEOF+KMEANS+PLOTKMEANS+TIMEFREQ+PLOTFREQ # ############################################################################## @@ -95,6 +96,7 @@ # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf +# ############################################################################## # MET Configuration @@ -107,6 +109,7 @@ # # parm/use_cases/met_tool_wrapper/RegridDataPlane/RegridDataPlane.py # parm/use_cases/met_tool_wrapper/PCPCombine/PCPCOmbine_derive.py +# ############################################################################## # Python Scripts From 1bcdc89566bbfb92cce8e75a0f922df1fa2459dd Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 08:16:56 -0600 Subject: [PATCH 08/39] call script to populate file list, output file list to output directory instead of under input base --- .../s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf index 7e295697fb..f5794b1b41 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf @@ -4,7 +4,7 @@ # All steps, including pre-processing: # PROCESS_LIST = RegridDataPlane(regrid_obs), PcpCombine(daily_mean_obs), UserScript(obs_wr_filelist), UserScript(script_wr) # Weather Regime Analysis only: -PROCESS_LIST = UserScript(script_wr) +PROCESS_LIST = UserScript(obs_wr_filelist), UserScript(script_wr) # time looping - options are INIT, VALID, RETRO, and REALTIME # If set to INIT or RETRO: @@ -52,7 +52,8 @@ CONFIG_DIR={PARM_BASE}/use_cases/model_applications/s2s OBS_WR_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily OBS_WR_INPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d}_NH.nc -OBS_WR_OUTPUT_DIR = {OBS_WR_INPUT_DIR} +#OBS_WR_OUTPUT_DIR = {OBS_WR_INPUT_DIR} +OBS_WR_OUTPUT_DIR = {OUTPUT_BASE} OBS_WR_OUTPUT_TEMPLATE = ERA_daily_files_lead{lead?fmt=%HHH}.txt From af998e10b52317e31ea5e521c775b034cf1165cd Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:37:25 -0600 Subject: [PATCH 09/39] added env file so pytests can run easily on seneca --- internal_tests/pytests/minimum_pytest.seneca.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 internal_tests/pytests/minimum_pytest.seneca.sh diff --git a/internal_tests/pytests/minimum_pytest.seneca.sh b/internal_tests/pytests/minimum_pytest.seneca.sh new file mode 100644 index 0000000000..9fac2a711f --- /dev/null +++ b/internal_tests/pytests/minimum_pytest.seneca.sh @@ -0,0 +1,4 @@ +export METPLUS_TEST_INPUT_BASE=/d1/projects/METplus/METplus_Data +export METPLUS_TEST_OUTPUT_BASE=/d1/personal/${USER}/pytest +export METPLUS_TEST_MET_INSTALL_DIR=/usr/local/met +export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp From 461cb095fb1197bf720ccc3a1b9ce7bd27906704 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:37:49 -0600 Subject: [PATCH 10/39] improved logging readability --- metplus/wrappers/command_builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index c6c802ca9b..44d85cd590 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -863,13 +863,14 @@ def write_list_file(self, filename, file_list, output_dir=None): if not os.path.exists(list_dir): os.makedirs(list_dir, mode=0o0775) - self.logger.debug(f"Writing list of filenames to {list_path}") + self.logger.debug("Writing list of filenames...") with open(list_path, 'w') as file_handle: file_handle.write('file_list\n') for f_path in file_list: self.logger.debug(f"Adding file to list: {f_path}") file_handle.write(f_path + '\n') + self.logger.debug(f"Wrote list of filenames to {list_path}") return list_path def find_and_check_output_file(self, time_info=None, From c8e0ef10e396a153986397f9fdabf0b9b768100b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:40:21 -0600 Subject: [PATCH 11/39] moved functions up to parent class so they can be used by UserScript --- metplus/wrappers/grid_diag_wrapper.py | 85 ---------------------- metplus/wrappers/runtime_freq_wrapper.py | 89 ++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 85 deletions(-) diff --git a/metplus/wrappers/grid_diag_wrapper.py b/metplus/wrappers/grid_diag_wrapper.py index 07f4058030..b10d9ee250 100755 --- a/metplus/wrappers/grid_diag_wrapper.py +++ b/metplus/wrappers/grid_diag_wrapper.py @@ -222,28 +222,6 @@ def set_data_field(self, time_info): return True - def find_input_files(self, time_info): - """! Loop over list of input templates and find files for each - - @param time_info time dictionary to use for string substitution - @returns Dictionary of key input number and value is list of - input file list if all files were found, None if not. - """ - all_input_files = {} - for idx, input_template in enumerate(self.c_dict['INPUT_TEMPLATES']): - self.c_dict['INPUT_TEMPLATE'] = input_template - input_files = self.find_data(time_info, return_list=True) - if not input_files: - continue - - all_input_files[f'input{idx}'] = input_files - - # return None if no matching input files were found - if not all_input_files: - return None - - return all_input_files - def set_command_line_arguments(self, time_info): """! add config file passing through do_string_sub to get custom string if set @@ -274,66 +252,3 @@ def get_files_from_time(self, time_info): file_dict[key] = value return file_dict - - def subset_input_files(self, time_info): - """! Obtain a subset of input files from the c_dict ALL_FILES based on - the time information for the current run. - - @param time_info dictionary containing time information - @returns the path to a ascii file containing the list of files - or None if could not find any files - """ - all_input_files = {} - for file_dict in self.c_dict['ALL_FILES']: - # compare time information for each input file - # add file to list of files to use if it matches - if not self.compare_time_info(time_info, file_dict['time_info']): - continue - - input_keys = [key for key in file_dict if key.startswith('input')] - for input_key in input_keys: - if input_key not in all_input_files: - all_input_files[input_key] = [] - all_input_files[input_key].extend(file_dict[input_key]) - - # return None if no matching input files were found - if not all_input_files: - return None - - # loop over all inputs and write a file list file for each - list_file_paths = [] - for identifier, input_files in all_input_files.items(): - list_file_name = self.get_list_file_name(time_info, identifier) - list_file_path = self.write_list_file(list_file_name, input_files) - list_file_paths.append(list_file_path) - - return list_file_paths - - @staticmethod - def get_list_file_name(time_info, identifier): - """! Build name of ascii file that contains a list of files to process. - If wildcard is set for init, valid, or lead then use the text ALL - in the filename. - - @param time_info dictionary containing time information - @param identifier string to identify which input is used - @returns filename i.e. - grid_diag_files_{identifier}_init_{init}_valid_{valid}_lead_{lead}.txt - """ - if time_info['init'] == '*': - init = 'ALL' - else: - init = time_info['init'].strftime('%Y%m%d%H%M%S') - - if time_info['valid'] == '*': - valid = 'ALL' - else: - valid = time_info['valid'].strftime('%Y%m%d%H%M%S') - - if time_info['lead'] == '*': - lead = 'ALL' - else: - lead = time_util.ti_get_seconds_from_lead(time_info['lead'], - time_info['valid']) - - return f"grid_diag_files_{identifier}_init_{init}_valid_{valid}_lead_{lead}.txt" diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 125c6510e0..b88bca526f 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -313,3 +313,92 @@ def compare_time_info(self, runtime, filetime): return runtime['lead'] == filetime['lead'] return runtime_lead == filetime_lead + + def find_input_files(self, time_info): + """! Loop over list of input templates and find files for each + + @param time_info time dictionary to use for string substitution + @returns Dictionary of key input number and value is list of + input file list if all files were found, None if not. + """ + all_input_files = {} + for idx, input_template in enumerate(self.c_dict['INPUT_TEMPLATES']): + self.c_dict['INPUT_TEMPLATE'] = input_template + input_files = self.find_data(time_info, return_list=True) + if not input_files: + continue + + all_input_files[f'input{idx}'] = input_files + + # return None if no matching input files were found + if not all_input_files: + return None + + return all_input_files + + def subset_input_files(self, time_info): + """! Obtain a subset of input files from the c_dict ALL_FILES based on + the time information for the current run. + + @param time_info dictionary containing time information + @returns dictionary with keys of the input identifier and the + value is the path to a ascii file containing the list of files + or None if could not find any files + """ + all_input_files = {} + if not self.c_dict.get('ALL_FILES'): + return all_input_files + + for file_dict in self.c_dict['ALL_FILES']: + # compare time information for each input file + # add file to list of files to use if it matches + if not self.compare_time_info(time_info, file_dict['time_info']): + continue + + input_keys = [key for key in file_dict if key.startswith('input')] + for input_key in input_keys: + if input_key not in all_input_files: + all_input_files[input_key] = [] + all_input_files[input_key].extend(file_dict[input_key]) + + # return None if no matching input files were found + if not all_input_files: + return all_input_files + + # loop over all inputs and write a file list file for each + list_file_dict = {} + for identifier, input_files in all_input_files.items(): + list_file_name = self.get_list_file_name(time_info, identifier) + list_file_path = self.write_list_file(list_file_name, input_files) + list_file_dict[identifier] = list_file_path + + return list_file_dict + + def get_list_file_name(self, time_info, identifier): + """! Build name of ascii file that contains a list of files to process. + If wildcard is set for init, valid, or lead then use the text ALL + in the filename. + + @param time_info dictionary containing time information + @param identifier string to identify which input is used + @returns filename i.e. + {app_name}_files_{identifier}_init_{init}_valid_{valid}_lead_{lead}.txt + """ + if time_info['init'] == '*': + init = 'ALL' + else: + init = time_info['init'].strftime('%Y%m%d%H%M%S') + + if time_info['valid'] == '*': + valid = 'ALL' + else: + valid = time_info['valid'].strftime('%Y%m%d%H%M%S') + + if time_info['lead'] == '*': + lead = 'ALL' + else: + lead = time_util.ti_get_seconds_from_lead(time_info['lead'], + time_info['valid']) + + return (f"{self.app_name}_files_{identifier}_" + f"init_{init}_valid_{valid}_lead_{lead}.txt") From 99385f2ba5c192260d81e19feaff8b66f4e89f1c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:41:30 -0600 Subject: [PATCH 12/39] updated logic to match change to subset_input_files function return value --- internal_tests/pytests/grid_diag/test_grid_diag.py | 6 +++--- metplus/wrappers/grid_diag_wrapper.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal_tests/pytests/grid_diag/test_grid_diag.py b/internal_tests/pytests/grid_diag/test_grid_diag.py index 3708ade52d..05202292aa 100644 --- a/internal_tests/pytests/grid_diag/test_grid_diag.py +++ b/internal_tests/pytests/grid_diag/test_grid_diag.py @@ -107,9 +107,9 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset): actual_files = [item for sub in actual_files for item in sub] assert(actual_files == expected_files) - file_list_files = wrapper.subset_input_files(time_info) - assert(file_list_files is not None) - with open(file_list_files[0], 'r') as file_handle: + file_list_dict = wrapper.subset_input_files(time_info) + assert file_list_dict + with open(file_list_dict['input0'], 'r') as file_handle: file_list = file_handle.readlines() file_list = file_list[1:] diff --git a/metplus/wrappers/grid_diag_wrapper.py b/metplus/wrappers/grid_diag_wrapper.py index b10d9ee250..e537dbb25e 100755 --- a/metplus/wrappers/grid_diag_wrapper.py +++ b/metplus/wrappers/grid_diag_wrapper.py @@ -163,11 +163,11 @@ def run_at_time_custom(self, time_info): self.clear() # subset input files as appropriate - input_list_files = self.subset_input_files(time_info) - if not input_list_files: + input_list_dict = self.subset_input_files(time_info) + if not input_list_dict: return - for input_list_file in input_list_files: + for input_list_file in input_list_dict.values(): self.infiles.append(input_list_file) # get output path From fc71217b138c648daf9742761bfb2b3ba9f720a1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:43:28 -0600 Subject: [PATCH 13/39] in run_at_time function loop over custom list and get files for the current run time before running since this function is called only if LOOP_BY=times and does not call the function that handles this -- consider refactoring to move logic for CUSTOM_LOOP_LIST so that it is handled consistently across all wrappers instead of being handled inside each --- metplus/wrappers/runtime_freq_wrapper.py | 49 ++++++++++++++++-------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index b88bca526f..0cb729e4d9 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -201,30 +201,45 @@ def run_at_time(self, input_dict): Args: @param input_dict dictionary containing time information """ + for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: + if custom_string: + self.logger.info(f"Processing custom string: {custom_string}") - # loop of forecast leads and process each - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - input_dict['lead'] = lead + input_dict['custom'] = custom_string - # set current lead time config and environment variables - time_info = time_util.ti_calculate(input_dict) + # loop of forecast leads and process each + lead_seq = get_lead_sequence(self.config, input_dict) + for lead in lead_seq: + input_dict['lead'] = lead - self.logger.info( - f"Processing forecast lead {time_info['lead_string']}" - ) + # set current lead time config and environment variables + time_info = time_util.ti_calculate(input_dict) - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue + self.logger.info( + f"Processing forecast lead {time_info['lead_string']}" + ) + + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + self.logger.debug('Skipping run time') + continue + + # since run_all_times was not called (LOOP_BY=times) then + # get files for current run time + file_dict = self.get_files_from_time(time_info) + all_files = [] + if file_dict: + if isinstance(file_dict, list): + all_files = file_dict + else: + all_files = [file_dict] + + self.c_dict['ALL_FILES'] = all_files - # Run for given init/valid time and forecast lead combination - self.run_at_time_once(time_info) + # Run for given init/valid time and forecast lead combination + self.run_at_time_once(time_info) def get_all_files(self, custom=None): - """! Get all files that can be processed with the app. Some wrappers - like UserScript do not need to obtain a list of possible files. In this - case, the function returns an empty dictionary. + """! Get all files that can be processed with the app. @returns A dictionary where the key is the type of data that was found, i.e. fcst or obs, and the value is a list of files that fit in that category From b7b1283a161ffffaf661a61d87a83c3d8b154b0b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:44:43 -0600 Subject: [PATCH 14/39] updated logic for UserScript to use the input dir/template variables to mimic logic in GridDiag to populate a list of files that are relevant for each run time of the use case --- metplus/wrappers/user_script_wrapper.py | 51 ++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/metplus/wrappers/user_script_wrapper.py b/metplus/wrappers/user_script_wrapper.py index 9d53a86cbb..d64b3fabeb 100755 --- a/metplus/wrappers/user_script_wrapper.py +++ b/metplus/wrappers/user_script_wrapper.py @@ -16,7 +16,7 @@ from ..util import met_util as util from ..util import time_util from . import RuntimeFreqWrapper -from ..util import do_string_sub +from ..util import do_string_sub, getlist '''!@namespace UserScriptWrapper @brief Parent class for wrappers that run over a grouping of times @@ -41,6 +41,15 @@ def create_c_dict(self): self.log_error("Must supply a command to run with " "USER_SCRIPT_COMMAND") + c_dict['INPUT_DIR'] = self.config.getraw('config', + 'USER_SCRIPT_INPUT_DIR', + '') + c_dict['INPUT_TEMPLATES'] = getlist( + self.config.getraw('config', + 'USER_SCRIPT_INPUT_TEMPLATE', + '') + ) + c_dict['IS_MET_CMD'] = False c_dict['LOG_THE_OUTPUT'] = True @@ -79,6 +88,10 @@ def run_at_time_once(self, time_info): or time_info.get('valid') != '*'): time_info = time_util.ti_calculate(time_info) + # create file list text files for the current run time criteria + # set c_dict to the input file dict to set as environment vars + self.c_dict['INPUT_LIST_DICT'] = self.subset_input_files(time_info) + self.set_environment_variables(time_info) # substitute values from dictionary into command @@ -97,9 +110,37 @@ def run_at_time_once(self, time_info): return success - def get_all_files(self, custom=None): - """! Don't get list of all files for UserScript wrapper + def get_files_from_time(self, time_info): + """! Create dictionary containing time information (key time_info) and + any relevant files for that runtime. The parent implementation of + this function creates a dictionary and adds the time_info to it. + This wrapper gets all files for the current runtime and adds it to + the dictionary with keys 'fcst' and 'obs' - @returns True to report that no failures occurred + @param time_info dictionary containing time information + @returns dictionary containing time_info dict and any relevant + files with a key representing a description of that file """ - return True + file_dict = super().get_files_from_time(time_info) + + input_files = self.find_input_files(time_info) + if input_files is None: + return None + + for key, value in input_files.items(): + file_dict[key] = value + + return file_dict + + def set_environment_variables(self, time_info): + """! Set environment variables that will be read set when running this + tool. Wrappers could override it to set wrapper-specific values, + then call super version to handle user configs and printing + + @param time_info dictionary containing timing info from current run + """ + + for identifier, file_path in self.c_dict['INPUT_LIST_DICT'].items(): + self.add_env_var(f'METPLUS_{identifier.upper()}', file_path) + + super().set_environment_variables(time_info) \ No newline at end of file From 581b29cf7db232b7750f84585bfe77545951d3fc Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:10:15 -0600 Subject: [PATCH 15/39] always return True from get_all_files for UserScript because handling file lists is optional --- metplus/wrappers/user_script_wrapper.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metplus/wrappers/user_script_wrapper.py b/metplus/wrappers/user_script_wrapper.py index d64b3fabeb..73a8beaeea 100755 --- a/metplus/wrappers/user_script_wrapper.py +++ b/metplus/wrappers/user_script_wrapper.py @@ -143,4 +143,13 @@ def set_environment_variables(self, time_info): for identifier, file_path in self.c_dict['INPUT_LIST_DICT'].items(): self.add_env_var(f'METPLUS_{identifier.upper()}', file_path) - super().set_environment_variables(time_info) \ No newline at end of file + super().set_environment_variables(time_info) + + def get_all_files(self, custom=None): + """! Call parent function to attempt to get files but always return + True because this functionality is optional + + @returns True + """ + super().get_all_files(custom) + return True From 6d3bf39bd9426372ba7eff6d96284fb378eab054 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:11:22 -0600 Subject: [PATCH 16/39] modified examples to call a script that checks environment variables and prints out the contents of file lists if they are set -- this better demonstrates how to use UserScript and how to get lists of files into a user's script, ci-run-diff --- .../UserScript/UserScript_run_once.conf | 2 +- .../UserScript_run_once_for_each.conf | 2 +- .../UserScript_run_once_per_init.conf | 2 +- .../UserScript_run_once_per_lead.conf | 2 +- .../UserScript_run_once_per_valid.conf | 2 +- .../UserScript/print_file_list.py | 30 +++++++++++++++++++ 6 files changed, 35 insertions(+), 5 deletions(-) create mode 100755 parm/use_cases/met_tool_wrapper/UserScript/print_file_list.py diff --git a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf index ebc7aab556..cf7383f6e6 100644 --- a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf +++ b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf @@ -53,4 +53,4 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE USER_SCRIPT_INPUT_TEMPLATE = init_{init?fmt=%Y%m%d%H%M%S}_valid_{valid?fmt=%Y%m%d%H%M%S}_lead_{lead?fmt=%3H}.{custom} USER_SCRIPT_INPUT_DIR = {INPUT_BASE}/met_test/new/test -USER_SCRIPT_COMMAND = ls {USER_SCRIPT_INPUT_DIR}/{USER_SCRIPT_INPUT_TEMPLATE} \ No newline at end of file +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/met_tool_wrapper/UserScript/print_file_list.py diff --git a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_for_each.conf b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_for_each.conf index 452d40c6dd..9e6691ef1a 100644 --- a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_for_each.conf +++ b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_for_each.conf @@ -53,4 +53,4 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_FOR_EACH USER_SCRIPT_INPUT_TEMPLATE = init_{init?fmt=%Y%m%d%H%M%S}_valid_{valid?fmt=%Y%m%d%H%M%S}_lead_{lead?fmt=%3H}.{custom} USER_SCRIPT_INPUT_DIR = {INPUT_BASE}/met_test/new/test -USER_SCRIPT_COMMAND = ls {USER_SCRIPT_INPUT_DIR}/{USER_SCRIPT_INPUT_TEMPLATE} \ No newline at end of file +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/met_tool_wrapper/UserScript/print_file_list.py diff --git a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_init.conf b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_init.conf index b0360af7e7..15b484615c 100644 --- a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_init.conf +++ b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_init.conf @@ -53,4 +53,4 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_INIT_OR_VALID USER_SCRIPT_INPUT_TEMPLATE = init_{init?fmt=%Y%m%d%H%M%S}_valid_{valid?fmt=%Y%m%d%H%M%S}_lead_{lead?fmt=%3H}.{custom} USER_SCRIPT_INPUT_DIR = {INPUT_BASE}/met_test/new/test -USER_SCRIPT_COMMAND = ls {USER_SCRIPT_INPUT_DIR}/{USER_SCRIPT_INPUT_TEMPLATE} \ No newline at end of file +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/met_tool_wrapper/UserScript/print_file_list.py diff --git a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_lead.conf b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_lead.conf index 0ef270f320..e8b6c95587 100644 --- a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_lead.conf +++ b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_lead.conf @@ -53,4 +53,4 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_LEAD USER_SCRIPT_INPUT_TEMPLATE = init_{init?fmt=%Y%m%d%H%M%S}_valid_{valid?fmt=%Y%m%d%H%M%S}_lead_{lead?fmt=%3H}.{custom} USER_SCRIPT_INPUT_DIR = {INPUT_BASE}/met_test/new/test -USER_SCRIPT_COMMAND = ls {USER_SCRIPT_INPUT_DIR}/{USER_SCRIPT_INPUT_TEMPLATE} \ No newline at end of file +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/met_tool_wrapper/UserScript/print_file_list.py diff --git a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_valid.conf b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_valid.conf index 5208e79812..fd80ab3573 100644 --- a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_valid.conf +++ b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once_per_valid.conf @@ -53,4 +53,4 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_INIT_OR_VALID USER_SCRIPT_INPUT_TEMPLATE = init_{init?fmt=%Y%m%d%H%M%S}_valid_{valid?fmt=%Y%m%d%H%M%S}_lead_{lead?fmt=%3H}.{custom} USER_SCRIPT_INPUT_DIR = {INPUT_BASE}/met_test/new/test -USER_SCRIPT_COMMAND = ls {USER_SCRIPT_INPUT_DIR}/{USER_SCRIPT_INPUT_TEMPLATE} \ No newline at end of file +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/met_tool_wrapper/UserScript/print_file_list.py diff --git a/parm/use_cases/met_tool_wrapper/UserScript/print_file_list.py b/parm/use_cases/met_tool_wrapper/UserScript/print_file_list.py new file mode 100755 index 0000000000..d9ecdb3c43 --- /dev/null +++ b/parm/use_cases/met_tool_wrapper/UserScript/print_file_list.py @@ -0,0 +1,30 @@ +#! /usr/bin/env python3 + +import os + +# Read environment variables that start with METPLUS_INPUT and +# print list of files found in text file set as value + +# check inputs 0-9 +for index in range(0, 10): + env_var_name = f'METPLUS_INPUT{index}' + print(f'Checking environment variable: {env_var_name}') + file_list_path = os.environ.get(env_var_name) + if not file_list_path: + print(f'{env_var_name} is not set or empty') + continue + + if not os.path.exists(file_list_path): + print(f'File does not exist: {file_list_path}') + continue + + with open(file_list_path, 'r') as file_handle: + input_lines = file_handle.read().splitlines() + + print(f'Contents of {file_list_path}') + for line in input_lines: + print(line) + + print() + +print(f'End of {__file__} script') From 9d0a68983aa136b81db8299c5c916e0a1820b193 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:35:59 -0600 Subject: [PATCH 17/39] don't loop to find files if time info is not set --- metplus/wrappers/runtime_freq_wrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 0cb729e4d9..78567b2abb 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -251,6 +251,10 @@ def get_all_files(self, custom=None): self.logger.debug("Finding all input files") all_files = [] + # if start time is not set, don't loop + if not self.c_dict.get('START_TIME'): + return False + # loop over all init/valid times loop_time = self.c_dict['START_TIME'] while loop_time <= self.c_dict['END_TIME']: From 175a56248af05ddc80659c5623491efb6b26cbf4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:36:15 -0600 Subject: [PATCH 18/39] rerun use case that failed --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index c5c83157a9..965463ca13 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -117,7 +117,7 @@ { "category": "s2s", "index_list": "5", - "new": false + "new": true }, { "category": "s2s", From 4fbd593ec74afb5d3f5c49de94ce444de98d9b5c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 13:01:58 -0600 Subject: [PATCH 19/39] trigger other failing use cases to run --- .github/parm/use_case_groups.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 965463ca13..d7bc67b8f4 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -67,7 +67,7 @@ { "category": "medium_range", "index_list": "3-5", - "new": false + "new": true }, { "category": "medium_range", @@ -107,12 +107,12 @@ { "category": "s2s", "index_list": "1-3", - "new": false + "new": true }, { "category": "s2s", "index_list": "4", - "new": false + "new": true }, { "category": "s2s", @@ -122,7 +122,7 @@ { "category": "s2s", "index_list": "6", - "new": false + "new": true }, { "category": "space_weather", @@ -132,7 +132,7 @@ { "category": "tc_and_extra_tc", "index_list": "0-2", - "new": false + "new": true }, { "category": "tc_and_extra_tc", From 819b1ec70ee0473902b85f956a18fc98fe5a64b5 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 13:36:16 -0600 Subject: [PATCH 20/39] changed name of config variable because name is now a reserved config variable --- .../UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf index d524045343..9a8d5f10a3 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf @@ -6,7 +6,7 @@ PROCESS_LIST = UserScript(untar_drop_file), Ascii2nc, PointStat USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_INIT_OR_VALID USER_SCRIPT_INPUT_DIR = {INPUT_BASE}/model_applications/tc_and_extra_tc/dropsonde/obs USER_SCRIPT_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/tc_and_extra_tc/dropsonde/obs -USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF/hrd_frd_sonde_find_tar.py {USER_SCRIPT_INPUT_TEMPLATE} +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF/hrd_frd_sonde_find_tar.py {USER_SCRIPT_ARGUMENTS} ASCII2NC_INPUT_FORMAT = python ASCII2NC_TIME_SUMMARY_FLAG = False @@ -106,7 +106,7 @@ POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/{OBTYPE} [filename_templates] -USER_SCRIPT_INPUT_TEMPLATE = {USER_SCRIPT_INPUT_DIR} {valid?fmt=%Y%m%d} {USER_SCRIPT_OUTPUT_DIR} +USER_SCRIPT_ARGUMENTS = {USER_SCRIPT_INPUT_DIR} {valid?fmt=%Y%m%d} {USER_SCRIPT_OUTPUT_DIR} ASCII2NC_OUTPUT_TEMPLATE = drop{valid?fmt=%Y%m%d}.nc FCST_POINT_STAT_INPUT_TEMPLATE = hafs.{valid?fmt=%Y%m%d%H}/dorian05l.{init?fmt=%Y%m%d%H}.hafsprs.synoptic.TMP600-900.0p03.f{lead?fmt=%3H}.grb2 OBS_POINT_STAT_INPUT_TEMPLATE = {ASCII2NC_OUTPUT_TEMPLATE} From ec13027d384a23e111213700b6a4c2ab94ad62ef Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Jul 2021 13:37:54 -0600 Subject: [PATCH 21/39] skip use cases that are no longer failing --- .github/parm/use_case_groups.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index d7bc67b8f4..8589e14f0a 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -67,7 +67,7 @@ { "category": "medium_range", "index_list": "3-5", - "new": true + "new": false }, { "category": "medium_range", @@ -112,17 +112,17 @@ { "category": "s2s", "index_list": "4", - "new": true + "new": false }, { "category": "s2s", "index_list": "5", - "new": true + "new": false }, { "category": "s2s", "index_list": "6", - "new": true + "new": false }, { "category": "space_weather", From 7bac06e65df9ff263f4852edd6044488c6f7d7a6 Mon Sep 17 00:00:00 2001 From: Christina Kalb Date: Fri, 16 Jul 2021 15:26:50 -0600 Subject: [PATCH 22/39] Fixed a typo --- .../Blocking_WeatherRegime_util.py | 4 +-- ...erScript_obsERA_obsOnly_WeatherRegime.conf | 23 ++++++------ .../WeatherRegime_driver.py | 35 ++++++++++--------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py index f73d77318e..d62b3b7bba 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Blocking/Blocking_WeatherRegime_util.py @@ -2,7 +2,7 @@ import netCDF4 import numpy as np import datetime -from metplus.util import pre_run_setup, config_metplus +#from metplus.util import pre_run_setup, config_metplus def parse_steps(): @@ -13,7 +13,7 @@ def parse_steps(): steps_param_obs = os.environ.get('OBS_STEPS','') steps_list_obs = steps_param_obs.split("+") - return steps_list_fcst, steps_list_obs + return steps_list_fcst, steps_list_obs def write_mpr_file(data_obs,data_fcst,lats_in,lons_in,time_obs,time_fcst,mname,fvar,flev,ovar,olev,maskname,obslev,outfile): diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf index f5794b1b41..d405602093 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf @@ -4,7 +4,8 @@ # All steps, including pre-processing: # PROCESS_LIST = RegridDataPlane(regrid_obs), PcpCombine(daily_mean_obs), UserScript(obs_wr_filelist), UserScript(script_wr) # Weather Regime Analysis only: -PROCESS_LIST = UserScript(obs_wr_filelist), UserScript(script_wr) +#PROCESS_LIST = UserScript(obs_wr_filelist), UserScript(script_wr) +PROCESS_LIST = UserScript(script_wr) # time looping - options are INIT, VALID, RETRO, and REALTIME # If set to INIT or RETRO: @@ -50,11 +51,11 @@ LOOP_ORDER = processes # location of configuration files used by MET applications CONFIG_DIR={PARM_BASE}/use_cases/model_applications/s2s -OBS_WR_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily -OBS_WR_INPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d}_NH.nc +#OBS_WR_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily +#OBS_WR_INPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d}_NH.nc #OBS_WR_OUTPUT_DIR = {OBS_WR_INPUT_DIR} -OBS_WR_OUTPUT_DIR = {OUTPUT_BASE} -OBS_WR_OUTPUT_TEMPLATE = ERA_daily_files_lead{lead?fmt=%HHH}.txt +#OBS_WR_OUTPUT_DIR = {OUTPUT_BASE} +#OBS_WR_OUTPUT_TEMPLATE = ERA_daily_files_lead{lead?fmt=%HHH}.txt # Regridding Pre-Processing Step @@ -145,11 +146,11 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d?shift=-64800}_NH. # Get a File list for the OBS IBL data (daily mean files) -[obs_wr_filelist] +#[obs_wr_filelist] # Find the files for each time -USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_FOR_EACH - -USER_SCRIPT_COMMAND = {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py {OBS_WR_INPUT_DIR}/{OBS_WR_INPUT_TEMPLATE} {OBS_WR_OUTPUT_DIR}/{OBS_WR_OUTPUT_TEMPLATE} +#USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_FOR_EACH +# +#USER_SCRIPT_COMMAND = {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking/save_input_files_txt_missing.py {OBS_WR_INPUT_DIR}/{OBS_WR_INPUT_TEMPLATE} {OBS_WR_OUTPUT_DIR}/{OBS_WR_OUTPUT_TEMPLATE} # Variables for the Weather Regime code @@ -168,7 +169,7 @@ NUM_SEASONS = 38 DAYS_PER_SEASON = 90 # Text files containing a list of input files for the IBL code -OBS_WR_INPUT_TEXTFILE = {OBS_WR_OUTPUT_DIR}/{OBS_WR_OUTPUT_TEMPLATE} +#OBS_WR_INPUT_TEXTFILE = {OBS_WR_OUTPUT_DIR}/{OBS_WR_OUTPUT_TEMPLATE} # Boolean indicating whether to keep the above text file or delete it at the end of the run KEEP_WR_FILE_LISTING = True @@ -229,5 +230,7 @@ USER_SCRIPT_CUSTOM_LOOP_LIST = nc # Run the user script once USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_LEAD +USER_SCRIPT_INPUT_TEMPLATE = {INPUT_BASE}/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/ERA/Daily/Z500_daily_{valid?fmt=%Y%m%d}_NH.nc + # Command to run the user script with input configuration file USER_SCRIPT_COMMAND = {METPLUS_BASE}/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py index ad9d503571..3bbec6102d 100755 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py @@ -4,7 +4,7 @@ import numpy as np import netCDF4 import warnings -import atexit +#import atexit from WeatherRegime import WeatherRegimeCalculation from metplus.util import getlist @@ -12,17 +12,17 @@ from Blocking_WeatherRegime_util import parse_steps, read_nc_met, write_mpr_file -def cleanup_daily_files(obs_dailyfile, fcst_dailyfile, keep_daily_files): - if keep_daily_files == 'false': - try: - os.remove(obs_anomfile) - except: - pass - - try: - os.remove(fcst_anomfile) - except: - pass +#def cleanup_daily_files(obs_dailyfile, fcst_dailyfile, keep_daily_files): +# if keep_daily_files == 'false': +# try: +# os.remove(obs_anomfile) +# except: +# pass +# +# try: +# os.remove(fcst_anomfile) +# except: +# pass def main(): @@ -59,15 +59,17 @@ def main(): dseasons = int(os.environ['DAYS_PER_SEASON']) # Grab the Daily text files - obs_wr_filetxt = os.environ.get('OBS_WR_INPUT_TEXTFILE','') - fcst_wr_filetxt = os.environ.get('FCST_WR_INPUT_TEXTFILE','') - keep_wr_textfile = os.environ.get('KEEP_WR_FILE_LISTING', 'False').lower() - atexit.register(cleanup_daily_files, obs_wr_filetxt, fcst_wr_filetxt, keep_wr_textfile) + obs_wr_filetxt = os.environ.get('METPLUS_INPUT0','') + fcst_wr_filetxt = os.environ.get('METPLUS_INPUT1','') + #keep_wr_textfile = os.environ.get('KEEP_WR_FILE_LISTING', 'False').lower() + #atexit.register(cleanup_daily_files, obs_wr_filetxt, fcst_wr_filetxt, keep_wr_textfile) if ("ELBOW" in steps_list_obs) or ("EOF" in steps_list_obs) or ("KMEANS" in steps_list_obs): with open(obs_wr_filetxt) as owl: obs_infiles = owl.read().splitlines() + # Remove the first line + obs_infiles = obs_infiles[1:] if len(obs_infiles) != (nseasons*dseasons): raise Exception('Invalid Obs data; each year must contain the same date range to calculate seasonal averages.') obs_invar = os.environ.get('OBS_WR_VAR','') @@ -77,6 +79,7 @@ def main(): if ("ELBOW" in steps_list_fcst) or ("EOF" in steps_list_fcst) or("KMEANS" in steps_list_fcst): with open(fcst_wr_filetxt) as fwl: fcst_infiles = fwl.read().splitlines() + fcst_infiles = fcst_infiles[1:] if len(fcst_infiles) != (nseasons*dseasons): raise Exception('Invalid Obs data; each year must contain the same date range to calculate seasonal averages.') fcst_invar = os.environ.get('FCST_WR_VAR','') From f20fca31dffdefb6a73526796f344f675b7af2b6 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Jul 2021 11:34:32 -0600 Subject: [PATCH 23/39] add skip times logic to function that finds all files to process --- metplus/wrappers/runtime_freq_wrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 78567b2abb..5ccb34e931 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -276,6 +276,10 @@ def get_all_files(self, custom=None): # set current lead time config and environment variables time_info = time_util.ti_calculate(input_dict) + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + self.logger.debug('Skip finding files from run time') + continue + file_dict = self.get_files_from_time(time_info) if file_dict: if isinstance(file_dict, list): From 934bb4aaa2ab2e42effeb859e104f4e9381212ba Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Jul 2021 12:05:03 -0600 Subject: [PATCH 24/39] added logic to put 'missing' for files that are not found for a given run time (for UserScript) --- metplus/wrappers/runtime_freq_wrapper.py | 16 +++++++++++++--- metplus/wrappers/user_script_wrapper.py | 4 ++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 5ccb34e931..6e6b404c47 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -337,19 +337,29 @@ def compare_time_info(self, runtime, filetime): return runtime_lead == filetime_lead - def find_input_files(self, time_info): + def find_input_files(self, time_info, fill_missing=False): """! Loop over list of input templates and find files for each @param time_info time dictionary to use for string substitution + @param fill_missing if True, add a placeholder if a file is not + found. Defaults to False. @returns Dictionary of key input number and value is list of input file list if all files were found, None if not. """ all_input_files = {} for idx, input_template in enumerate(self.c_dict['INPUT_TEMPLATES']): self.c_dict['INPUT_TEMPLATE'] = input_template - input_files = self.find_data(time_info, return_list=True) + # if fill missing is true, data is not mandatory to find + mandatory = not fill_missing + input_files = self.find_data(time_info, + return_list=True, + mandatory=mandatory) if not input_files: - continue + if not fill_missing: + continue + + # if no files are found and fill missing is set, add 'missing' + input_files = ['missing'] all_input_files[f'input{idx}'] = input_files diff --git a/metplus/wrappers/user_script_wrapper.py b/metplus/wrappers/user_script_wrapper.py index 73a8beaeea..d678d640b7 100755 --- a/metplus/wrappers/user_script_wrapper.py +++ b/metplus/wrappers/user_script_wrapper.py @@ -123,9 +123,9 @@ def get_files_from_time(self, time_info): """ file_dict = super().get_files_from_time(time_info) - input_files = self.find_input_files(time_info) + input_files = self.find_input_files(time_info, fill_missing=True) if input_files is None: - return None + return file_dict for key, value in input_files.items(): file_dict[key] = value From 004be4663e70a2f194514a25afb37f238c8ff76f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Jul 2021 12:06:03 -0600 Subject: [PATCH 25/39] added an extra lead time to example to demonstrate handling of missing files --- .../met_tool_wrapper/UserScript/UserScript_run_once.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf index cf7383f6e6..d0484030b4 100644 --- a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf +++ b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf @@ -32,7 +32,7 @@ VALID_INCREMENT = 12H # List of forecast leads to process for each run time (init or valid) # In hours if units are not specified # If unset, defaults to 0 (don't loop through forecast leads) -LEAD_SEQ = 0H, 12H, 24H, 120H +LEAD_SEQ = 0H, 12H, 15H, 24H, 120H # Order of loops to process data - Options are times, processes # Not relevant if only one item is in the PROCESS_LIST From e0f6886e31415c6e516b1a5fbf5683cbabd9a467 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Jul 2021 12:55:43 -0600 Subject: [PATCH 26/39] added logic to specify label for each input template file list, changed env var from METPLUS_