diff --git a/CHANGELOG.md b/CHANGELOG.md index cec8d6f7..daa19ef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added EASE grid option for remapping of land restarts in remap_restarts.py package (facilitates use of package in GEOSldas setup script) +- Added support for SLES15, NAS site and log for remap_lake_landice_saltwater in remap_restarts.py package +- Added "land_only" option for remapping of restarts + ### Changed ### Fixed diff --git a/pre/CMakeLists.txt b/pre/CMakeLists.txt index 4653c7c6..6605e1ee 100644 --- a/pre/CMakeLists.txt +++ b/pre/CMakeLists.txt @@ -2,9 +2,16 @@ add_subdirectory(NSIDC-OSTIA_SST-ICE_blend) add_subdirectory(prepare_ocnExtData) file(GLOB pythonscripts CONFIGURE_DEPENDS ./remap_restart/remap* ./remap_restart/bin2nc_merra2* ./remap_restart/tests/*.*) -install( - PROGRAMS ${pythonscripts} - DESTINATION bin) + +list(FILTER pythonscripts EXCLUDE REGEX "remap_utils.py") + +foreach (file ${pythonscripts}) + install(PROGRAMS ${file} DESTINATION bin) +endforeach() + +set(file ./remap_restart/remap_utils.py) +configure_file(${file} ${file} @ONLY) +install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${file} DESTINATION bin) install( PROGRAMS regrid.pl diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index 990456bc..f9ee8814 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -7,17 +7,14 @@ import ruamel.yaml import shutil import subprocess +from remap_utils import * class remap_base(object): def __init__(self, **configs): for key, value in configs.items(): if (key == 'params_file'): print( "use Config yaml file: " + value) - yaml = ruamel.yaml.YAML() - stream ='' - with open(value, 'r') as f: - stream = f.read() - self.config = yaml.load(stream) + self.config = get_config_from_file(value) out_dir = self.config['output']['shared']['out_dir'] if not os.path.exists(out_dir) : os.makedirs(out_dir) f = os.path.basename(value) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index c374f493..59af8629 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -18,9 +18,7 @@ import mimetypes import netCDF4 as nc from remap_base import remap_base -from remap_utils import get_landdir -from remap_utils import get_geomdir -from remap_utils import get_label +from remap_utils import * class catchANDcn(remap_base): def __init__(self, **configs): @@ -39,7 +37,7 @@ def remap(self): time = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] in_rstfiles = glob.glob(rst_dir+'/*'+model+'_*'+time+'*') if len(in_rstfiles) == 0: - print('\n try catchXX file without time stamp') + print('\n try catch[cn] restart file without time stamp') in_rstfiles = glob.glob(rst_dir+'/*'+model+'_*') if len(in_rstfiles) == 0: return @@ -60,12 +58,17 @@ def remap(self): surflay = config['output']['surface']['surflay'] in_tilefile = config['input']['surface']['catch_tilefile'] + if "gmao_SIteam/ModelData" in out_bc_base: + assert GEOS_SITE == "NAS", "wrong site to run the package" + if not in_tilefile : agrid = config['input']['shared']['agrid'] ogrid = config['input']['shared']['ogrid'] omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - bc_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) + EASE_grid = config['input']['surface'].get('EASE_grid', None) + + bc_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) in_tilefile = glob.glob(bc_geomdir + '/*.til')[0] agrid = config['output']['shared']['agrid'] @@ -73,12 +76,13 @@ def remap(self): omodel = config['output']['shared']['omodel'] stretch = config['output']['shared']['stretch'] out_tilefile = config['output']['surface']['catch_tilefile'] + EASE_grid = config['output']['surface'].get('EASE_grid', None) if not out_tilefile : - bc_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) + bc_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) out_tilefile = glob.glob(bc_geomdir+ '/*.til')[0] - out_bc_landdir = get_landdir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) + out_bc_landdir = get_landdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) # determine NPE based on *approximate* number of input and output tile @@ -104,14 +108,28 @@ def remap(self): else: NPE = 160 - QOS = "#SBATCH --qos="+config['slurm']['qos'] - TIME ="#SBATCH --time=1:00:00" - if NPE >= 160: - assert config['slurm']['qos'] != 'debug', "qos should be allnccs" - TIME = "#SBATCH --time=12:00:00" - PARTITION = "#SBATCH --partition=" + config['slurm']['partition'] - - account = config['slurm']['account'] + PARTITION ='' + QOS = config['slurm_pbs']['qos'] + TIME = "1:00:00" + if QOS != "debug": TIME="12:00:00" + + NNODE = '' + job = '' + if GEOS_SITE == 'NAS': + job = "PBS" + CONSTRAINT = 'cas_ait' + NNODE = (NPE-1)//40 + 1 + else: + job = "SLURM" + partition = config['slurm_pbs']['partition'] + if (partition != ''): + PARTITION = "#SBATCH --partition=" + partition + + CONSTRAINT = '"[cas|sky]"' + if BUILT_ON_SLES15: + CONSTRAINT = 'mil' + + account = config['slurm_pbs']['account'] # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py) label = get_label(config) @@ -140,18 +158,10 @@ def remap(self): in_rstfile = dest log_name = out_dir+'/'+'mk_catchANDcn_log' - mk_catch_j_template = """#!/bin/csh -f -#SBATCH --account={account} -#SBATCH --ntasks={NPE} -#SBATCH --job-name=mk_catchANDcn -#SBATCH --output={log_name} -{TIME} -{QOS} -{PARTITION} -# - + job_name = "mk_catchANDcn" + mk_catch_j_template = job_directive[job]+ \ +""" source {Bin}/g5_modules - limit stacksize unlimited set esma_mpirun_X = ( {Bin}/esma_mpirun -np {NPE} ) @@ -165,36 +175,45 @@ def remap(self): """ catch1script = mk_catch_j_template.format(Bin = bindir, account = account, out_bcs = out_bc_landdir, \ - model = model, out_dir = out_dir, surflay = surflay, log_name = log_name, NPE = NPE, \ + model = model, out_dir = out_dir, surflay = surflay, log_name = log_name, job_name = job_name, \ + NPE = NPE, NNODE=NNODE, \ in_wemin = in_wemin, out_wemin = out_wemin, out_tilefile = out_tilefile, in_tilefile = in_tilefile, \ - in_rstfile = in_rstfile, out_rstfile = out_rstfile, time = yyyymmddhh_, TIME = TIME, PARTITION = PARTITION, QOS=QOS ) + in_rstfile = in_rstfile, out_rstfile = out_rstfile, time = yyyymmddhh_, TIME = TIME, QOS=QOS, CONSTRAINT=CONSTRAINT, PARTITION=PARTITION ) script_name = './mk_catchANDcn.j' catch_scrpt = open(script_name,'wt') catch_scrpt.write(catch1script) catch_scrpt.close() - - interactive = os.getenv('SLURM_JOB_ID', default = None) + + interactive = None + if GEOS_SITE == 'NAS': + interactive = os.getenv('PBS_JOBID', default = None) + else: + interactive = os.getenv('SLURM_JOB_ID', default = None) + if ( interactive ) : print('interactive mode\n') - ntasks = os.getenv('SLURM_NTASKS', default = None) - if ( not ntasks): - nnodes = int(os.getenv('SLURM_NNODES', default = '1')) - ncpus = int(os.getenv('SLURM_CPUS_ON_NODE', default = '40')) - ntasks = nnodes * ncpus - ntasks = int(ntasks) + if GEOS_SITE != "NAS": + ntasks = os.getenv('SLURM_NTASKS', default = None) + if ( not ntasks): + nnodes = int(os.getenv('SLURM_NNODES', default = '1')) + ncpus = int(os.getenv('SLURM_CPUS_ON_NODE', default = '40')) + ntasks = nnodes * ncpus + ntasks = int(ntasks) - if (ntasks < NPE): - print("\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ".format(NPE=NPE, ntasks=ntasks)) + if (ntasks < NPE): + print("\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ".format(NPE=NPE, ntasks=ntasks)) subprocess.call(['chmod', '755', script_name]) print(script_name+ ' 1>' + log_name + ' 2>&1') os.system(script_name + ' 1>' + log_name+ ' 2>&1') - + elif GEOS_SITE == "NAS" : + print('qsub -W block=true '+ script_name +'\n') + subprocess.call(['qsub', '-W','block=true', script_name]) else: - print("sbatch -W " + script_name +"\n") - subprocess.call(['sbatch','-W', script_name]) + print('sbatch -W '+ script_name +'\n') + subprocess.call(['sbatch', '-W', script_name]) print( "cd " + cwdir) os.chdir(cwdir) @@ -225,6 +244,195 @@ def copy_merra2(self): print("Copy file "+f +" to " + rst_dir) shutil.copy(f, dest) -if __name__ == '__main__' : - catch = catchANDcn(params_file='remap_params.yaml') +def ask_catch_questions(): + catch_input_shared_rst_dir = '' + def has_rs_rc_out(path): + if os.path.exists(path): + dirs = os.listdir(path) + if 'rs' in dirs and 'rc_out' in dirs: + nonlocal catch_input_shared_rst_dir + catch_input_shared_rst_dir = path + return True + return False + + def has_catch_rst(text): + if len(text) !=10: return False + yyyy = text[0:4] + mm = text[4:6] + dd = text[6:8] + hh = text[8:10] + nonlocal catch_input_shared_rst_dir + rst_dir = catch_input_shared_rst_dir + '/rs/ens0000/Y'+yyyy +'/M'+mm+'/' + rst_files = glob.glob(rst_dir+'*catch*_internal_rst.'+yyyy+mm+dd+'_'+hh+'00') + if len(rst_files) !=1 : + return False + return True + + questions =[ + { + "type": "path", + "name": "input:shared:rst_dir", + "message": "Enter input directory that contains './rs' and './rc_out' subdirectories:\n", + "validate": lambda path : has_rs_rc_out(path) + }, + + { + "type": "text", + "name": "input:shared:yyyymmddhh", + "message": (message_datetime + " and rst file must exist.)\n"), + "validate": lambda text: has_catch_rst(text) + }, + + { + "type": "path", + "name": "output:shared:out_dir", + "message": message_out_dir, + }, + + { + "type": "text", + "name": "output:shared:expid", + "message": message_expid, + "default": "", + }, + + { + "type": "select", + "name": "output:shared:bc_base", + "message": ("\nSelect " + message_bc_base_new), + "choices": choices_bc_base, + "default": get_default_bc_base(), + }, + + { + "type": "select", + "name": "output:shared:bc_version", + "message": message_bc_ops_new, + "choices": choices_bc_ops, + "default": "NL3", + }, + { + "type": "select", + "name": "output:shared:bc_version", + "message": message_bc_other_new, + "choices": choices_bc_other, + "when": lambda x: x["output:shared:bc_version"] == 'Other', + }, + + { + "type": "select", + "name": "output:surface:EASE_grid", + "message": "Select grid for new restart:\n", + "choices": ['EASEv2_M03', 'EASEv2_M09', 'EASEv2_M25', 'EASEv2_M36', 'Cubed-Sphere'] + }, + + { + "type": "text", + "name": "output:shared:agrid", + "message": message_agrid_new, + "default": 'C360', + "validate": lambda text : text in validate_agrid, + "when": lambda x : x['output:surface:EASE_grid'] == 'Cubed-Sphere', + }, + + { + "type": "select", + "name": "output:shared:ogrid", + "message": message_ogrid_in, + "choices": choices_ogrid_data, + "default": lambda x: data_ocean_default(x.get('output:shared:agrid')), + "when": lambda x : x['output:surface:EASE_grid'] == 'Cubed-Sphere', + }, + + # dummy (invisible) question to remove parenthetical comments from selected output:shared:ogrid + { + "type": "text", + "name": "output:shared:ogrid", + "message": "remove the comment of ogrid", + # always return false, so questions never shows but changes ogrid + "when": lambda x: remove_ogrid_comment(x, 'OUT') + }, + + + { + "type": "text", + "name": "slurm_pbs:qos", + "message": message_qos, + "default": "debug", + }, + + { + "type": "text", + "name": "slurm_pbs:account", + "message": message_account, + "default": get_account(), + }, + { + "type": "text", + "name": "slurm_pbs:partition", + "message": message_partition, + "default": '', + }, + + ] + + answers = questionary.prompt(questions) + yyyy = answers['input:shared:yyyymmddhh'][0:4] + mm = answers['input:shared:yyyymmddhh'][4:6] + dd = answers['input:shared:yyyymmddhh'][6:8] + hh = answers['input:shared:yyyymmddhh'][8:10] + + rst_dir = answers['input:shared:rst_dir']+'/rs/ens0000/Y'+yyyy +'/M'+mm+'/' + rst_file = glob.glob(rst_dir+'*catch*_internal_rst.'+yyyy+mm+dd+'_'+hh+'00')[0] + idx1 = rst_file.find('catch') + idx2 = rst_file.find('_internal_rst') + catch_model = rst_file[idx1:idx2] + + tile_dir = answers['input:shared:rst_dir']+'/rc_out/' + in_tilefiles = glob.glob(tile_dir+'*.til') + for file in in_tilefiles: + answers['input:surface:catch_tilefile'] = file + if 'MAPL_' in file: + answers['input:surface:catch_tilefile'] = file + + answers['input:shared:rst_dir'] = rst_dir + answers['input:surface:catch_model'] = catch_model + answers['input:surface:wemin'] = "13" + answers['output:surface:wemin'] = "13" + answers['output:surface:remap_catch'] = True + bc_base= answers['output:shared:bc_base'].split(": ")[-1] + answers['output:shared:bc_base'] = bc_base + answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) + + if answers['output:surface:EASE_grid'] == 'Cubed-Sphere' : + remove_ogrid_comment(answers, 'OUT') + del answers['output:surface:EASE_grid'] + return answers + +def remap_land_only(): + print("\n Remap land (catch[cn]) restart only; global domain only; ens0000 only. \n") + answers = ask_catch_questions() + config = get_config_from_answers(answers, config_tpl = True) + config_yaml = config['output']['shared']['out_dir']+'/remap_params.yaml' + config_to_yaml(config, config_yaml) + print_config(config) + + question = [ + { + "type": "confirm", + "name": "Continue", + "message": "Above is the YAML config file, would you like to continue?", + "default": True + },] + answer = questionary.prompt(question) + + if not answer['Continue'] : + print("\nYou answered not to continue, exiting.\n") + sys.exit(0) + + catch = catchANDcn(params_file=config_yaml) catch.remap() + +if __name__ == '__main__' : + + remap_land_only() diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 2a7693ff..9b59632b 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -13,7 +13,6 @@ import glob import argparse from remap_utils import * -from remap_params import * def parse_args(program_description): @@ -30,6 +29,11 @@ def parse_args(program_description): p_config.add_argument('-c', '--config_file', help='YAML config file') + p_landonly = p_sub.add_parser( + 'land_only', + help = "Remap land (catch[cn]) restart only; global domain only; ens0000 only.\n", + ) + p_command = p_sub.add_parser( 'command_line', help = "Use command line as input", @@ -72,10 +76,10 @@ def parse_args(program_description): p_command.add_argument('-out_bc_base',default="", help="Boundary conditions base dir (w/o bc_version and resolution info) for new restarts") p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") - p_command.add_argument('-qos', default="debug", help="SLURM quality-of-service", choices=['debug', 'allnccs']) + p_command.add_argument('-qos', default="debug", help="slurm_pbs quality-of-service", choices=['debug', 'allnccs', 'normal']) account = get_account() - p_command.add_argument('-account', default=account, help="SLURM account") - p_command.add_argument('-partition', default='compute', help="SLURM partition") + p_command.add_argument('-account', default=account, help="slurm_pbs account") + p_command.add_argument('-partition', default='', help="slurm_pbs partition") p_command.add_argument('-rs', default='3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts @@ -121,12 +125,17 @@ def get_answers_from_command_line(cml): answers["output:analysis:bkg"] = not cml.nobkg answers["output:analysis:lcv"] = not cml.nolcv if cml.rs == '1': - answers["output:air:remap"] = True + answers["output:air:remap"] = True + answers["output:surface:remap_water"] = False + answers["output:surface:remap_catch"] = False if cml.rs == '2': - answers["output:surface:remap"] = True + answers["output:air:remap"] = False + answers["output:surface:remap_water"] = True + answers["output:surface:remap_catch"] = True if cml.rs == '3': - answers["output:surface:remap"] = True - answers["output:air:remap"] = True + answers["output:air:remap"] = True + answers["output:surface:remap_water"] = True + answers["output:surface:remap_catch"] = True answers["output:air:agcm_import_rst"] = not cml.noagcm_import_rst @@ -146,9 +155,9 @@ def get_answers_from_command_line(cml): else: answers["output:surface:wemin"] = wemin_default(answers['output:shared:bc_version']) - answers["slurm:account"] = cml.account - answers["slurm:qos"] = cml.qos - answers["slurm:partition"] = cml.partition + answers["slurm_pbs:account"] = cml.account + answers["slurm_pbs:qos"] = cml.qos + answers["slurm_pbs:partition"] = cml.partition return answers @@ -162,7 +171,7 @@ def get_answers_from_command_line(cml): with open("raw_command.yaml", "w") as f: yaml.dump(config, f) - params = remap_params(config) + config = get_config_from_answers(answers, config_tpl= True) with open("params_from_command.yaml", "w") as f: - yaml.dump(params.config, f) + yaml.dump(config, f) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index 486c4218..5603e1c0 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -10,7 +10,7 @@ # source g5_modules [csh] # import os -import subprocess +import subprocess as sp import shutil import glob import ruamel.yaml @@ -49,17 +49,17 @@ def remap(self): os.chdir(out_dir) InData_dir = out_dir+'/InData/' - if os.path.exists(InData_dir) : subprocess.call(['rm', '-rf',InData_dir]) + if os.path.exists(InData_dir) : sp.call(['rm', '-rf',InData_dir]) print ("mkdir " + InData_dir) os.makedirs(InData_dir) OutData_dir = out_dir+'/OutData/' - if os.path.exists(OutData_dir) : subprocess.call(['rm', '-rf',OutData_dir]) + if os.path.exists(OutData_dir) : sp.call(['rm', '-rf',OutData_dir]) print ("mkdir " + OutData_dir) os.makedirs(OutData_dir) types = '.bin' - type_str = subprocess.check_output(['file','-b', restarts_in[0]]) + type_str = sp.check_output(['file','-b', os.path.realpath(restarts_in[0])]) type_str = str(type_str) if 'Hierarchical' in type_str: types = '.nc4' @@ -91,14 +91,14 @@ def remap(self): ogrid = config['input']['shared']['ogrid'] omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - in_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) + in_geomdir = get_geomdir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) in_tile_file = glob.glob(in_geomdir+ '/*-Pfafstetter.til')[0] agrid = config['output']['shared']['agrid'] ogrid = config['output']['shared']['ogrid'] omodel = config['output']['shared']['omodel'] stretch = config['output']['shared']['stretch'] - out_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) + out_geomdir = get_geomdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) out_tile_file = glob.glob(out_geomdir+ '/*-Pfafstetter.til')[0] in_til = InData_dir+'/' + os.path.basename(in_tile_file) @@ -113,46 +113,44 @@ def remap(self): exe = bindir + '/mk_LakeLandiceSaltRestarts.x ' zoom = config['input']['surface']['zoom'] + log_name = out_dir+'/remap_lake_landice_saltwater_log' + if os.path.exists(log_name): + os.remove(log_name) if (saltwater): cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater + ' 0 ' + str(zoom) - print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + self.run_and_log(cmd, log_name) # split Saltwater if config['output']['surface']['split_saltwater']: print("\nSplitting Saltwater...\n") cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater - print('\n'+cmd) - subprocess.call(shlex.split(cmd)) +# subprocess.call(shlex.split(cmd)) openwater = '' seaice = '' + self.run_and_log(cmd, log_name) if (openwater): cmd = exe + out_til + ' ' + in_til + ' InData/' + openwater + ' 0 ' + str(zoom) - print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + self.run_and_log(cmd, log_name) if (seaice): cmd = exe + out_til + ' ' + in_til + ' InData/' + seaice + ' 0 ' + str(zoom) print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + self.run_and_log(cmd, log_name) if (lake): cmd = exe + out_til + ' ' + in_til + ' InData/' + lake + ' 19 ' + str(zoom) - print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + self.run_and_log(cmd, log_name) if (landice): cmd = exe + out_til + ' ' + in_til + ' InData/' + landice + ' 20 ' + str(zoom) - print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + self.run_and_log(cmd, log_name) if (route): route = bindir + '/mk_RouteRestarts.x ' cmd = route + out_til + ' ' + yyyymmddhh_[0:6] - print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + self.run_and_log(cmd, log_name) expid = config['output']['shared']['expid'] if (expid) : @@ -169,6 +167,19 @@ def remap(self): self.remove_merra2() + def run_and_log(self, cmd, log_name): + print('\n'+cmd) + process =sp.Popen(shlex.split(cmd), stdout=sp.PIPE, stderr=sp.PIPE) + stdout, stderr = process.communicate() + stdout = stdout.decode() + stderr = stderr.decode() + print('\n'+ stdout) + print('\n'+ stderr) + with open(log_name, "a") as log_: + log_.write(cmd) + log_.write(stdout) + log_.write(stderr) + def find_rst(self): surf_restarts =[ "route_internal_rst" , diff --git a/pre/remap_restart/remap_params.py b/pre/remap_restart/remap_params.py deleted file mode 100755 index 045edc17..00000000 --- a/pre/remap_restart/remap_params.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -# -# remap_restarts package: -# remap_params.py uses the dictionary `answers` (from remap_questions.py) as inputs and -# generates a yaml config file named `remap_params.yaml`. -# -import os,sys -import ruamel.yaml -import shutil -import glob -import time -import shlex -import subprocess -from datetime import datetime -from datetime import timedelta -from remap_utils import * - -class remap_params(object): - def __init__(self, config_from_question): - self.common_in = config_from_question['input']['shared'] - self.common_out = config_from_question['output']['shared'] - self.upper_out = config_from_question['output']['air'] - self.slurm_options = config_from_question['slurm'] - self.surf_in = config_from_question['input']['surface'] - self.surf_out = config_from_question['output']['surface'] - self.ana_out = config_from_question['output']['analysis'] - - # load input yaml - yaml = ruamel.yaml.YAML() - stream = '' - remap_tpl = os.path.dirname(os.path.realpath(__file__)) + '/remap_params.tpl' - with open(remap_tpl, 'r') as f: - stream = f.read() - config_tpl = yaml.load(stream) - - # params for shared - config_tpl['input']['shared']['MERRA-2'] = self.common_in.get('MERRA-2') - config_tpl['input']['shared']['agrid'] = self.common_in.get('agrid') - config_tpl['input']['shared']['ogrid'] = self.common_in.get('ogrid') - config_tpl['input']['shared']['omodel'] = self.common_in.get('omodel') - config_tpl['input']['shared']['rst_dir'] = self.common_in['rst_dir']+'/' - config_tpl['input']['shared']['expid'] = self.common_in.get('expid') - config_tpl['input']['shared']['yyyymmddhh'] = self.common_in['yyyymmddhh'] - config_tpl['input']['shared']['bc_version'] = self.common_in.get('bc_version') - config_tpl['input']['shared']['stretch'] = self.common_in.get('stretch') - config_tpl['input']['shared']['bc_base'] = self.common_in['bc_base'] - config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') - - config_tpl['output']['air']['nlevel'] = self.upper_out.get('nlevel') - config_tpl['output']['air']['agcm_import_rst'] = self.upper_out.get('agcm_import_rst') - config_tpl['output']['air']['remap'] = self.upper_out.get('remap') - config_tpl['output']['surface']['remap_water'] = self.surf_out.get('remap') - config_tpl['output']['surface']['remap_catch'] = self.surf_out.get('remap') - config_tpl['output']['shared']['agrid'] = self.common_out['agrid'] - config_tpl['output']['shared']['ogrid'] = self.common_out['ogrid'] - config_tpl['output']['shared']['omodel'] = self.common_out['omodel'] - config_tpl['output']['shared']['out_dir'] = self.common_out['out_dir'] + '/' - config_tpl['output']['shared']['expid'] = self.common_out['expid'] - config_tpl['output']['shared']['bc_version'] = self.common_out.get('bc_version') - config_tpl['output']['shared']['label'] = self.common_out.get('label') - config_tpl['output']['shared']['stretch'] = self.common_out.get('stretch') - config_tpl['output']['shared']['bc_base'] = self.common_out['bc_base'] - - # params for upper air - config_tpl = self.params_for_air(config_tpl) - config_tpl = self.params_for_surface(config_tpl) - config_tpl = self.params_for_analysis(config_tpl) - config_tpl = self.options_for_slurm(config_tpl) - - self.config = config_tpl - - def params_for_air(self, config_tpl): - if self.common_in['MERRA-2']: - return config_tpl - if ( not config_tpl['input']['air']['drymass']) : - config_tpl['input']['air']['drymass'] = 1 - - return config_tpl - - def options_for_slurm(self, config_tpl): - config_tpl['slurm']['account'] = self.slurm_options['account'] - config_tpl['slurm']['qos'] = self.slurm_options['qos'] - config_tpl['slurm']['partition'] = self.slurm_options['partition'] - return config_tpl - - def params_for_surface(self, config_tpl): - config_tpl['output']['surface']['surflay'] = 50. - bc_version = self.common_out['bc_version'] - ogrid = self.common_out['ogrid'] - config_tpl['output']['surface']['split_saltwater'] = True - if 'Ganymed' in bc_version : - config_tpl['output']['surface']['split_saltwater'] = False - config_tpl['input']['surface']['zoom'] = self.surf_in['zoom'] - config_tpl['input']['surface']['wemin'] = self.surf_in['wemin'] - config_tpl['output']['surface']['wemin'] = self.surf_out['wemin'] - config_tpl['input']['surface']['catch_model'] = self.surf_in.get('catch_model') - return config_tpl - - def params_for_analysis(self, config_tpl): - config_tpl['output']['analysis']['lcv'] = self.ana_out.get('lcv') - config_tpl['output']['analysis']['bkg'] = self.ana_out.get('bkg') - config_tpl['output']['analysis']['aqua'] = True - - return config_tpl - -if __name__ == "__main__": - yaml = ruamel.yaml.YAML() - stream ='' - with open("raw_answers.yaml", "r") as f: - stream = f.read() - config = yaml.load(stream) - param = remap_params(config) - config_to_yaml(param.config, 'remap_params.yaml') diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 3a78a200..1e7f6552 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -54,11 +54,13 @@ output: remap_catch: true # if catch_tilefile is null, it searches bc_dir catch_tilefile: null + # the name of ease grid + EASE_grid: null analysis: bkg: true - aqua: False + aqua: true lcv: false -slurm: +slurm_pbs: account: qos: - partition: + partition: '' diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index b3839005..a98688d5 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -14,26 +14,6 @@ import glob from remap_utils import * -def remove_ogrid_comment(x, opt): - ogrid = '' - if opt == "IN": - ogrid = x.get('input:shared:ogrid') - else: - ogrid = x.get('output:shared:ogrid') - if not ogrid: return False - - ogrid = ogrid.split()[0] - if opt == "IN": - if ogrid == 'CS': - ogrid = x['input:shared:agrid'] - x['input:shared:ogrid'] = ogrid - else: - if ogrid == 'CS': - ogrid = x['output:shared:agrid'] - x['output:shared:ogrid'] = ogrid - - return False - def echo_level(x): if x["output:air:nlevel"] != str(x.get("input:air:nlevel")) : print("NOTE: Different # atm levels in input and new restarts. Cannot remap agcm_import_rst (a.k.a. IAU) file.") @@ -46,18 +26,18 @@ def echo_bcs(x,opt): bcv = x.get(opt+':shared:bc_version') agrid = x.get(opt+':shared:agrid') ogrid = x.get(opt+':shared:ogrid') - model = x.get(opt+':shared:omodel') + omodel = x.get(opt+':shared:omodel') stretch = x.get(opt+':shared:stretch') x[opt+':shared:bc_base'] = base - land_dir = get_landdir(base, bcv, agrid, ogrid, model, stretch) + land_dir = get_landdir(base, bcv, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) if not os.path.isdir(land_dir): exit("cannot find grid subdirectory for agrid=" + agrid + " and ogrid=" + ogrid + " under " + base+'/'+bcv+'/land/') print("\n Land BCs for " + opt + " restarts: " + land_dir ) return False def default_partition(x): - if x['slurm:qos'] == 'debug': - x['slurm:partition'] = 'compute' + if x['slurm_pbs:qos'] == 'debug': + x['slurm_pbs:partition'] = 'compute' return False return True @@ -70,6 +50,11 @@ def validate_merra2_time(text): return False else: return False +def SITE_MERRA2(x): + if GEOS_SITE == "NAS": + x['input:shared:MERRA-2']= False + return False + return True def ask_questions(): @@ -82,6 +67,7 @@ def ask_questions(): "name": "input:shared:MERRA-2", "message": "Remap archived MERRA-2 restarts? (NCCS/Discover only; elsewhere, select 'N' and complete full config; requires nc4 restarts.)\n", "default": False, + "when": lambda x: SITE_MERRA2(x), }, { "type": "path", @@ -92,21 +78,21 @@ def ask_questions(): { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "Enter restart date/time: (Must be 10 digits: yyyymmddhh.)\n", + "message": (message_datetime + ".)\n"), "validate": lambda text: len(text)==10 , "when": lambda x: not x['input:shared:MERRA-2'] and not fvcore_info(x), }, { "type": "text", "name": "input:shared:yyyymmddhh", - "message": "Enter restart date: (Must be 10 digits: yyyymmddhh; hour = 03, 09, 15, or 21 [z].)\n", + "message": (message_datetime + "; hour = 03, 09, 15, or 21 [z].)\n"), "validate": lambda text: validate_merra2_time(text) , "when": lambda x: x['input:shared:MERRA-2'], }, { "type": "path", "name": "output:shared:out_dir", - "message": "Enter output directory for new restarts:\n" + "message": message_out_dir, }, # dummy (invisible) question to run function that initializes MERRA-2 config @@ -139,7 +125,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:ogrid", - "message": "Select data ocean grid/resolution of input restarts:\n", + "message": message_ogrid_in, "choices": choices_ogrid_data, "default": lambda x: data_ocean_default(x.get('input:shared:agrid')), "when": lambda x: x.get('input:shared:omodel') == 'data' and not x['input:shared:MERRA-2'], @@ -286,7 +272,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_version", - "message": "\nSelect BCs version of input restarts:\n", + "message": "\nSelect BCs version for new restarts:\n", "choices": choices_bc_other, "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11'], }, @@ -294,7 +280,7 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_base", - "message": "\nSelect BCs base directory for input restarts: \n", + "message": ("\nSelect " + message_bc_base_in), "choices": choices_bc_base, "default": get_default_bc_base(), "when": lambda x: not x.get('input:shared:bc_base'), @@ -303,7 +289,7 @@ def ask_questions(): { "type": "path", "name": "input:shared:bc_base", - "message": "\nEnter BCs base directory for input restarts: \n", + "message": ("\nEnter " + message_bc_base_in), "when": lambda x: 'Custom ' in x.get('input:shared:bc_base'), }, # dummy (invisible) question to retrieve input:shared:bc_base @@ -318,7 +304,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:bc_base", - "message": "\nSelect BCs base directory for new restarts: \n", + "message": ("\nSelect " + message_bc_base_new), "choices": choices_bc_base, "default": get_default_bc_base(), }, @@ -326,7 +312,7 @@ def ask_questions(): { "type": "path", "name": "output:shared:bc_base", - "message": "\nEnter BCs base directory for new restarts: \n", + "message": ("\nEnter " + message_bc_base_new), "when": lambda x: 'Custom ' in x.get('output:shared:bc_base'), }, # dummy (invisible) question to retrieve output:shared:bc_base @@ -398,7 +384,7 @@ def ask_questions(): { "type": "text", "name": "output:shared:expid", - "message": "Enter experiment ID for new restarts: (Added as prefix to new restart file names; can leave blank.)\n", + "message": message_expid, "default": "", }, { @@ -410,23 +396,22 @@ def ask_questions(): { "type": "text", - "name": "slurm:qos", - "message": "SLURM quality-of-service (qos)? (If on NCCS and atm resolution is c1440 or higher, enter allnccs.) ", + "name": "slurm_pbs:qos", + "message": message_qos, "default": "debug", }, { "type": "text", - "name": "slurm:account", - "message": "SLURM account?", + "name": "slurm_pbs:account", + "message": message_account, "default": get_account(), }, { "type": "text", - "name": "slurm:partition", - "message": "SLURM partition?", - "default": "compute", - "when": lambda x : default_partition(x), + "name": "slurm_pbs:partition", + "message": message_partition, + "default": '', }, ] answers = questionary.prompt(questions) @@ -436,6 +421,9 @@ def ask_questions(): if answers.get('input:air:nlevel') : del answers['input:air:nlevel'] if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"]: answers["input:surface:catch_model"] = catch_model(answers) + answers["output:surface:remap_water"] = answers["output:surface:remap"] + answers["output:surface:remap_catch"] = answers["output:surface:remap"] + del answers["output:surface:remap"] return answers diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index 4f1124c0..78ae59fc 100755 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -17,7 +17,6 @@ from remap_utils import * from remap_questions import * from remap_command_line import * -from remap_params import * from remap_upper import * from remap_lake_landice_saltwater import * from remap_analysis import * @@ -42,6 +41,9 @@ 3. Use command line arguments: ./remap_restarts.py command_line -ymdh 2004041421 .... + 4. For GEOSldas: Remap land (catch[cn]) restart only; global domain only; ens0000 only: + ./remap_restarts.py land_only + Help commands: ./remap_restarts.py -h ./remap_restarts.py config_file -h @@ -66,18 +68,20 @@ def main(): if sys.argv[1] == 'command_line': answers = get_answers_from_command_line(cmdl) noprompt = cmdl.np + if sys.argv[1] == 'land_only': + remap_land_only() + sys.exit(0) if (len(sys.argv) == 1 or answers) : if not answers: answers = ask_questions() question_flag = True - raw_config = get_config_from_answers(answers) cmd = get_command_line_from_answers(answers) write_cmd(answers['output:shared:out_dir'], cmd) # just for debugging + # raw_config = get_config_from_answers(answers) # with open("raw_answers.yaml", "w") as f: # yaml.dump(raw_config, f) - params = remap_params(raw_config) - config = params.config + config = get_config_from_answers(answers, config_tpl = True) config_yaml = answers['output:shared:out_dir']+'/remap_params.yaml' print('\n') diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 30ceacb7..1f28c8a1 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -15,9 +15,7 @@ import shutil import glob from remap_base import remap_base -from remap_utils import get_label -from remap_utils import STRETCH_GRID -from remap_utils import get_topodir +from remap_utils import * from remap_bin2nc import bin2nc class upperair(remap_base): @@ -74,7 +72,7 @@ def remap(self): print('\nUpper air restart file names link from "_rst" to "_restart_in" \n') types = '.bin' - type_str = subprocess.check_output(['file','-b', restarts_in[0]]) + type_str = subprocess.check_output(['file','-b', os.path.realpath(restarts_in[0])]) type_str = str(type_str) if type_str.find('Hierarchical') >=0: types = '.nc4' @@ -94,7 +92,10 @@ def remap(self): ogrid = config['input']['shared']['ogrid'] omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - topo_bcsdir = get_topodir(in_bc_base, in_bc_version, agrid, ogrid, omodel, stretch) + topo_bcsdir = get_topodir(in_bc_base, in_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) + + if "gmao_SIteam/ModelData" in in_bc_base: + assert GEOS_SITE == "NAS", "wrong site to run the package" topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] # link topo file @@ -109,7 +110,7 @@ def remap(self): ogrid = config['output']['shared']['ogrid'] omodel = config['output']['shared']['omodel'] stretch = config['output']['shared']['stretch'] - topo_bcsdir = get_topodir(out_bc_base, out_bc_version, agrid, ogrid, omodel, stretch) + topo_bcsdir = get_topodir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) topoout = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0] cmd = '/bin/ln -s ' + topoout + ' topo_dynave.data' @@ -143,15 +144,27 @@ def remap(self): elif (imout>=2880): NPE = 5400; nwrit = 6 - QOS = "#SBATCH --qos="+config['slurm']['qos'] - TIME ="#SBATCH --time=1:00:00" + PARTITION ='' + QOS = config['slurm_pbs']['qos'] + TIME = "1:00:00" if NPE > 532: - assert config['slurm']['qos'] != 'debug', "qos should be allnccs" - TIME = "#SBATCH --time=12:00:00" - - PARTITION = "#SBATCH --partition=" + config['slurm']['partition'] + assert config['slurm_pbs']['qos'] != 'debug', "qos should be 'allnccs' for NCCS or 'normal' for NAS" + TIME = "12:00:00" + NNODE = '' + job='' + if GEOS_SITE == "NAS": + CONSTRAINT = 'cas_ait' + NNODE = (NPE-1)//40 + 1 + job='PBS' + else: + job='SLURM' + partition = config['slurm_pbs']['partition'] + if (partition != ''): + PARTITION = "#SBATCH --partition=" + partition - log_name = out_dir+'/remap_upper_log' + CONSTRAINT = '"[cas|sky]"' + if BUILT_ON_SLES15: + CONSTRAINT = 'mil' # We need to create an input.nml file which is different if we are running stretched grid # If we are running stretched grid, we need to pass in the target lon+lat and stretch factor @@ -189,19 +202,12 @@ def remap(self): with open('input.nml', 'w') as f: f.write(nml_file) - remap_template="""#!/bin/csh -xf -#SBATCH --account={account} -#SBATCH --ntasks={NPE} -#SBATCH --job-name=remap_upper -#SBATCH --output={log_name} -{TIME} -{QOS} -{PARTITION} - -unlimit + remap_template = job_directive[job] + \ +""" +source {Bin}/g5_modules +limit stacksize unlimited cd {out_dir}/upper_data -source {Bin}/g5_modules /bin/touch input.nml # The MERRA fvcore_internal_restarts don't include W or DZ, but we can add them by setting @@ -252,15 +258,16 @@ def remap(self): -do_hydro {hydrostatic} $ioflag $dmflag -nwriter {nwrit} {stretch_str} """ - account = config['slurm']['account'] + account = config['slurm_pbs']['account'] drymassFLG = config['input']['air']['drymass'] hydrostatic = config['input']['air']['hydrostatic'] nlevel = config['output']['air']['nlevel'] - + log_name = out_dir+'/remap_upper_log' + job_name = 'remap_upper' remap_upper_script = remap_template.format(Bin=bindir, account = account, \ - out_dir = out_dir, log_name = log_name, drymassFLG = drymassFLG, \ - imout = imout, nwrit = nwrit, NPE = NPE, \ - QOS = QOS, TIME = TIME, PARTITION = PARTITION, nlevel = nlevel, hydrostatic = hydrostatic, + out_dir = out_dir, log_name = log_name, job_name= job_name, drymassFLG = drymassFLG, \ + imout = imout, nwrit = nwrit, NPE = NPE, NNODE = NNODE, \ + QOS = QOS, TIME = TIME, CONSTRAINT = CONSTRAINT, PARTITION = PARTITION, nlevel = nlevel, hydrostatic = hydrostatic, stretch_str = stretch_str) script_name = './remap_upper.j' @@ -269,23 +276,31 @@ def remap(self): upper.write(remap_upper_script) upper.close() - interactive = os.getenv('SLURM_JOB_ID', default = None) + interactive = None + if GEOS_SITE == 'NAS': + interactive = os.getenv('PBS_JOBID', default = None) + else: + interactive = os.getenv('SLURM_JOB_ID', default = None) if (interactive) : print('interactive mode\n') - ntasks = os.getenv('SLURM_NTASKS', default = None) - if ( not ntasks): - nnodes = int(os.getenv('SLURM_NNODES', default = '1')) - ncpus = int(os.getenv('SLURM_CPUS_ON_NODE', default = '28')) - ntasks = nnodes * ncpus - ntasks = int(ntasks) - if (ntasks < NPE ): - print("\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ".format(NPE=NPE, ntasks=ntasks)) + if GEOS_SITE != 'NAS': + ntasks = os.getenv('SLURM_NTASKS', default = None) + if ( not ntasks): + nnodes = int(os.getenv('SLURM_NNODES', default = '1')) + ncpus = int(os.getenv('SLURM_CPUS_ON_NODE', default = '28')) + ntasks = nnodes * ncpus + ntasks = int(ntasks) + if (ntasks < NPE ): + print("\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ".format(NPE=NPE, ntasks=ntasks)) subprocess.call(['chmod', '755', script_name]) print(script_name+ ' 1>' + log_name + ' 2>&1') os.system(script_name + ' 1>' + log_name+ ' 2>&1') - else : + elif GEOS_SITE == "NAS" : + print('qsub -W block=true '+ script_name +'\n') + subprocess.call(['qsub', '-W','block=true', script_name]) + else: print('sbatch -W '+ script_name +'\n') subprocess.call(['sbatch', '-W', script_name]) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 68b838ec..345a18bd 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -15,6 +15,16 @@ # shared global variables +#During cmake step, the string will be changed according to the system + +BUILT_ON_SLES15 = "NO@BUILT_ON_SLES15@" +GEOS_SITE = "@GEOS_SITE@" + +if BUILT_ON_SLES15== "NO": + BUILT_ON_SLES15 = False +else: + BUILT_ON_SLES15 = True + # top-level directory for BCs (machine-dependent) choices_bc_base =[ "NCCS/Discover : /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles", @@ -62,6 +72,15 @@ choices_res_SG002 = ['C1536'] +message_datetime = "Enter restart date/time: (Must be 10 digits: yyyymmddhh" + +message_out_dir = "Enter output directory for new restarts:\n" + +message_expid = "Enter experiment ID for new restarts: (Added as prefix to new restart file names; can leave blank.)\n" + +message_bc_base_in = "BCs base directory for input restarts: \n" +message_bc_base_new = "BCs base directory for new restarts: \n" + message_bc_ops = f'''\n BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- @@ -95,6 +114,37 @@ validate_agrid = ['C12','C24','C48','C90','C180','C360','C720','C1000','C1440','C2880','C5760'] +message_ogrid_in = "Select data ocean grid/resolution of input restarts:\n" + +message_qos = "SLURM or PBS quality-of-service (qos)? (If resolution is c1440 or higher, enter 'allnccs' for NCCS or 'normal' for NAS.)\n" + +message_account = "Select/enter SLURM or PBS account:\n", + +message_partition = "Enter SLURM or PBS partition: (If desired; can leave blank.)\n", + + +job_directive = {"SLURM": """#!/bin/csh -f +#SBATCH --account={account} +#SBATCH --ntasks={NPE} +#SBATCH --job-name={job_name} +#SBATCH --output={log_name} +#SBATCH --qos={QOS} +#SBATCH --time={TIME} +#SBATCH --constraint={CONSTRAINT} +{PARTITION} +""", +"PBS": """#!/bin/csh -f +#PBS -l walltime={TIME} +#PBS -l select={NNODE}:ncpus=40:mpiprocs=40:model={CONSTRAINT} +#PBS -N {job_name} +#PBS -q {QOS} +#PBS -W group_list={account} +#PBS -o {log_name} +#PBS -j oe +{PARTITION} +""" +} + # -------------------------------------------------------------------------------- def init_merra2(x): @@ -137,7 +187,7 @@ def fvcore_info(x): if not rst_dir : return False x['input:shared:rst_dir'] = rst_dir.strip() # remove extra space - files = glob.glob(rst_dir+'/*fvcore_*') + files = glob.glob(rst_dir+'/*fvcore_internal*') if len (files) == 0 : return False fname ='' @@ -147,7 +197,7 @@ def fvcore_info(x): ymdh = x.get('input:shared:yyyymmddhh') if (not ymdh): return False time = ymdh[0:8] + '_'+ymdh[8:10] - files = glob.glob(rst_dir+'/*fvcore_*'+time+'*') + files = glob.glob(rst_dir+'/*fvcore_internal*'+time+'*') fname = files[0] if len(files) == 1: @@ -200,13 +250,9 @@ def catch_model(x): files = glob.glob(rst_dir+'/*catch*') if len (files) == 0 : return False - fname= '' - if len(files) == 1: - fname = os.path.basename(files[0]) + fname = '' + fname = os.path.basename(files[0]) - if len(files) > 1 : - files = glob.glob(rst_dir+'/*fvcore_*'+time+'*') - fname = os.path.basename(files[0]) model = 'catch' if 'cnclm40' in fname.lower(): model = 'catchcnclm40' @@ -225,15 +271,19 @@ def get_label(config): if config['output']['shared']['label']: agrid = config['output']['shared']['agrid'] ogrid = config['output']['shared']['ogrid'] - model = config['output']['shared']['omodel'] + omodel = config['output']['shared']['omodel'] stretch = config['output']['shared']['stretch'] - out_resolution = get_resolutions(agrid, ogrid, model, stretch) + EASE_grid = config['output']['surface'].get('EASE_grid', None) + + out_resolution = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) agrid = config['input']['shared']['agrid'] ogrid = config['input']['shared']['ogrid'] - model = config['input']['shared']['omodel'] + omodel = config['input']['shared']['omodel'] stretch = config['input']['shared']['stretch'] - in_resolution = get_resolutions(agrid, ogrid, model, stretch) + EASE_grid = config['input']['surface'].get('EASE_grid', None) + + in_resolution = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid) in_bcv = config['input']['shared']['bc_version'] out_bcv = config['output']['shared']['bc_version'] @@ -291,6 +341,8 @@ def config_to_yaml(config, yaml_file, noprompt = False): shutil.move(yaml_file, new_name) break yaml = ruamel.yaml.YAML() + out_dir = os.path.dirname(yaml_file) + if not os.path.exists(out_dir) : os.mkdir(out_dir) with open(yaml_file, "w") as f: yaml.dump(config, f) @@ -386,18 +438,19 @@ def get_command_line_from_answers(answers): out_rs = " -rs " rs = 3 - if answers['output:air:remap'] and not answers['output:surface:remap']: + if answers['output:air:remap'] and not answers['output:surface:remap_catch']: rs = 1 - if answers['output:surface:remap'] and not answers['output:air:remap']: + if answers['output:surface:remap_catch'] and not answers['output:air:remap']: rs = 2 out_rs = out_rs + str(rs) noagcm_import_rst = '' if answers["output:air:agcm_import_rst"] else " -noagcm_import_rst " - account = " -account " + answers["slurm:account"] - qos = " -qos " + answers["slurm:qos"] - partition = " -partition " + answers["slurm:partition"] - + account = " -account " + answers["slurm_pbs:account"] + qos = " -qos " + answers["slurm_pbs:qos"] + partition = '' + if answers["slurm_pbs:partition"] != '': + partition = " -partition " + answers["slurm_pbs:partition"] cmdl = "remap_restarts.py command_line " + merra2 + \ ymdh + \ @@ -432,18 +485,42 @@ def get_command_line_from_answers(answers): return cmdl -def get_config_from_answers(answers): +def flatten_nested(nested_dict, result=None, prefix=''): + if result is None: + result = dict() + for k, v in nested_dict.items(): + new_k = ':'.join((prefix, k)) if prefix else k + if not (isinstance(v, dict) or isinstance(v, OrderedDict)): + result.update({new_k: v}) + else: + flatten_nested(v, result, new_k) + return result + +def get_config_from_file(file): + yaml = ruamel.yaml.YAML() + stream = '' + with open(file, 'r') as f: + stream = f.read() + config = yaml.load(stream) + return config + +def get_config_from_answers(answers, config_tpl = False): config = {} - config['input'] = {} - config['input']['air'] = {} - config['input']['shared'] = {} - config['input']['surface'] = {} - config['output'] = {} - config['output']['shared'] = {} - config['output']['air'] = {} - config['output']['surface'] = {} - config['output']['analysis'] = {} - config['slurm'] = {} + if config_tpl: + remap_tpl = os.path.dirname(os.path.realpath(__file__)) + '/remap_params.tpl' + config = get_config_from_file(remap_tpl) + else: + config['input'] = {} + config['input']['air'] = {} + config['input']['shared'] = {} + config['input']['surface'] = {} + config['output'] = {} + config['output']['shared'] = {} + config['output']['air'] = {} + config['output']['surface'] = {} + config['output']['analysis'] = {} + config['slurm_pbs'] = {} + for key, value in answers.items(): keys = key.split(":") if len(keys) == 2: @@ -451,9 +528,15 @@ def get_config_from_answers(answers): if len(keys) == 3: config[keys[0]][keys[1]][keys[2]] = value + bc_version = config['output']['shared'].get('bc_version') + config['output']['surface']['split_saltwater'] = True + if 'Ganymed' in bc_version : + config['output']['surface']['split_saltwater'] = False + return config -def get_resolutions(agrid, ogrid, model, stretch): +def get_resolutions(agrid=None, ogrid=None, omodel=None, stretch=None, grid=None): + if grid is not None : return grid aname = '' oname = '' if (agrid[0].upper() == 'C'): @@ -466,11 +549,11 @@ def get_resolutions(agrid, ogrid, model, stretch): xy = ogrid.upper().split('X') s0 = int(xy[0]) s1 = int(xy[1]) - if model == 'data': + if omodel == 'data': oname = 'DE{:04d}xPE{:04d}'.format(s0,s1) - if model == 'MOM5': + if omodel == 'MOM5': oname = 'M5TP{:04d}x{:04d}'.format(s0,s1) - if model == 'MOM6': + if omodel == 'MOM6': oname = 'M6TP{:04d}x{:04d}'.format(s0,s1) if stretch: aname = aname + '-' + stretch @@ -492,8 +575,8 @@ def get_default_bc_base(): return choices_bc_base[0] return choices_bc_base[1] -def get_topodir(bc_base, bc_version, agrid, ogrid, model, stretch): - gridStr = get_resolutions(agrid, ogrid, model,stretch) +def get_topodir(bc_base, bc_version, agrid=None, ogrid=None, omodel=None, stretch=None): + gridStr = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel,stretch=stretch) agrid_name = gridStr.split('_')[0] bc_topo = '' if 'GM4' == bc_version: @@ -503,15 +586,35 @@ def get_topodir(bc_base, bc_version, agrid, ogrid, model, stretch): return bc_topo -def get_landdir(bc_base, bc_version, agrid, ogrid, model, stretch): - gridStr = get_resolutions(agrid, ogrid, model,stretch) +def get_landdir(bc_base, bc_version, agrid=None, ogrid=None, omodel=None, stretch=None, grid=None): + gridStr = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel,stretch=stretch, grid=grid) bc_land = bc_base+'/'+ bc_version+'/land/'+gridStr return bc_land -def get_geomdir(bc_base, bc_version, agrid, ogrid, model, stretch): - bc_geom = get_landdir(bc_base, bc_version, agrid, ogrid, model, stretch). replace('/land/', '/geometry/') +def get_geomdir(bc_base, bc_version, agrid=None, ogrid=None, omodel=None, stretch=None, grid=None): + bc_geom = get_landdir(bc_base, bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=grid). replace('/land/', '/geometry/') return bc_geom +def remove_ogrid_comment(x, opt): + ogrid = '' + if opt == "IN": + ogrid = x.get('input:shared:ogrid') + else: + ogrid = x.get('output:shared:ogrid') + if not ogrid: return False + + ogrid = ogrid.split()[0] + if opt == "IN": + if ogrid == 'CS': + ogrid = x['input:shared:agrid'] + x['input:shared:ogrid'] = ogrid + else: + if ogrid == 'CS': + ogrid = x['output:shared:agrid'] + x['output:shared:ogrid'] = ogrid + + return False + if __name__ == '__main__' : config = yaml_to_config('c24Toc12.yaml') print_config(config) diff --git a/pre/remap_restart/tests/amip_c180Toc90.yaml b/pre/remap_restart/tests/amip_c180Toc90.yaml index b3b7096c..fbd55647 100644 --- a/pre/remap_restart/tests/amip_c180Toc90.yaml +++ b/pre/remap_restart/tests/amip_c180Toc90.yaml @@ -55,8 +55,8 @@ output: aqua: true lcv: false -slurm: +slurm_pbs: # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug - partition: compute + partition: '' diff --git a/pre/remap_restart/tests/c180Toc360.yaml b/pre/remap_restart/tests/c180Toc360.yaml index 10b4c325..e9e1c094 100644 --- a/pre/remap_restart/tests/c180Toc360.yaml +++ b/pre/remap_restart/tests/c180Toc360.yaml @@ -55,8 +55,8 @@ output: aqua: true lcv: false -slurm: +slurm_pbs: # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug - partition: compute + partition: '' diff --git a/pre/remap_restart/tests/c24Toc12.yaml b/pre/remap_restart/tests/c24Toc12.yaml index ed9ec246..fb7d67db 100644 --- a/pre/remap_restart/tests/c24Toc12.yaml +++ b/pre/remap_restart/tests/c24Toc12.yaml @@ -55,8 +55,8 @@ output: aqua: true lcv: false -slurm: +slurm_pbs: # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug - partition: compute + partition: '' diff --git a/pre/remap_restart/tests/c360Toc24.yaml b/pre/remap_restart/tests/c360Toc24.yaml index e566fc90..de7a55ab 100644 --- a/pre/remap_restart/tests/c360Toc24.yaml +++ b/pre/remap_restart/tests/c360Toc24.yaml @@ -55,8 +55,8 @@ output: aqua: true lcv: false -slurm: +slurm_pbs: # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug - partition: compute + partition: '' diff --git a/pre/remap_restart/tests/f522Toc360.yaml b/pre/remap_restart/tests/f522Toc360.yaml index 4c58aaba..2d2975f6 100644 --- a/pre/remap_restart/tests/f522Toc360.yaml +++ b/pre/remap_restart/tests/f522Toc360.yaml @@ -51,8 +51,8 @@ output: aqua: true lcv: true -slurm: +slurm_pbs: # account will be set to user's primary account automatically by test_remap_restarts.py account: qos: debug - partition: compute + partition: '' diff --git a/pre/remap_restart/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py index 0fcb0752..6d64a746 100755 --- a/pre/remap_restart/tests/test_remap_restarts.py +++ b/pre/remap_restart/tests/test_remap_restarts.py @@ -12,8 +12,7 @@ import glob import subprocess as sp import remap_restarts -import remap_utils -from remap_params import * +from remap_utils import * from remap_upper import * from remap_lake_landice_saltwater import * from remap_analysis import * @@ -59,6 +58,9 @@ def test_remap(config): if __name__ == '__main__' : + if GEOS_SITE != "NCCS" : + print("Test baseline data are only available at NCCS. Please run tests on Discover.") + exit() yaml = ruamel.yaml.YAML() stream ='' cases_yaml = 'test_remap_cases.yaml' @@ -84,7 +86,7 @@ def test_remap(config): out_dir = '/discover/nobackup/'+user+'/REMAP_TESTS/'+case+'/' config['output']['shared']['out_dir'] = out_dir - config['slurm']['account'] = get_account() + config['slurm_pbs']['account'] = get_account() test_remap(config)