Skip to content

Commit

Permalink
Merge pull request #879 from jedwards4b/buildnml_without_run_cmd
Browse files Browse the repository at this point in the history 
Buildnml without run cmd

Attempts to call buildnml scripts as modules before using run_cmd to call them. This results in a
20% speedup in performance of buildnml on an A case.

Test suite: scripts_regression_tests
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes #845

User interface changes?:

Code review:
  • Loading branch information
@jgfouca
jgfouca authored Dec 1, 2016
2 parents 70ae92f + 69748cb commit fce75f0
Show file tree
Hide file tree
Showing 19 changed files with 705 additions and 739 deletions.
185 changes: 84 additions & 101 deletions components/data_comps/datm/cime_config/buildnml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,10 @@ from standard_script_setup import *
from CIME.case import Case
from CIME.nmlgen import NamelistGenerator
from CIME.utils import expect
from CIME.utils import handle_standard_logging_options
from CIME.utils import setup_standard_logging_options
from CIME.buildnml import create_namelist_infile
from CIME.buildnml import create_namelist_infile, parse_input

logger = logging.getLogger(__name__)

###############################################################################
def _parse_input(argv):
###############################################################################

if "--test" in argv:
test_results = doctest.testmod(verbose=True)
sys.exit(1 if test_results.failed > 0 else 0)

parser = argparse.ArgumentParser()

setup_standard_logging_options(parser)

parser.add_argument("caseroot",
help="case directory")

args = parser.parse_args()

handle_standard_logging_options(args)

return args.caseroot

# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
####################################################################################
def _create_namelists(case, confdir, inst_string, infile, definition_file):
Expand All @@ -60,7 +37,6 @@ def _create_namelists(case, confdir, inst_string, infile, definition_file):
#----------------------------------------------------
# Get a bunch of information from the case.
#----------------------------------------------------
din_loc_root = case.get_value("DIN_LOC_ROOT")
atm_domain_file = case.get_value("ATM_DOMAIN_FILE")
atm_domain_path = case.get_value("ATM_DOMAIN_PATH")
datm_mode = case.get_value("DATM_MODE")
Expand Down Expand Up @@ -121,7 +97,11 @@ def _create_namelists(case, confdir, inst_string, infile, definition_file):
# Construct the list of streams.
#----------------------------------------------------
streams = nmlgen.get_streams()

#
# This disable is required because nmlgen.get_streams
# may return a string or a list. See issue #877 in ESMCI/cime
#
#pylint: disable=no-member
if datm_presaero == "pt1_pt1":
streams.append("presaero.%s.%s" % (datm_presaero, atm_grid))
elif datm_presaero != "none":
Expand Down Expand Up @@ -196,93 +176,96 @@ def _create_namelists(case, confdir, inst_string, infile, definition_file):
nmlgen.write_output_file(namelist_file, data_list_path, groups=['datm_nml','shr_strdata_nml'])

###############################################################################
def _main_func():
def buildnml(case, caseroot, compname):
###############################################################################

# Build the component namelist and required stream txt files
if compname != "datm":
raise AttributeError

compname = "datm"
caseroot = _parse_input(sys.argv)

with Case(caseroot) as case:
cimeroot = case.get_value("CIMEROOT")
rundir = case.get_value("RUNDIR")
ninst = case.get_value("NINST_ATM")
din_loc_root = case.get_value("DIN_LOC_ROOT")
cimeroot = case.get_value("CIMEROOT")
rundir = case.get_value("RUNDIR")
ninst = case.get_value("NINST_ATM")
din_loc_root = case.get_value("DIN_LOC_ROOT")

if not os.path.isdir(din_loc_root):
os.makedirs(din_loc_root)
logger.info("Created input root directory %s" %din_loc_root)
if not os.path.isdir(din_loc_root):
os.makedirs(din_loc_root)
logger.info("Created input root directory %s" %din_loc_root)

# determine directory for user modified namelist_definitions.xml and namelist_defaults.xml
user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname)
expect (os.path.isdir(user_xml_dir),
"user_xml_dir %s does not exist " %user_xml_dir)
user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname)
expect (os.path.isdir(user_xml_dir),
"user_xml_dir %s does not exist " %user_xml_dir)

# NOTE: User definition *replaces* existing definition.
namelist_xml_dir = os.path.join(cimeroot, "components", "data_comps", compname, "cime_config")
definition_file = [os.path.join(namelist_xml_dir, "namelist_definition_datm.xml")]
user_definition = os.path.join(user_xml_dir, "namelist_definition_datm.xml")
if os.path.isfile(user_definition):
definition_file = [user_definition]

# Checkout if definition file exists
for file_ in definition_file:
expect(os.path.isfile(file_), "Namelist XML file %s not found!" % file_)

confdir = os.path.join(caseroot,"Buildconf",compname + "conf")
if not os.path.isdir(confdir):
os.makedirs(confdir)

# Loop over instances
namelist_xml_dir = os.path.join(cimeroot, "components", "data_comps", compname, "cime_config")
definition_file = [os.path.join(namelist_xml_dir, "namelist_definition_datm.xml")]
user_definition = os.path.join(user_xml_dir, "namelist_definition_datm.xml")
if os.path.isfile(user_definition):
definition_file = [user_definition]

# Checkout if definition file exists
for file_ in definition_file:
expect(os.path.isfile(file_), "Namelist XML file %s not found!" % file_)

confdir = os.path.join(caseroot,"Buildconf",compname + "conf")
if not os.path.isdir(confdir):
os.makedirs(confdir)

# Loop over instances
inst_string = ""
inst_counter = 1
while (inst_counter <= ninst):

# determine instance string
inst_string = ""
inst_counter = 1
while (inst_counter <= ninst):

# determine instance string
inst_string = ""
if ninst > 1:
inst_string = '_' + '%04d' % inst_counter

# If multi-instance case does not have restart file, use
# single-case restart for each instance
rpointer = "rpointer." + compname
if (os.path.isfile(os.path.join(rundir,rpointer)) and
(not os.path.isfile(os.path.join(rundir,rpointer + inst_string)))):
shutil.copy(os.path.join(rundir, rpointer),
os.path.join(rundir, rpointer + inst_string))

inst_string_label = inst_string
if not inst_string_label:
inst_string_label = "\"\""

# create namelist output infile using user_nl_file as input
user_nl_file = os.path.join(caseroot, "user_nl_" + compname + inst_string)
expect(os.path.isfile(user_nl_file),
"Missing required user_nl_file %s " %(user_nl_file))
infile = os.path.join(confdir, "namelist_infile")
create_namelist_infile(case, user_nl_file, infile)
namelist_infile = [infile]

# create namelist and stream file(s) data component
_create_namelists(case, confdir, inst_string, namelist_infile, definition_file)

# copy namelist files and stream text files, to rundir
if os.path.isdir(rundir):
filename = compname + "_in"
file_src = os.path.join(confdir, filename)
file_dest = os.path.join(rundir, filename)
if inst_string:
file_dest += inst_string
shutil.copy(file_src,file_dest)

for txtfile in glob.glob(os.path.join(confdir, "*txt*")):
shutil.copy(txtfile, rundir)

# increment instance counter
inst_counter = inst_counter + 1
if ninst > 1:
inst_string = '_' + '%04d' % inst_counter

# If multi-instance case does not have restart file, use
# single-case restart for each instance
rpointer = "rpointer." + compname
if (os.path.isfile(os.path.join(rundir,rpointer)) and
(not os.path.isfile(os.path.join(rundir,rpointer + inst_string)))):
shutil.copy(os.path.join(rundir, rpointer),
os.path.join(rundir, rpointer + inst_string))

inst_string_label = inst_string
if not inst_string_label:
inst_string_label = "\"\""

# create namelist output infile using user_nl_file as input
user_nl_file = os.path.join(caseroot, "user_nl_" + compname + inst_string)
expect(os.path.isfile(user_nl_file),
"Missing required user_nl_file %s " %(user_nl_file))
infile = os.path.join(confdir, "namelist_infile")
create_namelist_infile(case, user_nl_file, infile)
namelist_infile = [infile]

# create namelist and stream file(s) data component
_create_namelists(case, confdir, inst_string, namelist_infile, definition_file)

# copy namelist files and stream text files, to rundir
if os.path.isdir(rundir):
filename = compname + "_in"
file_src = os.path.join(confdir, filename)
file_dest = os.path.join(rundir, filename)
if inst_string:
file_dest += inst_string
shutil.copy(file_src,file_dest)

for txtfile in glob.glob(os.path.join(confdir, "*txt*")):
shutil.copy(txtfile, rundir)

# increment instance counter
inst_counter = inst_counter + 1

###############################################################################
def _main_func():
caseroot = parse_input(sys.argv)
with Case(caseroot) as case:
buildnml(case, caseroot, "datm")


if __name__ == "__main__":
_main_func()
Loading

0 comments on commit fce75f0

Please sign in to comment.