+# -*- coding: utf-8 -*-
+"""
+Wrapper around all env XML for a case.
+
+All interaction with and between the module files in XML/ takes place
+through the Case module.
+"""
+from copy import deepcopy
+import sys
+import glob, os, shutil, math, time, hashlib, socket, getpass
+from CIME.XML.standard_module_setup import *
+
+# pylint: disable=import-error,redefined-builtin
+from CIME import utils
+from CIME.config import Config
+from CIME.utils import expect, get_cime_root, append_status
+from CIME.utils import convert_to_type, get_model, set_model
+from CIME.utils import get_project, get_charge_account, check_name
+from CIME.utils import get_current_commit, safe_copy, get_cime_default_driver
+from CIME.locked_files import LOCKED_DIR, lock_file
+from CIME.XML.machines import Machines
+from CIME.XML.pes import Pes
+from CIME.XML.files import Files
+from CIME.XML.testlist import Testlist
+from CIME.XML.component import Component
+from CIME.XML.compsets import Compsets
+from CIME.XML.grids import Grids
+from CIME.XML.batch import Batch
+from CIME.XML.workflow import Workflow
+from CIME.XML.pio import PIO
+from CIME.XML.archive import Archive
+from CIME.XML.env_test import EnvTest
+from CIME.XML.env_mach_specific import EnvMachSpecific
+from CIME.XML.env_case import EnvCase
+from CIME.XML.env_mach_pes import EnvMachPes
+from CIME.XML.env_build import EnvBuild
+from CIME.XML.env_run import EnvRun
+from CIME.XML.env_archive import EnvArchive
+from CIME.XML.env_batch import EnvBatch
+from CIME.XML.env_workflow import EnvWorkflow
+from CIME.XML.generic_xml import GenericXML
+from CIME.user_mod_support import apply_user_mods
+from CIME.aprun import get_aprun_cmd_for_case
+
+logger = logging.getLogger(__name__)
+
+config = Config.instance()
+
+
+
+
[docs]
+
class Case(object):
+
"""
+
https://github.com/ESMCI/cime/wiki/Developers-Introduction
+
The Case class is the heart of the CIME Case Control system. All
+
interactions with a Case take part through this class. All of the
+
variables used to create and manipulate a case are defined in xml
+
files and for every xml file there is a python class to interact
+
with that file.
+
+
XML files which are part of the CIME distribution and are meant to
+
be readonly with respect to a case are typically named
+
config_something.xml and the corresponding python Class is
+
Something and can be found in file CIME.XML.something.py. I'll
+
refer to these as the CIME config classes.
+
+
XML files which are part of a case and thus are read/write to a
+
case are typically named env_whatever.xml and the cooresponding
+
python modules are CIME.XML.env_whatever.py and classes are
+
EnvWhatever. I'll refer to these as the Case env classes.
+
+
The Case Class includes an array of the Case env classes, in the
+
configure function and it's supporting functions defined below
+
the case object creates and manipulates the Case env classes
+
by reading and interpreting the CIME config classes.
+
+
This class extends across multiple files, class members external to this file
+
are listed in the following imports
+
+
"""
+
+
from CIME.case.case_setup import case_setup
+
from CIME.case.case_clone import create_clone, _copy_user_modified_to_clone
+
from CIME.case.case_test import case_test
+
from CIME.case.case_submit import check_DA_settings, check_case, submit
+
from CIME.case.case_st_archive import (
+
case_st_archive,
+
restore_from_archive,
+
archive_last_restarts,
+
test_st_archive,
+
test_env_archive,
+
)
+
from CIME.case.case_run import case_run
+
from CIME.case.case_cmpgen_namelists import case_cmpgen_namelists
+
from CIME.case.check_lockedfiles import (
+
check_lockedfile,
+
check_lockedfiles,
+
check_pelayouts_require_rebuild,
+
)
+
from CIME.case.preview_namelists import create_dirs, create_namelists
+
from CIME.case.check_input_data import (
+
check_all_input_data,
+
stage_refcase,
+
check_input_data,
+
)
+
+
def __init__(self, case_root=None, read_only=True, record=False, non_local=False):
+
+
if case_root is None:
+
case_root = os.getcwd()
+
expect(
+
not os.path.isdir(case_root)
+
or os.path.isfile(os.path.join(case_root, "env_case.xml")),
+
"Directory {} does not appear to be a valid case directory".format(
+
case_root
+
),
+
)
+
+
self._caseroot = case_root
+
logger.debug("Initializing Case.")
+
self._read_only_mode = True
+
self._force_read_only = read_only
+
self._primary_component = None
+
+
self._env_entryid_files = []
+
self._env_generic_files = []
+
self._files = []
+
self._comp_interface = None
+
self.gpu_enabled = False
+
self._non_local = non_local
+
self.read_xml()
+
+
srcroot = self.get_value("SRCROOT")
+
+
# Propagate `srcroot` to `GenericXML` to resolve $SRCROOT
+
if srcroot is not None:
+
utils.GLOBAL["SRCROOT"] = srcroot
+
+
# srcroot may not be known yet, in the instance of creating
+
# a new case
+
customize_path = os.path.join(srcroot, "cime_config", "customize")
+
+
config.load(customize_path)
+
+
if record:
+
self.record_cmd()
+
+
cimeroot = get_cime_root()
+
+
# Insert tools path to support external code trying to import
+
# standard_script_setup
+
tools_path = os.path.join(cimeroot, "CIME", "Tools")
+
if tools_path not in sys.path:
+
sys.path.insert(0, tools_path)
+
+
# Hold arbitary values. In create_newcase we may set values
+
# for xml files that haven't been created yet. We need a place
+
# to store them until we are ready to create the file. At file
+
# creation we get the values for those fields from this lookup
+
# table and then remove the entry.
+
self.lookups = {}
+
self.set_lookup_value("CIMEROOT", cimeroot)
+
self._cime_model = get_model()
+
self.set_lookup_value("MODEL", self._cime_model)
+
self._compsetname = None
+
self._gridname = None
+
self._pesfile = None
+
self._gridfile = None
+
self._components = []
+
self._component_classes = []
+
self._component_description = {}
+
self._is_env_loaded = False
+
self._loaded_envs = None
+
+
# these are user_mods as defined in the compset
+
# Command Line user_mods are handled seperately
+
+
# Derived attributes
+
self.thread_count = None
+
self.total_tasks = None
+
self.tasks_per_node = None
+
self.ngpus_per_node = 0
+
self.num_nodes = None
+
self.spare_nodes = None
+
self.tasks_per_numa = None
+
self.cores_per_task = None
+
self.srun_binding = None
+
self.async_io = False
+
self.iotasks = 0
+
+
# check if case has been configured and if so initialize derived
+
if self.get_value("CASEROOT") is not None:
+
if not self._non_local:
+
mach = self.get_value("MACH")
+
extra_machdir = self.get_value("EXTRA_MACHDIR")
+
if extra_machdir:
+
machobj = Machines(machine=mach, extra_machines_dir=extra_machdir)
+
else:
+
machobj = Machines(machine=mach)
+
+
# This check should only be done on systems with a common filesystem but separate login nodes (ncar)
+
if "NCAR_HOST" in os.environ:
+
probed_machine = machobj.probe_machine_name()
+
if probed_machine:
+
expect(
+
mach == probed_machine,
+
f"Current machine {probed_machine} does not match case machine {mach}.",
+
)
+
+
self.initialize_derived_attributes()
+
+
+
[docs]
+
def get_baseline_dir(self):
+
baseline_root = self.get_value("BASELINE_ROOT")
+
+
baseline_name = self.get_value("BASECMP_CASE")
+
+
return os.path.join(baseline_root, baseline_name)
+
+
+
+
[docs]
+
def check_if_comp_var(self, vid):
+
for env_file in self._env_entryid_files:
+
new_vid, new_comp, iscompvar = env_file.check_if_comp_var(vid)
+
if iscompvar:
+
return new_vid, new_comp, iscompvar
+
+
return vid, None, False
+
+
+
+
[docs]
+
def initialize_derived_attributes(self):
+
"""
+
These are derived variables which can be used in the config_* files
+
for variable substitution using the {{ var }} syntax
+
"""
+
set_model(self.get_value("MODEL"))
+
env_mach_pes = self.get_env("mach_pes")
+
env_mach_spec = self.get_env("mach_specific")
+
comp_classes = self.get_values("COMP_CLASSES")
+
max_mpitasks_per_node = self.get_value("MAX_MPITASKS_PER_NODE")
+
self.async_io = {}
+
asyncio = False
+
for comp in comp_classes:
+
self.async_io[comp] = self.get_value("PIO_ASYNC_INTERFACE", subgroup=comp)
+
if self.async_io[comp]:
+
asyncio = True
+
+
self.iotasks = (
+
self.get_value("PIO_ASYNCIO_NTASKS")
+
if self.get_value("PIO_ASYNCIO_NTASKS")
+
else 0
+
)
+
+
self.thread_count = env_mach_pes.get_max_thread_count(comp_classes)
+
+
mpi_attribs = {
+
"compiler": self.get_value("COMPILER"),
+
"mpilib": self.get_value("MPILIB"),
+
"threaded": self.get_build_threaded(),
+
}
+
+
job = self.get_primary_job()
+
executable = env_mach_spec.get_mpirun(self, mpi_attribs, job, exe_only=True)[0]
+
if executable is not None and "aprun" in executable:
+
(
+
_,
+
self.num_nodes,
+
self.total_tasks,
+
self.tasks_per_node,
+
self.thread_count,
+
) = get_aprun_cmd_for_case(self, "e3sm.exe")
+
self.spare_nodes = env_mach_pes.get_spare_nodes(self.num_nodes)
+
self.num_nodes += self.spare_nodes
+
else:
+
self.total_tasks = env_mach_pes.get_total_tasks(comp_classes, asyncio)
+
self.tasks_per_node = env_mach_pes.get_tasks_per_node(
+
self.total_tasks, self.thread_count
+
)
+
+
self.num_nodes, self.spare_nodes = env_mach_pes.get_total_nodes(
+
self.total_tasks, self.thread_count
+
)
+
self.num_nodes += self.spare_nodes
+
+
logger.debug(
+
"total_tasks {} thread_count {}".format(self.total_tasks, self.thread_count)
+
)
+
+
max_gpus_per_node = self.get_value("MAX_GPUS_PER_NODE")
+
+
if max_gpus_per_node:
+
self.ngpus_per_node = self.get_value("NGPUS_PER_NODE")
+
# update the maximum MPI tasks for a GPU node (could differ from a pure-CPU node)
+
if self.ngpus_per_node > 0:
+
max_mpitasks_per_node = self.get_value("MAX_CPUTASKS_PER_GPU_NODE")
+
+
self.tasks_per_numa = int(math.ceil(self.tasks_per_node / 2.0))
+
smt_factor = max(
+
1, int(self.get_value("MAX_TASKS_PER_NODE") / max_mpitasks_per_node)
+
)
+
+
threads_per_node = self.tasks_per_node * self.thread_count
+
threads_per_core = (
+
1 if (threads_per_node <= max_mpitasks_per_node) else smt_factor
+
)
+
self.cores_per_task = self.thread_count / threads_per_core
+
+
os.environ["OMP_NUM_THREADS"] = str(self.thread_count)
+
+
self.srun_binding = math.floor(
+
smt_factor * max_mpitasks_per_node / self.tasks_per_node
+
)
+
self.srun_binding = max(1, int(self.srun_binding))
+
+
+
# Define __enter__ and __exit__ so that we can use this as a context manager
+
# and force a flush on exit.
+
def __enter__(self):
+
if not self._force_read_only:
+
self._read_only_mode = False
+
return self
+
+
def __exit__(self, *_):
+
self.flush()
+
self._read_only_mode = True
+
return False
+
+
+
[docs]
+
def read_xml(self):
+
for env_file in self._files:
+
expect(
+
not env_file.needsrewrite,
+
"Potential loss of unflushed changes in {}".format(env_file.filename),
+
)
+
+
self._env_entryid_files = []
+
self._env_entryid_files.append(
+
EnvCase(self._caseroot, components=None, read_only=self._force_read_only)
+
)
+
components = self._env_entryid_files[0].get_values("COMP_CLASSES")
+
self._env_entryid_files.append(
+
EnvRun(
+
self._caseroot, components=components, read_only=self._force_read_only
+
)
+
)
+
self._env_entryid_files.append(
+
EnvBuild(
+
self._caseroot, components=components, read_only=self._force_read_only
+
)
+
)
+
self._comp_interface = self._env_entryid_files[-1].get_value("COMP_INTERFACE")
+
+
self._env_entryid_files.append(
+
EnvMachPes(
+
self._caseroot,
+
components=components,
+
read_only=self._force_read_only,
+
comp_interface=self._comp_interface,
+
)
+
)
+
self._env_entryid_files.append(
+
EnvBatch(self._caseroot, read_only=self._force_read_only)
+
)
+
self._env_entryid_files.append(
+
EnvWorkflow(self._caseroot, read_only=self._force_read_only)
+
)
+
+
if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")):
+
self._env_entryid_files.append(
+
EnvTest(
+
self._caseroot,
+
components=components,
+
read_only=self._force_read_only,
+
)
+
)
+
self._env_generic_files = []
+
self._env_generic_files.append(
+
EnvMachSpecific(
+
self._caseroot,
+
read_only=self._force_read_only,
+
comp_interface=self._comp_interface,
+
)
+
)
+
self._env_generic_files.append(
+
EnvArchive(self._caseroot, read_only=self._force_read_only)
+
)
+
self._files = self._env_entryid_files + self._env_generic_files
+
+
+
+
[docs]
+
def get_case_root(self):
+
"""Returns the root directory for this case."""
+
return self._caseroot
+
+
+
+
[docs]
+
def get_env(self, short_name, allow_missing=False):
+
full_name = "env_{}.xml".format(short_name)
+
for env_file in self._files:
+
if os.path.basename(env_file.filename) == full_name:
+
return env_file
+
if allow_missing:
+
return None
+
expect(False, "Could not find object for {} in case".format(full_name))
+
+
+
+
[docs]
+
def check_timestamps(self, short_name=None):
+
if short_name is not None:
+
env_file = self.get_env(short_name)
+
env_file.check_timestamp()
+
else:
+
for env_file in self._files:
+
env_file.check_timestamp()
+
+
+
+
[docs]
+
def copy(self, newcasename, newcaseroot, newcimeroot=None, newsrcroot=None):
+
newcase = deepcopy(self)
+
for env_file in newcase._files: # pylint: disable=protected-access
+
basename = os.path.basename(env_file.filename)
+
newfile = os.path.join(newcaseroot, basename)
+
env_file.change_file(newfile, copy=True)
+
+
if newcimeroot is not None:
+
newcase.set_value("CIMEROOT", newcimeroot)
+
+
if newsrcroot is not None:
+
newcase.set_value("SRCROOT", newsrcroot)
+
+
newcase.set_value("CASE", newcasename)
+
newcase.set_value("CASEROOT", newcaseroot)
+
newcase.set_value("CONTINUE_RUN", "FALSE")
+
newcase.set_value("RESUBMIT", 0)
+
newcase.set_value("CASE_HASH", newcase.new_hash())
+
+
# Important, and subtle: Writability should NOT be copied because
+
# this allows the copy to be modified without needing a "with" statement
+
# which opens the door to tricky errors such as unflushed writes.
+
newcase._read_only_mode = True # pylint: disable=protected-access
+
+
return newcase
+
+
+
+
[docs]
+
def flush(self, flushall=False):
+
if not os.path.isdir(self._caseroot):
+
# do not flush if caseroot wasnt created
+
return
+
+
for env_file in self._files:
+
env_file.write(force_write=flushall)
+
+
+
+
[docs]
+
def get_values(self, item, attribute=None, resolved=True, subgroup=None):
+
for env_file in self._files:
+
# Wait and resolve in self rather than in env_file
+
results = env_file.get_values(
+
item, attribute, resolved=False, subgroup=subgroup
+
)
+
if len(results) > 0:
+
new_results = []
+
if resolved:
+
for result in results:
+
if isinstance(result, str):
+
result = self.get_resolved_value(result)
+
vtype = env_file.get_type_info(item)
+
if vtype is not None or vtype != "char":
+
result = convert_to_type(result, vtype, item)
+
+
new_results.append(result)
+
+
else:
+
new_results.append(result)
+
+
else:
+
new_results = results
+
+
return new_results
+
+
# Return empty result
+
return []
+
+
+
+
[docs]
+
def get_value(self, item, attribute=None, resolved=True, subgroup=None):
+
if item == "GPU_ENABLED":
+
if not self.gpu_enabled:
+
if (
+
self.get_value("GPU_TYPE") != "none"
+
and self.get_value("NGPUS_PER_NODE") > 0
+
):
+
self.gpu_enabled = True
+
return "true" if self.gpu_enabled else "false"
+
+
result = None
+
for env_file in self._files:
+
# Wait and resolve in self rather than in env_file
+
result = env_file.get_value(
+
item, attribute, resolved=False, subgroup=subgroup
+
)
+
+
if result is not None:
+
if resolved and isinstance(result, str):
+
result = self.get_resolved_value(result)
+
vtype = env_file.get_type_info(item)
+
if vtype is not None and vtype != "char":
+
result = convert_to_type(result, vtype, item)
+
+
return result
+
+
# Return empty result
+
return result
+
+
+
+
[docs]
+
def get_record_fields(self, variable, field):
+
"""get_record_fields gets individual requested field from an entry_id file
+
this routine is used only by xmlquery"""
+
# Empty result
+
result = []
+
+
for env_file in self._env_entryid_files:
+
# Wait and resolve in self rather than in env_file
+
logger.debug(
+
"(get_record_field) Searching in {}".format(env_file.__class__.__name__)
+
)
+
if field == "varid":
+
roots = env_file.scan_children("entry")
+
else:
+
roots = env_file.get_nodes_by_id(variable)
+
+
for root in roots:
+
if root is not None:
+
if field == "raw":
+
result.append(env_file.get_raw_record(root))
+
elif field == "desc":
+
result.append(env_file.get_description(root))
+
elif field == "varid":
+
result.append(env_file.get(root, "id"))
+
elif field == "group":
+
result.extend(env_file.get_groups(root))
+
elif field == "valid_values":
+
# pylint: disable=protected-access
+
vv = env_file._get_valid_values(root)
+
if vv:
+
result.extend(vv)
+
elif field == "file":
+
result.append(env_file.filename)
+
+
if not result:
+
for env_file in self._env_generic_files:
+
roots = env_file.scan_children(variable)
+
for root in roots:
+
if root is not None:
+
if field == "raw":
+
result.append(env_file.get_raw_record(root))
+
elif field == "group":
+
result.extend(env_file.get_groups(root))
+
elif field == "file":
+
result.append(env_file.filename)
+
+
return list(set(result))
+
+
+
+
[docs]
+
def get_type_info(self, item):
+
result = None
+
for env_file in self._env_entryid_files:
+
result = env_file.get_type_info(item)
+
if result is not None:
+
return result
+
+
return result
+
+
+
+
[docs]
+
def get_resolved_value(self, item, recurse=0, allow_unresolved_envvars=False):
+
num_unresolved = item.count("$") if item else 0
+
recurse_limit = 10
+
if num_unresolved > 0 and recurse < recurse_limit:
+
for env_file in self._env_entryid_files:
+
item = env_file.get_resolved_value(
+
item, allow_unresolved_envvars=allow_unresolved_envvars
+
)
+
if "$" not in item:
+
return item
+
else:
+
item = self.get_resolved_value(
+
item,
+
recurse=recurse + 1,
+
allow_unresolved_envvars=allow_unresolved_envvars,
+
)
+
+
return item
+
+
+
+
[docs]
+
def set_value(
+
self,
+
item,
+
value,
+
subgroup=None,
+
ignore_type=False,
+
allow_undefined=False,
+
return_file=False,
+
):
+
"""
+
If a file has been defined, and the variable is in the file,
+
then that value will be set in the file object and the resovled value
+
is returned unless return_file is True, in which case (resolved_value, filename)
+
is returned where filename is the name of the modified file.
+
"""
+
expect(
+
not self._read_only_mode,
+
"Cannot modify case, read_only. "
+
"Case must be opened with read_only=False and can only be modified within a context manager",
+
)
+
+
if item == "CASEROOT":
+
self._caseroot = value
+
result = None
+
+
for env_file in self._files:
+
result = env_file.set_value(item, value, subgroup, ignore_type)
+
if result is not None:
+
logger.debug("Will rewrite file {} {}".format(env_file.filename, item))
+
return (result, env_file.filename) if return_file else result
+
+
if len(self._files) == 1:
+
expect(
+
allow_undefined or result is not None,
+
"No variable {} found in file {}".format(item, self._files[0].filename),
+
)
+
else:
+
expect(
+
allow_undefined or result is not None,
+
"No variable {} found in case".format(item),
+
)
+
+
+
+
[docs]
+
def set_valid_values(self, item, valid_values):
+
"""
+
Update or create a valid_values entry for item and populate it
+
"""
+
expect(
+
not self._read_only_mode,
+
"Cannot modify case, read_only. "
+
"Case must be opened with read_only=False and can only be modified within a context manager",
+
)
+
+
result = None
+
for env_file in self._env_entryid_files:
+
result = env_file.set_valid_values(item, valid_values)
+
if result is not None:
+
logger.debug("Will rewrite file {} {}".format(env_file.filename, item))
+
return result
+
+
+
+
[docs]
+
def set_lookup_value(self, item, value):
+
if item in self.lookups and self.lookups[item] is not None:
+
logger.warning(
+
"Item {} already in lookups with value {}".format(
+
item, self.lookups[item]
+
)
+
)
+
else:
+
logger.debug("Setting in lookups: item {}, value {}".format(item, value))
+
self.lookups[item] = value
+
+
+
+
[docs]
+
def clean_up_lookups(self, allow_undefined=False):
+
# put anything in the lookups table into existing env objects
+
for key, value in list(self.lookups.items()):
+
logger.debug("lookup key {} value {}".format(key, value))
+
result = self.set_value(key, value, allow_undefined=allow_undefined)
+
if result is not None:
+
del self.lookups[key]
+
+
+
def _set_compset(self, compset_name, files):
+
"""
+
Loop through all the compset files and find the compset
+
specifation file that matches either the input 'compset_name'.
+
Note that the input compset name (i.e. compset_name) can be
+
either a longname or an alias. This will set various compset-related
+
info.
+
+
Returns a tuple: (compset_alias, science_support, component_defining_compset)
+
(For a user-defined compset - i.e., a compset without an alias - these
+
return values will be None, [], None.)
+
"""
+
science_support = []
+
compset_alias = None
+
components = files.get_components("COMPSETS_SPEC_FILE")
+
logger.debug(
+
" Possible components for COMPSETS_SPEC_FILE are {}".format(components)
+
)
+
+
self.set_lookup_value("COMP_INTERFACE", self._comp_interface)
+
if config.set_comp_root_dir_cpl:
+
if config.use_nems_comp_root_dir:
+
ufs_driver = os.environ.get("UFS_DRIVER")
+
attribute = None
+
if ufs_driver:
+
attribute = {"component": "nems"}
+
comp_root_dir_cpl = files.get_value(
+
"COMP_ROOT_DIR_CPL", attribute=attribute
+
)
+
else:
+
comp_root_dir_cpl = files.get_value("COMP_ROOT_DIR_CPL")
+
+
self.set_lookup_value("COMP_ROOT_DIR_CPL", comp_root_dir_cpl)
+
+
# Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
+
# that has a match for either the alias or the longname in that order
+
for component in components:
+
+
# Determine the compsets file for this component
+
compsets_filename = files.get_value(
+
"COMPSETS_SPEC_FILE", {"component": component}
+
)
+
+
# If the file exists, read it and see if there is a match for the compset alias or longname
+
if os.path.isfile(compsets_filename):
+
compsets = Compsets(compsets_filename)
+
match, compset_alias, science_support = compsets.get_compset_match(
+
name=compset_name
+
)
+
if match is not None:
+
self._compsetname = match
+
logger.info("Compset longname is {}".format(match))
+
logger.info(
+
"Compset specification file is {}".format(compsets_filename)
+
)
+
break
+
+
if compset_alias is None:
+
logger.info(
+
"Did not find an alias or longname compset match for {} ".format(
+
compset_name
+
)
+
)
+
self._compsetname = compset_name
+
+
# Fill in compset name
+
self._compsetname, self._components = self.valid_compset(
+
self._compsetname, compset_alias, files
+
)
+
+
# if this is a valiid compset longname there will be at least 7 components.
+
components = self.get_compset_components()
+
expect(
+
len(components) > 6,
+
"No compset alias {} found and this does not appear to be a compset longname.".format(
+
compset_name
+
),
+
)
+
+
return compset_alias, science_support
+
+
+
[docs]
+
def get_primary_component(self):
+
if self._primary_component is None:
+
self._primary_component = self._find_primary_component()
+
return self._primary_component
+
+
+
def _find_primary_component(self):
+
"""
+
try to glean the primary component based on compset name
+
"""
+
progcomps = {}
+
spec = {}
+
primary_component = None
+
for comp in self._component_classes:
+
if comp == "CPL":
+
continue
+
spec[comp] = self.get_value("COMP_{}".format(comp))
+
notprogcomps = ("D{}".format(comp), "X{}".format(comp), "S{}".format(comp))
+
if spec[comp].upper() in notprogcomps:
+
progcomps[comp] = False
+
else:
+
progcomps[comp] = True
+
expect(
+
"ATM" in progcomps
+
and "LND" in progcomps
+
and "OCN" in progcomps
+
and "ICE" in progcomps,
+
" Not finding expected components in {}".format(self._component_classes),
+
)
+
if (
+
progcomps["ATM"]
+
and progcomps["LND"]
+
and progcomps["OCN"]
+
and progcomps["ICE"]
+
):
+
primary_component = "allactive"
+
elif progcomps["LND"] and progcomps["OCN"] and progcomps["ICE"]:
+
# this is a "J" compset
+
primary_component = "allactive"
+
elif progcomps["ATM"] and progcomps["OCN"] and progcomps["ICE"]:
+
# this is a ufs s2s compset
+
primary_component = "allactive"
+
elif progcomps["ATM"]:
+
if "DOCN%SOM" in self._compsetname and progcomps["LND"]:
+
# This is an "E" compset
+
primary_component = "allactive"
+
else:
+
# This is an "F" or "Q" compset
+
primary_component = spec["ATM"]
+
elif progcomps["LND"]:
+
# This is an "I" compset
+
primary_component = spec["LND"]
+
elif progcomps["OCN"]:
+
# This is a "C" or "G" compset
+
primary_component = spec["OCN"]
+
elif progcomps["ICE"]:
+
# This is a "D" compset
+
primary_component = spec["ICE"]
+
elif "GLC" in progcomps and progcomps["GLC"]:
+
# This is a "TG" compset
+
primary_component = spec["GLC"]
+
elif progcomps["ROF"]:
+
# This is a "R" compset
+
primary_component = spec["ROF"]
+
elif progcomps["WAV"]:
+
# This is a "V" compset
+
primary_component = spec["WAV"]
+
else:
+
# This is "A", "X" or "S"
+
primary_component = "drv"
+
+
return primary_component
+
+
def _valid_compset_impl(self, compset_name, compset_alias, comp_classes, comp_hash):
+
"""Add stub models missing in <compset_name>, return full compset name.
+
<comp_classes> is a list of all supported component classes.
+
<comp_hash> is a dictionary where each key is a supported component
+
(e.g., datm) and the associated value is the index in <comp_classes> of
+
that component's class (e.g., 1 for atm).
+
>>> import os, shutil, tempfile
+
>>> workdir = tempfile.mkdtemp()
+
>>> caseroot = os.path.join(workdir, 'caseroot') # use non-existent caseroot to avoid error about not being a valid case directory in Case __init__ method
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'ESP'], {'datm':1,'satm':1,'dlnd':2,'slnd':2,'dice':3,'sice':3,'docn':4,'socn':4,'drof':5,'srof':5,'sglc':6,'swav':7,'ww3':7,'sesp':8})
+
('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_SESP', ['2000', 'DATM%NYF', 'SLND', 'DICE%SSMI', 'DOCN%DOM', 'DROF%NYF', 'SGLC', 'SWAV', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'ESP'], {'datm':1,'satm':1,'dlnd':2,'slnd':2,'dice':3,'sice':3,'docn':4,'socn':4,'drof':5,'srof':5,'sglc':6,'swav':7,'ww3':7,'sesp':8})
+
('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_SESP', ['2000', 'DATM%NYF', 'SLND', 'DICE%SSMI', 'DOCN%DOM', 'DROF%NYF', 'SGLC', 'SWAV', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('atm:DATM%NYF_rof:DROF%NYF_scn:2000_ice:DICE%SSMI_ocn:DOCN%DOM', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'ESP'], {'datm':1,'satm':1,'dlnd':2,'slnd':2,'dice':3,'sice':3,'docn':4,'socn':4,'drof':5,'srof':5,'sglc':6,'swav':7,'ww3':7,'sesp':8})
+
('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_SESP', ['2000', 'DATM%NYF', 'SLND', 'DICE%SSMI', 'DOCN%DOM', 'DROF%NYF', 'SGLC', 'SWAV', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('2000_DATM%NYF_DICE%SSMI_DOCN%DOM_DROF%NYF', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'ESP'], {'datm':1,'satm':1,'dlnd':2,'slnd':2,'dice':3,'sice':3,'docn':4,'socn':4,'drof':5,'srof':5,'sglc':6,'swav':7,'ww3':7,'sesp':8})
+
('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_SESP', ['2000', 'DATM%NYF', 'SLND', 'DICE%SSMI', 'DOCN%DOM', 'DROF%NYF', 'SGLC', 'SWAV', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('2000_DICE%SSMI_DOCN%DOM_DATM%NYF_DROF%NYF', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'ESP'], {'datm':1,'satm':1,'dlnd':2,'slnd':2,'dice':3,'sice':3,'docn':4,'socn':4,'drof':5,'srof':5,'sglc':6,'swav':7,'ww3':7,'sesp':8})
+
('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_SESP', ['2000', 'DATM%NYF', 'SLND', 'DICE%SSMI', 'DOCN%DOM', 'DROF%NYF', 'SGLC', 'SWAV', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('2000_DICE%SSMI_DOCN%DOM_DATM%NYF_DROF%NYF_TEST', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'ESP'], {'datm':1,'satm':1,'dlnd':2,'slnd':2,'dice':3,'sice':3,'docn':4,'socn':4,'drof':5,'srof':5,'sglc':6,'swav':7,'ww3':7,'sesp':8})
+
('2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_SESP_TEST', ['2000', 'DATM%NYF', 'SLND', 'DICE%SSMI', 'DOCN%DOM', 'DROF%NYF', 'SGLC', 'SWAV', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('1850_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO%ABIO-DIC_MOSART_CISM2%NOEVOLVE_WW3_BGC%BDRD', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'ESP'], {'datm':1,'satm':1, 'cam':1,'dlnd':2,'clm':2,'slnd':2,'cice':3,'dice':3,'sice':3,'pop':4,'docn':4,'socn':4,'mosart':5,'drof':5,'srof':5,'cism':6,'sglc':6,'ww':7,'swav':7,'ww3':7,'sesp':8})
+
('1850_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO%ABIO-DIC_MOSART_CISM2%NOEVOLVE_WW3_SESP_BGC%BDRD', ['1850', 'CAM60', 'CLM50%BGC-CROP', 'CICE', 'POP2%ECO%ABIO-DIC', 'MOSART', 'CISM2%NOEVOLVE', 'WW3', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('1850_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO%ABIO-DIC_MOSART_CISM2%NOEVOLVE_WW3_BGC%BDRD_TEST', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'IAC', 'ESP'], {'datm':1,'satm':1, 'cam':1,'dlnd':2,'clm':2,'slnd':2,'cice':3,'dice':3,'sice':3,'pop':4,'docn':4,'socn':4,'mosart':5,'drof':5,'srof':5,'cism':6,'sglc':6,'ww':7,'swav':7,'ww3':7,'sesp':8})
+
('1850_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO%ABIO-DIC_MOSART_CISM2%NOEVOLVE_WW3_SIAC_SESP_BGC%BDRD_TEST', ['1850', 'CAM60', 'CLM50%BGC-CROP', 'CICE', 'POP2%ECO%ABIO-DIC', 'MOSART', 'CISM2%NOEVOLVE', 'WW3', 'SIAC', 'SESP'])
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('1850_SATM_SLND_SICE_SOCN_SGLC_SWAV', 'S', ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'IAC', 'ESP'], {'datm':1,'satm':1, 'cam':1,'dlnd':2,'clm':2,'slnd':2,'cice':3,'dice':3,'sice':3,'pop':4,'docn':4,'socn':4,'mosart':5,'drof':5,'srof':5,'cism':6,'sglc':6,'ww':7,'swav':7,'ww3':7,'sesp':8})
+
('1850_SATM_SLND_SICE_SOCN_SROF_SGLC_SWAV_SIAC_SESP', ['1850', 'SATM', 'SLND', 'SICE', 'SOCN', 'SROF', 'SGLC', 'SWAV', 'SIAC', 'SESP'])
+
+
>>> Case(caseroot, read_only=False)._valid_compset_impl('1850_SATM_SLND_SICE_SOCN_SGLC_SWAV', None, ['CPL', 'ATM', 'LND', 'ICE', 'OCN', 'ROF', 'GLC', 'WAV', 'IAC', 'ESP'], {'datm':1,'satm':1, 'cam':1,'dlnd':2,'clm':2,'slnd':2,'cice':3,'dice':3,'sice':3,'pop':4,'docn':4,'socn':4,'mosart':5,'drof':5,'srof':5,'cism':6,'sglc':6,'ww':7,'swav':7,'ww3':7,'sesp':8}) #doctest: +IGNORE_EXCEPTION_DETAIL
+
Traceback (most recent call last):
+
CIMEError: ERROR: Invalid compset name, 1850_SATM_SLND_SICE_SOCN_SGLC_SWAV, all stub components generated
+
>>> shutil.rmtree(workdir, ignore_errors=True)
+
"""
+
# Find the models declared in the compset
+
model_set = [None] * len(comp_classes)
+
components = compset_name.split("_")
+
noncomps = []
+
allstubs = True
+
colonformat = ":" in compset_name
+
if colonformat:
+
# make sure that scn: is component[0] as expected
+
for i in range(1, len(components)):
+
if components[i].startswith("scn:"):
+
tmp = components[0]
+
components[0] = components[i]
+
components[i] = tmp
+
break
+
+
model_set[0] = components[0][4:]
+
else:
+
model_set[0] = components[0]
+
+
for model in components[1:]:
+
match = Case.__mod_match_re__.match(model.lower())
+
expect(match is not None, "No model match for {}".format(model))
+
mod_match = match.group(1)
+
# Check for noncomponent appends (BGC & TEST)
+
if mod_match in ("bgc", "test"):
+
noncomps.append(model)
+
elif ":" in mod_match:
+
comp_ind = comp_hash[mod_match[4:]]
+
model_set[comp_ind] = model
+
else:
+
expect(mod_match in comp_hash, "Unknown model type, {}".format(model))
+
comp_ind = comp_hash[mod_match]
+
model_set[comp_ind] = model
+
+
# Fill in missing components with stubs
+
for comp_ind in range(1, len(model_set)):
+
if model_set[comp_ind] is None:
+
comp_class = comp_classes[comp_ind]
+
stub = "S" + comp_class
+
logger.info("Automatically adding {} to compset".format(stub))
+
model_set[comp_ind] = stub
+
elif ":" in model_set[comp_ind]:
+
model_set[comp_ind] = model_set[comp_ind][4:]
+
+
if model_set[comp_ind][0] != "S":
+
allstubs = False
+
+
expect(
+
(compset_alias is not None) or (not allstubs),
+
"Invalid compset name, {}, all stub components generated".format(
+
compset_name
+
),
+
)
+
# Return the completed compset
+
compsetname = "_".join(model_set)
+
for noncomp in noncomps:
+
compsetname = compsetname + "_" + noncomp
+
return compsetname, model_set
+
+
# RE to match component type name without optional piece (stuff after %).
+
# Drop any trailing digits (e.g., the 60 in CAM60) to ensure match
+
# Note, this will also drop trailing digits such as in ww3 but since it
+
# is handled consistenly, this should not affect functionality.
+
# Note: interstitial digits are included (e.g., in FV3GFS).
+
__mod_match_re__ = re.compile(r"([^%]*[^0-9%]+)")
+
+
+
[docs]
+
def valid_compset(self, compset_name, compset_alias, files):
+
"""Add stub models missing in <compset_name>, return full compset name.
+
<files> is used to collect set of all supported components.
+
"""
+
# First, create hash of model names
+
# A note about indexing. Relevant component classes start at 1
+
# because we ignore CPL for finding model components.
+
# Model components would normally start at zero but since we are
+
# dealing with a compset, 0 is reserved for the time field
+
drv_config_file = files.get_value("CONFIG_CPL_FILE")
+
drv_comp = Component(drv_config_file, "CPL")
+
comp_classes = drv_comp.get_valid_model_components()
+
comp_hash = {} # Hash model name to component class index
+
for comp_ind in range(1, len(comp_classes)):
+
comp = comp_classes[comp_ind]
+
# Find list of models for component class
+
# List can be in different locations, check CONFIG_XXX_FILE
+
node_name = "CONFIG_{}_FILE".format(comp)
+
models = files.get_components(node_name)
+
if (models is None) or (None in models):
+
# Backup, check COMP_ROOT_DIR_XXX
+
node_name = "COMP_ROOT_DIR_" + comp
+
models = files.get_components(node_name)
+
+
expect(
+
(models is not None) and (None not in models),
+
"Unable to find list of supported components",
+
)
+
+
for model in models:
+
mod_match = Case.__mod_match_re__.match(model.lower()).group(1)
+
comp_hash[mod_match] = comp_ind
+
+
return self._valid_compset_impl(
+
compset_name, compset_alias, comp_classes, comp_hash
+
)
+
+
+
def _set_info_from_primary_component(self, files, pesfile=None):
+
"""
+
Sets file and directory paths that depend on the primary component of
+
this compset.
+
+
Assumes that self._primary_component has already been set.
+
"""
+
component = self.get_primary_component()
+
+
compset_spec_file = files.get_value(
+
"COMPSETS_SPEC_FILE", {"component": component}, resolved=False
+
)
+
+
self.set_lookup_value("COMPSETS_SPEC_FILE", compset_spec_file)
+
if pesfile is None:
+
self._pesfile = files.get_value("PES_SPEC_FILE", {"component": component})
+
pesfile_unresolved = files.get_value(
+
"PES_SPEC_FILE", {"component": component}, resolved=False
+
)
+
logger.info("Pes specification file is {}".format(self._pesfile))
+
else:
+
self._pesfile = pesfile
+
pesfile_unresolved = pesfile
+
expect(
+
self._pesfile is not None,
+
"No pesfile found for component {}".format(component),
+
)
+
+
self.set_lookup_value("PES_SPEC_FILE", pesfile_unresolved)
+
+
tests_filename = files.get_value(
+
"TESTS_SPEC_FILE", {"component": component}, resolved=False
+
)
+
tests_mods_dir = files.get_value(
+
"TESTS_MODS_DIR", {"component": component}, resolved=False
+
)
+
user_mods_dir = files.get_value(
+
"USER_MODS_DIR", {"component": component}, resolved=False
+
)
+
self.set_lookup_value("TESTS_SPEC_FILE", tests_filename)
+
self.set_lookup_value("TESTS_MODS_DIR", tests_mods_dir)
+
self.set_lookup_value("USER_MODS_DIR", user_mods_dir)
+
+
+
[docs]
+
def get_compset_components(self):
+
# If are doing a create_clone then, self._compsetname is not set yet
+
components = []
+
compset = self.get_value("COMPSET")
+
if compset is None:
+
compset = self._compsetname
+
expect(compset is not None, "compset is not set")
+
# the first element is always the date operator - skip it
+
elements = compset.split("_")[1:] # pylint: disable=maybe-no-member
+
for element in elements:
+
if ":" in element:
+
element = element[4:]
+
# ignore the possible BGC or TEST modifier
+
if element.startswith("BGC%") or element.startswith("TEST"):
+
continue
+
else:
+
element_component = element.split("%")[0].lower()
+
if (
+
"ww" not in element_component
+
and "fv3" not in element_component
+
and "cice" not in element_component
+
):
+
element_component = re.sub(r"[0-9]*", "", element_component)
+
components.append(element_component)
+
return components
+
+
+
def __iter__(self):
+
for entryid_file in self._env_entryid_files:
+
for key, val in entryid_file:
+
if isinstance(val, str) and "$" in val:
+
yield key, self.get_resolved_value(val)
+
else:
+
yield key, val
+
+
+
[docs]
+
def set_comp_classes(self, comp_classes):
+
self._component_classes = comp_classes
+
for env_file in self._env_entryid_files:
+
env_file.set_components(comp_classes)
+
+
+
def _get_component_config_data(self, files):
+
# attributes used for multi valued defaults
+
# attlist is a dictionary used to determine the value element that has the most matches
+
attlist = {
+
"compset": self._compsetname,
+
"grid": self._gridname,
+
"cime_model": self._cime_model,
+
}
+
+
# Determine list of component classes that this coupler/driver knows how
+
# to deal with. This list follows the same order as compset longnames follow.
+
+
# Add the group and elements for the config_files.xml
+
for env_file in self._env_entryid_files:
+
env_file.add_elements_by_group(files, attlist)
+
+
drv_config_file = files.get_value("CONFIG_CPL_FILE")
+
drv_comp = Component(drv_config_file, "CPL")
+
for env_file in self._env_entryid_files:
+
env_file.add_elements_by_group(drv_comp, attributes=attlist)
+
+
drv_config_file_model_specific = files.get_value(
+
"CONFIG_CPL_FILE_MODEL_SPECIFIC"
+
)
+
expect(
+
os.path.isfile(drv_config_file_model_specific),
+
"No {} specific file found for driver {}".format(
+
get_model(), self._comp_interface
+
),
+
)
+
drv_comp_model_specific = Component(drv_config_file_model_specific, "CPL")
+
+
self._component_description[
+
"forcing"
+
] = drv_comp_model_specific.get_forcing_description(self._compsetname)
+
logger.info(
+
"Compset forcing is {}".format(self._component_description["forcing"])
+
)
+
self._component_description["CPL"] = drv_comp_model_specific.get_description(
+
self._compsetname
+
)
+
if len(self._component_description["CPL"]) > 0:
+
logger.info("Com forcing is {}".format(self._component_description["CPL"]))
+
for env_file in self._env_entryid_files:
+
env_file.add_elements_by_group(drv_comp_model_specific, attributes=attlist)
+
+
self.clean_up_lookups(allow_undefined=True)
+
+
# loop over all elements of both component_classes and components - and get config_component_file for
+
# for each component
+
self.set_comp_classes(drv_comp.get_valid_model_components())
+
+
# will need a change here for new cpl components
+
root_dir_node_name = "COMP_ROOT_DIR_CPL"
+
comp_root_dir = files.get_value(
+
root_dir_node_name, {"component": self._comp_interface}, resolved=False
+
)
+
+
if comp_root_dir is not None:
+
self.set_value(root_dir_node_name, comp_root_dir)
+
+
for i in range(1, len(self._component_classes)):
+
comp_class = self._component_classes[i]
+
comp_name = self._components[i - 1]
+
if ":" in comp_name:
+
comp_name = comp_name[4:]
+
root_dir_node_name = "COMP_ROOT_DIR_" + comp_class
+
node_name = "CONFIG_" + comp_class + "_FILE"
+
compatt = {"component": comp_name}
+
comp_root_dir = files.get_value(root_dir_node_name, compatt, resolved=False)
+
if comp_root_dir is not None:
+
self.set_value(root_dir_node_name, comp_root_dir)
+
+
# Add the group and elements for the config_files.xml
+
+
comp_config_file = files.get_value(node_name, compatt, resolved=False)
+
expect(
+
comp_config_file is not None,
+
"No component {} found for class {}".format(comp_name, comp_class),
+
)
+
self.set_value(node_name, comp_config_file)
+
comp_config_file = files.get_value(node_name, compatt)
+
+
expect(
+
comp_config_file is not None and os.path.isfile(comp_config_file),
+
"Config file {} for component {} not found.".format(
+
comp_config_file, comp_name
+
),
+
)
+
compobj = Component(comp_config_file, comp_class)
+
# For files following version 3 schema this also checks the compsetname validity
+
+
self._component_description[comp_class] = compobj.get_description(
+
self._compsetname
+
)
+
expect(
+
self._component_description[comp_class] is not None,
+
"No description found in file {} for component {} in comp_class {}".format(
+
comp_config_file, comp_name, comp_class
+
),
+
)
+
logger.info(
+
"{} component is {}".format(
+
comp_class, self._component_description[comp_class]
+
)
+
)
+
for env_file in self._env_entryid_files:
+
env_file.add_elements_by_group(compobj, attributes=attlist)
+
self.clean_up_lookups(allow_undefined=self._comp_interface == "nuopc")
+
+
def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib):
+
# --------------------------------------------
+
# pe layout
+
# --------------------------------------------
+
mach_pes_obj = None
+
# self._pesfile may already be env_mach_pes.xml if so we can just return
+
gfile = GenericXML(infile=self._pesfile)
+
ftype = gfile.get_id()
+
expect(
+
ftype == "env_mach_pes.xml" or ftype == "config_pes",
+
" Do not recognize {} as a valid CIME pes file {}".format(
+
self._pesfile, ftype
+
),
+
)
+
if ftype == "env_mach_pes.xml":
+
new_mach_pes_obj = EnvMachPes(
+
infile=self._pesfile,
+
components=self._component_classes,
+
comp_interface=self._comp_interface,
+
)
+
self.update_env(new_mach_pes_obj, "mach_pes", blow_away=True)
+
return new_mach_pes_obj.get_value("TOTALPES")
+
+
pesobj = Pes(self._pesfile)
+
+
match1 = re.match("(.+)x([0-9]+)", "" if pecount is None else pecount)
+
match2 = re.match("([0-9]+)", "" if pecount is None else pecount)
+
+
pes_ntasks = {}
+
pes_nthrds = {}
+
pes_rootpe = {}
+
pes_pstrid = {}
+
other = {}
+
comment = None
+
force_tasks = None
+
force_thrds = None
+
if match1:
+
opti_tasks = match1.group(1)
+
if opti_tasks.isdigit():
+
force_tasks = int(opti_tasks)
+
else:
+
pes_ntasks = pesobj.find_pes_layout(
+
self._gridname,
+
self._compsetname,
+
machine_name,
+
pesize_opts=opti_tasks,
+
mpilib=mpilib,
+
)[0]
+
force_thrds = int(match1.group(2))
+
elif match2:
+
force_tasks = int(match2.group(1))
+
pes_nthrds = pesobj.find_pes_layout(
+
self._gridname, self._compsetname, machine_name, mpilib=mpilib
+
)[1]
+
else:
+
(
+
pes_ntasks,
+
pes_nthrds,
+
pes_rootpe,
+
pes_pstrid,
+
other,
+
comment,
+
) = pesobj.find_pes_layout(
+
self._gridname,
+
self._compsetname,
+
machine_name,
+
pesize_opts=pecount,
+
mpilib=mpilib,
+
)
+
+
if match1 or match2:
+
for component_class in self._component_classes:
+
if force_tasks is not None:
+
string_ = "NTASKS_" + component_class
+
pes_ntasks[string_] = force_tasks
+
+
if force_thrds is not None:
+
string_ = "NTHRDS_" + component_class
+
pes_nthrds[string_] = force_thrds
+
+
# Always default to zero rootpe if user forced procs and or threads
+
string_ = "ROOTPE_" + component_class
+
pes_rootpe[string_] = 0
+
+
mach_pes_obj = self.get_env("mach_pes")
+
mach_pes_obj.add_comment(comment)
+
+
if other is not None:
+
logger.info("setting additional fields from config_pes: {}".format(other))
+
for key, value in list(other.items()):
+
self.set_value(key, value)
+
+
totaltasks = []
+
for comp_class in self._component_classes:
+
ntasks_str = "NTASKS_{}".format(comp_class)
+
nthrds_str = "NTHRDS_{}".format(comp_class)
+
rootpe_str = "ROOTPE_{}".format(comp_class)
+
pstrid_str = "PSTRID_{}".format(comp_class)
+
+
ntasks = pes_ntasks[ntasks_str] if ntasks_str in pes_ntasks else 1
+
nthrds = pes_nthrds[nthrds_str] if nthrds_str in pes_nthrds else 1
+
rootpe = pes_rootpe[rootpe_str] if rootpe_str in pes_rootpe else 0
+
pstrid = pes_pstrid[pstrid_str] if pstrid_str in pes_pstrid else 1
+
+
totaltasks.append((ntasks + rootpe) * nthrds)
+
mach_pes_obj.set_value(ntasks_str, ntasks)
+
mach_pes_obj.set_value(nthrds_str, nthrds)
+
mach_pes_obj.set_value(rootpe_str, rootpe)
+
mach_pes_obj.set_value(pstrid_str, pstrid)
+
+
# Make sure that every component has been accounted for
+
# set, nthrds and ntasks to 1 otherwise. Also set the ninst values here.
+
for compclass in self._component_classes:
+
key = "NINST_{}".format(compclass)
+
if compclass == "CPL":
+
continue
+
mach_pes_obj.set_value(key, ninst)
+
+
key = "NTASKS_{}".format(compclass)
+
if key not in pes_ntasks:
+
mach_pes_obj.set_value(key, 1)
+
+
key = "NTHRDS_{}".format(compclass)
+
if key not in pes_nthrds:
+
mach_pes_obj.set_value(key, 1)
+
+
if multi_driver:
+
mach_pes_obj.set_value("MULTI_DRIVER", True)
+
+
+
+
+
+
[docs]
+
def get_compset_var_settings(self, files):
+
infile = files.get_value(
+
"COMPSETS_SPEC_FILE", attribute={"component": self._primary_component}
+
)
+
compset_obj = Compsets(infile=infile, files=files)
+
matches = compset_obj.get_compset_var_settings(
+
self._compsetname, self._gridname
+
)
+
for name, value in matches:
+
if len(value) > 0:
+
logger.info(
+
"Compset specific settings: name is {} and value is {}".format(
+
name, value
+
)
+
)
+
self.set_lookup_value(name, value)
+
+
+
+
[docs]
+
def set_initial_test_values(self):
+
testobj = self.get_env("test")
+
testobj.set_initial_values(self)
+
+
+
+
[docs]
+
def get_batch_jobs(self):
+
batchobj = self.get_env("batch")
+
return batchobj.get_jobs()
+
+
+
def _set_pio_xml(self):
+
pioobj = PIO(self._component_classes)
+
grid = self.get_value("GRID")
+
compiler = self.get_value("COMPILER")
+
mach = self.get_value("MACH")
+
compset = self.get_value("COMPSET")
+
mpilib = self.get_value("MPILIB")
+
+
defaults = pioobj.get_defaults(
+
grid=grid, compset=compset, mach=mach, compiler=compiler, mpilib=mpilib
+
)
+
+
for vid, value in list(defaults.items()):
+
self.set_value(vid, value)
+
+
def _create_caseroot_tools(self):
+
machines_dir = os.path.abspath(self.get_value("MACHDIR"))
+
machine = self.get_value("MACH")
+
toolsdir = os.path.join(self.get_value("CIMEROOT"), "CIME", "Tools")
+
casetools = os.path.join(self._caseroot, "Tools")
+
# setup executable files in caseroot/
+
exefiles = (
+
os.path.join(toolsdir, "case.setup"),
+
os.path.join(toolsdir, "case.build"),
+
os.path.join(toolsdir, "case.submit"),
+
os.path.join(toolsdir, "case.qstatus"),
+
os.path.join(toolsdir, "case.cmpgen_namelists"),
+
os.path.join(toolsdir, "preview_namelists"),
+
os.path.join(toolsdir, "preview_run"),
+
os.path.join(toolsdir, "check_input_data"),
+
os.path.join(toolsdir, "check_case"),
+
os.path.join(toolsdir, "xmlchange"),
+
os.path.join(toolsdir, "xmlquery"),
+
os.path.join(toolsdir, "pelayout"),
+
)
+
try:
+
for exefile in exefiles:
+
destfile = os.path.join(self._caseroot, os.path.basename(exefile))
+
os.symlink(exefile, destfile)
+
except Exception as e:
+
logger.warning("FAILED to set up exefiles: {}".format(str(e)))
+
+
toolfiles = [
+
os.path.join(toolsdir, "check_lockedfiles"),
+
os.path.join(toolsdir, "get_standard_makefile_args"),
+
os.path.join(toolsdir, "getTiming"),
+
os.path.join(toolsdir, "save_provenance"),
+
os.path.join(toolsdir, "Makefile"),
+
os.path.join(toolsdir, "mkSrcfiles"),
+
os.path.join(toolsdir, "mkDepends"),
+
]
+
+
# used on Titan
+
if os.path.isfile(os.path.join(toolsdir, "mdiag_reduce.csh")):
+
toolfiles.append(os.path.join(toolsdir, "mdiag_reduce.csh"))
+
toolfiles.append(os.path.join(toolsdir, "mdiag_reduce.pl"))
+
+
for toolfile in toolfiles:
+
destfile = os.path.join(casetools, os.path.basename(toolfile))
+
expect(os.path.isfile(toolfile), " File {} does not exist".format(toolfile))
+
try:
+
os.symlink(toolfile, destfile)
+
except Exception as e:
+
logger.warning(
+
"FAILED to set up toolfiles: {} {} {}".format(
+
str(e), toolfile, destfile
+
)
+
)
+
+
if config.copy_e3sm_tools:
+
if os.path.exists(os.path.join(machines_dir, "syslog.{}".format(machine))):
+
safe_copy(
+
os.path.join(machines_dir, "syslog.{}".format(machine)),
+
os.path.join(casetools, "mach_syslog"),
+
)
+
else:
+
safe_copy(
+
os.path.join(machines_dir, "syslog.noop"),
+
os.path.join(casetools, "mach_syslog"),
+
)
+
+
srcroot = self.get_value("SRCROOT")
+
customize_path = os.path.join(srcroot, "cime_config", "customize")
+
safe_copy(os.path.join(customize_path, "e3sm_compile_wrap.py"), casetools)
+
+
# add archive_metadata to the CASEROOT but only for CESM
+
if config.copy_cesm_tools:
+
try:
+
exefile = os.path.join(toolsdir, "archive_metadata")
+
destfile = os.path.join(self._caseroot, os.path.basename(exefile))
+
os.symlink(exefile, destfile)
+
except Exception as e:
+
logger.warning("FAILED to set up exefiles: {}".format(str(e)))
+
+
def _create_caseroot_sourcemods(self):
+
components = self.get_compset_components()
+
components.extend(["share", "drv"])
+
if self._comp_interface == "nuopc":
+
components.extend(["cdeps"])
+
+
readme_message_start = (
+
"Put source mods for the {component} library in this directory."
+
)
+
readme_message_end = """
+
+
WARNING: SourceMods are not kept under version control, and can easily
+
become out of date if changes are made to the source code on which they
+
are based. We only recommend using SourceMods for small, short-term
+
changes that just apply to one or two cases. For larger or longer-term
+
changes, including gradual, incremental changes towards a final
+
solution, we highly recommend making changes in the main source tree,
+
leveraging version control (git or svn).
+
"""
+
+
for component in components:
+
directory = os.path.join(
+
self._caseroot, "SourceMods", "src.{}".format(component)
+
)
+
# don't make SourceMods for stub components
+
if not os.path.exists(directory) and component not in (
+
"satm",
+
"slnd",
+
"sice",
+
"socn",
+
"sesp",
+
"sglc",
+
"swav",
+
):
+
os.makedirs(directory)
+
# Besides giving some information on SourceMods, this
+
# README file serves one other important purpose: By
+
# putting a file inside each SourceMods subdirectory, we
+
# prevent aggressive scrubbers from scrubbing these
+
# directories due to being empty (which can cause builds
+
# to fail).
+
readme_file = os.path.join(directory, "README")
+
with open(readme_file, "w") as fd:
+
fd.write(readme_message_start.format(component=component))
+
+
if component == "cdeps":
+
readme_message_extra = """
+
+
Note that this subdirectory should only contain files from CDEPS's
+
dshr and streams source code directories.
+
Files related to specific data models should go in SourceMods subdirectories
+
for those data models (e.g., src.datm)."""
+
fd.write(readme_message_extra)
+
+
fd.write(readme_message_end)
+
+
if config.copy_cism_source_mods:
+
# Note: this is CESM specific, given that we are referencing cism explitly
+
if "cism" in components:
+
directory = os.path.join(
+
self._caseroot, "SourceMods", "src.cism", "source_cism"
+
)
+
if not os.path.exists(directory):
+
os.makedirs(directory)
+
readme_file = os.path.join(directory, "README")
+
str_to_write = """Put source mods for the source_cism library in this subdirectory.
+
This includes any files from $COMP_ROOT_DIR_GLC/source_cism. Anything
+
else (e.g., mods to source_glc or drivers) goes in the src.cism
+
directory, NOT in this subdirectory."""
+
+
with open(readme_file, "w") as fd:
+
fd.write(str_to_write)
+
+
+
[docs]
+
def create_caseroot(self, clone=False):
+
if not os.path.exists(self._caseroot):
+
# Make the case directory
+
logger.info(" Creating Case directory {}".format(self._caseroot))
+
os.makedirs(self._caseroot)
+
os.chdir(self._caseroot)
+
+
# Create relevant directories in $self._caseroot
+
if clone:
+
newdirs = (LOCKED_DIR, "Tools")
+
else:
+
newdirs = ("SourceMods", LOCKED_DIR, "Buildconf", "Tools")
+
for newdir in newdirs:
+
os.makedirs(newdir)
+
+
# Open a new README.case file in $self._caseroot
+
append_status(" ".join(sys.argv), "README.case", caseroot=self._caseroot)
+
compset_info = "Compset longname is {}".format(self.get_value("COMPSET"))
+
append_status(compset_info, "README.case", caseroot=self._caseroot)
+
append_status(
+
"Compset specification file is {}".format(
+
self.get_value("COMPSETS_SPEC_FILE")
+
),
+
"README.case",
+
caseroot=self._caseroot,
+
)
+
append_status(
+
"Pes specification file is {}".format(self.get_value("PES_SPEC_FILE")),
+
"README.case",
+
caseroot=self._caseroot,
+
)
+
if "forcing" in self._component_description:
+
append_status(
+
"Forcing is {}".format(self._component_description["forcing"]),
+
"README.case",
+
caseroot=self._caseroot,
+
)
+
for component_class in self._component_classes:
+
if (
+
component_class in self._component_description
+
and len(self._component_description[component_class]) > 0
+
):
+
append_status(
+
"Component {} is {}".format(
+
component_class, self._component_description[component_class]
+
),
+
"README.case",
+
caseroot=self._caseroot,
+
)
+
if component_class == "CPL":
+
append_status(
+
"Using %s coupler instances" % (self.get_value("NINST_CPL")),
+
"README.case",
+
caseroot=self._caseroot,
+
)
+
continue
+
comp_grid = "{}_GRID".format(component_class)
+
+
append_status(
+
"{} is {}".format(comp_grid, self.get_value(comp_grid)),
+
"README.case",
+
caseroot=self._caseroot,
+
)
+
comp = str(self.get_value("COMP_{}".format(component_class)))
+
user_mods = self._get_comp_user_mods(comp)
+
if user_mods is not None:
+
note = "This component includes user_mods {}".format(user_mods)
+
append_status(note, "README.case", caseroot=self._caseroot)
+
logger.info(note)
+
if not clone:
+
self._create_caseroot_sourcemods()
+
self._create_caseroot_tools()
+
+
+
+
[docs]
+
def apply_user_mods(self, user_mods_dirs=None):
+
"""
+
User mods can be specified on the create_newcase command line (usually when called from create test)
+
or they can be in the compset definition, or both.
+
+
If user_mods_dirs is specified, it should be a list of paths giving the user mods
+
specified on the create_newcase command line.
+
"""
+
all_user_mods = []
+
for comp in self._component_classes:
+
component = str(self.get_value("COMP_{}".format(comp)))
+
if component == self._primary_component:
+
continue
+
comp_user_mods = self._get_comp_user_mods(component)
+
if comp_user_mods is not None:
+
all_user_mods.append(comp_user_mods)
+
# get the primary last so that it takes precidence over other components
+
comp_user_mods = self._get_comp_user_mods(self._primary_component)
+
if comp_user_mods is not None:
+
all_user_mods.append(comp_user_mods)
+
if user_mods_dirs is not None:
+
all_user_mods.extend(user_mods_dirs)
+
+
# This looping order will lead to the specified user_mods_dirs taking
+
# precedence over self._user_mods, if there are any conflicts.
+
for user_mods in all_user_mods:
+
if os.path.isabs(user_mods):
+
user_mods_path = user_mods
+
else:
+
user_mods_path = self.get_value("USER_MODS_DIR")
+
user_mods_path = os.path.join(user_mods_path, user_mods)
+
apply_user_mods(self._caseroot, user_mods_path)
+
+
# User mods may have modified underlying XML files
+
if all_user_mods:
+
self.read_xml()
+
+
+
def _get_comp_user_mods(self, component):
+
"""
+
For a component 'foo', gets the value of FOO_USER_MODS.
+
+
Returns None if no value was found, or if the value is an empty string.
+
"""
+
comp_user_mods = self.get_value("{}_USER_MODS".format(component.upper()))
+
# pylint: disable=no-member
+
if comp_user_mods is None or comp_user_mods == "" or comp_user_mods.isspace():
+
return None
+
else:
+
return comp_user_mods
+
+
+
[docs]
+
def submit_jobs(
+
self,
+
no_batch=False,
+
job=None,
+
skip_pnl=None,
+
prereq=None,
+
allow_fail=False,
+
resubmit_immediate=False,
+
mail_user=None,
+
mail_type=None,
+
batch_args=None,
+
dry_run=False,
+
workflow=True,
+
):
+
env_batch = self.get_env("batch")
+
result = env_batch.submit_jobs(
+
self,
+
no_batch=no_batch,
+
skip_pnl=skip_pnl,
+
job=job,
+
user_prereq=prereq,
+
allow_fail=allow_fail,
+
resubmit_immediate=resubmit_immediate,
+
mail_user=mail_user,
+
mail_type=mail_type,
+
batch_args=batch_args,
+
dry_run=dry_run,
+
workflow=workflow,
+
)
+
return result
+
+
+
+
[docs]
+
def get_job_info(self):
+
"""
+
Get information on batch jobs associated with this case
+
"""
+
xml_job_ids = self.get_value("JOB_IDS")
+
if not xml_job_ids:
+
return {}
+
else:
+
result = {}
+
job_infos = xml_job_ids.split(", ") # pylint: disable=no-member
+
for job_info in job_infos:
+
jobname, jobid = job_info.split(":")
+
result[jobname] = jobid
+
+
return result
+
+
+
+
[docs]
+
def get_job_id(self, output):
+
env_batch = self.get_env("batch")
+
return env_batch.get_job_id(output)
+
+
+
+
[docs]
+
def report_job_status(self):
+
jobmap = self.get_job_info()
+
if not jobmap:
+
logger.info(
+
"No job ids associated with this case. Either case.submit was not run or was run with no-batch"
+
)
+
else:
+
for jobname, jobid in list(jobmap.items()):
+
status = self.get_env("batch").get_status(jobid)
+
if status:
+
logger.info("{}: {}".format(jobname, status))
+
else:
+
logger.info(
+
"{}: Unable to get status. Job may be complete already.".format(
+
jobname
+
)
+
)
+
+
+
+
[docs]
+
def cancel_batch_jobs(self, jobids):
+
env_batch = self.get_env("batch")
+
for jobid in jobids:
+
success = env_batch.cancel_job(jobid)
+
if not success:
+
logger.warning("Failed to kill {}".format(jobid))
+
+
+
+
[docs]
+
def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True, overrides=None):
+
if job is None:
+
job = self.get_primary_job()
+
+
env_mach_specific = self.get_env("mach_specific")
+
run_exe = env_mach_specific.get_value("run_exe")
+
run_misc_suffix = env_mach_specific.get_value("run_misc_suffix")
+
run_misc_suffix = "" if run_misc_suffix is None else run_misc_suffix
+
+
mpirun_cmd_override = self.get_value("MPI_RUN_COMMAND")
+
if mpirun_cmd_override not in ["", None, "UNSET"]:
+
return self.get_resolved_value(
+
mpirun_cmd_override + " " + run_exe + " " + run_misc_suffix
+
)
+
queue = self.get_value("JOB_QUEUE", subgroup=job)
+
+
# Things that will have to be matched against mpirun element attributes
+
mpi_attribs = {
+
"compiler": self.get_value("COMPILER"),
+
"mpilib": self.get_value("MPILIB"),
+
"threaded": self.get_build_threaded(),
+
"queue": queue,
+
"unit_testing": False,
+
"comp_interface": self._comp_interface,
+
}
+
+
(
+
executable,
+
mpi_arg_list,
+
custom_run_exe,
+
custom_run_misc_suffix,
+
) = env_mach_specific.get_mpirun(self, mpi_attribs, job)
+
if custom_run_exe:
+
logger.info("Using a custom run_exe {}".format(custom_run_exe))
+
run_exe = custom_run_exe
+
if custom_run_misc_suffix:
+
logger.info(
+
"Using a custom run_misc_suffix {}".format(custom_run_misc_suffix)
+
)
+
run_misc_suffix = custom_run_misc_suffix
+
+
aprun_mode = env_mach_specific.get_aprun_mode(mpi_attribs)
+
+
# special case for aprun
+
if (
+
executable is not None
+
and "aprun" in executable
+
and aprun_mode != "ignore"
+
# and not "theta" in self.get_value("MACH")
+
):
+
extra_args = env_mach_specific.get_aprun_args(
+
self, mpi_attribs, job, overrides=overrides
+
)
+
+
aprun_args, num_nodes, _, _, _ = get_aprun_cmd_for_case(
+
self,
+
run_exe,
+
overrides=overrides,
+
extra_args=extra_args,
+
)
+
if job in ("case.run", "case.test"):
+
expect(
+
(num_nodes + self.spare_nodes) == self.num_nodes,
+
"Not using optimized num nodes",
+
)
+
return self.get_resolved_value(
+
executable + aprun_args + " " + run_misc_suffix,
+
allow_unresolved_envvars=allow_unresolved_envvars,
+
)
+
+
else:
+
mpi_arg_string = " ".join(mpi_arg_list)
+
+
if self.get_value("BATCH_SYSTEM") == "cobalt":
+
mpi_arg_string += " : "
+
+
ngpus_per_node = self.get_value("NGPUS_PER_NODE")
+
if ngpus_per_node and ngpus_per_node > 0:
+
mpi_gpu_run_script = self.get_value("MPI_GPU_WRAPPER_SCRIPT")
+
if mpi_gpu_run_script:
+
mpi_arg_string = mpi_arg_string + " " + mpi_gpu_run_script
+
+
return self.get_resolved_value(
+
"{} {} {} {}".format(
+
executable if executable is not None else "",
+
mpi_arg_string,
+
run_exe,
+
run_misc_suffix,
+
),
+
allow_unresolved_envvars=allow_unresolved_envvars,
+
)
+
+
+
+
[docs]
+
def set_model_version(self, model):
+
version = "unknown"
+
srcroot = self.get_value("SRCROOT")
+
version = get_current_commit(True, srcroot, tag=(model == "cesm"))
+
+
self.set_value("MODEL_VERSION", version)
+
+
if version != "unknown":
+
logger.info("{} model version found: {}".format(model, version))
+
else:
+
logger.warning("WARNING: No {} Model version found.".format(model))
+
+
+
+
[docs]
+
def load_env(self, reset=False, job=None, verbose=False):
+
if not self._is_env_loaded or reset:
+
if job is None:
+
job = self.get_primary_job()
+
os.environ["OMP_NUM_THREADS"] = str(self.thread_count)
+
env_module = self.get_env("mach_specific")
+
self._loaded_envs = env_module.load_env(self, job=job, verbose=verbose)
+
self._loaded_envs.append(("OMP_NUM_THREADS", os.environ["OMP_NUM_THREADS"]))
+
self._is_env_loaded = True
+
+
return self._loaded_envs
+
+
+
+
[docs]
+
def get_build_threaded(self):
+
"""
+
Returns True if current settings require a threaded build/run.
+
"""
+
force_threaded = self.get_value("FORCE_BUILD_SMP")
+
if not self.thread_count:
+
return False
+
smp_present = force_threaded or self.thread_count > 1
+
return smp_present
+
+
+
def _check_testlists(self, compset_alias, grid_name, files):
+
"""
+
CESM only: check the testlist file for tests of this compset grid combination
+
+
compset_alias should be None for a user-defined compset (i.e., a compset
+
without an alias)
+
"""
+
if "TESTS_SPEC_FILE" in self.lookups:
+
tests_spec_file = self.get_resolved_value(self.lookups["TESTS_SPEC_FILE"])
+
else:
+
tests_spec_file = self.get_value("TESTS_SPEC_FILE")
+
+
testcnt = 0
+
if os.path.isfile(tests_spec_file) and compset_alias is not None:
+
# It's important that we not try to find matching tests if
+
# compset_alias is None, since compset=None tells get_tests to find
+
# tests of all compsets!
+
# Only collect supported tests as this _check_testlists is only
+
# called if run_unsupported is False.
+
tests = Testlist(tests_spec_file, files)
+
testlist = tests.get_tests(
+
compset=compset_alias, grid=grid_name, supported_only=True
+
)
+
test_categories = ["prealpha", "prebeta"]
+
for test in testlist:
+
if (
+
test["category"] in test_categories
+
or "aux_" in test["category"]
+
or get_cime_default_driver() in test["category"]
+
):
+
testcnt += 1
+
if testcnt > 0:
+
logger.warning(
+
"\n*********************************************************************************************************************************"
+
)
+
logger.warning(
+
"This compset and grid combination is not scientifically supported, however it is used in {:d} tests.".format(
+
testcnt
+
)
+
)
+
logger.warning(
+
"*********************************************************************************************************************************\n"
+
)
+
else:
+
expect(
+
False,
+
"\nThis compset and grid combination is untested in CESM. "
+
"Override this warning with the --run-unsupported option to create_newcase.",
+
error_prefix="STOP: ",
+
)
+
+
+
[docs]
+
def set_file(self, xmlfile):
+
"""
+
force the case object to consider only xmlfile
+
"""
+
expect(os.path.isfile(xmlfile), "Could not find file {}".format(xmlfile))
+
+
if not self._read_only_mode:
+
self.flush(flushall=True)
+
+
gfile = GenericXML(infile=xmlfile)
+
ftype = gfile.get_id()
+
+
logger.warning("setting case file to {}".format(xmlfile))
+
components = self.get_value("COMP_CLASSES")
+
new_env_file = None
+
for env_file in self._files:
+
if os.path.basename(env_file.filename) == ftype:
+
if ftype == "env_run.xml":
+
new_env_file = EnvRun(infile=xmlfile, components=components)
+
elif ftype == "env_build.xml":
+
new_env_file = EnvBuild(infile=xmlfile, components=components)
+
elif ftype == "env_case.xml":
+
new_env_file = EnvCase(infile=xmlfile, components=components)
+
elif ftype == "env_mach_pes.xml":
+
new_env_file = EnvMachPes(
+
infile=xmlfile,
+
components=components,
+
comp_interface=self._comp_interface,
+
)
+
elif ftype == "env_batch.xml":
+
new_env_file = EnvBatch(infile=xmlfile)
+
elif ftype == "env_workflow.xml":
+
new_env_file = EnvWorkflow(infile=xmlfile)
+
elif ftype == "env_test.xml":
+
new_env_file = EnvTest(infile=xmlfile)
+
elif ftype == "env_archive.xml":
+
new_env_file = EnvArchive(infile=xmlfile)
+
elif ftype == "env_mach_specific.xml":
+
new_env_file = EnvMachSpecific(
+
infile=xmlfile, comp_interface=self._comp_interface
+
)
+
else:
+
expect(False, "No match found for file type {}".format(ftype))
+
+
if new_env_file is not None:
+
self._env_entryid_files = []
+
self._env_generic_files = []
+
if ftype in ["env_archive.xml", "env_mach_specific.xml"]:
+
self._env_generic_files = [new_env_file]
+
else:
+
self._env_entryid_files = [new_env_file]
+
+
break
+
+
expect(
+
new_env_file is not None, "No match found for file type {}".format(ftype)
+
)
+
self._files = [new_env_file]
+
+
+
+
[docs]
+
def update_env(self, new_object, env_file, blow_away=False):
+
"""
+
Replace a case env object file
+
"""
+
old_object = self.get_env(env_file)
+
if not blow_away:
+
expect(
+
not old_object.needsrewrite,
+
"Potential loss of unflushed changes in {}".format(env_file),
+
)
+
+
new_object.filename = old_object.filename
+
if old_object in self._env_entryid_files:
+
self._env_entryid_files.remove(old_object)
+
self._env_entryid_files.append(new_object)
+
elif old_object in self._env_generic_files:
+
self._env_generic_files.remove(old_object)
+
self._env_generic_files.append(new_object)
+
self._files.remove(old_object)
+
self._files.append(new_object)
+
+
+
+
[docs]
+
def get_latest_cpl_log(self, coupler_log_path=None, cplname="cpl"):
+
"""
+
find and return the latest cpl log file in the
+
coupler_log_path directory
+
"""
+
if coupler_log_path is None:
+
coupler_log_path = self.get_value("RUNDIR")
+
cpllog = None
+
cpllogs = glob.glob(os.path.join(coupler_log_path, "{}.log.*".format(cplname)))
+
if cpllogs:
+
cpllog = max(cpllogs, key=os.path.getctime)
+
return cpllog
+
else:
+
return None
+
+
+
+
[docs]
+
def record_cmd(self, cmd=None, init=False):
+
lines = []
+
caseroot = self.get_value("CASEROOT")
+
cimeroot = self.get_value("CIMEROOT")
+
+
if cmd is None:
+
cmd = self.fix_sys_argv_quotes(list(sys.argv))
+
+
if init:
+
ctime = time.strftime("%Y-%m-%d %H:%M:%S")
+
+
lines.append("#!/bin/bash\n\n")
+
# stop script on error, prevents `create_newcase` from failing
+
# and continuing to execute commands
+
lines.append("set -e\n\n")
+
lines.append("# Created {}\n\n".format(ctime))
+
lines.append('CASEDIR="{}"\n\n'.format(caseroot))
+
lines.append('cd "${CASEDIR}"\n\n')
+
+
# Ensure program path is absolute
+
cmd[0] = re.sub("^./", "{}/scripts/".format(cimeroot), cmd[0])
+
else:
+
expect(
+
caseroot
+
and os.path.isdir(caseroot)
+
and os.path.isfile(os.path.join(caseroot, "env_case.xml")),
+
"Directory {} does not appear to be a valid case directory".format(
+
caseroot
+
),
+
)
+
+
cmd = " ".join(cmd)
+
+
# Replace instances of caseroot with variable
+
cmd = re.sub(caseroot, '"${CASEDIR}"', cmd)
+
+
lines_len = len(lines)
+
lines.insert(lines_len - 1 if init else lines_len, "{}\n\n".format(cmd))
+
+
try:
+
with open(os.path.join(caseroot, "replay.sh"), "a") as fd:
+
fd.writelines(lines)
+
except PermissionError:
+
logger.warning("Could not write to 'replay.sh' script")
+
+
+
+
[docs]
+
def fix_sys_argv_quotes(self, cmd):
+
"""Fixes removed quotes from argument list.
+
+
Restores quotes to `--val` and `KEY=VALUE` from sys.argv.
+
"""
+
# handle fixing quotes
+
# case 1: "--val", " -nlev 276 "
+
# case 2: "-val" , " -nlev 276 "
+
# case 3: CAM_CONFIG_OPTS=" -nlev 276 "
+
for i, item in enumerate(cmd):
+
if re.match("[-]{1,2}val", item) is not None:
+
if i + 1 >= len(cmd):
+
continue
+
+
# only quote if value contains spaces
+
if " " in cmd[i + 1]:
+
cmd[i + 1] = f'"{cmd[i + 1]}"'
+
else:
+
m = re.search("([^=]*)=(.*)", item)
+
+
if m is None:
+
continue
+
+
g = m.groups()
+
+
# only quote if value contains spaces
+
if " " in g[1]:
+
cmd[i] = f'{g[0]}="{g[1]}"'
+
+
return cmd
+
+
+
+
[docs]
+
def create(
+
self,
+
casename,
+
srcroot,
+
compset_name,
+
grid_name,
+
user_mods_dirs=None,
+
machine_name=None,
+
project=None,
+
pecount=None,
+
compiler=None,
+
mpilib=None,
+
pesfile=None,
+
gridfile=None,
+
multi_driver=False,
+
ninst=1,
+
test=False,
+
walltime=None,
+
queue=None,
+
output_root=None,
+
run_unsupported=False,
+
answer=None,
+
input_dir=None,
+
driver=None,
+
workflowid="default",
+
non_local=False,
+
extra_machines_dir=None,
+
case_group=None,
+
ngpus_per_node=0,
+
gpu_type=None,
+
gpu_offload=None,
+
):
+
try:
+
# Set values for env_case.xml
+
self.set_lookup_value("CASE", os.path.basename(casename))
+
self.set_lookup_value("CASEROOT", self._caseroot)
+
self.set_lookup_value("SRCROOT", srcroot)
+
self.set_lookup_value("CASE_HASH", self.new_hash())
+
+
# Propagate to `GenericXML` to resolve $SRCROOT
+
utils.GLOBAL["SRCROOT"] = srcroot
+
+
customize_path = os.path.join(srcroot, "cime_config", "customize")
+
+
config.load(customize_path)
+
+
# If any of the top level user_mods_dirs contain a config_grids.xml file and
+
# gridfile was not set on the command line, use it. However, if there are
+
# multiple user_mods_dirs, it is an error for more than one of them to contain
+
# a config_grids.xml file, because it would be ambiguous which one we should
+
# use.
+
if user_mods_dirs:
+
found_um_config_grids = False
+
for this_user_mods_dir in user_mods_dirs:
+
um_config_grids = os.path.join(
+
this_user_mods_dir, "config_grids.xml"
+
)
+
if os.path.exists(um_config_grids):
+
if gridfile:
+
# Either a gridfile was found in an earlier user_mods
+
# directory or a gridfile was given on the command line. The
+
# first case (which would set found_um_config_grids to True)
+
# is an error; the second case just generates a warning.
+
expect(
+
not found_um_config_grids,
+
"Cannot handle multiple usermods directories with config_grids.xml files: {} and {}".format(
+
gridfile, um_config_grids
+
),
+
)
+
logger.warning(
+
"A config_grids file was found in {} but also provided on the command line {}, command line takes precedence".format(
+
um_config_grids, gridfile
+
)
+
)
+
else:
+
gridfile = um_config_grids
+
found_um_config_grids = True
+
+
# Configure the Case
+
self.configure(
+
compset_name,
+
grid_name,
+
machine_name=machine_name,
+
project=project,
+
pecount=pecount,
+
compiler=compiler,
+
mpilib=mpilib,
+
pesfile=pesfile,
+
gridfile=gridfile,
+
multi_driver=multi_driver,
+
ninst=ninst,
+
test=test,
+
walltime=walltime,
+
queue=queue,
+
output_root=output_root,
+
run_unsupported=run_unsupported,
+
answer=answer,
+
input_dir=input_dir,
+
driver=driver,
+
workflowid=workflowid,
+
non_local=non_local,
+
extra_machines_dir=extra_machines_dir,
+
case_group=case_group,
+
ngpus_per_node=ngpus_per_node,
+
gpu_type=gpu_type,
+
gpu_offload=gpu_offload,
+
)
+
+
self.create_caseroot()
+
+
# Write out the case files
+
self.flush(flushall=True)
+
self.apply_user_mods(user_mods_dirs)
+
+
# Lock env_case.xml
+
lock_file("env_case.xml", self._caseroot)
+
except Exception:
+
if os.path.exists(self._caseroot):
+
if not logger.isEnabledFor(logging.DEBUG) and not test:
+
logger.warning(
+
"Failed to setup case, removing {}\nUse --debug to force me to keep caseroot".format(
+
self._caseroot
+
)
+
)
+
shutil.rmtree(self._caseroot)
+
else:
+
logger.warning("Leaving broken case dir {}".format(self._caseroot))
+
+
raise
+
+
+
+
[docs]
+
def new_hash(self):
+
"""Creates a hash"""
+
args = "".join(sys.argv)
+
ctime = time.strftime("%Y-%m-%d %H:%M:%S")
+
hostname = socket.getfqdn()
+
user = getpass.getuser()
+
+
data = "{}{}{}{}".format(args, ctime, hostname, user)
+
+
return hashlib.sha256(data.encode()).hexdigest()
+
+
+
+
[docs]
+
def is_save_timing_dir_project(self, project):
+
"""
+
Check whether the project is permitted to archive performance data in the location
+
specified for the current machine
+
"""
+
save_timing_dir_projects = self.get_value("SAVE_TIMING_DIR_PROJECTS")
+
if not save_timing_dir_projects:
+
return False
+
else:
+
save_timing_dir_projects = save_timing_dir_projects.split(
+
","
+
) # pylint: disable=no-member
+
for save_timing_dir_project in save_timing_dir_projects:
+
regex = re.compile(save_timing_dir_project)
+
if regex.match(project):
+
return True
+
+
return False
+
+
+
+
[docs]
+
def get_primary_job(self):
+
return "case.test" if self.get_value("TEST") else "case.run"
+
+
+
+
[docs]
+
def get_first_job(self):
+
env_workflow = self.get_env("workflow")
+
jobs = env_workflow.get_jobs()
+
return jobs[0]
+
+
+
+
[docs]
+
def preview_run(self, write, job):
+
write("CASE INFO:")
+
write(" nodes: {}".format(self.num_nodes))
+
write(" total tasks: {}".format(self.total_tasks))
+
write(" tasks per node: {}".format(self.tasks_per_node))
+
write(" thread count: {}".format(self.thread_count))
+
write(" ngpus per node: {}".format(self.ngpus_per_node))
+
write("")
+
+
write("BATCH INFO:")
+
if not job:
+
job = self.get_first_job()
+
+
job_id_to_cmd = self.submit_jobs(dry_run=True, job=job)
+
+
env_batch = self.get_env("batch")
+
for job_id, cmd in job_id_to_cmd:
+
write(" FOR JOB: {}".format(job_id))
+
write(" ENV:")
+
loaded_envs = self.load_env(job=job_id, reset=True, verbose=False)
+
+
for name, value in iter(sorted(loaded_envs, key=lambda x: x[0])):
+
write(" Setting Environment {}={}".format(name, value))
+
+
write("")
+
write(" SUBMIT CMD:")
+
write(" {}".format(self.get_resolved_value(cmd)))
+
write("")
+
if job_id in ("case.run", "case.test"):
+
# get_job_overrides must come after the case.load_env since the cmd may use
+
# env vars.
+
overrides = env_batch.get_job_overrides(job_id, self)
+
write(" MPIRUN (job={}):".format(job_id))
+
write(" {}".format(self.get_resolved_value(overrides["mpirun"])))
+
write("")
+