Skip to content

Commit

Permalink
Merge pull request #337 from ESMCI/jgfouca/autosave_env_info
Browse files Browse the repository at this point in the history
Autosave environment information in case_setup.
  • Loading branch information
jgfouca authored Aug 3, 2016
2 parents 33ce89b + edc1671 commit e061505
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 21 deletions.
68 changes: 52 additions & 16 deletions utils/python/CIME/XML/env_mach_specific.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ def __init__(self, caseroot, infile="env_mach_specific.xml"):
"""
fullpath = infile if os.path.isabs(infile) else os.path.join(caseroot, infile)
EnvBase.__init__(self, caseroot, fullpath)
self._cshscript = []
self._shscript = []

def get_values(self, item, attribute=None, resolved=True, subgroup=None):
"""Returns the value as a string of the first xml element with item as attribute value.
Expand Down Expand Up @@ -59,27 +57,32 @@ def get_values(self, item, attribute=None, resolved=True, subgroup=None):

return results

def load_env_for_case(self, compiler, debug, mpilib):
def _get_env_for_case(self, compiler, debug, mpilib):
module_nodes = self.get_nodes("modules")
env_nodes = self.get_nodes("environment_variables")

if (module_nodes is not None):
modules_to_load = None
if module_nodes is not None:
modules_to_load = self._compute_module_actions(module_nodes, compiler, debug, mpilib)
self.load_modules(modules_to_load)
if (env_nodes is not None):

envs_to_set = None
if env_nodes is not None:
envs_to_set = self._compute_env_actions(env_nodes, compiler, debug, mpilib)

return modules_to_load, envs_to_set

def load_env_for_case(self, compiler, debug, mpilib):
modules_to_load, envs_to_set = self._get_env_for_case(compiler, debug, mpilib)

if (modules_to_load is not None):
self.load_modules(modules_to_load)
if (envs_to_set is not None):
self.load_envs(envs_to_set)
with open(".env_mach_specific.csh",'w') as f:
f.write("\n".join(self._cshscript))
with open(".env_mach_specific.sh",'w') as f:
f.write("\n".join(self._shscript))

def load_modules(self, modules_to_load):
module_system = self.get_module_system_type()
if (module_system == "module"):
self._load_module_modules(modules_to_load)
self._cshscript += self._get_module_module_commands(modules_to_load,"csh")
self._shscript += self._get_module_module_commands(modules_to_load,"sh")
elif (module_system == "soft"):
self._load_soft_modules(modules_to_load)
elif (module_system == "dotkit"):
Expand All @@ -89,12 +92,44 @@ def load_modules(self, modules_to_load):
else:
expect(False, "Unhandled module system '%s'" % module_system)

def list_modules(self):
module_system = self.get_module_system_type()
if (module_system == "module"):
return run_cmd_no_fail("module list 2>&1")
elif (module_system == "soft"):
return run_cmd_no_fail("softenv")
elif (module_system == "dotkit"):
return run_cmd_no_fail("use -lv")
elif (module_system == "none"):
return ""
else:
expect(False, "Unhandled module system '%s'" % module_system)

def make_env_mach_specific_file(self, compiler, debug, mpilib, shell):
modules_to_load, envs_to_set = self._get_env_for_case(compiler, debug, mpilib)

filename = ".env_mach_specific.%s" % shell
lines = []
if modules_to_load is not None:
lines.extend(self._get_module_commands(modules_to_load, shell))

if envs_to_set is not None:
for env_name, env_value in envs_to_set:
# Let bash do the work on evaluating and resolving env_value
if shell == "sh":
lines.append("export %s=%s" % (env_name, env_value))
elif shell == "csh":
lines.append("setenv %s %s" % (env_name, env_value))
else:
expect(False, "Unknown shell type: '%s'" % shell)

with open(filename, "w") as fd:
fd.write("\n".join(lines))

def load_envs(self, envs_to_set):
for env_name, env_value in envs_to_set:
# Let bash do the work on evaluating and resolving env_value
os.environ[env_name] = run_cmd_no_fail("echo %s" % env_value)
self._cshscript.append("setenv %s %s"%(env_name,env_value))
self._shscript.append("export %s=%s"%(env_name,env_value))

# Private API

Expand Down Expand Up @@ -135,7 +170,8 @@ def _match(self, my_value, xml_value):
else:
return my_value == xml_value

def _get_module_module_commands(self, modules_to_load, shell):
def _get_module_commands(self, modules_to_load, shell):
# Note this is independent of module system type
mod_cmd = self.get_module_system_cmd_path(shell)
cmds = []
for action, argument in modules_to_load:
Expand All @@ -145,7 +181,7 @@ def _get_module_module_commands(self, modules_to_load, shell):
return cmds

def _load_module_modules(self, modules_to_load):
for cmd in self._get_module_module_commands(modules_to_load, "python"):
for cmd in self._get_module_commands(modules_to_load, "python"):
py_module_code = run_cmd_no_fail(cmd)
exec(py_module_code)

Expand Down
19 changes: 14 additions & 5 deletions utils/python/CIME/case_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,19 +141,20 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
models = drv_comp.get_valid_model_components()
models.remove("DRV")

mach = case.get_value("MACH")
mach, compiler, debug, mpilib = \
case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB")
expect(mach is not None, "xml variable MACH is not set")

# Create Macros file only if it does not exist
if not os.path.exists("Macros"):
logger.debug("Creating Macros file for %s" % mach)
compilers = Compilers(compiler=case.get_value("COMPILER"), machine=mach, os_=case.get_value("OS"), mpilib=case.get_value("MPILIB"))
compilers = Compilers(compiler=compiler, machine=mach, os_=case.get_value("OS"), mpilib=mpilib)
compilers.write_macros_file()
else:
logger.debug("Macros script already created ...skipping")

# Set tasks to 1 if mpi-serial library
if case.get_value("MPILIB") == "mpi-serial":
if mpilib == "mpi-serial":
for vid, value in case:
if vid.startswith("NTASKS_") and value != 1:
case.set_value(vid, 1)
Expand All @@ -172,7 +173,7 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
else:
expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst[comp_model], comp, ntasks))

expect(not (case.get_value("BUILD_THREADED") and case.get_value("COMPILER") == "nag"),
expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"),
"it is not possible to run with OpenMP if using the NAG Fortran compiler")

if os.path.exists("case.run"):
Expand All @@ -198,7 +199,7 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
pcost = 3 - pcnt / 10 # (3 is 64 with 6)

# Compute cost based on DEBUG
dcost = 3 if case.get_value("DEBUG") else 0
dcost = 3 if debug else 0

# Compute cost based on run length
# For simplicity, we use a heuristic just based on STOP_OPTION (not considering
Expand Down Expand Up @@ -279,3 +280,11 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
msg = "case.setup complete"
append_status(msg, caseroot=caseroot, sfile="CaseStatus")

# Record env information
env_module = case.get_env("mach_specific")
env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
with open("software_environment.txt", "w") as f:
f.write(env_module.list_modules())
run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \
env >> software_environment.txt")

0 comments on commit e061505

Please sign in to comment.