Revert "Revert "Merge branch 'azamat/mira/add-smp-present' into next …

…(PR #1549)""

This reverts commit 5f14228.

cime/config/acme/machines/config_machines.xml

@@ -1041,7 +1041,6 @@
          <DOUT_L_MSROOT>/home/$USER/csm/$CASE/</DOUT_L_MSROOT>
          <BASELINE_ROOT>/projects/ccsm/ccsm_baselines/</BASELINE_ROOT>
          <CCSM_CPRNC>/projects/ccsm/tools/cprnc/cprnc</CCSM_CPRNC>
-         <SAVE_TIMING_DIR>/projects/$PROJECT</SAVE_TIMING_DIR>
          <OS>BGQ</OS>
          <BATCH_SYSTEM>cobalt</BATCH_SYSTEM>
          <SUPPORTED_BY>   jayesh -at- mcs.anl.gov</SUPPORTED_BY>
@@ -1054,16 +1053,11 @@
          <mpirun mpilib="default">
            <executable>/usr/bin/runjob</executable>
              <arguments>
-               <arg name="label"> --label short</arg>
-               <!-- Ranks per node!! -->
-               <arg name="tasks_per_node"> --ranks-per-node $PES_PER_NODE</arg>
-               <!-- Total MPI Tasks -->
-               <arg name="num_tasks"> --np $TOTALPES</arg>
-               <arg name="locargs"> --block $COBALT_PARTNAME $LOCARGS</arg>
-               <arg name="bg_threadlayout"> --envs BG_THREADLAYOUT=1</arg>
-	       <arg name="xl_bg_spreadlayout"> --envs XL_BG_SPREADLAYOUT=YES</arg>
-               <arg name="omp_stacksize"> --envs OMP_STACKSIZE=64M</arg>
-               <arg name="thread_count"> --envs OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS}</arg>
+               <arg name="label">--label short</arg>
+               <arg name="tasks_per_node">--ranks-per-node $PES_PER_NODE</arg>
+               <arg name="num_tasks">--np $TOTALPES</arg>
+               <arg name="locargs">--block $COBALT_PARTNAME $LOCARGS</arg>
+               <arg name="bgq_smp_vars">$ENV{BGQ_SMP_VARS}</arg>
              </arguments>
          </mpirun>
     <module_system type="soft">
@@ -1080,8 +1074,10 @@
     </module_system>
     <environment_variables>
       <env name="MPI_TYPE_MAX">10000</env>
-      <env name="OMP_DYNAMIC">FALSE</env>
-      <env name="OMP_STACKSIZE">64M</env>
+      <env name="BGQ_SMP_VARS"></env>
+    </environment_variables>
+    <environment_variables SMP_PRESENT="TRUE">
+      <env name="BGQ_SMP_VARS">--envs BG_THREADLAYOUT=1 XL_BG_SPREADLAYOUT=YES OMP_DYNAMIC=FALSE OMP_STACKSIZE=64M OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS}</env>
     </environment_variables>
 </machine>
 
@@ -1219,16 +1215,11 @@
          <mpirun mpilib="default">
            <executable>/usr/bin/runjob</executable>
              <arguments>
-               <arg name="label"> --label short</arg>
-               <!-- Ranks per node!! -->
-               <arg name="tasks_per_node"> --ranks-per-node $PES_PER_NODE</arg>
-               <!-- Total MPI Tasks -->
-               <arg name="num_tasks"> --np $TOTALPES</arg>
-               <arg name="locargs"> --block $COBALT_PARTNAME $LOCARGS</arg>
-               <arg name="bg_threadlayout"> --envs BG_THREADLAYOUT=1</arg>
-	       <arg name="xl_bg_spreadlayout"> --envs XL_BG_SPREADLAYOUT=YES</arg>
-               <arg name="omp_stacksize"> --envs OMP_STACKSIZE=64M</arg>
-               <arg name="thread_count"> --envs OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS}</arg>
+               <arg name="label">--label short</arg>
+               <arg name="tasks_per_node">--ranks-per-node $PES_PER_NODE</arg>
+               <arg name="num_tasks">--np $TOTALPES</arg>
+               <arg name="locargs">--block $COBALT_PARTNAME $LOCARGS</arg>
+               <arg name="bgq_smp_vars">$ENV{BGQ_SMP_VARS}</arg>
              </arguments>
          </mpirun>
     <module_system type="soft">
@@ -1245,8 +1236,10 @@
     </module_system>
     <environment_variables>
       <env name="MPI_TYPE_MAX">10000</env>
-      <env name="OMP_DYNAMIC">FALSE</env>
-      <env name="OMP_STACKSIZE">64M</env>
+      <env name="BGQ_SMP_VARS"></env>
+    </environment_variables>
+    <environment_variables SMP_PRESENT="TRUE">
+      <env name="BGQ_SMP_VARS">--envs BG_THREADLAYOUT=1 XL_BG_SPREADLAYOUT=YES OMP_DYNAMIC=FALSE OMP_STACKSIZE=64M OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS}</env>
     </environment_variables>
 </machine>
 

cime/scripts/Tools/preview_run

@@ -58,6 +58,7 @@ def _main_func(description):
 
     with Case(caseroot, read_only=False) as case:
         print "BATCH SUBMIT:"
+        case.load_env()
         job = "case.test" if case.get_value("TEST") else "case.run"
         job_id_to_cmd = case.submit_jobs(dry_run=True, job=job)
         for job_id, cmd in job_id_to_cmd:

cime/scripts/fortran_unit_testing/run_tests.py

@@ -7,7 +7,7 @@
 sys.path.append(os.path.join(_CIMEROOT, "scripts", "fortran_unit_testing", "python"))
 
 from standard_script_setup import *
-from CIME.BuildTools.configure import configure
+from CIME.BuildTools.configure import configure, FakeCase
 from CIME.utils import run_cmd_no_fail, stringify_bool, expect
 from CIME.XML.machines import Machines
 from CIME.XML.compilers import Compilers
@@ -318,7 +318,8 @@ def _main():
               unit_testing=True)
     machspecific = EnvMachSpecific(build_dir, unit_testing=True)
 
-    machspecific.load_env(compiler, debug, mpilib)
+    fake_case = FakeCase(compiler, mpilib, debug)
+    machspecific.load_env(fake_case)
     os.environ["OS"] = os_
     os.environ["COMPILER"] = compiler
     os.environ["DEBUG"] = stringify_bool(debug)

cime/scripts/lib/CIME/BuildTools/configure.py

@@ -66,6 +66,14 @@ def _copy_depends_files(machine_name, machines_dir, output_dir, compiler):
             if os.path.isfile(dfile) and not os.path.isfile(outputdfile):
                 shutil.copyfile(dfile, outputdfile)
 
+class FakeCase(object):
+
+    def __init__(self, compiler, mpilib, debug):
+        self._vals = {"COMPILER":compiler, "MPILIB":mpilib, "DEBUG":debug}
+
+    def get_value(self, attrib):
+        expect(attrib in self._vals, "FakeCase does not support getting value of '%s'" % attrib)
+        return self._vals[attrib]
 
 def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
                                 sysos, unit_testing):
@@ -79,8 +87,9 @@ def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
     ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing)
     ems_file.populate(machobj)
     ems_file.write()
+    fake_case = FakeCase(compiler, mpilib, debug)
     for shell in ('sh', 'csh'):
-        ems_file.make_env_mach_specific_file(compiler, debug, mpilib, shell)
+        ems_file.make_env_mach_specific_file(shell, fake_case)
         shell_path = os.path.join(output_dir, ".env_mach_specific." + shell)
         with open(shell_path, 'a') as shell_file:
             if shell == 'sh':

cime/scripts/lib/CIME/XML/env_mach_specific.py

@@ -53,34 +53,34 @@ def populate(self, machobj):
                 for node in nodes:
                     self.add_child(node)
 
-    def _get_modules_for_case(self, compiler, debug, mpilib):
+    def _get_modules_for_case(self, case):
         module_nodes = self.get_nodes("modules")
         modules_to_load = None
         if module_nodes is not None:
-            modules_to_load = self._compute_module_actions(module_nodes, compiler, debug, mpilib)
+            modules_to_load = self._compute_module_actions(module_nodes, case)
 
         return modules_to_load
 
-    def _get_envs_for_case(self, compiler, debug, mpilib):
+    def _get_envs_for_case(self, case):
         env_nodes = self.get_nodes("environment_variables")
 
         envs_to_set = None
         if env_nodes is not None:
-            envs_to_set = self._compute_env_actions(env_nodes, compiler, debug, mpilib)
+            envs_to_set = self._compute_env_actions(env_nodes, case)
 
         return envs_to_set
 
-    def load_env(self, compiler, debug, mpilib):
+    def load_env(self, case):
         """
         Should only be called by case.load_env
         """
         # Do the modules so we can refer to env vars set by the modules
         # in the environment_variables block
-        modules_to_load = self._get_modules_for_case(compiler, debug, mpilib)
+        modules_to_load = self._get_modules_for_case(case)
         if (modules_to_load is not None):
             self.load_modules(modules_to_load)
 
-        envs_to_set = self._get_envs_for_case(compiler, debug, mpilib)
+        envs_to_set = self._get_envs_for_case(case)
         if (envs_to_set is not None):
             self.load_envs(envs_to_set)
 
@@ -130,9 +130,9 @@ def save_all_env_info(self, filename):
             f.write(self.list_modules())
         run_cmd_no_fail("echo -e '\n' && env", arg_stdout=filename)
 
-    def make_env_mach_specific_file(self, compiler, debug, mpilib, shell):
-        modules_to_load = self._get_modules_for_case(compiler, debug, mpilib)
-        envs_to_set = self._get_envs_for_case(compiler, debug, mpilib)
+    def make_env_mach_specific_file(self, shell, case):
+        modules_to_load = self._get_modules_for_case(case)
+        envs_to_set = self._get_envs_for_case(case)
         filename = ".env_mach_specific.%s" % shell
         lines = []
         if modules_to_load is not None:
@@ -152,24 +152,25 @@ def make_env_mach_specific_file(self, compiler, debug, mpilib, shell):
 
     def load_envs(self, envs_to_set):
         for env_name, env_value in envs_to_set:
-            os.environ[env_name] = env_value
+            os.environ[env_name] = "" if env_value is None else env_value
 
     # Private API
 
-    def _compute_module_actions(self, module_nodes, compiler, debug, mpilib):
-        return self._compute_actions(module_nodes, "command", compiler, debug, mpilib)
+    def _compute_module_actions(self, module_nodes, case):
+        return self._compute_actions(module_nodes, "command", case)
 
-    def _compute_env_actions(self, env_nodes, compiler, debug, mpilib):
-        return self._compute_actions(env_nodes, "env", compiler, debug, mpilib)
+    def _compute_env_actions(self, env_nodes, case):
+        return self._compute_actions(env_nodes, "env", case)
 
-    def _compute_actions(self, nodes, child_tag, compiler, debug, mpilib):
+    def _compute_actions(self, nodes, child_tag, case):
         result = [] # list of tuples ("name", "argument")
+        compiler, mpilib = case.get_value("COMPILER"), case.get_value("MPILIB")
 
         for node in nodes:
-            if (self._match_attribs(node.attrib, compiler, debug, mpilib)):
+            if (self._match_attribs(node.attrib, case)):
                 for child in node:
                     expect(child.tag == child_tag, "Expected %s element" % child_tag)
-                    if (self._match_attribs(child.attrib, compiler, debug, mpilib)):
+                    if (self._match_attribs(child.attrib, case)):
                         val = child.text
                         if val is not None:
                             # We allow a couple special substitutions for these fields
@@ -178,36 +179,39 @@ def _compute_actions(self, nodes, child_tag, compiler, debug, mpilib):
 
                             val = self.get_resolved_value(val)
                             expect("$" not in val, "Not safe to leave unresolved items in env var value: '%s'" % val)
+
                         # intentional unindent, result is appended even if val is None
                         result.append( (child.get("name"), val) )
 
         return result
 
-    def _match_attribs(self, attribs, compiler, debug, mpilib):
-        if ("compiler" in attribs and
-            not self._match(compiler, attribs["compiler"])):
-            return False
-        elif ("mpilib" in attribs and
-            not self._match(mpilib, attribs["mpilib"])):
-            return False
-        elif ("debug" in attribs and
-            not self._match("TRUE" if debug else "FALSE", attribs["debug"].upper())):
-            return False
-        elif ("unit_testing" in attribs and
-              not self._match("TRUE" if self._unit_testing else "FALSE",
-                              attribs["unit_testing"].upper())):
-            return False
+    def _match_attribs(self, attribs, case):
+        # check for matches with case-vars
+        for attrib in attribs:
+            if attrib == "unit_testing": # special case
+                if not self._match(self._unit_testing, attribs["unit_testing"].upper()):
+                    return False
+            elif attrib == "name":
+                pass
+            else:
+                val = case.get_value(attrib.upper())
+                expect(val is not None, "Cannot match attrib '%s', case has no value for it" % attrib.upper())
+                if not self._match(val, attribs[attrib]):
+                    return False
 
         return True
 
     def _match(self, my_value, xml_value):
-        if (xml_value.startswith("!")):
+        if xml_value.startswith("!"):
             result = my_value != xml_value[1:]
+        elif isinstance(my_value, bool):
+            if my_value: result = xml_value == "TRUE"
+            else: result = xml_value == "FALSE"
         else:
             result = my_value == xml_value
-        logger.debug("(env_mach_specific) _match %s %s %s"%(my_value, xml_value, result))
-        return result
 
+        logger.debug("(env_mach_specific) _match %s %s %s" % (my_value, xml_value, result))
+        return result
 
     def _get_module_commands(self, modules_to_load, shell):
         # Note this is independent of module system type

cime/scripts/lib/CIME/case.py

@@ -8,7 +8,7 @@
 import glob, os, shutil, math, string
 from CIME.XML.standard_module_setup import *
 
-from CIME.utils                     import expect, get_cime_root, append_status
+from CIME.utils                     import expect, get_cime_root, append_status, stringify_bool
 from CIME.utils                     import convert_to_type, get_model, get_project
 from CIME.utils                     import get_current_commit
 from CIME.check_lockedfiles         import LOCKED_DIR, lock_file
@@ -1168,19 +1168,19 @@ def set_model_version(self, model):
 
     def load_env(self):
         if not self._is_env_loaded:
-            compiler = self.get_value("COMPILER")
-            debug=self.get_value("DEBUG")
-            mpilib=self.get_value("MPILIB")
             env_module = self.get_env("mach_specific")
-            env_module.load_env(compiler=compiler,debug=debug, mpilib=mpilib)
+            env_module.load_env(self)
             self._is_env_loaded = True
 
     def get_build_threaded(self):
         """
         Returns True if current settings require a threaded build/run.
         """
         force_threaded = self.get_value("BUILD_THREADED")
-        return bool(force_threaded) or self.thread_count > 1
+        smp_present = bool(force_threaded) or self.thread_count > 1
+        self.set_value("SMP_PRESENT", stringify_bool(smp_present))
+        os.environ["OMP_NUM_THREADS"] = str(self.thread_count)
+        return smp_present
 
     def _check_testlists(self, compset_alias, grid_name, files):
         """

cime/scripts/lib/CIME/case_setup.py

@@ -212,8 +212,8 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
 
         # Record env information
         env_module = case.get_env("mach_specific")
-        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
-        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
+        env_module.make_env_mach_specific_file("sh", case)
+        env_module.make_env_mach_specific_file("csh", case)
         env_module.save_all_env_info("software_environment.txt")
 
 ###############################################################################

cime/scripts/lib/CIME/utils.py

@@ -1243,7 +1243,7 @@ def _check_for_invalid_args(args):
         if arg.startswith("-") and len(arg) > 2:
 #       Uncomment these lines when we want to enforce --mulitchararg syntax
 #            if arg == "-value" or arg == "-noecho":
-                logger.warn("This argument is depricated, please use -%s"%arg)
+            logger.warn("This argument is depricated, please use -%s"%arg)
 #            else:
 #                expect(False, "Invalid argument %s\n  Multi-character arguments should begin with \"--\" and single character with \"-\"\n  Use --help for a complete list of available options"%arg)
 

cime/src/drivers/mct/cime_config/config_component.xml

@@ -761,6 +761,15 @@
       If TRUE, the component libraries are always built with OpenMP capability.</desc>
   </entry>
 
+  <entry id="SMP_PRESENT">
+    <type>logical</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc>TRUE implies that at least one of the components is built threaded (DO NOT EDIT)</desc>
+  </entry>
+
   <entry id="USE_PETSC">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>