Skip to content

Commit

Permalink
Merge branch 'jgfouca/cime/omp_num_threads' (PR #1359)
Browse files Browse the repository at this point in the history 
* Use proper syntax to refer to OMP_NUM_THREADS in config_machines.xml.
    Must use ENV syntax.
* Move setting of OMP_NUM_THREADS earlier so it will be captured
    by provenance saving.
* Avoid hyper-threading on Edison.

Fixes #1355

[BFB]
  • Loading branch information
@amametjanov
amametjanov committed Apr 1, 2017
2 parents fa7ecc1 + f546d8b commit 9f08b65
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 14 deletions.
19 changes: 9 additions & 10 deletions cime_config/acme/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>cseg</SUPPORTED_BY>
<GMAKE_J>8</GMAKE_J>
<MAX_TASKS_PER_NODE>48</MAX_TASKS_PER_NODE>
<MAX_TASKS_PER_NODE>24</MAX_TASKS_PER_NODE>
<PES_PER_NODE>24</PES_PER_NODE>
<PROJECT>acme</PROJECT>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
Expand All @@ -83,7 +83,7 @@
<arguments>
<arg name="label"> --label</arg>
<arg name="num_tasks" > -n $TOTALPES</arg>
<arg name="thread_count" > -c $OMP_NUM_THREADS</arg>
<arg name="thread_count" > -c $ENV{OMP_NUM_THREADS}</arg>
</arguments>
</mpirun>
<module_system type="module">
Expand Down Expand Up @@ -939,7 +939,7 @@
<executable>mpiexec</executable>
<arguments>
<arg name="num_tasks"> -n $TOTALPES </arg>
<arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$OMP_NUM_THREADS --bind-to core</arg>
<arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to core</arg>
</arguments>
</mpirun>
<mpirun mpilib="mvapich">
Expand Down Expand Up @@ -1048,7 +1048,7 @@
<arg name="bg_threadlayout"> --envs BG_THREADLAYOUT=1</arg>
<arg name="xl_bg_spreadlayout"> --envs XL_BG_SPREADLAYOUT=YES</arg>
<arg name="omp_stacksize"> --envs OMP_STACKSIZE=64M</arg>
<arg name="thread_count"> --envs OMP_NUM_THREADS=$OMP_NUM_THREADS</arg>
<arg name="thread_count"> --envs OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS}</arg>
</arguments>
</mpirun>
<module_system type="soft">
Expand Down Expand Up @@ -1170,7 +1170,7 @@
<arg name="bg_threadlayout"> --envs BG_THREADLAYOUT=1</arg>
<arg name="xl_bg_spreadlayout"> --envs XL_BG_SPREADLAYOUT=YES</arg>
<arg name="omp_stacksize"> --envs OMP_STACKSIZE=64M</arg>
<arg name="thread_count"> --envs OMP_NUM_THREADS=$OMP_NUM_THREADS</arg>
<arg name="thread_count"> --envs OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS}</arg>
</arguments>
</mpirun>
<module_system type="soft">
Expand Down Expand Up @@ -1548,7 +1548,6 @@
<OS>CNL</OS>
<BATCH_SYSTEM>pbs</BATCH_SYSTEM>
<PES_PER_NODE>16</PES_PER_NODE>
<!-- difference to MAX_TASKS_PER_NODE-->
<SUPPORTED_BY>acme</SUPPORTED_BY>
<GMAKE_J>8</GMAKE_J>
<MAX_TASKS_PER_NODE>16</MAX_TASKS_PER_NODE>
Expand All @@ -1562,7 +1561,7 @@
<arg name="tasks_per_numa" > -S {{ tasks_per_numa }}</arg>
<arg name="num_tasks" > -n $TOTALPES</arg>
<arg name="tasks_per_node" > -N $PES_PER_NODE</arg>
<arg name="thread_count" > -d $OMP_NUM_THREADS</arg>
<arg name="thread_count" > -d $ENV{OMP_NUM_THREADS}</arg>
<arg name="numa_node" > -cc numa_node </arg>
-->
<arg name="aprun"> {{ aprun }}</arg>
Expand Down Expand Up @@ -1727,7 +1726,7 @@
<arg name="tasks_per_numa" > -S {{ tasks_per_numa }}</arg>
<arg name="num_tasks" > -n $TOTALPES</arg>
<arg name="tasks_per_node" > -N $PES_PER_NODE</arg>
<arg name="thread_count" > -d $OMP_NUM_THREADS</arg>
<arg name="thread_count" > -d $ENV{OMP_NUM_THREADS}</arg>
<arg name="numa_node" > -cc numa_node</arg>
</arguments>
</mpirun>
Expand Down Expand Up @@ -2085,7 +2084,7 @@
<arg name="num_tasks"> -n $TOTALPES</arg>
<arg name="tasks_per_numa"> -S {{ tasks_per_numa }}</arg>
<arg name="tasks_per_node"> -N $PES_PER_NODE</arg>
<arg name="thread_count"> -d $OMP_NUM_THREADS</arg>
<arg name="thread_count"> -d $ENV{OMP_NUM_THREADS}</arg>
</arguments>
</mpirun>
</machine>
Expand Down Expand Up @@ -2124,7 +2123,7 @@
<arg name="num_tasks"> -n $TOTALPES</arg>
<arg name="tasks_per_numa"> -S {{ tasks_per_numa }}</arg>
<arg name="tasks_per_node"> -N $PES_PER_NODE</arg>
<arg name="thread_count"> -d $OMP_NUM_THREADS</arg>
<arg name="thread_count"> -d $ENV{OMP_NUM_THREADS}</arg>
</arguments>
</mpirun>
</machine>
Expand Down
9 changes: 5 additions & 4 deletions utils/python/CIME/case_run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,6 @@ def pre_run_check(case, lid):
def run_model(case):
###############################################################################

# Set OMP_NUM_THREADS
env_mach_pes = case.get_env("mach_pes")
os.environ["OMP_NUM_THREADS"] = str(env_mach_pes.get_max_thread_count(case.get_values("COMP_CLASSES")))

# Run the model
logger.info("%s MODEL EXECUTION BEGINS HERE" %(time.strftime("%Y-%m-%d %H:%M:%S")))

Expand Down Expand Up @@ -214,6 +210,11 @@ def case_run(case):
# set up the LID
lid = new_lid()

# Set OMP_NUM_THREADS. Want this to happen before prerun provenance
# so it gets properly logged in the environment provenance files.
env_mach_pes = case.get_env("mach_pes")
os.environ["OMP_NUM_THREADS"] = str(env_mach_pes.get_max_thread_count(case.get_values("COMP_CLASSES")))

save_prerun_provenance(case)

for cycle in range(data_assimilation_cycles):
Expand Down

0 comments on commit 9f08b65

Please sign in to comment.