deploy: 8e6a7aa

_sources/olcf-compilers.rst.txt

@@ -69,15 +69,16 @@ load modules:
    module load PrgEnv-gnu
    module load cray-mpich/8.1.28
    module load craype-accel-amd-gfx90a
-   module load amd-mixed/6.0.0
+   module load amd-mixed/6.1.3
    module unload darshan-runtime
 
-this will load ROCm 6.0.0
+this will load ROCm 6.1.3
 
 .. note::
 
-   Tabulate rates seem to exhibit a strange slow down on Frontier, so
-   it is best to run without rate tabulation.
+   In the past, tabulated rates seem to exhibit a strange slow down on
+   Frontier, so it is best to run test with and without rate
+   tabulation to see if there is a performance issue.
 
 build via:
 

_sources/olcf-workflow.rst.txt

@@ -502,6 +502,23 @@ https://github.com/AMReX-Astro/workflow/blob/main/job_scripts/frontier/frontier.
 Also see the WarpX docs: https://warpx.readthedocs.io/en/latest/install/hpc/frontier.html
 
 
+GPU-aware MPI
+^^^^^^^^^^^^^
+
+Some codes run better with GPU-aware MPI.  To enable this add the following to your
+submission script:
+
+.. code:: bash
+
+   export MPICH_GPU_SUPPORT_ENABLED=1
+   export FI_MR_CACHE_MONITOR=memhooks
+
+and set the runtime parameter:
+
+.. code::
+
+   amrex.use_gpu_aware_mpi=1
+
 Job Status
 ^^^^^^^^^^
 

olcf-compilers.html

@@ -148,13 +148,14 @@ <h2>Frontier<a class="headerlink" href="#frontier" title="Link to this heading">
 <div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">module<span class="w"> </span>load<span class="w"> </span>PrgEnv-gnu</span>
 <span class="prompt1">module<span class="w"> </span>load<span class="w"> </span>cray-mpich/8.1.28</span>
 <span class="prompt1">module<span class="w"> </span>load<span class="w"> </span>craype-accel-amd-gfx90a</span>
-<span class="prompt1">module<span class="w"> </span>load<span class="w"> </span>amd-mixed/6.0.0</span>
+<span class="prompt1">module<span class="w"> </span>load<span class="w"> </span>amd-mixed/6.1.3</span>
 <span class="prompt1">module<span class="w"> </span>unload<span class="w"> </span>darshan-runtime</span>
-</pre></div></div><p>this will load ROCm 6.0.0</p>
+</pre></div></div><p>this will load ROCm 6.1.3</p>
 <div class="admonition note">
 <p class="admonition-title">Note</p>
-<p>Tabulate rates seem to exhibit a strange slow down on Frontier, so
-it is best to run without rate tabulation.</p>
+<p>In the past, tabulated rates seem to exhibit a strange slow down on
+Frontier, so it is best to run test with and without rate
+tabulation to see if there is a performance issue.</p>
 </div>
 <p>build via:</p>
 <div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">make<span class="w"> </span><span class="nv">USE_HIP</span><span class="o">=</span>TRUE</span>

olcf-workflow.html

@@ -70,6 +70,7 @@
 <li class="toctree-l3"><a class="reference internal" href="#frontier">Frontier</a><ul>
 <li class="toctree-l4"><a class="reference internal" href="#machine-details">Machine details</a></li>
 <li class="toctree-l4"><a class="reference internal" href="#submitting-jobs">Submitting jobs</a></li>
+<li class="toctree-l4"><a class="reference internal" href="#gpu-aware-mpi">GPU-aware MPI</a></li>
 <li class="toctree-l4"><a class="reference internal" href="#job-status">Job Status</a></li>
 <li class="toctree-l4"><a class="reference internal" href="#job-chaining">Job Chaining</a></li>
 <li class="toctree-l4"><a class="reference internal" href="#debugging">Debugging</a></li>
@@ -600,6 +601,19 @@ <h3>Submitting jobs<a class="headerlink" href="#submitting-jobs" title="Link to
 <a class="reference external" href="https://github.com/AMReX-Astro/workflow/blob/main/job_scripts/frontier/frontier.slurm">https://github.com/AMReX-Astro/workflow/blob/main/job_scripts/frontier/frontier.slurm</a></p>
 <p>Also see the WarpX docs: <a class="reference external" href="https://warpx.readthedocs.io/en/latest/install/hpc/frontier.html">https://warpx.readthedocs.io/en/latest/install/hpc/frontier.html</a></p>
 </section>
+<section id="gpu-aware-mpi">
+<h3>GPU-aware MPI<a class="headerlink" href="#gpu-aware-mpi" title="Link to this heading">¶</a></h3>
+<p>Some codes run better with GPU-aware MPI.  To enable this add the following to your
+submission script:</p>
+<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span><span class="w"> </span><span class="nv">MPICH_GPU_SUPPORT_ENABLED</span><span class="o">=</span><span class="m">1</span>
+<span class="nb">export</span><span class="w"> </span><span class="nv">FI_MR_CACHE_MONITOR</span><span class="o">=</span>memhooks
+</pre></div>
+</div>
+<p>and set the runtime parameter:</p>
+<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>amrex.use_gpu_aware_mpi<span class="o">=</span><span class="m">1</span>
+</pre></div>
+</div>
+</section>
 <section id="job-status">
 <h3>Job Status<a class="headerlink" href="#job-status" title="Link to this heading">¶</a></h3>
 <p>You can check on the status of your jobs via:</p>

olcf.html

@@ -115,6 +115,7 @@ <h1>Working at OLCF<a class="headerlink" href="#working-at-olcf" title="Link to
 <li class="toctree-l2"><a class="reference internal" href="olcf-workflow.html#frontier">Frontier</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="olcf-workflow.html#machine-details">Machine details</a></li>
 <li class="toctree-l3"><a class="reference internal" href="olcf-workflow.html#submitting-jobs">Submitting jobs</a></li>
+<li class="toctree-l3"><a class="reference internal" href="olcf-workflow.html#gpu-aware-mpi">GPU-aware MPI</a></li>
 <li class="toctree-l3"><a class="reference internal" href="olcf-workflow.html#job-status">Job Status</a></li>
 <li class="toctree-l3"><a class="reference internal" href="olcf-workflow.html#job-chaining">Job Chaining</a></li>
 <li class="toctree-l3"><a class="reference internal" href="olcf-workflow.html#debugging">Debugging</a></li>