Merge pull request #89 from omlins/revision

Address remaining issues of review 1 and 2

paper/.latexmkrc

@@ -0,0 +1,6 @@
+
+sub build_header {
+ system("ruby ./prep.rb")
+}
+
+build_header()

paper/README.md

@@ -0,0 +1,27 @@
+# JuliaCon-proceeding-IGG
+
+## Paper dependencies
+
+The document can be built locally, the following dependencies need to be installed:
+- Ruby
+- latexmk
+
+## Build process
+
+Build the paper using:
+```
+$ latexmk -bibtex -pdf paper.tex
+```
+
+Clean up temporary files using:
+```
+$ latexmk -c
+```
+
+## Paper metadata
+
+**IMPORTANT**
+Some information for building the document (such as the title and keywords)
+is provided through the `paper.yml` file and not through the usual `\title`
+command. Respecting the process is important to avoid build errors when
+submitting your work.

paper/bib.tex

@@ -0,0 +1,4 @@
+% **************GENERATED FILE, DO NOT EDIT**************
+
+\bibliographystyle{juliacon}
+\bibliography{ref.bib}

paper/codes/benchmarks/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+ImplicitGlobalGrid = "4d7a3746-15be-11ea-1130-334b0c4f5fa0"
+MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+ParallelStencil = "94395366-693c-11ea-3b26-d9b7aac5d958"

paper/codes/benchmarks/daint_run_all.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+sbatch daint_runme_1.sh
+sbatch daint_runme_8.sh
+sbatch daint_runme_27.sh
+sbatch daint_runme_64.sh
+sbatch daint_runme_125.sh
+sbatch daint_runme_216.sh
+sbatch daint_runme_512.sh
+sbatch daint_runme_1000.sh
+sbatch daint_runme_2197.sh

paper/codes/benchmarks/daint_runme_1.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_1"
+#SBATCH --output=diff3D_1.%j.o
+#SBATCH --error=diff3D_1.%j.e
+#SBATCH --time=00:30:00
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_1000.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_1000"
+#SBATCH --output=diff3D_1000.%j.o
+#SBATCH --error=diff3D_1000.%j.e
+#SBATCH --time=00:50:00
+#SBATCH --nodes=1000
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_125.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_125"
+#SBATCH --output=diff3D_125.%j.o
+#SBATCH --error=diff3D_125.%j.e
+#SBATCH --time=00:30:00
+#SBATCH --nodes=125
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_216.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_216"
+#SBATCH --output=diff3D_216.%j.o
+#SBATCH --error=diff3D_216.%j.e
+#SBATCH --time=00:45:00
+#SBATCH --nodes=216
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_2197.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_2197"
+#SBATCH --output=diff3D_2197.%j.o
+#SBATCH --error=diff3D_2197.%j.e
+#SBATCH --time=00:50:00
+#SBATCH --nodes=2197
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_27.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_27"
+#SBATCH --output=diff3D_27.%j.o
+#SBATCH --error=diff3D_27.%j.e
+#SBATCH --time=00:30:00
+#SBATCH --nodes=27
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_512.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_512"
+#SBATCH --output=diff3D_512.%j.o
+#SBATCH --error=diff3D_512.%j.e
+#SBATCH --time=00:50:00
+#SBATCH --nodes=512
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_64.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_64"
+#SBATCH --output=diff3D_64.%j.o
+#SBATCH --error=diff3D_64.%j.e
+#SBATCH --time=00:30:00
+#SBATCH --nodes=64
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_runme_8.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -l
+#SBATCH --job-name="diff3D_8"
+#SBATCH --output=diff3D_8.%j.o
+#SBATCH --error=diff3D_8.%j.e
+#SBATCH --time=00:30:00
+#SBATCH --nodes=8
+#SBATCH --ntasks-per-node=1
+#SBATCH --partition=normal
+#SBATCH --constraint=gpu
+#SBATCH --account c23
+
+module load daint-gpu
+module load Julia/1.7.2-CrayGNU-21.09-cuda
+module load cray-hdf5-parallel
+
+export JULIA_HDF5_PATH=$HDF5_ROOT
+export JULIA_CUDA_MEMORY_POOL=none
+
+export IGG_CUDAAWARE_MPI=1
+export MPICH_RDMA_ENABLED_CUDA=1
+
+scp diff3D.jl daint_submit_pareff.sh $SCRATCH/diff3D
+
+pushd $SCRATCH/diff3D
+
+chmod +x *.sh
+
+srun daint_submit_pareff.sh
+
+scp out_diff3D_pareff* $HOME/diff3D

paper/codes/benchmarks/daint_submit_pareff.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+RUN="diff3D"
+
+for ie in {1..20}; do
+
+ # RDMA CUDA-aware
+ LD_PRELOAD="/usr/lib64/libcuda.so:/usr/local/cuda/lib64/libcudart.so" julia --project --check-bounds=no -O3 "$RUN".jl
+
+ # no RDMA
+ # julia --project --check-bounds=no -O3 "$RUN".jl
+
+done