Parameters updates, default to Broadwell

README.md

@@ -48,6 +48,10 @@ $$ srun --mpi=pmi2 python3 my_fenics_script.py
 You can adjust the build location and the installation location in the file
 `env-build-fenics.sh`.
 
+Default compiler optimisation flags are `-O3 -march=haswell`. If you would like to
+target AVX-512 nodes please change `FENICSX_OPTFLAGS` environmental variable in `env-build-fenics.sh`
+and the content of `dolfinx_jit_parameters.json` in `env-fenics.sh` accordingly.
+
 ## Running FEniCS MPI jobs ##
 
 Included in this repository is a very simple example launcher script to submit
@@ -56,7 +60,7 @@ jobs on the cluster.
 ```
 $ cd $HOME
 $ cd fenicsx-iris-cluster
-$ sbatch -n 4 fenics-launcher.sh python3 poisson.py
+$ sbatch -n 4 fenicsx-launcher.sh python3 poisson.py
 ```
 
 ## Experimental: LLNL Spindle Support

build-fenics.sh

@@ -20,7 +20,7 @@ cd $BUILD_DIR && \
    mkdir build && \
    cd build && \
    cmake -DDOLFINX_SKIP_BUILD_TESTS=True -DCMAKE_INSTALL_PREFIX=${PREFIX} \
-     -DCMAKE_BUILD_TYPE="Release" -DCMAKE_CXX_FLAGS_RELEASE="-O2 -march=native" ../cpp && \
+     -DCMAKE_BUILD_TYPE="Release" -DCMAKE_CXX_FLAGS_RELEASE=${FENICSX_OPTFLAGS} ../cpp && \
    make install && \
    cd ../python && \
-   python3 -m pip install --ignore-installed --no-dependencies .
+   CXXFLAGS=${FENICSX_OPTFLAGS} python3 -m pip install --ignore-installed --no-dependencies .

build-petsc.sh

@@ -13,9 +13,9 @@ cd ${BUILD_DIR} && \
    cd petsc && \
    python3 ./configure \
                --with-blaslapack-dir=${EBROOTIMKL}/mkl \
-               --COPTFLAGS="-march=native -O2" \
-               --CXXOPTFLAGS="-march=native -O2" \
-               --FOPTFLAGS="-march=native -O2" \
+               --COPTFLAGS=${FENICSX_OPTFLAGS} \
+               --CXXOPTFLAGS=${FENICSX_OPTFLAGS} \
+               --FOPTFLAGS=${FENICSX_OPTFLAGS} \
                --with-cc=${MPICC} --with-cxx=${MPICXX} --with-fc=${MPIFC} \
                --with-mpiexec="srun -n 1 --mpi=pmi2" \
                --download-metis \

builder.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-srun -p interactive --qos qos-interactive --time=02:00:00 -N 1 -n 1 --pty bash -i
+srun -p interactive --time=02:00:00 -N 1 -n 1 --pty bash -i

env-build-fenics.sh

@@ -28,6 +28,8 @@ export MPICC=mpigcc
 export MPICXX=mpigxx
 export MPIFC=mpif90
 
+export FENICSX_OPTFLAGS="-O3 -march=haswell"
+
 # Using Intel Compilers (doesn't work at the moment).
 # Note that some components (e.g. Boost) are always built with GCC.
 #export CC=icc

env-fenics.sh

@@ -3,8 +3,8 @@ source ${HOME}/fenicsx-iris-cluster/env-build-fenics.sh
 
 # Note: These must be unset initially, need to code this in somehow.
 export PETSC_DIR=${PREFIX}
-export DOLFINX_JIT_CACHE_DIR=${PREFIX}/cache
-export DOLFINX_JIT_CFLAGS="-O3 -march=native"
+mkdir -p ${HOME}/.config/dolfinx
+echo '{"cache_dir":"'${PREFIX}/.cache/fenics'", "cffi_extra_compile_args": ["-O3", "-march=haswell"]}' > ${HOME}/.config/dolfinx/dolfinx_jit_parameters.json
 
 # Bring in virtualenv with python package
 source $HOME/.local/bin/virtualenvwrapper.sh

fenicsx-launcher.sh

@@ -1,6 +1,5 @@
 #!/bin/bash -l
 #SBATCH --time=0-00:03:00
-#SBATCH --qos=qos-batch
 source $HOME/fenicsx-iris-cluster/env-fenics.sh
 cd $SLURM_SUBMIT_DIR
 #srun --mpi=pmi2 "$@"