[Manual] Devito on CSD3@Cambridge

Installing Devito on CSD3@CAMBRIDGE

Update 11/06/24

ssh -i .ssh/<your id_rsa_csd3> <username>@login-icelake.hpc.cam.ac.uk
<TOTP CODE WILL BE THEN ASKED>

# Do `ssh` to your login node (This example logs in to Intel(R) Xeon(R) Platinum 8368Q login node (login-q-3))

# Python3.11 is available on the nodes by default, so we create a venv

mkdir environments/python311
python3.11 -m venv environments/python311

# Following instructions from https://github.com/devitocodes/devito
git clone https://github.com/devitocodes/devito.git
cd devito
source environments/python311/bin/activate

pip3 install --upgrade -e .

# Load the default-dawn it also "purges"
module load purge # Redundant, if you do default-dawn
module load default-dawn
module load intel-oneapi-compilers/
# icx should already be available
icx --version
# Intel(R) oneAPI DPC++/C++ Compiler 2024.1.0 (2024.1.0.20240308)
and
icpx --version
# Intel(R) oneAPI DPC++/C++ Compiler 2024.1.0 (2024.1.0.20240308)


# If everything went fine, you should be able to run an example operator.
# Quickly try a cheap one, on a login node, i.e.:
DEVITO_LOGGING=DEBUG DEVITO_ARCH=intel python examples/seismic/acoustic/acoustic_example.py -d 40 40 40

Now let's try to install mpi4py

# First check what mpiicc is available
mpiicc --version
module load intel-oneapi-compilers/
module load intel-oneapi-mpi
# Then install mpi4py
pip3 install --ignore-installed --no-cache-dir -r requirements-mpi.txt
# and try a small example

# Start an interactive job:
sintr -t 01:00:00 --exclusive -A TRAINING-DAWN-GPU -p pvc -n 1 -c 24 --gres=gpu:1

# If you have a problem with some shortcuts in an interactive job on dawn,  you can try to launch a job with the command below, to avoid
# running screen, which sintr script does:

salloc  -t 01:00:00 -A TRAINING-DAWN-GPU -p pvc -n 1 -c 24 --gres=gpu:1 srun --interactive --preserve-env --pty bash

icpx -fPIC -O3 -xHost -Wall -fsycl -fsycl-targets=spir64 -shared /tmp/devito-jitcache-uid29380/e75e2fffacb07eb51f934279fb75ff2f520b703f.c -lm -o /tmp/devito-jitcache-uid29380/e75e2fffacb07eb51f934279fb75ff2f520b703f.so

icpx -O3 -g -fPIC -Wall -xHost -fsycl -fsycl-targets=spir64 -shared /tmp/devito-jitcache-uid29380/69d519b0008bed583ec3664a764c9f0f13d057fb.c -lm -o /tmp/devito-jitcache-uid29380/69d519b0008bed583ec3664a764c9f0f13d057fb.so