487 add singularity container download

dfm_tools/data.py

@@ -12,6 +12,10 @@
 import pooch
 import zipfile
 from dfm_tools import open_partitioned_dataset, preprocess_hisnc, open_dataset_delft3d4
+import tarfile
+import glob
+import warnings
+import getpass
 
 
 def get_dir_testdata():
@@ -253,3 +257,41 @@
 
  return dir_gshhs
 
+
+def fm_singularity_container():
+ dir_testdata = get_dir_testdata()
+
+ fname_zip = 'Stable_release.zip' # this can be any name, but is the default name from the download portal
+ filepath_zip = os.path.join(dir_testdata,fname_zip)
+ dir_container = os.path.join(dir_testdata,'delft3dfm_container')
+
+ #download zipfile if not present
+ if not os.path.exists(dir_container) and not os.path.exists(filepath_zip):
+ download_key = getpass.getpass("Enter your deltares download key: ")
+
+ file_url = f'https://deltares.thegood.cloud/s/{download_key}/download'
+ print(f'downloading "{fname_zip}" from deltares.thegood.cloud to cachedir')
+ r = requests.get(file_url, allow_redirects=True)
+ r.raise_for_status() #raise HTTPError if url not exists
+ with open(filepath_zip, 'wb') as f:
+ f.write(r.content)
+
+ if not os.path.exists(dir_container):
+ print(f'extracting "{fname_zip}"')
+ with zipfile.ZipFile(filepath_zip, 'r') as zip_ref:
+ file_list = zip_ref.namelist()
+ zip_ref.extractall(dir_testdata)
+
+ #get tarfile name
+ zipdir_tar = [x for x in file_list if x.endswith('.tar.gz')][0]
+ filepath_tar = os.path.join(dir_testdata,zipdir_tar)
+ fname_tar = os.path.basename(filepath_tar)
+
+ print(f'extracting "{fname_tar}"')
+ with tarfile.open(filepath_tar, 'r:gz') as tar_ref:
+ tar_ref.extractall(dir_container)
+
+ filepath_sif_list = glob.glob(os.path.join(dir_container,'*.sif'))
+ if len(filepath_sif_list)!=1:
+ raise ValueError(f'no or multiple delft3dfm singularity containers (.sif file) found:\n{filepath_sif_list}')
+ return dir_container

docs/notebooks/runmodel_singularity_example.ipynb

@@ -0,0 +1,135 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "e02e4ff7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "04aa1354",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "C:\\Users\\veenstra\\AppData\\Local\\dfm_tools\\dfm_tools\\Cache\\delft3dfm_container\n"
+ ]
+ }
+ ],
+ "source": [
+ "# download singularity container\n",
+ "from dfm_tools.data import fm_singularity_container\n",
+ "dir_container = fm_singularity_container()\n",
+ "print(dir_container)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "638fcf4e",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "singularitydir=\"C:\\Users\\veenstra\\AppData\\Local\\dfm_tools\\dfm_tools\\Cache\\delft3dfm_container\"\n",
+ "\n",
+ "# DIMR input file; must already exist!\n",
+ "dimrFile=dimr_config.xml\n",
+ "\n",
+ "# stop after an error occurred:\n",
+ "set -e\n",
+ "\n",
+ "# Replace number of processes in DIMR file\n",
+ "PROCESSSTR=\"$(seq -s \" \" 0 $((nPart-1)))\"\n",
+ "sed -i \"s/\\(<process.*>\\)[^<>]*\\(<\\/process.*\\)/\\1$PROCESSSTR\\2/\" $dimrFile\n",
+ "\n",
+ "# Read MDU file from DIMR file\n",
+ "mduFile=\"$(sed -n 's/\\r//; s/<inputFile>\\(.*\\).mdu<\\/inputFile>/\\1/p' $dimrFile)\".mdu\n",
+ "\n",
+ "# Sequential computation\n",
+ "# \"-p\": See above. Arguments after \"run_dimr.sh\" are explained in run_dimr.sh\n",
+ "$singularitydir/execute_singularity.sh -p 2 run_dimr.sh -m $dimrFile\n",
+ "\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "1"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#run model with singularity container (only works if singularity is installed on machine)\n",
+ "# this is a test without MPI, including MPI makes it a bit more complex\n",
+ "\n",
+ "exec_command = rf\"\"\"singularitydir=\"{dir_container}\"\n",
+ "\n",
+ "# DIMR input file; must already exist!\n",
+ "dimrFile=dimr_config.xml\n",
+ "\n",
+ "# stop after an error occurred:\n",
+ "set -e\n",
+ "\n",
+ "# Replace number of processes in DIMR file\n",
+ "PROCESSSTR=\"$(seq -s \" \" 0 $((nPart-1)))\"\n",
+ "sed -i \"s/\\(<process.*>\\)[^<>]*\\(<\\/process.*\\)/\\1$PROCESSSTR\\2/\" $dimrFile\n",
+ "\n",
+ "# Read MDU file from DIMR file\n",
+ "mduFile=\"$(sed -n 's/\\r//; s/<inputFile>\\(.*\\).mdu<\\/inputFile>/\\1/p' $dimrFile)\".mdu\n",
+ "\n",
+ "# Sequential computation\n",
+ "# \"-p\": See above. Arguments after \"run_dimr.sh\" are explained in run_dimr.sh\n",
+ "$singularitydir/execute_singularity.sh -p 2 run_dimr.sh -m $dimrFile\n",
+ "\"\"\"\n",
+ "\n",
+ "#os.chdir(dir_output)\n",
+ "print(exec_command)\n",
+ "os.system(exec_command)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "219b38b7",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}