Settings centralised

.gitignore

@@ -0,0 +1,2 @@
+env/
+noresm2/

LandsitesTools/prepare_input.py

@@ -0,0 +1,82 @@
+import os
+import sys
+import shutil
+import re
+import subprocess
+from configparser import ConfigParser, ExtendedInterpolation
+from pathlib import Path
+import pandas as pd
+
+class LandSiteInput:
+    """Land site input processor for NorESM"""
+
+    def __init__(self, pathsettings: Path):
+        """Parse the NorESM Land Site Platform settings file
+        Arguments:
+            pathsettings: path to settings file
+        Returns:
+            instance of LandSiteInput class
+        """
+        # Settings parser
+        self.settings = ConfigParser(interpolation=ExtendedInterpolation(),
+                                      inline_comment_prefixes='#')
+        self.settings.read(pathsettings)
+        # Directory of NorESM code and script paths
+        self.dir_noresm = self.readPath('paths', 'dir_noresm')
+        self.dir_clm_tools = self.readPath('scripts', 'dir_clm_tools')
+        self.path_mknoocnmap = self.readPath('scripts', 'mknoocnmap')
+        # Directory of raw NorESM input data
+        self.input_raw = self.readPath('paths', 'input_raw')
+        # Names and coordinates of all available sites
+        self.path_sites_table = self.readPath('paths', 'sites_table')
+        self.coordinate_sites = pd.read_csv(self.path_sites_table,
+                                            index_col='name')
+        # Sites to be simulated
+        self.sites2run = re.split('[ ,;]+',
+                                  self.settings.get('user', 'sites2run'))
+        # Flow control switches
+        self.switches = {k: eval(s) for k, s in self.settings.items('switches')}
+        # Version and root directory of data to be created
+        self.version = self.settings.get('user', 'version')
+        self.input_dir = self.readPath('paths', 'input_dir')
+
+    def readPath(self, section, parameter):
+        """Read and parse path from settings (configparser.ConfigParser format)
+        Arguments:
+            section: settings file section
+            parameter: settings file parameter name
+        """
+        return Path(self.settings.get(section, parameter)).expanduser()
+
+    def createGrid(self, name_site):
+        """Create site grid for a land site"""
+        # Load NCL module
+#       subprocess.run("module purge && module load NCL/6.6.2-intel-2019b",
+        subprocess.run("module purge && module load NCL/6.6.2-intel-2018b",
+                       shell=True, check=True)
+        # Create destination folder
+        dir_scripgrid = self.input_dir / f"share/scripgrids/{name_site}"
+        dir_scripgrid.mkdir(parents=True, exist_ok=True)
+        # Run mknoocnmap.pl using site's coordinates
+        yx = self.coordinate_sites.loc[name_site]
+        cmd = f"{self.path_mknoocnmap} -dx 0.01 -dy 0.01 " +\
+              f"-centerpoint {yx.latitude},{yx.longitude} -name {name_site}"
+        subprocess.run(cmd, shell=True, check=True)
+        # Move created files from temporary (current folder) to destination folder
+        tmpfiles_grid = [f for f in os.listdir(".") if name_site in f]
+        for f in tmpfiles_grid:
+            shutil.move(f, dir_scripgrid / f)
+        print(f"Files ./*{name_site}*.nc moved to {dir_scripgrid}")
+
+
+if __name__ == "__main__":
+    # 1. Read input from settings file, passed as shell argument
+    processor = LandSiteInput(sys.argv[1])
+    # 2. Loop through sites to be simulated
+    print(f"Input will be prepared for sites {processor.sites2run}")
+    for name_site in processor.sites2run:
+        case_name = f"{name_site}_default_{processor.version}"
+        # 2.1 Create script grid files
+        if processor.switches["creat_script"]:
+            processor.createGrid(name_site)
+    import pdb; pdb.set_trace()

LandsitesTools/prepare_input.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/bash
+
+#SBATCH --account=nn2806k 
+#SBATCH --job-name=mkmapdata
+#SBATCH --mem-per-cpu=256G --partition=bigmem
+#SBATCH --ntasks=1
+#SBATCH --time=07:00:00
+
+# 1. Directory of this script
+dir_script=$PWD/$(dirname "${BASH_SOURCE[0]}")
+
+# 2. Load conda module
+module purge && module load Anaconda3/2019.07
+source $(conda info --base)/etc/profile.d/conda.sh
+
+# 3. Activate virtual environment
+conda activate $dir_script/../env
+
+# 4. Run input preparation Python script
+python $dir_script/prepare_input.py $dir_script/../settings.txt

config/sites.csv

@@ -0,0 +1,13 @@
+name,latitude,longitude
+ALP1,61.0243,8.12343
+ALP2,60.8231,7.27596
+ALP3,60.8328,7.17561
+ALP4,60.9335,6.41504
+SUB1,60.8203,8.70466
+SUB2,60.876,7.17666
+SUB3,61.0866,6.63028
+SUB4,60.5445,6.51468
+BOR1,61.0355,9.07876
+BOR2,60.8803,7.16982
+BOR3,60.6652,6.33738
+BOR4,60.6901,5.96487

environment_test.yml

@@ -0,0 +1,17 @@
+name: sites_platform
+channels:
+  - conda-forge
+
+dependencies:
+  - vim=8.2
+  - cmake
+  - compilers
+  - gdal
+  - cdo>=1.9.7
+  - eccodes!=2.19.0  # cdo dependency; something messed up with libeccodes.so
+  - mpich<3.4  # avoid 3.4.1 external_2 from conda-forge like the plague
+  - nco
+  - ncl>=6.5.0  # this should always install 6.6.0 though
+  - python=3.8
+  - pip
+  - pandas=1.2

install.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# 1. Directories of this script; and NorESM (from settings file)
+dir_script=$PWD/$(dirname "${BASH_SOURCE[0]}")
+path_settings=$dir_script/settings.txt
+dir_platform=$(grep -oP 'dir_platform\s*[=:]\s*\K(.+)' $path_settings)
+dir_platform="${dir_platform/#~/$HOME}" # needed to get dir_noresm
+dir_noresm=$(grep -oP 'dir_noresm\s*[=:]\s*\K(.+)' $path_settings)
+dir_noresm=`eval echo $dir_noresm` # evaluate $dir_platform
+
+# 2. Install conda virtual environment
+module purge && module load Anaconda3/2019.07
+dir_env=$dir_script/env
+if ! [ -d $dir_env ]; then
+    conda env create --prefix $dir_env --file $dir_script/environment_test.yml
+else
+    echo "$dir_env exists: make sure it is the required conda environment!"
+fi;
+module purge
+# (run "conda clean -all" to avoid large number of cached files in ~/.conda/pkgs)
+
+# 3. Clone/update NorESM and get external tools
+url_noresm=$(grep -oP 'url_noresm\s*[=:]\s*\K(.+)' $path_settings)
+branch_noresm=$(grep -oP 'branch_noresm\s*[=:]\s*\K(.+)' $path_settings)
+mkdir -p $dir_noresm
+cd $dir_noresm
+if ! [ -d $dir_noresm/.git ]; then
+    git clone -b $branch_noresm $url_noresm $dir_noresm
+    echo "NorESM cloned into $dir_noresm: on branch $branch_noresm"
+else
+    git checkout $branch_noresm && git pull
+    echo "NorESM already exists in $dir_noresm: $branch_noresm branch up to date"
+fi;
+python2 manage_externals/checkout_externals # TO CHECK: not working with Python3
+# TO DO: run $dir_platform/LandsitesTools/ctsm_patching.ipynb ?

settings.txt

@@ -0,0 +1,33 @@
+[user]
+sites2run = ALP1, ALP2
+version = 2.0.0
+
+[paths]
+dir_platform = ~/NorESM_LandSites_Platform
+sites_table = ${dir_platform}/config/sites.csv
+input_raw = /cluster/shared/noresm/inputdata
+input_dir = /cluster/shared/noresm/test_inputdata_fates_platform
+dir_noresm = ${dir_platform}/noresm2
+url_noresm = https://github.com/NorESMhub/NorESM.git
+branch_noresm = noresm_landsites
+
+[switches]
+creat_script = True      # create script grid or not
+creat_domain = False     # create domain file
+creat_mapping = False    # create mapping file
+creat_surfdat = False    # create surface data file
+creat_aero = False       # create aerosol depostion file for each site. If False, model will use global data as input.               
+creat_topo = False       # create topography file for each site. If False, model will use global data as input. 
+creat_urb = False        # create urban data file for each site. If False, model will use global data as input. 
+creat_fire = False       # create fire data file for each site. If False, model will use global data as input. 
+creat_atm = False        # create atmospheric forcing. 
+tar_input = False        # tar all the input data required for each site.
+run_case = False         # run site simulations automatically. True can only be used when all the inputdata are ready!!!!  
+run_case_first = False   # create, build and submit short test runs
+run_case_second = False  # run long experiments
+run_archive = True       # archive experiments. 
+merge_hist = True        # merge history experiments. 
+
+[scripts]
+dir_clm_tools = ${paths:dir_noresm}/components/clm/tools
+mknoocnmap = ${dir_clm_tools}/mkmapdata/mknoocnmap.pl