Merge pull request #88 from NorESMhub/platform_dev

First release

.dockerignore

@@ -0,0 +1,6 @@
+build
+dist
+data
+docs
+notebooks
+venv

.github/workflows/docker.yml

@@ -0,0 +1,47 @@
+name: Build and publish the docker image on Github Container Registry
+
+on:
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checkout source code
+      - name: Check out source code
+        uses: actions/checkout@v2
+
+      # Calculate some variables that are used later
+      - name: Version information
+        run: |
+          if [ "${{ github.event.release.target_commitish }}" != "" ]; then
+            BRANCH="${{ github.event.release.target_commitish }}"
+          elif [[ $GITHUB_REF =~ pull ]]; then
+            BRANCH="$(echo $GITHUB_REF | sed 's#refs/pull/\([0-9]*\)/merge#PR-\1#')"
+          else
+            BRANCH=${GITHUB_REF##*/}
+          fi
+          echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
+          if [ "$BRANCH" == "main" ]; then
+            TAGS=latest
+          else
+            TAGS=${BRANCH}
+          fi
+          echo "TAGS=${TAGS}" >> $GITHUB_ENV
+
+      # build image
+      - name: Build image
+        uses: elgohr/Publish-Docker-Github-Action@3.04
+        env:
+          BRANCH: ${{ env.GITHUB_BRANCH }}
+          BUILDNUMBER: ${{ github.run_number }}
+          GITSHA1: ${{ github.sha }}
+        with:
+          name: noresmhub/noresm_land_sites_platform
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+          dockerfile: docker/Dockerfile
+          tags: "${{ env.TAGS }}"
+          buildargs: BRANCH,BUILDNUMBER,GITSHA1

.gitignore

@@ -0,0 +1,64 @@
+### Project specific
+# ignore...
+data/cases/*
+data/input/clm/*
+data/input/user_observations/atmosphere/*
+data/input/user_observations/soil/*
+data/output/*
+landsites_tools/custom_settings/*
+*.remote-sync.json
+*test.py
+site/
+
+# keep... (don't track user files, but make sure folders are in place)
+!cases/.gitkeep
+!data/input/clm/.gitkeep
+!data/input/user_observations/.gitkeep
+!data/input/user_observations/atmosphere/example.csv
+!data/input/user_observations/soil/example.csv
+!output/.gitkeep
+!landsites_tools/custom_settings/.gitkeep
+
+### Environment
+env/
+.env
+.venv
+venv/
+ENV/
+env.bak/
+venv.bak/
+### NorESM model
+noresm2/
+
+### Python and notebooks
+__pycache__/
+.*.swp
+.ipynb_checkpoints
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+*.py[cod]
+*$py.class
+profile_default/
+ipython_config.py
+src/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+

.machine/NREC_VM_setup/.bashrc

@@ -0,0 +1,18 @@
+# .bashrc
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# Uncomment the following line if you don't like systemctl's auto-paging feature:
+# export SYSTEMD_PAGER=
+
+# User specific aliases and functions
+alias view='vim -R'
+alias du='du -h'
+alias df='df -h'
+alias grep='egrep --color=always -n --exclude-dir=.git'
+alias tom='top -o %MEM'
+alias ncdump='ncdump -h'
+alias jlremote='jupyter lab --no-browser --port=8899'

.machine/NREC_VM_setup/.tmux.conf

@@ -0,0 +1,32 @@
+# remap prefix from 'C-b' to 'C-a'
+unbind C-b
+set-option -g prefix C-a
+bind-key C-a send-prefix
+
+# switch panes using Alt-arrow without prefix
+bind -n M-Left select-pane -L
+bind -n M-Right select-pane -R
+bind -n M-Up select-pane -U
+bind -n M-Down select-pane -D
+
+# split panes using | and -
+bind | split-window -h
+bind - split-window -v
+unbind '"'
+unbind %
+
+# don't rename windows automatically
+set-option -g allow-rename off
+
+# eliminate Esc delays
+set -sg escape-time 0
+
+# do not spawn login shells
+set -g default-command "${SHELL}"
+
+# Quick window selection
+bind -r C-h select-window -t :-   # Jump to window on the left
+bind -r C-l select-window -t :+   # Jump to window on the right
+
+# Use 256 colors
+set -g default-terminal "xterm-256color"

.machine/NREC_VM_setup/.vimrc

@@ -0,0 +1,105 @@
+set nocompatible              " required
+filetype off                  " required
+
+" ruler: (line,column) bottom right
+set ruler
+
+" visual bell
+set visualbell
+set t_vb=
+
+" always highlight search results
+set hlsearch
+hi Search cterm=NONE ctermfg=Yellow ctermbg=DarkGrey
+
+" enable file-specific settings
+filetype plugin on
+
+" set the runtime path to include Vundle and initialize
+set rtp+=~/.vim/bundle/Vundle.vim
+call vundle#begin()
+
+" alternatively, pass a path where Vundle should install plugins
+" "call vundle#begin('~/some/path/here')
+
+" let Vundle manage Vundle, required
+Plugin 'gmarik/Vundle.vim'
+
+" Add all your plugins here (note older versions of Vundle used Bundle instead of Plugin)
+Plugin 'vim-scripts/indentpython.vim'
+Bundle 'Valloric/YouCompleteMe'
+Plugin 'ervandew/supertab'
+Plugin 'jpalardy/vim-slime'
+Plugin 'JuliaEditorSupport/julia-vim'
+
+" All of your Plugins must be added before the following line
+call vundle#end()            " required
+filetype plugin indent on    " required
+
+" youcompleteme settings
+let g:ycm_autoclose_preview_window_after_completion=1
+map <leader>g  :YcmCompleter GoToDefinitionElseDeclaration<CR>
+
+"" omnicompletion (instead of Valloric/YouCompleteMe on vim 7)
+"filetype plugin on
+"set omnifunc=syntaxcomplete#Complete
+
+" Other Python settings
+syntax on
+set tabstop=4
+set softtabstop=4
+set shiftwidth=4
+set expandtab
+set autoindent
+set fileformat=unix
+
+" Show current line number and relative numbers around it
+set number
+set relativenumber
+highlight CursorLineNR ctermbg=DarkGrey cterm=bold ctermfg=Yellow
+
+" colors
+set term=screen-256color
+"if exists('$TMUX')
+"  set term=screen-256color
+"endif
+highlight Visual cterm=bold ctermbg=Black ctermfg=NONE
+
+" Slime plugin (send text to REPL via tmux)
+if exists('$TMUX')
+    let g:slime_target = "tmux"
+    let g:slime_paste_file = tempname()
+    let g:slime_default_config = {"socket_name": split($TMUX, ",")[0], "target_pane": "1"}
+    let g:slime_python_ipython = 1
+    xmap ,q <Plug>SlimeRegionSend
+    xmap ,a <Plug>SlimeLineSend
+endif
+nmap ,q <Plug>SlimeParagraphSend
+nmap ,a <Plug>SlimeLineSend
+
+" for tmux to automatically set paste and nopaste mode at the time pasting (as happens in VIM UI)
+function! WrapForTmux(s)
+  if !exists('$TMUX')
+    return a:s
+  endif       
+  let tmux_start = "\<Esc>Ptmux;"
+  let tmux_end = "\<Esc>\\"        
+  return tmux_start . substitute(a:s, "\<Esc>","\<Esc>\<Esc>", 'g') . tmux_end
+endfunction
+
+let &t_SI .= WrapForTmux("\<Esc>[?2004h")
+let &t_EI .= WrapForTmux("\<Esc>[?2004l")
+
+function! XTermPasteBegin()
+  set pastetoggle=<Esc>[201~
+  set paste
+  return ""
+endfunction
+
+inoremap <special> <expr> <Esc>[200~ XTermPasteBegin()
+
+" wrap text in diff mode
+au VimEnter * if &diff | execute 'windo set wrap' | endif
+
+" backspace
+set backspace=indent,eol,start

.machine/NREC_VM_setup/README.md

@@ -0,0 +1,73 @@
+# Tools to set up a NREC virtual machine for development
+
+## 1. Access to NREC
+Execute these steps according to the
+[NREC documentation](https://docs.nrec.no/index.html):
+1. Register as a user and save your API password, which you will need in step 3
+   (https://docs.nrec.no/login.html#first-time-login).
+2. Install OpenStack command line interface tools
+   (https://docs.nrec.no/api.html#openstack-command-line-interface-cli).
+3. Create a `keystone_rc.sh` file in your local home folder
+   (https://docs.nrec.no/api.html#using-the-cli-tools),
+   where `OS_REGION_NAME` should be set either to `osl` or `bgo`
+   depending on available project resources, which can be checked on
+   [the NREC dashboard](https://dashboard.nrec.no/dashboard/project/)
+   by switching regions in the dialog at the top to the right of the
+   project name.
+
+
+## 2. Create a NREC virtual machine from the latest image
+The latest image on NREC is [NorESM_platform_0.5](https://dashboard.nrec.no/dashboard/ngdetails/OS::Glance::Image/e09310ea-1d68-4a9a-9be0-5cec7051f225).
+Instances can be created from this image by executing the script [`create_VM_latest_image.sh`](https://github.com/NorESMhub/NorESM_LandSites_Platform/blob/main/.machine/NREC_VM_setup/create_VM_latest_image.sh),
+where some parameters, which are set under the "Default names and paths" header,
+can be modified manually if needed (see also Sect. 3.1).
+
+To build an instance from the base CentOS 8 image follow the steps in Sect. 3.
+
+
+## 3. Create and configure a NREC virtual machine from scratch
+On your local Linux machine run the steps in `source setup_VM_NREC.sh`
+to set up the remote virtual machine (VM) and install the dependencies.
+
+Set the `image` variable in `setup_VM_NREC.sh` to decide from which
+image (snapshot) your VM will be created.
+
+Manual execution is recommended to avoid problems due to delays or
+connection errors when calling `openstack` commands.
+
+Login information is printed at the end of `setup_VM_NREC.sh` execution.
+
+### 3.1 Notes
+- The following parameters are set at the beginning of the `setup_VM_NREC.sh`
+  under "Default names and paths": VM name, SSH key path and
+  name, and security group name.
+- Several steps (Create security group and rules, Upload public SSH key,
+  Instance creation) can be done manually on the
+  [NREC dashboard](https://dashboard.nrec.no): links to the documentation are
+  provided in `setup_VM_NREC.sh`.
+- Sources for CLM dependencies and configuration are the CESM Dockerfiles:
+  * https://github.com/ESCOMP/ESCOMP-Containers/blob/master/base/centos8/Dockerfile
+  * https://github.com/ESCOMP/ESCOMP-Containers/blob/master/CESM/2.2/Dockerfile
+
+### 3.2 To do
+In `setup_VM_NREC.sh`:
+- Create, attach and mount storage volume
+- Mount cluster storage via sshfs (needed to create data for new sites?)
+- Use shell arguments in to set machine parameters now hard-coded under
+  "Default names and paths".
+
+
+## 4. Remote development with Jupyter Lab
+To edit files and run programs on the remote VM from the web browser on
+your local Linux machine (Ubuntu under Windows needs testing):
+1. On the VM run `jlremote` (alias defined in the `.bashrc` that is copied to
+   the VM during the setup process described in Sect. 1).
+   It starts a Jupyter Lab server without graphical interface.
+   An URL is printed in the terminal: copy the string after `token=` that is
+   needed as password for the first connection from the local machine.
+2. On the local machine run the bash script `jllocal.sh` (located in the same
+   folder of this README file): `bash jllocal.sh centos $address`,
+   where `address` is the VM IP address environment variable.
+3. A web-browser tab should open and you may need to type in the token;
+   otherwise you have to open the URL in the browser (see point 1).
+   The webpage may need to be refreshed to display the Jupyter Lab interface.

.machine/NREC_VM_setup/create_VM_latest_image.sh

@@ -0,0 +1,32 @@
+#! /usr/bin/bash
+set -e # Exit if any command fails
+#unalias grep # only needed if grep is aliased .bashrc or similar
+
+# Default names and paths (TO DO: use command line arguments)
+name_server=${USER}_dev # name of virtual machine
+path_key=~/.ssh/id_rsa.pub # path of local SSH public key to be uploaded
+name_key=mykey # key name once uploaded to NREC
+security_group=SSH_ICMP_all # name of project's security group
+machine_size=m1.medium # amount of resources allocated to VM
+
+# Image (snapshot) to build VM from (TO DO: use command line arguments)
+image="NorESM_platform_0.5"
+
+# Instance creation (https://docs.nrec.no/create-virtual-machine.html)
+if ! openstack server list | grep -q $name_server; then
+    openstack server create --image "$image" \
+                            --flavor $machine_size \
+                            --security-group $security_group \
+                            --security-group default \
+                            --key-name $name_key \
+                            --nic net-id=dualStack $name_server
+fi
+
+# Get virtual machine's IP address and add it to SSH known hosts
+echo "Wait 1 min for the virtual machine to build..."
+sleep 1m
+address=`openstack server list | grep "$name_server" | grep -oP '\K([0-9\.]{10,})'`
+echo "NREC machine IP address: $address"
+
+# Login
+ssh centos@$address