wrf.pbs

#!/bin/bash --login

# See http://www.archer.ac.uk/documentation/user-guide/batch.php

#PBS -l select=2
#PBS -N wrf
#PBS -A <your_archer_account>
#PBS -l walltime=12:00:00
#PBS -q standard

# Number of processes (i.e. number of nodes selected times number of cores per
# node (24 on ARCHER)).
np=$(echo "$(wc -l < "$PBS_NODEFILE") * 24" | bc -l)

set -eu

# Use the ARCHER compiled version of WRF 3.7.1.
module load wrf-wrf_wps_chem/3.7.1_build1-em_real-basic_nesting-dm+sm-cray-s

real=$WRF_WRF_WPS_CHEM_DIR/WRFV3/run/real.exe
wrf=$WRF_WRF_WPS_CHEM_DIR/WRFV3/run/wrf.exe
years=$(seq 2000 2010)

# Use restart files for relaunching a job?
restart=${restart:-yes}

# Do a coldstart? i.e. run real.exe. This is only checked for the first month
# (i.e. January).
coldstart=${coldstart:-yes}

# Skip n months? 0 = no skipping.
skipmonths=${skipmonths=0}

# Make sure any symbolic links are resolved to absolute path.
export PBS_O_WORKDIR=$(readlink -f $PBS_O_WORKDIR)

# Set the number of threads to 1
#   This prevents any system libraries from automatically
#   using threading.
export OMP_NUM_THREADS=1

# Change to the directory from which the job was submitted. This should be the
# project "run" directory as all the paths are assumed relative to that.
cd $PBS_O_WORKDIR

mv wrf_*.{e,o}* logs || true
if [ -f $(echo ./core.* | cut -f1 -d' ') ]; then
    rm ./core.*
fi

for year in $years; do

    # Remove run file symlinks so we don't bomb out trying to recreate them.
    find ./ -maxdepth 1 -type l -delete

    # Symlink the input files to the current directory. I really need to figure
    # out how to tell WRF to look for the files in a specific directory.
    lndir input/$year/ .
    # Symlink all the common files needed for the run.
    lndir input/common/ .

    for rawmonth in {1..12}; do

        month=$(printf %02d $rawmonth)
        if [ $rawmonth -le $skipmonths ]; then
            echo "Skipping $month."
            continue
        fi

        # Remove the symlinks to the non-existent binaries.
        for i in {wrf,real,ndown,tc}.exe; do
            if [ -h $i ]; then rm $i; fi
        done

        # Make somewhere to store the output and logs.
        mkdir -p output/$year/{$month,restart} logs

        # Link to the current namelist.
        if [ -h ./namelist.input ]; then
            rm ./namelist.input || true
        fi
        ln -s $(readlink -f models/${year}-${month}.input) ./namelist.input

        # Check if we have restart files and link accordingly.
        if [ -d output/$year/restart/ -a $restart == 'yes' ]; then
            lndir output/$year/restart/
        fi

        # Launch the parallel jobs. Only do the real.exe if this is not a
        # restart run. This assumes you only want a cold start in the first
        # month.
        if [ $coldstart == 'yes' -a $rawmonth -eq 1 ]; then
            aprun -n $np $real
        fi
        aprun -n $np $wrf

        # Move the output stuff to the relevant subdirectories.
        mv wrfbdy_d?? output/$year/$month || true
        mv wrfinput_d?? output/$year/$month || true
        mv "wrfout_d??_*" output/$year/$month || true
        # Remove the symlinked restart files.
        find ./ -maxdepth 1 -type l -iname "wrfrst_d??_*" -delete
        # Move the remaining restart files to the common restart directory
        mv "wrfrst_d??_*" output/$year/restart/ || true
        mv namelist.output output/$year/$month/namelist-${year}-${month}-${PBS_JOBID%.*}.output || true

        # Save the input files used in this model run to a text file.
        echo -n > output/$year/$month/inputs.txt
        for i in input/$year/*; do
            readlink -f ${i} >> output/$year/$month/inputs.txt
        done

        # Save the model configuration for this run.
        cp -a $(readlink -f ./namelist.input) output/$year/$month/

        # Copy the WRF binaries too.
        cp -a $(readlink -f $real) $(readlink -f $wrf) output/$year/$month/

        # Remove the current model symlink.
        if [ -h ./namelist.input ]; then
            rm ./namelist.input
        fi

        # Rename and move the logs.
        rename error error-${year}-${month}-${PBS_JOBID} rsl.error.*
        rename out out-${year}-${month}-${PBS_JOBID} rsl.out.*
        # Only keep a single log file.
        mv ./rsl.{error,out}*.0000 logs || true
        rm ./rsl.{error,out}*.????

    done

done