get_grib.run

#!/bin/bash

# Cd into our working directory in case we're not into it already
cd "$(dirname "$0")";

echo "gens: Starting processing of GEFS model data - `date`"

export MODEL_DATA_FOLDER=/home/ekman/ssd/guido/gens/
export IMGDIR=/home/ekman/ssd/guido/gens/
export HOME_FOLDER=$(pwd)
export NCFTP_BOOKMARK="mid"
DATA_DOWNLOAD=true
DATA_PLOTTING=true
DATA_UPLOAD=true

# Wrapper to merge the members
merge_gefs_member()
{
    echo "Merging perturbation ${1}"
    # We need sellonlatbox to shift the grid from 0,360 to -180, 180. Somehow it is
    # easier to do it now that afterwars in Python
    cdo -f nc copy -sellonlatbox,-180,180,-90,90 -mergetime \
                    $MODEL_DATA_FOLDER"grib_gefs_"$YEAR$MONTH$DAY"_"$RUN"_*_${1}" \
                    $MODEL_DATA_FOLDER"grib_gefs_"$YEAR$MONTH$DAY"_"$RUN"_${1}.nc"
}
export -f merge_gefs_member
# Make sure we're using bash
export SHELL=$(type -p bash)
# We need to open many files at the same time
ulimit -Sn 8192
########################################### 

mkdir -p ${MODEL_DATA_FOLDER}euratl
mkdir -p ${MODEL_DATA_FOLDER}nh

##### COMPUTE the date variables to determine the run
export MONTH=$(date -u +"%m")
export DAY=$(date -u +"%d")
export YEAR=$(date -u +"%Y")
export HOUR=$(date -u +"%H")

if [ $HOUR -ge 6 ] && [ $HOUR -lt 12 ]
then
    export RUN=00
elif [ $HOUR -ge 12 ] && [ $HOUR -lt 18 ]
then
    export RUN=06
elif [ $HOUR -ge 18 ]
then
    export RUN=12
elif [ $HOUR -ge 0 ] && [ $HOUR -lt 6 ]
then
    DAY=$(date -u -d'yesterday' +"%d")
    export RUN=18
else
    echo "Invalid hour!"
fi

echo "----------------------------------------------------------------------------------------------"
echo "gens: run ${YEAR}${MONTH}${DAY}${RUN}"
echo "----------------------------------------------------------------------------------------------"

# Move to the data folder to do processing
cd ${MODEL_DATA_FOLDER} || { echo 'Cannot change to DATA folder' ; exit 1; }

# SECTION 1 - DATA DOWNLOAD ############################################################

if [ "$DATA_DOWNLOAD" = true ]; then
    echo "-----------------------------------------------"
    echo "gens: Starting downloading of data - `date`"
    echo "-----------------------------------------------"
    cp ${HOME_FOLDER}/*.py ${MODEL_DATA_FOLDER}
    #loop through forecast hours
    # hours_download=$(seq -s " f" -f %03g 6 6 384)
    # hours_download="f${hours_download}"
    # Finally convert to array
    # hours_download=($hours_download)
    # f006 f012 f018 f024 f030...
    members_download=$(seq -s " gep" -f %02g 0 1 30)
    members_download="gec${members_download}"
    # Finally convert to array
    members_download=($members_download)
    # gec00 gep01 gep02 gep03...
    # This is needed to avoid some errors in CDO
    export SKIP_SAME_TIME=1
    # clean out the old grib data
    rm ${MODEL_DATA_FOLDER}/grib_gefs*
    # Call the Python script to do the downloading
    python downloader.py \
    -d "${YEAR}${MONTH}${DAY}" \
    -r "${RUN}" \
    -v "APCP" "CSNOW" "CRAIN" "TMP" "TMP" "HGT" "UGRD" "VGRD" "CAPE" \
    -l "surface" "surface" "surface" "2 m above ground" "850 mb" "500 mb" "10 m above ground" "10 m above ground" "180-0 mb above ground" \
    -o "${MODEL_DATA_FOLDER}"
    # We have to pass the hours and members...
    # For the moment we let Python handle that
    if [[ $? = 0 ]]; then
        echo "Downloaded files succesfully"
    else
        echo "Could not download data, exiting"
        exit 1
    fi

    # Delete empty files
    FILENAMES=$MODEL_DATA_FOLDER"grib_gefs_"$YEAR$MONTH$DAY"_"$RUN"_*"
    find $FILENAMES -empty -type f -delete

    # merging files 
    parallel -j 5 merge_gefs_member ::: ${members_download[@]}

    # Remove all the non-netcdf files which are not necessary 
    rm $MODEL_DATA_FOLDER/grib_gefs*[!.nc]
fi

# SECTION 2 - DATA PLOTTING ############################################################

if [ "$DATA_PLOTTING" = true ]; then
    echo "-----------------------------------------------"
    echo "gens: Starting plotting of data - `date`"
    echo "-----------------------------------------------"
    python --version
    export QT_QPA_PLATFORM=offscreen 

    python plot_meteogram.py Milano Roma Palermo Hamburg Pisa Storrs Utrecht Toulouse Sassari Cagliari Munich Napoli 

    scripts=("plot_gph_500.py" "plot_t_850.py" "plot_prec.py")
    projections=("euratl" "nh")

    parallel -j 2 python ::: "${scripts[@]}" ::: "${projections[@]}"
fi


# SECTION 3 - IMAGES UPLOAD ############################################################
# Use ncftpbookmarks to add a new FTP server with credentials
if [ "$DATA_UPLOAD" = true ]; then
    echo "-----------------------------------------------"
    echo "gens: Starting FTP uploading - `date`"
    echo "-----------------------------------------------"

    ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} gens ${IMGDIR}/meteogram_*

    ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} gens/nh/t_850 ${IMGDIR}/nh/t_850_* &
    ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} gens/nh/gph_500 ${IMGDIR}/nh/gph_500_* &
    ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} gens/nh/prob_snow ${IMGDIR}/nh/prob_snow_* &
    wait

    ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} gens/euratl/t_850 ${IMGDIR}/euratl/t_850_* &
    ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} gens/euratl/gph_500 ${IMGDIR}/euratl/gph_500_* &
    ncftpput -R -v -DD -m ${NCFTP_BOOKMARK} gens/euratl/prob_snow ${IMGDIR}/euratl/prob_snow_* &
    wait
fi

# SECTION 4 - CLEANING ############################################################

echo "-----------------------------------------------"
echo "gens: Finished cleaning up - `date`"
echo "----------------------------------------------_"

############################################################

cd -