Build and Install (#5)

* Changes to build script

* build the GDASApp in its entirety, not just the bundle

* build or skip bundle

* Update unittests.yaml

* checkout and cache ecbuild

* ignore built files

Co-authored-by: Cory Martin <cory.r.martin@noaa.gov>

.github/workflows/unittests.yaml

@@ -13,19 +13,31 @@ jobs:
       with:
         path: GDASApp
 
+    - name: Cache ecBuild
+      id: cache-ecbuild
+      uses: actions/cache@v2
+      with:
+        path: ecbuild
+        key: ${{ runner.os }}-ecbuild
+
+    - name: Install ecBuild
+      if: steps.cache-ecbuild.outputs.cache-hit != 'true'
+      run: |
+        git clone https://github.com/ecmwf/ecbuild.git ecbuild
+        cd ecbuild
+        git checkout 3.6.1
+
     - name: Configure with cmake
       run: |
-        cd $GITHUB_WORKSPACE/GDASApp
-        mkdir build
-        cd build
-        cmake ../
+        mkdir $GITHUB_WORKSPACE/build && cd $GITHUB_WORKSPACE/build
+        cmake -DBUILD_GDASBUNDLE=OFF $GITHUB_WORKSPACE/GDASApp
 
     - name: Build GDASApp
       run: |
-        cd $GITHUB_WORKSPACE/GDASApp/build
+        cd $GITHUB_WORKSPACE/build
         make
 
     - name: Run ctest
       run: |
-        cd $GITHUB_WORKSPACE/GDASApp/build
-        ctest
+        cd $GITHUB_WORKSPACE/build
+        ctest --output-on-failure

.gitignore

@@ -128,5 +128,44 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-# bin directory
+# Ignore the following
+*.[aox]
+*.mod
+*.la
+*.lo
+
+# Ignore editor caches
+.*.sw[a-p]
+*DS_Store*
+*~
+
+# Ignore the following directories anywhere in the tree
 bin/
+build*/
+install*/
+
+# Ignore the following bundle repositories
+ecbuild/
+atlas/
+crtm/
+eckit/
+fckit/
+femps/
+fms/
+fv3/
+fv3-jedi/
+fv3-jedi-data/
+fv3-jedi-lm/
+gsw/
+ioda/
+ioda-data/
+iodaconv/
+jedicmake/
+mom6/
+oops/
+saber/
+saber-data/
+soca/
+test-data-release/
+ufo/
+ufo-data/

CMakeLists.txt

@@ -1,10 +1,149 @@
-cmake_minimum_required(VERSION 3.10)
+# ------------------------------------------------------------------------- #
+# Application for all the components needed for the GDAS system             #
+# ------------------------------------------------------------------------- #
 
-# set the project name
-project(GDASApp)
+# Check for minimim cmake requirement
+cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 
-find_package(Python3 REQUIRED COMPONENTS Interpreter)
+find_package(ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
+
+project(GDASApp VERSION 1.0.0 LANGUAGES C CXX Fortran )
 
+include(GNUInstallDirs)
 enable_testing()
 
+# Build type.
+if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
+  message(STATUS "Setting build type to 'Release' as none was specified.")
+  set(CMAKE_BUILD_TYPE
+      "Release"
+      CACHE STRING "Choose the type of build." FORCE)
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
+                                               "MinSizeRel" "RelWithDebInfo")
+endif()
+
+# Find dependencies.
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+# Include ecbuild_bundle macro
+include( ecbuild_bundle )
+
+# Enable MPI
+set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
+
+# Handle user options.
+option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON)
+
+# Initialize bundle
+# -----------------
+ecbuild_bundle_initialize()
+
+# Build bundle source code.
+if(BUILD_GDASBUNDLE)
+
+# jedi-cmake
+  ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda/jedi-cmake.git" BRANCH develop UPDATE )
+
+# ECMWF libraries
+  option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
+  option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF
+  option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF
+
+# turn off optional OOPS toy models
+  option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF )
+  option( ENABLE_QG_MODEL "Build QG toy model" OFF )
+
+  ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 )
+  ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 )
+  ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 )
+
+# External (required) observation operators
+  ecbuild_bundle( PROJECT crtm GIT "https://github.com/jcsda/crtm.git"   TAG v2.3-jedi.3 )
+
+# Core JEDI repositories
+  ecbuild_bundle( PROJECT oops  GIT "https://github.com/jcsda/oops.git"  BRANCH develop UPDATE )
+  ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE )
+  ecbuild_bundle( PROJECT ioda  GIT "https://github.com/jcsda/ioda.git"  BRANCH develop UPDATE )
+  ecbuild_bundle( PROJECT ufo   GIT "https://github.com/noaa-emc/ufo.git"   BRANCH feature/ufsda UPDATE )
+
+# FMS and FV3 dynamical core
+  ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable UPDATE )
+  ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable UPDATE )
+
+# fv3-jedi and associated repositories
+  ecbuild_bundle( PROJECT femps       GIT "https://github.com/jcsda/femps.git"                BRANCH develop UPDATE )
+  ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE )
+  ecbuild_bundle( PROJECT fv3-jedi    GIT "https://github.com/jcsda/fv3-jedi.git"             BRANCH develop UPDATE )
+
+# SOCA associated repositories
+  ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" UPDATE BRANCH develop )
+  ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" UPDATE BRANCH main-ecbuild RECURSIVE )
+  ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" UPDATE BRANCH develop )
+
+# Build IODA converters
+  option(BUILD_IODA_CONVERTERS "Build IODA Converters" OFF)
+  if(BUILD_IODA_CONVERTERS)
+    ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop UPDATE )
+  endif()
+
+# ioda, ufo, fv3-jedi, and saber test data
+#---------------------------------
+
+# If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch.
+# If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case,
+# IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned.
+# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
+# in a local directory, ioda-data repo will not be cloned
+
+  find_branch_name(REPO_DIR_NAME ioda)
+# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
+# in a local directory, ioda-data repo will not be cloned
+  if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
+    ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE )
+
+    # If IODA's current branch is available in ioda-data repo, that branch will be checked out
+    branch_checkout (REPO_DIR_NAME ioda-data
+                    BRANCH ${GIT_BRANCH_FUNC} )
+  endif()
+
+# same procedure for ufo-data
+  find_branch_name(REPO_DIR_NAME ufo)
+  if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
+    ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE )
+
+    # If UFO's current branch is available in ioda-data repo, that branch will be checked out
+    branch_checkout (REPO_DIR_NAME ufo-data
+                    BRANCH ${GIT_BRANCH_FUNC} )
+  endif()
+
+# same procedure for fv3-jedi-data
+  find_branch_name(REPO_DIR_NAME fv3-jedi)
+  if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
+    ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE )
+
+    # If fv3-jedi's current branch is available in ioda-data repo, that branch will be checked out
+    branch_checkout (REPO_DIR_NAME fv3-jedi-data
+                    BRANCH ${GIT_BRANCH_FUNC} )
+  endif()
+
+# same procedure for saber-data
+  find_branch_name(REPO_DIR_NAME saber)
+  if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
+    ecbuild_bundle( PROJECT saber-data GIT "https://github.com/JCSDA-internal/saber-data.git" BRANCH develop UPDATE )
+
+    # If saber's current branch is available in saber-data repo, that branch will be checked out
+    branch_checkout (REPO_DIR_NAME saber-data
+                    BRANCH ${GIT_BRANCH_FUNC} )
+  endif()
+
+endif(BUILD_GDASBUNDLE)
+
+# Install utility scripts.
+add_subdirectory(ush)
+
+# Include testing.
 add_subdirectory(test)
+
+# Finalize bundle
+# ---------------
+ecbuild_bundle_finalize()

build.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# build.sh
+# 1 - determine host, load modules on supported hosts; proceed w/o otherwise
+# 2 - configure; build; install
+# 4 - optional, run unit tests
+
+set -eu
+
+dir_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# ==============================================================================
+usage() {
+  set +x
+  echo
+  echo "Usage: $0 -p <prefix> | -t <target> -h"
+  echo
+  echo "  -p  installation prefix <prefix>    DEFAULT: <none>"
+  echo "  -t  target to build for <target>    DEFAULT: $(hostname)"
+  echo "  -c  additional CMake options        DEFAULT: <none>"
+  echo "  -v  build with verbose output       DEFAULT: NO"
+  echo "  -h  display this message and quit"
+  echo
+  exit 1
+}
+
+# ==============================================================================
+
+# Defaults:
+INSTALL_PREFIX=""
+CMAKE_OPTS=""
+BUILD_TARGET="$(hostname)"
+BUILD_VERBOSE="NO"
+
+while getopts "p:t:c:hv" opt; do
+  case $opt in
+    p)
+      INSTALL_PREFIX=$OPTARG
+      ;;
+    t)
+      BUILD_TARGET=$OPTARG
+      ;;
+    c)
+      CMAKE_OPTS=$OPTARG
+      ;;
+    v)
+      BUILD_VERBOSE=YES
+      ;;
+    h|\?|:)
+      usage
+      ;;
+  esac
+done
+
+case ${BUILD_TARGET} in
+  hera | orion)
+    echo "Building GDASApp on $BUILD_TARGET"
+    source $MODULESHOME/init/sh
+    module purge
+    module use $dir_root/modulefiles
+    module load GDAS/$BUILD_TARGET
+    CMAKE_OPTS+=" -DMPIEXEC_EXECUTABLE=$MPIEXEC_EXEC -DMPIEXEC_NUMPROC_FLAG=$MPIEXEC_NPROC"
+    module list
+    ;;
+  $(hostname))
+    echo "Building GDASApp on $BUILD_TARGET"
+    ;;
+  *)
+    echo "Building GDASApp on unknown target: $BUILD_TARGET"
+    ;;
+esac
+
+BUILD_DIR=${BUILD_DIR:-$dir_root/build}
+[[ -d ${BUILD_DIR} ]] && rm -rf ${BUILD_DIR}
+mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR}
+
+# If INSTALL_PREFIX is not empty; install at INSTALL_PREFIX
+[[ -n "${INSTALL_PREFIX:-}" ]] && CMAKE_OPTS+=" -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}"
+
+# Configure
+echo "Configuring ..."
+set -x
+cmake \
+  ${CMAKE_OPTS:-} \
+  $dir_root
+set +x
+
+# Build
+echo "Building ..."
+set -x
+make -j ${BUILD_JOBS:-6} VERBOSE=$BUILD_VERBOSE
+set +x
+
+# Install
+if [[ -n ${INSTALL_PREFIX:-} ]]; then
+  echo "Installing ..."
+  set -x
+  make install
+  set +x
+fi
+
+exit 0