[develop] Add integration test job

doc/UsersGuide/BuildingRunningTesting/RunSRW.rst

@@ -791,6 +791,8 @@ The ``FV3LAM_wflow.xml`` file runs the specific j-job scripts (``jobs/JREGIONAL_
      - Run the forecast model (UFS Weather Model)
    * - run_post_*
      - Run the post-processing tool (UPP)
+   * - integration_test_*
+     - Run integration test 
 
 In addition to the baseline tasks described in :numref:`Table %s <WorkflowTasksTable>` above, users may choose to run a variety of optional tasks, including plotting and verification tasks. 
 
@@ -983,6 +985,7 @@ The workflow run is complete when all tasks have "SUCCEEDED". If everything goes
    201906151800   run_post_mem000_f001     4953245       SUCCEEDED         0          1          4.0
    ...
    201906151800   run_post_mem000_f012     4953381       SUCCEEDED         0          1          7.0
+   201906151800   integration_test_mem000     4953237       SUCCEEDED         0          1          7.0
 
 If users choose to run METplus verification tasks as part of their experiment, the output above will include additional lines after ``run_post_mem000_f012``. The output will resemble the following but may be significantly longer when using ensemble verification: 
 
@@ -1058,6 +1061,7 @@ This will output the last 40 lines of the log file, which lists the status of th
    201906151800   run_post_mem000_f004                           -            -             -       -         -
    201906151800   run_post_mem000_f005                           -            -             -       -         -
    201906151800   run_post_mem000_f006                           -            -             -       -         -
+   201906151800   integration_test_mem000
 
    Summary of workflow status:
    ~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1168,6 +1172,7 @@ The SRW App workflow can be run using standalone shell scripts in cases where th
       ./run_make_lbcs.sh
       ./run_fcst.sh
       ./run_post.sh
+      ./run_integration_test.sh
 
 Each task should finish with error code 0. For example: 
 
@@ -1184,31 +1189,33 @@ Check the batch script output file in your experiment directory for a “SUCCESS
             processors and wall clock time is a good starting point for NOAA HPC systems
             when running a 48-h forecast on the 25-km CONUS domain. For a brief description of tasks, see :numref:`Table %s <WorkflowTasksTable>`. 
 
-   +------------+------------------------+----------------+----------------------------+
-   | **Stage/** | **Task Run Script**    | **Number of**  | **Wall Clock Time (H:mm)** |
-   |            |                        | **Processors** |                            |             
-   +============+========================+================+============================+
-   | 1          | run_get_ics.sh         | 1              | 0:20 (depends on HPSS vs   |
-   |            |                        |                | FTP vs staged-on-disk)     |
-   +------------+------------------------+----------------+----------------------------+
-   | 1          | run_get_lbcs.sh        | 1              | 0:20 (depends on HPSS vs   |
-   |            |                        |                | FTP vs staged-on-disk)     |
-   +------------+------------------------+----------------+----------------------------+
-   | 1          | run_make_grid.sh       | 24             | 0:20                       |
-   +------------+------------------------+----------------+----------------------------+
-   | 2          | run_make_orog.sh       | 24             | 0:20                       |
-   +------------+------------------------+----------------+----------------------------+
-   | 3          | run_make_sfc_climo.sh  | 48             | 0:20                       |
-   +------------+------------------------+----------------+----------------------------+
-   | 4          | run_make_ics.sh        | 48             | 0:30                       |
-   +------------+------------------------+----------------+----------------------------+
-   | 4          | run_make_lbcs.sh       | 48             | 0:30                       |
-   +------------+------------------------+----------------+----------------------------+
-   | 5          | run_fcst.sh            | 48             | 0:30                       |
-   +------------+------------------------+----------------+----------------------------+
-   | 6          | run_post.sh            | 48             | 0:25 (2 min per output     |
-   |            |                        |                | forecast hour)             |
-   +------------+------------------------+----------------+----------------------------+
+   +------------+--------------------------+----------------+----------------------------+
+   | **Stage/** | **Task Run Script**      | **Number of**  | **Wall Clock Time (H:mm)** |
+   |            |                          | **Processors** |                            |             
+   +============+==========================+================+============================+
+   | 1          | run_get_ics.sh           | 1              | 0:20 (depends on HPSS vs   |
+   |            |                          |                | FTP vs staged-on-disk)     |
+   +------------+--------------------------+----------------+----------------------------+
+   | 1          | run_get_lbcs.sh          | 1              | 0:20 (depends on HPSS vs   |
+   |            |                          |                | FTP vs staged-on-disk)     |
+   +------------+--------------------------+----------------+----------------------------+
+   | 1          | run_make_grid.sh         | 24             | 0:20                       |
+   +------------+--------------------------+----------------+----------------------------+
+   | 2          | run_make_orog.sh         | 24             | 0:20                       |
+   +------------+--------------------------+----------------+----------------------------+
+   | 3          | run_make_sfc_climo.sh    | 48             | 0:20                       |
+   +------------+--------------------------+----------------+----------------------------+
+   | 4          | run_make_ics.sh          | 48             | 0:30                       |
+   +------------+--------------------------+----------------+----------------------------+
+   | 4          | run_make_lbcs.sh         | 48             | 0:30                       |
+   +------------+--------------------------+----------------+----------------------------+
+   | 5          | run_fcst.sh              | 48             | 0:30                       |
+   +------------+--------------------------+----------------+----------------------------+
+   | 6          | run_post.sh              | 48             | 0:25 (2 min per output     |
+   |            |                          |                | forecast hour)             |
+   +------------+--------------------------+----------------+----------------------------+
+   | 7          | run_integration_test.sh  | 1              | 0:05                       |
+   +------------+--------------------------+----------------+----------------------------+
 
 Users can access log files for specific tasks in the ``$EXPTDIR/log`` directory. To see how the experiment is progressing, users can also check the end of the ``log.launch_FV3LAM_wflow`` file from the command line:
 

jobs/JREGIONAL_INTEGRATION_TEST

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+#
+#-----------------------------------------------------------------------
+#
+# Source the variable definitions file and the bash utility functions.
+#
+#-----------------------------------------------------------------------
+#
+. $USHdir/source_util_funcs.sh
+source_config_for_task "task_integration_test|task_run_fcst" ${GLOBAL_VAR_DEFNS_FP}
+. $USHdir/job_preamble.sh
+#
+#-----------------------------------------------------------------------
+#
+# Save current shell options (in a global array).  Then set new options
+# for this script/function.
+#
+#-----------------------------------------------------------------------
+#
+{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1
+#
+#-----------------------------------------------------------------------
+#
+# Get the full path to the file in which this script/function is located 
+# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in
+# which the file is located (scrfunc_dir).
+#
+#-----------------------------------------------------------------------
+#
+scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" )
+scrfunc_fn=$( basename "${scrfunc_fp}" )
+scrfunc_dir=$( dirname "${scrfunc_fp}" )
+#
+#-----------------------------------------------------------------------
+#
+# Print message indicating entry into script.
+#
+#-----------------------------------------------------------------------
+#
+print_info_msg "
+========================================================================
+Entering script:  \"${scrfunc_fn}\"
+In directory:     \"${scrfunc_dir}\"
+
+This is the J-job script for the plotting task
+========================================================================"
+#
+#-----------------------------------------------------------------------
+#
+# Set grid name and COMOUT locations.
+#
+#-----------------------------------------------------------------------
+#
+if [ "${RUN_ENVIR}" != "nco" ]; then
+  export fcst_dir="${FCST_DIR}${SLASH_ENSMEM_SUBDIR}"
+else
+  export fcst_dir="${FCST_DIR}"
+fi
+#
+#-----------------------------------------------------------------------
+#
+# Call the ex-script for this J-job and pass to it the necessary variables.
+#
+#-----------------------------------------------------------------------
+#
+$SCRIPTSdir/exregional_integration_test.py \
+           --fcst_dir=$fcst_dir \
+           --fcst_len=${FCST_LEN_HRS} || \
+print_err_msg_exit "\
+Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
+#
+#-----------------------------------------------------------------------
+#
+# Run job postamble.
+#
+#-----------------------------------------------------------------------
+#
+job_postamble
+#
+#-----------------------------------------------------------------------
+#
+# Restore the shell options saved at the beginning of this script/function.
+#
+#-----------------------------------------------------------------------
+#
+{ restore_shell_opts; } > /dev/null 2>&1
+

parm/wflow/default_workflow.yaml

@@ -45,4 +45,4 @@ rocoto:
       - !startstopfreq ['{%- if workflow.DATE_FIRST_CYCL != workflow.DATE_LAST_CYCL %}{{ [workflow.DATE_FIRST_CYCL[0:8], "{:02d}".format(workflow.INCR_CYCL_FREQ)]|join }}{%- else %}{{workflow.DATE_FIRST_CYCL}}{%- endif %}', '{{workflow.DATE_LAST_CYCL}}', '{{workflow.INCR_CYCL_FREQ}}']
   log: !cycstr '&LOGDIR;/FV3LAM_wflow.{% if user.RUN_ENVIR == "nco" %}{{ workflow.WORKFLOW_ID + "." }}{% endif %}log'
   tasks:
-    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}'
+    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/test.yaml"]|include }}'

parm/wflow/test.yaml

@@ -0,0 +1,42 @@
+# Settings that will run tasks needed per-experiment to create "fix
+# files" for the stationary grid.
+
+default_task_test: &default_task
+  account: '&ACCOUNT;'
+  attrs:
+    cycledefs: forecast
+    maxtries: '2'
+  envars: &default_envars
+    GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;'
+    USHdir: '&USHdir;'
+    FCST_DIR: !cycstr '&FCST_DIR;'
+    PDY: !cycstr "@Y@m@d"
+    cyc: !cycstr "@H"
+    subcyc: !cycstr "@M"
+    LOGDIR: !cycstr "&LOGDIR;"
+    SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;'
+    ENSMEM_INDX: '#mem#'
+  native: '{{ platform.SCHED_NATIVE_CMD }}'
+  nnodes: 1
+  nodes: '{{ nnodes }}:ppn={{ ppn }}'
+  partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}'
+  ppn: 24
+  queue: '&QUEUE_DEFAULT;'
+  walltime: 00:05:00
+
+metatask_integration_test:
+  var:
+    mem: '{% if global.DO_ENSEMBLE  %}{%- for m in range(1, global.NUM_ENS_MEMBERS+1) -%}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}'
+  task_integration_test_mem#mem#:
+    <<: *default_task
+    command: '&LOAD_MODULES_RUN_TASK_FP; "integration_test" "&JOBSdir;/JREGIONAL_INTEGRATION_TEST"'
+    join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+    dependency:
+      and_run_fcst: 
+        taskvalid:
+          attrs:
+            task: run_fcst_mem#mem#
+        taskdep:
+          attrs:
+            task: run_fcst_mem#mem#
+

scripts/exregional_integration_test.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+
+################################################################################
+####  Python Script Documentation Block
+#
+# Script name:       	exregional_integration_test.py
+# Script description:  	Ensures the correct number of netcdf files are generated  
+# 			for each experiment
+#
+# Author:  Eddie Snyder 	Org: NOAA EPIC		Date: 2024-02-05
+#           
+# Instructions:		1. Pass the appropriate info for the required arguments:
+#                              --fcst_dir=/path/to/forecast/files
+#                              --fcst_len=<forecast length as Int>
+#                       2. Run script with arguments
+#                       
+# Notes/future work:    - Currently SRW App only accepts netcdf as the UFS WM 
+#                         output file format. If that changes, then additional
+#                         logic is needed to address the other file formats.
+#                       - SRW App doesn't have a variable that updates the 
+#                         forecast increment. The UFS WM does with the 
+#                         output_fh variable, which can be found in the 
+#                         model_configure file. If it becomes available with 
+#                         the SRW App, then logic is needed to account for the 
+#                         forecast increment variable.
+#
+################################################################################
+
+# -------------Import modules --------------------------#
+import os
+import sys
+import logging
+import argparse
+import unittest
+
+# --------------Define some functions ------------------#
+
+
+class TestExptFiles(unittest.TestCase):
+    fcst_dir = ''
+    filename_list = ''
+
+    def test_fcst_files(self):
+
+        for filename in self.filename_list:
+            filename_fp = "{0}/{1}".format(self.fcst_dir, filename)
+
+            logging.info("Checking existence of: {0}".format(filename_fp))
+            err_msg = "Missing file: {0}".format(filename_fp)
+            self.assertTrue(os.path.exists(filename_fp), err_msg)
+
+def setup_logging(debug=False):
+
+    """Calls initialization functions for logging package, and sets the
+    user-defined level for logging in the script."""
+
+    level = logging.INFO
+    if debug:
+        level = logging.DEBUG
+
+    logging.basicConfig(format="%(levelname)s: %(message)s ", level=level)
+    if debug:
+        logging.info("Logging level set to DEBUG")
+
+
+# -------------Start of script -------------------------#
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--fcst_dir",
+        help="Directory to forecast files.",
+        required=True,
+    )
+    parser.add_argument(
+        "--fcst_len",
+        help="Forecast length.",
+        required=True,
+    )
+    parser.add_argument(
+        "--fcst_inc",
+        default="1",
+        help="Increment of forecast in hours.",
+        required=False,
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Print debug messages.",
+        required=False,
+    )
+    parser.add_argument('unittest_args', nargs='*')
+    args = parser.parse_args()
+    sys.argv[1:] = args.unittest_args
+
+    fcst_dir = str(args.fcst_dir) 
+    fcst_len = int(args.fcst_len)
+    fcst_inc = int(args.fcst_inc)
+
+    # Start logger
+    setup_logging()
+
+    # Check if model_configure exists
+    model_configure_fp = "{0}/model_configure".format(fcst_dir)
+
+    if not os.path.isfile(model_configure_fp):
+        logging.error("Experiments model_configure file is missing! Exiting!")
+        sys.exit(1)
+
+    # Loop through model_configure file to find the netcdf base names
+    f = open(model_configure_fp, 'r')
+
+    for line in f:
+        if line.startswith("filename_base"):
+            filename_base_1 = line.split("'")[1]
+            filename_base_2 = line.split("'")[3]
+            break
+    f.close()
+
+    # Create list of expected filenames from the experiment
+    fcst_len = fcst_len + 1
+    filename_list = []
+
+    for x in range(0, fcst_len, fcst_inc):
+        fhour = str(x).zfill(3)
+        filename_1 = "{0}f{1}.nc".format(filename_base_1, fhour)
+        filename_2 = "{0}f{1}.nc".format(filename_base_2, fhour)
+        filename_list.append(filename_1)
+        filename_list.append(filename_2)
+
+    # Call unittest class 
+    TestExptFiles.fcst_dir = fcst_dir
+    TestExptFiles.filename_list = filename_list
+    unittest.main()

tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml

@@ -20,7 +20,7 @@ nco:
   NET_default: aqm
 rocoto:
   tasks:
-    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}'
+    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/aqm_prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/test.yaml"]|include }}'
     task_aqm_ics_ext:
     metatask_run_ensemble:
       task_run_fcst_mem#mem#:

tests/WE2E/test_configs/custom_grids/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml

@@ -15,7 +15,7 @@ workflow:
   PREEXISTING_DIR_METHOD: rename
 rocoto:
   tasks:
-    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}'
+    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/test.yaml"]|include }}'
     metatask_run_ensemble:
       task_run_fcst_mem#mem#:
         walltime: 01:00:00

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot.yaml

@@ -14,7 +14,7 @@ workflow:
   PREEXISTING_DIR_METHOD: rename
 rocoto:
   tasks:
-    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/plot.yaml"]|include }}'
+    taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/plot.yaml", "parm/wflow/test.yaml"]|include }}'
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS
   USE_USER_STAGED_EXTRN_FILES: true